Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions backend/app/api/v1/render.py
Original file line number Diff line number Diff line change
Expand Up @@ -974,6 +974,11 @@ async def image_models(user: User = Depends(require_auth)):
# are affordability floors; exact capture pricing is checked at admission.
return {
**availability,
"street_view_context_available": bool(
settings.direct_3d_images_enabled
and settings.direct_3d_street_view_context_enabled
and settings.google_maps_api_key
),
"credit_balance": effective_render_credit_balance(user),
"credit_exempt": is_admin_or_above(user),
"models": [
Expand Down
1 change: 1 addition & 0 deletions backend/app/core/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,7 @@ def cors_allow_origin_regex(self) -> str | None:
kling_animation_cost_per_second_usd: float = Field(default=0.112, gt=0)
kling_animation_credit_cost: int = Field(default=50, gt=0)
google_maps_api_key: str = ""
direct_3d_street_view_context_enabled: bool = False
fal_key: str = ""
fal_style_model: str = "fal-ai/fast-sdxl/image-to-image"
gemini_2d_image_model: str = "gemini-3-pro-image-preview"
Expand Down
15 changes: 15 additions & 0 deletions backend/app/schemas/direct_3d_render.py
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,17 @@ class Direct3DArchetypeReference(BaseModel):
zone_ids: list[str] = Field(default_factory=list, max_length=2048)


class Direct3DStreetViewContext(BaseModel):
latitude: float = Field(ge=-90, le=90, allow_inf_nan=False)
longitude: float = Field(ge=-180, le=180, allow_inf_nan=False)
heading: float = Field(ge=0, lt=360, allow_inf_nan=False)


class Direct3DStreetViewReferenceDiagnostics(BaseModel):
pano_date: str | None = None
offset_m: float = Field(ge=0)


class Direct3DRenderRequest(BaseModel):
"""A clean 3D capture plus mode-specific presentation and design authority."""

Expand Down Expand Up @@ -250,6 +261,7 @@ class Direct3DRenderRequest(BaseModel):
"in this first version. Street requires presentation_mode='scene'."
),
)
street_view_context: Direct3DStreetViewContext | None = None
style: Direct3DStyle = Field(
default="photorealistic",
description="The render panel style identifier governing the presentation treatment.",
Expand Down Expand Up @@ -390,6 +402,8 @@ def validate_material_id_manifest(
def validate_object_id_pair(self) -> "Direct3DRenderRequest":
if self.view_mode == "street" and self.presentation_mode != "scene":
raise ValueError("view_mode='street' requires presentation_mode='scene'")
if self.street_view_context is not None and self.view_mode != "street":
raise ValueError("street_view_context requires view_mode='street'")
if bool(self.object_id_image_base64) != bool(self.object_id_manifest):
raise ValueError("object_id_image_base64 and object_id_manifest must be supplied together")
if self.presentation_mode in {"scene", "reproject"} and not self.object_id_image_base64:
Expand Down Expand Up @@ -602,6 +616,7 @@ class Direct3DFinishFusionDiagnostics(BaseModel):

class Direct3DRenderDiagnostics(BaseModel):
control_bundle_version: Literal[1, 2] = 1
street_view_context: Direct3DStreetViewReferenceDiagnostics | None = None
processing_mode: Direct3DPresentationMode = "source_anchored"
view_lock: Literal[
"source_pixel_locked",
Expand Down
58 changes: 58 additions & 0 deletions backend/app/services/direct_3d_render.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import numpy as np
from PIL import Image, ImageDraw

from app.core.config import get_settings
from app.core.image_models import (
DEFAULT_OPENAI_IMAGE_MODEL,
PAID_IMAGE_CREDIT_MULTIPLIERS,
Expand Down Expand Up @@ -4102,6 +4103,51 @@ def __init__(self, api_key: str, *, gemini_api_key: str = ""):
self.api_key = api_key
self.gemini_api_key = gemini_api_key
self._nano_original: str | None = None
self._street_view_context: dict[str, Any] | None = None

async def _load_street_view_context(
self,
req: Direct3DRenderRequest,
capture: PreparedDirect3DCapture,
) -> tuple[str, bytes, str]:
from app.services.site_context import fetch_streetview_plate

context = req.street_view_context
settings = get_settings()
if context is None or not (settings.direct_3d_street_view_context_enabled and settings.google_maps_api_key):
raise Direct3DProviderError(
"Google Street View context is unavailable on this server.", billing_status="unproduced"
)

aspect = capture.normalized_beauty.width / capture.normalized_beauty.height
horizontal_fov = 90.0
if req.camera and req.camera.fov is not None:
horizontal_fov = math.degrees(2 * math.atan(math.tan(math.radians(req.camera.fov / 2)) * aspect))
plate = await fetch_streetview_plate(
context.latitude,
context.longitude,
context.heading,
horizontal_fov,
640,
round(640 / aspect),
max_offset_m=20,
)
if plate is None:
raise Direct3DProviderError(
"No usable Google Street View image was returned within 20 m. Move the camera or turn off the reference.",
billing_status="unproduced",
)
try:
raw = _decode_base64_payload(plate["photo_base64"], label="Google Street View reference")
_load_image(raw, label="Google Street View reference", allowed_formats={"PNG", "JPEG"}).close()
except Direct3DValidationError as exc:
raise Direct3DProviderError(
"Google Street View returned an invalid image. Move the camera or turn off the reference.",
billing_status="unproduced",
) from exc
self._street_view_context = {"pano_date": plate["pano_date"], "offset_m": plate["pano_offset_m"]}
mime = plate["mime_type"]
return ("street-view-context.png" if mime == "image/png" else "street-view-context.jpg", raw, mime)

async def _call_openai(
self,
Expand Down Expand Up @@ -4253,6 +4299,15 @@ async def _call_openai(
max_length=30_000,
)
prompt = apply_presentation_selections(prompt, add_people=req.add_people, add_vehicles=req.add_vehicles)
if req.street_view_context is not None:
files.append(("image[]", await self._load_street_view_context(req, capture)))
prompt += (
"\nEXISTING STREET REFERENCE: The last image is a photograph from a nearby panorama. "
"Use it for the appearance of existing façades, pavement, sidewalk edges and street furniture "
"only where they correspond to the captured 3D view. Keep the first image's camera, visibility, "
"street layout and every placed design object. Keep changes to materials and lighting. "
"Leave objects that are outside the frame, occluded or absent from the capture out of the result."
)
if req.model == NANO_BANANA_MODEL:
from app.services.nano_banana_image import (
generate_nano_banana_image,
Expand Down Expand Up @@ -4419,6 +4474,7 @@ async def generate(
) -> Direct3DServiceResult:
capture = capture or prepare_direct_3d_capture(req)
self._nano_original = None
self._street_view_context = None
generated = await self._call_openai(
req,
capture,
Expand Down Expand Up @@ -4496,6 +4552,8 @@ async def generate(
provider_image_base64,
)
street_result.diagnostics["view_mode"] = "street"
if self._street_view_context is not None:
street_result.diagnostics["street_view_context"] = self._street_view_context
return street_result

if DIRECT_3D_PRESENTATION_FIRST and req.presentation_mode == "scene":
Expand Down
3 changes: 3 additions & 0 deletions backend/app/services/render_attempts.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,9 @@ async def submit_attempt(db, user, req: Direct3DRenderRequest | VideoAppearanceR
await check_project_permission(req.project_id, user, db, required="editor")
user_id = user.id
payload = req.model_dump(mode="json")
# Retries of pre-Street View jobs must hash the same request as before rollout.
if payload.get("street_view_context") is None:
payload.pop("street_view_context", None)
fingerprint = revision_sha256(payload)
# Recover existing jobs without capture work or waiting for global admission.
existing = await db.scalar(
Expand Down
34 changes: 20 additions & 14 deletions backend/app/services/site_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import asyncio
import base64
import logging
import math
import time

import httpx
Expand All @@ -39,6 +40,8 @@ async def fetch_streetview_plate(
fov: float,
width: int,
height: int,
*,
max_offset_m: float | None = None,
) -> dict | None:
"""Fetch ONE real Street View photo aligned to a given camera, for use as the
composite background plate in SV-snap photomontage mode.
Expand Down Expand Up @@ -67,6 +70,17 @@ async def fetch_streetview_plate(
return None
pano = meta["pano_id"]
ploc = meta.get("location", {})
offset_m = None
if ploc.get("lat") is not None and ploc.get("lng") is not None:
dlat = math.radians(ploc["lat"] - lat)
dlng = math.radians(ploc["lng"] - lng)
a = (
math.sin(dlat / 2) ** 2
+ math.cos(math.radians(lat)) * math.cos(math.radians(ploc["lat"])) * math.sin(dlng / 2) ** 2
)
offset_m = 2 * 6_371_000 * math.asin(math.sqrt(min(1.0, a)))
if max_offset_m is not None and (offset_m is None or offset_m > max_offset_m):
return None
r = await client.get(
"https://maps.googleapis.com/maps/api/streetview",
params={
Expand All @@ -75,30 +89,22 @@ async def fetch_streetview_plate(
"heading": round(heading),
"fov": round(fov),
"pitch": 0,
"return_error_code": "true",
"key": key,
},
)
if r.status_code != 200 or not r.headers.get("content-type", "").startswith("image/"):
mime_type = r.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if r.status_code != 200 or mime_type not in {"image/jpeg", "image/png"}:
logger.warning("sv-plate: image fetch HTTP %s", r.status_code)
return None
except Exception as exc: # noqa: BLE001
logger.warning("sv-plate fetch failed: %s", exc)
logger.warning("sv-plate fetch failed (%s)", type(exc).__name__)
return None

offset_m = None
if ploc.get("lat") is not None:
import math

dlat = math.radians(ploc["lat"] - lat)
dlng = math.radians(ploc["lng"] - lng)
a = (
math.sin(dlat / 2) ** 2
+ math.cos(math.radians(lat)) * math.cos(math.radians(ploc["lat"])) * math.sin(dlng / 2) ** 2
)
offset_m = round(2 * 6_371_000 * math.asin(math.sqrt(a)), 1)
return {
"photo_base64": base64.b64encode(r.content).decode(),
"pano_offset_m": offset_m,
"mime_type": mime_type,
"pano_offset_m": round(offset_m, 1) if offset_m is not None else None,
"pano_date": meta.get("date"),
"width": w,
"height": h,
Expand Down
89 changes: 88 additions & 1 deletion backend/tests/test_direct_3d_street_mode.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,23 @@
"""Street view_mode contract for the Direct 3D pipeline (Stage 3 v1)."""

from types import SimpleNamespace
from unittest.mock import AsyncMock

import httpx
import numpy as np
import pytest
from PIL import Image
from pydantic import ValidationError

from app.schemas.direct_3d_render import Direct3DRenderRequest
from app.services import direct_3d_render as direct_service, site_context
from app.services.direct_3d_render import (
Direct3DProviderError,
Direct3DRenderService,
_presentation_prompt,
prepare_direct_3d_capture,
)
from tests.test_direct_3d_render import _capture_images, _request
from tests.test_direct_3d_render import _FakeResponse, _RecordingClient, _capture_images, _png_b64, _request


def _street_request(**overrides) -> Direct3DRenderRequest:
Expand All @@ -32,6 +39,86 @@ def test_street_with_scene_is_valid():
assert _street_request().view_mode == "street"


def test_aerial_requests_reject_street_view_context():
payload = _request(presentation_mode="scene").model_dump()
payload["street_view_context"] = {"latitude": 51.045535, "longitude": -114.067772, "heading": 90}
with pytest.raises(ValidationError, match="street"):
Direct3DRenderRequest.model_validate(payload)


@pytest.mark.parametrize("coverage", ["available", "missing", "invalid"])
async def test_street_context_is_attached_once_or_fails_before_the_image_provider(monkeypatch, coverage):
location = {"latitude": 51.045535, "longitude": -114.067772, "heading": 90}
request = Direct3DRenderRequest.model_validate(
{
**_street_request().model_dump(),
"street_view_context": location,
}
)
capture = prepare_direct_3d_capture(request)
fixture = {
"photo_base64": _png_b64(capture.normalized_beauty),
"mime_type": "image/png",
"pano_date": "2024-06",
"pano_offset_m": 15.0,
}
if coverage == "invalid":
fixture["photo_base64"] = "invalid-image"
lookup = AsyncMock(return_value=None if coverage == "missing" else fixture)
monkeypatch.setattr(site_context, "fetch_streetview_plate", lookup)
monkeypatch.setattr(
direct_service,
"get_settings",
lambda: SimpleNamespace(
direct_3d_street_view_context_enabled=True,
google_maps_api_key="test-maps-key",
),
)
monkeypatch.setattr(direct_service.httpx, "AsyncClient", _RecordingClient)
_RecordingClient.calls = []
_RecordingClient.response = _FakeResponse(200, {"data": [{"b64_json": _png_b64(capture.normalized_beauty)}]})
service = Direct3DRenderService("test-image-key")
if coverage == "available":
result = await service.generate(request, capture)
assert result.diagnostics["street_view_context"] == {"pano_date": "2024-06", "offset_m": 15.0}
assert len(_RecordingClient.calls) == 1
call = _RecordingClient.calls[0]
assert call["files"][0][1][0] == "direct-3d-beauty.png"
assert call["files"][-1][1][0] == "street-view-context.png"
assert "EXISTING STREET REFERENCE" in call["data"]["prompt"]
else:
message = "within 20 m" if coverage == "missing" else "invalid image"
with pytest.raises(Direct3DProviderError, match=message) as error:
await service.generate(request, capture)
assert error.value.billing_status == "unproduced"
assert _RecordingClient.calls == []
lookup.assert_awaited_once()
assert lookup.await_args.args[:3] == (location["latitude"], location["longitude"], location["heading"])
assert lookup.await_args.kwargs == {"max_offset_m": 20}


async def test_street_view_rejects_placeholder_when_panorama_image_is_unavailable(monkeypatch):
lat, lng = 51.045535, -114.067772

def google(request: httpx.Request) -> httpx.Response:
if request.url.path.endswith("/metadata"):
return httpx.Response(
200, json={"status": "OK", "pano_id": "test-pano", "location": {"lat": lat, "lng": lng}}
)
if request.url.params.get("return_error_code") == "true":
return httpx.Response(404)
return httpx.Response(200, headers={"content-type": "image/jpeg"}, content=b"placeholder")

async_client = httpx.AsyncClient
monkeypatch.setattr(
site_context.httpx,
"AsyncClient",
lambda **kwargs: async_client(transport=httpx.MockTransport(google), **kwargs),
)
monkeypatch.setattr(site_context, "get_settings", lambda: SimpleNamespace(google_maps_api_key="test-maps-key"))
assert await site_context.fetch_streetview_plate(lat, lng, 90, 90, 640, 360, max_offset_m=20) is None


def _street_framed_mask(size: tuple[int, int]) -> Image.Image:
"""Proposal fills everything below a context (sky) band — a street frame.

Expand Down
4 changes: 3 additions & 1 deletion backend/tests/test_render_attempt_admission.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,9 @@ async def test_due_weekly_allowance_admits_without_persisting_reset(admission, r
async def test_existing_request_recovery_ignores_current_balance(admission, status, monkeypatch):
f = admission
f.user.render_credits = 0
existing = SimpleNamespace(status=status, request_sha256=revision_sha256(f.req.model_dump(mode="json")))
# A queued row may predate the optional Street View field.
legacy_payload = f.req.model_dump(mode="json", exclude={"street_view_context"})
existing = SimpleNamespace(status=status, request_sha256=revision_sha256(legacy_payload))
f.db.scalar.return_value = existing
prepare = MagicMock(side_effect=AssertionError("Existing jobs must not prepare another capture"))
monkeypatch.setattr(direct_service, "prepare_direct_3d_capture", prepare)
Expand Down
25 changes: 25 additions & 0 deletions backend/tests/test_render_attempts.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,31 @@ async def execute(fixture, attempt):
await jobs.execute_attempt(db, attempt.id)


@pytest.mark.asyncio
async def test_retry_recovers_attempt_submitted_before_street_view_context(isolated):
f = isolated
key = str(uuid.uuid4())
legacy_payload = f.req.model_dump(mode="json", exclude={"street_view_context"})
legacy = RenderAttempt(
id=uuid.uuid4(),
user_id=f.user.id,
project_id=f.project.id,
idempotency_key=key,
request_sha256=jobs.revision_sha256(legacy_payload),
status="completed",
model=f.req.model,
style=f.req.style,
)
async with f.sessions() as db:
db.add(legacy)
await db.commit()

attempt = await submit(f, key)
assert attempt.id == legacy.id
f.delivered.assert_not_awaited()
f.provider.assert_not_awaited()


@pytest.mark.asyncio
async def test_credit_admission_preserves_real_trial_slot_and_existing_recovery(isolated):
from app.services.render_trial import TRIAL_KEY
Expand Down
Loading