diff --git a/.env.example b/.env.example index 57f76b7..35bbb00 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,11 @@ # OpenRouter API Key - Get yours at https://openrouter.ai/keys OPENROUTER_API_KEY=your_openrouter_key_here +# Optional: Direct provider keys (used when provider != openrouter) +ANTHROPIC_API_KEY= # Required for claude-* models +GROQ_API_KEY= # Required for Groq provider (fast Llama/Mixtral) +OLLAMA_BASE_URL=http://localhost:11434 # Local Ollama endpoint + # --- VOICE CONFIGURATION (LIVEKIT AGENTS) --- # Deepgram API Key (STT) - Get yours at https://console.deepgram.com/ DEEPGRAM_API_KEY=your_deepgram_key_here diff --git a/ai-service/app/api/v1/chat.py b/ai-service/app/api/v1/chat.py index 725912f..d42f2ce 100644 --- a/ai-service/app/api/v1/chat.py +++ b/ai-service/app/api/v1/chat.py @@ -1,7 +1,8 @@ -import re +import json import logging - +import asyncio from fastapi import APIRouter, HTTPException +from fastapi.responses import StreamingResponse from app.services.memory_service import memory_service from app.models.chat import ChatRequest, ChatResponse from app.services.brain.graph import brain @@ -10,7 +11,7 @@ router = APIRouter() logger = logging.getLogger(__name__) -@router.post("", response_model=ChatResponse) +@router.post("") async def chat(request: ChatRequest): # Run Graph try: @@ -24,16 +25,98 @@ async def chat(request: ChatRequest): "messages": [HumanMessage(content=request.message)], "emotion": "neutral", "conversation_id": conversation_id, + "identity": request.identity or "anonymous", + "stream": request.stream } config = {"configurable": {"thread_id": conversation_id}} - result = brain.invoke(initial_state, config=config) + + if request.stream: + async def event_generator(): + # 1. Start with emotion detection (sequential but fast) + try: + from app.services.brain.nodes.emotion import detect_emotion + emotion_res = await detect_emotion(initial_state) + detected_emotion = emotion_res.get("emotion", "neutral") + yield f"data: {json.dumps({'emotion': detected_emotion})}\n\n" + except Exception as ex: + logger.warning(f"Emotion detection failed: {ex}") + detected_emotion = "neutral" + + # 2. Setup the full context for generation + from app.services.brain.nodes.generate import session_history_window + from app.services.llm import llm_service + from app.services.persona import persona_engine + from app.services.settings_service import settings_service + from datetime import datetime + from uuid import UUID + + # Fetch context + user_msg = request.message + history_model, memories, facts = await asyncio.gather( + memory_service.get_history(UUID(conversation_id), session_history_window), + memory_service.search(query=user_msg, limit=3), + memory_service.get_long_term_memories(identity=request.identity or "anonymous", limit=5), + ) + + # Build Persona + db_settings = settings_service.get_settings() + custom_sys = (db_settings.get("system_prompt") or "").strip() + persona = custom_sys if custom_sys else persona_engine.get_persona() + time_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + system_content = ( + "You are AURA (Advanced Universal Responsive Avatar), steward of the ASE Lab.\n\n" + f"{persona}\n\n" + "IMPORTANT: Do NOT include bracketed emotions like [happy] or [sad] in your response content. " + "I have already detected your emotion separately.\n\n" + f"**Context:**\n- Current Time: {time_str}" + ) + if facts: system_content += f"\nWhat I know about you:\n{facts}\n" + if memories: + memory_block = "\n".join(f"- {m}" for m in memories) + system_content += f"\nRelevant past snippets:\n{memory_block}\n" + + messages_format = [{"role":"system", "content":system_content}] + history_model + [{"role":"user", "content":user_msg}] + + import re + full_text = "" + # 3. Stream from the registry directly + from app.services.providers.base import TextDelta + async for chunk in llm_service.stream(messages_format): + # Only yield incremental deltas to the dashboard + if isinstance(chunk, TextDelta): + txt = chunk.text + full_text += txt + yield f"data: {json.dumps({'text': txt})}\n\n" + # StreamDone is handled silently for background persistence below + + # 4. Final sync/persistence - SCRUBBED + scrubbed_final = re.sub(r'\[.*?\]', '', full_text).strip() + asyncio.create_task(memory_service.add_interaction( + conversation_id=UUID(conversation_id), + user_text=user_msg, + assistant_text=scrubbed_final, + user_emotion=detected_emotion, + assistant_emotion="neutral" + )) + asyncio.create_task(memory_service.store( + text=f"User: {user_msg} \n AURA: {scrubbed_final}", + metadata={"conversation_id": str(conversation_id)} + )) + + yield "data: [DONE]\n\n" + + return StreamingResponse(event_generator(), media_type="text/event-stream") + + # Non-streaming fallback + result = await brain.ainvoke(initial_state, config=config) # Extract response last_msg = result["messages"][-1].content emotion = result.get("emotion", "neutral") - # Look for tool calls in the last turn + # Look for tool calls tools_used = [] for msg in result["messages"]: if hasattr(msg, "tool_calls") and msg.tool_calls: @@ -43,22 +126,21 @@ async def chat(request: ChatRequest): "args": tc.get("args", {}) }) - # Clean tags - text = last_msg - if text.startswith("["): - match = re.match(r'^\[(.*?)\]', text) - if match: - text = text[match.end():].strip() - return ChatResponse( - text=text, + text=last_msg, emotion=emotion, conversation_id=conversation_id, tools_used=tools_used if tools_used else None ) except Exception as e: - logger.error(f"Chat error: {e}") + logger.error(f"Chat error: {e}", exc_info=True) + # If it was a stream request, we should yield an error event + if request.stream: + return StreamingResponse( + iter([f"data: {json.dumps({'text': f'Brain Freeze: {str(e)}', 'emotion': 'confused'})}\n\n"]), + media_type="text/event-stream" + ) return ChatResponse( text=f"Brain Freeze: {str(e)}", diff --git a/ai-service/app/api/v1/settings.py b/ai-service/app/api/v1/settings.py index 264c47e..77941dd 100644 --- a/ai-service/app/api/v1/settings.py +++ b/ai-service/app/api/v1/settings.py @@ -4,23 +4,29 @@ router = APIRouter() +PROVIDERS = ["openrouter", "openai", "anthropic", "groq", "ollama"] + class SettingsPatch(BaseModel): system_prompt: str | None = None - model: str | None = None - temperature: float | None = None - max_tokens: int | None = None - empathy: int | None = None - humor: int | None = None - formality: int | None = None + model: str | None = None + provider: str | None = None + temperature: float | None = None + max_tokens: int | None = None + empathy: int | None = None + humor: int | None = None + formality: int | None = None class ApiKeysPatch(BaseModel): openrouter_api_key: str | None = None - deepgram_api_key: str | None = None - cartesia_api_key: str | None = None - livekit_url: str | None = None - livekit_api_key: str | None = None + deepgram_api_key: str | None = None + cartesia_api_key: str | None = None + anthropic_api_key: str | None = None + groq_api_key: str | None = None + ollama_base_url: str | None = None + livekit_url: str | None = None + livekit_api_key: str | None = None livekit_api_secret: str | None = None @@ -35,11 +41,18 @@ def update_settings(patch: SettingsPatch): return settings_service.update_settings(data) +@router.get("/providers") +def list_providers(): + """Return available provider names for the UI dropdown.""" + return {"providers": PROVIDERS} + + @router.get("/keys") def get_api_keys(): keys = settings_service.get_api_keys() - # Mask values in response — only reveal whether each key is set - return {k: ("••••••••" if v else None) for k, v in keys.items() if k != "id"} + # Return masked values — just signals whether the key is configured + return {k: ("set" if (v and str(v).strip()) else None) + for k, v in keys.items() if k != "id"} @router.put("/keys") diff --git a/ai-service/app/core/config.py b/ai-service/app/core/config.py index 5eff0f1..51de459 100644 --- a/ai-service/app/core/config.py +++ b/ai-service/app/core/config.py @@ -28,6 +28,9 @@ class Settings(BaseSettings): LLM_API_KEY: str | None = None OPENAI_API_KEY: str | None = None OPENROUTER_API_KEY: str | None = None + ANTHROPIC_API_KEY: str | None = None + GROQ_API_KEY: str | None = None + OLLAMA_BASE_URL: str = "http://localhost:11434" OPENAI_MODEL: str = "gpt-3.5-turbo" # Supabase diff --git a/ai-service/app/models/chat.py b/ai-service/app/models/chat.py index 6ffcc57..c55136a 100644 --- a/ai-service/app/models/chat.py +++ b/ai-service/app/models/chat.py @@ -4,6 +4,8 @@ class ChatRequest(BaseModel): message: str conversation_id: Optional[str] = None + identity: Optional[str] = None + stream: bool = False class ChatResponse(BaseModel): text: str diff --git a/ai-service/app/services/brain/nodes/emotion.py b/ai-service/app/services/brain/nodes/emotion.py index 66cd899..ec37427 100644 --- a/ai-service/app/services/brain/nodes/emotion.py +++ b/ai-service/app/services/brain/nodes/emotion.py @@ -2,7 +2,7 @@ from app.services.llm import llm_service # Node to detect emotion -def detect_emotion(state: BrainState) -> dict: +async def detect_emotion(state: BrainState) -> dict: # Get last user message last_message = state["messages"][-1].content @@ -13,7 +13,7 @@ def detect_emotion(state: BrainState) -> dict: """ # Call LLM to detect emotion - emotion = llm_service.generate([{"role": "system", "content": prompt}]) + response = await llm_service.generate([{"role": "system", "content": prompt}]) # Return detected emotion - return {"emotion": emotion["emotion"].strip().lower()} \ No newline at end of file + return {"emotion": response.get("emotion", "neutral").strip().lower()} \ No newline at end of file diff --git a/ai-service/app/services/brain/nodes/generate.py b/ai-service/app/services/brain/nodes/generate.py index 0c1cfad..8c0207a 100644 --- a/ai-service/app/services/brain/nodes/generate.py +++ b/ai-service/app/services/brain/nodes/generate.py @@ -11,10 +11,9 @@ session_history_window = 9999 -def generate_response(state: BrainState) -> dict: - with concurrent.futures.ThreadPoolExecutor() as pool: - future = pool.submit(asyncio.run, generate(state)) - return future.result() +async def generate_response(state: BrainState) -> dict: + """Async wrapper for the generation node.""" + return await generate(state) # Node to generate response based on persona, conversation history and detected emotion (convesation history not being tested yet) @@ -45,46 +44,90 @@ async def generate(state: BrainState) -> dict: else: user_message = "" - # Load History - history_model, memories = await asyncio.gather( + # Load History & Long-term memories + history_model, memories, facts = await asyncio.gather( memory_service.get_history(conversation_id, session_history_window), memory_service.search(query=user_message, limit=3), + memory_service.get_long_term_memories(identity=state.get("identity", "anonymous"), limit=5), ) history = history_model - # System Prompt - system_message = prompter.build("", context=None)[0] + # Save User message IMMEDIATELY to DB so it persists even if AI fails or disconnects + await memory_service.add_interaction( + conversation_id=conversation_id, + user_text=user_message, + assistant_text=None, # Update later + user_emotion=detected_emotion, + assistant_emotion=None + ) - if memories: - memory_block = "\n".join(f"-{message}" for message in memories) - system_message = { - "role" : "system", - "content": (system_message["content"] + f"Ingatan sebelumnya: \n {memory_block}") - } + # System Prompt (Pulling from DB via settings_service) + from app.services.settings_service import settings_service + db_settings = settings_service.get_settings() + custom_sys = (db_settings.get("system_prompt") or "").strip() + from app.services.persona import persona_engine + persona = custom_sys if custom_sys else persona_engine.get_persona() - # Add system prompt with persona and current time + from datetime import datetime + time_str = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + system_content = ( + "You are AURA (Advanced Universal Responsive Avatar), " + "the spirited AI steward of the ASE Lab.\n\n" + f"{persona}\n\n" + f"**Context:**\n- Current Time: {time_str}" + ) + + # Combine RAG (memories) and LTS (facts) + combined_memory = "" + if facts: + combined_memory += f"\nWhat I know about you:\n{facts}\n" + if memories: + memory_block = "\n".join(f"- {message}" for message in memories) + combined_memory += f"\nRelevant past snippets:\n{memory_block}\n" + + if combined_memory: + system_content += f"\n\n**Memory Retrieval:**{combined_memory}" + + system_message = {"role": "system", "content": system_content} + + # Build payload messages_format = [system_message] + history + current_message + # Check for stream request + is_stream = state.get("stream", False) + + if is_stream: + # For streaming, we yield chunks. + # But this is a node, so we return the final state but can use callbacks? + # Actually, chat.py will call brain.astream(). + # We handle the stream here if we want to return the stream object, + # but LangGraph nodes should return the update. + # So we update chat.py to use a different strategy. + pass + # Generate response from LLM - response = llm_service.generate(messages_format) + response = await llm_service.generate(messages_format) + text = response.get("text", "") emotion = response.get("emotion", "neutral") await asyncio.gather( + # Complete the interaction in DB memory_service.add_interaction( conversation_id=conversation_id, user_text=user_message, - assistant_text=response["text"], + assistant_text=text, user_emotion=detected_emotion, assistant_emotion=emotion ), memory_service.store( - text=f"User: {user_message} \n AURA: {response['text']}", + text=f"User: {user_message} \n AURA: {text}", metadata={"conversation_id": str(conversation_id)}, ), ) # Return response - return {"messages": [AIMessage(content=response["text"])], "emotion": response["emotion"]} \ No newline at end of file + return {"messages": [AIMessage(content=text)], "emotion": emotion} \ No newline at end of file diff --git a/ai-service/app/services/brain/state.py b/ai-service/app/services/brain/state.py index c4047a7..bab52f5 100644 --- a/ai-service/app/services/brain/state.py +++ b/ai-service/app/services/brain/state.py @@ -7,4 +7,5 @@ class BrainState(TypedDict): messages: Annotated[List[BaseMessage], operator.add] emotion: str - conversation_id: str \ No newline at end of file + conversation_id: str + identity: str \ No newline at end of file diff --git a/ai-service/app/services/llm.py b/ai-service/app/services/llm.py index 7b3cad1..0a1629c 100644 --- a/ai-service/app/services/llm.py +++ b/ai-service/app/services/llm.py @@ -1,74 +1,33 @@ -from openai import OpenAI -from app.core.config import settings +""" +LLMService — thin facade over the Provider Abstraction Layer. + +All routing logic lives in providers/registry.py. +This class exists so existing callers (brain nodes, etc.) don't need to change. +""" import logging -import re +import asyncio +from app.services.providers.registry import provider_registry logger = logging.getLogger(__name__) class LLMService: - def __init__(self): - self._env_key = settings.OPENROUTER_API_KEY or settings.OPENAI_API_KEY - self.base_url = "https://openrouter.ai/api/v1" if settings.OPENROUTER_API_KEY else None - self.client = None - - if self._env_key: - self.client = OpenAI(api_key=self._env_key, base_url=self.base_url) - logger.info(f"LLM Service Initialized. Base: {self.base_url or 'Default'}") - else: - logger.warning("API Key not set. LLMService will fail.") - - def _get_client(self): - """Return a client using the DB key if set, falling back to the env key.""" - from app.services.settings_service import settings_service - db_key = settings_service.get_api_keys().get("openrouter_api_key") - if db_key and db_key.strip(): - return OpenAI(api_key=db_key, base_url="https://openrouter.ai/api/v1") - return self.client - - def generate(self, messages: list, model: str = None, temperature: float = None, max_tokens: int = None) -> dict: - client = self._get_client() - if not client: - return {"text": "Error: API Key is missing.", "emotion": "[dizzy]"} - - # Import here to avoid circular imports at module load time - from app.services.settings_service import settings_service - db = settings_service.get_settings() - - actual_model = model or db.get("model") or "deepseek/deepseek-v3.2" - actual_temp = temperature if temperature is not None else db.get("temperature", 0.8) - actual_max_tokens = max_tokens or db.get("max_tokens") or 300 - - try: - extra_headers = {} - if settings.OPENROUTER_API_KEY: - extra_headers = { - "HTTP-Referer": "http://localhost:5173", - "X-Title": "Project AURA", - } - - response = client.chat.completions.create( - model=actual_model, - messages=messages, - temperature=actual_temp, - max_tokens=actual_max_tokens, - extra_headers=extra_headers, - ) - - content = response.choices[0].message.content - emotion_match = re.match(r'^\[(.*?)\]', content) - emotion = "neutral" - text = content - - if emotion_match: - emotion = emotion_match.group(1) - text = content[emotion_match.end():].strip() - - return {"text": text, "emotion": emotion, "raw": content} - - except Exception as e: - logger.error(f"LLM Generation Error: {e}") - return {"text": f"I lost my train of thought. ({str(e)})", "emotion": "[confused]"} + async def generate( + self, + messages: list, + model: str | None = None, + temperature: float | None = None, + max_tokens: int | None = None, + ) -> dict: + return await provider_registry.generate( + messages, + model=model, + temperature=temperature, + max_tokens=max_tokens, + ) + + def stream(self, *args, **kwargs): + return provider_registry.stream(*args, **kwargs) llm_service = LLMService() diff --git a/ai-service/app/services/memory_service.py b/ai-service/app/services/memory_service.py index b401291..dbb4acd 100644 --- a/ai-service/app/services/memory_service.py +++ b/ai-service/app/services/memory_service.py @@ -4,17 +4,28 @@ """ from __future__ import annotations from typing import List +import urllib.request from supabase import create_client from langchain_openai import OpenAIEmbeddings from app.core.config import settings from uuid import UUID + from app.models.database import (Conversation, CreateConversation, Message, CreateMesssage, Memory, CreateMemory) import logging logger = logging.getLogger(__name__) + +def _ollama_is_running(base_url: str) -> bool: + """Return True if an Ollama server is reachable at base_url.""" + try: + urllib.request.urlopen(f"{base_url}/api/tags", timeout=2) + return True + except Exception: + return False + class MemoryService: def __init__(self): self.client = None @@ -27,16 +38,36 @@ def __init__(self): else: logger.warning("Supabase credentials not set. Memory service disabled.") - # Initialize embeddings model via OpenRouter - api_key = settings.OPENROUTER_API_KEY - if api_key: + # Initialize embeddings — try providers in order of preference + if settings.OPENAI_API_KEY: self.embeddings = OpenAIEmbeddings( - api_key=api_key, + api_key=settings.OPENAI_API_KEY, + model="text-embedding-3-small", + ) + logger.info("RAG: Using OpenAI Directly for semantic embeddings (best-in-class mapping).") + print("INFO: Memory Service using OpenAI Embeddings for search mapping.") + elif settings.OPENROUTER_API_KEY: + self.embeddings = OpenAIEmbeddings( + api_key=settings.OPENROUTER_API_KEY, model="openai/text-embedding-3-small", - base_url="https://openrouter.ai/api/v1" + base_url="https://openrouter.ai/api/v1", ) + logger.info("RAG: Using OpenRouter for semantic embeddings.") + print("INFO: Memory Service using OpenRouter Embeddings.") + elif _ollama_is_running(settings.OLLAMA_BASE_URL): + self.embeddings = OpenAIEmbeddings( + api_key="ollama", + model="nomic-embed-text", + base_url=f"{settings.OLLAMA_BASE_URL}/v1", + ) + logger.info("RAG: Using local Ollama for semantic embeddings.") + print("INFO: Memory Service using local Ollama Embeddings.") else: - logger.warning("OPENROUTER_API_KEY not set. Memory embedding disabled.") + logger.warning( + "No embedding provider available " + "(OPENAI_API_KEY / OPENROUTER_API_KEY not set; Ollama not reachable). " + "Memory store/search disabled." + ) async def create_conversation(self, title: str = "New Conversation") -> UUID | None: if not self.client: @@ -76,26 +107,30 @@ async def get_conversation(self, conversation_id: UUID) -> Conversation | None: logger.error(f"Memory Service Get Conversation Error: {error}") return None - async def add_interaction(self, conversation_id: UUID, user_text: str, assistant_text: str, user_emotion: str = "neutral", assistant_emotion: str = "neutral") -> None: + async def add_interaction(self, conversation_id: UUID, user_text: str, assistant_text: str | None, user_emotion: str = "neutral", assistant_emotion: str = "neutral") -> None: if not self.client: return None try: - self.client.table("messages").insert([ - CreateMesssage( + msgs = [] + if user_text: + msgs.append(CreateMesssage( conversation_id=conversation_id, role="user", content=user_text, emotion=user_emotion, - ).model_dump(mode="json"), + ).model_dump(mode="json")) - CreateMesssage( + if assistant_text: + msgs.append(CreateMesssage( conversation_id=conversation_id, role="aura", content=assistant_text, emotion=assistant_emotion - ).model_dump(mode="json") - ]).execute() + ).model_dump(mode="json")) + + if msgs: + self.client.table("messages").insert(msgs).execute() self.client.table("conversations") \ .update({"updated_at": "now()"}) \ @@ -202,4 +237,31 @@ async def search(self, query: str, limit: int = 3) -> list[str]: return [] + async def get_long_term_memories(self, identity: str, limit: int = 10) -> str: + """Retrieve the last N non-embedded 'user_facts' memories for this identity.""" + if not self.client: + return "" + + try: + result = self.client.table("memories") \ + .select("content, created_at") \ + .eq("metadata->>type", "user_facts") \ + .eq("metadata->>identity", identity) \ + .order("created_at", desc=True) \ + .limit(limit) \ + .execute() + + rows = result.data or [] + if not rows: + return "" + + # Reverse to get chronological order in the prompt + facts_list = [row["content"] for row in reversed(rows)] + return "\n---\n".join(facts_list) + + except Exception as e: + logger.error(f"Memory Service Get Long Term Memories error: {e}") + return "" + + memory_service = MemoryService() \ No newline at end of file diff --git a/ai-service/app/services/prompter.py b/ai-service/app/services/prompter.py index e541fec..57cdfc3 100644 --- a/ai-service/app/services/prompter.py +++ b/ai-service/app/services/prompter.py @@ -7,13 +7,14 @@ class Prompter: def build(self, message: str, context: dict = None) -> list: current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - # Pull live settings — custom system_prompt overrides the hardcoded persona - db_settings = settings_service.get_settings() - custom_prompt = (db_settings.get("system_prompt") or "").strip() - persona = custom_prompt if custom_prompt else persona_engine.get_persona() + # Custom system_prompt from admin panel overrides the hardcoded persona + db = settings_service.get_settings() + custom = (db.get("system_prompt") or "").strip() + persona = custom if custom else persona_engine.get_persona() formatted_system = ( - f"You are AURA (Advanced Universal Responsive Avatar), the spirited AI steward of the ASE Lab.\n\n" + "You are AURA (Advanced Universal Responsive Avatar), " + "the spirited AI steward of the ASE Lab.\n\n" f"{persona}\n\n" f"**Context:**\n- Current Time: {current_time}" ) diff --git a/ai-service/app/services/providers/__init__.py b/ai-service/app/services/providers/__init__.py new file mode 100644 index 0000000..292fc89 --- /dev/null +++ b/ai-service/app/services/providers/__init__.py @@ -0,0 +1,3 @@ +from app.services.providers.registry import provider_registry + +__all__ = ["provider_registry"] diff --git a/ai-service/app/services/providers/anthropic_provider.py b/ai-service/app/services/providers/anthropic_provider.py new file mode 100644 index 0000000..89685fe --- /dev/null +++ b/ai-service/app/services/providers/anthropic_provider.py @@ -0,0 +1,183 @@ +""" +Anthropic / Claude provider. + +Key differences from OpenAI-compatible providers: + +1. System message → separate `system` parameter (not in messages list). +2. Streaming: chunks are `content_block_delta` with type "text_delta" + (vs GPT's `choices[0].delta.content`). +3. Tool calls: come as `content_block_start` with type "tool_use" + (vs OpenAI's `message.tool_calls`). +4. Tool definitions: Anthropic uses a different schema than OpenAI. + We accept the OpenAI schema and translate it internally. + +Normalized output is always the same result dict as every other provider. +""" +from __future__ import annotations + +import json +import logging +from typing import AsyncGenerator + +from app.services.providers.base import LLMProvider, TextDelta, StreamDone, make_result, RetryableError, NonRetryableError + +logger = logging.getLogger(__name__) + + +def _split_system(messages: list[dict]) -> tuple[str, list[dict]]: + """Separate the system prompt from the rest of the message list.""" + system_parts = [] + rest = [] + for m in messages: + if m.get("role") == "system": + system_parts.append(m.get("content", "")) + else: + rest.append(m) + return "\n\n".join(system_parts), rest + + +def _openai_tools_to_anthropic(tools: list[dict]) -> list[dict]: + """ + Translate OpenAI tool schema to Anthropic's format. + + OpenAI: { "type": "function", "function": { "name", "description", "parameters" } } + Anthropic: { "name", "description", "input_schema" } + """ + result = [] + for t in tools: + fn = t.get("function", t) # handle both wrapped and unwrapped + result.append({ + "name": fn["name"], + "description": fn.get("description", ""), + "input_schema": fn.get("parameters", {"type": "object", "properties": {}}), + }) + return result + + +def _extract_tool_calls(content_blocks) -> list | None: + """Normalize Anthropic tool_use blocks to our common schema.""" + calls = [ + { + "id": block.id, + "name": block.name, + "arguments": json.dumps(block.input), + } + for block in content_blocks + if getattr(block, "type", None) == "tool_use" + ] + return calls or None + + +class AnthropicProvider(LLMProvider): + name = "anthropic" + + def __init__(self, api_key: str): + try: + import anthropic as _anthropic + self._anthropic = _anthropic + self._client = _anthropic.Anthropic(api_key=api_key) + self._async_client = _anthropic.AsyncAnthropic(api_key=api_key) + logger.info("[anthropic] provider ready") + except ImportError: + raise RuntimeError( + "The 'anthropic' package is required for the Anthropic provider. " + "Run: pip install anthropic" + ) + + # ── Blocking ────────────────────────────────────────────────────────────── + + def generate( + self, + messages: list[dict], + *, + model: str, + temperature: float, + max_tokens: int, + tools: list[dict] | None = None, + ) -> dict: + system, user_messages = _split_system(messages) + kwargs = dict( + model=model, + system=system, + messages=user_messages, + temperature=temperature, + max_tokens=max_tokens, + ) + if tools: + kwargs["tools"] = _openai_tools_to_anthropic(tools) + + _a = self._anthropic # local ref so except clauses can reference it + try: + response = self._client.messages.create(**kwargs) + + # Text from text blocks + raw = "".join( + block.text for block in response.content + if getattr(block, "type", None) == "text" + ) + tool_calls = _extract_tool_calls(response.content) + + if tool_calls and not raw: + raw = f"[tool_call: {tool_calls[0]['name']}]" + + return make_result(raw, self.name, model, tool_calls=tool_calls) + + except _a.RateLimitError as e: + raise RetryableError(str(e), status_code=429) + except (_a.APIConnectionError, _a.APITimeoutError) as e: + raise RetryableError(str(e)) + except _a.InternalServerError as e: + raise RetryableError(str(e), status_code=getattr(e, "status_code", 500)) + except _a.AuthenticationError as e: + raise NonRetryableError(str(e), status_code=401) + except _a.BadRequestError as e: + raise NonRetryableError(str(e), status_code=400) + except Exception as e: + raise RetryableError(str(e)) + + # ── Streaming ───────────────────────────────────────────────────────────── + + async def stream( + self, + messages: list[dict], + *, + model: str, + temperature: float, + max_tokens: int, + tools: list[dict] | None = None, + ) -> AsyncGenerator[TextDelta | StreamDone, None]: + system, user_messages = _split_system(messages) + assembled = "" + kwargs = dict( + model=model, + system=system, + messages=user_messages, + temperature=temperature, + max_tokens=max_tokens, + ) + if tools: + kwargs["tools"] = _openai_tools_to_anthropic(tools) + + try: + async with self._async_client.messages.stream(**kwargs) as stream: + async for event in stream: + if ( + event.type == "content_block_delta" + and hasattr(event, "delta") + and getattr(event.delta, "type", None) == "text_delta" + ): + chunk = event.delta.text or "" + if chunk: + assembled += chunk + yield TextDelta(text=chunk) + except Exception as e: + logger.error(f"[anthropic] stream error: {e}") + + result = make_result(assembled, self.name, model) + yield StreamDone( + text=result["text"], + emotion=result["emotion"], + raw=assembled, + provider=self.name, + model=model, + ) diff --git a/ai-service/app/services/providers/base.py b/ai-service/app/services/providers/base.py new file mode 100644 index 0000000..cfb6b84 --- /dev/null +++ b/ai-service/app/services/providers/base.py @@ -0,0 +1,147 @@ +""" +Provider Abstraction Layer — base types and interface. + +Every LLM provider normalizes its output into the same result dict +so the rest of the system never needs to know which model is running. + +Normalized result: + { text, emotion, raw, provider, model, tool_calls } + +Tool calls are always normalized to: + [{ "id": str, "name": str, "arguments": str (JSON) }] + — regardless of whether the provider used OpenAI function_call deltas + or Anthropic content_block tool_use blocks. + +Stream events (for future streaming endpoints): + TextDelta — incremental text chunk + StreamDone — final assembled result +""" +from __future__ import annotations + +import re +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import AsyncGenerator + + +# ── Normalized event types ──────────────────────────────────────────────────── + +@dataclass +class TextDelta: + """A chunk of text from a streaming response.""" + text: str + + +@dataclass +class StreamDone: + """Final event — carries the fully assembled response.""" + text: str + emotion: str + raw: str + provider: str + model: str + tool_calls: list | None = None + + +# ── Error types ─────────────────────────────────────────────────────────────── + +class RetryableError(Exception): + """ + Rate limit (429), server error (5xx), or transient network issue. + The registry will retry with exponential backoff, then try the next provider. + """ + def __init__(self, msg: str, status_code: int | None = None): + super().__init__(msg) + self.status_code = status_code + + +class NonRetryableError(Exception): + """ + Auth failure (401) or bad request (400). + - 401: key is wrong for this provider → skip to next provider. + - 400: our message is malformed → no provider will fix it; abort immediately. + """ + def __init__(self, msg: str, status_code: int | None = None): + super().__init__(msg) + self.status_code = status_code + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + +def parse_emotion(raw: str) -> tuple[str, str]: + """ + Extract the leading [emotion, tag] from a raw LLM response. + Returns (emotion_string, cleaned_text). + """ + stripped = raw.strip() + match = re.match(r'^\[(.*?)\]', stripped) + if match: + return match.group(1), stripped[match.end():].strip() + return "neutral", stripped + + +def make_result( + raw: str, + provider: str, + model: str, + tool_calls: list | None = None, +) -> dict: + """Build the normalized result dict that the rest of the system expects.""" + emotion, text = parse_emotion(raw) + return { + "text": text, + "emotion": emotion, + "raw": raw, + "provider": provider, + "model": model, + "tool_calls": tool_calls or None, + } + + +# ── Abstract base ───────────────────────────────────────────────────────────── + +class LLMProvider(ABC): + """ + All providers implement this interface. + `generate` is the blocking path used by the brain pipeline. + `stream` is the async-generator path for future streaming endpoints. + + Tool definitions follow the OpenAI schema: + [{ "type": "function", "function": { "name": ..., "description": ..., + "parameters": {...} } }] + Providers that use a different native schema (e.g. Anthropic) translate + internally — callers always pass the OpenAI format. + """ + + name: str = "base" + + @abstractmethod + def generate( + self, + messages: list[dict], + *, + model: str, + temperature: float, + max_tokens: int, + tools: list[dict] | None = None, + ) -> dict: + """ + Blocking generation. Returns the normalized result dict: + { text, emotion, raw, provider, model, tool_calls } + """ + + @abstractmethod + async def stream( + self, + messages: list[dict], + *, + model: str, + temperature: float, + max_tokens: int, + tools: list[dict] | None = None, + ) -> AsyncGenerator[TextDelta | StreamDone, None]: + """ + Streaming generation. + Yields TextDelta chunks, ends with one StreamDone. + """ + yield # type: ignore diff --git a/ai-service/app/services/providers/openai_compat.py b/ai-service/app/services/providers/openai_compat.py new file mode 100644 index 0000000..35c005e --- /dev/null +++ b/ai-service/app/services/providers/openai_compat.py @@ -0,0 +1,194 @@ +""" +OpenAI-compatible provider. + +Covers every backend that speaks the OpenAI chat-completions API: + • OpenRouter (base_url = https://openrouter.ai/api/v1) + • OpenAI (base_url = None → default) + • Groq (base_url = https://api.groq.com/openai/v1) + • Ollama (base_url = http://localhost:11434/v1) + +Tool call normalization: + OpenAI sends tool_calls on the response message. + Each tool call has: id, function.name, function.arguments (JSON string). + We surface these as [{ "id", "name", "arguments" }] in the result dict. +""" +from __future__ import annotations + +import logging +from typing import AsyncGenerator + +import openai as _openai_lib +from openai import OpenAI, AsyncOpenAI + +from app.services.providers.base import LLMProvider, TextDelta, StreamDone, make_result, RetryableError, NonRetryableError + +logger = logging.getLogger(__name__) + +_OPENROUTER_HEADERS = { + "HTTP-Referer": "http://localhost:5173", + "X-Title": "Project AURA", +} + + +def _extract_tool_calls(response_message) -> list | None: + """Normalize OpenAI tool_calls to our common schema.""" + raw_calls = getattr(response_message, "tool_calls", None) + if not raw_calls: + return None + return [ + { + "id": tc.id, + "name": tc.function.name, + "arguments": tc.function.arguments, # already a JSON string + } + for tc in raw_calls + ] + + +class OpenAICompatProvider(LLMProvider): + + def __init__( + self, + api_key: str, + base_url: str | None = None, + extra_headers: dict | None = None, + provider_name: str = "openai", + ): + self.name = provider_name + self._extra_headers = extra_headers or {} + self._client = OpenAI(api_key=api_key, base_url=base_url) + self._async_client = AsyncOpenAI(api_key=api_key, base_url=base_url) + logger.info(f"[{self.name}] provider ready (base_url={base_url or 'default'})") + + # ── Blocking ────────────────────────────────────────────────────────────── + + def generate( + self, + messages: list[dict], + *, + model: str, + temperature: float, + max_tokens: int, + tools: list[dict] | None = None, + ) -> dict: + kwargs = dict( + model=model, + messages=messages, + temperature=temperature, + max_tokens=max_tokens, + extra_headers=self._extra_headers, + ) + if tools: + kwargs["tools"] = tools + kwargs["tool_choice"] = "auto" + + try: + response = self._client.chat.completions.create(**kwargs) + msg = response.choices[0].message + raw = msg.content or "" + tool_calls = _extract_tool_calls(msg) + + # When the model only returns a tool call (no text), give a placeholder + # so make_result always has something to parse. + if tool_calls and not raw: + raw = f"[tool_call: {tool_calls[0]['name']}]" + + return make_result(raw, self.name, model, tool_calls=tool_calls) + + except _openai_lib.RateLimitError as e: + raise RetryableError(str(e), status_code=429) + except (_openai_lib.APIConnectionError, _openai_lib.APITimeoutError) as e: + raise RetryableError(str(e)) + except _openai_lib.InternalServerError as e: + raise RetryableError(str(e), status_code=getattr(e, "status_code", 500)) + except _openai_lib.AuthenticationError as e: + raise NonRetryableError(str(e), status_code=401) + except (_openai_lib.BadRequestError, _openai_lib.NotFoundError) as e: + raise NonRetryableError(str(e), status_code=getattr(e, "status_code", 400)) + except Exception as e: + # Unknown error — treat as retryable so the registry can decide + raise RetryableError(str(e)) + + # ── Streaming ───────────────────────────────────────────────────────────── + + async def stream( + self, + messages: list[dict], + *, + model: str, + temperature: float, + max_tokens: int, + tools: list[dict] | None = None, + ) -> AsyncGenerator[TextDelta | StreamDone, None]: + assembled = "" + kwargs = dict( + model=model, + messages=messages, + temperature=temperature, + max_tokens=max_tokens, + extra_headers=self._extra_headers, + ) + if tools: + kwargs["tools"] = tools + kwargs["tool_choice"] = "auto" + + try: + response = await self._async_client.chat.completions.create(**kwargs, stream=True) + async for chunk in response: + if not chunk.choices: + continue + + delta = chunk.choices[0].delta + + # Handle reasoning tokens (DeepSeek R1 / OpenRouter) + # These are internal thoughts we don't want to show the user + reasoning = getattr(delta, "reasoning_content", None) or getattr(delta, "reasoning", None) + if reasoning: + continue + + if delta.content: + txt = delta.content + assembled += txt + yield TextDelta(text=txt) + except Exception as e: + logger.error(f"[{self.name}] stream error: {e}") + + result = make_result(assembled, self.name, model) + yield StreamDone( + text=result["text"], + emotion=result["emotion"], + raw=assembled, + provider=self.name, + model=model, + ) + + +# ── Named constructors ──────────────────────────────────────────────────────── + +def openrouter_provider(api_key: str) -> OpenAICompatProvider: + return OpenAICompatProvider( + api_key=api_key, + base_url="https://openrouter.ai/api/v1", + extra_headers=_OPENROUTER_HEADERS, + provider_name="openrouter", + ) + + +def openai_provider(api_key: str) -> OpenAICompatProvider: + return OpenAICompatProvider(api_key=api_key, base_url=None, provider_name="openai") + + +def groq_provider(api_key: str) -> OpenAICompatProvider: + return OpenAICompatProvider( + api_key=api_key, + base_url="https://api.groq.com/openai/v1", + provider_name="groq", + ) + + +def ollama_provider(base_url: str = "http://localhost:11434") -> OpenAICompatProvider: + return OpenAICompatProvider( + api_key="ollama", + base_url=f"{base_url.rstrip('/')}/v1", + provider_name="ollama", + ) diff --git a/ai-service/app/services/providers/registry.py b/ai-service/app/services/providers/registry.py new file mode 100644 index 0000000..8d51c79 --- /dev/null +++ b/ai-service/app/services/providers/registry.py @@ -0,0 +1,297 @@ +""" +Provider Registry — the single entry point for LLM calls. + +Responsibilities: + 1. Read active model / provider / temperature / max_tokens from settings_service + 2. Read the matching API key from settings_service (DB) or fall back to env vars + 3. Instantiate the right LLMProvider + 4. Call provider.generate() and return the normalized result + +Provider inference (when `provider` field is "auto" or missing): + model starts with "claude-" → anthropic + model contains "/" → openrouter (e.g. "deepseek/deepseek-v3.2") + model starts with gpt-/o1-/o3- → openai + model starts with llama/mistral… → ollama + explicit groq_ prefix → groq + fallback → openrouter +""" +from __future__ import annotations + +import logging +import asyncio +import os +import random +import time + +from app.services.providers.base import LLMProvider, RetryableError, NonRetryableError + +logger = logging.getLogger(__name__) + +_MAX_ATTEMPTS = 3 # attempts per provider before giving up on it +_BACKOFF_BASE = 1.0 # seconds; delay = base * 2^attempt + jitter + +# Ordered fallback chain — first provider with an available key wins +_FALLBACK_ORDER = ["openrouter", "openai", "groq", "ollama"] + +# ── Provider inference ──────────────────────────────────────────────────────── + +_OPENAI_PREFIXES = ("gpt-", "o1-", "o3-", "text-davinci", "babbage", "ada") +_OLLAMA_PREFIXES = ("llama", "mistral", "gemma", "phi", "qwen", "codellama", "deepseek-r1") + + +def infer_provider(model: str) -> str: + m = model.lower() + if m.startswith("claude-"): + return "anthropic" + if "/" in m: + return "openrouter" + if any(m.startswith(p) for p in _OPENAI_PREFIXES): + return "openai" + if any(m.startswith(p) for p in _OLLAMA_PREFIXES): + return "ollama" + return "openrouter" + + +# ── Registry ────────────────────────────────────────────────────────────────── + +class ProviderRegistry: + """ + Resolves and calls the correct LLM provider on every request. + Providers are constructed lazily and cached by (provider_name, key_hash). + """ + + def __init__(self): + self._cache: dict[str, LLMProvider] = {} + + # ── Public API ──────────────────────────────────────────────────────────── + + async def generate( + self, + messages: list[dict], + *, + model: str | None = None, + temperature: float | None = None, + max_tokens: int | None = None, + tools: list[dict] | None = None, + ) -> dict: + # Lazy import avoids circular imports at module load time + from app.services.settings_service import settings_service + + db = settings_service.get_settings() + keys = settings_service.get_api_keys() + + actual_model = model or db.get("model") or "deepseek/deepseek-v3.2" + actual_temp = temperature if temperature is not None else float(db.get("temperature", 0.8)) + actual_max_tokens = max_tokens or int(db.get("max_tokens", 300)) + + configured_provider = (db.get("provider") or "auto").lower() + primary = ( + configured_provider + if configured_provider != "auto" + else infer_provider(actual_model) + ) + + # Build candidate list: primary first, then any fallback with an available key + candidates = [primary] + [ + p for p in _FALLBACK_ORDER + if p != primary and (p == "ollama" or self._pick_key(p, keys)) + ] + + call_kwargs = dict( + model=actual_model, + temperature=actual_temp, + max_tokens=actual_max_tokens, + tools=tools, + ) + + last_error: Exception | None = None + + for provider_name in candidates: + try: + provider = self._get_provider(provider_name, keys) + except (ValueError, RuntimeError) as e: + # Missing key or missing package — skip silently + logger.debug(f"[registry] skipping {provider_name}: {e}") + last_error = e + continue + + logger.info(f"[registry] trying {provider_name} / {actual_model}") + try: + result = await self._call_with_retry(provider, messages, **call_kwargs) + if provider_name != primary: + logger.warning(f"[registry] fell back to {provider_name} (primary={primary} failed)") + return result + + except NonRetryableError as e: + last_error = e + if e.status_code == 400: + # Bad request — our message is wrong, no other provider will help + logger.error(f"[registry] bad request ({provider_name}): {e}") + break + # 401 auth failure — key is bad for this provider, try next + logger.warning(f"[registry] auth failed for {provider_name} (HTTP {e.status_code}), trying next") + continue + + except RetryableError as e: + # All retries for this provider exhausted — try next + logger.warning(f"[registry] {provider_name} exhausted retries: {e}") + last_error = e + continue + + logger.error(f"[registry] all providers failed. Last: {last_error}") + return { + "text": "I seem to be having trouble connecting right now. Please try again in a moment.", + "emotion": "confused", + "raw": "", + "provider": primary, + "model": actual_model, + "tool_calls": None, + } + + async def stream( + self, + messages: list[dict], + *, + model: str | None = None, + temperature: float | None = None, + max_tokens: int | None = None, + tools: list[dict] | None = None, + ) -> AsyncGenerator[TextDelta | StreamDone, None]: + from app.services.settings_service import settings_service + + db = settings_service.get_settings() + keys = settings_service.get_api_keys() + + actual_model = model or db.get("model") or "deepseek/deepseek-v3.2" + actual_temp = temperature if temperature is not None else float(db.get("temperature", 0.8)) + actual_max_tokens = max_tokens or int(db.get("max_tokens", 300)) + + configured_provider = (db.get("provider") or "auto").lower() + primary = ( + configured_provider + if configured_provider != "auto" + else infer_provider(actual_model) + ) + + candidates = [primary] + [ + p for p in _FALLBACK_ORDER + if p != primary and (p == "ollama" or self._pick_key(p, keys)) + ] + + # Note: Fallbacks for streaming are harder to implement gracefully mid-stream. + # We try the primary and first available. + for provider_name in candidates: + try: + provider = self._get_provider(provider_name, keys) + logger.info(f"[registry] streaming {provider_name} / {actual_model}") + + async for chunk in provider.stream( + messages, + model=actual_model, + temperature=actual_temp, + max_tokens=actual_max_tokens, + tools=tools + ): + yield chunk + return + except Exception as e: + logger.warning(f"[registry] stream failed for {provider_name}: {e}") + continue + + async def _call_with_retry(self, provider: LLMProvider, messages: list[dict], **kwargs) -> dict: + """ + Call provider.generate() with exponential backoff on RetryableError. + Raises RetryableError if all attempts fail. + Raises NonRetryableError immediately (no retry). + """ + for attempt in range(_MAX_ATTEMPTS): + try: + # Use thread pool for sync generate calls to keep registry async-friendly + return await asyncio.to_thread(provider.generate, messages, **kwargs) + except NonRetryableError: + raise # propagate immediately + except RetryableError as e: + if attempt == _MAX_ATTEMPTS - 1: + raise # all attempts exhausted + delay = _BACKOFF_BASE * (2 ** attempt) + random.uniform(0.0, 0.5) + logger.warning( + f"[{provider.name}] attempt {attempt + 1}/{_MAX_ATTEMPTS} failed " + f"(status={e.status_code}): {e} — retrying in {delay:.1f}s" + ) + await asyncio.sleep(delay) + + # ── Provider instantiation ──────────────────────────────────────────────── + + def _get_provider(self, provider_name: str, keys: dict) -> LLMProvider: + # Cache key: provider name + first 8 chars of api key (detects key rotation) + raw_key = self._pick_key(provider_name, keys) + cache_key = f"{provider_name}:{(raw_key or '')[:8]}" + + if cache_key not in self._cache: + self._cache[cache_key] = self._build(provider_name, keys) + + return self._cache[cache_key] + + def _build(self, provider_name: str, keys: dict) -> LLMProvider: + from app.services.providers.openai_compat import ( + openrouter_provider, openai_provider, groq_provider, ollama_provider, + ) + from app.services.providers.anthropic_provider import AnthropicProvider + + if provider_name == "anthropic": + key = self._pick_key("anthropic", keys) + if not key: + raise ValueError("Anthropic API key not set. Add it via the dashboard or ANTHROPIC_API_KEY env var.") + return AnthropicProvider(api_key=key) + + if provider_name == "groq": + key = self._pick_key("groq", keys) + if not key: + raise ValueError("Groq API key not set. Add it via the dashboard or GROQ_API_KEY env var.") + return groq_provider(api_key=key) + + if provider_name == "openai": + key = self._pick_key("openai", keys) + if not key: + raise ValueError("OpenAI API key not set. Add it via the dashboard or OPENAI_API_KEY env var.") + return openai_provider(api_key=key) + + if provider_name == "ollama": + ollama_url = ( + (keys.get("ollama_base_url") or "").strip() + or os.getenv("OLLAMA_BASE_URL", "http://localhost:11434") + ) + return ollama_provider(base_url=ollama_url) + + # Default: openrouter + key = self._pick_key("openrouter", keys) + if not key: + raise ValueError("OpenRouter API key not set. Add it via the dashboard or OPENROUTER_API_KEY env var.") + return openrouter_provider(api_key=key) + + @staticmethod + def _pick_key(provider_name: str, keys: dict) -> str | None: + """DB key takes precedence over env var.""" + env_map = { + "openrouter": "OPENROUTER_API_KEY", + "openai": "OPENAI_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", + "groq": "GROQ_API_KEY", + } + db_key_map = { + "openrouter": "openrouter_api_key", + "openai": "openrouter_api_key", # share the same field for now + "anthropic": "anthropic_api_key", + "groq": "groq_api_key", + } + + db_field = db_key_map.get(provider_name) + db_val = (keys.get(db_field) or "").strip() if db_field else "" + if db_val: + return db_val + + env_var = env_map.get(provider_name) + return os.getenv(env_var, "") if env_var else "" + + +provider_registry = ProviderRegistry() diff --git a/ai-service/app/services/settings_service.py b/ai-service/app/services/settings_service.py index 194c0f1..a56dbe9 100644 --- a/ai-service/app/services/settings_service.py +++ b/ai-service/app/services/settings_service.py @@ -1,4 +1,5 @@ import logging +import time from supabase import create_client, Client from app.core.config import settings as app_settings @@ -6,20 +7,24 @@ _DEFAULTS = { "system_prompt": None, - "model": "deepseek/deepseek-v3.2", - "temperature": 0.8, - "max_tokens": 300, - "empathy": 50, - "humor": 50, - "formality": 50, + "model": "deepseek/deepseek-v3.2", + "provider": "openrouter", + "temperature": 0.8, + "max_tokens": 300, + "empathy": 50, + "humor": 50, + "formality": 50, } _KEY_DEFAULTS = { "openrouter_api_key": None, - "deepgram_api_key": None, - "cartesia_api_key": None, - "livekit_url": None, - "livekit_api_key": None, + "deepgram_api_key": None, + "cartesia_api_key": None, + "anthropic_api_key": None, + "groq_api_key": None, + "ollama_base_url": "http://localhost:11434", + "livekit_url": None, + "livekit_api_key": None, "livekit_api_secret": None, } @@ -29,23 +34,43 @@ def __init__(self): self._client: Client | None = None if app_settings.SUPABASE_URL and app_settings.SUPABASE_SERVICE_KEY: self._client = create_client(app_settings.SUPABASE_URL, app_settings.SUPABASE_SERVICE_KEY) + + # Simple cache + self._cache = {} + self._cache_expiry = { + "settings": 0, + "keys": 0 + } + self._TTL = 60 # seconds for settings + self._KEY_TTL = 5 # seconds for keys (re-check faster) def get_settings(self) -> dict: if not self._client: return dict(_DEFAULTS) + + now = time.time() + if "settings" in self._cache and now < self._cache_expiry["settings"]: + return self._cache["settings"] + try: result = self._client.table("personality_settings").select("*").eq("id", 1).single().execute() if result.data: - return {**_DEFAULTS, **result.data} + settings = {**_DEFAULTS, **result.data} + self._cache["settings"] = settings + self._cache_expiry["settings"] = now + self._TTL + return settings except Exception as e: logger.warning(f"SettingsService.get_settings failed: {e}") - return dict(_DEFAULTS) + return self._cache.get("settings", dict(_DEFAULTS)) def update_settings(self, patch: dict) -> dict: if not self._client: return dict(_DEFAULTS) try: result = self._client.table("personality_settings").update(patch).eq("id", 1).execute() + # Invalidate cache + if "settings" in self._cache: + del self._cache["settings"] if result.data: return {**_DEFAULTS, **result.data[0]} except Exception as e: @@ -55,19 +80,30 @@ def update_settings(self, patch: dict) -> dict: def get_api_keys(self) -> dict: if not self._client: return dict(_KEY_DEFAULTS) + + now = time.time() + if "keys" in self._cache and now < self._cache_expiry["keys"]: + return self._cache["keys"] + try: result = self._client.table("api_keys").select("*").eq("id", 1).single().execute() if result.data: - return {**_KEY_DEFAULTS, **result.data} + keys = {**_KEY_DEFAULTS, **result.data} + self._cache["keys"] = keys + self._cache_expiry["keys"] = now + self._KEY_TTL + return keys except Exception as e: logger.warning(f"SettingsService.get_api_keys failed: {e}") - return dict(_KEY_DEFAULTS) + return self._cache.get("keys", dict(_KEY_DEFAULTS)) def update_api_keys(self, patch: dict) -> dict: if not self._client: return dict(_KEY_DEFAULTS) try: result = self._client.table("api_keys").update(patch).eq("id", 1).execute() + # Invalidate cache + if "keys" in self._cache: + del self._cache["keys"] if result.data: return {**_KEY_DEFAULTS, **result.data[0]} except Exception as e: diff --git a/ai-service/requirements.txt b/ai-service/requirements.txt index 2c42291..747f7f8 100644 --- a/ai-service/requirements.txt +++ b/ai-service/requirements.txt @@ -3,6 +3,7 @@ aiohttp==3.13.3 aiosignal==1.4.0 annotated-doc==0.0.4 annotated-types==0.7.0 +anthropic anyio==4.11.0 attrs==25.4.0 cachetools==6.2.6 diff --git a/ai-service/tests/__init__.py b/ai-service/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ai-service/tests/conftest.py b/ai-service/tests/conftest.py new file mode 100644 index 0000000..8eb1151 --- /dev/null +++ b/ai-service/tests/conftest.py @@ -0,0 +1,71 @@ +""" +Shared pytest fixtures and env setup. +Loads the project .env so integration tests can use real API keys. +""" +import os +import sys +from pathlib import Path + +import pytest +from dotenv import load_dotenv + +# ── Add ai-service root to sys.path so `app.*` imports resolve ─────────────── +AI_SERVICE_DIR = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(AI_SERVICE_DIR)) + +# ── Load .env from project root ─────────────────────────────────────────────── +PROJECT_ROOT = AI_SERVICE_DIR.parent +env_path = PROJECT_ROOT / ".env" +if not env_path.exists(): + env_path = AI_SERVICE_DIR / ".env" +load_dotenv(env_path) + + +# ── Reusable message lists ──────────────────────────────────────────────────── + +@pytest.fixture +def simple_messages(): + return [ + {"role": "system", "content": "You are a helpful assistant. Reply very briefly."}, + {"role": "user", "content": "Say exactly: [smile] Hello!"}, + ] + + +@pytest.fixture +def tool_messages(): + return [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is the weather in Tokyo? Use the get_weather tool."}, + ] + + +@pytest.fixture +def sample_tools(): + return [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather for a city.", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string", "description": "City name"}, + }, + "required": ["city"], + }, + }, + } + ] + + +# ── Key availability helpers (used by integration marks) ───────────────────── + +def has_openrouter_key(): + return bool(os.getenv("OPENROUTER_API_KEY", "").strip()) + +def has_openai_key(): + return bool(os.getenv("OPENAI_API_KEY", "").strip()) + +def has_anthropic_key(): + return bool(os.getenv("ANTHROPIC_API_KEY", "").strip()) diff --git a/ai-service/tests/providers/__init__.py b/ai-service/tests/providers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dashboard/package.json b/dashboard/package.json index c7fdc9a..465db32 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -15,7 +15,7 @@ "@supabase/supabase-js": "^2.95.3", "@tailwindcss/vite": "^4.1.18", "livekit-client": "^2.17.1", - "pixi.js": "^6.5.10", + "pixi.js": "^8.17.1", "react": "^19.2.0", "react-dom": "^19.2.0", "react-router-dom": "^7.13.0", @@ -39,4 +39,4 @@ "overrides": { "vite": "npm:rolldown-vite@7.2.2" } -} +} \ No newline at end of file diff --git a/dashboard/src/components/ApiKeys.jsx b/dashboard/src/components/ApiKeys.jsx index 037d9b8..b2e2369 100644 --- a/dashboard/src/components/ApiKeys.jsx +++ b/dashboard/src/components/ApiKeys.jsx @@ -3,10 +3,13 @@ import { supabase } from '../lib/supabaseClient' const KEY_GROUPS = [ { - label: 'LLM Provider', + label: 'LLM Providers', icon: 'psychology', fields: [ - { key: 'openrouter_api_key', label: 'OpenRouter API Key', placeholder: 'sk-or-v1-...' }, + { key: 'openrouter_api_key', label: 'OpenRouter API Key', placeholder: 'sk-or-v1-...', hint: 'Routes to DeepSeek, GPT, Mistral, and more' }, + { key: 'anthropic_api_key', label: 'Anthropic API Key', placeholder: 'sk-ant-...', hint: 'Required for claude-* models' }, + { key: 'groq_api_key', label: 'Groq API Key', placeholder: 'gsk_...', hint: 'Fast inference for Llama / Mixtral' }, + { key: 'ollama_base_url', label: 'Ollama Base URL', placeholder: 'http://localhost:11434', hint: 'Local LLMs via Ollama', isUrl: true }, ], }, { @@ -14,13 +17,13 @@ const KEY_GROUPS = [ icon: 'mic', fields: [ { key: 'deepgram_api_key', label: 'Deepgram API Key (STT)', placeholder: 'your_deepgram_key' }, - { key: 'cartesia_api_key', label: 'Cartesia API Key (TTS)', placeholder: 'your_cartesia_key', note: 'Requires agent restart to apply' }, + { key: 'cartesia_api_key', label: 'Cartesia API Key (TTS)', placeholder: 'your_cartesia_key', note: 'Requires agent restart' }, ], }, { label: 'LiveKit', icon: 'cell_tower', - note: 'Changes require agent restart', + note: 'Requires agent restart', fields: [ { key: 'livekit_url', label: 'LiveKit URL', placeholder: 'wss://your-project.livekit.cloud' }, { key: 'livekit_api_key', label: 'LiveKit API Key', placeholder: 'API key' }, @@ -32,24 +35,16 @@ const KEY_GROUPS = [ export default function ApiKeys() { const [draft, setDraft] = useState({}) const [visible, setVisible] = useState({}) - const [saveState, setSaveState] = useState('idle') // 'idle' | 'saving' | 'saved' | 'error' + const [saveState, setSaveState] = useState('idle') const [loaded, setLoaded] = useState(false) useEffect(() => { - supabase - .from('api_keys') - .select('*') - .eq('id', 1) - .single() - .then(({ data }) => { - if (data) setDraft(data) - setLoaded(true) - }) + supabase.from('api_keys').select('*').eq('id', 1).single() + .then(({ data }) => { if (data) setDraft(data); setLoaded(true) }) }, []) - const patch = (key, value) => setDraft((d) => ({ ...d, [key]: value })) - - const toggleVisible = (key) => setVisible((v) => ({ ...v, [key]: !v[key] })) + const patch = (key, value) => setDraft(d => ({ ...d, [key]: value })) + const toggleVisible = key => setVisible(v => ({ ...v, [key]: !v[key] })) const saveKeys = async () => { setSaveState('saving') @@ -57,12 +52,7 @@ export default function ApiKeys() { const payload = { ...draft } delete payload.id payload.updated_at = new Date().toISOString() - - const { error } = await supabase - .from('api_keys') - .update(payload) - .eq('id', 1) - + const { error } = await supabase.from('api_keys').update(payload).eq('id', 1) if (error) throw error setSaveState('saved') setTimeout(() => setSaveState('idle'), 2500) @@ -73,11 +63,11 @@ export default function ApiKeys() { } } - const btnProps = { - idle: { label: 'Save API Keys', icon: 'key', cls: 'bg-primary hover:bg-primary/90 shadow-primary/20' }, - saving: { label: 'Saving...', icon: 'hourglass_top', cls: 'bg-primary/70 cursor-not-allowed' }, - saved: { label: 'Keys Saved!', icon: 'check_circle', cls: 'bg-emerald-500 shadow-emerald-200' }, - error: { label: 'Save Failed', icon: 'error', cls: 'bg-red-500 shadow-red-200' }, + const btn = { + idle: { label: 'Save API Keys', icon: 'key', cls: 'bg-primary hover:bg-primary/90 shadow-primary/20' }, + saving: { label: 'Saving...', icon: 'hourglass_top', cls: 'bg-primary/70 cursor-not-allowed' }, + saved: { label: 'Keys Saved!', icon: 'check_circle', cls: 'bg-emerald-500 shadow-emerald-200' }, + error: { label: 'Save Failed', icon: 'error', cls: 'bg-red-500 shadow-red-200' }, }[saveState] return ( @@ -90,10 +80,10 @@ export default function ApiKeys() { @@ -105,37 +95,35 @@ export default function ApiKeys() { {label} {note && ( - info - {note} + info{note} )}
- {fields.map(({ key, label: fieldLabel, placeholder, note: fieldNote }) => ( + {fields.map(({ key, label: fl, placeholder, note: fn, hint, isUrl }) => (
- + + {hint &&

{hint}

}
patch(key, e.target.value)} + onChange={e => patch(key, e.target.value)} placeholder={loaded ? placeholder : '••••••••'} className="w-full bg-bg-light border border-slate-200 rounded-lg px-3 py-2 pr-10 text-sm font-mono focus:ring-1 focus:ring-primary focus:border-primary outline-none" /> - + {!isUrl && ( + + )}
- {fieldNote && ( + {fn && (

- info - {fieldNote} + info{fn}

)}
@@ -147,7 +135,7 @@ export default function ApiKeys() {

lock - Keys are stored in your private Supabase database. Leave a field empty to use the value from the server's .env file. + Stored in your private Supabase database. Leave a field empty to use the server's .env value.

) diff --git a/dashboard/src/components/AvatarRenderer.jsx b/dashboard/src/components/AvatarRenderer.jsx index 73c9081..23fd6cd 100644 --- a/dashboard/src/components/AvatarRenderer.jsx +++ b/dashboard/src/components/AvatarRenderer.jsx @@ -1,57 +1,104 @@ /** - * AvatarRenderer — Phase 2 - * Renders the Hu Tao Live2D model on a transparent canvas using - * pixi-live2d-display. Exposes an imperative ref API so CallOverlay - * can drive expressions in sync with AURA's speech. + * AvatarRenderer — Phase 3 + * Idle / Speaking state machine with richer moods and cute micro-animations: + * • 6 weighted moods per state (neutral, happy, curious, playful, sleepy, thinking) + * • Cute head-tilt event during idle + * • Occasional double-blink during idle + * • Sleepy: half-closed eyes, slow blink + * • Speaking: gentle nod, tighter saccade, snappier blink, slight smile boost * - * Usage: - * const avatarRef = useRef(null) - * - * avatarRef.current.setExpression(['smile', 'shadow'], 2.3) - * avatarRef.current.resetNeutral() + * Ref API: + * setExpression(names[], duration) — play expression(s) for N seconds + * setSpeaking(bool) — switch idle ↔ speaking state + * setMouthOpen(0–1) — drive lip sync each frame + * setParameter(id, value) — raw Core Model parameter override + * resetNeutral() — cancel active expression, return to idle */ import { forwardRef, useEffect, useImperativeHandle, useRef } from 'react' import * as PIXI from 'pixi.js' import { Live2DModel } from 'pixi-live2d-display/cubism4' -// Register PIXI Ticker so Live2D animations update every frame Live2DModel.registerTicker(PIXI.Ticker) -// Model path relative to dashboard/public/ const MODEL_URL = '/models/hutao/Hu Tao.model3.json' -/** - * Per-expression Cubism 4 parameter overrides. - * Applied smoothly every frame in coreModel.update() to flawlessly - * override the base idle animation values. - */ -const EXPRESSION_OVERRIDES = { - smile: { ParamMouthForm: 1.0, ParamEyeLSmile: 0.9, ParamEyeRSmile: 0.9, Param37: 0.4 }, - sad: { ParamMouthForm: -1.0, ParamBrowLForm: -1.0, ParamBrowRForm: -1.0, ParamBrowLAngle: 0.75, ParamBrowRAngle: 0.75 }, - angry: { ParamMouthForm: -0.5, ParamEyeRSmile: 0.0, ParamEyeLSmile: 0.0, ParamBrowLAngle: -1.0, ParamBrowRAngle: -1.0, ParamBrowRForm: -0.5, ParamBrowLForm: -0.5 }, - ghost: { Param80: 1.0 }, - ghost_nervous: { Param75: 1.0 }, - shadow: { Param2: 1.0 }, - pupil_shrink: { Param38: 1.0 }, - eyeshine_off: { Param3: 1.0 }, - wink: { ParamEyeLOpen: 0.0, ParamEyeLSmile: 1.0, ParamBrowLForm: 0.5, ParamMouthForm: 0.5 }, - tongue: { Param70: 1.0, ParamMouthForm: -1.0 }, // Param70 = TongueOut mesh +const EXPRESSION_FILES = { + smile: 'SmileLock.exp3.json', + sad: 'SadLock.exp3.json', + angry: 'Angry.exp3.json', + ghost: 'Ghost.exp3.json', + ghost_nervous: 'GhostChange.exp3.json', + shadow: 'Shadow.exp3.json', + pupil_shrink: 'PupilShrink.exp3.json', + eyeshine_off: 'EyeshineOff.exp3.json', +} + +// Maps LLM-annotated expression names → the closest ambient mood. +// Applied after the expression fades so the idle baseline stays emotionally coherent. +const EXPRESSION_TO_MOOD = { + smile: 'happy', + sad: 'neutral', // no sad mood — settle to calm neutral + angry: 'thinking', // furrowed brows, withdrawn + ghost: 'playful', // mischievous + ghost_nervous: 'curious', // uncertain, alert + shadow: 'thinking', // serious / dark + pupil_shrink: 'curious', // surprised / wide-eyed + eyeshine_off: 'sleepy', // dull / fatigued + wink: 'playful', + tongue: 'playful', +} + +// ── State machine ────────────────────────────────────────────────────────── +const STATE = { IDLE: 'idle', SPEAKING: 'speaking' } + +// ── Mood definitions (target parameter values) ───────────────────────────── +const MOODS = { + neutral: { mouthForm: 0, browForm: 0, browRaise: 0, eyeSmile: 0 }, + happy: { mouthForm: 0.65, browForm: 0.30, browRaise: 0.45, eyeSmile: 0.55 }, + curious: { mouthForm: 0.20, browForm: -0.10, browRaise: 0.50, eyeSmile: 0 }, + playful: { mouthForm: 0.90, browForm: 0.50, browRaise: 0.70, eyeSmile: 0.30 }, + sleepy: { mouthForm: -0.05, browForm: 0.10, browRaise: -0.15, eyeSmile: 0 }, + thinking: { mouthForm: 0.10, browForm: -0.20, browRaise: 0.35, eyeSmile: 0 }, } -let sharedApp = null -let sharedModelPromise = null -let sharedModel = null -let sharedMouthOpen = 0 -let sharedExpressionOverride = null +// Weighted mood pool per state — [moodKey, weight], weights sum to 1.0 +const MOOD_POOLS = { + [STATE.IDLE]: [ + ['neutral', 0.15], ['happy', 0.35], ['curious', 0.20], + ['playful', 0.10], ['sleepy', 0.10], ['thinking', 0.10], + ], + [STATE.SPEAKING]: [ + ['neutral', 0.10], ['happy', 0.45], ['curious', 0.20], + ['playful', 0.20], ['thinking', 0.05], + ], +} -function initSharedApp(width, height) { - if (sharedApp) { - sharedApp.renderer.resize(width, height) - return +function pickWeightedMood(state) { + const pool = MOOD_POOLS[state] ?? MOOD_POOLS[STATE.IDLE] + const r = Math.random() + let acc = 0 + for (const [key, w] of pool) { + acc += w + if (r < acc) return MOODS[key] } + return MOODS.neutral +} + +// ── Module-scoped Singleton State ────────────────────────────────────────── +let _app = null +let _model = null +let _loaded = false +let _mouthOpen = 0 +let _expressionActive = false +let _mouthYLocked = false // true while tongue expression holds MouthOpenY +let _state = STATE.IDLE +let _pendingMood = null // set by setExpression, consumed by update loop on expiry + +function initSingleton(width, height) { + if (_app) return - sharedApp = new PIXI.Application({ + _app = new PIXI.Application({ backgroundAlpha: 0, width, height, @@ -60,193 +107,300 @@ function initSharedApp(width, height) { autoDensity: true, }) - sharedModelPromise = Live2DModel.from(MODEL_URL, { autoInteract: false }).then((model) => { - sharedModel = model - sharedApp.stage.addChild(model) - - const logicalW = sharedApp.screen.width - const logicalH = sharedApp.screen.height - const autoScale = (logicalH / model.height) * 1.9 - model.scale.set(autoScale) - model.anchor.set(0.5, 0.0) - model.position.set(logicalW * 0.5, 0) - - const core = model.internalModel.coreModel - let lastMs = performance.now() - const clamp = (v, lo, hi) => v < lo ? lo : v > hi ? hi : v - - let blinkTimer = 0, blinkPhase = 0, nextBlink = 2 + Math.random() * 4 - let saccadeTimer = 0, nextSaccade = 1 + Math.random() * 2 - let eyeTargetX = 0, eyeTargetY = 0, eyeX = 0, eyeY = 0 - - let moodTimer = 0, nextMoodChange = 3 + Math.random() * 4 - let mouthFormT = 0, mouthFormC = 0 - let browFormT = 0, browFormC = 0 - let browRaiseT = 0, browRaiseC = 0 - let eyeSmileT = 0, eyeSmileC = 0 - - function pickMood() { - const roll = Math.random() - if (roll < 0.30) { - mouthFormT = 0; browFormT = 0; browRaiseT = 0; eyeSmileT = 0 - } else if (roll < 0.60) { - mouthFormT = 0.55 + Math.random() * 0.35 - browFormT = 0.35; browRaiseT = 0.4; eyeSmileT = 0.45 - } else if (roll < 0.80) { - mouthFormT = -0.1; browFormT = 0.1; browRaiseT = 0.2; eyeSmileT = 0 - eyeTargetY = 0.45 + Math.random() * 0.3 - nextSaccade = saccadeTimer + 2.8 - } else { - mouthFormT = 0.9; browFormT = 0.5; browRaiseT = 0.7; eyeSmileT = 0.25 - } - nextMoodChange = 3 + Math.random() * 5 - } - - const currentOverrides = {} - - sharedApp.ticker.add(() => { - if (!sharedModel) return - const now = performance.now() / 1000 - const elapsed = Math.min((performance.now() - lastMs) / 1000, 0.1) - lastMs = performance.now() - - core.setParameterValueById('ParamAngleX', Math.sin(now * 0.31) * 12 + Math.sin(now * 0.73) * 3) - core.setParameterValueById('ParamAngleY', Math.sin(now * 0.19) * 5 + Math.sin(now * 0.47) * 2) - core.setParameterValueById('ParamAngleZ', Math.sin(now * 0.13) * 5 + Math.sin(now * 0.41) * 2) - core.setParameterValueById('ParamBodyAngleX', Math.sin(now * 0.28) * 4) - core.setParameterValueById('ParamBodyAngleZ', Math.sin(now * 0.21) * 3) - core.setParameterValueById('ParamBreath', Math.sin(now * 0.9) * 0.5 + 0.5) - core.setParameterValueById('ParamMouthOpenY', sharedMouthOpen) - - moodTimer += elapsed - if (moodTimer >= nextMoodChange) { moodTimer = 0; pickMood() } - const lm = elapsed * 4 - mouthFormC += (mouthFormT - mouthFormC) * lm - browFormC += (browFormT - browFormC) * lm - browRaiseC += (browRaiseT - browRaiseC) * lm - eyeSmileC += (eyeSmileT - eyeSmileC) * lm - core.setParameterValueById('ParamMouthForm', mouthFormC) - core.setParameterValueById('ParamBrowLForm', browFormC) - core.setParameterValueById('ParamBrowRForm', browFormC) - core.setParameterValueById('Param37', browRaiseC) - core.setParameterValueById('ParamEyeLSmile', eyeSmileC) - core.setParameterValueById('ParamEyeRSmile', eyeSmileC) - - saccadeTimer += elapsed - if (saccadeTimer >= nextSaccade) { - eyeTargetX = (Math.random() * 2 - 1) * 0.65 - const r = Math.random() - if (r < 0.20) eyeTargetY = 0.5 + Math.random() * 0.35 - else if (r < 0.35) eyeTargetY = -0.3 - Math.random() * 0.25 - else eyeTargetY = (Math.random() * 2 - 1) * 0.4 - nextSaccade = saccadeTimer + 1.5 + Math.random() * 2.5 - } - eyeX += (eyeTargetX - eyeX) * elapsed * 3.5 - eyeY += (eyeTargetY - eyeY) * elapsed * 3.5 - core.setParameterValueById('ParamEyeBallX', clamp(eyeX, -1, 1)) - core.setParameterValueById('ParamEyeBallY', clamp(eyeY, -1, 1)) - - blinkTimer += elapsed - const bspd = 9 - if (blinkPhase === 0 && blinkTimer >= nextBlink) { blinkPhase = 1; blinkTimer = 0 } - if (blinkPhase === 1) { - const v = clamp(1 - blinkTimer * bspd, 0, 1) - core.setParameterValueById('ParamEyeLOpen', v) - core.setParameterValueById('ParamEyeROpen', v) - if (v <= 0) { blinkPhase = 2; blinkTimer = 0 } - } else if (blinkPhase === 2) { - const v = clamp(blinkTimer * bspd, 0, 1) - core.setParameterValueById('ParamEyeLOpen', v) - core.setParameterValueById('ParamEyeROpen', v) - if (v >= 1) { blinkPhase = 0; blinkTimer = 0; nextBlink = 3 + Math.random() * 5 } - } else { - core.setParameterValueById('ParamEyeLOpen', 1) - core.setParameterValueById('ParamEyeROpen', 1) - } - - const targetOv = sharedExpressionOverride + Live2DModel.from(MODEL_URL, { autoInteract: false }) + .then((model) => { + _model = model + _app.stage.addChild(model) + + const logicalW = _app.screen.width + const logicalH = _app.screen.height + const autoScale = (logicalH / model.height) * 1.4 + model.scale.set(autoScale) + model.anchor.set(0.5, 0.0) + model.position.set(logicalW * 0.5, 0) + + const core = model.internalModel.coreModel + const clamp = (v, lo, hi) => Math.max(lo, Math.min(hi, v)) + let lastMs = performance.now() + + // ── Blink state ────────────────────────────────────────────────────── + let blinkTimer = 0, blinkPhase = 0, nextBlink = 2 + Math.random() * 3 + // Double-blink: blink twice in quick succession (cute quirk) + let dblBlinkPending = false + let dblBlinkTimer = 0, nextDblBlink = 10 + Math.random() * 10 + + // ── Saccade state ───────────────────────────────────────────────────── + let saccadeTimer = 0, nextSaccade = 1 + Math.random() * 2 + let eyeTargetX = 0, eyeTargetY = 0, eyeX = 0, eyeY = 0 + + // ── Mood state ──────────────────────────────────────────────────────── + let moodTimer = 0, nextMoodChange = 3 + Math.random() * 4 + let currentMood = MOODS.happy + let mouthFormC = 0, browFormC = 0, browRaiseC = 0, eyeSmileC = 0 + + // ── Head tilt micro-animation (idle only) ───────────────────────────── + // Occasionally snaps to a cute side-tilt, holds briefly, then eases back + let tiltTimer = 0, nextTilt = 6 + Math.random() * 8 + let tiltTarget = 0, tiltC = 0 + let tiltHolding = false, tiltHoldTimer = 0, tiltHoldDuration = 0 + + // ── Speaking nod ────────────────────────────────────────────────────── + let nodPhase = 0 + + const origCoreUpdate = core.update.bind(core) + + core.update = function () { + const now = performance.now() / 1000 + const elapsed = Math.min((performance.now() - lastMs) / 1000, 0.1) + lastMs = performance.now() + + const speaking = _state === STATE.SPEAKING + const lerpSpd = speaking ? 5.0 : 3.5 + + // ── Breathing ──────────────────────────────────────────────────── + // Slightly faster when speaking (more energetic) + core.setParameterValueById('ParamBreath', + Math.sin(now * (speaking ? 1.1 : 0.75)) * 0.5 + 0.5) + + // ── Head movement ───────────────────────────────────────────────── + const swayAmt = speaking ? 0.35 : 1.0 + const bX = (Math.sin(now * 0.31) * 12 + Math.sin(now * 0.73) * 3) * swayAmt + const bY = (Math.sin(now * 0.19) * 5 + Math.sin(now * 0.47) * 2) * swayAmt + const bZ = (Math.sin(now * 0.13) * 5 + Math.sin(now * 0.41) * 2) * swayAmt + + // Gentle speaking nod — Y oscillation in rough speech rhythm + let nodY = 0 + if (speaking) { + nodPhase += elapsed * 2.6 + nodY = Math.sin(nodPhase) * 3.5 + } else { + nodPhase = 0 + } - const overrideKeys = new Set([...Object.keys(currentOverrides), ...(targetOv ? Object.keys(targetOv) : [])]) - const lerpSpeed = elapsed * 5 + // Cute idle head tilt — snap in quickly, ease back slowly + if (!speaking) { + tiltTimer += elapsed + if (!tiltHolding && tiltTimer >= nextTilt) { + tiltTarget = (Math.random() < 0.5 ? 1 : -1) * (7 + Math.random() * 7) + tiltTimer = 0 + nextTilt = 6 + Math.random() * 8 + tiltHolding = true + tiltHoldTimer = 0 + tiltHoldDuration = 0.9 + Math.random() * 0.8 + } + } + if (tiltHolding) { + tiltHoldTimer += elapsed + if (tiltHoldTimer >= tiltHoldDuration) { tiltTarget = 0; tiltHolding = false } + } + tiltC += (tiltTarget - tiltC) * elapsed * (tiltTarget !== 0 ? 6.0 : 2.2) + + core.setParameterValueById('ParamAngleX', bX) + core.setParameterValueById('ParamAngleY', bY + nodY) + core.setParameterValueById('ParamAngleZ', bZ + tiltC) + core.setParameterValueById('ParamBodyAngleX', Math.sin(now * 0.28) * 4 * swayAmt) + core.setParameterValueById('ParamBodyAngleZ', Math.sin(now * 0.21) * 3 * swayAmt) + + // ── Lip sync ────────────────────────────────────────────────────── + // Skip when tongue expression is holding MouthOpenY at 1.0 + if (!_mouthYLocked) core.setParameterValueById('ParamMouthOpenY', _mouthOpen) + + // ── Mood interpolation ──────────────────────────────────────────── + if (!_expressionActive) { + // Expression just expired — align ambient mood to the emotion the LLM set + if (_pendingMood) { + currentMood = _pendingMood + _pendingMood = null + moodTimer = 0 + nextMoodChange = 3 + Math.random() * 3 // hold this mood for 3-6s before drifting + } + + moodTimer += elapsed + if (moodTimer >= nextMoodChange) { + moodTimer = 0 + nextMoodChange = speaking + ? 2 + Math.random() * 2.5 + : 3 + Math.random() * 5 + currentMood = pickWeightedMood(_state) + + // Curious: look upward with a lingering gaze + if (currentMood === MOODS.curious) { + eyeTargetY = 0.45 + Math.random() * 0.30 + nextSaccade = saccadeTimer + 3 + } + // Thinking: look up-left (classic thinking glance) + if (currentMood === MOODS.thinking) { + eyeTargetX = -(0.4 + Math.random() * 0.3) + eyeTargetY = 0.4 + Math.random() * 0.3 + nextSaccade = saccadeTimer + 4 + } + } + + const lm = elapsed * lerpSpd + mouthFormC += (currentMood.mouthForm - mouthFormC) * lm + browFormC += (currentMood.browForm - browFormC) * lm + browRaiseC += (currentMood.browRaise - browRaiseC) * lm + eyeSmileC += (currentMood.eyeSmile - eyeSmileC) * lm + + // Speaking: add a slight smile boost (engaged / expressive look) + const mfBoost = speaking ? 0.20 : 0 + core.setParameterValueById('ParamMouthForm', clamp(mouthFormC + mfBoost, -1, 1)) + core.setParameterValueById('ParamBrowLForm', browFormC) + core.setParameterValueById('ParamBrowRForm', browFormC) + core.setParameterValueById('Param37', browRaiseC) + core.setParameterValueById('ParamEyeLSmile', eyeSmileC) + core.setParameterValueById('ParamEyeRSmile', eyeSmileC) + } - for (const id of overrideKeys) { - let targetVal = 0 - let isIdleParam = false + // ── Saccade ─────────────────────────────────────────────────────── + saccadeTimer += elapsed + if (saccadeTimer >= nextSaccade) { + if (speaking) { + // Focus on "listener" — small central range, frequent updates + eyeTargetX = (Math.random() * 2 - 1) * 0.25 + eyeTargetY = (Math.random() * 2 - 1) * 0.15 + nextSaccade = saccadeTimer + 0.8 + Math.random() * 1.0 + } else { + eyeTargetX = (Math.random() * 2 - 1) * 0.65 + const r = Math.random() + if (r < 0.20) eyeTargetY = 0.5 + Math.random() * 0.35 + else if (r < 0.35) eyeTargetY = -0.3 - Math.random() * 0.25 + else eyeTargetY = (Math.random() * 2 - 1) * 0.4 + nextSaccade = saccadeTimer + 1.5 + Math.random() * 2.5 + } + } + const gzSpd = speaking ? 5.0 : 3.5 + eyeX += (eyeTargetX - eyeX) * elapsed * gzSpd + eyeY += (eyeTargetY - eyeY) * elapsed * gzSpd + core.setParameterValueById('ParamEyeBallX', clamp(eyeX, -1, 1)) + core.setParameterValueById('ParamEyeBallY', clamp(eyeY, -1, 1)) + + // ── Double-blink scheduler (idle only) ──────────────────────────── + if (!speaking) { + dblBlinkTimer += elapsed + if (dblBlinkTimer >= nextDblBlink) { + dblBlinkPending = true + dblBlinkTimer = 0 + nextDblBlink = 10 + Math.random() * 12 + } + } - if (id === 'ParamMouthForm') { isIdleParam = true; targetVal = mouthFormC; } - else if (id === 'ParamBrowLForm' || id === 'ParamBrowRForm') { isIdleParam = true; targetVal = browFormC; } - else if (id === 'Param37') { isIdleParam = true; targetVal = browRaiseC; } - else if (id === 'ParamEyeLSmile' || id === 'ParamEyeRSmile') { isIdleParam = true; targetVal = eyeSmileC; } - else if (id === 'ParamEyeLOpen' || id === 'ParamEyeROpen') { isIdleParam = true; targetVal = core.getParameterValueById(id); } + // ── Blink ───────────────────────────────────────────────────────── + const isSleepy = currentMood === MOODS.sleepy + // Speaking: snappy blink (11). Sleepy: slow droopy blink (6). Normal: 9 + const bspd = speaking ? 11 : (isSleepy ? 6 : 9) + blinkTimer += elapsed - if (targetOv && targetOv[id] !== undefined) { - targetVal = targetOv[id] + // Don't start a new blink while an expression is holding eye parameters (e.g. wink) + if (blinkPhase === 0 && blinkTimer >= nextBlink && !_expressionActive) { + blinkPhase = 1; blinkTimer = 0 } - - if (currentOverrides[id] === undefined) { - currentOverrides[id] = isIdleParam ? targetVal : 0; + if (blinkPhase === 1) { + const v = clamp(1 - blinkTimer * bspd, 0, 1) + core.setParameterValueById('ParamEyeLOpen', v) + core.setParameterValueById('ParamEyeROpen', v) + if (v <= 0) { blinkPhase = 2; blinkTimer = 0 } + } else if (blinkPhase === 2) { + const v = clamp(blinkTimer * bspd, 0, 1) + core.setParameterValueById('ParamEyeLOpen', v) + core.setParameterValueById('ParamEyeROpen', v) + if (v >= 1) { + blinkPhase = 0; blinkTimer = 0 + if (dblBlinkPending) { + nextBlink = 0.06 + Math.random() * 0.08 // blink again almost immediately + dblBlinkPending = false + } else if (isSleepy) { + nextBlink = 1.5 + Math.random() * 2.0 // sleepy: blinks more often + } else if (speaking) { + nextBlink = 4.0 + Math.random() * 3.0 // speaking: eyes stay open longer + } else { + nextBlink = 3.0 + Math.random() * 5.0 // normal idle + } + } + } else { + // Resting open — sleepy mode: eyes only 72% open (heavy lidded) + if (!_expressionActive) { + const restOpen = isSleepy ? 0.72 : 1.0 + core.setParameterValueById('ParamEyeLOpen', restOpen) + core.setParameterValueById('ParamEyeROpen', restOpen) + } } - currentOverrides[id] += (targetVal - currentOverrides[id]) * lerpSpeed - core.setParameterValueById(id, currentOverrides[id]) + origCoreUpdate() } - if (currentOverrides['Param70'] > 0) { - const currentMouth = core.getParameterValueById('ParamMouthOpenY') - core.setParameterValueById('ParamMouthOpenY', Math.max(currentMouth, currentOverrides['Param70'] * 0.4)) - } + _loaded = true }) - }).catch((err) => { - console.error('[AvatarRenderer] Failed to load Live2D model:', err) - }) + .catch((err) => console.error('[AvatarRenderer] Failed to load Live2D model:', err)) } export const AvatarRenderer = forwardRef(function AvatarRenderer(props, ref) { const { width = 400, height = 600 } = props const containerRef = useRef(null) - // ── Boot PIXI + load model ──────────────────────────────────────────────── useEffect(() => { - initSharedApp(width, height) - - if (sharedApp && sharedApp.view && containerRef.current) { - containerRef.current.appendChild(sharedApp.view) - } - + initSingleton(width, height) + const container = containerRef.current + if (container && _app) container.appendChild(_app.view) return () => { - if (sharedApp && sharedApp.view && containerRef.current && sharedApp.view.parentNode === containerRef.current) { - containerRef.current.removeChild(sharedApp.view) - } + if (container && _app && _app.view.parentNode === container) + container.removeChild(_app.view) } }, [width, height]) - // ── Imperative API ──────────────────────────────────────────────────────── useImperativeHandle(ref, () => ({ setExpression(names, duration) { - if (!sharedModel) return + if (!_loaded || !_model) return + _expressionActive = true - const merged = {} + // Queue the mood that best matches this expression — applied when it expires for (const name of names) { - const overrides = EXPRESSION_OVERRIDES[name] - if (overrides) Object.assign(merged, overrides) + const moodKey = EXPRESSION_TO_MOOD[name] + if (moodKey) { _pendingMood = MOODS[moodKey]; break } } - sharedExpressionOverride = Object.keys(merged).length > 0 ? merged : null + const merged = {} + for (const name of names) { + const file = EXPRESSION_FILES[name] + if (file) _model.expression(file) + if (name === 'wink') { + const c = _model.internalModel.coreModel + c.setParameterValueById('ParamEyeLOpen', 0.0) + c.setParameterValueById('ParamBrowLForm', -1.0) + c.setParameterValueById('ParamMouthForm', 1.0) + } + if (name === 'tongue') { + _mouthYLocked = true // prevent lip-sync loop from overriding MouthOpenY + const c = _model.internalModel.coreModel + // Hu Tao specific: Param70 is TongueOut + c.setParameterValueById('Param70', 1.0) + c.setParameterValueById('ParamMouthOpenY', 1.0) + c.setParameterValueById('ParamMouthForm', -1.0) + } + } setTimeout(() => { - sharedExpressionOverride = null + _expressionActive = false + _mouthYLocked = false + if (_model) _model.expression() }, duration * 1000) }, + /** Switch between idle and speaking animation state */ + setSpeaking(active) { + _state = active ? STATE.SPEAKING : STATE.IDLE + }, + setParameter(name, value) { - sharedModel?.internalModel.coreModel.setParameterValueById(name, value) + _model?.internalModel.coreModel.setParameterValueById(name, value) }, resetNeutral() { - sharedExpressionOverride = null + _expressionActive = false + _model?.expression() }, setMouthOpen(v) { - sharedMouthOpen = Math.max(0, Math.min(1, v)) + _mouthOpen = Math.max(0, Math.min(1, v)) }, }), []) diff --git a/dashboard/src/components/AvatarRenderer.test.jsx b/dashboard/src/components/AvatarRenderer.test.jsx index ea12b4e..f7469f4 100644 --- a/dashboard/src/components/AvatarRenderer.test.jsx +++ b/dashboard/src/components/AvatarRenderer.test.jsx @@ -1,7 +1,7 @@ /** - * Phase 2 tests — AvatarRenderer component + * AvatarRenderer tests — Phase 3 * All GPU / PIXI / Live2D dependencies are mocked so these run in jsdom - * without a real GPU or network. + * without a real WebGL context or network. * * Run: cd dashboard && npm test */ @@ -11,11 +11,14 @@ import { render, act } from '@testing-library/react' import { createRef } from 'react' import { AvatarRenderer } from './AvatarRenderer' -// ── Mock heavy GPU dependencies ──────────────────────────────────────────── +// ── Mocks ────────────────────────────────────────────────────────────────── const mockSetParameterValueById = vi.fn() const mockExpression = vi.fn() +const mockCoreUpdate = vi.fn() + const mockModel = { + height: 600, // needed for auto-scale calculation expression: mockExpression, scale: { set: vi.fn() }, anchor: { set: vi.fn() }, @@ -23,30 +26,31 @@ const mockModel = { internalModel: { coreModel: { setParameterValueById: mockSetParameterValueById, - update: vi.fn(), + update: mockCoreUpdate, // needed for core.update.bind() in initSingleton }, }, } -const mockStage = { addChild: vi.fn() } -const mockRenderer = { width: 400, height: 600 } + +// A real canvas element so container.appendChild / removeChild work in jsdom +const mockCanvas = document.createElement('canvas') + +const mockApp = { + view: mockCanvas, + stage: { addChild: vi.fn() }, + screen: { width: 400, height: 600 }, // used for model positioning + renderer: { width: 400, height: 600 }, + destroy: vi.fn(), +} + vi.mock('pixi.js', () => ({ - Application: vi.fn((opts) => { - const canvas = document.createElement('canvas') - if (opts?.width) canvas.setAttribute('width', opts.width.toString()) - if (opts?.height) canvas.setAttribute('height', opts.height.toString()) - return { - stage: mockStage, - renderer: mockRenderer, - screen: mockRenderer, - view: canvas, - destroy: vi.fn(), - } - }), - Ticker: {}, + Application: vi.fn(() => mockApp), + Ticker: {}, // passed to Live2DModel.registerTicker })) +// Must mock the cubism4 sub-path — that's what the component imports vi.mock('pixi-live2d-display/cubism4', () => ({ Live2DModel: { + registerTicker: vi.fn(), from: vi.fn(() => Promise.resolve(mockModel)), registerTicker: vi.fn(), }, @@ -54,11 +58,11 @@ vi.mock('pixi-live2d-display/cubism4', () => ({ // ── Helpers ──────────────────────────────────────────────────────────────── -/** Mount the component and wait for the async model load to complete. */ +/** Mount the component and wait for the async model load to settle. */ async function mountAndLoad(props = {}) { const ref = createRef() const result = render() - await act(async () => { }) // flush the Live2DModel.from() promise + await act(async () => { }) // flush Live2DModel.from() promise + React effects return { ref, ...result } } @@ -69,37 +73,111 @@ describe('AvatarRenderer', () => { vi.clearAllMocks() }) - // ── DOM ────────────────────────────────────────────────────────────────── + // ── Rendering ───────────────────────────────────────────────────────────── - it('renders a canvas element', async () => { + it('renders a container div', async () => { + const { container } = await mountAndLoad() + expect(container.firstChild).toBeTruthy() + }) + + it('renders a canvas element inside the container', async () => { const { container } = await mountAndLoad() expect(container.querySelector('canvas')).toBeTruthy() }) - it('canvas has correct width and height attributes', async () => { + it('wrapper div reflects width and height props', async () => { const { container } = await mountAndLoad({ width: 320, height: 480 }) - const canvas = container.querySelector('canvas') - expect(canvas.getAttribute('width')).toBe('320') - expect(canvas.getAttribute('height')).toBe('480') + const div = container.firstChild + expect(div.style.width).toBe('320px') + expect(div.style.height).toBe('480px') }) // ── Expression overriding states ───────────────────────────────────────── - it('setExpression updates the ref state for the core.update loop to consume', async () => { + it.each([ + ['smile', 'SmileLock.exp3.json'], + ['sad', 'SadLock.exp3.json'], + ['angry', 'Angry.exp3.json'], + ['ghost', 'Ghost.exp3.json'], + ['ghost_nervous', 'GhostChange.exp3.json'], + ['shadow', 'Shadow.exp3.json'], + ['pupil_shrink', 'PupilShrink.exp3.json'], + ['eyeshine_off', 'EyeshineOff.exp3.json'], + ])('setExpression maps "%s" → %s', async (tag, file) => { const { ref } = await mountAndLoad() - // Ticker doesn't run in tests, so we just verify it doesn't throw and accepts names - expect(() => ref.current.setExpression(['smile', 'angry', 'ghost'], 2.0)).not.toThrow() + ref.current.setExpression([tag], 2.0) + expect(mockExpression).toHaveBeenCalledWith(file) + }) + + it('setExpression applies all tags in the array', async () => { + const { ref } = await mountAndLoad() + ref.current.setExpression(['smile', 'shadow'], 2.0) + expect(mockExpression).toHaveBeenCalledWith('SmileLock.exp3.json') + expect(mockExpression).toHaveBeenCalledWith('Shadow.exp3.json') + }) + + // ── Parameter-based expressions ─────────────────────────────────────────── + + it('wink sets correct Cubism4 Core Model parameters', async () => { + const { ref } = await mountAndLoad() + ref.current.setExpression(['wink'], 1.5) + expect(mockSetParameterValueById).toHaveBeenCalledWith('ParamEyeLOpen', 0.0) + expect(mockSetParameterValueById).toHaveBeenCalledWith('ParamBrowLForm', -1.0) + expect(mockSetParameterValueById).toHaveBeenCalledWith('ParamMouthForm', 1.0) + }) + + it('tongue sets correct Cubism4 Core Model parameters', async () => { + const { ref } = await mountAndLoad() + ref.current.setExpression(['tongue'], 1.5) + expect(mockSetParameterValueById).toHaveBeenCalledWith('ParamMouthOpenY', 1.0) + expect(mockSetParameterValueById).toHaveBeenCalledWith('ParamMouthForm', -1.0) }) // ── Auto-reset ──────────────────────────────────────────────────────────── - it('setExpression schedules auto-reset after duration ms', async () => { + it('setExpression resets to neutral after the given duration', async () => { vi.useFakeTimers() const { ref } = await mountAndLoad() - expect(() => { - ref.current.setExpression(['smile'], 2.0) - vi.advanceTimersByTime(2000) - }).not.toThrow() + ref.current.setExpression(['smile'], 2.0) + mockExpression.mockClear() + vi.advanceTimersByTime(2000) + expect(mockExpression).toHaveBeenCalledWith() // no-arg call = reset to default + vi.useRealTimers() + }) + + it('auto-reset does not fire before the duration elapses', async () => { + vi.useFakeTimers() + const { ref } = await mountAndLoad() + ref.current.setExpression(['angry'], 1.5) + mockExpression.mockClear() + vi.advanceTimersByTime(1499) + expect(mockExpression).not.toHaveBeenCalled() + vi.advanceTimersByTime(1) + expect(mockExpression).toHaveBeenCalledWith() + vi.useRealTimers() + }) + + // ── Mood memory ─────────────────────────────────────────────────────────── + // The full mood-rendering loop requires a live PIXI ticker (unavailable in jsdom). + // These tests verify the pending-mood pipeline is wired without crashing. + + it('setExpression with a mood-mapped name does not throw', async () => { + const { ref } = await mountAndLoad() + // Each of these has an EXPRESSION_TO_MOOD entry and should queue a pending mood + for (const tag of ['smile', 'sad', 'angry', 'ghost', 'ghost_nervous', + 'shadow', 'pupil_shrink', 'eyeshine_off']) { + expect(() => ref.current.setExpression([tag], 1.0)).not.toThrow() + } + }) + + it('mood memory: expression → expiry → state transition completes cleanly', async () => { + vi.useFakeTimers() + const { ref } = await mountAndLoad() + ref.current.setExpression(['smile'], 2.0) // queues _pendingMood = MOODS.happy + vi.advanceTimersByTime(2000) // triggers auto-reset; _pendingMood consumed on next frame + // After expiry the avatar should accept further API calls without errors + expect(() => ref.current.setSpeaking(false)).not.toThrow() + expect(() => ref.current.resetNeutral()).not.toThrow() vi.useRealTimers() }) @@ -110,19 +188,56 @@ describe('AvatarRenderer', () => { expect(() => ref.current.resetNeutral()).not.toThrow() }) - // ── setParameter ───────────────────────────────────────────────────────── + // ── setParameter ────────────────────────────────────────────────────────── - it('setParameter forwards name and value to coreModel', async () => { + it('setParameter forwards the id and value to coreModel', async () => { const { ref } = await mountAndLoad() ref.current.setParameter('ParamMouthOpenY', 0.8) expect(mockSetParameterValueById).toHaveBeenCalledWith('ParamMouthOpenY', 0.8) }) + // ── setSpeaking ─────────────────────────────────────────────────────────── + + it('setSpeaking(true) switches to speaking state without throwing', async () => { + const { ref } = await mountAndLoad() + expect(() => ref.current.setSpeaking(true)).not.toThrow() + }) + + it('setSpeaking(false) switches to idle state without throwing', async () => { + const { ref } = await mountAndLoad() + expect(() => ref.current.setSpeaking(false)).not.toThrow() + }) + + it('setSpeaking can toggle states repeatedly without side effects', async () => { + const { ref } = await mountAndLoad() + ref.current.setSpeaking(true) + ref.current.setSpeaking(false) + ref.current.setSpeaking(true) + // State changes should not trigger expressions + expect(mockExpression).not.toHaveBeenCalled() + }) + + // ── setMouthOpen ────────────────────────────────────────────────────────── + + it('setMouthOpen accepts values within [0, 1]', async () => { + const { ref } = await mountAndLoad() + expect(() => ref.current.setMouthOpen(0)).not.toThrow() + expect(() => ref.current.setMouthOpen(0.5)).not.toThrow() + expect(() => ref.current.setMouthOpen(1)).not.toThrow() + }) + + it('setMouthOpen silently clamps out-of-range values', async () => { + const { ref } = await mountAndLoad() + expect(() => ref.current.setMouthOpen(-1.0)).not.toThrow() + expect(() => ref.current.setMouthOpen(2.5)).not.toThrow() + }) + // ── Guard rails ─────────────────────────────────────────────────────────── - it('unknown expression name is silently ignored (no throw)', async () => { + it('unknown expression tag is silently ignored', async () => { const { ref } = await mountAndLoad() expect(() => ref.current.setExpression(['nonexistent_tag'], 1.0)).not.toThrow() + expect(mockExpression).not.toHaveBeenCalled() }) it('empty expression list does not throw', async () => { diff --git a/dashboard/src/components/CallOverlay.jsx b/dashboard/src/components/CallOverlay.jsx index 9fb821b..0f99fa5 100644 --- a/dashboard/src/components/CallOverlay.jsx +++ b/dashboard/src/components/CallOverlay.jsx @@ -1,19 +1,9 @@ import { useState, useEffect, useRef, useCallback } from 'react' import { AvatarRenderer } from './AvatarRenderer' -function getOrCreateIdentity() { - const KEY = 'aura_user_identity' - let id = localStorage.getItem(KEY) +import { getOrCreateIdentity } from '../lib/user' - if (!id) { - id = `user-${crypto.randomUUID().slice(0, 8)}` - localStorage.setItem(KEY, id) - } - - return id -} - -export default function CallOverlay({ onClose }) { +export default function CallOverlay({ onClose, conversationId }) { const [status, setStatus] = useState('connecting') const [elapsed, setElapsed] = useState(0) const roomRef = useRef(null) @@ -22,6 +12,7 @@ export default function CallOverlay({ onClose }) { const audioCtxRef = useRef(null) const analyserRef = useRef(null) const lipRafRef = useRef(null) + const speakTimeoutRef = useRef(null) // ─── Connect to LiveKit ────────────────────── useEffect(() => { @@ -33,19 +24,19 @@ export default function CallOverlay({ onClose }) { const connect = async () => { try { - // Dynamically import to avoid bundling when not needed const { Room, RoomEvent, Track } = await import('livekit-client') - const identity = getOrCreateIdentity() // Fetch token from token server - const res = await fetch(`http://${window.location.hostname}:8082/getToken?identity=${encodeURIComponent(identity)}`) + let url = `http://${window.location.hostname}:8082/getToken?room=aura-room&identity=${encodeURIComponent(identity)}` + if (conversationId) url += `&conversation_id=${encodeURIComponent(conversationId)}` + + const res = await fetch(url) if (!res.ok) throw new Error(`Token server error: ${res.status}`) - const { token, url } = await res.json() + const { token, url: lkUrl } = await res.json() if (cancelled) return - // Connect to room const room = new Room() roomRef.current = room @@ -58,22 +49,30 @@ export default function CallOverlay({ onClose }) { const analyser = ctx.createAnalyser() analyser.fftSize = 2048 analyser.smoothingTimeConstant = 0.8 - const src = ctx.createMediaStreamSource( - new MediaStream([track.mediaStreamTrack]) - ) + const src = ctx.createMediaStreamSource(new MediaStream([track.mediaStreamTrack])) src.connect(analyser) analyserRef.current = analyser const buf = new Float32Array(analyser.fftSize) const tick = () => { + if (cancelled) return lipRafRef.current = requestAnimationFrame(tick) analyser.getFloatTimeDomainData(buf) let sum = 0 for (let i = 0; i < buf.length; i++) sum += buf[i] * buf[i] const rms = Math.sqrt(sum / buf.length) - avatarRef.current?.setMouthOpen( - rms > 0.008 ? Math.min(0.55, rms * 10) : 0 - ) + const active = rms > 0.008 + avatarRef.current?.setMouthOpen(active ? Math.min(0.55, rms * 10) : 0) + + if (active) { + if (speakTimeoutRef.current) { clearTimeout(speakTimeoutRef.current); speakTimeoutRef.current = null } + avatarRef.current?.setSpeaking(true) + } else if (!speakTimeoutRef.current) { + speakTimeoutRef.current = setTimeout(() => { + avatarRef.current?.setSpeaking(false) + speakTimeoutRef.current = null + }, 600) + } } tick() } @@ -83,19 +82,16 @@ export default function CallOverlay({ onClose }) { track.detach().forEach((el) => el.remove()) }) - // ── Expression events from Python avatar_bridge.py ────────── room.on(RoomEvent.DataReceived, (payload) => { try { const msg = JSON.parse(new TextDecoder().decode(payload)) if (msg.type === 'expression') { avatarRef.current?.setExpression(msg.expressions, msg.duration) } - } catch { - // malformed payload — silently ignore - } + } catch { } }) - await room.connect(url, token) + await room.connect(lkUrl, token) await room.localParticipant.setMicrophoneEnabled(true) if (!cancelled) { @@ -117,11 +113,12 @@ export default function CallOverlay({ onClose }) { window.removeEventListener('beforeunload', handleUnload) cleanup() } - }, []) + }, [conversationId]) const cleanup = useCallback(() => { if (timerRef.current) clearInterval(timerRef.current) if (lipRafRef.current) cancelAnimationFrame(lipRafRef.current) + if (speakTimeoutRef.current) { clearTimeout(speakTimeoutRef.current); speakTimeoutRef.current = null } if (audioCtxRef.current) { audioCtxRef.current.close(); audioCtxRef.current = null } if (roomRef.current) { roomRef.current.disconnect() @@ -141,48 +138,53 @@ export default function CallOverlay({ onClose }) { const vh = window.innerHeight return ( - // Full-screen container — avatar fills the whole background, - // controls float as an overlay on the right side (same as AIRI). -
- - {/* ── Live2D Avatar — full-screen canvas ── */} - - - {/* ── Controls — overlaid panel, right side ── */} -
-

AURA

- -

- {status === 'connecting' && 'Connecting...'} - {status === 'connected' && formatTime(elapsed)} - {status === 'error' && 'Connection failed'} -

- - {/* Waveform */} - {status === 'connected' && ( -
- {[0, 1, 2, 3, 4].map((i) => ( -
- ))} +
+ {/* Background Branding */} +
+

PROJECT AURA

+
+ + {/* ── Live2D Avatar — centered ── */} +
+ +
+ + {/* ── Controls — bottom center ── */} +
+
+

Project AURA

+

+ {status === 'connecting' && 'Establishing Connection...'} + {status === 'connected' && `Live Interaction — ${formatTime(elapsed)}`} + {status === 'error' && 'Neural Link Failed'} +

+
+ +
+ {/* Visualizer */} + {status === 'connected' && ( +
+ {[...Array(12)].map((_, i) => ( +
+ ))} +
+ )} + + {/* Hangup */} + + + {/* Placeholder for future mic toggle/settings */} +
+ mic
- )} - - {/* Hangup */} - +
) diff --git a/dashboard/src/components/ChatFeed.jsx b/dashboard/src/components/ChatFeed.jsx index 744a263..6d11ad4 100644 --- a/dashboard/src/components/ChatFeed.jsx +++ b/dashboard/src/components/ChatFeed.jsx @@ -1,49 +1,64 @@ export default function ChatFeed({ messages = [] }) { if (messages.length === 0) { return ( -
-
- wb_sunny +
+
+ auto_awesome +
-

Hello! I'm AURA

-

- Your personal AI companion. Ask me anything, or start a voice call! +

Project AURA

+

+ Advanced Universal Responsive Avatar.
+ Ready for your next inquiry.

) } return ( -
+
{messages.map((msg) => ( -
- {/* Avatar */} -
+ {/* Avatar Icon */} +
- {msg.role === 'user' ? 'U' : '☀'} + {msg.role === 'user' ? 'ME' : 'AURA'}
- {/* Bubble Container */} -
- {/* Tool Usage Indicator */} + {/* Content Column */} +
+ + {/* Tool Execution Details */} {msg.tools_used && msg.tools_used.map((tool, idx) => ( -
- travel_explore -
- {tool.name} - {JSON.stringify(tool.args.query || tool.args)} +
+ api +
+ {tool.name} + — + + {typeof tool.args === 'string' ? tool.args : (tool.args.query || JSON.stringify(tool.args))} +
))} -
{msg.content}
+ + {/* Emotion Tag */} + {msg.role === 'aura' && msg.emotion && ( +
+ + {msg.emotion} +
+ )}
))} diff --git a/dashboard/src/components/ChatHeader.jsx b/dashboard/src/components/ChatHeader.jsx index ceaa671..32333ad 100644 --- a/dashboard/src/components/ChatHeader.jsx +++ b/dashboard/src/components/ChatHeader.jsx @@ -1,36 +1,42 @@ -import { useNavigate } from 'react-router-dom' - -export default function ChatHeader({ onCallStart }) { - const navigate = useNavigate() - +export default function ChatHeader({ onCallStart, isCallActive, onTuningOpen }) { return ( -
+
-
- wb_sunny +
+ auto_awesome
-

AURA

-

Active • High Precision Mode

+

Project AURA

+

+ {isCallActive ? 'Interactive Mode' : 'Ready to Assist'} +

-
+
+ {/* Voice Interaction Toggle */} + + {/* Personality Tuning Toggle */}
diff --git a/dashboard/src/components/ChatInput.jsx b/dashboard/src/components/ChatInput.jsx index 4ae0a63..8e74c18 100644 --- a/dashboard/src/components/ChatInput.jsx +++ b/dashboard/src/components/ChatInput.jsx @@ -28,30 +28,28 @@ export default function ChatInput({ onSend, disabled }) { } return ( -
-
+
+