diff --git a/.env.example b/.env.example index b570d0f..4b194c3 100644 --- a/.env.example +++ b/.env.example @@ -4,7 +4,7 @@ # # 启动顺序： # 1) pnpm dev:api # 读 .env，监听 http://localhost:8787（含 WS /stt-realtime） -# 2) pnpm dev:playground # Vite 把 /stt /stt-realtime /llm 代理到 8787 +# 2) pnpm dev:playground # Vite 把 /stt-realtime /llm 代理到 8787 # ── 代理端口 ───────────────────────────────────────────── PORT=8787 @@ -39,39 +39,12 @@ PORT=8787 # PLAYGROUND_API_ORIGIN=http://localhost:8787 # ── 语音转写 (STT) ──────────────────────────────────────── -# 三个 batch provider（可同时配置，playground 从 GET /stt/providers 切换）： -# - openrouter ：云端，JSON → /audio/transcriptions -# - openai ：Whisper 形 multipart /v1/audio/transcriptions（本地 mlx-qwen3-asr 等，见 ADR 0002） -# - dashscope ：阿里云百炼 batch ASR（OpenAI 兼容 chat/completions + input_audio） -# - dashscope-realtime：同 key + STT_DASHSCOPE_WS_URL → WS /stt-realtime（ADR 0004） -# STT__MODEL 不填时回退到注册默认值。 -STT_ACTIVE=openrouter -# 生产 realtime 与 batch 并行；batch provider 单独冻结。 -# STT_BATCH_ACTIVE=dashscope -STT_OPENROUTER_BASE_URL=https://openrouter.ai/api/v1 -STT_OPENROUTER_API_KEY=sk-or-xxxxxxxxxxxxxxxx -STT_OPENROUTER_MODEL=openai/gpt-4o-transcribe - -# 本地 Qwen3-ASR（Apple Silicon，mlx-qwen3-asr serve 默认监听 8765）： -# pip install "mlx-qwen3-asr[serve]" -# mlx-qwen3-asr serve --api-key $(openssl rand -hex 16) -# 仍经 apps/api /stt 代理转发；浏览器发 /stt?provider=openai，key 留在服务端。 -# 需 apps/api 与 mlx-qwen3-asr 同机（本地 dev）。切到本地：STT_ACTIVE=openai，或浏览器加 ?provider=openai。 -# STT_ACTIVE=openai -# STT_OPENAI_BASE_URL=http://localhost:8765/v1 -# STT_OPENAI_API_KEY=本地 serve 启动时生成的 key -# STT_OPENAI_MODEL=Qwen3-ASR-1.7B - -# 阿里云百炼 Qwen3-ASR batch（将 {WorkspaceId} 换成业务空间 ID）： -# STT_ACTIVE=dashscope -# STT_DASHSCOPE_BASE_URL=https://{WorkspaceId}.cn-beijing.maas.aliyuncs.com/compatible-mode/v1 -# STT_DASHSCOPE_API_KEY=sk-xxxxxxxx -# STT_DASHSCOPE_MODEL=qwen3-asr-flash -# -# Realtime（WS /stt-realtime?provider=dashscope-realtime，ADR 0004）： -# 复用 STT_DASHSCOPE_API_KEY；另需 WS_URL。浏览器只连同源代理，不直连上游。 -# STT_DASHSCOPE_WS_URL=wss://{WorkspaceId}.cn-beijing.maas.aliyuncs.com/api-ws/v1/realtime -# STT_DASHSCOPE_REALTIME_MODEL=qwen3-asr-flash-realtime +# Realtime only（WS /stt-realtime?provider=dashscope-realtime，ADR 0004）。 +# 浏览器只连同源代理，不直连上游。 +STT_ACTIVE=dashscope-realtime +STT_DASHSCOPE_API_KEY=sk-xxxxxxxx +STT_DASHSCOPE_WS_URL=wss://{WorkspaceId}.ap-northeast-1.maas.aliyuncs.com/api-ws/v1/realtime +STT_DASHSCOPE_REALTIME_MODEL=qwen3-asr-flash-realtime # ── 大语言模型 (LLM) ───────────────────────────────────── # 走 xsai（OpenAI 兼容）。provider 与 STT 可不同。 diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml index 71b0f8e..86882ac 100644 --- a/.github/workflows/deploy-production.yml +++ b/.github/workflows/deploy-production.yml @@ -27,10 +27,13 @@ jobs: CN_SERVER_HOST: ${{ secrets.CN_SERVER_HOST }} CN_SERVER_USER: ${{ secrets.CN_SERVER_USER }} CN_RELAY_ORIGIN: ${{ secrets.CN_RELAY_ORIGIN }} + JP_STT_DASHSCOPE_API_KEY: ${{ secrets.JP_STT_DASHSCOPE_API_KEY }} + JP_STT_DASHSCOPE_WS_URL: ${{ secrets.JP_STT_DASHSCOPE_WS_URL }} run: | for secret_name in \ DEPLOY_SSH_KEY SERVER_HOST SERVER_USER \ - CN_DEPLOY_SSH_KEY CN_SERVER_HOST CN_SERVER_USER CN_RELAY_ORIGIN; do + CN_DEPLOY_SSH_KEY CN_SERVER_HOST CN_SERVER_USER CN_RELAY_ORIGIN \ + JP_STT_DASHSCOPE_API_KEY JP_STT_DASHSCOPE_WS_URL; do if [ -z "${!secret_name}" ]; then echo "::error::Missing Actions secret: ${secret_name}" exit 1 @@ -49,26 +52,44 @@ jobs: chmod 600 ~/.ssh/id_ed25519_primary ~/.ssh/id_ed25519_cn ssh-keyscan -H "$SERVER_HOST" >> ~/.ssh/known_hosts ssh-keyscan -H "$CN_SERVER_HOST" >> ~/.ssh/known_hosts - - name: Configure relay origin on primary + - name: Configure primary routes and providers env: SERVER_HOST: ${{ secrets.SERVER_HOST }} SERVER_USER: ${{ secrets.SERVER_USER }} CN_RELAY_ORIGIN: ${{ secrets.CN_RELAY_ORIGIN }} + JP_STT_DASHSCOPE_API_KEY: ${{ secrets.JP_STT_DASHSCOPE_API_KEY }} + JP_STT_DASHSCOPE_WS_URL: ${{ secrets.JP_STT_DASHSCOPE_WS_URL }} run: | case "$CN_RELAY_ORIGIN" in http://*|https://*) ;; *) echo "::error::CN_RELAY_ORIGIN must be an absolute HTTP(S) origin"; exit 1 ;; esac - printf '%s\n' "$CN_RELAY_ORIGIN" \ + case "$JP_STT_DASHSCOPE_WS_URL" in + wss://*.ap-northeast-1.maas.aliyuncs.com/api-ws/v1/realtime) ;; + *) echo "::error::JP_STT_DASHSCOPE_WS_URL must use the Alibaba Cloud Tokyo realtime endpoint"; exit 1 ;; + esac + printf '%s\n%s\n%s\n' \ + "$CN_RELAY_ORIGIN" \ + "$JP_STT_DASHSCOPE_API_KEY" \ + "$JP_STT_DASHSCOPE_WS_URL" \ | ssh -i ~/.ssh/id_ed25519_primary "${SERVER_USER}@${SERVER_HOST}" ' set -eu IFS= read -r origin - escaped=$(printf "%s" "$origin" | sed "s/[|&]/\\\\&/g") - if grep -q "^RELAY_CN_ORIGIN=" /opt/kibotalk/.env; then - sed -i "s|^RELAY_CN_ORIGIN=.*|RELAY_CN_ORIGIN=${escaped}|" /opt/kibotalk/.env - else - printf "RELAY_CN_ORIGIN=%s\n" "$origin" >> /opt/kibotalk/.env - fi + IFS= read -r stt_key + IFS= read -r stt_url + set_env() { + name=$1 + value=$2 + escaped=$(printf "%s" "$value" | sed "s/[|&]/\\\\&/g") + if grep -q "^${name}=" /opt/kibotalk/.env; then + sed -i "s|^${name}=.*|${name}=${escaped}|" /opt/kibotalk/.env + else + printf "%s=%s\n" "$name" "$value" >> /opt/kibotalk/.env + fi + } + set_env RELAY_CN_ORIGIN "$origin" + set_env STT_DASHSCOPE_API_KEY "$stt_key" + set_env STT_DASHSCOPE_WS_URL "$stt_url" chmod 600 /opt/kibotalk/.env ' - name: Validate server environments @@ -84,7 +105,8 @@ jobs: for name in \ RELAY_PRIMARY_ORIGIN RELAY_CN_ORIGIN RELAY_CN_ENABLED \ RELAY_TOKEN_PRIVATE_KEY RELAY_TOKEN_PUBLIC_KEY RELAY_NODE_SECRET \ - STT_ACTIVE STT_BATCH_ACTIVE LLM_ACTIVE; do + STT_ACTIVE STT_DASHSCOPE_API_KEY STT_DASHSCOPE_WS_URL \ + LLM_ACTIVE; do grep -q "^${name}=." /opt/kibotalk/.env done ' @@ -94,7 +116,7 @@ jobs: for name in \ RELAY_PRIMARY_ORIGIN RELAY_ACCEPT_NEW_SESSIONS \ RELAY_TOKEN_PUBLIC_KEY RELAY_NODE_SECRET \ - STT_ACTIVE STT_BATCH_ACTIVE LLM_ACTIVE; do + STT_ACTIVE LLM_ACTIVE; do grep -q "^${name}=." /opt/kibotalk-relay/.env done relay_port=$(sed -n "s/^RELAY_HTTP_PORT=//p" /opt/kibotalk-relay/.env) diff --git a/AGENTS.md b/AGENTS.md index e7bb406..1774c90 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,15 +31,14 @@ update the spec — don't silently drift. Client orchestration + thin proxy (ADR 0001). The browser runs the pipeline; `apps/api` is a stateless Hono proxy that hides provider keys and forwards STT/LLM. -**STT modes** (ADR 0004): **batch** (`POST /stt`) and **realtime** -(`WS /stt-realtime`) run in parallel—not a migration. Local VAD + speaker -verification own turn boundaries (`pauseMs`); realtime uses Manual commit -(no server VAD). Timeline may show partial drafts; formal turns + LLM fire -only on finalized transcript. +**STT** (ADR 0004): **realtime only** via `WS /stt-realtime` (DashScope +`qwen3-asr-flash-realtime`). Local VAD + speaker verification own turn +boundaries (`pauseMs`); upstream uses Manual commit (no server VAD). Timeline +may show partial drafts; formal turns + LLM fire only on finalized transcript. ``` apps/ - api/ Hono proxy: /stt, /stt-realtime (WS), /llm (SSE). Keys server-side only. + api/ Hono proxy: /stt-realtime (WS), /llm (SSE). Keys server-side only. playground/ Vite + React dev panel (Chinese UI) for testing each layer (声纹页 covers enrollment + free-speech verify / threshold tuning). web/ PWA shell (not yet built). @@ -49,8 +48,8 @@ packages/ prompts/ Reply-suggestion prompts (Velin TSX → markdown). speaker/ Speaker verification (wavlm-base-plus-sv, WASM + IndexedDB); `verify` returns raw `similarity` plus label `confidence`. - stt/ Provider-agnostic STT: batch adapters + DashScope realtime mapper - helpers (server-side). Providers declare mode batch|realtime. + stt/ DashScope realtime mapper helpers (server-side); `apps/api` relays + `WS /stt-realtime` only. pipeline/ Conversation store + turn state machine. ui/ shadcn/ui primitives on Tailwind v4 (shared). app-shared/ Shared client types/config shell. @@ -62,21 +61,20 @@ These are spec-named choices. **Do not rewrite or substitute them** with hand-ro - **LLM → `xsai`** (`@xsai/stream-text`). Never hand-roll fetch/SSE for LLM. An official `xsai` skill is installed at `.agents/skills/xsai/` — read its `SKILL.md` + `references/` before touching `packages/llm`. - **Prompts → Velin** (`@velin-dev/core-react`). Render TSX components to markdown strings; don't template with raw string concat. -- **STT → provider-agnostic factory** in `packages/stt`, reached via the `apps/api` `/stt` proxy. Local ASR (`mlx-qwen3-asr`) also goes through the proxy (`?provider=openai`), **never browser-direct** (ADR 0002). +- **STT → DashScope realtime** via `packages/stt` mapper helpers, reached through the `apps/api` `WS /stt-realtime` relay. **Never browser-direct** to upstream (ADR 0004). - **VAD → Silero** via `@huggingface/transformers`. v6.2 needs a 64-sample context prepended to each 512-sample chunk (576 input), carried across calls; v5 takes 512 raw. See `apps/playground/src/audio/silero-vad.ts`. - **Speaker → `Xenova/wavlm-base-plus-sv`** via transformers.js in a Web Worker; embeddings persist in IndexedDB. ## Audio pipeline specifics -- **Sample rate**: 16 kHz mono PCM everywhere (VAD, STT uploads, speaker embeddings). +- **Sample rate**: 16 kHz mono PCM everywhere (VAD, realtime STT uplink, speaker embeddings). - **VAD chunk size**: 512 samples (32 ms) per `processAudio` call (`packages/audio/src/vad.ts` `newBufferSize`). Silero v6.2 prepends 64-sample context → 576 input; v5 takes 512 raw. See `docs/solutions/silero-vad-v6-context-frame.md`. -- **STT upload format**: WAV 16 kHz mono PCM via `encodeWav` (`packages/audio/src`). The `/stt` proxy forwards the WAV body as-is. - **Speaker embeddings**: computed in a Web Worker (`apps/playground/src/audio/speaker-worker.ts`), persisted in IndexedDB. Don't run the WASM model on the main thread. -- **Segment aggregation / TurnGate** (`packages/audio/src/aggregator.ts`): sits between VAD (+ speaker verification) and batch `ingestSegment` or realtime `append`/`commit`. Accumulates same-speaker speech and flushes on `pauseMs`, `maxMs` (speech only), or speaker change. PCM is direct-concatenated (no gap fill). Spec §2.4 / ADR 0004. Pipeline fires LLM per finalized turn and does NOT wait on pause itself. +- **Segment aggregation / TurnGate** (`packages/audio/src/aggregator.ts`): sits between VAD (+ speaker verification) and realtime `append`/`commit` → `ingestFinalizedTurn`. Merges same-speaker fragment transcripts and flushes on `pauseMs`, `maxMs` (speech only), or speaker change. Spec §2.4 / ADR 0004. Pipeline fires LLM per finalized turn and does NOT wait on pause itself. ## Conventions -- **Keys in env, never client.** All provider keys/config live in `.env` (see `.env.example`), loaded by `apps/api`. Naming: `__` (e.g. `LLM_OPENROUTER_API_KEY`, `STT_OPENAI_BASE_URL`). +- **Keys in env, never client.** All provider keys/config live in `.env` (see `.env.example`), loaded by `apps/api`. Naming: `__` (e.g. `LLM_OPENROUTER_API_KEY`, `STT_DASHSCOPE_API_KEY`). - **Pure functions; no new classes** unless the framework/API requires it. Imports at top. TS unions: exhaustive `switch`. - **Full words.** No obscure abbreviations; only use ones common in software. - **Smallest correct diff.** Only change what was asked. No drive-by refactor, tests, or docs unless asked. When you touch code, small progressive refactors alongside the change are welcome. @@ -84,7 +82,7 @@ These are spec-named choices. **Do not rewrite or substitute them** with hand-ro - **No backward-compatibility guards.** If a rename/breakage is needed, do it directly and update callers in the same change. - **Playground UI is Chinese** — labels, examples, and sample content in Chinese. - **Tailwind v4 + shadcn/ui** for all UI (playground included). Shared primitives in `packages/ui`（Button、Card、Badge、Input、Textarea、Label、Tabs、Separator、Accordion、Collapsible、Dialog、DropdownMenu、Popover、Progress、ScrollArea、Select、Sheet、Skeleton、Slider、Switch、Tooltip、Toaster）。缺组件先 `shadcn add` 进该包并导出；**改样式改源组件 / token，不要平行重造**。 -- **Shared playground config lives in one Zustand store** (`apps/playground/src/config-store.ts`, `useConfig`) — the React analog of a Pinia store. VAD/ASR/merge/speaker knobs, language prefs (`uiLang` / `conversationLang` / `level` / `languagesConfirmed`, persisted; `meaningLang` is derived from `uiLang` in the session snapshot), and selectors (provider, VAD model, transcribe mode) are shared across the VAD panel and the live session: change one on a tab and it's already aligned on the other. Subscribe per-field (`useConfig(s => s.field)`); in async callbacks read `useConfig.getState()`. Stage-grouped field components live in `apps/playground/src/components/ConfigFields.tsx` (`VadParamsFields`, `AsrPadFields`, `MergeParamsFields`, `LanguagePrefsFields`, `VadModelSelect`, `TranscribeModeSelect`, `TranscribeProviderSelect`, `NumberField`) — reuse these instead of re-declaring the same knobs. +- **Shared playground config lives in one Zustand store** (`apps/playground/src/config-store.ts`, `useConfig`) — the React analog of a Pinia store. VAD/merge/speaker knobs, language prefs (`uiLang` / `conversationLang` / `level` / `languagesConfirmed`, persisted; `meaningLang` is derived from `uiLang` in the session snapshot), and selectors (VAD model, STT provider) are shared across the VAD panel and the live session: change one on a tab and it's already aligned on the other. Subscribe per-field (`useConfig(s => s.field)`); in async callbacks read `useConfig.getState()`. Stage-grouped field components live in `apps/playground/src/components/ConfigFields.tsx` (`VadParamsFields`, `MergeParamsFields`, `LanguagePrefsFields`, `VadModelSelect`, `TranscribeProviderSelect`, `NumberField`) — reuse these instead of re-declaring the same knobs. ## Commands diff --git a/apps/api/src/app.ts b/apps/api/src/app.ts index 4f11375..dc98c59 100644 --- a/apps/api/src/app.ts +++ b/apps/api/src/app.ts @@ -3,7 +3,7 @@ import { serveStatic } from '@hono/node-server/serve-static' import type { AppLanguage, ConversationTurn, LearnerLevel } from '@kibotalk/conversation' import { createLlmClient, llmConfigFromEnv, type LlmUsage } from '@kibotalk/llm' import { buildReplySuggestionsMessages, buildSessionReviewMessages } from '@kibotalk/prompts' -import { createSttClient, listSttProviders, sttConfigFromEnv } from '@kibotalk/stt' +import { listSttProviders } from '@kibotalk/stt' import { Hono } from 'hono' import { bodyLimit } from 'hono/body-limit' import { cors } from 'hono/cors' @@ -47,7 +47,6 @@ import { } from './sync' import { recordTelemetry, recordTelemetryLater } from './telemetry' import { redeemVoucher } from './vouchers' -import { billCompletedRelayTurn } from './relay-billing' import { requireRelayRequestAuth } from './relay-request-auth' import { registerRelayRoutes } from './relay-routes' import { @@ -116,9 +115,7 @@ const RELAY_DATA_PATHS = new Set([ '/api/latency', '/api/relay/handshake', '/api/relay/ws-ticket', - '/api/stt', '/api/llm', - '/stt', '/llm', ]) @@ -225,79 +222,10 @@ app.get('/api/stt/providers', async (context) => { const auth = await requireRequestAuth(context) if (auth instanceof Response) return auth const providers = listSttProviders(process.env) - .filter((provider) => provider.configured) + .filter((provider) => provider.configured && provider.mode === 'realtime') return context.json({ providers }) }) -app.post('/api/stt', async (context) => { - const relayAuth = requireRelayRequestAuth(context, 'stt') - if (relayAuth instanceof Response) return relayAuth - const startedAt = Date.now() - const requestId = randomUUID() - const wav = await context.req.arrayBuffer() - const providerOverride = - process.env.APP_ENV === 'production' - ? process.env.STT_BATCH_ACTIVE - : context.req.query('provider') || undefined - const language = context.req.query('language') || undefined - let sttClient - try { - sttClient = createSttClient(sttConfigFromEnv(process.env, providerOverride)) - } catch (cause) { - return context.json({ error: (cause as Error).message }, 500) - } - try { - const text = await sttClient.transcribe(wav, { - signal: context.req.raw.signal, - language, - }) - const config = sttConfigFromEnv(process.env, providerOverride) - if ( - process.env.APP_ENV === 'production' - && config.provider !== relayAuth.claims.sttBatchProvider - ) throw new Error('RELAY_CAPABILITY_MISMATCH') - const durationMs = Date.now() - startedAt - const deduction = await billCompletedRelayTurn({ - role, - auth: relayAuth, - requestId, - audioSeconds: Math.max(1, (wav.byteLength - 44) / (16_000 * 2)), - provider: config.provider, - model: config.model, - durationMs, - }) - recordTelemetryLater(relayAuth.requestAuth, { - requestId, - eventType: 'stt_batch', - provider: config.provider, - model: config.model, - status: 'ok', - durationMs, - billedAudioSeconds: deduction.billedSeconds, - metadata: { - nodeId: relayAuth.claims.nodeId, - deductedSeconds: deduction.deductedSeconds, - overdrawSeconds: deduction.overdrawSeconds, - }, - }) - return context.json({ - text, - quotaExhausted: deduction.exhausted, - remainingSeconds: deduction.remainingSeconds, - }) - } catch (cause) { - recordTelemetryLater(relayAuth.requestAuth, { - requestId, - eventType: 'stt_batch', - provider: providerOverride ?? process.env.STT_ACTIVE, - status: 'error', - durationMs: Date.now() - startedAt, - errorCode: cause instanceof Error ? cause.name : 'UPSTREAM_ERROR', - }) - return context.json({ error: (cause as Error).message }, 502) - } -}) - app.post('/api/llm', async (context) => { const relayAuth = requireRelayRequestAuth(context, 'llm') if (relayAuth instanceof Response) return relayAuth @@ -524,7 +452,6 @@ app.post('/api/session-review', async (context) => { // the same handlers. app.route('/', new Hono() .get('/stt/providers', (context) => app.fetch(new Request(new URL('/api/stt/providers', context.req.url), context.req.raw))) - .post('/stt', (context) => app.fetch(new Request(new URL(`/api/stt${new URL(context.req.url).search}`, context.req.url), context.req.raw))) .post('/llm', (context) => app.fetch(new Request(new URL('/api/llm', context.req.url), context.req.raw))) .post('/session-review', (context) => app.fetch(new Request(new URL('/api/session-review', context.req.url), context.req.raw)))) diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts index 34766e3..7fff948 100644 --- a/apps/api/src/index.ts +++ b/apps/api/src/index.ts @@ -29,7 +29,6 @@ if (process.env.APP_ENV === 'production') { 'STT_DASHSCOPE_API_KEY', 'STT_DASHSCOPE_WS_URL', 'STT_DASHSCOPE_REALTIME_MODEL', - 'STT_BATCH_ACTIVE', 'LLM_ACTIVE', ] as const const roleRequired = role === 'primary' @@ -62,14 +61,6 @@ if (process.env.APP_ENV === 'production') { if (process.env.STT_DASHSCOPE_REALTIME_MODEL !== 'qwen3-asr-flash-realtime') { throw new Error('Production STT model must be qwen3-asr-flash-realtime') } - const batchProvider = process.env.STT_BATCH_ACTIVE! - const batchGroup = `STT_${batchProvider.toUpperCase()}_` - const missingBatch = ['BASE_URL', 'API_KEY', 'MODEL'] - .map((field) => `${batchGroup}${field}`) - .filter((name) => !process.env[name]) - if (missingBatch.length > 0) { - throw new Error(`Missing production environment variables: ${missingBatch.join(', ')}`) - } const llmGroup = `LLM_${process.env.LLM_ACTIVE!.toUpperCase()}_` const llmRequired = [`${llmGroup}BASE_URL`, `${llmGroup}API_KEY`, `${llmGroup}MODEL`] const missingLlm = llmRequired.filter((name) => !process.env[name]) diff --git a/apps/api/src/provider-health.ts b/apps/api/src/provider-health.ts index ded0230..f942cea 100644 --- a/apps/api/src/provider-health.ts +++ b/apps/api/src/provider-health.ts @@ -1,14 +1,9 @@ let cachedProviderHealthy = false -const REQUIRED_UPSTREAM_COUNT = 3 +const REQUIRED_UPSTREAM_COUNT = 2 function configuredUpstreamUrls(env: NodeJS.ProcessEnv): string[] { const urls: string[] = [] if (env.STT_DASHSCOPE_WS_URL) urls.push(env.STT_DASHSCOPE_WS_URL) - const batchProvider = env.STT_BATCH_ACTIVE - if (batchProvider) { - const batchUrl = env[`STT_${batchProvider.toUpperCase()}_BASE_URL`] - if (batchUrl) urls.push(batchUrl) - } const llmProvider = env.LLM_ACTIVE if (llmProvider) { const llmUrl = env[`LLM_${llmProvider.toUpperCase()}_BASE_URL`] diff --git a/apps/api/src/relay-control.ts b/apps/api/src/relay-control.ts index 3606663..8e44129 100644 --- a/apps/api/src/relay-control.ts +++ b/apps/api/src/relay-control.ts @@ -14,15 +14,13 @@ const SESSION_ID_MAX_LENGTH = 200 function providerSnapshot(env: NodeJS.ProcessEnv): { sttProvider: string - sttBatchProvider: string llmProvider: string llmModel: string } { const sttProvider = env.STT_ACTIVE ?? 'dashscope-realtime' - const sttBatchProvider = env.STT_BATCH_ACTIVE ?? 'dashscope' const llmProvider = env.LLM_ACTIVE ?? 'openrouter' const llmModel = env[`LLM_${llmProvider.toUpperCase()}_MODEL`] ?? 'development' - return { sttProvider, sttBatchProvider, llmProvider, llmModel } + return { sttProvider, llmProvider, llmModel } } async function existingRelaySession( @@ -135,7 +133,7 @@ export async function grantRelaySession(args: { deviceSessionId: args.auth.deviceSessionId, conversationSessionId: args.conversationSessionId, nodeId: node.id, - scopes: ['llm', 'stt', 'stt-realtime'], + scopes: ['llm', 'stt-realtime'], ...providerSnapshot(env), quotaSeconds, }, { env }) diff --git a/apps/api/src/relay-request-auth.ts b/apps/api/src/relay-request-auth.ts index e556843..1aed5af 100644 --- a/apps/api/src/relay-request-auth.ts +++ b/apps/api/src/relay-request-auth.ts @@ -32,7 +32,6 @@ function developmentClaims(scope: RelayScope): RelaySessionClaims { nodeId: relayNodeId(), scopes: [scope], sttProvider: process.env.STT_ACTIVE ?? 'dashscope-realtime', - sttBatchProvider: process.env.STT_BATCH_ACTIVE ?? process.env.STT_ACTIVE ?? 'openrouter', llmProvider: process.env.LLM_ACTIVE ?? 'openrouter', llmModel: process.env.LLM_OPENROUTER_MODEL ?? 'development', quotaSeconds: 30 * 60, diff --git a/apps/api/src/relay-routes.ts b/apps/api/src/relay-routes.ts index 5a47b30..8842c3f 100644 --- a/apps/api/src/relay-routes.ts +++ b/apps/api/src/relay-routes.ts @@ -72,14 +72,17 @@ function heartbeatSessions(value: unknown): RelayActiveSessionHeartbeat[] { } function registerCommonRelayRoutes(app: Hono): void { + // Reachability / RTT only — must not 503 when upstreams are degraded + // (ADR 0006: latency excludes node→provider time). app.get('/api/latency', (context) => context.json( { - ok: providerHealthy(), + ok: true, + providersOk: providerHealthy(), nodeId: relayNodeId(), timestamp: Date.now(), }, - providerHealthy() ? 200 : 503, + 200, { 'cache-control': 'no-store', 'server-timing': 'relay;dur=0', diff --git a/apps/api/src/relay-token.ts b/apps/api/src/relay-token.ts index c9639b4..6aabf49 100644 --- a/apps/api/src/relay-token.ts +++ b/apps/api/src/relay-token.ts @@ -17,7 +17,7 @@ const TOKEN_ISSUER = 'kibotalk-primary' const CLOCK_SKEW_SECONDS = 30 const SESSION_SECONDS = 30 * 60 const RENEW_AFTER_SECONDS = 20 * 60 -const RELAY_SCOPES = new Set(['llm', 'stt', 'stt-realtime']) +const RELAY_SCOPES = new Set(['llm', 'stt-realtime']) let developmentKeys: { privateKey: KeyObject; publicKey: KeyObject } | undefined @@ -64,7 +64,6 @@ function parseClaims(value: unknown): RelaySessionClaims | null { || !Array.isArray(claims.scopes) || claims.scopes.some((scope) => !RELAY_SCOPES.has(scope)) || typeof claims.sttProvider !== 'string' - || typeof claims.sttBatchProvider !== 'string' || typeof claims.llmProvider !== 'string' || typeof claims.llmModel !== 'string' || typeof claims.quotaSeconds !== 'number' diff --git a/apps/api/test/provider-health.test.ts b/apps/api/test/provider-health.test.ts index 4d9a16b..c203ff6 100644 --- a/apps/api/test/provider-health.test.ts +++ b/apps/api/test/provider-health.test.ts @@ -6,8 +6,6 @@ import { const env = { STT_DASHSCOPE_WS_URL: 'wss://stt.example/realtime', - STT_BATCH_ACTIVE: 'dashscope', - STT_DASHSCOPE_BASE_URL: 'https://stt.example/v1', LLM_ACTIVE: 'openrouter', LLM_OPENROUTER_BASE_URL: 'https://llm.example/v1', } @@ -15,23 +13,23 @@ const env = { beforeEach(() => setProviderHealthForTests(true)) describe('provider health', () => { - it('requires realtime STT, batch STT, and LLM upstreams', async () => { + it('requires realtime STT and LLM upstreams', async () => { const fetchImpl = vi.fn(async ( _input: URL | RequestInfo, _init?: RequestInit, ) => new Response(null, { status: 401 })) expect(await refreshProviderHealth(env, fetchImpl as typeof fetch)).toBe(true) - expect(fetchImpl).toHaveBeenCalledTimes(3) + expect(fetchImpl).toHaveBeenCalledTimes(2) expect(fetchImpl.mock.calls[0]?.[0].toString()).toBe( 'https://stt.example/realtime', ) }) it('fails closed when a required upstream is missing or unavailable', async () => { - const missingBatch = { ...env, STT_DASHSCOPE_BASE_URL: undefined } + const missingRealtime = { ...env, STT_DASHSCOPE_WS_URL: undefined } expect(await refreshProviderHealth( - missingBatch, + missingRealtime, vi.fn(async () => new Response(null, { status: 200 })) as typeof fetch, )).toBe(false) @@ -39,7 +37,6 @@ describe('provider health', () => { .fn() .mockResolvedValueOnce(new Response(null, { status: 200 })) .mockResolvedValueOnce(new Response(null, { status: 503 })) - .mockResolvedValueOnce(new Response(null, { status: 200 })) expect(await refreshProviderHealth( env, fetchImpl as typeof fetch, diff --git a/apps/api/test/relay-session-state.test.ts b/apps/api/test/relay-session-state.test.ts index e139606..3711716 100644 --- a/apps/api/test/relay-session-state.test.ts +++ b/apps/api/test/relay-session-state.test.ts @@ -22,7 +22,6 @@ const claims: RelaySessionClaims = { nodeId: 'cn-relay', scopes: ['llm', 'stt-realtime'], sttProvider: 'dashscope-realtime', - sttBatchProvider: 'dashscope', llmProvider: 'openai', llmModel: 'deepseek-v4-flash', quotaSeconds: 5, diff --git a/apps/api/test/relay-token.test.ts b/apps/api/test/relay-token.test.ts index 9e6bcb3..0c60289 100644 --- a/apps/api/test/relay-token.test.ts +++ b/apps/api/test/relay-token.test.ts @@ -23,9 +23,8 @@ const input = { deviceSessionId: 'device-1', conversationSessionId: 'conversation-1', nodeId: 'cn-relay', - scopes: ['llm', 'stt-realtime'] as const, + scopes: ['llm'] as const, sttProvider: 'dashscope-realtime', - sttBatchProvider: 'dashscope', llmProvider: 'openai', llmModel: 'deepseek-v4-flash', quotaSeconds: 120, @@ -61,7 +60,7 @@ describe('relay session token', () => { expect(verifyRelaySessionToken(token, { env, nodeId: 'cn-relay', - requiredScope: 'stt', + requiredScope: 'stt-realtime', now, })).toBeNull() }) diff --git a/apps/api/test/stt.test.ts b/apps/api/test/stt.test.ts deleted file mode 100644 index b800995..0000000 --- a/apps/api/test/stt.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest' -import { serve } from '@hono/node-server' -import { app } from '../src/app' -import { encodeWav } from '@kibotalk/audio' - -const ENV = { - STT_ACTIVE: 'openrouter', - STT_OPENROUTER_BASE_URL: 'https://openrouter.example/api/v1', - STT_OPENROUTER_API_KEY: 'sk-test-secret-do-not-leak', - STT_OPENROUTER_MODEL: 'openai/gpt-4o-transcribe', -} - -let server: ReturnType -let baseUrl: string -let realFetch: typeof globalThis.fetch - -beforeAll(async () => { - realFetch = globalThis.fetch.bind(globalThis) - server = serve({ fetch: app.fetch, port: 0 }) - const { port } = server.address() as { port: number } - baseUrl = `http://localhost:${port}` -}) - -afterAll(() => { - server.close() -}) - -afterEach(() => { - vi.restoreAllMocks() -}) - -function setEnv() { - for (const [k, v] of Object.entries(ENV)) process.env[k] = v - for (const k of Object.keys(process.env)) { - if (k.startsWith('STT_') && !(k in ENV)) delete process.env[k] - } -} - -/** Mock only upstream provider calls; delegate localhost (the proxy) to real fetch. */ -function mockUpstream(response: Response) { - return vi.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => { - const url = typeof input === 'string' ? input : (input as Request).url - if (url.startsWith(baseUrl)) return realFetch(input as RequestInfo, init as RequestInit) - return response - }) -} - -function upstreamCall(calls: Array<[unknown, RequestInit | undefined]>): [string, RequestInit] { - const upstream = calls.find(([url]) => { - const u = typeof url === 'string' ? url : (url as Request).url - return u.includes('/audio/transcriptions') - }) - if (!upstream) throw new Error('upstream /audio/transcriptions call not captured') - return [upstream[0] as string, upstream[1] as RequestInit] -} - -describe('T2 — real /stt through proxy', () => { - it('returns the real transcription from the upstream provider', async () => { - setEnv() - const fetchSpy = mockUpstream( - new Response(JSON.stringify({ text: 'こんにちは' }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }), - ) - - const wav = encodeWav(new Float32Array([0, 0.5, -0.5, 0]), 16000) - const res = await fetch(`${baseUrl}/stt`, { method: 'POST', body: wav }) - - expect(res.ok).toBe(true) - const json = (await res.json()) as { text: string } - expect(json.text).toBe('こんにちは') - - // upstream was called with Bearer auth + the WAV base64 payload - const [url, init] = upstreamCall(fetchSpy.mock.calls as Array<[unknown, RequestInit | undefined]>) - expect(url).toBe(`${ENV.STT_OPENROUTER_BASE_URL}/audio/transcriptions`) - expect(init.headers).toMatchObject({ - Authorization: `Bearer ${ENV.STT_OPENROUTER_API_KEY}`, - 'Content-Type': 'application/json', - }) - const sentBody = JSON.parse(String(init.body)) as { - model: string - input_audio: { format: string; data: string } - } - expect(sentBody.model).toBe(ENV.STT_OPENROUTER_MODEL) - expect(sentBody.input_audio.format).toBe('wav') - expect(sentBody.input_audio.data.length).toBeGreaterThan(0) - }) - - it('never leaks the env API key in the response body or headers', async () => { - setEnv() - mockUpstream(new Response(JSON.stringify({ text: 'transcribed text' }), { status: 200 })) - - const wav = encodeWav(new Float32Array(32), 16000) - const res = await fetch(`${baseUrl}/stt`, { method: 'POST', body: wav }) - const bodyText = await res.text() - - expect(bodyText).not.toContain(ENV.STT_OPENROUTER_API_KEY) - for (const [h, v] of res.headers.entries()) { - expect(h).not.toContain(ENV.STT_OPENROUTER_API_KEY) - expect(v).not.toContain(ENV.STT_OPENROUTER_API_KEY) - } - }) - - it('returns 500 with a clear error when STT env is not configured', async () => { - for (const k of Object.keys(process.env)) { - if (k.startsWith('STT_')) delete process.env[k] - } - const res = await fetch(`${baseUrl}/stt`, { method: 'POST', body: new ArrayBuffer(44) }) - expect(res.status).toBe(500) - const json = (await res.json()) as { error: string } - expect(json.error).toMatch(/STT_ACTIVE/) - }) -}) diff --git a/apps/desktop/build/tray/kibotalk.png b/apps/desktop/build/tray/kibotalk.png deleted file mode 100644 index 5192894..0000000 Binary files a/apps/desktop/build/tray/kibotalk.png and /dev/null differ diff --git a/apps/desktop/build/tray/kibotalk@2x.png b/apps/desktop/build/tray/kibotalk@2x.png deleted file mode 100644 index 72e39f2..0000000 Binary files a/apps/desktop/build/tray/kibotalk@2x.png and /dev/null differ diff --git a/apps/desktop/build/tray/kibotalkTemplate.png b/apps/desktop/build/tray/kibotalkTemplate.png new file mode 100644 index 0000000..2c1583b Binary files /dev/null and b/apps/desktop/build/tray/kibotalkTemplate.png differ diff --git a/apps/desktop/build/tray/kibotalkTemplate@2x.png b/apps/desktop/build/tray/kibotalkTemplate@2x.png new file mode 100644 index 0000000..c9e8c4a Binary files /dev/null and b/apps/desktop/build/tray/kibotalkTemplate@2x.png differ diff --git a/apps/desktop/electron-builder.config.ts b/apps/desktop/electron-builder.config.ts index c02c03d..eafe4f0 100644 --- a/apps/desktop/electron-builder.config.ts +++ b/apps/desktop/electron-builder.config.ts @@ -61,8 +61,8 @@ const config: Configuration = { // Populated by `pnpm download-models` — bundled models, not runtime-fetched (see `src/main/model-protocol.ts`). extraResources: [ { from: 'resources/bundle-models', to: 'models' }, - { from: 'build/tray/kibotalk.png', to: 'tray/kibotalk.png' }, - { from: 'build/tray/kibotalk@2x.png', to: 'tray/kibotalk@2x.png' }, + { from: 'build/tray/kibotalkTemplate.png', to: 'tray/kibotalkTemplate.png' }, + { from: 'build/tray/kibotalkTemplate@2x.png', to: 'tray/kibotalkTemplate@2x.png' }, ], mac: { category: 'public.app-category.productivity', diff --git a/apps/desktop/scripts/verify-island-flip.ts b/apps/desktop/scripts/verify-island-flip.ts new file mode 100644 index 0000000..09e27da --- /dev/null +++ b/apps/desktop/scripts/verify-island-flip.ts @@ -0,0 +1,186 @@ +/** + * Agent-runnable flip harness using the user's logged dual-monitor geometry. + * Run: pnpm exec tsx apps/desktop/scripts/verify-island-flip.ts + */ +import { + boundsKeepingIslandBarFixed, + decideIslandContentSide, + islandBarPoint, + menuSideForContentSide, + type Rect, +} from '../src/main/windows/island-content-side' +import { createElement } from 'react' +import { renderToStaticMarkup } from 'react-dom/server' +import { IslandShell } from '../../../packages/pages/src/IslandShell' + +/** From logged dual-monitor geometry (unequal vertical stack). */ +const DISPLAYS = [ + { id: 1, workArea: { x: 0, y: 34, width: 1710, height: 1073 } }, + { id: 3, workArea: { x: -409, y: -1410, width: 2560, height: 1348 } }, +] + +const WINDOW = { width: 420, height: 723 } +const BAR_EDGE_OFFSET = 33 + +function barOffsetForSide(side: 'above' | 'below') { + return { + x: WINDOW.width / 2, + y: side === 'above' ? WINDOW.height - BAR_EDGE_OFFSET : BAR_EDGE_OFFSET, + } +} + +function placeBarOnDisplay( + displayWorkArea: Rect, + vertical: 'upper' | 'lower', + currentSide: 'above' | 'below', +): Rect { + const x = displayWorkArea.x + displayWorkArea.width - WINDOW.width - 24 + const barY = + displayWorkArea.y + + displayWorkArea.height * (vertical === 'upper' ? 0.25 : 0.75) + const barOffset = barOffsetForSide(currentSide) + const y = Math.round(barY - barOffset.y) + return { x, y, width: WINDOW.width, height: WINDOW.height } +} + +type Case = { + name: string + bounds: Rect + currentSide: 'above' | 'below' + expect: 'above' | 'below' + expectDisplay: number +} + +const cases: Case[] = [ + { + name: 'lower screen upper-half bar with content currently above → below', + bounds: placeBarOnDisplay(DISPLAYS[0]!.workArea, 'upper', 'above'), + currentSide: 'above', + expect: 'below', + expectDisplay: 1, + }, + { + name: 'lower screen upper-half bar stays below', + bounds: placeBarOnDisplay(DISPLAYS[0]!.workArea, 'upper', 'below'), + currentSide: 'below', + expect: 'below', + expectDisplay: 1, + }, + { + name: 'lower screen lower-half bar with content currently below → above', + bounds: placeBarOnDisplay(DISPLAYS[0]!.workArea, 'lower', 'below'), + currentSide: 'below', + expect: 'above', + expectDisplay: 1, + }, + { + name: 'lower screen lower-half bar stays above', + bounds: placeBarOnDisplay(DISPLAYS[0]!.workArea, 'lower', 'above'), + currentSide: 'above', + expect: 'above', + expectDisplay: 1, + }, + { + name: 'upper screen upper-half bar with content currently above → below', + bounds: placeBarOnDisplay(DISPLAYS[1]!.workArea, 'upper', 'above'), + currentSide: 'above', + expect: 'below', + expectDisplay: 3, + }, + { + name: 'upper screen upper-half bar stays below', + bounds: placeBarOnDisplay(DISPLAYS[1]!.workArea, 'upper', 'below'), + currentSide: 'below', + expect: 'below', + expectDisplay: 3, + }, + { + name: 'upper screen lower-half bar with content currently below → above', + bounds: placeBarOnDisplay(DISPLAYS[1]!.workArea, 'lower', 'below'), + currentSide: 'below', + expect: 'above', + expectDisplay: 3, + }, + { + name: 'upper screen lower-half bar stays above', + bounds: placeBarOnDisplay(DISPLAYS[1]!.workArea, 'lower', 'above'), + currentSide: 'above', + expect: 'above', + expectDisplay: 3, + }, +] + +let failed = 0 + +for (const side of ['above', 'below'] as const) { + const markup = renderToStaticMarkup(createElement(IslandShell, { + contentSide: side, + content: null, + island: createElement('div', { 'data-test-island': true }), + })) + const contentIndex = markup.indexOf('data-island-content-slot') + const islandIndex = markup.indexOf('data-test-island') + const contentSlotExists = contentIndex >= 0 + const orderMatches = + side === 'above' + ? contentIndex < islandIndex + : islandIndex < contentIndex + const ok = contentSlotExists && orderMatches + if (!ok) failed += 1 + console.log( + `${ok ? 'PASS' : 'FAIL'} no-content renderer slot ${side}\n` + + ` contentSlotExists=${contentSlotExists} orderMatches=${orderMatches}`, + ) +} + +for (const item of cases) { + const barOffset = barOffsetForSide(item.currentSide) + const result = decideIslandContentSide({ + bounds: item.bounds, + currentSide: item.currentSide, + barOffset, + displays: DISPLAYS, + }) + const menu = menuSideForContentSide(result.desired) + const actualBarY = islandBarPoint(item.bounds, barOffset).y + const settled = boundsKeepingIslandBarFixed( + item.bounds, + item.currentSide, + result.desired, + barOffset, + ) + const settledBarY = islandBarPoint(settled.bounds, settled.barOffset).y + const repeated = decideIslandContentSide({ + bounds: settled.bounds, + currentSide: result.desired, + barOffset: settled.barOffset, + displays: DISPLAYS, + }) + const ok = + result.desired === item.expect + && result.displayId === item.expectDisplay + && settledBarY === actualBarY + && repeated.desired === result.desired + && repeated.displayId === result.displayId + if (!ok) failed += 1 + console.log( + `${ok ? 'PASS' : 'FAIL'} ${item.name}\n` + + ` bounds.y=${item.bounds.y} current=${item.currentSide} → ${result.desired} (expect ${item.expect})\n` + + ` display=${result.displayId} (expect ${item.expectDisplay}) mid=${result.midpoint} barY=${actualBarY} settledY=${settled.bounds.y} repeat=${repeated.desired} menu=${menu}`, + ) +} + +// Menu must match content side (single value). +for (const side of ['above', 'below'] as const) { + const menu = menuSideForContentSide(side) + const expect = side === 'above' ? 'top' : 'bottom' + const ok = menu === expect + if (!ok) failed += 1 + console.log(`${ok ? 'PASS' : 'FAIL'} menu mapping ${side} → ${menu} (expect ${expect})`) +} + +if (failed > 0) { + console.error(`\n${failed} failure(s)`) + process.exit(1) +} +console.log('\nAll island flip cases passed.') diff --git a/apps/desktop/src/main/ipc.ts b/apps/desktop/src/main/ipc.ts index 9f16bfc..f807d6b 100644 --- a/apps/desktop/src/main/ipc.ts +++ b/apps/desktop/src/main/ipc.ts @@ -2,6 +2,7 @@ import { app, BrowserWindow, ipcMain } from 'electron' import { IPC_CHANNEL, type DesktopSessionState, + type IslandBarOffset, type OnboardingContentSize, type ProductWindowView, } from '../shared/ipc' @@ -27,6 +28,8 @@ import { openOnboardingWindow, resizeOnboardingWindow, } from './windows/onboarding' +import { settleIslandContentSide } from './windows/island' +import type { IslandContentSide } from '../shared/ipc' function broadcastAuthChanged(sourceWebContentsId: number): void { for (const window of BrowserWindow.getAllWindows()) { @@ -36,6 +39,12 @@ function broadcastAuthChanged(sourceWebContentsId: number): void { } } +function isIslandBarOffset(value: unknown): value is IslandBarOffset { + if (!value || typeof value !== 'object') return false + const offset = value as Partial + return Number.isFinite(offset.x) && Number.isFinite(offset.y) +} + /** Registers every `ipcMain.handle` channel the preload bridge exposes as `window.kibotalk`. */ export function registerIpcHandlers(params: { getIslandWindow: () => BrowserWindow | null @@ -82,6 +91,20 @@ export function registerIpcHandlers(params: { ipcMain.handle(IPC_CHANNEL.systemAudioStop, () => stopSystemAudioCapture()) ipcMain.handle(IPC_CHANNEL.islandGetContentSide, () => readConfig().islandContentSide) + ipcMain.handle( + IPC_CHANNEL.islandSettleContentSide, + (_event, current: IslandContentSide, barOffset: unknown) => { + const window = params.getIslandWindow() + if ( + !window + || (current !== 'above' && current !== 'below') + || !isIslandBarOffset(barOffset) + ) { + return readConfig().islandContentSide + } + return settleIslandContentSide(window, current, barOffset) + }, + ) ipcMain.handle(IPC_CHANNEL.islandHide, () => params.getIslandWindow()?.hide()) ipcMain.handle(IPC_CHANNEL.islandShow, () => { params.getIslandWindow()?.show() @@ -90,6 +113,19 @@ export function registerIpcHandlers(params: { ipcMain.handle(IPC_CHANNEL.islandSetPointerThrough, (_event, ignored: boolean) => { params.getIslandWindow()?.setIgnoreMouseEvents(ignored, { forward: true }) }) + ipcMain.handle(IPC_CHANNEL.islandGetBounds, () => { + const window = params.getIslandWindow() + if (!window) return { x: 0, y: 0, width: 0, height: 0 } + return window.getBounds() + }) + ipcMain.handle(IPC_CHANNEL.islandSetPosition, (_event, x: number, y: number) => { + const window = params.getIslandWindow() as + | (BrowserWindow & { __scheduleMoveSettled?: () => void }) + | null + if (!window || !Number.isFinite(x) || !Number.isFinite(y)) return + window.setPosition(Math.round(x), Math.round(y)) + window.__scheduleMoveSettled?.() + }) ipcMain.handle(IPC_CHANNEL.sessionUpdateState, (_event, state: DesktopSessionState) => { params.onSessionState(state) diff --git a/apps/desktop/src/main/tray.ts b/apps/desktop/src/main/tray.ts index 39f2434..d9e556a 100644 --- a/apps/desktop/src/main/tray.ts +++ b/apps/desktop/src/main/tray.ts @@ -93,9 +93,9 @@ const copy = { function trayImagePath(): string { const candidates = [ - join(process.resourcesPath, 'tray', 'kibotalk.png'), - join(process.cwd(), 'apps', 'desktop', 'build', 'tray', 'kibotalk.png'), - join(process.cwd(), 'build', 'tray', 'kibotalk.png'), + join(process.resourcesPath, 'tray', 'kibotalkTemplate.png'), + join(process.cwd(), 'apps', 'desktop', 'build', 'tray', 'kibotalkTemplate.png'), + join(process.cwd(), 'build', 'tray', 'kibotalkTemplate.png'), ] return candidates.find(existsSync) ?? candidates[0] } @@ -105,6 +105,7 @@ export function createTrayController(params: { onQuitConfirmed: () => void }) { const image = nativeImage.createFromPath(trayImagePath()) + if (process.platform === 'darwin') image.setTemplateImage(true) const tray = new Tray(image) let sessionState: DesktopSessionState = { lifecycle: 'stopped', diff --git a/apps/desktop/src/main/windows/island-content-side.ts b/apps/desktop/src/main/windows/island-content-side.ts new file mode 100644 index 0000000..bcb9b9a --- /dev/null +++ b/apps/desktop/src/main/windows/island-content-side.ts @@ -0,0 +1,97 @@ +import type { IslandBarOffset, IslandContentSide } from '../../shared/ipc' + +export type Rect = { x: number; y: number; width: number; height: number } +type Point = { x: number; y: number } + +export const FLIP_HYSTERESIS = 48 + +export function clamp(value: number, minimum: number, maximum: number): number { + return Math.min(Math.max(value, minimum), maximum) +} + +/** Display containing the draggable Island bar, or the nearest one. */ +export function pickDisplayForPoint( + point: Point, + displays: Array<{ id: number; workArea: Rect }>, +): { id: number; workArea: Rect } { + let nearest = displays[0] + if (!nearest) throw new Error('no displays') + let nearestDistance = Number.POSITIVE_INFINITY + for (const display of displays) { + const { workArea } = display + const clampedX = clamp(point.x, workArea.x, workArea.x + workArea.width) + const clampedY = clamp(point.y, workArea.y, workArea.y + workArea.height) + const distance = (clampedX - point.x) ** 2 + (clampedY - point.y) ** 2 + if (distance < nearestDistance) { + nearest = display + nearestDistance = distance + } + } + return nearest +} + +export function islandBarPoint(bounds: Rect, offset: IslandBarOffset): Point { + return { + x: bounds.x + offset.x, + y: bounds.y + offset.y, + } +} + +/** Move only the window shell so the visible Island bar stays fixed on screen. */ +export function boundsKeepingIslandBarFixed( + bounds: Rect, + currentSide: IslandContentSide, + nextSide: IslandContentSide, + currentBarOffset: IslandBarOffset, +): { bounds: Rect; barOffset: IslandBarOffset } { + if (currentSide === nextSide) { + return { bounds, barOffset: currentBarOffset } + } + const nextBarOffset = { + x: currentBarOffset.x, + y: bounds.height - currentBarOffset.y, + } + return { + bounds: { + ...bounds, + y: bounds.y + currentBarOffset.y - nextBarOffset.y, + }, + barOffset: nextBarOffset, + } +} + +export function decideIslandContentSide(input: { + bounds: Rect + currentSide: IslandContentSide + barOffset: IslandBarOffset + displays: Array<{ id: number; workArea: Rect }> +}): { + desired: IslandContentSide + displayId: number + workArea: Rect + midpoint: number + islandBarY: number +} { + const barPoint = islandBarPoint(input.bounds, input.barOffset) + const display = pickDisplayForPoint(barPoint, input.displays) + const workArea = display.workArea + const midpoint = workArea.y + workArea.height / 2 + const desired = + barPoint.y < midpoint - FLIP_HYSTERESIS + ? 'below' + : barPoint.y > midpoint + FLIP_HYSTERESIS + ? 'above' + : input.currentSide + return { + desired, + displayId: display.id, + workArea, + midpoint, + islandBarY: barPoint.y, + } +} + +/** Menu open direction locked to content side (no second heuristic). */ +export function menuSideForContentSide(contentSide: IslandContentSide): 'top' | 'bottom' { + return contentSide === 'above' ? 'top' : 'bottom' +} diff --git a/apps/desktop/src/main/windows/island.ts b/apps/desktop/src/main/windows/island.ts index a7c7e4f..1f51cf2 100644 --- a/apps/desktop/src/main/windows/island.ts +++ b/apps/desktop/src/main/windows/island.ts @@ -1,10 +1,19 @@ import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' import { BrowserWindow, screen, type Display, type Rectangle } from 'electron' -import { IPC_CHANNEL } from '../../shared/ipc' +import { + IPC_CHANNEL, + type IslandBarOffset, + type IslandContentSide, +} from '../../shared/ipc' import { loadRendererEntry } from '../location' import { readConfig, updateConfig } from '../config' import { protectPrivilegedWindowNavigation, transparentWindowConfig } from './shared' +import { + boundsKeepingIslandBarFixed, + decideIslandContentSide, + islandBarPoint, +} from './island-content-side' const isMacOS = process.platform === 'darwin' const mainDirname = dirname(fileURLToPath(import.meta.url)) @@ -15,8 +24,11 @@ const MIN_WIDTH = 360 const MIN_HEIGHT = 420 const MAX_WIDTH = 680 const MARGIN = 24 -const ISLAND_CENTER_OFFSET = 29 -const FLIP_HYSTERESIS = 32 + +type IslandBrowserWindow = BrowserWindow & { + __scheduleMoveSettled?: () => void + __suppressMoveSettle?: boolean +} function clamp(value: number, minimum: number, maximum: number): number { return Math.min(Math.max(value, minimum), maximum) @@ -49,6 +61,50 @@ function boundsInsideDisplay(bounds: Rectangle, display: Display): Rectangle { } } +/** + * Compute content side from the renderer's current value + window geometry. + * The renderer owns the live value; main only persists. + */ +export function settleIslandContentSide( + window: BrowserWindow, + currentSide: IslandContentSide, + barOffset: IslandBarOffset, +): IslandContentSide { + const islandWindow = window as IslandBrowserWindow + const boundsNow = window.getBounds() + const displays = screen.getAllDisplays().map((item) => ({ + id: item.id, + workArea: item.workArea, + })) + const { desired } = decideIslandContentSide({ + bounds: boundsNow, + currentSide, + barOffset, + displays, + }) + if (desired === currentSide) return currentSide + const next = boundsKeepingIslandBarFixed( + boundsNow, + currentSide, + desired, + barOffset, + ) + const display = screen.getDisplayNearestPoint(islandBarPoint(next.bounds, next.barOffset)) + islandWindow.__suppressMoveSettle = true + window.setBounds(next.bounds) + updateConfig({ + islandContentSide: desired, + islandBoundsByDisplay: { + ...readConfig().islandBoundsByDisplay, + [displayKey(display)]: next.bounds, + }, + }) + setTimeout(() => { + if (!window.isDestroyed()) islandWindow.__suppressMoveSettle = false + }, 80) + return desired +} + /** AIRI-style transparent, always-on-top, edge-resizable floating window. */ export async function createIslandWindow(): Promise { const cursorDisplay = screen.getDisplayNearestPoint(screen.getCursorScreenPoint()) @@ -70,9 +126,8 @@ export async function createIslandWindow(): Promise { sandbox: false, }, ...transparentWindowConfig(), - }) + }) as IslandBrowserWindow - let contentSide = config.islandContentSide let moveSettledTimer: ReturnType | null = null function persistBounds() { @@ -87,47 +142,24 @@ export async function createIslandWindow(): Promise { }) } - function settleVerticalFlip() { - const boundsNow = window.getBounds() - const display = screen.getDisplayMatching(boundsNow) - const midpoint = display.workArea.y + display.workArea.height / 2 - const islandCenter = - contentSide === 'above' - ? boundsNow.y + boundsNow.height - ISLAND_CENTER_OFFSET - : boundsNow.y + ISLAND_CENTER_OFFSET - const desired = - islandCenter < midpoint - FLIP_HYSTERESIS - ? 'below' - : islandCenter > midpoint + FLIP_HYSTERESIS - ? 'above' - : contentSide - if (desired === contentSide) return - - const nextY = - desired === 'below' - ? islandCenter - ISLAND_CENTER_OFFSET - : islandCenter - boundsNow.height + ISLAND_CENTER_OFFSET - const clampedY = clamp( - Math.round(nextY), - display.workArea.y, - display.workArea.y + display.workArea.height - boundsNow.height, - ) - contentSide = desired - updateConfig({ islandContentSide: desired }) - window.setPosition(boundsNow.x, clampedY) - window.webContents.send(IPC_CHANNEL.islandContentSideChanged, desired) - } - function scheduleMoveSettled() { + if (window.__suppressMoveSettle) return persistBounds() if (moveSettledTimer) clearTimeout(moveSettledTimer) - moveSettledTimer = setTimeout(settleVerticalFlip, 140) + // Renderer owns contentSide — notify it to settle with its single value. + moveSettledTimer = setTimeout(() => { + if (window.isDestroyed()) return + window.webContents.send(IPC_CHANNEL.islandMoveSettledEvent) + }, 140) } + window.__scheduleMoveSettled = scheduleMoveSettled window.on('move', scheduleMoveSettled) + window.on('moved', scheduleMoveSettled) window.on('resize', persistBounds) window.on('closed', () => { if (moveSettledTimer) clearTimeout(moveSettledTimer) + delete window.__scheduleMoveSettled }) window.setAlwaysOnTop(true, 'screen-saver', 1) diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index b9202fc..b676be1 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -38,13 +38,20 @@ const api: KiboTalkDesktopApi = { }, island: { getContentSide: () => ipcRenderer.invoke(IPC_CHANNEL.islandGetContentSide), + settleContentSide: (current, barOffset) => ipcRenderer.invoke( + IPC_CHANNEL.islandSettleContentSide, + current, + barOffset, + ), hide: () => ipcRenderer.invoke(IPC_CHANNEL.islandHide), show: () => ipcRenderer.invoke(IPC_CHANNEL.islandShow), setPointerThrough: (ignored) => ipcRenderer.invoke(IPC_CHANNEL.islandSetPointerThrough, ignored), - onContentSideChanged: (callback) => { - const listener = (_event: Electron.IpcRendererEvent, side: Parameters[0]) => callback(side) - ipcRenderer.on(IPC_CHANNEL.islandContentSideChanged, listener) - return () => ipcRenderer.removeListener(IPC_CHANNEL.islandContentSideChanged, listener) + getBounds: () => ipcRenderer.invoke(IPC_CHANNEL.islandGetBounds), + setPosition: (x, y) => ipcRenderer.invoke(IPC_CHANNEL.islandSetPosition, x, y), + onMoveSettled: (callback) => { + const listener = () => callback() + ipcRenderer.on(IPC_CHANNEL.islandMoveSettledEvent, listener) + return () => ipcRenderer.removeListener(IPC_CHANNEL.islandMoveSettledEvent, listener) }, }, session: { diff --git a/apps/desktop/src/renderer/IslandApp.tsx b/apps/desktop/src/renderer/IslandApp.tsx index 1dc9cfd..29f0c52 100644 --- a/apps/desktop/src/renderer/IslandApp.tsx +++ b/apps/desktop/src/renderer/IslandApp.tsx @@ -46,6 +46,8 @@ function ReadyIsland({ }) { const { language } = useI18n() const [contentSide, setContentSide] = useState('above') + const contentSideRef = useRef(contentSide) + contentSideRef.current = contentSide const snapshot = createSessionSnapshot(prefs) const controller = useProductSession({ languageSnapshot: snapshot, @@ -65,8 +67,24 @@ function ReadyIsland({ }, []) useEffect(() => { - void window.kibotalk.island.getContentSide().then(setContentSide) - return window.kibotalk.island.onContentSideChanged(setContentSide) + void window.kibotalk.island.getContentSide().then((side) => { + setContentSide(side) + }) + return window.kibotalk.island.onMoveSettled(() => { + const bar = document.querySelector('.island-bar') + if (!(bar instanceof HTMLElement)) return + const bounds = bar.getBoundingClientRect() + const barOffset = { + x: bounds.left + bounds.width / 2, + y: bounds.top + bounds.height / 2, + } + void window.kibotalk.island.settleContentSide( + contentSideRef.current, + barOffset, + ).then((next) => { + setContentSide(next) + }) + }) }, []) useEffect(() => { @@ -122,6 +140,8 @@ function ReadyIsland({ useEffect(() => { let ignored = false + let resizing = false + let resizeIdleTimer: number | null = null const setOutlineActive = (active: boolean) => { document .querySelector('.island-window-shell') @@ -129,32 +149,57 @@ function ReadyIsland({ } const handlePointerMove = (event: MouseEvent) => { const edge = - event.clientX <= 8 || - event.clientY <= 8 || - event.clientX >= window.innerWidth - 8 || - event.clientY >= window.innerHeight - 8 + event.clientX <= 12 || + event.clientY <= 12 || + event.clientX >= window.innerWidth - 12 || + event.clientY >= window.innerHeight - 12 const target = document.elementFromPoint(event.clientX, event.clientY) + const overlayOpen = !!( + document.querySelector('[role="dialog"]') + || document.querySelector('[role="menu"]') + ) const interactive = - !!document.querySelector('[role="dialog"]') || - (target instanceof Element && - !!target.closest('.desktop-interactive, [role="dialog"], [role="menu"]')) - setOutlineActive(edge || interactive) + overlayOpen + || (target instanceof Element + && !!target.closest( + '.desktop-interactive, [role="dialog"], [role="menu"], [role="tooltip"], [data-radix-popper-content-wrapper], [data-island-drag-handle]', + )) + // Keep yellow outline sticky while resizing so it does not flicker off + // when the OS briefly leaves the edge hit zone. + setOutlineActive(edge || interactive || overlayOpen || resizing) const nextIgnored = !edge && !interactive if (nextIgnored === ignored) return ignored = nextIgnored void window.kibotalk.island.setPointerThrough(ignored) } const handlePointerLeave = () => { + if (resizing) return setOutlineActive(false) if (ignored) return ignored = true void window.kibotalk.island.setPointerThrough(true) } + const handleResize = () => { + resizing = true + setOutlineActive(true) + if (ignored) { + ignored = false + void window.kibotalk.island.setPointerThrough(false) + } + if (resizeIdleTimer !== null) window.clearTimeout(resizeIdleTimer) + resizeIdleTimer = window.setTimeout(() => { + resizing = false + resizeIdleTimer = null + }, 250) + } window.addEventListener('mousemove', handlePointerMove) + window.addEventListener('resize', handleResize) document.documentElement.addEventListener('mouseleave', handlePointerLeave) return () => { window.removeEventListener('mousemove', handlePointerMove) + window.removeEventListener('resize', handleResize) document.documentElement.removeEventListener('mouseleave', handlePointerLeave) + if (resizeIdleTimer !== null) window.clearTimeout(resizeIdleTimer) setOutlineActive(false) void window.kibotalk.island.setPointerThrough(false) } diff --git a/apps/desktop/src/renderer/OnboardingApp.tsx b/apps/desktop/src/renderer/OnboardingApp.tsx index 74f6f0b..45af265 100644 --- a/apps/desktop/src/renderer/OnboardingApp.tsx +++ b/apps/desktop/src/renderer/OnboardingApp.tsx @@ -220,6 +220,7 @@ function DesktopWindowContent({ void window.kibotalk.onboarding.close()} onRetryReview={async (sessionId) => { @@ -265,6 +267,7 @@ function DesktopWindowContent({ return withSyncStatus( Promise + /** Single source of truth stays in the renderer; main only computes + persists. */ + settleContentSide: ( + current: IslandContentSide, + barOffset: IslandBarOffset, + ) => Promise hide: () => Promise show: () => Promise setPointerThrough: (ignored: boolean) => Promise - onContentSideChanged: (callback: (side: IslandContentSide) => void) => () => void + getBounds: () => Promise<{ x: number; y: number; width: number; height: number }> + setPosition: (x: number, y: number) => Promise + onMoveSettled: (callback: () => void) => () => void } session: { updateState: (state: DesktopSessionState) => Promise diff --git a/apps/playground/src/App.tsx b/apps/playground/src/App.tsx index bd03112..b4746ca 100644 --- a/apps/playground/src/App.tsx +++ b/apps/playground/src/App.tsx @@ -208,7 +208,7 @@ function LanguageOnboarding({ return (
-
+
-
+
= { } export default function DirectApi() { - const { providers, provider } = useTranscribeProvider() - const patch = useConfig((s) => s.patch) const productSurfaceMode = useConfig((s) => s.productSurfaceMode) - const [transcription, setTranscription] = useState('') - const [sttError, setSttError] = useState('') - const [sttBusy, setSttBusy] = useState(false) - const [recording, setRecording] = useState(false) - const audioRef = useRef(null) - const chunksRef = useRef([]) - const [contextText, setContextText] = useState( 'other: 本日はお忙しい中お越しいただきありがとうございます。まずは簡単に自己紹介をお願いします。\nuser: 〇〇大学で情報工学を専攻しております、田中と申します。\nother: では、数ある企業の中で、なぜ弊社を志望されたのでしょうか。', ) @@ -95,75 +83,11 @@ export default function DirectApi() { const abortRef = useRef(null) const batchRef = useRef({ chars: 0, lastFlush: 0 }) - function failStt(message: string) { - setSttError(message) - toast.error(message) - } - function failLlm(message: string) { setLlmError(message) toast.error(message) } - async function sendWav(wav: ArrayBuffer) { - setSttBusy(true) - setSttError('') - setTranscription('') - try { - const res = await fetch( - sttUrl( - useConfig.getState().transcribeProvider, - useConfig.getState().conversationLang, - ), - { method: 'POST', body: wav }, - ) - const json = (await res.json()) as { text?: string; error?: string } - if (!res.ok) throw new Error(json.error ?? `HTTP ${res.status}`) - setTranscription(json.text ?? '') - } catch (e) { - failStt((e as Error).message) - } finally { - setSttBusy(false) - } - } - - async function startRecording() { - setSttError('') - setTranscription('') - try { - const audio = new AudioSource() - audioRef.current = audio - chunksRef.current = [] - await audio.start((chunk) => { - chunksRef.current.push(new Float32Array(chunk)) - }) - setRecording(true) - } catch (e) { - failStt((e as Error).message) - audioRef.current?.stop() - audioRef.current = null - } - } - - async function stopAndTranscribe() { - const audio = audioRef.current - const sampleRate = audio?.sampleRate ?? 16000 - audio?.stop() - audioRef.current = null - setRecording(false) - const chunks = chunksRef.current - chunksRef.current = [] - if (chunks.length === 0) return - const total = chunks.reduce((n, c) => n + c.length, 0) - const pcm = new Float32Array(total) - let off = 0 - for (const c of chunks) { - pcm.set(c, off) - off += c.length - } - await sendWav(encodeWav(pcm, sampleRate)) - } - function parseContext(text: string): ConversationTurn[] { return text .split('\n') @@ -354,50 +278,6 @@ export default function DirectApi() { debug={
-
-

/stt 探针

-
- - (p.mode ?? 'batch') === 'batch')} - value={ - provider && providers.some((p) => p.id === provider && (p.mode ?? 'batch') === 'batch') - ? provider - : null - } - onChange={(p) => patch({ transcribeProvider: p })} - allowOff={false} - offLabel="" - /> -
-
- {!recording ? ( - - ) : ( - - )} -
- {sttBusy ? ( -

- - 转写中… -

- ) : null} - {sttError ?

{sttError}

: null} - {transcription ? ( -

- 文本： - {transcription} -

- ) : null} -
- {metrics.status !== 'idle' ? (
diff --git a/apps/playground/src/LiveSession.tsx b/apps/playground/src/LiveSession.tsx index 9754bf1..897607f 100644 --- a/apps/playground/src/LiveSession.tsx +++ b/apps/playground/src/LiveSession.tsx @@ -19,19 +19,16 @@ import { SILERO_VARIANTS, createWorkerEmbedAudio, createCurrentSpeakerEmbeddingStorage, - ProxySttClient, ProxyLlmClient, connectRealtimeSttWithRetry, finalizedTurnFromRealtimeSegments, isTranscriptionFailed, type RealtimeSttClient, type TranscribedAudioSegment, - providerMode, fetchRelayNodes, openRelaySession, probeRelayNodes, releaseRelaySession, - type SttProvider, } from '@kibotalk/app-shared' import { readLanguageSnapshot, useConfig } from './config-store' import { @@ -51,7 +48,6 @@ import { startSpan, } from '@kibotalk/observability' import { useIoTracerStore } from './io-tracer/store' -import { useTranscribeProvider } from './SttProviderSelect' type TurnView = ConversationTurn & { candidates?: ReplyCandidate[] } @@ -69,7 +65,6 @@ export default function LiveSession({ hasEmbedding: boolean onGoEnroll: () => void }) { - const [speaker, setSpeaker] = useState<'user' | 'other'>('other') const [running, setRunning] = useState(false) const [loading, setLoading] = useState('') const [error, setError] = useState('') @@ -79,17 +74,14 @@ export default function LiveSession({ const [draft, setDraft] = useState(null) const [candidateRounds, setCandidateRounds] = useState([]) const [vadStatus, setVadStatus] = useState<'idle' | 'speech' | 'silence'>('idle') - const [mode, setMode] = useState<'auto' | 'manual' | 'checking'>('checking') + const [mode, setMode] = useState<'auto' | 'checking'>('checking') const [confidence, setConfidence] = useState(null) - /** Actual STT path for the running session (may differ from UI after session-only R4 degrade). */ - const [activeSttPath, setActiveSttPath] = useState<'idle' | 'realtime' | 'batch'>('idle') + const [activeSttPath, setActiveSttPath] = useState<'idle' | 'realtime'>('idle') const speechThreshold = useConfig((s) => s.speechThreshold) const exitThreshold = useConfig((s) => s.exitThreshold) const minSilenceDurationMs = useConfig((s) => s.minSilenceDurationMs) const minSpeechDurationMs = useConfig((s) => s.minSpeechDurationMs) - const prePadMs = useConfig((s) => s.prePadMs) - const postPadMs = useConfig((s) => s.postPadMs) const pauseMs = useConfig((s) => s.pauseMs) const mergeMaxMs = useConfig((s) => s.mergeMaxMs) const speakerThreshold = useConfig((s) => s.speakerThreshold) @@ -99,33 +91,22 @@ export default function LiveSession({ const productSurfaceMode = useConfig((s) => s.productSurfaceMode) const patch = useConfig((s) => s.patch) const setLiveSessionRunning = useConfig((s) => s.setLiveSessionRunning) - const { providers } = useTranscribeProvider() - const speakerRef = useRef(speaker) - speakerRef.current = speaker - const stableSpeakerRef = useRef(speaker) + const stableSpeakerRef = useRef<'user' | 'other'>('other') const llmRef = useRef(null) const audioRef = useRef(null) const pipelineRef = useRef(null) const storageRef = useRef(new InMemoryConversationStorage()) const verifierRef = useRef(null) const embeddingRef = useRef(null) - const autoRef = useRef(false) const vadRef = useRef(null) - const sttRef = useRef(null) - const aggregatorRef = useRef(null) const realtimeAggregatorRef = useRef | null>(null) const realtimeRef = useRef(null) - const realtimeModeRef = useRef(false) const realtimeBusyRef = useRef(Promise.resolve()) const pipelineBusyRef = useRef(Promise.resolve()) - const providersRef = useRef(providers) - providersRef.current = providers const draftMetaRef = useRef<{ speaker: 'user' | 'other'; startedAt: number } | null>(null) - /** Realtime: stream mic while Silero says in-speech (not wait for speech-ready). */ const inSpeechRef = useRef(false) - /** True after append until commit completes — blocks next speech stream from mixing. */ const uncommittedRef = useRef(false) const relaySessionIdRef = useRef(null) @@ -142,22 +123,11 @@ export default function LiveSession({ verifierRef.current?.setThreshold(speakerThreshold) }, [speakerThreshold]) useEffect(() => { - sttRef.current?.configurePadding(prePadMs, postPadMs) - }, [prePadMs, postPadMs]) - useEffect(() => { - aggregatorRef.current?.updateConfig({ - pauseMs, - maxMs: mergeMaxMs, - }) realtimeAggregatorRef.current?.updateConfig({ pauseMs, maxMs: mergeMaxMs, }) }, [pauseMs, mergeMaxMs]) - const transcribeMode = useConfig((s) => s.transcribeMode) - useEffect(() => { - if (transcribeMode !== 'aggregated') aggregatorRef.current?.flush() - }, [transcribeMode]) useEffect(() => { return () => { @@ -171,34 +141,6 @@ export default function LiveSession({ toast.error(message) } - function degradeToBatch(reason: string) { - const batch = providersRef.current.find((p) => p.mode !== 'realtime' && p.id) - if (!batch) { - reportError(`实时转写失败：${reason}（无可用 batch provider 可降级）`) - return false - } - // Session-only: keep UI on realtime selection, but STT POSTs must use a batch id. - realtimeRef.current?.close() - realtimeRef.current = null - realtimeModeRef.current = false - sttRef.current?.setProviderOverride(batch.id) - setActiveSttPath('batch') - const degradeSpan = startSpan(IOSpanNames.SpeechRecognition, { - attrs: { - [IOAttributes.Subsystem]: IOSubsystems.STT, - [IOAttributes.SttPath]: 'realtime', - [IOAttributes.SttDegraded]: true, - }, - }) - degradeSpan.end() - setStatusNote( - `本会话实时转写已降级为 batch（${batch.label}）：${reason}。停止后重新开始可再试实时。`, - ) - setDraft(null) - draftMetaRef.current = null - return true - } - async function verifyWithSpan( buffer: ArrayBuffer, ): Promise<'user' | 'other'> { @@ -209,11 +151,8 @@ export default function LiveSession({ }, }) try { - if (!autoRef.current || !embeddingRef.current || !verifierRef.current) { - stableSpeakerRef.current = speakerRef.current - span.setAttribute(IOAttributes.SpeakerResult, speakerRef.current) - span.end() - return speakerRef.current + if (!embeddingRef.current || !verifierRef.current) { + throw new Error('VOICEPRINT_REQUIRED') } const r = await verifierRef.current.verify(buffer, embeddingRef.current) setConfidence(r.confidence) @@ -256,7 +195,10 @@ export default function LiveSession({ } async function start() { - stableSpeakerRef.current = speakerRef.current + if (!hasEmbedding) { + reportError('VOICEPRINT_REQUIRED') + return + } setError('') setStatusNote('') setLoading('正在检查声纹录入…') @@ -277,12 +219,12 @@ export default function LiveSession({ } const embedding = await verifierRef.current.loadEmbedding() embeddingRef.current = embedding - autoRef.current = !!embedding - setMode(embedding ? 'auto' : 'manual') + if (!embedding) throw new Error('VOICEPRINT_REQUIRED') + setMode('auto') const cfg = useConfig.getState() const selectedProvider = cfg.transcribeProvider - const isRealtime = providerMode(providers, selectedProvider) === 'realtime' + if (!selectedProvider) throw new Error('STT_PROVIDER_REQUIRED') const relaySessionId = globalThis.crypto?.randomUUID?.() ?? `playground-${Date.now()}-${Math.random().toString(36).slice(2)}` @@ -310,104 +252,59 @@ export default function LiveSession({ exitThreshold, minSilenceDurationMs, minSpeechDurationMs, + maxSpeechDurationMs: cfg.mergeMaxMs, speechPadMs: 0, sampleRate: audio.sampleRate, }) vadRef.current = vad - const stt = new ProxySttClient( - audio.sampleRate, - cfg.conversationLang, - () => useConfig.getState().islandSttEnabled, - () => useConfig.getState().transcribeProvider, - ) - stt.configurePadding(prePadMs, postPadMs) - stt.setProviderOverride(null) - sttRef.current = stt const llm = new ProxyLlmClient(readLanguageSnapshot(), () => useConfig.getState().islandReplyEnabled) llmRef.current = llm const storage = storageRef.current - const pipeline = new Pipeline({ stt, llm, conversation: storage }) + const pipeline = new Pipeline({ llm, conversation: storage }) pipelineRef.current = pipeline - realtimeModeRef.current = isRealtime - setActiveSttPath(isRealtime ? 'realtime' : 'batch') - if (!isRealtime) { - setStatusNote( - '当前为 batch STT：无实时草稿，停顿后整段上传。要边说边出字请把 STT 选成带「· 实时」的项（如 dashscope-realtime）。', - ) - } - if (isRealtime && selectedProvider) { - setLoading('正在连接实时转写…') - try { - const rt = await connectRealtimeSttWithRetry({ - provider: selectedProvider, - language: cfg.conversationLang, - sessionId: relaySessionId, - handlers: { - onPartial: (text) => { - const meta = draftMetaRef.current - if (!meta) return - setDraft({ - speaker: meta.speaker, - text, - startedAt: meta.startedAt, - endedAt: Date.now(), - }) - }, - onError: (message) => { - reportError(`实时转写：${message}`) - }, - }, - }) - realtimeRef.current = rt - setStatusNote(`实时转写已连接 · ${relayStatus}`) - setActiveSttPath('realtime') - } catch (e) { - if (!degradeToBatch((e as Error).message)) { - throw e - } - realtimeModeRef.current = false - } - } + setActiveSttPath('realtime') + setLoading('正在连接实时转写…') + const rt = await connectRealtimeSttWithRetry({ + provider: selectedProvider, + language: cfg.conversationLang, + sessionId: relaySessionId, + handlers: { + onPartial: (text) => { + const meta = draftMetaRef.current + if (!meta) return + setDraft({ + speaker: meta.speaker, + text, + startedAt: meta.startedAt, + endedAt: Date.now(), + }) + }, + onError: (message) => { + reportError(`实时转写：${message}`) + }, + }, + }) + realtimeRef.current = rt + setStatusNote(`实时转写已连接 · ${relayStatus}`) - const aggregator = createSegmentAggregator({ + const realtimeAggregator = createSegmentAggregator({ sampleRate: audio.sampleRate, pauseMs: cfg.pauseMs, maxMs: cfg.mergeMaxMs, }) - aggregator.onFlush((merged) => { + realtimeAggregator.onFlush((merged) => { + const turn = finalizedTurnFromRealtimeSegments( + merged, + cfg.conversationLang, + ) + if (!turn) return pipelineBusyRef.current = pipelineBusyRef.current - .then(() => - pipeline.ingestSegment({ - pcm: merged.pcm, - speaker: merged.speaker, - startedAt: merged.startedAt, - endedAt: merged.endedAt, - }), - ) + .then(() => pipeline.ingestFinalizedTurn(turn)) .catch(() => {}) }) - aggregatorRef.current = aggregator - - if (realtimeModeRef.current) { - const realtimeAggregator = createSegmentAggregator({ - sampleRate: audio.sampleRate, - pauseMs: cfg.pauseMs, - maxMs: cfg.mergeMaxMs, - }) - realtimeAggregator.onFlush((merged) => { - const turn = finalizedTurnFromRealtimeSegments( - merged, - cfg.conversationLang, - ) - if (!turn) return - pipelineBusyRef.current = pipelineBusyRef.current - .then(() => pipeline.ingestFinalizedTurn(turn)) - .catch(() => {}) - }) - realtimeAggregatorRef.current = realtimeAggregator - } + realtimeAggregatorRef.current = realtimeAggregator pipeline.on((e: PipelineEvent) => { switch (e.type) { @@ -429,6 +326,8 @@ export default function LiveSession({ prev.map((t) => (t.id === e.turnId ? { ...t, candidates: e.candidates } : t)), ) break + case 'candidatesStreaming': + case 'candidateDelta': case 'llmAborted': case 'llmFailed': break @@ -437,8 +336,11 @@ export default function LiveSession({ prev.map((t) => (t.id === e.turnId ? { ...t, sttFailed: true } as TurnView : t)), ) break - default: + default: { + const _exhaustive: never = e + void _exhaustive break + } } }) @@ -449,12 +351,10 @@ export default function LiveSession({ return } const startedAt = Date.now() - if (realtimeModeRef.current) { - realtimeAggregatorRef.current?.hold() - const who = draftMetaRef.current?.speaker ?? speakerRef.current - draftMetaRef.current = { speaker: who, startedAt } - setDraft({ speaker: who, text: '', startedAt, endedAt: startedAt }) - } + realtimeAggregatorRef.current?.hold() + const who = draftMetaRef.current?.speaker ?? stableSpeakerRef.current + draftMetaRef.current = { speaker: who, startedAt } + setDraft({ speaker: who, text: '', startedAt, endedAt: startedAt }) inSpeechRef.current = true }) vad.on('speech-end', () => { @@ -469,122 +369,70 @@ export default function LiveSession({ draftMetaRef.current?.startedAt ?? now - e.duration * 1000 const endedAt = now - if (realtimeModeRef.current) { - const rt = realtimeRef.current - if (!rt || !uncommittedRef.current) return + const rtConn = realtimeRef.current + if (!rtConn || !uncommittedRef.current) return - rt.commit() - uncommittedRef.current = false - const buffer = e.buffer - const provisional = draftMetaRef.current?.speaker ?? speakerRef.current - const transcription = waitRealtimeCompletedWithSpan(rt).then( - (text) => ({ ok: true as const, text }), - (error: unknown) => ({ ok: false as const, error }), - ) - const verifiedSpeaker = verifyWithSpan(buffer.buffer as ArrayBuffer) - realtimeBusyRef.current = realtimeBusyRef.current - .then(async () => { - const [outcome, who] = await Promise.all([ - transcription, - verifiedSpeaker, - ]) - if (outcome.ok) { - draftMetaRef.current = { speaker: who, startedAt } - setDraft((current) => - current?.startedAt === startedAt ? null : current, - ) - realtimeAggregatorRef.current?.feed({ - buffer, - speaker: who, - text: outcome.text, - startedAt, - endedAt, - }) - return - } - if (isTranscriptionFailed(outcome.error)) { - realtimeAggregatorRef.current?.feed({ - buffer, - speaker: provisional, - text: '', - sttFailed: true, - startedAt, - endedAt, - }) - return - } - if (degradeToBatch(String(outcome.error))) { - await pipeline.ingestSegment({ - pcm: buffer, - speaker: who, - startedAt, - endedAt, - }) - return - } + rtConn.commit() + uncommittedRef.current = false + const buffer = e.buffer + const provisional = draftMetaRef.current?.speaker ?? stableSpeakerRef.current + const transcription = waitRealtimeCompletedWithSpan(rtConn).then( + (text) => ({ ok: true as const, text }), + (error: unknown) => ({ ok: false as const, error }), + ) + const verifiedSpeaker = verifyWithSpan(buffer.buffer as ArrayBuffer) + realtimeBusyRef.current = realtimeBusyRef.current + .then(async () => { + const [outcome, who] = await Promise.all([ + transcription, + verifiedSpeaker, + ]) + if (outcome.ok) { + draftMetaRef.current = { speaker: who, startedAt } + setDraft((current) => + current?.startedAt === startedAt ? null : current, + ) realtimeAggregatorRef.current?.feed({ buffer, speaker: who, - text: '', - sttFailed: true, + text: outcome.text, startedAt, endedAt, }) - }) - .catch(() => {}) - return - } - - // Batch: verify || STT in parallel, then ingest finalized text. - const buffer = e.buffer - void (async () => { - try { - const verifyPromise = verifyWithSpan(buffer.buffer as ArrayBuffer) - - if (useConfig.getState().transcribeMode === 'aggregated') { - const who = await verifyPromise - aggregatorRef.current?.feed({ buffer, speaker: who, startedAt, endedAt }) return } - - const stt = sttRef.current - if (!stt) { - const who = await verifyPromise - void pipeline.ingestSegment({ pcm: buffer, speaker: who, startedAt, endedAt }) + if (isTranscriptionFailed(outcome.error)) { + realtimeAggregatorRef.current?.feed({ + buffer, + speaker: provisional, + text: '', + sttFailed: true, + startedAt, + endedAt, + }) return } - const [who, text] = await Promise.all([ - verifyPromise, - stt.transcribe(buffer, new AbortController().signal), - ]) - await pipeline.ingestFinalizedTurn({ - speaker: who, - text, - startedAt, - endedAt, - }) - } catch (err) { - reportError(`转写失败：${String(err)}`) - const who = speakerRef.current - await pipeline.ingestFinalizedTurn({ + reportError(`实时转写：${String(outcome.error)}`) + realtimeAggregatorRef.current?.feed({ + buffer, speaker: who, text: '', + sttFailed: true, startedAt, endedAt, - sttFailed: true, }) - } - })() + }) + .catch(() => {}) }) await audio.start(async (chunk) => { await vad.processAudio(chunk) if (!useConfig.getState().islandSttEnabled) return - if (!realtimeModeRef.current || !inSpeechRef.current) return - const rt = realtimeRef.current - if (!rt) return + if (!inSpeechRef.current) return + const rtConn = realtimeRef.current + if (!rtConn) return uncommittedRef.current = true - rt.append(chunk) + rtConn.append(chunk) }) setRunning(true) setLiveSessionRunning(true) @@ -600,13 +448,9 @@ export default function LiveSession({ realtimeAggregatorRef.current?.flush() realtimeAggregatorRef.current?.dispose() realtimeAggregatorRef.current = null - aggregatorRef.current?.flush() - aggregatorRef.current?.dispose() - aggregatorRef.current = null realtimeRef.current?.finish() realtimeRef.current?.close() realtimeRef.current = null - realtimeModeRef.current = false inSpeechRef.current = false uncommittedRef.current = false audioRef.current?.stop() @@ -614,7 +458,6 @@ export default function LiveSession({ pipelineRef.current = null llmRef.current = null vadRef.current = null - sttRef.current = null draftMetaRef.current = null setDraft(null) setRunning(false) @@ -650,8 +493,6 @@ export default function LiveSession({ activeSttPath={activeSttPath} mode={mode} confidence={confidence} - speaker={speaker} - onSpeakerChange={setSpeaker} hasEmbedding={hasEmbedding} statusNote={statusNote} error={error} @@ -703,7 +544,7 @@ export default function LiveSession({

窗口模式 · 应用内卡片

{!running ? ( - ) : ( diff --git a/apps/playground/src/SttProviderSelect.tsx b/apps/playground/src/SttProviderSelect.tsx index 502fd55..2049a35 100644 --- a/apps/playground/src/SttProviderSelect.tsx +++ b/apps/playground/src/SttProviderSelect.tsx @@ -8,21 +8,18 @@ import { import { useEffect, useState } from 'react' import { useConfig } from './config-store' import type { SttProvider } from '@kibotalk/app-shared' -import { defaultSttProvider, providerMode, sttUrl } from '@kibotalk/app-shared' +import { defaultRealtimeFirstProvider, fetchSttProviders } from '@kibotalk/app-shared' export type { SttProvider } -export { defaultSttProvider, providerMode, sttUrl } +export { defaultRealtimeFirstProvider as defaultSttProvider } export function useSttProviders(): SttProvider[] { const [providers, setProviders] = useState([]) useEffect(() => { let cancelled = false - fetch('/stt/providers') - .then((r) => (r.ok ? r.json() : { providers: [] })) - .then((d: { providers?: SttProvider[] }) => { - if (!cancelled) setProviders(d.providers ?? []) - }) - .catch(() => {}) + void fetchSttProviders().then((list) => { + if (!cancelled) setProviders(list) + }) return () => { cancelled = true } @@ -50,7 +47,7 @@ type SttProviderSelectProps = { id?: string } -/** Shared STT provider selector (shadcn Select). */ +/** Shared realtime STT provider selector (shadcn Select). */ export function SttProviderSelect({ providers, value, @@ -76,14 +73,13 @@ export function SttProviderSelect({ {providers.map((p) => ( {p.label} - {p.mode === 'realtime' ? ' · 实时' : ' · batch'} {p.active ? ' · 默认' : ''} ))} {providers.length === 0 ? ( -

服务端未配置 STT provider

+

服务端未配置实时 STT provider

) : null}
) diff --git a/apps/playground/src/VadPanel.tsx b/apps/playground/src/VadPanel.tsx index 102de11..1a85265 100644 --- a/apps/playground/src/VadPanel.tsx +++ b/apps/playground/src/VadPanel.tsx @@ -1,11 +1,9 @@ import { useEffect, useRef, useState } from 'react' import { createVAD } from '@kibotalk/audio/vad' import type { VAD } from '@kibotalk/audio/vad' -import { encodeWav, padBuffer } from '@kibotalk/audio' import { createSegmentAggregator } from '@kibotalk/audio/aggregator' import type { SegmentAggregator, AggregatedSegment } from '@kibotalk/audio/aggregator' import { useConfig } from './config-store' -import { sttUrl, providerMode, useSttProviders } from './SttProviderSelect' import { Badge, Button, @@ -24,11 +22,8 @@ import { import { AudioSource, createSileroInfer, SILERO_VARIANTS } from '@kibotalk/app-shared' import { VadParamsFields, - AsrPadFields, MergeParamsFields, VadModelSelect, - TranscribeModeSelect, - TranscribeProviderSelect, } from './components/ConfigFields' import { StageShell } from './components/StageShell' @@ -36,20 +31,12 @@ type Segment = { id: number duration: number buffer: Float32Array - text?: string - transcribing?: boolean - sttError?: string - sttMs?: number } type MergedSegment = { id: number duration: number buffer: Float32Array - text?: string - transcribing?: boolean - sttError?: string - sttMs?: number constituents: { buffer: Float32Array; duration: number }[] } @@ -65,19 +52,12 @@ export default function VadPanel() { const [prob, setProb] = useState(0) const [probHistory, setProbHistory] = useState([]) - // Shared config (zustand). UI reads via hooks; async callbacks read - // useConfig.getState() so they always see the latest values without refs. const speechThreshold = useConfig((s) => s.speechThreshold) const exitThreshold = useConfig((s) => s.exitThreshold) const minSilenceDurationMs = useConfig((s) => s.minSilenceDurationMs) const minSpeechDurationMs = useConfig((s) => s.minSpeechDurationMs) const pauseMs = useConfig((s) => s.pauseMs) - const transcribeProvider = useConfig((s) => s.transcribeProvider) - const transcribeMode = useConfig((s) => s.transcribeMode) const mergeMaxMs = useConfig((s) => s.mergeMaxMs) - const sttProviders = useSttProviders() - const sttProvidersRef = useRef(sttProviders) - sttProvidersRef.current = sttProviders const audioRef = useRef(null) const vadRef = useRef(null) @@ -87,8 +67,6 @@ export default function VadPanel() { const sampleRateRef = useRef(16000) const playCtxRef = useRef(null) - // Live-tune VAD knobs without restarting. speechPadMs is forced to 0 so VAD - // cuts stay tight; pre/post padding is applied at ASR-send time (see padBuffer). useEffect(() => { vadRef.current?.updateConfig({ speechThreshold, @@ -99,12 +77,10 @@ export default function VadPanel() { }) }, [speechThreshold, exitThreshold, minSilenceDurationMs, minSpeechDurationMs]) - // Live-tune the aggregator's merge/scheduling knobs. useEffect(() => { aggregatorRef.current?.updateConfig({ pauseMs, maxMs: mergeMaxMs }) }, [pauseMs, mergeMaxMs]) - // Stop playback when the panel unmounts. useEffect(() => { return () => { playCtxRef.current?.close().catch(() => {}) @@ -128,87 +104,6 @@ export default function VadPanel() { src.start() } - async function transcribeSegment(id: number, buffer: Float32Array) { - setSegments((prev) => prev.map((s) => (s.id === id ? { ...s, transcribing: true } : s))) - const { prePadMs, postPadMs, transcribeProvider } = useConfig.getState() - if (providerMode(sttProvidersRef.current, transcribeProvider) === 'realtime') { - setSegments((prev) => - prev.map((s) => - s.id === id - ? { - ...s, - transcribing: false, - sttError: '实时 STT 请用「实时会话」页；本页仅支持 batch provider', - } - : s, - ), - ) - return - } - const padded = padBuffer(buffer, prePadMs, postPadMs, sampleRateRef.current) - const _wav = encodeWav(padded, sampleRateRef.current) - const startedAt = performance.now() - try { - const res = await fetch( - sttUrl(transcribeProvider, useConfig.getState().conversationLang), - { method: 'POST', body: _wav }, - ) - const json = (await res.json()) as { text?: string; error?: string } - if (!res.ok) throw new Error(json.error ?? `HTTP ${res.status}`) - const sttMs = Math.round(performance.now() - startedAt) - setSegments((prev) => - prev.map((s) => (s.id === id ? { ...s, text: json.text ?? '', transcribing: false, sttMs } : s)), - ) - } catch (e) { - const sttMs = Math.round(performance.now() - startedAt) - setSegments((prev) => - prev.map((s) => - s.id === id ? { ...s, transcribing: false, sttError: (e as Error).message, sttMs } : s, - ), - ) - } - } - - async function transcribeMerged(id: number, buffer: Float32Array) { - setMergedSegments((prev) => prev.map((s) => (s.id === id ? { ...s, transcribing: true } : s))) - const { transcribeProvider } = useConfig.getState() - if (providerMode(sttProvidersRef.current, transcribeProvider) === 'realtime') { - setMergedSegments((prev) => - prev.map((s) => - s.id === id - ? { - ...s, - transcribing: false, - sttError: '实时 STT 请用「实时会话」页；本页仅支持 batch provider', - } - : s, - ), - ) - return - } - const startedAt = performance.now() - try { - const wav = encodeWav(buffer, sampleRateRef.current) - const res = await fetch( - sttUrl(transcribeProvider, useConfig.getState().conversationLang), - { method: 'POST', body: wav }, - ) - const json = (await res.json()) as { text?: string; error?: string } - if (!res.ok) throw new Error(json.error ?? `HTTP ${res.status}`) - const sttMs = Math.round(performance.now() - startedAt) - setMergedSegments((prev) => - prev.map((s) => (s.id === id ? { ...s, text: json.text ?? '', transcribing: false, sttMs } : s)), - ) - } catch (e) { - const sttMs = Math.round(performance.now() - startedAt) - setMergedSegments((prev) => - prev.map((s) => - s.id === id ? { ...s, transcribing: false, sttError: (e as Error).message, sttMs } : s, - ), - ) - } - } - async function start() { setError('') setLoading('正在启动麦克风与音频处理…') @@ -229,9 +124,6 @@ export default function VadPanel() { const vad = createVAD(infer, { sampleRate: audio.sampleRate }) vadRef.current = vad - // Aggregator: same shared module the live session uses. The VAD panel has - // no speaker verification, so every segment is fed as 'other' (the panel's - // pause threshold is therefore `pauseMs`). const aggregator = createSegmentAggregator({ sampleRate: audio.sampleRate, pauseMs: cfg.pauseMs, @@ -239,15 +131,13 @@ export default function VadPanel() { }) aggregator.onFlush((merged: AggregatedSegment) => { const id = ++mergedIdRef.current - const { prePadMs, postPadMs, transcribeProvider } = useConfig.getState() - const buffer = padBuffer(merged.pcm, prePadMs, postPadMs, sampleRateRef.current) + const buffer = merged.pcm const duration = buffer.length / sampleRateRef.current const constituents = merged.segments.map((s) => ({ buffer: s.buffer, duration: s.buffer.length / sampleRateRef.current, })) setMergedSegments((prev) => [...prev, { id, duration, buffer, constituents }].slice(-20)) - if (transcribeProvider !== null) void transcribeMerged(id, buffer) }) aggregatorRef.current = aggregator @@ -260,17 +150,12 @@ export default function VadPanel() { vad.on('speech-ready', (e) => { const id = ++segIdRef.current setSegments((prev) => [...prev, { id, duration: e.duration, buffer: e.buffer }].slice(-20)) - const { transcribeMode, transcribeProvider } = useConfig.getState() - if (transcribeMode === 'aggregated') { - aggregator.feed({ - buffer: e.buffer, - speaker: 'other', - startedAt: Date.now() - e.duration * 1000, - endedAt: Date.now(), - }) - } else if (transcribeProvider !== null) { - void transcribeSegment(id, e.buffer) - } + aggregator.feed({ + buffer: e.buffer, + speaker: 'other', + startedAt: Date.now() - e.duration * 1000, + endedAt: Date.now(), + }) }) await audio.start((chunk) => void vad.processAudio(chunk)) setRunning(true) @@ -349,9 +234,7 @@ export default function VadPanel() {
- -

@@ -360,8 +243,7 @@ export default function VadPanel() {

- - +
@@ -395,65 +277,24 @@ export default function VadPanel() {

- {transcribeMode === 'aggregated' ? '合并片段' : '语音片段'} + 合并片段

- {transcribeMode === 'aggregated' ? ( - mergedSegments.length === 0 ? ( -

- 停顿 {pauseMs}ms 以上会触发一次合并转写 -

- ) : ( -
    - {[...mergedSegments].reverse().map((m) => ( -
  1. -
    - #{m.id} - {(m.duration * 1000).toFixed(0)} ms - {m.constituents.length} 段 - -
    - {transcribeProvider !== null ? ( -
    - {m.transcribing ? ( - 转写中… - ) : m.sttError ? ( - {m.sttError} - ) : ( - {m.text ?? ''} - )} - {m.sttMs != null && !m.transcribing ? ( - {m.sttMs} ms - ) : null} -
    - ) : null} -
    2. - ))} -
- ) - ) : segments.length === 0 ? ( -

还没有检测到语音

+ {mergedSegments.length === 0 ? ( +

+ 停顿 {pauseMs}ms 以上会触发一次合并 +

) : ( -
    - {[...segments].reverse().map((s) => ( -
  1. - #{s.id} - {(s.duration * 1000).toFixed(0)} ms - - {transcribeProvider !== null ? ( - - {s.transcribing ? ( - 转写中… - ) : s.sttError ? ( - {s.sttError} - ) : ( - {s.text ?? ''} - )} - - ) : null} +
      + {[...mergedSegments].reverse().map((m) => ( +
    1. +
      + #{m.id} + {(m.duration * 1000).toFixed(0)} ms + {m.constituents.length} 段 + +
      2. ))}
    diff --git a/apps/playground/src/components/ConfigFields.tsx b/apps/playground/src/components/ConfigFields.tsx index a607647..9da0a11 100644 --- a/apps/playground/src/components/ConfigFields.tsx +++ b/apps/playground/src/components/ConfigFields.tsx @@ -1,3 +1,4 @@ +import { useEffect, useState } from 'react' import { Input, Label, @@ -20,12 +21,12 @@ import { import { fetchRelayNodes, probeRelayNodes, + relayNodeLabelKind, SILERO_VARIANTS, type RelayProbeResult, } from '@kibotalk/app-shared' import { SttProviderSelect } from '../SttProviderSelect' import { useTranscribeProvider } from '../SttProviderSelect' -import type { TranscribeMode } from '../config-store' /** Shared numeric field. Reads/writes nothing itself — fully controlled. */ export function NumberField({ @@ -122,17 +123,17 @@ export function VadParamsFields() { <> patch({ speechThreshold: v })} /> patch({ exitThreshold: v })} /> - patch({ minSilenceDurationMs: v })} /> patch({ minSpeechDurationMs: v })} /> @@ -140,21 +141,6 @@ export function VadParamsFields() { ) } -/** ASR-send padding (applied at ASR send; VAD cuts stay tight). */ -export function AsrPadFields() { - const prePadMs = useConfig((s) => s.prePadMs) - const postPadMs = useConfig((s) => s.postPadMs) - const patch = useConfig((s) => s.patch) - return ( - <> - patch({ prePadMs: v })} /> - patch({ postPadMs: v })} /> - - ) -} - /** Merge / scheduling: single pause + max speech length (TurnGate). */ export function MergeParamsFields({ disabled }: { disabled?: boolean }) { const pauseMs = useConfig((s) => s.pauseMs) @@ -195,64 +181,20 @@ export function VadModelSelect({ disabled }: { disabled?: boolean }) { ) } -/** Transcribe mode: aggregate (merge) vs per-segment. */ -export function TranscribeModeSelect({ disabled }: { disabled?: boolean }) { - const transcribeMode = useConfig((s) => s.transcribeMode) - const patch = useConfig((s) => s.patch) - return ( -
    - - -
    - ) -} - -/** STT provider selector wired to the shared store (auto-bootstraps to active). */ -export function TranscribeProviderSelect({ - allowOff = true, - modeFilter, -}: { - allowOff?: boolean - modeFilter?: 'batch' | 'realtime' -}) { +/** Realtime STT provider selector wired to the shared store. */ +export function TranscribeProviderSelect({ allowOff = true }: { allowOff?: boolean }) { const { providers, provider } = useTranscribeProvider() const patch = useConfig((s) => s.patch) - const filtered = modeFilter - ? providers.filter((p) => (p.mode ?? 'batch') === modeFilter) - : providers - const value = - provider && filtered.some((p) => p.id === provider) ? provider : null - const warnRealtimeOnBatchPage = - modeFilter === 'batch' - && provider - && providers.find((p) => p.id === provider)?.mode === 'realtime' return (
    - + patch({ transcribeProvider: p })} allowOff={allowOff} /> - {warnRealtimeOnBatchPage ? ( -

    - 当前为实时 provider，本页仅 batch；请另选 batch，或到「实时会话」使用 -

    - ) : null}
    ) } @@ -294,14 +236,19 @@ export function RelayNodeSelect({ disabled }: { disabled?: boolean }) { - {results.map(({ node, latencyMs }) => ( - - {node.role === 'primary' ? '日本主节点' : '国内中转'} - {' · '} - {latencyMs === null ? '不可达' : `${Math.round(latencyMs)} ms`} - {node.origin.startsWith('http:') ? ' · HTTP 未加密' : ''} - - ))} + {results.map(({ node, latencyMs }) => { + const kind = relayNodeLabelKind(node) + const title = + kind === 'local' ? '本地节点' : kind === 'primary' ? '日本主节点' : '国内中转' + return ( + + {title} + {' · '} + {latencyMs === null ? '不可达' : `${Math.round(latencyMs)} ms`} + {node.origin.startsWith('http:') ? ' · HTTP 未加密' : ''} + + ) + })}
@@ -372,4 +319,3 @@ export function LanguagePrefsFields({ disabled }: { disabled?: boolean }) {
) } -import { useEffect, useState } from 'react' diff --git a/apps/playground/src/components/DebugPanel.tsx b/apps/playground/src/components/DebugPanel.tsx index dc39e39..6608519 100644 --- a/apps/playground/src/components/DebugPanel.tsx +++ b/apps/playground/src/components/DebugPanel.tsx @@ -1,23 +1,28 @@ import { Button, + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogTrigger, Label, ScrollArea, Separator, } from '@kibotalk/ui' -import { RotateCcw, SlidersHorizontal } from 'lucide-react' +import { Activity, RotateCcw, SlidersHorizontal } from 'lucide-react' import { useConfig } from '../config-store' import { VadParamsFields, - AsrPadFields, MergeParamsFields, RelayNodeSelect, VadModelSelect, - TranscribeModeSelect, NumberField, ThresholdSlider, } from './ConfigFields' -import { SttProviderSelect, useTranscribeProvider, providerMode } from '../SttProviderSelect' +import { SttProviderSelect, useTranscribeProvider } from '../SttProviderSelect' import { IoTracer } from '../io-tracer/IoTracer' +import { useIoTracerStore } from '../io-tracer/store' export type DebugPanelProps = { running: boolean @@ -26,11 +31,10 @@ export type DebugPanelProps = { export function DebugPanel({ running }: DebugPanelProps) { const speakerThreshold = useConfig((s) => s.speakerThreshold) const candidateRoundsMax = useConfig((s) => s.candidateRoundsMax) - const transcribeMode = useConfig((s) => s.transcribeMode) - const mergeEnabled = transcribeMode === 'aggregated' const patch = useConfig((s) => s.patch) const { providers, provider } = useTranscribeProvider() - const sttIsRealtime = providerMode(providers, provider) === 'realtime' + const isRecording = useIoTracerStore((s) => s.isRecording) + const turnCount = useIoTracerStore((s) => s.turns.length) return (
@@ -46,7 +50,6 @@ export function DebugPanel({ running }: DebugPanelProps) {
- {!sttIsRealtime && }
- patch({ speakerThreshold: v })} /> - + -
-

时间流 · IO 追踪

- -
+ + + + + + + 时间流 · IO 追踪 + + 查看会话期间的子系统时序与 IO 跨度；不影响产品主舞台布局。 + + +
+ +
+ +
diff --git a/apps/playground/src/components/SessionToolbar.tsx b/apps/playground/src/components/SessionToolbar.tsx index 92adf9e..708e240 100644 --- a/apps/playground/src/components/SessionToolbar.tsx +++ b/apps/playground/src/components/SessionToolbar.tsx @@ -1,11 +1,6 @@ import { Badge, Button, - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, Tooltip, TooltipContent, TooltipTrigger, @@ -17,8 +12,6 @@ import { Mic, Play, Square, - User, - Users, } from 'lucide-react' const STATE_VARIANT: Record = { @@ -33,11 +26,9 @@ export type SessionToolbarProps = { loading: string state: string vadStatus: string - activeSttPath: 'idle' | 'realtime' | 'batch' - mode: 'auto' | 'manual' | 'checking' + activeSttPath: 'idle' | 'realtime' + mode: 'auto' | 'checking' confidence: number | null - speaker: 'user' | 'other' - onSpeakerChange: (s: 'user' | 'other') => void hasEmbedding: boolean statusNote: string error: string @@ -55,8 +46,6 @@ export function SessionToolbar({ activeSttPath, mode, confidence, - speaker, - onSpeakerChange, hasEmbedding, statusNote, error, @@ -65,22 +54,20 @@ export function SessionToolbar({ onClear, onGoEnroll, }: SessionToolbarProps) { - const sttLabel = - activeSttPath === 'realtime' ? '实时' : activeSttPath === 'batch' ? 'batch' : null + const sttLabel = activeSttPath === 'realtime' ? '实时' : null const vadLabel = vadStatus === 'speech' ? '说话中' : vadStatus === 'silence' ? '静音' : null const speakerLabel = mode === 'auto' - ? `自动${confidence !== null ? ` ${confidence.toFixed(2)}` : ''}` - : mode === 'manual' - ? '手动' - : null + ? `声纹${confidence !== null ? ` ${confidence.toFixed(2)}` : ''}` + : null + const startBlocked = !hasEmbedding return (
{!running ? ( - @@ -94,29 +81,6 @@ export function SessionToolbar({ 清空 - {!hasEmbedding ? (
+ {startBlocked && !running ? ( +

请先录入声纹后再开始会话。

+ ) : null} {statusNote ? (

{statusNote}

) : null} diff --git a/apps/playground/src/components/StageShell.tsx b/apps/playground/src/components/StageShell.tsx index 0ae3799..7a13c2f 100644 --- a/apps/playground/src/components/StageShell.tsx +++ b/apps/playground/src/components/StageShell.tsx @@ -101,8 +101,8 @@ export function StageShell({ ) } - const leftCol = hasLeft ? (leftOpen ? '15rem' : '2.5rem') : null - const debugCol = debugOpen ? '17rem' : '2.5rem' + const leftCol = hasLeft ? (leftOpen ? '14rem' : '2.5rem') : null + const debugCol = debugOpen ? '22rem' : '2.5rem' return (
((set, get) => ({ set({ ...audioDefaults, ...languageSlice(lang), - providerBootstrapped: true, + providerBootstrapped: false, liveSessionRunning: false, }) }, bootstrapProvider: (providers) => - set((s) => - s.providerBootstrapped - ? s - : { transcribeProvider: defaultSttProvider(providers), providerBootstrapped: true }, - ), + set((s) => { + // Don't mark bootstrapped on the empty first paint — wait for /api/stt/providers. + if (providers.length === 0) return s + if (s.providerBootstrapped) { + if (!s.transcribeProvider) return s + if (providers.some((provider) => provider.id === s.transcribeProvider)) return s + const next = defaultRealtimeFirstProvider(providers) + return next ? { transcribeProvider: next } : s + } + const next = defaultRealtimeFirstProvider(providers) + if (!next) return s + return { transcribeProvider: next, providerBootstrapped: true } + }), })) /** Session snapshot of language prefs (frozen at session start). */ diff --git a/apps/playground/src/io-tracer/IoTracer.tsx b/apps/playground/src/io-tracer/IoTracer.tsx index c553294..8fce5b7 100644 --- a/apps/playground/src/io-tracer/IoTracer.tsx +++ b/apps/playground/src/io-tracer/IoTracer.tsx @@ -1,6 +1,4 @@ import type { IOSubsystem } from '@kibotalk/observability' -import { Button, Collapsible, CollapsibleContent, CollapsibleTrigger } from '@kibotalk/ui' -import { ChevronDown, Activity } from 'lucide-react' import { useRef, useState } from 'react' import { IoTracerChart, type IoTracerChartHandle } from './IoTracerChart' import { IoTracerControls } from './IoTracerControls' @@ -9,13 +7,8 @@ import { IoTracerMetrics } from './IoTracerMetrics' import { IoTracerTurnList } from './IoTracerTurnList' import { useIoTracerStore } from './store' -export type IoTracerProps = { - /** When true, panel starts expanded. Default collapsed. */ - defaultOpen?: boolean -} - -export function IoTracer({ defaultOpen = false }: IoTracerProps) { - const [open, setOpen] = useState(defaultOpen) +/** Full IO timeline panel — intended for Dialog / wide surfaces. */ +export function IoTracer() { const [hiddenSubsystems, setHiddenSubsystems] = useState(() => new Set()) const [selectedTurnId, setSelectedTurnId] = useState(null) const chartRef = useRef(null) @@ -46,65 +39,43 @@ export function IoTracer({ defaultOpen = false }: IoTracerProps) { } return ( - -
-
- - - - - {isRecording ? '录制中' : '已停止'} - {turns.length > 0 ? ` · ${turns.length} 轮` : ''} - -
- - -
- { - if (isRecording) stopRecording() - else startRecording() - }} - onClear={clear} - onAutoFit={() => chartRef.current?.autoFit()} - onToggleSubsystem={toggleSubsystem} - onExportOtlp={exportOTLP} - /> - -
- - - selectSpan(null)} - onSelectSpan={selectSpan} - /> -
-
- +
+ { + if (isRecording) stopRecording() + else startRecording() + }} + onClear={clear} + onAutoFit={() => chartRef.current?.autoFit()} + onToggleSubsystem={toggleSubsystem} + onExportOtlp={exportOTLP} + /> + +
+ + + selectSpan(null)} + onSelectSpan={selectSpan} + />
- +
) } diff --git a/apps/playground/src/io-tracer/IoTracerChart.tsx b/apps/playground/src/io-tracer/IoTracerChart.tsx index fa07167..edc7861 100644 --- a/apps/playground/src/io-tracer/IoTracerChart.tsx +++ b/apps/playground/src/io-tracer/IoTracerChart.tsx @@ -104,7 +104,8 @@ export const IoTracerChart = forwardRef return { min: min - pad, max: max + pad } }, [visibleSpans, timeOrigin]) - const chartWidth = Math.max(1, containerWidth - LABEL_COL_WIDTH) + const labelColWidth = visibleSpans.length > 0 ? LABEL_COL_WIDTH : 0 + const chartWidth = Math.max(1, containerWidth - labelColWidth) const minViewDuration = globalRange.max - globalRange.min const maxViewDuration = Math.max(minViewDuration, 10) const minZoomDuration = 1 @@ -146,7 +147,12 @@ export const IoTracerChart = forwardRef useImperativeHandle(ref, () => ({ autoFit }), [autoFit]) useEffect(() => { - if (visibleSpans.length > 0 && !hasUserInteracted.current) { + if (visibleSpans.length === 0) { + hasUserInteracted.current = false + setViewport(globalRange.min, globalRange.max) + return + } + if (!hasUserInteracted.current) { setViewport(globalRange.min, globalRange.max) } }, [visibleSpans.length, globalRange, setViewport]) @@ -464,14 +470,14 @@ export const IoTracerChart = forwardRef : undefined return ( -
+
{/* Minimap */}
}} /> ))} -
-
-
-
-
-
-
- {viewStart !== globalRange.min || viewEnd !== globalRange.max ? ( - + {visibleSpans.length > 0 ? ( + <> +
+
+
+
+
+
+
+ {viewStart !== globalRange.min || viewEnd !== globalRange.max ? ( + + ) : null} + ) : null}
{/* Time axis */}
{ticks.map((tick, i) => (
+
点击 span 查看详情
) @@ -80,7 +80,7 @@ export function IoTracerDetail({ span, turn, onClose, onSelectSpan }: IoTracerDe const color = SUBSYSTEM_CONFIG_MAP.get(span.subsystem)?.color return ( -
+
diff --git a/apps/playground/src/io-tracer/IoTracerTurnList.tsx b/apps/playground/src/io-tracer/IoTracerTurnList.tsx index eaa4bba..60d1264 100644 --- a/apps/playground/src/io-tracer/IoTracerTurnList.tsx +++ b/apps/playground/src/io-tracer/IoTracerTurnList.tsx @@ -38,7 +38,7 @@ export function IoTracerTurnList({ onSelectTurn, }: IoTracerTurnListProps) { return ( -
+
轮次
diff --git a/apps/playground/src/session.ts b/apps/playground/src/session.ts index 01161f4..48ed670 100644 --- a/apps/playground/src/session.ts +++ b/apps/playground/src/session.ts @@ -1,31 +1,15 @@ import { Pipeline } from '@kibotalk/pipeline' -import type { CandidateStreamEvent, LlmClient, SttClient } from '@kibotalk/pipeline' +import type { CandidateStreamEvent, LlmClient } from '@kibotalk/pipeline' import { InMemoryConversationStorage } from '@kibotalk/conversation' import type { ConversationTurn, ReplyCandidate } from '@kibotalk/conversation' import { StubSpeakerVerifier } from '@kibotalk/speaker' /** * In-browser mock providers for the playground session simulator. T4 ships no - * real STT/LLM/WASM speaker — these stand in so the state machine can be driven - * visually. Swap for real clients (T2/T3/T6) without touching this wiring. + * real LLM/WASM speaker — these stand in so the state machine can be driven + * visually. Swap for real clients without touching this wiring. */ -export class PlaygroundStt implements SttClient { - constructor(private getText: () => string, private failNext = false) {} - - setFailNext(value: boolean): void { - this.failNext = value - } - - async transcribe(_pcm: Float32Array, _signal: AbortSignal): Promise { - if (this.failNext) { - this.failNext = false - throw new Error('mock stt failure') - } - return this.getText() - } -} - export class PlaygroundLlm implements LlmClient { async *streamCandidates( context: ConversationTurn[], @@ -49,15 +33,13 @@ export class PlaygroundLlm implements LlmClient { export type SessionHandle = { pipeline: Pipeline storage: InMemoryConversationStorage - stt: PlaygroundStt speaker: StubSpeakerVerifier } -export function createSession(getText: () => string): SessionHandle { +export function createSession(): SessionHandle { const storage = new InMemoryConversationStorage() - const stt = new PlaygroundStt(getText) const llm = new PlaygroundLlm() const speaker = new StubSpeakerVerifier('other') - const pipeline = new Pipeline({ stt, llm, conversation: storage }) - return { pipeline, storage, stt, speaker } + const pipeline = new Pipeline({ llm, conversation: storage }) + return { pipeline, storage, speaker } } diff --git a/apps/playground/vite.config.ts b/apps/playground/vite.config.ts index 601a866..4ffa7df 100644 --- a/apps/playground/vite.config.ts +++ b/apps/playground/vite.config.ts @@ -16,11 +16,6 @@ export default defineConfig({ changeOrigin: true, ws: true, }, - '/stt': { - target: apiOrigin, - changeOrigin: true, - ws: true, - }, '/llm': { target: apiOrigin, changeOrigin: true, diff --git a/docs/adr/0001-client-orchestration-thin-proxy.md b/docs/adr/0001-client-orchestration-thin-proxy.md index a815808..dd9c480 100644 --- a/docs/adr/0001-client-orchestration-thin-proxy.md +++ b/docs/adr/0001-client-orchestration-thin-proxy.md @@ -4,7 +4,7 @@ > “服务端仅无状态代理”的部署假设已由 [ADR 0005](./0005-competition-production-platform.md) > 覆盖。 -会话编排（VAD / 说话人判定 / STT 上行 / conversation store）全部跑在浏览器，服务端只做一个薄 Hono 代理转发 LLM 与 STT 请求、藏 API key、透传 streaming。部署在 Railway 常驻进程上。MVP 不做账号，将来加账号时用 hosted Supabase。 +会话编排（VAD / 说话人判定 / realtime STT 上行 / conversation store）全部跑在浏览器，服务端只做一个薄 Hono 代理转发 LLM 与 `WS /stt-realtime`、藏 API key、透传 streaming。部署在 Railway 常驻进程上。MVP 不做账号，将来加账号时用 hosted Supabase。 ## 为何不把编排放服务端 diff --git a/docs/adr/0002-local-stt-mlx-qwen3-asr.md b/docs/adr/0002-local-stt-mlx-qwen3-asr.md index 6254692..b6a834e 100644 --- a/docs/adr/0002-local-stt-mlx-qwen3-asr.md +++ b/docs/adr/0002-local-stt-mlx-qwen3-asr.md @@ -1,5 +1,8 @@ # 本地 STT = 外部 mlx-qwen3-asr（OpenAI 兼容，经代理） +> **Status: Superseded** — 2026-07-27。Batch / 本地 mlx 路径已移除；产品 STT 仅保留 +> realtime（`WS /stt-realtime`，DashScope `qwen3-asr-flash-realtime`）。下文为历史记录。 + STT 增加一条本地低延迟路径：经 `apps/api` 的 `/stt` 代理转发到一个外部 Python 服务 `mlx-qwen3-asr`（`serve` 模式），它暴露标准 OpenAI `/v1/audio/transcriptions`（multipart）。该服务不纳入本仓库，作为独立进程在本机运行。 ## 为何要本地 STT diff --git a/docs/adr/0003-multilingual-conversation-and-meaning.md b/docs/adr/0003-multilingual-conversation-and-meaning.md index bb0d689..a7b9a55 100644 --- a/docs/adr/0003-multilingual-conversation-and-meaning.md +++ b/docs/adr/0003-multilingual-conversation-and-meaning.md @@ -38,6 +38,6 @@ ## 后果 - `/llm` body 与 Velin args 携带 `conversationLang` / `meaningLang` / `level`（不再默认 JLPT `N5`）。 -- `/stt` 接受 `?language=`，与会话快照的 `conversationLang` 对齐。 +- `WS /stt-realtime` query 接受 `language=`，与会话快照的 `conversationLang` 对齐。 - 评测矩阵以 ja↔zh 为主；en/zh case 套件后置；字段名已改为 `meaning`。 - Playground 语言偏好进 Zustand + localStorage；未确认前挡一层确认卡。 diff --git a/docs/adr/0004-realtime-stt-parallel-to-batch.md b/docs/adr/0004-realtime-stt-parallel-to-batch.md index 0a44cc6..0bb7231 100644 --- a/docs/adr/0004-realtime-stt-parallel-to-batch.md +++ b/docs/adr/0004-realtime-stt-parallel-to-batch.md @@ -1,16 +1,15 @@ -# Realtime STT 与 batch 并行（经代理 WebSocket） +# Realtime-only STT（经代理 WebSocket） -> **2026-07-25 生产约束**：开发 / playground 仍保留两条路径；比赛生产只开放 -> `qwen3-asr-flash-realtime`，不做 batch 降级。见 -> [ADR 0005](./0005-competition-production-platform.md)。 +> **2026-07-27 更新**：batch `POST /stt`、本地 mlx-qwen3-asr 与 batch 降级已移除。 +> 产品仅保留 realtime。历史「与 batch 并行」叙述见文末 changelog。 -STT 增加一条 **realtime** 路径，与现有 batch（`POST /stt`）并行，不是替换。首家上游：阿里云 DashScope `qwen3-asr-flash-realtime`（WSS）。浏览器只连同源 `WS /stt-realtime`；`apps/api` 中继并注入 key，把薄客户端 JSON 映射为上游事件。 +STT 仅走 **realtime** 路径：阿里云 DashScope `qwen3-asr-flash-realtime`（WSS）。浏览器只连同源 `WS /stt-realtime`；`apps/api` 中继并注入 key，把薄客户端 JSON 映射为上游事件。 -## 为何要 realtime（且不迁走 batch） +## 为何 realtime（且不再保留 batch） -Batch 路径是：本地 VAD 判停顿 → 再上传整段 WAV → 才开始转写。停顿阈值（默认 1s）之后还要等一轮 STT RTT，体感「说完很久才出字」。Realtime 在说话过程中持续 `append` 音频，停顿后只需 `commit` 取定稿，转写与说话重叠，砍掉的是 **pause 之后的 STT 等待**（不是取消 pause 本身——pause 仍是 turn 边界）。 +Batch 路径曾是：本地 VAD 判停顿 → 再上传整段 WAV → 才开始转写。停顿阈值（默认 1s）之后还要等一轮 STT RTT，体感「说完很久才出字」。Realtime 在说话过程中持续 `append` 音频，停顿后只需 `commit` 取定稿，转写与说话重叠，砍掉的是 **pause 之后的 STT 等待**（不是取消 pause 本身——pause 仍是 turn 边界）。 -Batch 仍保留：本地 mlx-qwen3-asr、OpenRouter、DashScope file/batch 等不提供或不必上长连接的场景；realtime 失败时可降级到已配置的 batch provider。 +2026-07-27 起 batch / 本地 mlx 路径从代码与生产配置中移除；失败时**不**降级到 `POST /stt`。 ## 为何 Manual + 本地 turn gate（不用 server VAD） @@ -20,36 +19,40 @@ Batch 仍保留：本地 mlx-qwen3-asr、OpenRouter、DashScope file/batch 等 ## 为何 WebSocket 经 apps/api 中继（不浏览器直连） -与 ADR 0001 / 0002 同一原则：key 不出浏览器；换 provider 只改服务端。LLM 仍用 SSE；**仅 realtime STT** 使用 WebSocket（双向：上行音频事件 + 下行 partial/completed）。 +与 ADR 0001 同一原则：key 不出浏览器；换 provider 只改服务端。LLM 仍用 SSE；**STT** 使用 WebSocket（双向：上行音频事件 + 下行 partial/completed）。 整场会话一条长连接（开麦建连，停会话再 finish）；每轮 turn 只 `commit`，不停连。 ## 文本与教练契约 - 时间轴可显示进行中草稿（partial，客户端本地状态） -- 正式 `ConversationTurn` 与 LLM 请求只在 upstream `completed` 之后（`pipeline.ingestFinalizedTurn`） +- 正式 `ConversationTurn` 与 LLM 请求只在 TurnGate flush 之后（`pipeline.ingestFinalizedTurn`） - 不做中途 partial 触发 LLM -## 失败策略（R4） +## 失败策略 -Realtime 断线：短退避重连同一 provider；仍失败且已配置任意 batch provider → 本会话降级为 `POST /stt` 并明示用户；否则停转写并显示错误。 +Realtime 断线：短退避重连同一 provider；仍失败则停止转写并显示错误。**不**降级到 batch。 ## 接线 | 项 | 说明 | |----|------| -| Provider id | `dashscope-realtime`（`mode: realtime`） | +| Provider id | `dashscope-realtime` | | 浏览器 | `WS /stt-realtime?provider=dashscope-realtime&language=ja` | -| Env | 复用 `STT_DASHSCOPE_API_KEY`；`STT_DASHSCOPE_WS_URL`；`STT_DASHSCOPE_REALTIME_MODEL`（默认 `qwen3-asr-flash-realtime`） | +| Env | `STT_ACTIVE=dashscope-realtime`；`STT_DASHSCOPE_API_KEY`；`STT_DASHSCOPE_WS_URL`；`STT_DASHSCOPE_REALTIME_MODEL`（默认 `qwen3-asr-flash-realtime`） | | 薄协议 | 客户端：`session.start` / `append` / `commit` / `finish`；服务端：`ready` / `partial` / `completed` / `error` | | 映射 | `packages/stt` 的 DashScope realtime 辅助函数；仅 `apps/api` 调用 | -| TurnGate | `packages/audio` `createSegmentAggregator`：batch 直拼 PCM；realtime 合并 fragment 定稿文本；均使用单 `pauseMs`、无 gap 填零 | -| Pipeline | batch 仍 `ingestSegment`；realtime 定稿走 `ingestFinalizedTurn` | -| Playground | LiveSession 按 provider `mode` 选 POST 或 WS；草稿 UI；R4 降级 | +| TurnGate | `packages/audio` `createSegmentAggregator`：合并 fragment 定稿文本；使用单 `pauseMs`、无 gap 填零 | +| Pipeline | 定稿经 TurnGate flush 后 `ingestFinalizedTurn` | +| Playground | LiveSession 经 WS；草稿 UI；失败重连，无 batch 降级 | ## 后果 - `apps/api` 持有长连接；开发期 Vite 须代理 WebSocket；Railway 须支持 WS upgrade -- Batch 合并不再为「准确率填静音」；realtime 上下文由上游 session buffer 累积 - 双 pause（user/other）产品旋钮取消；卡壳仍 = user 在统一 `pauseMs` 后入库的半句 - M1 仅 playground + 一家 realtime；`apps/web` 与第二家厂商另开 + +## Changelog + +- **2026-07-25**：初版标题为「Realtime STT 与 batch 并行」；生产约束见 ADR 0005。 +- **2026-07-27**：batch `POST /stt`、mlx-qwen3-asr、R4 降级到 batch 移除；STT 改为 realtime-only；TurnGate 保留（合并 realtime fragment 定稿文本）。 diff --git a/docs/adr/0005-competition-production-platform.md b/docs/adr/0005-competition-production-platform.md index a722dfa..f0582ea 100644 --- a/docs/adr/0005-competition-production-platform.md +++ b/docs/adr/0005-competition-production-platform.md @@ -2,9 +2,9 @@ - **状态**：Accepted - **日期**：2026-07-25 -- **覆盖**：ADR 0001 中 Railway / Supabase / 无账号的部署假设；ADR 0004 中生产降级策略 +- **覆盖**：ADR 0001 中 Railway / Supabase / 无账号的部署假设；ADR 0004 中 realtime-only STT -> **2026-07-26 更新**：单 VPS 数据面、生产 batch STT 禁用和免费 10 分钟规则已由 +> **2026-07-26 更新**：单 VPS 数据面、生产 realtime-only STT 和免费 10 分钟规则已由 > [ADR 0006](./0006-session-pinned-api-relay.md) 覆盖。日本 VPS 仍是唯一控制面。 ## 决策 @@ -72,13 +72,10 @@ ID，防止凭据切换竞态。客户端只缓存不含 token 的最小账户 产品默认使用： -- STT：`STT_ACTIVE=dashscope-realtime`，DashScope；生产 batch 路径由 - `STT_BATCH_ACTIVE` 同时保留 - `qwen3-asr-flash-realtime`； +- STT：`STT_ACTIVE=dashscope-realtime`，DashScope `qwen3-asr-flash-realtime`（仅 realtime，无 batch 路径）； - LLM：DeepSeek `deepseek-v4-flash`，thinking disabled。 -Batch STT 继续用于本地 playground 和开发 provider；生产 API 返回 -`REALTIME_STT_REQUIRED`，不自动降级到 batch。 +Realtime 断线短退避重连；仍失败则停止转写并显示错误，不自动降级。 ## 发布与运维 diff --git a/docs/adr/0006-session-pinned-api-relay.md b/docs/adr/0006-session-pinned-api-relay.md index 0caf639..09841fa 100644 --- a/docs/adr/0006-session-pinned-api-relay.md +++ b/docs/adr/0006-session-pinned-api-relay.md @@ -2,7 +2,7 @@ - **状态**：Accepted - **日期**：2026-07-26 -- **覆盖**：ADR 0005 的单 VPS 数据面、生产 batch STT 禁用和每月 10 分钟免费额度 +- **覆盖**：ADR 0005 的单 VPS 数据面、生产 realtime-only STT 和每月 10 分钟免费额度 ## 背景 @@ -40,8 +40,8 @@ ### 控制面与短令牌 日本服务器使用 Ed25519 私钥签发 30 分钟短令牌，两节点使用公钥离线验签。令牌绑定 -用户、设备、conversation session、选定节点、STT/LLM scope、provider/model 和签发 -时可用额度。客户端在第 20 分钟开始静默续签；续签只能得到相同节点。 +用户、设备、conversation session、选定节点、STT-realtime / LLM scope、`sttProvider`、 +model 和签发时可用额度。客户端在第 20 分钟开始静默续签；续签只能得到相同节点。 实时 STT 建连前，用短令牌向所选节点交换 60 秒、单次使用的 WebSocket ticket， 完整短令牌不进入 WebSocket URL。日本仍通过活跃会话租约保证同一账号只有一个 @@ -89,8 +89,8 @@ HTTPS Web 页面会因浏览器 mixed-content 策略探测失败并回退日本 普通用户北京时间自然月免费额度从 10 分钟提高为 30 分钟。既有当月 10 分钟桶在 首次读取时补足 grant 差额；节点选择与剩余额度无关，实际 AI 使用仍在额度归零时停止。 -生产同时保留 realtime 与 batch STT；产品默认 realtime，batch 由 -`STT_BATCH_ACTIVE` 冻结并通过同一短令牌授权。 +生产 STT 仅 realtime（`STT_ACTIVE=dashscope-realtime`）；短令牌授权 +`stt-realtime` scope 与冻结的 `sttProvider`。 ## 后果 diff --git a/docs/brainstorm/2026-07-23-drama-subtitle-benchmark.md b/docs/brainstorm/2026-07-23-drama-subtitle-benchmark.md index 3ebc3cb..6164a30 100644 --- a/docs/brainstorm/2026-07-23-drama-subtitle-benchmark.md +++ b/docs/brainstorm/2026-07-23-drama-subtitle-benchmark.md @@ -25,11 +25,10 @@ Silero v6.2 原始概率分布为： | 0.20 | 0.10 | 200ms | 200ms | 23 | 0.937 | 0.563 | 0.703 | 0.582 | | 0.50 | 0.20 | 200ms | 200ms | 17 | 0.953 | 0.350 | 0.512 | 0.404 | -结论不是把生产 enter threshold 直接改成 0.20。中文字幕不是逐字日语字幕， -字幕持续时间包含编辑偏差；中心声道归一化也不同于真实麦克风 AGC。0.20 -展示的是本片段的召回上界，同时会增加环境声误触发风险。生产先修复 -realtime completion FIFO 和 TurnGate 绕过问题，让 200ms VAD fragment -不再直接成为正式轮次；阈值改动等带人工标注的真实麦克风噪声集。 +**已采纳（2026-07-27）**：产品 / playground 默认改为网格冠军 +`speechThreshold=0.2`、`exitThreshold=0.1`、`minSilenceDurationMs=400`、 +`minSpeechDurationMs=200`，模型仍为 Silero v6.2。字幕与棚录偏差仍在，真麦噪声 +集到位后可再网格；不再空测不落地。 这份 `.sc.ass` 没有角色名或说话人样式，所以不能调声纹阈值，也不能可靠 评估 TurnGate 的换人 flush。它还是简体中文翻译，并非日语逐字稿，因此也 diff --git a/docs/production-deployment.md b/docs/production-deployment.md index 2f7af84..4062dc1 100644 --- a/docs/production-deployment.md +++ b/docs/production-deployment.md @@ -68,7 +68,8 @@ Create `/opt/kibotalk/.env` from `.env.example` and set at least: - `POSTGRES_PASSWORD` - `AUTH_SECRET` - `SYNC_ENCRYPTION_KEY` -- `STT_DASHSCOPE_API_KEY` +- `STT_DASHSCOPE_API_KEY` (Japan (Tokyo) workspace key) +- `STT_DASHSCOPE_WS_URL=wss://{WorkspaceId}.ap-northeast-1.maas.aliyuncs.com/api-ws/v1/realtime` - `LLM_OPENROUTER_API_KEY` - `RESEND_API_KEY` - `RESEND_FROM_EMAIL` @@ -88,6 +89,10 @@ Create `/opt/kibotalk/.env` from `.env.example` and set at least: `RESEND_FROM_EMAIL` must use a sender domain verified in Resend. Keep `ALLOW_DEV_OTP=false` in production. +The China relay keeps its own China (Beijing) `STT_DASHSCOPE_API_KEY` and +`STT_DASHSCOPE_WS_URL`; production deployment only replaces the primary +server's STT values from the `JP_STT_DASHSCOPE_*` GitHub secrets. + ## China relay environment Create `/opt/kibotalk-relay/.env` with mode `0600`. Copy the same provider diff --git a/docs/solutions/README.md b/docs/solutions/README.md index 6fddfb8..c417d73 100644 --- a/docs/solutions/README.md +++ b/docs/solutions/README.md @@ -28,3 +28,4 @@ problem_type: - `silero-vad-v6-context-frame.md` — Silero VAD v6.2 needs 64-sample context (576 input), not 512 raw. - `transformers-js-automodel-callable.md` — `AutoModel.from_pretrained` returns a callable; invoke `model({...})`, not `model.__call__({...})`. - `api-dotenv-path.md` — `apps/api/src` → repo root is `../../../.env`, not `../../.env`. +- `island-flip-bar-anchor.md` — use the visible Island bar, not the transparent window shell, for multi-monitor flip decisions. diff --git a/docs/solutions/api-dotenv-path.md b/docs/solutions/api-dotenv-path.md index 119e9a7..5b4326f 100644 --- a/docs/solutions/api-dotenv-path.md +++ b/docs/solutions/api-dotenv-path.md @@ -8,11 +8,15 @@ problem_type: config ## 症状 / Symptom -`apps/api` started but every `/stt` request failed with +`apps/api` started but STT requests failed with `HTTP 500: Missing STT config for provider "openai"`, even though `.env` at the -repo root had `STT_OPENAI_*` set. `process.env.STT_*` were undefined inside the +repo root had provider vars set. `process.env.STT_*` were undefined inside the server. +> **Note (2026-07-27)**：当时症状来自 `STT_OPENAI_*`（本地 mlx batch 路径）。 +> 该路径已移除；生产与开发现仅 `STT_ACTIVE=dashscope-realtime` + +> `STT_DASHSCOPE_*`。dotenv 路径问题本身仍适用。 + ## 原因 / Cause `apps/api/src/index.ts` loaded `.env` with @@ -37,5 +41,5 @@ never runs in tests — the bug only surfaced at dev/runtime. ## 参考 / References -- `.env.example` documents the `STT_OPENAI_*` / `LLM_*` naming. +- `.env.example` documents `STT_DASHSCOPE_*` / `LLM_*` naming. - AGENTS.md "Keys in env, never client". diff --git a/docs/solutions/island-flip-bar-anchor.md b/docs/solutions/island-flip-bar-anchor.md new file mode 100644 index 0000000..e929053 --- /dev/null +++ b/docs/solutions/island-flip-bar-anchor.md @@ -0,0 +1,38 @@ +--- +module: desktop +tags: [electron, island, multi-monitor, window-position] +problem_type: regression +--- + +# Island flip must use the visible bar as its anchor + +## 症状 / Symptom + +上下排列且分辨率不同的双显示器上，较高的 Island 窗口可能跨越两个屏幕。使用窗口 +重叠面积或窗口中心选择屏幕时，岛条明明位于当前屏幕上半区，内容却可能翻到上方； +React 重排内容后还可能再次触发相反判定。 + +## 原因 / Cause + +用户实际拖动和观察的是岛条，而不是透明窗口外壳。内容从岛条上方切到下方时， +岛条在窗口内的相对位置也会改变；若只重排 DOM、不同步移动窗口外壳，下一次判定 +看到的是另一个屏幕坐标，形成反馈回路。 + +此外，无转写和建议卡片时，`content` 为 `null`，原 flex 布局没有占位内容区。 +`contentSide=above` 虽然表示岛条应在底部，实际 DOM 却仍把它放在顶部，任何固定 +偏移计算都会读错位置。 + +## 修复 / Fix + +- renderer 始终渲染可伸缩的内容槽，即使当前内容为空； +- renderer 实测岛条中心相对窗口的偏移并通过 IPC 传给主进程，不硬编码高度； +- 用岛条中心点选择其所在或最近的显示器； +- 用该显示器工作区中点和固定滞回区判定内容方向； +- 翻转时只移动窗口外壳，保持岛条的绝对屏幕坐标不变； +- 主进程移动期间抑制一次 settle，renderer 继续持有唯一的 `contentSide`， + 菜单方向也直接由该值派生。 + +## 证据 / Evidence + +`apps/desktop/scripts/verify-island-flip.ts` 使用用户记录的两块不等高显示器坐标， +覆盖两个屏幕的上下半区、两种初始方向，并验证翻转后岛条坐标不变且重复判定不振荡。 diff --git a/docs/spec/live-reply-coach-mvp.md b/docs/spec/live-reply-coach-mvp.md index 2900b09..1794be6 100644 --- a/docs/spec/live-reply-coach-mvp.md +++ b/docs/spec/live-reply-coach-mvp.md @@ -6,7 +6,7 @@ - **作者**：路路（与神奈子需求对齐后整理） > **比赛生产补充（2026-07-25）**：本文件中早期的 Railway、Supabase、 -> “MVP 无账号”和生产 batch fallback 描述已被 +> “MVP 无账号”描述已被 > [ADR 0005](../adr/0005-competition-production-platform.md) 覆盖。正式入口为 > `https://app.kibotalk.app`，账号、强制加密同步、额度和发布要求以 ADR 0005 > 及本文新增 F12–F15 为准。 @@ -112,7 +112,7 @@ | TTS / AI 代说 | 产品定位是用户自己开口 | | iPhone 通话监听 | Web / PWA 做不到 | | 登录账号 | 本地会话即可 | -| 离线 LLM | LLM 走在线 API。本地 ASR 可选（低延迟，见 §2.9 本地 STT） | +| 离线 LLM | LLM 走在线 API；STT 走 realtime 云端代理（§2.9） | | 会话中热切换语言 | 时间轴与 STT hint 会混乱；仅会话外改、新会话快照 | | 对方说异于对话语言的语 | 同场双方都用 `conversationLang`；异语为后续愿景 | @@ -249,7 +249,7 @@ type ReplyCandidate = { | 语音 Pipeline | `packages/pipeline` | [webai-example-realtime-voice-chat](https://github.com/proj-airi/webai-example-realtime-voice-chat)（VAD + STT，无 TTS） | | 提示词 | **Velin** `@velin-dev/core-react`（TSX） | [moeru-ai/velin](https://github.com/moeru-ai/velin) | | LLM | **xsai** | AIRI 生态 | -| STT | 统一走代理；生产仅 DashScope `qwen3-asr-flash-realtime`，开发保留 batch / mlx-qwen3-asr | 见 §2.9、ADR 0005 | +| STT | 统一走代理 `WS /stt-realtime`；DashScope `qwen3-asr-flash-realtime` | 见 §2.9、ADR 0004、ADR 0005 | | 服务端 | **Hono** 薄代理（转发 LLM + STT，藏 key，streaming） | — | | 部署 | **日本 VPS**：Docker Compose + Caddy + Hono + PostgreSQL | ADR 0005 | | Prompt 评估 | **vieval**（根目录 config + `evals/`） | [vieval-dev/vieval](https://github.com/vieval-dev/vieval) | @@ -381,7 +381,7 @@ Velin(repliesPrompt) → xsAI → JSON: 3 candidates | [] 3. **多轮无用户**：可连续多个 other turn——每次 other 停顿达标都触发 LLM；若返回 3 条则刷新候选，若 `[]` 则保留原展示 4. **完整对话进下一轮**：被打断后新一轮 LLM 的 context = **所有已完成的 ConversationTurn**。半截候选与 realtime partial **不进 context**（§1.4「以 STT 为准」） 5. **抢说 / 连说**：对方或用户停说后的阈值倒计时中若另一方（或同方）又开口 → **取消待触发或在途的 LLM**，先完成新 turn 的 STT/入库，再按规则 1 **重新请求教练**（user 入库后同样触发，不再「只入库不出 LLM」） -6. **STT 失败**：batch（仅开发）自动重试 1 次（1s 退避）→ 仍失败 → `appendTurn({ text: '', sttFailed: true })`，UI 标红（**不可补字**）→ **仍可触发 LLM**（上下文带 `sttFailed`，由模型决定是否给提示）→ 循环继续，**不杀会话**。生产 realtime 短退避重连，仍失败则停止转写并明示错误，不降级到 batch（ADR 0005） +6. **STT 失败**：realtime 短退避重连 → 仍失败 → `appendTurn({ text: '', sttFailed: true })` 或停转写并明示错误（生产不降级）→ **仍可触发 LLM**（上下文带 `sttFailed`，由模型决定是否给提示）→ 循环继续，**不杀会话**（ADR 0005） 7. **LLM 失败**：自动重试 1 次（1s 退避）→ 仍失败 → 候选区可提示「出候选失败，重试」，但**不清空**上一轮已提交候选；turn 已入库不动 → 循环继续听下一轮，**不杀会话** 8. **不做**：熔断、多轮指数退避、离线缓存重放、客户端预过滤闸门、双模型、中途 partial 触发 LLM——MVP 过度 @@ -395,8 +395,7 @@ Velin(repliesPrompt) → xsAI → JSON: 3 candidates | [] - 说话人切换（先 flush 旧 turn，再开始新 turn） - 累计**语音**时长 ≥ `VAD_MERGE_MAX_MS` -**Batch**：flush 时把组成段 **直接拼接** PCM（**不**按时间轴填静音 gap）→ 一次 `POST /stt` / `ingestSegment`。 -**Realtime**：每段 VAD speech fragment 上行 `append` 并 Manual `commit` → 等该 fragment 的 `completed`；定稿文本 + 声纹结果再进入 TurnGate，flush 后才 `ingestFinalizedTurn`（不传合并 WAV）。`completed` 与 commit 严格 FIFO 一一对应；单个 `TRANSCRIPTION_FAILED` 只失败对应 fragment，不得清空其他等待项或杀死连接。详见 [ADR 0004](../adr/0004-realtime-stt-parallel-to-batch.md)。 +**Realtime（唯一路径）**：每段 VAD speech fragment 上行 `append` 并 Manual `commit` → 等该 fragment 的 `completed`；定稿文本 + 声纹结果再进入 TurnGate，flush 后才 `ingestFinalizedTurn`。`completed` 与 commit 严格 FIFO 一一对应；单个 `TRANSCRIPTION_FAILED` 只失败对应 fragment，不得清空其他等待项或杀死连接。详见 [ADR 0004](../adr/0004-realtime-stt-parallel-to-batch.md)。 **配置**： @@ -434,7 +433,7 @@ VAD 停顿阈值与说话人判定阈值为**频繁调试参数**，在 playgrou - 完整 diarization（多人、重叠）往往比 ASR 更脆。 - 你们这种 **1 人 enroll + 二分类** 比多语种 STT **更简单**（固定向量 + 阈值，不依赖语言）。 -- 产品风险更大：ASR 错字可改；speaker 判错会乱触候选 / 漏出候选。**不**用 LLM 纠 speaker（成本翻倍且自身会错），**不**做事后纠错；误判对策 = 修 gate 本身（enrollment / 阈值 / 模型 / 安静 demo）。开发期测下游 pipeline 用 Playground 注入 mock speaker 标签，**不**在生产 pipeline 开 manual 分支。 +- 产品风险更大：ASR 错字可改；speaker 判错会乱触候选 / 漏出候选。**不**用 LLM 纠 speaker（成本翻倍且自身会错），**不**做事后纠错；误判对策 = 修 gate 本身（enrollment / 阈值 / 模型 / 安静 demo）。生产 pipeline 仅走声纹 verification，**不**开 manual speaker 分支；Playground 声纹页用于 enrollment 与阈值调参。 **Demo 减误判**：安静环境、两人音色有差、enrollment 念够约 5–10 秒。 @@ -492,8 +491,8 @@ clearHistory(): Promise - running / paused 在刷新、崩溃或设备中断后恢复为 `pauseReason: 'unexpected'`，可继续或停止。 - 时长不计暂停区间。 -Realtime 连续语音也必须在 30 秒切成正式 turn（不能只依赖 batch -`SegmentAggregator.maxMs`）；服务端按 PCM 样本数再做同样上限。余额耗尽时， +Realtime 连续语音也必须在 30 秒切成正式 turn（`SegmentAggregator.maxMs`）； +服务端按 PCM 样本数再做同样上限。余额耗尽时， 服务端只给该 session 一次最终建议和一次复盘 allowance，其余零余额 LLM 请求 直接拒绝；上游失败不消费 allowance。 @@ -582,17 +581,15 @@ packages/pipeline、speaker、llm、conversation （真实实现） |------|------|------|------| | `POST /api/llm` | **SSE 流式** | 接收对话上下文，转发 DeepSeek，流式回 3 条候选或 `[]` | 必须登录；key 只在服务端 env | | `POST /api/session-review` | 普通 POST | 用停止会话的冻结语言与 turn 文本生成短标题和总结 | 必须登录；客户端负责重试 | -| `WS /api/stt-realtime` | **WebSocket** | 中转短令牌换单次票据；speech PCM / commit；DashScope 中继；计时扣额度 | 产品默认 STT；不保存音频 | +| `WS /api/stt-realtime` | **WebSocket** | 中转短令牌换单次票据；speech PCM / commit；DashScope 中继；计时扣额度 | 唯一 STT 路径；不保存音频 | | `/api/auth/*` | REST | 邮箱 OTP、当前账户、设备撤销、WS 单次票据 | Resend + 安全 cookie / bearer | | `/api/sync/*` | REST | 加密会话、删除 tombstone 与偏好同步 | AES-256-GCM；不含声纹 / 音频 | | `/api/admin/*` | REST | 用户、账本、赠送、兑换码、运行面板 | 管理邮箱白名单 | -| `POST /api/stt` | 普通 POST | provider-agnostic batch STT | 生产由 `STT_BATCH_ACTIVE` 冻结 | **流式协议选型**： - **LLM 用 SSE**（Server-Sent Events）：单向服务端→客户端，Hono `streamSSE` 原生支持，是 LLM 流式的事实标准。代理透传 provider 的原始 token 流，浏览器用 `fetch` + `ReadableStream` 读，边收边增量解析结构化输出（3 候选的 JSON，用 partial-json 类库增量 parse）——第一个候选生成时用户就能开始读 -- **Batch STT**：本地 TurnGate 切段后 `POST /stt`（音频 → JSON 转写） -- **Realtime STT 用 WebSocket**：说话中持续上行、下行 partial；LLM 仍不用 WS。浏览器只连同源 `/stt-realtime`，不直连上游 +- **Realtime STT 用 WebSocket**：说话中持续上行 PCM、下行 partial；定稿经 TurnGate 后入库。浏览器只连同源 `/stt-realtime`，不直连上游 **Realtime 薄协议（浏览器 ↔ 代理）**： @@ -621,25 +618,24 @@ app.post('/llm', streamSSE(async (c) => { - 不保存原始音频或声纹 embedding；文本密文与最小查询元数据才入库 - 不做 SSR / 模板渲染 -**STT 上行音频格式**：WAV，16kHz 单声道 PCM。 +**Realtime 上行音频**：16 kHz 单声道 PCM（base64 pcm16le），经 `append` 事件上行；不经 batch WAV POST。 -- 浏览器里音频本就是 PCM（VAD、speaker gate 都吃 PCM），WAV = 加 44 字节头，零依赖零编码 -- 不用 MediaRecorder/WebM——MediaRecorder 是实时录流器，不能对任意 PCM 缓冲事后编码，切片复杂度会渗进 pipeline 状态机 -- 16kHz mono 是语音采样标准，体积可控（3s ≈ 96KB），所有 STT provider 都认 -- `packages/audio` 暴露 `encodeWav(pcm: Float32Array, sampleRate = 16000): ArrayBuffer`；`/stt` 收 WAV 转发 OpenRouter `/audio/transcriptions`（`input_audio.format: "wav"`） +- 浏览器里音频本就是 PCM（VAD、speaker gate 都吃 PCM） +- 16 kHz mono 是语音采样标准；realtime 按 speech fragment 流式上行 +- `packages/audio` 仍暴露 `encodeWav`（调试 / 录音导出等）；STT 路径不依赖 WAV 上传 **静态托管（`apps/web` 产物）**：Web 与 API 打进同一生产镜像，由 Caddy 在 `app.kibotalk.app` 前置 TLS。Web 的 WeSpeaker ResNet34-LM 与 Silero 都使用 Q8：首选固定 commit 的 Hugging Face 文件并进入浏览器缓存，加载失败自动重试 VPS 同源镜像； 桌面模型打进 DMG。DMG 仅通过 GitHub Release 分发，VPS 不托管安装包。 -- Hono 用 `serveStatic` 把 `apps/web/dist` 挂到根路径，API 路由挂 `/llm` `/stt`，PWA manifest / service worker 同源加载（iOS Safari 添加到主屏幕最稳） -- 开发期各自 dev server（Vite 5173 + Hono 8787），Vite proxy 把 `/llm` `/stt` 转发到 Hono，避免开发期 CORS +- Hono 用 `serveStatic` 把 `apps/web/dist` 挂到根路径，API 路由挂 `/llm` `/stt-realtime`，PWA manifest / service worker 同源加载（iOS Safari 添加到主屏幕最稳） +- 开发期各自 dev server（Vite 5173 + Hono 8787），Vite proxy 把 `/llm` `/stt-realtime` 转发到 Hono，避免开发期 CORS - 不构成锁定：`apps/web` 仍是独立 Vite 包，产物纯静态，将来要拆到 Cloudflare Pages 只需改 Hono 不 serve 静态 + 加 CORS ```ts app.use('/llm', ...) -app.use('/stt', ...) +app.use('/stt-realtime', ...) app.use('/*', serveStatic({ root: '../web/dist' })) ``` @@ -657,9 +653,10 @@ app.use('/*', serveStatic({ root: '../web/dist' })) 浏览器（Renderer + Web Worker） VPS（Caddy + Hono + PostgreSQL） ───────────────────────── ────────────────────────────── VAD → SpeakerGate → TurnGate - batch: 合并 PCM ──POST /stt────────→ 转发 batch STT - realtime: append/commit ──WS /stt-realtime──→ 中继上游 realtime - ↓ ←── 定稿文本（realtime 另有 partial→UI 草稿） + append/commit ──WS /stt-realtime──→ 中继上游 realtime + ↓ ←── partial（草稿）/ completed（fragment 定稿） + TurnGate flush → ingestFinalizedTurn + ↓ conversation.appendTurn（仅定稿） ↓ 任一 speaker: ──POST /llm───→ 转发 LLM provider @@ -676,36 +673,36 @@ VAD → SpeakerGate → TurnGate **生产固定**：LLM 直连 DeepSeek `deepseek-v4-flash`（thinking disabled）； STT 以 `STT_ACTIVE=dashscope-realtime` 直连 DashScope -`qwen3-asr-flash-realtime`。下方 OpenRouter / batch 配置仅用于本地开发和 -playground，不是生产 fallback。 +`qwen3-asr-flash-realtime`。 ```bash -# 本地开发可选：LLM 与 STT 共用 OpenRouter +# 日本主节点 STT（realtime only；东京 Workspace + 东京 Key） +STT_ACTIVE=dashscope-realtime +STT_DASHSCOPE_API_KEY=sk-xxxxxxxx +STT_DASHSCOPE_WS_URL=wss://{WorkspaceId}.ap-northeast-1.maas.aliyuncs.com/api-ws/v1/realtime +STT_DASHSCOPE_REALTIME_MODEL=qwen3-asr-flash-realtime + +# 国内 relay 使用同名变量，但配置北京 Workspace + 北京 Key +# STT_DASHSCOPE_WS_URL=wss://{WorkspaceId}.cn-beijing.maas.aliyuncs.com/api-ws/v1/realtime + +# 本地开发可选：LLM 走 OpenRouter LLM_ACTIVE=openrouter LLM_OPENROUTER_BASE_URL=https://openrouter.ai/api/v1 LLM_OPENROUTER_API_KEY=sk-or-... LLM_OPENROUTER_MODEL=deepseek/deepseek-chat # 或 anthropic/claude-...，随时切 -STT_ACTIVE=openrouter -STT_OPENROUTER_BASE_URL=https://openrouter.ai/api/v1 -STT_OPENROUTER_API_KEY=sk-or-... # 同一个 key -STT_OPENROUTER_MODEL=openai/gpt-4o-transcribe # fallback: groq/whisper-large-v3-turbo - # 将来要直连某家（绕开 OpenRouter）时再加一组，无需改代码 # LLM_DEEPSEEK_BASE_URL=https://api.deepseek.com/v1 # LLM_DEEPSEEK_API_KEY=sk-... # LLM_DEEPSEEK_MODEL=deepseek-chat -# STT_OPENAI_BASE_URL=https://api.openai.com/v1 -# STT_OPENAI_API_KEY=sk-... -# STT_OPENAI_MODEL=gpt-4o-transcribe ``` **与代码的接口（provider 无关，不写死 OpenRouter）**： - `packages/llm` 暴露 `createLlmClient({ provider, baseUrl, apiKey, model })`，启动时按 `LLM_ACTIVE` 选一组 env 注入 -- `packages/audio`（或新建 `packages/stt`）对 STT 同构：`createSttClient({ provider, baseUrl, apiKey, model })` -- `apps/api` 的 `/llm` `/stt` 路由接受可选 `provider` 字段做 per-request 覆盖（默认走 `LLM_ACTIVE` / `STT_ACTIVE`），将来按用户偏好路由就靠这个口子 -- **OpenRouter 只是 `provider` 的一个取值，不是代码里的硬编码假设**。加新 provider = 加一个 adapter（实现 `createLlmClient` / `createSttClient` 的接口）+ 加一组 env，不动现有代码、不动其他 adapter。LLM 走 `/chat/completions`、STT 走 `/audio/transcriptions`，是 OpenRouter adapter 自己的事，不渗到工厂接口层 +- `packages/stt` 暴露 DashScope realtime 映射辅助函数；`apps/api` 的 `WS /stt-realtime` 中继上游 +- `apps/api` 的 `/llm` 路由接受可选 `provider` 字段做 per-request 覆盖（默认走 `LLM_ACTIVE`），将来按用户偏好路由就靠这个口子 +- **OpenRouter 只是 LLM `provider` 的一个取值，不是代码里的硬编码假设**。加新 provider = 加一个 adapter + 加一组 env，不动现有代码、不动其他 adapter **分层（key 永不进 DB）**： @@ -717,26 +714,9 @@ STT_OPENROUTER_MODEL=openai/gpt-4o-transcribe # fallback: groq/whisper-large-v 用户不自带 key（已定"走我们中转"），所以 DB 只存"用哪个 provider/model"的选择，不存 key 本身。 -**STT provider 形态**：每个已配置 provider 带 `mode: 'batch' | 'realtime'`（`GET /stt/providers`）。Playground 选 batch → `POST /stt`；选 realtime → `WS /stt-realtime`。`POST /stt` 与 WS 均接受可选 `language=` / query（BCP-47 短码，与 `conversationLang` 对齐）。默认 batch 仍可走 OpenRouter `openai/gpt-4o-transcribe` 等。 - -**本地 STT（可选，低延迟 batch）**：`packages/stt` 注册 `openai` provider——标准 OpenAI 兼容 multipart `/v1/audio/transcriptions`，默认指向本机 [`mlx-qwen3-asr`](https://github.com/moona3k/mlx-qwen3-asr)。**仍经 `apps/api` 的 `/stt` 代理**。详见 [ADR 0002](../adr/0002-local-stt-mlx-qwen3-asr.md)。 +**STT provider**：`dashscope-realtime`（`GET /stt/providers` 列出已配置 realtime provider）。`WS /stt-realtime` 接受可选 `language=` / query（BCP-47 短码，与 `conversationLang` 对齐）。详见 [ADR 0004](../adr/0004-realtime-stt-parallel-to-batch.md)。 -**阿里云 DashScope**： -- **batch** `dashscope`：`POST …/compatible-mode/v1/chat/completions` + `input_audio`，默认 `qwen3-asr-flash` -- **realtime** `dashscope-realtime`：`WS /stt-realtime` → 上游 WSS Manual 模式（本地 TurnGate + commit）。Env：`STT_DASHSCOPE_API_KEY`（与 batch 共用）、`STT_DASHSCOPE_WS_URL`、`STT_DASHSCOPE_REALTIME_MODEL`（默认 `qwen3-asr-flash-realtime`）。详见 [ADR 0004](../adr/0004-realtime-stt-parallel-to-batch.md)。 - -**Realtime 失败（生产）**：短退避重连同一 realtime provider；仍失败则停止转写并显示错误。开发 / playground 可继续验证 batch，但生产不降级。 - -```bash -# 本地 Qwen3-ASR（仅 Apple Silicon，与 apps/api 同机） -pip install "mlx-qwen3-asr[serve]" -mlx-qwen3-asr serve --api-key $(openssl rand -hex 16) # localhost:8765 -# 服务端 .env： -STT_OPENAI_BASE_URL=http://localhost:8765/v1 -STT_OPENAI_API_KEY=本地 serve 启动时生成的 key -STT_OPENAI_MODEL=Qwen/Qwen3-ASR-1.7B # 想更快切 Qwen/Qwen3-ASR-0.6B -# 浏览器：POST /stt?provider=openai -``` +**Realtime 失败**：短退避重连同一 realtime provider；仍失败则停止转写并显示错误。 --- @@ -820,7 +800,7 @@ macOS 状态栏应用：apps/desktop（Electron，共用 packages） | 风险 | 对策 | |------|------| -| 双人同麦 / 短句 / 音色接近导致 speaker 误判 | 本地 verification + 阈值调参；安静 demo；enrollment 念够 5–10 秒；必要时换 NeXt-TDNN mobile / Eagle 等更稳模型；测 user↔other 混淆率。**不**用 LLM 纠 speaker（成本翻倍且自身会错）；**不**做事后纠错；manual 标注仅活在 Playground（注入 mock 标签测下游），不进生产 env | +| 双人同麦 / 短句 / 音色接近导致 speaker 误判 | 本地声纹 verification + 阈值调参；安静 demo；enrollment 念够 5–10 秒；必要时换 NeXt-TDNN mobile / Eagle 等更稳模型；测 user↔other 混淆率。**不**用 LLM 纠 speaker；**不**做事后纠错；生产仅声纹判定，Playground 声纹页用于 enrollment / 阈值调参 | | PWA 本地模型体积大 / iOS 慢 | 生产 WeSpeaker Q8 speaker 权重约 6.4MB；Worker + 缓存；**首次打开后台预下载全部权重**（§1.2，填表并行 + 右上角进度圆，下完才能进），会话中途再下会卡死演示 | | 权限延后申请被拒 / 打断录音 | 首次打开与模型预下载同期申请麦克风等权限；勿拖到「开始会话」或 enrollment 才弹 | | PWA iOS 后台杀进程 | 每轮即时写 IndexedDB；重开后以意外暂停恢复同一会话 | diff --git a/infra/production/env.example b/infra/production/env.example index f9f27a1..8816947 100644 --- a/infra/production/env.example +++ b/infra/production/env.example @@ -10,7 +10,7 @@ RESEND_FROM_NAME=Kibo Ai STT_ACTIVE=dashscope-realtime STT_DASHSCOPE_API_KEY=sk-xxxxxxxxx -STT_DASHSCOPE_WS_URL=wss://YOUR_WORKSPACE.cn-beijing.maas.aliyuncs.com/api-ws/v1/realtime +STT_DASHSCOPE_WS_URL=wss://YOUR_WORKSPACE.ap-northeast-1.maas.aliyuncs.com/api-ws/v1/realtime STT_DASHSCOPE_REALTIME_MODEL=qwen3-asr-flash-realtime LLM_ACTIVE=openai diff --git a/packages/app-shared/src/config.ts b/packages/app-shared/src/config.ts index b40e6c0..5fd1262 100644 --- a/packages/app-shared/src/config.ts +++ b/packages/app-shared/src/config.ts @@ -21,8 +21,6 @@ export type AppConfig = { minSpeechDurationMs: number } vadVariantId: string - /** ASR-send padding (VAD cuts stay tight; padding applied at ASR send). */ - asrPadMs: { pre: number; post: number } /** Segment-aggregator flush triggers (same-speaker accumulation → one turn). */ aggregator: { pauseMs: number; maxMs: number } speakerThreshold: number @@ -36,7 +34,6 @@ export const defaultAppConfig: AppConfig = { minSpeechDurationMs: defaultVadConfig.minSpeechDurationMs, }, vadVariantId: SILERO_VARIANTS[0].id, - asrPadMs: { pre: 80, post: 80 }, aggregator: { pauseMs: 500, maxMs: 30000 }, speakerThreshold: 0.49, } diff --git a/packages/app-shared/src/i18n.tsx b/packages/app-shared/src/i18n.tsx index ff66ffe..8a03177 100644 --- a/packages/app-shared/src/i18n.tsx +++ b/packages/app-shared/src/i18n.tsx @@ -135,6 +135,7 @@ const messages = { selectNetworkNode: '选择本次会话的网络节点', networkNodeDescription: '以下延迟为你的设备到节点的实测往返时间。选择后，本次会话中不会切换节点。', checkingLatency: '正在测量节点延迟…', + localNode: '本地节点', japanNode: '日本主节点', chinaNode: '国内中转节点', encryptedConnection: 'HTTPS 加密连接', @@ -273,6 +274,7 @@ const messages = { selectNetworkNode: 'このセッションのネットワークノードを選択', networkNodeDescription: '表示される遅延は、この端末から各ノードまでの実測往復時間です。セッション中はノードを切り替えません。', checkingLatency: 'ノードの遅延を測定しています…', + localNode: 'ローカルノード', japanNode: '日本メインノード', chinaNode: '中国リレーノード', encryptedConnection: 'HTTPS 暗号化接続', @@ -411,6 +413,7 @@ const messages = { selectNetworkNode: 'Choose a network node for this session', networkNodeDescription: 'Latency is the measured round trip from this device to each node. The node will not change during the session.', checkingLatency: 'Measuring node latency…', + localNode: 'Local node', japanNode: 'Japan primary', chinaNode: 'China relay', encryptedConnection: 'Encrypted with HTTPS', diff --git a/packages/app-shared/src/index.ts b/packages/app-shared/src/index.ts index 966ae85..65398e3 100644 --- a/packages/app-shared/src/index.ts +++ b/packages/app-shared/src/index.ts @@ -44,7 +44,9 @@ export { type RelaySessionSelection, } from './api-runtime' export { + isLocalRelayOrigin, probeRelayNodes, + relayNodeLabelKind, type RelayProbeResult, } from './relay-routing' export { useRelayNodeProbes } from './use-relay-node-probes' @@ -68,10 +70,8 @@ export { } from './cloud-conversation-storage' export { extractCandidates, extractCompleteObjects } from './partial-json' export { - sttUrl, defaultSttProvider, defaultRealtimeFirstProvider, - providerMode, fetchSttProviders, } from './stt-providers' export type { SttProvider } from './stt-providers' @@ -84,7 +84,7 @@ export { export type { RealtimeSttClient, RealtimeSttHandlers } from './realtime-stt-client' export { finalizedTurnFromRealtimeSegments } from './session/realtime-turn' export type { TranscribedAudioSegment } from './session/realtime-turn' -export { ProxySttClient, ProxyLlmClient } from './proxy-clients' +export { ProxyLlmClient } from './proxy-clients' export type { SessionLanguageSnapshot } from './proxy-clients' export type { RelayNode } from '@kibotalk/shared' diff --git a/packages/app-shared/src/proxy-clients.ts b/packages/app-shared/src/proxy-clients.ts index 50ca344..0d743a6 100644 --- a/packages/app-shared/src/proxy-clients.ts +++ b/packages/app-shared/src/proxy-clients.ts @@ -1,14 +1,12 @@ -import type { LlmClient, SttClient, CandidateStreamEvent } from '@kibotalk/pipeline' +import type { LlmClient, CandidateStreamEvent } from '@kibotalk/pipeline' import type { AppLanguage, ConversationTurn, LearnerLevel, ReplyCandidate, } from '@kibotalk/conversation' -import { encodeWav, padBuffer } from '@kibotalk/audio' import { parseSseStream } from './sse' import { extractCandidates } from './partial-json' -import { sttUrl } from './stt-providers' import { relayFetch } from './api-runtime' /** Session snapshot of language prefs (frozen at session start). */ @@ -18,67 +16,14 @@ export type SessionLanguageSnapshot = { level: LearnerLevel } -/** - * Pipeline STT client that talks to the /stt proxy. The proxy holds the - * provider key; the browser just ships WAV. `pcm` is the VAD-cut segment at - * `sampleRate` (16kHz mono). Pre/post silence padding is applied here (ASR - * preprocessing) so VAD cuts can stay tight (speechPadMs = 0). - * - * `isEnabled` / `getProvider` are caller-supplied so this stays agnostic of - * where STT on/off state and provider selection live (playground's dev - * config store vs. a product app's session state) — both wire the same - * client through their own state. - */ -export class ProxySttClient implements SttClient { - private prePadMs = 0 - private postPadMs = 0 - /** Session-only override (e.g. R4 degrade to batch while UI still shows realtime). */ - private providerOverride: string | null = null - - constructor( - private sampleRate = 16000, - private language: AppLanguage = 'ja', - private isEnabled: () => boolean = () => true, - private getProvider: () => string | null = () => null, - ) {} - - configureLanguage(language: AppLanguage): void { - this.language = language - } - - /** Live-tune ASR-level padding without restarting the session. */ - configurePadding(prePadMs: number, postPadMs: number): void { - this.prePadMs = prePadMs - this.postPadMs = postPadMs - } - - setProviderOverride(provider: string | null): void { - this.providerOverride = provider - } - - async transcribe(pcm: Float32Array, signal: AbortSignal): Promise { - if (!this.isEnabled()) return '' - const provider = this.providerOverride ?? this.getProvider() - const padded = padBuffer(pcm, this.prePadMs, this.postPadMs, this.sampleRate) - const wav = encodeWav(padded, this.sampleRate) - const res = await relayFetch( - sttUrl(provider, this.language), - { method: 'POST', body: wav, signal }, - ) - const json = (await res.json().catch(() => ({}))) as { text?: string; error?: string } - if (!res.ok) throw new Error(json.error ?? `STT HTTP ${res.status}`) - return json.text ?? '' - } -} - /** * Pipeline LLM client that talks to the /llm SSE proxy. The proxy renders the * reply-suggestions prompt and streams raw LLM JSON tokens; here we incrementally * parse the candidate JSON array (exactly 3 objects, or []) and map each completed * candidate onto the pipeline's CandidateStreamEvent. * - * `isEnabled` is caller-supplied (see `ProxySttClient`) so the reply-suggestion - * on/off toggle can live wherever the host app keeps its state. + * `isEnabled` is caller-supplied so the reply-suggestion on/off toggle can live + * wherever the host app keeps its state. */ export class ProxyLlmClient implements LlmClient { private conversationLang: AppLanguage diff --git a/packages/app-shared/src/relay-routing.ts b/packages/app-shared/src/relay-routing.ts index c838ca3..fb52998 100644 --- a/packages/app-shared/src/relay-routing.ts +++ b/packages/app-shared/src/relay-routing.ts @@ -9,6 +9,24 @@ export type RelayProbeResult = { successfulAttempts: number } +/** True when the data-plane origin is this machine (dev / local primary). */ +export function isLocalRelayOrigin(origin: string): boolean { + try { + const host = new URL(origin).hostname + return host === 'localhost' || host === '127.0.0.1' || host === '[::1]' + } catch { + return false + } +} + +/** Display label key for a relay node: local vs primary vs china relay. */ +export function relayNodeLabelKind( + node: Pick, +): 'local' | 'primary' | 'relay' { + if (isLocalRelayOrigin(node.origin)) return 'local' + return node.role === 'primary' ? 'primary' : 'relay' +} + type ProbeOptions = RelayNodeList['probe'] & { fetch?: typeof globalThis.fetch now?: () => number diff --git a/packages/app-shared/src/session/use-conversation-session.ts b/packages/app-shared/src/session/use-conversation-session.ts index 3335221..0d60eaa 100644 --- a/packages/app-shared/src/session/use-conversation-session.ts +++ b/packages/app-shared/src/session/use-conversation-session.ts @@ -22,13 +22,13 @@ import { AudioSource } from '../audio/audio-source' import { createSileroInfer, SILERO_VARIANTS } from '../audio/silero-vad' import { createWorkerEmbedAudio } from '../audio/speaker-embed' import { createCurrentSpeakerEmbeddingStorage } from '../speaker-embedding-storage' -import { ProxySttClient, ProxyLlmClient, type SessionLanguageSnapshot } from '../proxy-clients' +import { ProxyLlmClient, type SessionLanguageSnapshot } from '../proxy-clients' import { connectRealtimeSttWithRetry, isTranscriptionFailed, type RealtimeSttClient, } from '../realtime-stt-client' -import { providerMode, type SttProvider } from '../stt-providers' +import type { SttProvider } from '../stt-providers' import { finalizedTurnFromRealtimeSegments, type TranscribedAudioSegment, @@ -60,19 +60,16 @@ export type ConversationSessionParams = { minSilenceDurationMs: number minSpeechDurationMs: number vadVariantId: string - prePadMs: number - postPadMs: number pauseMs: number mergeMaxMs: number speakerThreshold: number - transcribeMode: 'aggregated' | 'perSegment' candidateRoundsMax: number sttEnabled: boolean replyEnabled: boolean languageSnapshot: SessionLanguageSnapshot - /** STT providers the `/stt` proxy has configured (batch + realtime). */ + /** Realtime STT providers from `/api/stt/providers`. */ providers: SttProvider[] - /** Preferred provider id; realtime if its mode is `realtime`, else batch. */ + /** Preferred realtime provider id. */ selectedProvider: string | null /** Defaults to a fresh `InMemoryConversationStorage` (playground behavior). */ storage?: ConversationStorage @@ -89,14 +86,10 @@ export type ConversationSessionParams = { /** * The full live-session orchestration: mic capture → VAD → speaker - * verification → segment aggregation → STT (realtime WS with Manual commit, - * session-only degrade to batch on failure, per ADR 0004) → LLM reply - * candidates via `Pipeline`. One hook, two presentations: the playground's - * `LiveSession` (dev toolbar + debug panel) and `packages/pages`' - * `SessionPage` (product UI) both drive it — only the JSX differs. + * verification → realtime STT (Manual commit) → TurnGate merge → + * `ingestFinalizedTurn` → LLM reply candidates via `Pipeline`. */ export function useConversationSession(params: ConversationSessionParams) { - const [speaker, setSpeaker] = useState<'user' | 'other'>('other') const [running, setRunning] = useState(false) const [loading, setLoading] = useState('') const [error, setError] = useState('') @@ -106,10 +99,9 @@ export function useConversationSession(params: ConversationSessionParams) { const [draft, setDraft] = useState(null) const [candidateRounds, setCandidateRounds] = useState([]) const [vadStatus, setVadStatus] = useState<'idle' | 'speech' | 'silence'>('idle') - const [mode, setMode] = useState<'auto' | 'manual' | 'checking'>('checking') + const [mode, setMode] = useState<'auto' | 'checking'>('checking') const [confidence, setConfidence] = useState(null) - /** Actual STT path for the running session (may differ from the selected provider after R4 degrade). */ - const [activeSttPath, setActiveSttPath] = useState<'idle' | 'realtime' | 'batch'>('idle') + const [activeSttPath, setActiveSttPath] = useState<'idle' | 'realtime'>('idle') const [lifecycle, setLifecycle] = useState( params.persistSessionLifecycle ? 'restoring' : 'stopped', ) @@ -119,9 +111,7 @@ export function useConversationSession(params: ConversationSessionParams) { const [relayNodeId, setRelayNodeId] = useState(null) const [relayLatencyMs, setRelayLatencyMs] = useState(null) - const speakerRef = useRef(speaker) - speakerRef.current = speaker - const stableSpeakerRef = useRef(speaker) + const stableSpeakerRef = useRef<'user' | 'other'>('other') const paramsRef = useRef(params) paramsRef.current = params @@ -133,20 +123,14 @@ export function useConversationSession(params: ConversationSessionParams) { const storageRef = useRef(params.storage ?? new InMemoryConversationStorage()) const verifierRef = useRef(null) const embeddingRef = useRef(null) - const autoRef = useRef(false) const vadRef = useRef(null) const systemVadRef = useRef(null) - const sttRef = useRef(null) - const aggregatorRef = useRef(null) - const systemAggregatorRef = useRef(null) const realtimeAggregatorRef = useRef | null>(null) const systemRealtimeAggregatorRef = useRef | null>(null) const realtimeRef = useRef(null) const systemRealtimeRef = useRef(null) - const realtimeModeRef = useRef(false) - const systemRealtimeModeRef = useRef(false) const realtimeBusyRef = useRef(Promise.resolve()) const pipelineBusyRef = useRef(Promise.resolve()) const startInFlightRef = useRef(false) @@ -220,49 +204,22 @@ export function useConversationSession(params: ConversationSessionParams) { useEffect(() => { verifierRef.current?.setThreshold(params.speakerThreshold) }, [params.speakerThreshold]) - useEffect(() => { - sttRef.current?.configurePadding(params.prePadMs, params.postPadMs) - }, [params.prePadMs, params.postPadMs]) useEffect(() => { const config = { pauseMs: params.pauseMs, maxMs: params.mergeMaxMs } - aggregatorRef.current?.updateConfig(config) - systemAggregatorRef.current?.updateConfig(config) realtimeAggregatorRef.current?.updateConfig(config) systemRealtimeAggregatorRef.current?.updateConfig(config) }, [params.pauseMs, params.mergeMaxMs]) - useEffect(() => { - if (params.transcribeMode !== 'aggregated') aggregatorRef.current?.flush() - }, [params.transcribeMode]) function reportError(message: string) { setError(message) } - function degradeToBatch(reason: string) { - const batch = paramsRef.current.providers.find((p) => p.mode !== 'realtime' && p.id) - if (!batch) { - reportError(`实时转写失败：${reason}（无可用 batch provider 可降级）`) - return false - } - // Session-only: keep the caller's provider selection, but STT POSTs must use a batch id. - realtimeRef.current?.close() - realtimeRef.current = null - realtimeModeRef.current = false - sttRef.current?.setProviderOverride(batch.id) - setActiveSttPath('batch') - setStatusNote(`本会话实时转写已降级为 batch（${batch.label}）：${reason}。停止后重新开始可再试实时。`) - setDraft(null) - draftMetaRef.current = null - return true - } - async function verifySpeaker(buffer: ArrayBuffer): Promise<'user' | 'other'> { const audioSource = paramsRef.current.sessionSnapshot?.audioSource if (audioSource === 'both') return 'user' if (audioSource === 'system') return 'other' - if (!autoRef.current || !embeddingRef.current || !verifierRef.current) { - stableSpeakerRef.current = speakerRef.current - return speakerRef.current + if (!embeddingRef.current || !verifierRef.current) { + throw new Error('VOICEPRINT_REQUIRED') } try { const r = await verifierRef.current.verify(buffer, embeddingRef.current) @@ -286,12 +243,11 @@ export function useConversationSession(params: ConversationSessionParams) { } = {}) { if (startInFlightRef.current || lifecycle === 'starting' || running) return startInFlightRef.current = true - stableSpeakerRef.current = speakerRef.current - const resuming = options.resume === true setError('') setStatusNote('') setLoading('正在检查声纹录入…') setLifecycle('starting') + const resuming = options.resume === true if (!resuming) setTurns([]) setDraft(null) if (!resuming) setCandidateRounds([]) @@ -315,11 +271,8 @@ export function useConversationSession(params: ConversationSessionParams) { } const embedding = await verifierRef.current.loadEmbedding() embeddingRef.current = embedding - autoRef.current = !!embedding - setMode(embedding ? 'auto' : 'manual') - if (paramsRef.current.persistSessionLifecycle && !embedding) { - throw new Error('VOICEPRINT_REQUIRED') - } + if (!embedding) throw new Error('VOICEPRINT_REQUIRED') + setMode('auto') if (paramsRef.current.persistSessionLifecycle) { if (resuming) { @@ -347,6 +300,8 @@ export function useConversationSession(params: ConversationSessionParams) { } const p = paramsRef.current + const selectedProvider = p.selectedProvider + if (!selectedProvider) throw new Error('STT_PROVIDER_REQUIRED') const selectedRelayNodeId = frozenRelayNodeId ?? options.relayNodeId if (!selectedRelayNodeId) throw new Error('RELAY_NODE_SELECTION_REQUIRED') setLoading('正在连接所选网络节点…') @@ -368,10 +323,7 @@ export function useConversationSession(params: ConversationSessionParams) { }) setActiveSession(created) } - const selectedProvider = p.selectedProvider const audioSourceMode = p.sessionSnapshot?.audioSource ?? 'microphone' - const isRealtime = - providerMode(p.providers, selectedProvider) === 'realtime' setLoading('正在启动麦克风…') const primaryStream = @@ -401,15 +353,6 @@ export function useConversationSession(params: ConversationSessionParams) { }) vadRef.current = vad - const stt = new ProxySttClient( - audio.sampleRate, - p.languageSnapshot.conversationLang, - () => paramsRef.current.sttEnabled, - () => paramsRef.current.selectedProvider, - ) - stt.configurePadding(p.prePadMs, p.postPadMs) - stt.setProviderOverride(null) - sttRef.current = stt const llm = new ProxyLlmClient( p.languageSnapshot, () => paramsRef.current.replyEnabled, @@ -417,91 +360,54 @@ export function useConversationSession(params: ConversationSessionParams) { ) llmRef.current = llm const storage = storageRef.current - const pipeline = new Pipeline({ stt, llm, conversation: storage }) + const pipeline = new Pipeline({ llm, conversation: storage }) pipelineRef.current = pipeline - realtimeModeRef.current = isRealtime - setActiveSttPath(isRealtime ? 'realtime' : 'batch') - if (!isRealtime) { - setStatusNote( - `${relayStatus}；当前为 batch STT：无实时草稿，停顿后整段上传。`, - ) - } - if (isRealtime && selectedProvider) { - setLoading('正在连接实时转写…') - try { - const rt = await connectRealtimeSttWithRetry({ - provider: selectedProvider, - language: p.languageSnapshot.conversationLang, - sessionId: conversationSessionId, - handlers: { - onPartial: (text) => { - const meta = draftMetaRef.current - if (!meta) return - setDraft({ speaker: meta.speaker, text, startedAt: meta.startedAt, endedAt: Date.now() }) - }, - onError: (message) => { - reportError(`实时转写：${message}`) - }, - onQuotaExhausted: () => { - setQuotaExhausted(true) - setStatusNote('本轮已完成；本月可用分钟数已用完，会话将在最终建议生成后停止。') - globalThis.dispatchEvent?.(new CustomEvent('kibotalk:quota-changed')) - void (async () => { - await pipeline.idle().catch(() => {}) - await stop() - })() - }, - }, - }) - realtimeRef.current = rt - setStatusNote(`${relayStatus}；实时转写已连接。`) - setActiveSttPath('realtime') - } catch (e) { - if (!degradeToBatch((e as Error).message)) { - throw e - } - realtimeModeRef.current = false - } - } + setActiveSttPath('realtime') + setLoading('正在连接实时转写…') + const rt = await connectRealtimeSttWithRetry({ + provider: selectedProvider, + language: p.languageSnapshot.conversationLang, + sessionId: conversationSessionId, + handlers: { + onPartial: (text) => { + const meta = draftMetaRef.current + if (!meta) return + setDraft({ speaker: meta.speaker, text, startedAt: meta.startedAt, endedAt: Date.now() }) + }, + onError: (message) => { + reportError(`实时转写：${message}`) + }, + onQuotaExhausted: () => { + setQuotaExhausted(true) + setStatusNote('本轮已完成；本月可用分钟数已用完，会话将在最终建议生成后停止。') + globalThis.dispatchEvent?.(new CustomEvent('kibotalk:quota-changed')) + void (async () => { + await pipeline.idle().catch(() => {}) + await stop() + })() + }, + }, + }) + realtimeRef.current = rt + setStatusNote(`${relayStatus}；实时转写已连接。`) - const aggregator = createSegmentAggregator({ + const realtimeAggregator = createSegmentAggregator({ sampleRate: audio.sampleRate, pauseMs: p.pauseMs, maxMs: p.mergeMaxMs, }) - aggregator.onFlush((merged) => { + realtimeAggregator.onFlush((merged) => { + const turn = finalizedTurnFromRealtimeSegments( + merged, + p.languageSnapshot.conversationLang, + ) + if (!turn) return pipelineBusyRef.current = pipelineBusyRef.current - .then(() => - pipeline.ingestSegment({ - pcm: merged.pcm, - speaker: merged.speaker, - startedAt: merged.startedAt, - endedAt: merged.endedAt, - }), - ) + .then(() => pipeline.ingestFinalizedTurn(turn)) .catch(() => {}) }) - aggregatorRef.current = aggregator - - if (realtimeModeRef.current) { - const realtimeAggregator = createSegmentAggregator({ - sampleRate: audio.sampleRate, - pauseMs: p.pauseMs, - maxMs: p.mergeMaxMs, - }) - realtimeAggregator.onFlush((merged) => { - const turn = finalizedTurnFromRealtimeSegments( - merged, - p.languageSnapshot.conversationLang, - ) - if (!turn) return - pipelineBusyRef.current = pipelineBusyRef.current - .then(() => pipeline.ingestFinalizedTurn(turn)) - .catch(() => {}) - }) - realtimeAggregatorRef.current = realtimeAggregator - } + realtimeAggregatorRef.current = realtimeAggregator if (audioSourceMode === 'both') { const systemStream = await p.getSystemAudioStream?.() @@ -523,74 +429,51 @@ export function useConversationSession(params: ConversationSessionParams) { sampleRate: systemAudio.sampleRate, }) systemVadRef.current = systemVad - if (isRealtime && selectedProvider) { - systemRealtimeRef.current = await connectRealtimeSttWithRetry({ - provider: selectedProvider, - language: p.languageSnapshot.conversationLang, - sessionId: conversationSessionId, - handlers: { - onPartial: (text) => { - setDraft((current) => - current?.speaker === 'user' - ? current - : { - speaker: 'other', - text, - startedAt: current?.startedAt ?? Date.now(), - endedAt: Date.now(), - }, - ) - }, - onError: (message) => reportError(`系统音频实时转写：${message}`), - onQuotaExhausted: () => { - setQuotaExhausted(true) - setStatusNote('本轮已完成；本月可用分钟数已用完，会话将在最终建议生成后停止。') - globalThis.dispatchEvent?.(new CustomEvent('kibotalk:quota-changed')) - void (async () => { - await pipeline.idle().catch(() => {}) - await stop() - })() - }, + systemRealtimeRef.current = await connectRealtimeSttWithRetry({ + provider: selectedProvider, + language: p.languageSnapshot.conversationLang, + sessionId: conversationSessionId, + handlers: { + onPartial: (text) => { + setDraft((current) => + current?.speaker === 'user' + ? current + : { + speaker: 'other', + text, + startedAt: current?.startedAt ?? Date.now(), + endedAt: Date.now(), + }, + ) }, - }) - systemRealtimeModeRef.current = true - } - const systemAggregator = createSegmentAggregator({ + onError: (message) => reportError(`系统音频实时转写：${message}`), + onQuotaExhausted: () => { + setQuotaExhausted(true) + setStatusNote('本轮已完成；本月可用分钟数已用完，会话将在最终建议生成后停止。') + globalThis.dispatchEvent?.(new CustomEvent('kibotalk:quota-changed')) + void (async () => { + await pipeline.idle().catch(() => {}) + await stop() + })() + }, + }, + }) + const systemRealtimeAggregator = createSegmentAggregator({ sampleRate: systemAudio.sampleRate, pauseMs: p.pauseMs, maxMs: p.mergeMaxMs, }) - systemAggregator.onFlush((merged) => { + systemRealtimeAggregator.onFlush((merged) => { + const turn = finalizedTurnFromRealtimeSegments( + merged, + p.languageSnapshot.conversationLang, + ) + if (!turn) return pipelineBusyRef.current = pipelineBusyRef.current - .then(() => - pipeline.ingestSegment({ - pcm: merged.pcm, - speaker: 'other', - startedAt: merged.startedAt, - endedAt: merged.endedAt, - }), - ) + .then(() => pipeline.ingestFinalizedTurn(turn)) .catch(() => {}) }) - systemAggregatorRef.current = systemAggregator - if (systemRealtimeModeRef.current) { - const realtimeAggregator = createSegmentAggregator({ - sampleRate: systemAudio.sampleRate, - pauseMs: p.pauseMs, - maxMs: p.mergeMaxMs, - }) - realtimeAggregator.onFlush((merged) => { - const turn = finalizedTurnFromRealtimeSegments( - merged, - p.languageSnapshot.conversationLang, - ) - if (!turn) return - pipelineBusyRef.current = pipelineBusyRef.current - .then(() => pipeline.ingestFinalizedTurn(turn)) - .catch(() => {}) - }) - systemRealtimeAggregatorRef.current = realtimeAggregator - } + systemRealtimeAggregatorRef.current = systemRealtimeAggregator systemVad.on('speech-start', () => { systemInSpeechRef.current = true systemRealtimeAggregatorRef.current?.hold() @@ -607,59 +490,46 @@ export function useConversationSession(params: ConversationSessionParams) { }) systemVad.on('speech-ready', (event) => { const endedAt = Date.now() - if (systemRealtimeModeRef.current) { - const realtime = systemRealtimeRef.current - if (!realtime || !systemUncommittedRef.current) return - systemUncommittedRef.current = false - realtime.commit() - const startedAt = endedAt - event.duration * 1000 - const completed = realtime.waitCompleted() - realtimeBusyRef.current = realtimeBusyRef.current - .then(async () => { - try { - const text = await completed - setDraft((current) => - current?.startedAt === startedAt ? null : current, - ) + const realtime = systemRealtimeRef.current + if (!realtime || !systemUncommittedRef.current) return + systemUncommittedRef.current = false + realtime.commit() + const startedAt = endedAt - event.duration * 1000 + const completed = realtime.waitCompleted() + realtimeBusyRef.current = realtimeBusyRef.current + .then(async () => { + try { + const text = await completed + setDraft((current) => + current?.startedAt === startedAt ? null : current, + ) + systemRealtimeAggregatorRef.current?.feed({ + buffer: event.buffer, + speaker: 'other', + text, + startedAt, + endedAt, + }) + } catch (cause) { + if (isTranscriptionFailed(cause)) { systemRealtimeAggregatorRef.current?.feed({ buffer: event.buffer, speaker: 'other', - text, + text: '', + sttFailed: true, startedAt, endedAt, }) - } catch (cause) { - if (isTranscriptionFailed(cause)) { - systemRealtimeAggregatorRef.current?.feed({ - buffer: event.buffer, - speaker: 'other', - text: '', - sttFailed: true, - startedAt, - endedAt, - }) - return - } - reportError(`系统音频实时转写：${String(cause)}`) + return } - }) - .catch(() => {}) - return - } - systemAggregator?.feed({ - buffer: event.buffer, - speaker: 'other', - startedAt: endedAt - event.duration * 1000, - endedAt, - }) + reportError(`系统音频实时转写：${String(cause)}`) + } + }) + .catch(() => {}) }) await systemAudio.start(async (chunk) => { await systemVad.processAudio(chunk) - if ( - systemRealtimeModeRef.current - && systemInSpeechRef.current - && systemRealtimeRef.current - ) { + if (systemInSpeechRef.current && systemRealtimeRef.current) { systemUncommittedRef.current = true systemRealtimeRef.current.append(chunk) } @@ -682,14 +552,19 @@ export function useConversationSession(params: ConversationSessionParams) { setTurns((prev) => prev.map((t) => (t.id === e.turnId ? { ...t, candidates: e.candidates } : t))) void storageRef.current.updateTurnSuggestions(e.turnId, e.candidates) break + case 'candidatesStreaming': + case 'candidateDelta': case 'llmAborted': case 'llmFailed': break case 'sttFailed': setTurns((prev) => prev.map((t) => (t.id === e.turnId ? { ...t, sttFailed: true } as SessionTurn : t))) break - default: + default: { + const _exhaustive: never = e + void _exhaustive break + } } }) @@ -700,12 +575,10 @@ export function useConversationSession(params: ConversationSessionParams) { return } const startedAt = Date.now() - if (realtimeModeRef.current) { - realtimeAggregatorRef.current?.hold() - const who = draftMetaRef.current?.speaker ?? speakerRef.current - draftMetaRef.current = { speaker: who, startedAt } - setDraft({ speaker: who, text: '', startedAt, endedAt: startedAt }) - } + realtimeAggregatorRef.current?.hold() + const who = draftMetaRef.current?.speaker ?? stableSpeakerRef.current + draftMetaRef.current = { speaker: who, startedAt } + setDraft({ speaker: who, text: '', startedAt, endedAt: startedAt }) inSpeechRef.current = true }) vad.on('speech-end', () => { @@ -719,111 +592,73 @@ export function useConversationSession(params: ConversationSessionParams) { const startedAt = draftMetaRef.current?.startedAt ?? now - e.duration * 1000 const endedAt = now - if (realtimeModeRef.current) { - const rt = realtimeRef.current - if (!rt || !uncommittedRef.current) return - - rt.commit() - uncommittedRef.current = false - const buffer = e.buffer - const provisional = draftMetaRef.current?.speaker ?? speakerRef.current - const transcription = rt.waitCompleted().then( - (text) => ({ ok: true as const, text }), - (error: unknown) => ({ ok: false as const, error }), - ) - const verifiedSpeaker = verifySpeaker(buffer.buffer as ArrayBuffer) - realtimeBusyRef.current = realtimeBusyRef.current - .then(async () => { - const [outcome, who] = await Promise.all([ - transcription, - verifiedSpeaker, - ]) - if (outcome.ok) { - draftMetaRef.current = { speaker: who, startedAt } - setDraft((current) => - current?.startedAt === startedAt ? null : current, - ) - realtimeAggregatorRef.current?.feed({ - buffer, - speaker: who, - text: outcome.text, - startedAt, - endedAt, - }) - return - } - if (isTranscriptionFailed(outcome.error)) { - setDraft((current) => - current?.startedAt === startedAt ? null : current, - ) - realtimeAggregatorRef.current?.feed({ - buffer, - speaker: provisional, - text: '', - sttFailed: true, - startedAt, - endedAt, - }) - return - } - if (degradeToBatch(String(outcome.error))) { - await pipeline.ingestSegment({ - pcm: buffer, - speaker: who, - startedAt, - endedAt, - }) - return - } + const rt = realtimeRef.current + if (!rt || !uncommittedRef.current) return + + rt.commit() + uncommittedRef.current = false + const buffer = e.buffer + const provisional = draftMetaRef.current?.speaker ?? stableSpeakerRef.current + const transcription = rt.waitCompleted().then( + (text) => ({ ok: true as const, text }), + (error: unknown) => ({ ok: false as const, error }), + ) + const verifiedSpeaker = verifySpeaker(buffer.buffer as ArrayBuffer) + realtimeBusyRef.current = realtimeBusyRef.current + .then(async () => { + const [outcome, who] = await Promise.all([ + transcription, + verifiedSpeaker, + ]) + if (outcome.ok) { + draftMetaRef.current = { speaker: who, startedAt } + setDraft((current) => + current?.startedAt === startedAt ? null : current, + ) realtimeAggregatorRef.current?.feed({ buffer, speaker: who, - text: '', - sttFailed: true, + text: outcome.text, startedAt, endedAt, }) - }) - .catch(() => {}) - return - } - - // Batch: verify || STT in parallel, then ingest finalized text. - const buffer = e.buffer - void (async () => { - try { - const verifyPromise = verifySpeaker(buffer.buffer as ArrayBuffer) - - if (paramsRef.current.transcribeMode === 'aggregated') { - const who = await verifyPromise - aggregatorRef.current?.feed({ buffer, speaker: who, startedAt, endedAt }) return } - - const stt = sttRef.current - if (!stt) { - const who = await verifyPromise - void pipeline.ingestSegment({ pcm: buffer, speaker: who, startedAt, endedAt }) + if (isTranscriptionFailed(outcome.error)) { + setDraft((current) => + current?.startedAt === startedAt ? null : current, + ) + realtimeAggregatorRef.current?.feed({ + buffer, + speaker: provisional, + text: '', + sttFailed: true, + startedAt, + endedAt, + }) return } - const [who, text] = await Promise.all([verifyPromise, stt.transcribe(buffer, new AbortController().signal)]) - await pipeline.ingestFinalizedTurn({ speaker: who, text, startedAt, endedAt }) - } catch (err) { - reportError(`转写失败：${String(err)}`) - const who = speakerRef.current - await pipeline.ingestFinalizedTurn({ speaker: who, text: '', startedAt, endedAt, sttFailed: true }) - } - })() + reportError(`实时转写：${String(outcome.error)}`) + realtimeAggregatorRef.current?.feed({ + buffer, + speaker: who, + text: '', + sttFailed: true, + startedAt, + endedAt, + }) + }) + .catch(() => {}) }) await audio.start(async (chunk) => { await vad.processAudio(chunk) if (!paramsRef.current.sttEnabled) return - if (!realtimeModeRef.current || !inSpeechRef.current) return - const rt = realtimeRef.current - if (!rt) return + if (!inSpeechRef.current) return + const rtConn = realtimeRef.current + if (!rtConn) return uncommittedRef.current = true - rt.append(chunk) + rtConn.append(chunk) }) setRunning(true) setLifecycle('running') @@ -850,26 +685,24 @@ export function useConversationSession(params: ConversationSessionParams) { } function releaseCapture() { - systemAggregatorRef.current?.flush() - systemAggregatorRef.current?.dispose() - systemAggregatorRef.current = null + systemRealtimeAggregatorRef.current?.flush() + systemRealtimeAggregatorRef.current?.dispose() + systemRealtimeAggregatorRef.current = null systemAudioRef.current?.stop() systemAudioRef.current = null systemVadRef.current = null systemRealtimeRef.current?.finish() systemRealtimeRef.current?.close() systemRealtimeRef.current = null - systemRealtimeModeRef.current = false systemInSpeechRef.current = false systemUncommittedRef.current = false void paramsRef.current.stopSystemAudioStream?.() - aggregatorRef.current?.flush() - aggregatorRef.current?.dispose() - aggregatorRef.current = null + realtimeAggregatorRef.current?.flush() + realtimeAggregatorRef.current?.dispose() + realtimeAggregatorRef.current = null realtimeRef.current?.finish() realtimeRef.current?.close() realtimeRef.current = null - realtimeModeRef.current = false uncommittedRef.current = false inSpeechRef.current = false audioRef.current?.stop() @@ -878,7 +711,6 @@ export function useConversationSession(params: ConversationSessionParams) { pipelineRef.current = null llmRef.current = null vadRef.current = null - sttRef.current = null draftMetaRef.current = null setDraft(null) setRunning(false) @@ -980,8 +812,6 @@ export function useConversationSession(params: ConversationSessionParams) { } return { - speaker, - setSpeaker, running, loading, error, diff --git a/packages/app-shared/src/session/use-product-session.ts b/packages/app-shared/src/session/use-product-session.ts index 0afc445..94baa6f 100644 --- a/packages/app-shared/src/session/use-product-session.ts +++ b/packages/app-shared/src/session/use-product-session.ts @@ -69,12 +69,9 @@ export function useProductSession({ minSilenceDurationMs: defaultAppConfig.vad.minSilenceDurationMs, minSpeechDurationMs: defaultAppConfig.vad.minSpeechDurationMs, vadVariantId: defaultAppConfig.vadVariantId, - prePadMs: defaultAppConfig.asrPadMs.pre, - postPadMs: defaultAppConfig.asrPadMs.post, pauseMs: defaultAppConfig.aggregator.pauseMs, mergeMaxMs: defaultAppConfig.aggregator.maxMs, speakerThreshold: defaultAppConfig.speakerThreshold, - transcribeMode: 'aggregated', candidateRoundsMax, sttEnabled: true, replyEnabled, diff --git a/packages/app-shared/src/stt-providers.ts b/packages/app-shared/src/stt-providers.ts index 8056945..87de40a 100644 --- a/packages/app-shared/src/stt-providers.ts +++ b/packages/app-shared/src/stt-providers.ts @@ -5,33 +5,14 @@ export type SttProvider = { label: string model: string active: boolean - mode?: 'batch' | 'realtime' -} - -export function sttUrl( - provider: string | null, - language?: string | null, -): string { - const params = new URLSearchParams() - if (provider) params.set('provider', provider) - if (language) params.set('language', language) - const qs = params.toString() - return qs ? `/api/stt?${qs}` : '/api/stt' + mode: 'realtime' } export function defaultSttProvider(providers: SttProvider[]): string | null { return providers.find((p) => p.active)?.id ?? providers[0]?.id ?? null } -export function providerMode( - providers: SttProvider[], - id: string | null, -): 'batch' | 'realtime' { - if (!id) return 'batch' - return providers.find((p) => p.id === id)?.mode ?? 'batch' -} - -/** Fetch the STT providers the `/stt` proxy has configured server-side. */ +/** Fetch realtime STT providers configured server-side. */ export async function fetchSttProviders(): Promise { const res = await authorizedFetch('/api/stt/providers') if (!res.ok) return [] @@ -39,11 +20,11 @@ export async function fetchSttProviders(): Promise { return data.providers ?? [] } -/** Prefer a `realtime` provider (product default transport); fall back to any active one. */ +/** Prefer an active realtime provider; fall back to any configured one. */ export function defaultRealtimeFirstProvider(providers: SttProvider[]): string | null { return ( - providers.find((p) => p.mode === 'realtime' && p.active)?.id - ?? providers.find((p) => p.mode === 'realtime')?.id - ?? defaultSttProvider(providers) + providers.find((p) => p.active)?.id + ?? providers[0]?.id + ?? null ) } diff --git a/packages/app-shared/test/api-runtime-relay.test.ts b/packages/app-shared/test/api-runtime-relay.test.ts index d1fe266..0ba3b05 100644 --- a/packages/app-shared/test/api-runtime-relay.test.ts +++ b/packages/app-shared/test/api-runtime-relay.test.ts @@ -61,9 +61,8 @@ describe('relay API runtime', () => { deviceSessionId: 'device-1', conversationSessionId: body.conversationSessionId, nodeId: node.id, - scopes: ['llm', 'stt', 'stt-realtime'], + scopes: ['llm', 'stt-realtime'], sttProvider: 'dashscope-realtime', - sttBatchProvider: 'dashscope', llmProvider: 'openai', llmModel: 'deepseek-v4-flash', quotaSeconds: 1_800, diff --git a/packages/app-shared/test/relay-routing.test.ts b/packages/app-shared/test/relay-routing.test.ts index 7fb4d1a..b077a44 100644 --- a/packages/app-shared/test/relay-routing.test.ts +++ b/packages/app-shared/test/relay-routing.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import type { RelayNode, RelayNodeList } from '@kibotalk/shared' -import { probeRelayNodes } from '../src/relay-routing' +import { probeRelayNodes, relayNodeLabelKind } from '../src/relay-routing' const primary: RelayNode = { id: 'jp-primary', @@ -14,6 +14,20 @@ const relay: RelayNode = { role: 'relay', acceptingNewSessions: true, } +const localPrimary: RelayNode = { + id: 'jp-primary', + origin: 'http://localhost:8787', + role: 'primary', + acceptingNewSessions: true, +} + +describe('relayNodeLabelKind', () => { + it('labels localhost primary as local, not japan', () => { + expect(relayNodeLabelKind(localPrimary)).toBe('local') + expect(relayNodeLabelKind(primary)).toBe('primary') + expect(relayNodeLabelKind(relay)).toBe('relay') + }) +}) describe('relay node latency probes', () => { it('returns measured latency for the user-facing manual picker', async () => { diff --git a/packages/audio/src/vad.ts b/packages/audio/src/vad.ts index 4445724..b6c2437 100644 --- a/packages/audio/src/vad.ts +++ b/packages/audio/src/vad.ts @@ -42,10 +42,11 @@ export type VadConfig = { export const defaultVadConfig: VadConfig = { sampleRate: 16000, - speechThreshold: 0.5, - exitThreshold: 0.3, - /** Tight cut so speaker turns with short gaps don't merge into one blob. */ - minSilenceDurationMs: 200, + // Drama VAD grid champion (Silero v6.2, FC/-23 LUFS clip): enter 0.20 / exit 0.10 / + // silence 400ms / minSpeech 200ms — see docs/brainstorm/2026-07-23-drama-subtitle-benchmark.md + speechThreshold: 0.2, + exitThreshold: 0.1, + minSilenceDurationMs: 400, speechPadMs: 80, minSpeechDurationMs: 200, maxSpeechDurationMs: 30000, diff --git a/packages/audio/test/vad.test.ts b/packages/audio/test/vad.test.ts index 7837b3a..0d9ed98 100644 --- a/packages/audio/test/vad.test.ts +++ b/packages/audio/test/vad.test.ts @@ -113,7 +113,9 @@ describe('createVAD', () => { it('defaultVadConfig has expected shape', () => { expect(defaultVadConfig.sampleRate).toBe(16000) - expect(defaultVadConfig.minSilenceDurationMs).toBe(200) + expect(defaultVadConfig.speechThreshold).toBe(0.2) + expect(defaultVadConfig.exitThreshold).toBe(0.1) + expect(defaultVadConfig.minSilenceDurationMs).toBe(400) expect(defaultVadConfig.minSpeechDurationMs).toBe(200) expect(defaultVadConfig.maxSpeechDurationMs).toBe(30000) }) diff --git a/packages/observability/src/io-trace.ts b/packages/observability/src/io-trace.ts index 205b2bf..231ab7b 100644 --- a/packages/observability/src/io-trace.ts +++ b/packages/observability/src/io-trace.ts @@ -30,7 +30,6 @@ export const IOAttributes = { ASRText: `${customPrefix}.asr.text`, ASRAbort: `${customPrefix}.asr.abort`, SttPath: `${customPrefix}.stt.path`, - SttDegraded: `${customPrefix}.stt.degraded`, SpeakerConfidence: `${customPrefix}.speaker.confidence`, SpeakerThreshold: `${customPrefix}.speaker.threshold`, SpeakerResult: `${customPrefix}.speaker.result`, diff --git a/packages/pages/src/AccountPage.tsx b/packages/pages/src/AccountPage.tsx index 3e9a3ec..9a95429 100644 --- a/packages/pages/src/AccountPage.tsx +++ b/packages/pages/src/AccountPage.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from 'react' +import { useEffect, useState, type ReactNode } from 'react' import type { AccountDevice, AccountSession } from '@kibotalk/app-shared' import { deleteCloudAccount, @@ -23,6 +23,7 @@ import { DialogFooter, DialogHeader, DialogTitle, + DesktopProductWindowFrame, Input, Label, Separator, @@ -93,7 +94,13 @@ function LoginCard({ return (
- + 登录 KiboTalk @@ -166,6 +173,7 @@ function LoginCard({ function AccountContent({ account, + embedded, showAdminLink, onAccountChange, onBack, @@ -222,10 +230,14 @@ function AccountContent({ } } - return ( -
-
-