diff --git a/README.md b/README.md index 0667fd0..5a57e6c 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ Useful shortcuts: The free on-device engine is **Supertonic 3** through ONNX Runtime in a dedicated browser worker. The model is downloaded from Hugging Face only after you accept its OpenRAIL-M terms; once installed, speech generation runs locally with WebGPU acceleration when available. Generation quality is adjustable from 2 to 16 steps, with 10 steps as the default. Voicebook generates the current passage first, buffers upcoming passages, and stores generated audio locally so it does not need to repeat work. -Prefer premium voices? Paste an **ElevenLabs** API key and pick any of your voices — no download, native word-level timing for read-along highlighting, and the economical Flash v2.5 model by default. +Prefer premium voices? Paste an **ElevenLabs** API key and pick any of your voices — no download and native word-level timing for read-along highlighting. Both generations are available: **Eleven v3** and **Eleven v3 Conversational** for the most expressive read, or the **v2** models (Flash v2.5, Turbo v2.5, Multilingual v2) for a cheaper, steadier one. Flash v2.5 is the default, and each model keeps its own settings — stability, similarity, style, speed, and speaker boost, wherever that model honours them. ## Spoken descriptions diff --git a/src/lib/components/ElevenLabsOptions.svelte b/src/lib/components/ElevenLabsOptions.svelte new file mode 100644 index 0000000..98df88d --- /dev/null +++ b/src/lib/components/ElevenLabsOptions.svelte @@ -0,0 +1,291 @@ + + +
+
+
+ {model.label} settings + + {customized + ? 'Changing these re-generates audio the next time you play.' + : "Untouched — each voice's own saved settings apply."} + +
+ {#if customized} + + {/if} +
+ + {#if model.options.stability === 'discrete'} +
+ Stability +
+ {#each ELEVENLABS_V3_STABILITY as point (point.value)} + + {/each} +
+
+ {:else} + + {/if} + + {#if model.options.similarity} + + {/if} + + {#if model.options.style} + + {/if} + + {#if model.options.speed} + + {/if} + + {#if model.options.speakerBoost} +
+
+ Speaker boost + Sharpens resemblance to the voice, at a little latency. +
+ +
+ {/if} +
+ + diff --git a/src/lib/domain/provider-catalog.spec.ts b/src/lib/domain/provider-catalog.spec.ts index 85c63de..af64f13 100644 --- a/src/lib/domain/provider-catalog.spec.ts +++ b/src/lib/domain/provider-catalog.spec.ts @@ -1,10 +1,18 @@ import { describe, expect, it } from 'vitest'; import { CLOUD_LLM_PROVIDERS, + DEFAULT_ELEVENLABS_MODEL, + DEFAULT_ELEVENLABS_OPTIONS, + ELEVENLABS_FAMILIES, ELEVENLABS_MODELS, defaultCloudLlmModel, + elevenLabsRevision, + elevenLabsVoiceSettings, getCloudLlmProvider, - isCloudLlmProvider + getElevenLabsModel, + isCloudLlmProvider, + normalizeElevenLabsOptions, + type ElevenLabsModelSpec } from './provider-catalog'; describe('cloud LLM provider catalog', () => { @@ -29,9 +37,109 @@ describe('cloud LLM provider catalog', () => { }); describe('elevenlabs model catalog', () => { - it('defaults to the cheapest timestamp-capable model', () => { - expect(ELEVENLABS_MODELS[0].id).toBe('eleven_flash_v2_5'); - // v3 has no with-timestamps support — word highlighting depends on it. - expect(ELEVENLABS_MODELS.some((model) => model.id === 'eleven_v3')).toBe(false); + it('offers both generations and defaults to the cheapest v2 model', () => { + expect(getElevenLabsModel(DEFAULT_ELEVENLABS_MODEL)).not.toBeNull(); + expect(DEFAULT_ELEVENLABS_MODEL).toBe('eleven_flash_v2_5'); + // v3 now returns character alignment from the with-timestamps + // endpoint, so word highlighting survives the switch. + expect(ELEVENLABS_MODELS.map((model) => model.id)).toEqual([ + 'eleven_v3', + 'eleven_v3_conversational', + 'eleven_flash_v2_5', + 'eleven_turbo_v2_5', + 'eleven_multilingual_v2' + ]); + }); + + it('groups every model into exactly one family', () => { + const grouped = ELEVENLABS_FAMILIES.flatMap((family) => family.models); + expect(grouped).toHaveLength(ELEVENLABS_MODELS.length); + expect(new Set(grouped.map((model) => model.id)).size).toBe(ELEVENLABS_MODELS.length); + expect(ELEVENLABS_FAMILIES[0].id).toBe('v3'); + }); +}); + +describe('elevenlabs voice options', () => { + const v3 = getElevenLabsModel('eleven_v3') as ElevenLabsModelSpec; + const conversational = getElevenLabsModel('eleven_v3_conversational') as ElevenLabsModelSpec; + const flash = getElevenLabsModel('eleven_flash_v2_5') as ElevenLabsModelSpec; + const multilingual = getElevenLabsModel('eleven_multilingual_v2') as ElevenLabsModelSpec; + + it('snaps v3 stability to the three named points', () => { + expect(normalizeElevenLabsOptions(v3, { stability: 0.2 }).stability).toBe(0); + expect(normalizeElevenLabsOptions(v3, { stability: 0.4 }).stability).toBe(0.5); + expect(normalizeElevenLabsOptions(v3, { stability: 0.9 }).stability).toBe(1); + // v2 keeps the continuous value. + expect(normalizeElevenLabsOptions(flash, { stability: 0.2 }).stability).toBe(0.2); + }); + + it('clamps out-of-range and non-finite values', () => { + expect(normalizeElevenLabsOptions(flash, { similarity: 4 }).similarity).toBe(1); + expect(normalizeElevenLabsOptions(flash, { speed: 0.1 }).speed).toBe(0.7); + expect(normalizeElevenLabsOptions(flash, { speed: 9 }).speed).toBe(1.2); + expect(normalizeElevenLabsOptions(flash, { stability: Number.NaN }).stability).toBe( + DEFAULT_ELEVENLABS_OPTIONS.stability + ); + }); + + it('forces knobs a model ignores back to their defaults', () => { + // v3 accepts `speed` on the wire but produces identical audio, and + // reports can_use_style / can_use_speaker_boost false. + const tuned = normalizeElevenLabsOptions(v3, { + speed: 1.2, + style: 0.8, + similarity: 0.1, + speakerBoost: false + }); + expect(tuned.speed).toBe(1); + expect(tuned.style).toBe(0); + expect(tuned.similarity).toBe(DEFAULT_ELEVENLABS_OPTIONS.similarity); + expect(tuned.speakerBoost).toBe(true); + }); + + it('sends nothing while the options are untouched', () => { + for (const model of ELEVENLABS_MODELS) { + const options = normalizeElevenLabsOptions(model, null); + expect(elevenLabsVoiceSettings(model, options)).toBeUndefined(); + // The bare id keeps audio cached before these controls existed. + expect(elevenLabsRevision(model, options)).toBe(model.id); + } + }); + + it('sends only the fields the model honours', () => { + const v3Body = elevenLabsVoiceSettings(v3, normalizeElevenLabsOptions(v3, { stability: 0 })); + expect(v3Body).toEqual({ stability: 0 }); + + const boostBody = elevenLabsVoiceSettings( + conversational, + normalizeElevenLabsOptions(conversational, { speakerBoost: false }) + ); + expect(boostBody).toEqual({ stability: 0.5, use_speaker_boost: false }); + + const flashBody = elevenLabsVoiceSettings( + flash, + normalizeElevenLabsOptions(flash, { speed: 1.1 }) + ); + expect(flashBody).toEqual({ stability: 0.5, similarity_boost: 0.75, speed: 1.1 }); + + const fullBody = elevenLabsVoiceSettings( + multilingual, + normalizeElevenLabsOptions(multilingual, { style: 0.4 }) + ); + expect(fullBody).toEqual({ + stability: 0.5, + similarity_boost: 0.75, + style: 0.4, + speed: 1, + use_speaker_boost: true + }); + }); + + it('gives tuned options a stable, order-independent revision', () => { + const a = elevenLabsRevision(flash, normalizeElevenLabsOptions(flash, { speed: 1.1 })); + const b = elevenLabsRevision(flash, normalizeElevenLabsOptions(flash, { speed: 1.1 })); + expect(a).toBe(b); + expect(a).toBe('eleven_flash_v2_5#similarity_boost=0.75,speed=1.1,stability=0.5'); + expect(a).not.toBe(elevenLabsRevision(flash, normalizeElevenLabsOptions(flash, { speed: 1 }))); }); }); diff --git a/src/lib/domain/provider-catalog.ts b/src/lib/domain/provider-catalog.ts index f3ed8ef..1792d5e 100644 --- a/src/lib/domain/provider-catalog.ts +++ b/src/lib/domain/provider-catalog.ts @@ -96,29 +96,249 @@ export function isCloudLlmProvider(value: string): value is CloudLlmProvider { /* ── ElevenLabs speech ───────────────────────────────────────────────────── */ +/** v3 is a different generation with its own controls — grouped in the UI and + * gated separately when building the request body. */ +export type ElevenLabsFamily = 'v3' | 'v2'; + +/** Which voice_settings a model actually honours. Probed against + * GET /v1/models (can_use_style, can_use_speaker_boost) and by measuring + * generated audio — v3 accepts `speed` but ignores it, so it is off here. */ +export interface ElevenLabsModelOptionSupport { + /** v3 exposes three named stability points; v2 takes any 0–1 value. */ + stability: 'discrete' | 'continuous'; + similarity: boolean; + style: boolean; + speed: boolean; + speakerBoost: boolean; +} + export interface ElevenLabsModelSpec { id: string; label: string; tagline: string; + family: ElevenLabsFamily; + /** Per-request cap from the model's maximum_text_length_per_request. + * Passages are capped far below this; the guard is for safety. */ + maxCharacters: number; + options: ElevenLabsModelOptionSupport; } -/** TTS models that support the with-timestamps endpoint (word highlighting - * needs character alignment, so v3 alpha is deliberately absent). First entry - * is the default. */ +/** Every TTS model that returns character alignment from the with-timestamps + * endpoint — word highlighting depends on it, which v3 now supports too. + * DEFAULT_ELEVENLABS_MODEL, not the array order, picks the default. */ export const ELEVENLABS_MODELS: ElevenLabsModelSpec[] = [ + { + id: 'eleven_v3', + label: 'Eleven v3', + tagline: 'most expressive · 70+ languages', + family: 'v3', + maxCharacters: 5_000, + options: { + stability: 'discrete', + similarity: false, + style: false, + speed: false, + speakerBoost: false + } + }, + { + id: 'eleven_v3_conversational', + label: 'Eleven v3 Conversational', + tagline: 'expressive · half the credits', + family: 'v3', + maxCharacters: 5_000, + options: { + stability: 'discrete', + similarity: false, + style: false, + speed: false, + speakerBoost: true + } + }, { id: 'eleven_flash_v2_5', label: 'Flash v2.5', - tagline: 'half the credits · recommended' + tagline: 'half the credits · recommended', + family: 'v2', + maxCharacters: 40_000, + options: { + stability: 'continuous', + similarity: true, + style: false, + speed: true, + speakerBoost: false + } + }, + { + id: 'eleven_turbo_v2_5', + label: 'Turbo v2.5', + tagline: 'fast · half the credits', + family: 'v2', + maxCharacters: 40_000, + options: { + stability: 'continuous', + similarity: true, + style: false, + speed: true, + speakerBoost: false + } }, - { id: 'eleven_turbo_v2_5', label: 'Turbo v2.5', tagline: 'fast · half the credits' }, { id: 'eleven_multilingual_v2', label: 'Multilingual v2', - tagline: 'highest quality · double credits' + tagline: 'steadiest long-form read', + family: 'v2', + maxCharacters: 10_000, + options: { + stability: 'continuous', + similarity: true, + style: true, + speed: true, + speakerBoost: true + } + } +]; + +/** Flash v2.5 stays the default: cheapest per character and the steadiest on + * the short passages the segmenter produces. v3 is one click away. */ +export const DEFAULT_ELEVENLABS_MODEL = 'eleven_flash_v2_5'; + +export function getElevenLabsModel(id: string): ElevenLabsModelSpec | null { + return ELEVENLABS_MODELS.find((model) => model.id === id) ?? null; +} + +/** The catalog in display order: v3 first, then the v2 generation. */ +export const ELEVENLABS_FAMILIES: Array<{ + id: ElevenLabsFamily; + label: string; + note: string; + models: ElevenLabsModelSpec[]; +}> = [ + { + id: 'v3', + label: 'Eleven v3', + note: 'Newest generation — richer delivery, more variation between takes.', + models: ELEVENLABS_MODELS.filter((model) => model.family === 'v3') + }, + { + id: 'v2', + label: 'Eleven v2', + note: 'Predictable and cheap — the steady choice for a long read.', + models: ELEVENLABS_MODELS.filter((model) => model.family === 'v2') } ]; +/** The three stability points the v3 models expose, in ElevenLabs' order. */ +export const ELEVENLABS_V3_STABILITY: Array<{ + value: number; + label: string; + tagline: string; +}> = [ + { value: 0, label: 'Creative', tagline: 'most emotion · can drift' }, + { value: 0.5, label: 'Natural', tagline: 'closest to the voice' }, + { value: 1, label: 'Robust', tagline: 'steadiest · least directable' } +]; + +/** Per-model synthesis knobs. Values match the ElevenLabs defaults, so an + * untouched set is indistinguishable from sending nothing. */ +export interface ElevenLabsVoiceOptions { + stability: number; + similarity: number; + style: number; + speed: number; + speakerBoost: boolean; +} + +export const DEFAULT_ELEVENLABS_OPTIONS: ElevenLabsVoiceOptions = { + stability: 0.5, + similarity: 0.75, + style: 0, + speed: 1, + speakerBoost: true +}; + +/** Snap to the nearest of the three v3 stability points. */ +function snapStability(value: number): number { + return ELEVENLABS_V3_STABILITY.reduce((best, point) => + Math.abs(point.value - value) < Math.abs(best.value - value) ? point : best + ).value; +} + +function clamp(value: number, low: number, high: number, fallback: number): number { + if (!Number.isFinite(value)) return fallback; + return Math.min(high, Math.max(low, value)); +} + +/** + * Fill in and clamp a stored option set for one model. Knobs the model does + * not honour are forced back to their default so they can never leak into a + * request (or into the cache signature) for a model that ignores them. + */ +export function normalizeElevenLabsOptions( + model: ElevenLabsModelSpec, + stored?: Partial | null +): ElevenLabsVoiceOptions { + const raw = { ...DEFAULT_ELEVENLABS_OPTIONS, ...(stored ?? {}) }; + const stability = clamp(raw.stability, 0, 1, DEFAULT_ELEVENLABS_OPTIONS.stability); + const support = model.options; + return { + stability: support.stability === 'discrete' ? snapStability(stability) : stability, + similarity: support.similarity + ? clamp(raw.similarity, 0, 1, DEFAULT_ELEVENLABS_OPTIONS.similarity) + : DEFAULT_ELEVENLABS_OPTIONS.similarity, + style: support.style ? clamp(raw.style, 0, 1, DEFAULT_ELEVENLABS_OPTIONS.style) : 0, + speed: support.speed ? clamp(raw.speed, 0.7, 1.2, DEFAULT_ELEVENLABS_OPTIONS.speed) : 1, + speakerBoost: support.speakerBoost + ? Boolean(raw.speakerBoost) + : DEFAULT_ELEVENLABS_OPTIONS.speakerBoost + }; +} + +export function isDefaultElevenLabsOptions(options: ElevenLabsVoiceOptions): boolean { + return ( + options.stability === DEFAULT_ELEVENLABS_OPTIONS.stability && + options.similarity === DEFAULT_ELEVENLABS_OPTIONS.similarity && + options.style === DEFAULT_ELEVENLABS_OPTIONS.style && + options.speed === DEFAULT_ELEVENLABS_OPTIONS.speed && + options.speakerBoost === DEFAULT_ELEVENLABS_OPTIONS.speakerBoost + ); +} + +/** The voice_settings body, or undefined to let the voice's own saved + * settings apply — which is what an untouched set has always meant, so + * audio cached before these controls existed still matches. */ +export function elevenLabsVoiceSettings( + model: ElevenLabsModelSpec, + options: ElevenLabsVoiceOptions +): Record | undefined { + const normalized = normalizeElevenLabsOptions(model, options); + if (isDefaultElevenLabsOptions(normalized)) return undefined; + const settings: Record = { stability: normalized.stability }; + if (model.options.similarity) settings.similarity_boost = normalized.similarity; + if (model.options.style) settings.style = normalized.style; + if (model.options.speed) settings.speed = normalized.speed; + if (model.options.speakerBoost) settings.use_speaker_boost = normalized.speakerBoost; + return settings; +} + +/** + * Cache-key fragment for a model plus its options: the bare model id while the + * options are untouched (so audio generated before the controls shipped keeps + * matching), and a stable suffix once any knob moves. + */ +export function elevenLabsRevision( + model: ElevenLabsModelSpec, + options: ElevenLabsVoiceOptions +): string { + const settings = elevenLabsVoiceSettings(model, options); + if (!settings) return model.id; + const suffix = Object.keys(settings) + .sort() + .map((key) => `${key}=${settings[key]}`) + .join(','); + return `${model.id}#${suffix}`; +} + /** George — a warm narrator that suits long-form reading. */ export const DEFAULT_ELEVENLABS_VOICE = 'JBFqnCBsd6RMkjVDRZzb'; diff --git a/src/lib/services/elevenlabs.spec.ts b/src/lib/services/elevenlabs.spec.ts index d093638..fc6ee20 100644 --- a/src/lib/services/elevenlabs.spec.ts +++ b/src/lib/services/elevenlabs.spec.ts @@ -1,5 +1,10 @@ -import { describe, expect, it } from 'vitest'; -import { pcm16ToFloat32, wordTimingsFromAlignment } from './elevenlabs'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + ElevenLabsError, + pcm16ToFloat32, + synthesizeElevenLabs, + wordTimingsFromAlignment +} from './elevenlabs'; describe('word timings from character alignment', () => { it('groups characters into whitespace-delimited words with span timing', () => { @@ -73,3 +78,77 @@ describe('pcm decoding', () => { expect(pcm16ToFloat32(bytes).length).toBe(1); }); }); + +describe('synthesis request', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + function stubFetch(): ReturnType { + const samples = new Int16Array([0, 128, -128, 0]); + const bytes = new Uint8Array(samples.buffer); + const fetchMock = vi.fn(async () => ({ + ok: true, + status: 200, + json: async () => ({ + audio_base64: btoa(String.fromCharCode(...bytes)), + alignment: { + characters: ['H', 'i'], + character_start_times_seconds: [0, 0.1], + character_end_times_seconds: [0.1, 0.2] + } + }) + })); + vi.stubGlobal('fetch', fetchMock); + return fetchMock; + } + + function bodyOf(fetchMock: ReturnType): Record { + const init = fetchMock.mock.calls[0][1] as RequestInit; + return JSON.parse(init.body as string) as Record; + } + + it('sends the bare model id and omits voice_settings when none are given', async () => { + const fetchMock = stubFetch(); + const result = await synthesizeElevenLabs({ + apiKey: 'k', + voiceId: 'v', + modelId: 'eleven_v3', + text: 'Hi' + }); + expect(fetchMock.mock.calls[0][0]).toContain('/v1/text-to-speech/v/with-timestamps'); + expect(bodyOf(fetchMock)).toEqual({ text: 'Hi', model_id: 'eleven_v3' }); + expect(result.timing.confidence).toBe('native'); + expect(result.timing.words.map((word) => word.word)).toEqual(['Hi']); + }); + + it('forwards voice settings verbatim when the user has tuned them', async () => { + const fetchMock = stubFetch(); + await synthesizeElevenLabs({ + apiKey: 'k', + voiceId: 'v', + modelId: 'eleven_v3', + text: 'Hi', + voiceSettings: { stability: 1 } + }); + expect(bodyOf(fetchMock)).toEqual({ + text: 'Hi', + model_id: 'eleven_v3', + voice_settings: { stability: 1 } + }); + }); + + it('refuses a passage longer than the model accepts before spending credits', async () => { + const fetchMock = stubFetch(); + await expect( + synthesizeElevenLabs({ + apiKey: 'k', + voiceId: 'v', + // v3 caps at 5,000 characters where Flash takes 40,000. + modelId: 'eleven_v3', + text: 'x'.repeat(5_001) + }) + ).rejects.toThrow(ElevenLabsError); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/services/elevenlabs.ts b/src/lib/services/elevenlabs.ts index d9ffb46..f13e7dd 100644 --- a/src/lib/services/elevenlabs.ts +++ b/src/lib/services/elevenlabs.ts @@ -7,7 +7,7 @@ import { wordsFor } from '$lib/domain/speech-words'; import type { TimingMap, WordTiming } from '$lib/domain/types'; import type { SynthesisResult } from './tts-client'; -import type { ElevenLabsVoice } from '$lib/domain/provider-catalog'; +import { getElevenLabsModel, type ElevenLabsVoice } from '$lib/domain/provider-catalog'; const API = 'https://api.elevenlabs.io'; const SAMPLE_RATE = 24_000; @@ -118,17 +118,30 @@ export interface ElevenLabsSynthesisRequest { voiceId: string; modelId: string; text: string; + /** Built by elevenLabsVoiceSettings — omitted while the user has not + * moved any knob, which leaves the voice's own saved settings in force. */ + voiceSettings?: Record; signal?: AbortSignal; } /** * Timestamped synthesis at PCM 24 kHz — drops straight into the player's - * AudioBuffer/word-highlight pipeline with native timing confidence. + * AudioBuffer/word-highlight pipeline with native timing confidence. Every + * catalog model returns character alignment here, v3 included. */ export async function synthesizeElevenLabs( options: ElevenLabsSynthesisRequest ): Promise { const started = performance.now(); + // Passages are capped far below every model's limit; this catches a + // mismatch (say a 5,000-character v3 request) with a readable message + // instead of a bare 422 from the API. + const spec = getElevenLabsModel(options.modelId); + if (spec && options.text.length > spec.maxCharacters) { + throw new ElevenLabsError( + `This passage is ${options.text.length.toLocaleString()} characters — ${spec.label} accepts ${spec.maxCharacters.toLocaleString()} per request.` + ); + } const timeout = AbortSignal.timeout(90_000); const signal = options.signal ? AbortSignal.any([options.signal, timeout]) : timeout; const data = (await request( @@ -136,7 +149,11 @@ export async function synthesizeElevenLabs( options.apiKey, { method: 'POST', - body: JSON.stringify({ text: options.text, model_id: options.modelId }), + body: JSON.stringify({ + text: options.text, + model_id: options.modelId, + ...(options.voiceSettings ? { voice_settings: options.voiceSettings } : {}) + }), signal } )) as { diff --git a/src/lib/state/player.svelte.ts b/src/lib/state/player.svelte.ts index 1e341bd..f2b6607 100644 --- a/src/lib/state/player.svelte.ts +++ b/src/lib/state/player.svelte.ts @@ -28,7 +28,6 @@ import { import { encodeDocumentMp3, mp3Filename } from '$lib/services/mp3-export'; import { ttsClient, type SynthesisResult } from '$lib/services/tts-client'; import { synthesizeElevenLabs } from '$lib/services/elevenlabs'; -import { ELEVENLABS_MODELS } from '$lib/domain/provider-catalog'; import { generationPlan } from '$lib/services/generation-plan'; import { absoluteTimelinePosition, @@ -284,7 +283,9 @@ export class VoicebookPlayer { return { modelId: 'elevenlabs', repository: 'elevenlabs', - revision: providersState.elevenLabsModelId, + // Carries the tuning knobs so a changed setting re-synthesizes + // instead of replaying audio made with the old ones. + revision: providersState.elevenLabsRevision, voiceId: providersState.elevenLabsVoiceId, backend: 'cloud', dtype: 'pcm24' @@ -302,12 +303,7 @@ export class VoicebookPlayer { } get runtimeLabel(): string { - if (this.usesElevenLabs) { - const model = ELEVENLABS_MODELS.find( - (candidate) => candidate.id === providersState.elevenLabsModelId - ); - return `ElevenLabs · ${model?.label ?? providersState.elevenLabsModelId}`; - } + if (this.usesElevenLabs) return `ElevenLabs · ${providersState.elevenLabsModel.label}`; if (!this.engineBackend) return 'Engine warming'; return `${this.engineBackend === 'webgpu' ? 'WebGPU' : 'WASM'} · ${this.engineDtype}`; } @@ -733,7 +729,10 @@ export class VoicebookPlayer { generated = await synthesizeElevenLabs({ apiKey: providersState.keyFor('elevenlabs') ?? '', voiceId: variant.voiceId, - modelId: variant.revision, + // variant.revision folds the tuning knobs in for cache keys; + // the wire needs the bare model id and its settings. + modelId: providersState.elevenLabsModel.id, + voiceSettings: providersState.elevenLabsVoiceSettings, text: segment.normalizedText, signal: controller.signal }); @@ -1269,7 +1268,10 @@ export class VoicebookPlayer { generated = await synthesizeElevenLabs({ apiKey: providersState.keyFor('elevenlabs') ?? '', voiceId: variant.voiceId, - modelId: variant.revision, + // variant.revision folds the tuning knobs in for cache keys; + // the wire needs the bare model id and its settings. + modelId: providersState.elevenLabsModel.id, + voiceSettings: providersState.elevenLabsVoiceSettings, text: spoken, signal: controller.signal }); diff --git a/src/lib/state/providers.svelte.ts b/src/lib/state/providers.svelte.ts index e978e70..e3bd78e 100644 --- a/src/lib/state/providers.svelte.ts +++ b/src/lib/state/providers.svelte.ts @@ -10,10 +10,15 @@ * bundle contains none of this (the DEV branch is compiled out). */ import { + DEFAULT_ELEVENLABS_MODEL, DEFAULT_ELEVENLABS_VOICE, DEFAULT_REALTIME_EFFORT, DEFAULT_REALTIME_VOICE, ELEVENLABS_MODELS, + elevenLabsRevision, + elevenLabsVoiceSettings, + getElevenLabsModel, + normalizeElevenLabsOptions, REALTIME_MODELS, REALTIME_VOICES, defaultCloudLlmModel, @@ -24,7 +29,9 @@ import { type ApiProvider, type CloudLlmProvider, type DescriptionEngine, + type ElevenLabsModelSpec, type ElevenLabsVoice, + type ElevenLabsVoiceOptions, type RealtimeEffort, type SpeechEngine, type StudyEngine @@ -71,8 +78,10 @@ export class ProvidersState { realtimeModelId = $state(REALTIME_MODELS[0].id); realtimeVoice = $state(DEFAULT_REALTIME_VOICE); realtimeEffort = $state(DEFAULT_REALTIME_EFFORT); - elevenLabsModelId = $state(ELEVENLABS_MODELS[0].id); + elevenLabsModelId = $state(DEFAULT_ELEVENLABS_MODEL); elevenLabsVoiceId = $state(DEFAULT_ELEVENLABS_VOICE); + /** Synthesis knobs kept per model id — v3 and v2 tune independently. */ + private elevenLabsOptions = $state>>({}); /** Cached voice list (refreshed whenever settings opens with a key). */ elevenLabsVoices = $state([]); private voicesLoad?: Promise; @@ -94,6 +103,7 @@ export class ProvidersState { elModel, elVoice, elVoices, + elOptions, realtimeModel, rtVoice, rtEffort, @@ -104,9 +114,10 @@ export class ProvidersState { getSetting('description-engine', 'local'), getSetting>>('cloud-llm-models', {}), getSetting('speech-engine', 'local'), - getSetting('elevenlabs-model', ELEVENLABS_MODELS[0].id), + getSetting('elevenlabs-model', DEFAULT_ELEVENLABS_MODEL), getSetting('elevenlabs-voice', DEFAULT_ELEVENLABS_VOICE), getSetting('elevenlabs-voices-cache', []), + getSetting>>('elevenlabs-model-options', {}), getSetting('realtime-model', REALTIME_MODELS[0].id), getSetting('realtime-voice', DEFAULT_REALTIME_VOICE), getSetting('realtime-effort', DEFAULT_REALTIME_EFFORT), @@ -122,9 +133,10 @@ export class ProvidersState { this.speechEngine = speech === 'elevenlabs' ? 'elevenlabs' : 'local'; this.elevenLabsModelId = ELEVENLABS_MODELS.some((model) => model.id === elModel) ? elModel - : ELEVENLABS_MODELS[0].id; + : DEFAULT_ELEVENLABS_MODEL; this.elevenLabsVoiceId = elVoice || DEFAULT_ELEVENLABS_VOICE; this.elevenLabsVoices = elVoices ?? []; + this.elevenLabsOptions = elOptions ?? {}; this.realtimeModelId = REALTIME_MODELS.some((model) => model.id === realtimeModel) ? realtimeModel : REALTIME_MODELS[0].id; @@ -284,6 +296,63 @@ export class ProvidersState { await setSetting('elevenlabs-model', modelId); } + /** The selected model's spec, falling back to the default if a stored id + * ever outlives the catalog. */ + get elevenLabsModel(): ElevenLabsModelSpec { + return ( + getElevenLabsModel(this.elevenLabsModelId) ?? + (getElevenLabsModel(DEFAULT_ELEVENLABS_MODEL) as ElevenLabsModelSpec) + ); + } + + /** Fully-populated, clamped options for one model — knobs it does not + * honour read back as their defaults. */ + elevenLabsOptionsFor(modelId: string): ElevenLabsVoiceOptions { + const model = getElevenLabsModel(modelId); + if (!model) return normalizeElevenLabsOptions(this.elevenLabsModel, null); + return normalizeElevenLabsOptions(model, this.elevenLabsOptions[modelId]); + } + + /** voice_settings for the active model, or undefined to let the voice's + * own saved settings apply. */ + get elevenLabsVoiceSettings(): Record | undefined { + const model = this.elevenLabsModel; + return elevenLabsVoiceSettings(model, this.elevenLabsOptionsFor(model.id)); + } + + /** Cache-key fragment: the model id, plus a suffix once options move. */ + get elevenLabsRevision(): string { + const model = this.elevenLabsModel; + return elevenLabsRevision(model, this.elevenLabsOptionsFor(model.id)); + } + + async setElevenLabsOptions( + modelId: string, + patch: Partial + ): Promise { + const model = getElevenLabsModel(modelId); + if (!model) return; + const next = normalizeElevenLabsOptions(model, { + ...this.elevenLabsOptionsFor(modelId), + ...patch + }); + this.elevenLabsOptions = { ...this.elevenLabsOptions, [modelId]: next }; + await this.persistElevenLabsOptions(); + } + + async resetElevenLabsOptions(modelId: string): Promise { + const next = { ...this.elevenLabsOptions }; + delete next[modelId]; + this.elevenLabsOptions = next; + await this.persistElevenLabsOptions(); + } + + /** The stored records are nested objects, so they read back as $state + * proxies — IndexedDB cannot structured-clone those. Snapshot first. */ + private async persistElevenLabsOptions(): Promise { + await setSetting('elevenlabs-model-options', $state.snapshot(this.elevenLabsOptions)); + } + async setElevenLabsVoice(voiceId: string): Promise { this.elevenLabsVoiceId = voiceId; await setSetting('elevenlabs-voice', voiceId); diff --git a/src/routes/settings/+page.svelte b/src/routes/settings/+page.svelte index 98ef7e2..4f79ed0 100644 --- a/src/routes/settings/+page.svelte +++ b/src/routes/settings/+page.svelte @@ -31,7 +31,7 @@ import { LLM_CATALOG, type LlmModelSpec } from '$lib/domain/llm-catalog'; import { CLOUD_LLM_PROVIDERS, - ELEVENLABS_MODELS, + ELEVENLABS_FAMILIES, REALTIME_EFFORTS, REALTIME_MODELS, REALTIME_VOICES, @@ -45,6 +45,7 @@ getCloudLlmProvider } from '$lib/domain/provider-catalog'; import ApiKeyField from '$lib/components/ApiKeyField.svelte'; + import ElevenLabsOptions from '$lib/components/ElevenLabsOptions.svelte'; import { verifyCloudLlmKey } from '$lib/services/cloud-llm'; import { elevenLabsUsage, type ElevenLabsUsage } from '$lib/services/elevenlabs'; import { player } from '$lib/state/player.svelte'; @@ -743,20 +744,36 @@ {#if providersState.speechEngine === 'elevenlabs'}
-
- {#each ELEVENLABS_MODELS as model (model.id)} - - {/each} -
+ {#each ELEVENLABS_FAMILIES as family (family.id)} +
+
+ {family.label} + {family.note} +
+
+ {#each family.models as model (model.id)} + + {/each} +
+
+ {/each} + + providersState.setElevenLabsOptions(providersState.elevenLabsModelId, patch)} + onReset={() => + providersState.resetElevenLabsOptions(providersState.elevenLabsModelId)} + />