From 71eed3d80e45e656a8a073243c490d6ef2367437 Mon Sep 17 00:00:00 2001 From: NeoVand Date: Wed, 26 Aug 2026 22:29:37 -0500 Subject: [PATCH] feat: offer Eleven v3 alongside the v2 models, each with its own settings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eleven v3 was deliberately absent from the catalog because it could not return character alignment, and word highlighting depends on it. Probing the live API shows that is no longer true: both eleven_v3 and eleven_v3_conversational return alignment from /with-timestamps at pcm_24000, so they can join the picker without losing read-along. The two generations are grouped in Settings and every v2 model stays. Flash v2.5 remains the default — half the credits, and steadiest on the short passages the segmenter produces. Each model now carries its own synthesis knobs, persisted per model id in the elevenlabs-model-options setting, and only the ones that model actually honours are shown. The support matrix comes from GET /v1/models plus measurement, not the docs: v3 accepts `speed` and ignores it (identical audio at 0.7 and 1.2, where Flash tracked both exactly), and reports can_use_style false, so v3 exposes stability alone — the three named points ElevenLabs documents. v3 Conversational adds speaker boost; Multilingual v2 exposes all five. Cached audio is preserved. Untouched options send no voice_settings at all, which is exactly what the code did before, and elevenLabsRevision() returns the bare model id so existing cache entries still match. Moving any knob appends a sorted suffix to the revision, which flows into the audio cache key and re-synthesizes that variant. The wire always receives the bare model id. Also guards a request longer than the model's limit (v3 caps at 5,000 characters against Flash's 40,000) before spending credits, and persists the options with $state.snapshot — nested $state proxies cannot be structured-cloned into IndexedDB, and the resulting DataCloneError rejected silently while the UI looked correct until a reload. Co-Authored-By: Claude Opus 5 --- README.md | 2 +- src/lib/components/ElevenLabsOptions.svelte | 291 ++++++++++++++++++++ src/lib/domain/provider-catalog.spec.ts | 118 +++++++- src/lib/domain/provider-catalog.ts | 232 +++++++++++++++- src/lib/services/elevenlabs.spec.ts | 83 +++++- src/lib/services/elevenlabs.ts | 23 +- src/lib/state/player.svelte.ts | 22 +- src/lib/state/providers.svelte.ts | 75 ++++- src/routes/settings/+page.svelte | 64 ++++- 9 files changed, 865 insertions(+), 45 deletions(-) create mode 100644 src/lib/components/ElevenLabsOptions.svelte diff --git a/README.md b/README.md index 0667fd0..5a57e6c 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ Useful shortcuts: The free on-device engine is **Supertonic 3** through ONNX Runtime in a dedicated browser worker. The model is downloaded from Hugging Face only after you accept its OpenRAIL-M terms; once installed, speech generation runs locally with WebGPU acceleration when available. Generation quality is adjustable from 2 to 16 steps, with 10 steps as the default. Voicebook generates the current passage first, buffers upcoming passages, and stores generated audio locally so it does not need to repeat work. -Prefer premium voices? Paste an **ElevenLabs** API key and pick any of your voices — no download, native word-level timing for read-along highlighting, and the economical Flash v2.5 model by default. +Prefer premium voices? Paste an **ElevenLabs** API key and pick any of your voices — no download and native word-level timing for read-along highlighting. Both generations are available: **Eleven v3** and **Eleven v3 Conversational** for the most expressive read, or the **v2** models (Flash v2.5, Turbo v2.5, Multilingual v2) for a cheaper, steadier one. Flash v2.5 is the default, and each model keeps its own settings — stability, similarity, style, speed, and speaker boost, wherever that model honours them. ## Spoken descriptions diff --git a/src/lib/components/ElevenLabsOptions.svelte b/src/lib/components/ElevenLabsOptions.svelte new file mode 100644 index 0000000..98df88d --- /dev/null +++ b/src/lib/components/ElevenLabsOptions.svelte @@ -0,0 +1,291 @@ + + +
+
+
+ {model.label} settings + + {customized + ? 'Changing these re-generates audio the next time you play.' + : "Untouched — each voice's own saved settings apply."} + +
+ {#if customized} + + {/if} +
+ + {#if model.options.stability === 'discrete'} +
+ Stability +
+ {#each ELEVENLABS_V3_STABILITY as point (point.value)} + + {/each} +
+
+ {:else} + + {/if} + + {#if model.options.similarity} + + {/if} + + {#if model.options.style} + + {/if} + + {#if model.options.speed} + + {/if} + + {#if model.options.speakerBoost} +
+
+ Speaker boost + Sharpens resemblance to the voice, at a little latency. +
+ +
+ {/if} +
+ + diff --git a/src/lib/domain/provider-catalog.spec.ts b/src/lib/domain/provider-catalog.spec.ts index 85c63de..af64f13 100644 --- a/src/lib/domain/provider-catalog.spec.ts +++ b/src/lib/domain/provider-catalog.spec.ts @@ -1,10 +1,18 @@ import { describe, expect, it } from 'vitest'; import { CLOUD_LLM_PROVIDERS, + DEFAULT_ELEVENLABS_MODEL, + DEFAULT_ELEVENLABS_OPTIONS, + ELEVENLABS_FAMILIES, ELEVENLABS_MODELS, defaultCloudLlmModel, + elevenLabsRevision, + elevenLabsVoiceSettings, getCloudLlmProvider, - isCloudLlmProvider + getElevenLabsModel, + isCloudLlmProvider, + normalizeElevenLabsOptions, + type ElevenLabsModelSpec } from './provider-catalog'; describe('cloud LLM provider catalog', () => { @@ -29,9 +37,109 @@ describe('cloud LLM provider catalog', () => { }); describe('elevenlabs model catalog', () => { - it('defaults to the cheapest timestamp-capable model', () => { - expect(ELEVENLABS_MODELS[0].id).toBe('eleven_flash_v2_5'); - // v3 has no with-timestamps support — word highlighting depends on it. - expect(ELEVENLABS_MODELS.some((model) => model.id === 'eleven_v3')).toBe(false); + it('offers both generations and defaults to the cheapest v2 model', () => { + expect(getElevenLabsModel(DEFAULT_ELEVENLABS_MODEL)).not.toBeNull(); + expect(DEFAULT_ELEVENLABS_MODEL).toBe('eleven_flash_v2_5'); + // v3 now returns character alignment from the with-timestamps + // endpoint, so word highlighting survives the switch. + expect(ELEVENLABS_MODELS.map((model) => model.id)).toEqual([ + 'eleven_v3', + 'eleven_v3_conversational', + 'eleven_flash_v2_5', + 'eleven_turbo_v2_5', + 'eleven_multilingual_v2' + ]); + }); + + it('groups every model into exactly one family', () => { + const grouped = ELEVENLABS_FAMILIES.flatMap((family) => family.models); + expect(grouped).toHaveLength(ELEVENLABS_MODELS.length); + expect(new Set(grouped.map((model) => model.id)).size).toBe(ELEVENLABS_MODELS.length); + expect(ELEVENLABS_FAMILIES[0].id).toBe('v3'); + }); +}); + +describe('elevenlabs voice options', () => { + const v3 = getElevenLabsModel('eleven_v3') as ElevenLabsModelSpec; + const conversational = getElevenLabsModel('eleven_v3_conversational') as ElevenLabsModelSpec; + const flash = getElevenLabsModel('eleven_flash_v2_5') as ElevenLabsModelSpec; + const multilingual = getElevenLabsModel('eleven_multilingual_v2') as ElevenLabsModelSpec; + + it('snaps v3 stability to the three named points', () => { + expect(normalizeElevenLabsOptions(v3, { stability: 0.2 }).stability).toBe(0); + expect(normalizeElevenLabsOptions(v3, { stability: 0.4 }).stability).toBe(0.5); + expect(normalizeElevenLabsOptions(v3, { stability: 0.9 }).stability).toBe(1); + // v2 keeps the continuous value. + expect(normalizeElevenLabsOptions(flash, { stability: 0.2 }).stability).toBe(0.2); + }); + + it('clamps out-of-range and non-finite values', () => { + expect(normalizeElevenLabsOptions(flash, { similarity: 4 }).similarity).toBe(1); + expect(normalizeElevenLabsOptions(flash, { speed: 0.1 }).speed).toBe(0.7); + expect(normalizeElevenLabsOptions(flash, { speed: 9 }).speed).toBe(1.2); + expect(normalizeElevenLabsOptions(flash, { stability: Number.NaN }).stability).toBe( + DEFAULT_ELEVENLABS_OPTIONS.stability + ); + }); + + it('forces knobs a model ignores back to their defaults', () => { + // v3 accepts `speed` on the wire but produces identical audio, and + // reports can_use_style / can_use_speaker_boost false. + const tuned = normalizeElevenLabsOptions(v3, { + speed: 1.2, + style: 0.8, + similarity: 0.1, + speakerBoost: false + }); + expect(tuned.speed).toBe(1); + expect(tuned.style).toBe(0); + expect(tuned.similarity).toBe(DEFAULT_ELEVENLABS_OPTIONS.similarity); + expect(tuned.speakerBoost).toBe(true); + }); + + it('sends nothing while the options are untouched', () => { + for (const model of ELEVENLABS_MODELS) { + const options = normalizeElevenLabsOptions(model, null); + expect(elevenLabsVoiceSettings(model, options)).toBeUndefined(); + // The bare id keeps audio cached before these controls existed. + expect(elevenLabsRevision(model, options)).toBe(model.id); + } + }); + + it('sends only the fields the model honours', () => { + const v3Body = elevenLabsVoiceSettings(v3, normalizeElevenLabsOptions(v3, { stability: 0 })); + expect(v3Body).toEqual({ stability: 0 }); + + const boostBody = elevenLabsVoiceSettings( + conversational, + normalizeElevenLabsOptions(conversational, { speakerBoost: false }) + ); + expect(boostBody).toEqual({ stability: 0.5, use_speaker_boost: false }); + + const flashBody = elevenLabsVoiceSettings( + flash, + normalizeElevenLabsOptions(flash, { speed: 1.1 }) + ); + expect(flashBody).toEqual({ stability: 0.5, similarity_boost: 0.75, speed: 1.1 }); + + const fullBody = elevenLabsVoiceSettings( + multilingual, + normalizeElevenLabsOptions(multilingual, { style: 0.4 }) + ); + expect(fullBody).toEqual({ + stability: 0.5, + similarity_boost: 0.75, + style: 0.4, + speed: 1, + use_speaker_boost: true + }); + }); + + it('gives tuned options a stable, order-independent revision', () => { + const a = elevenLabsRevision(flash, normalizeElevenLabsOptions(flash, { speed: 1.1 })); + const b = elevenLabsRevision(flash, normalizeElevenLabsOptions(flash, { speed: 1.1 })); + expect(a).toBe(b); + expect(a).toBe('eleven_flash_v2_5#similarity_boost=0.75,speed=1.1,stability=0.5'); + expect(a).not.toBe(elevenLabsRevision(flash, normalizeElevenLabsOptions(flash, { speed: 1 }))); }); }); diff --git a/src/lib/domain/provider-catalog.ts b/src/lib/domain/provider-catalog.ts index f3ed8ef..1792d5e 100644 --- a/src/lib/domain/provider-catalog.ts +++ b/src/lib/domain/provider-catalog.ts @@ -96,29 +96,249 @@ export function isCloudLlmProvider(value: string): value is CloudLlmProvider { /* ── ElevenLabs speech ───────────────────────────────────────────────────── */ +/** v3 is a different generation with its own controls — grouped in the UI and + * gated separately when building the request body. */ +export type ElevenLabsFamily = 'v3' | 'v2'; + +/** Which voice_settings a model actually honours. Probed against + * GET /v1/models (can_use_style, can_use_speaker_boost) and by measuring + * generated audio — v3 accepts `speed` but ignores it, so it is off here. */ +export interface ElevenLabsModelOptionSupport { + /** v3 exposes three named stability points; v2 takes any 0–1 value. */ + stability: 'discrete' | 'continuous'; + similarity: boolean; + style: boolean; + speed: boolean; + speakerBoost: boolean; +} + export interface ElevenLabsModelSpec { id: string; label: string; tagline: string; + family: ElevenLabsFamily; + /** Per-request cap from the model's maximum_text_length_per_request. + * Passages are capped far below this; the guard is for safety. */ + maxCharacters: number; + options: ElevenLabsModelOptionSupport; } -/** TTS models that support the with-timestamps endpoint (word highlighting - * needs character alignment, so v3 alpha is deliberately absent). First entry - * is the default. */ +/** Every TTS model that returns character alignment from the with-timestamps + * endpoint — word highlighting depends on it, which v3 now supports too. + * DEFAULT_ELEVENLABS_MODEL, not the array order, picks the default. */ export const ELEVENLABS_MODELS: ElevenLabsModelSpec[] = [ + { + id: 'eleven_v3', + label: 'Eleven v3', + tagline: 'most expressive · 70+ languages', + family: 'v3', + maxCharacters: 5_000, + options: { + stability: 'discrete', + similarity: false, + style: false, + speed: false, + speakerBoost: false + } + }, + { + id: 'eleven_v3_conversational', + label: 'Eleven v3 Conversational', + tagline: 'expressive · half the credits', + family: 'v3', + maxCharacters: 5_000, + options: { + stability: 'discrete', + similarity: false, + style: false, + speed: false, + speakerBoost: true + } + }, { id: 'eleven_flash_v2_5', label: 'Flash v2.5', - tagline: 'half the credits · recommended' + tagline: 'half the credits · recommended', + family: 'v2', + maxCharacters: 40_000, + options: { + stability: 'continuous', + similarity: true, + style: false, + speed: true, + speakerBoost: false + } + }, + { + id: 'eleven_turbo_v2_5', + label: 'Turbo v2.5', + tagline: 'fast · half the credits', + family: 'v2', + maxCharacters: 40_000, + options: { + stability: 'continuous', + similarity: true, + style: false, + speed: true, + speakerBoost: false + } }, - { id: 'eleven_turbo_v2_5', label: 'Turbo v2.5', tagline: 'fast · half the credits' }, { id: 'eleven_multilingual_v2', label: 'Multilingual v2', - tagline: 'highest quality · double credits' + tagline: 'steadiest long-form read', + family: 'v2', + maxCharacters: 10_000, + options: { + stability: 'continuous', + similarity: true, + style: true, + speed: true, + speakerBoost: true + } + } +]; + +/** Flash v2.5 stays the default: cheapest per character and the steadiest on + * the short passages the segmenter produces. v3 is one click away. */ +export const DEFAULT_ELEVENLABS_MODEL = 'eleven_flash_v2_5'; + +export function getElevenLabsModel(id: string): ElevenLabsModelSpec | null { + return ELEVENLABS_MODELS.find((model) => model.id === id) ?? null; +} + +/** The catalog in display order: v3 first, then the v2 generation. */ +export const ELEVENLABS_FAMILIES: Array<{ + id: ElevenLabsFamily; + label: string; + note: string; + models: ElevenLabsModelSpec[]; +}> = [ + { + id: 'v3', + label: 'Eleven v3', + note: 'Newest generation — richer delivery, more variation between takes.', + models: ELEVENLABS_MODELS.filter((model) => model.family === 'v3') + }, + { + id: 'v2', + label: 'Eleven v2', + note: 'Predictable and cheap — the steady choice for a long read.', + models: ELEVENLABS_MODELS.filter((model) => model.family === 'v2') } ]; +/** The three stability points the v3 models expose, in ElevenLabs' order. */ +export const ELEVENLABS_V3_STABILITY: Array<{ + value: number; + label: string; + tagline: string; +}> = [ + { value: 0, label: 'Creative', tagline: 'most emotion · can drift' }, + { value: 0.5, label: 'Natural', tagline: 'closest to the voice' }, + { value: 1, label: 'Robust', tagline: 'steadiest · least directable' } +]; + +/** Per-model synthesis knobs. Values match the ElevenLabs defaults, so an + * untouched set is indistinguishable from sending nothing. */ +export interface ElevenLabsVoiceOptions { + stability: number; + similarity: number; + style: number; + speed: number; + speakerBoost: boolean; +} + +export const DEFAULT_ELEVENLABS_OPTIONS: ElevenLabsVoiceOptions = { + stability: 0.5, + similarity: 0.75, + style: 0, + speed: 1, + speakerBoost: true +}; + +/** Snap to the nearest of the three v3 stability points. */ +function snapStability(value: number): number { + return ELEVENLABS_V3_STABILITY.reduce((best, point) => + Math.abs(point.value - value) < Math.abs(best.value - value) ? point : best + ).value; +} + +function clamp(value: number, low: number, high: number, fallback: number): number { + if (!Number.isFinite(value)) return fallback; + return Math.min(high, Math.max(low, value)); +} + +/** + * Fill in and clamp a stored option set for one model. Knobs the model does + * not honour are forced back to their default so they can never leak into a + * request (or into the cache signature) for a model that ignores them. + */ +export function normalizeElevenLabsOptions( + model: ElevenLabsModelSpec, + stored?: Partial | null +): ElevenLabsVoiceOptions { + const raw = { ...DEFAULT_ELEVENLABS_OPTIONS, ...(stored ?? {}) }; + const stability = clamp(raw.stability, 0, 1, DEFAULT_ELEVENLABS_OPTIONS.stability); + const support = model.options; + return { + stability: support.stability === 'discrete' ? snapStability(stability) : stability, + similarity: support.similarity + ? clamp(raw.similarity, 0, 1, DEFAULT_ELEVENLABS_OPTIONS.similarity) + : DEFAULT_ELEVENLABS_OPTIONS.similarity, + style: support.style ? clamp(raw.style, 0, 1, DEFAULT_ELEVENLABS_OPTIONS.style) : 0, + speed: support.speed ? clamp(raw.speed, 0.7, 1.2, DEFAULT_ELEVENLABS_OPTIONS.speed) : 1, + speakerBoost: support.speakerBoost + ? Boolean(raw.speakerBoost) + : DEFAULT_ELEVENLABS_OPTIONS.speakerBoost + }; +} + +export function isDefaultElevenLabsOptions(options: ElevenLabsVoiceOptions): boolean { + return ( + options.stability === DEFAULT_ELEVENLABS_OPTIONS.stability && + options.similarity === DEFAULT_ELEVENLABS_OPTIONS.similarity && + options.style === DEFAULT_ELEVENLABS_OPTIONS.style && + options.speed === DEFAULT_ELEVENLABS_OPTIONS.speed && + options.speakerBoost === DEFAULT_ELEVENLABS_OPTIONS.speakerBoost + ); +} + +/** The voice_settings body, or undefined to let the voice's own saved + * settings apply — which is what an untouched set has always meant, so + * audio cached before these controls existed still matches. */ +export function elevenLabsVoiceSettings( + model: ElevenLabsModelSpec, + options: ElevenLabsVoiceOptions +): Record | undefined { + const normalized = normalizeElevenLabsOptions(model, options); + if (isDefaultElevenLabsOptions(normalized)) return undefined; + const settings: Record = { stability: normalized.stability }; + if (model.options.similarity) settings.similarity_boost = normalized.similarity; + if (model.options.style) settings.style = normalized.style; + if (model.options.speed) settings.speed = normalized.speed; + if (model.options.speakerBoost) settings.use_speaker_boost = normalized.speakerBoost; + return settings; +} + +/** + * Cache-key fragment for a model plus its options: the bare model id while the + * options are untouched (so audio generated before the controls shipped keeps + * matching), and a stable suffix once any knob moves. + */ +export function elevenLabsRevision( + model: ElevenLabsModelSpec, + options: ElevenLabsVoiceOptions +): string { + const settings = elevenLabsVoiceSettings(model, options); + if (!settings) return model.id; + const suffix = Object.keys(settings) + .sort() + .map((key) => `${key}=${settings[key]}`) + .join(','); + return `${model.id}#${suffix}`; +} + /** George — a warm narrator that suits long-form reading. */ export const DEFAULT_ELEVENLABS_VOICE = 'JBFqnCBsd6RMkjVDRZzb'; diff --git a/src/lib/services/elevenlabs.spec.ts b/src/lib/services/elevenlabs.spec.ts index d093638..fc6ee20 100644 --- a/src/lib/services/elevenlabs.spec.ts +++ b/src/lib/services/elevenlabs.spec.ts @@ -1,5 +1,10 @@ -import { describe, expect, it } from 'vitest'; -import { pcm16ToFloat32, wordTimingsFromAlignment } from './elevenlabs'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { + ElevenLabsError, + pcm16ToFloat32, + synthesizeElevenLabs, + wordTimingsFromAlignment +} from './elevenlabs'; describe('word timings from character alignment', () => { it('groups characters into whitespace-delimited words with span timing', () => { @@ -73,3 +78,77 @@ describe('pcm decoding', () => { expect(pcm16ToFloat32(bytes).length).toBe(1); }); }); + +describe('synthesis request', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + function stubFetch(): ReturnType { + const samples = new Int16Array([0, 128, -128, 0]); + const bytes = new Uint8Array(samples.buffer); + const fetchMock = vi.fn(async () => ({ + ok: true, + status: 200, + json: async () => ({ + audio_base64: btoa(String.fromCharCode(...bytes)), + alignment: { + characters: ['H', 'i'], + character_start_times_seconds: [0, 0.1], + character_end_times_seconds: [0.1, 0.2] + } + }) + })); + vi.stubGlobal('fetch', fetchMock); + return fetchMock; + } + + function bodyOf(fetchMock: ReturnType): Record { + const init = fetchMock.mock.calls[0][1] as RequestInit; + return JSON.parse(init.body as string) as Record; + } + + it('sends the bare model id and omits voice_settings when none are given', async () => { + const fetchMock = stubFetch(); + const result = await synthesizeElevenLabs({ + apiKey: 'k', + voiceId: 'v', + modelId: 'eleven_v3', + text: 'Hi' + }); + expect(fetchMock.mock.calls[0][0]).toContain('/v1/text-to-speech/v/with-timestamps'); + expect(bodyOf(fetchMock)).toEqual({ text: 'Hi', model_id: 'eleven_v3' }); + expect(result.timing.confidence).toBe('native'); + expect(result.timing.words.map((word) => word.word)).toEqual(['Hi']); + }); + + it('forwards voice settings verbatim when the user has tuned them', async () => { + const fetchMock = stubFetch(); + await synthesizeElevenLabs({ + apiKey: 'k', + voiceId: 'v', + modelId: 'eleven_v3', + text: 'Hi', + voiceSettings: { stability: 1 } + }); + expect(bodyOf(fetchMock)).toEqual({ + text: 'Hi', + model_id: 'eleven_v3', + voice_settings: { stability: 1 } + }); + }); + + it('refuses a passage longer than the model accepts before spending credits', async () => { + const fetchMock = stubFetch(); + await expect( + synthesizeElevenLabs({ + apiKey: 'k', + voiceId: 'v', + // v3 caps at 5,000 characters where Flash takes 40,000. + modelId: 'eleven_v3', + text: 'x'.repeat(5_001) + }) + ).rejects.toThrow(ElevenLabsError); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/services/elevenlabs.ts b/src/lib/services/elevenlabs.ts index d9ffb46..f13e7dd 100644 --- a/src/lib/services/elevenlabs.ts +++ b/src/lib/services/elevenlabs.ts @@ -7,7 +7,7 @@ import { wordsFor } from '$lib/domain/speech-words'; import type { TimingMap, WordTiming } from '$lib/domain/types'; import type { SynthesisResult } from './tts-client'; -import type { ElevenLabsVoice } from '$lib/domain/provider-catalog'; +import { getElevenLabsModel, type ElevenLabsVoice } from '$lib/domain/provider-catalog'; const API = 'https://api.elevenlabs.io'; const SAMPLE_RATE = 24_000; @@ -118,17 +118,30 @@ export interface ElevenLabsSynthesisRequest { voiceId: string; modelId: string; text: string; + /** Built by elevenLabsVoiceSettings — omitted while the user has not + * moved any knob, which leaves the voice's own saved settings in force. */ + voiceSettings?: Record; signal?: AbortSignal; } /** * Timestamped synthesis at PCM 24 kHz — drops straight into the player's - * AudioBuffer/word-highlight pipeline with native timing confidence. + * AudioBuffer/word-highlight pipeline with native timing confidence. Every + * catalog model returns character alignment here, v3 included. */ export async function synthesizeElevenLabs( options: ElevenLabsSynthesisRequest ): Promise { const started = performance.now(); + // Passages are capped far below every model's limit; this catches a + // mismatch (say a 5,000-character v3 request) with a readable message + // instead of a bare 422 from the API. + const spec = getElevenLabsModel(options.modelId); + if (spec && options.text.length > spec.maxCharacters) { + throw new ElevenLabsError( + `This passage is ${options.text.length.toLocaleString()} characters — ${spec.label} accepts ${spec.maxCharacters.toLocaleString()} per request.` + ); + } const timeout = AbortSignal.timeout(90_000); const signal = options.signal ? AbortSignal.any([options.signal, timeout]) : timeout; const data = (await request( @@ -136,7 +149,11 @@ export async function synthesizeElevenLabs( options.apiKey, { method: 'POST', - body: JSON.stringify({ text: options.text, model_id: options.modelId }), + body: JSON.stringify({ + text: options.text, + model_id: options.modelId, + ...(options.voiceSettings ? { voice_settings: options.voiceSettings } : {}) + }), signal } )) as { diff --git a/src/lib/state/player.svelte.ts b/src/lib/state/player.svelte.ts index 1e341bd..f2b6607 100644 --- a/src/lib/state/player.svelte.ts +++ b/src/lib/state/player.svelte.ts @@ -28,7 +28,6 @@ import { import { encodeDocumentMp3, mp3Filename } from '$lib/services/mp3-export'; import { ttsClient, type SynthesisResult } from '$lib/services/tts-client'; import { synthesizeElevenLabs } from '$lib/services/elevenlabs'; -import { ELEVENLABS_MODELS } from '$lib/domain/provider-catalog'; import { generationPlan } from '$lib/services/generation-plan'; import { absoluteTimelinePosition, @@ -284,7 +283,9 @@ export class VoicebookPlayer { return { modelId: 'elevenlabs', repository: 'elevenlabs', - revision: providersState.elevenLabsModelId, + // Carries the tuning knobs so a changed setting re-synthesizes + // instead of replaying audio made with the old ones. + revision: providersState.elevenLabsRevision, voiceId: providersState.elevenLabsVoiceId, backend: 'cloud', dtype: 'pcm24' @@ -302,12 +303,7 @@ export class VoicebookPlayer { } get runtimeLabel(): string { - if (this.usesElevenLabs) { - const model = ELEVENLABS_MODELS.find( - (candidate) => candidate.id === providersState.elevenLabsModelId - ); - return `ElevenLabs · ${model?.label ?? providersState.elevenLabsModelId}`; - } + if (this.usesElevenLabs) return `ElevenLabs · ${providersState.elevenLabsModel.label}`; if (!this.engineBackend) return 'Engine warming'; return `${this.engineBackend === 'webgpu' ? 'WebGPU' : 'WASM'} · ${this.engineDtype}`; } @@ -733,7 +729,10 @@ export class VoicebookPlayer { generated = await synthesizeElevenLabs({ apiKey: providersState.keyFor('elevenlabs') ?? '', voiceId: variant.voiceId, - modelId: variant.revision, + // variant.revision folds the tuning knobs in for cache keys; + // the wire needs the bare model id and its settings. + modelId: providersState.elevenLabsModel.id, + voiceSettings: providersState.elevenLabsVoiceSettings, text: segment.normalizedText, signal: controller.signal }); @@ -1269,7 +1268,10 @@ export class VoicebookPlayer { generated = await synthesizeElevenLabs({ apiKey: providersState.keyFor('elevenlabs') ?? '', voiceId: variant.voiceId, - modelId: variant.revision, + // variant.revision folds the tuning knobs in for cache keys; + // the wire needs the bare model id and its settings. + modelId: providersState.elevenLabsModel.id, + voiceSettings: providersState.elevenLabsVoiceSettings, text: spoken, signal: controller.signal }); diff --git a/src/lib/state/providers.svelte.ts b/src/lib/state/providers.svelte.ts index e978e70..e3bd78e 100644 --- a/src/lib/state/providers.svelte.ts +++ b/src/lib/state/providers.svelte.ts @@ -10,10 +10,15 @@ * bundle contains none of this (the DEV branch is compiled out). */ import { + DEFAULT_ELEVENLABS_MODEL, DEFAULT_ELEVENLABS_VOICE, DEFAULT_REALTIME_EFFORT, DEFAULT_REALTIME_VOICE, ELEVENLABS_MODELS, + elevenLabsRevision, + elevenLabsVoiceSettings, + getElevenLabsModel, + normalizeElevenLabsOptions, REALTIME_MODELS, REALTIME_VOICES, defaultCloudLlmModel, @@ -24,7 +29,9 @@ import { type ApiProvider, type CloudLlmProvider, type DescriptionEngine, + type ElevenLabsModelSpec, type ElevenLabsVoice, + type ElevenLabsVoiceOptions, type RealtimeEffort, type SpeechEngine, type StudyEngine @@ -71,8 +78,10 @@ export class ProvidersState { realtimeModelId = $state(REALTIME_MODELS[0].id); realtimeVoice = $state(DEFAULT_REALTIME_VOICE); realtimeEffort = $state(DEFAULT_REALTIME_EFFORT); - elevenLabsModelId = $state(ELEVENLABS_MODELS[0].id); + elevenLabsModelId = $state(DEFAULT_ELEVENLABS_MODEL); elevenLabsVoiceId = $state(DEFAULT_ELEVENLABS_VOICE); + /** Synthesis knobs kept per model id — v3 and v2 tune independently. */ + private elevenLabsOptions = $state>>({}); /** Cached voice list (refreshed whenever settings opens with a key). */ elevenLabsVoices = $state([]); private voicesLoad?: Promise; @@ -94,6 +103,7 @@ export class ProvidersState { elModel, elVoice, elVoices, + elOptions, realtimeModel, rtVoice, rtEffort, @@ -104,9 +114,10 @@ export class ProvidersState { getSetting('description-engine', 'local'), getSetting>>('cloud-llm-models', {}), getSetting('speech-engine', 'local'), - getSetting('elevenlabs-model', ELEVENLABS_MODELS[0].id), + getSetting('elevenlabs-model', DEFAULT_ELEVENLABS_MODEL), getSetting('elevenlabs-voice', DEFAULT_ELEVENLABS_VOICE), getSetting('elevenlabs-voices-cache', []), + getSetting>>('elevenlabs-model-options', {}), getSetting('realtime-model', REALTIME_MODELS[0].id), getSetting('realtime-voice', DEFAULT_REALTIME_VOICE), getSetting('realtime-effort', DEFAULT_REALTIME_EFFORT), @@ -122,9 +133,10 @@ export class ProvidersState { this.speechEngine = speech === 'elevenlabs' ? 'elevenlabs' : 'local'; this.elevenLabsModelId = ELEVENLABS_MODELS.some((model) => model.id === elModel) ? elModel - : ELEVENLABS_MODELS[0].id; + : DEFAULT_ELEVENLABS_MODEL; this.elevenLabsVoiceId = elVoice || DEFAULT_ELEVENLABS_VOICE; this.elevenLabsVoices = elVoices ?? []; + this.elevenLabsOptions = elOptions ?? {}; this.realtimeModelId = REALTIME_MODELS.some((model) => model.id === realtimeModel) ? realtimeModel : REALTIME_MODELS[0].id; @@ -284,6 +296,63 @@ export class ProvidersState { await setSetting('elevenlabs-model', modelId); } + /** The selected model's spec, falling back to the default if a stored id + * ever outlives the catalog. */ + get elevenLabsModel(): ElevenLabsModelSpec { + return ( + getElevenLabsModel(this.elevenLabsModelId) ?? + (getElevenLabsModel(DEFAULT_ELEVENLABS_MODEL) as ElevenLabsModelSpec) + ); + } + + /** Fully-populated, clamped options for one model — knobs it does not + * honour read back as their defaults. */ + elevenLabsOptionsFor(modelId: string): ElevenLabsVoiceOptions { + const model = getElevenLabsModel(modelId); + if (!model) return normalizeElevenLabsOptions(this.elevenLabsModel, null); + return normalizeElevenLabsOptions(model, this.elevenLabsOptions[modelId]); + } + + /** voice_settings for the active model, or undefined to let the voice's + * own saved settings apply. */ + get elevenLabsVoiceSettings(): Record | undefined { + const model = this.elevenLabsModel; + return elevenLabsVoiceSettings(model, this.elevenLabsOptionsFor(model.id)); + } + + /** Cache-key fragment: the model id, plus a suffix once options move. */ + get elevenLabsRevision(): string { + const model = this.elevenLabsModel; + return elevenLabsRevision(model, this.elevenLabsOptionsFor(model.id)); + } + + async setElevenLabsOptions( + modelId: string, + patch: Partial + ): Promise { + const model = getElevenLabsModel(modelId); + if (!model) return; + const next = normalizeElevenLabsOptions(model, { + ...this.elevenLabsOptionsFor(modelId), + ...patch + }); + this.elevenLabsOptions = { ...this.elevenLabsOptions, [modelId]: next }; + await this.persistElevenLabsOptions(); + } + + async resetElevenLabsOptions(modelId: string): Promise { + const next = { ...this.elevenLabsOptions }; + delete next[modelId]; + this.elevenLabsOptions = next; + await this.persistElevenLabsOptions(); + } + + /** The stored records are nested objects, so they read back as $state + * proxies — IndexedDB cannot structured-clone those. Snapshot first. */ + private async persistElevenLabsOptions(): Promise { + await setSetting('elevenlabs-model-options', $state.snapshot(this.elevenLabsOptions)); + } + async setElevenLabsVoice(voiceId: string): Promise { this.elevenLabsVoiceId = voiceId; await setSetting('elevenlabs-voice', voiceId); diff --git a/src/routes/settings/+page.svelte b/src/routes/settings/+page.svelte index 98ef7e2..4f79ed0 100644 --- a/src/routes/settings/+page.svelte +++ b/src/routes/settings/+page.svelte @@ -31,7 +31,7 @@ import { LLM_CATALOG, type LlmModelSpec } from '$lib/domain/llm-catalog'; import { CLOUD_LLM_PROVIDERS, - ELEVENLABS_MODELS, + ELEVENLABS_FAMILIES, REALTIME_EFFORTS, REALTIME_MODELS, REALTIME_VOICES, @@ -45,6 +45,7 @@ getCloudLlmProvider } from '$lib/domain/provider-catalog'; import ApiKeyField from '$lib/components/ApiKeyField.svelte'; + import ElevenLabsOptions from '$lib/components/ElevenLabsOptions.svelte'; import { verifyCloudLlmKey } from '$lib/services/cloud-llm'; import { elevenLabsUsage, type ElevenLabsUsage } from '$lib/services/elevenlabs'; import { player } from '$lib/state/player.svelte'; @@ -743,20 +744,36 @@ {#if providersState.speechEngine === 'elevenlabs'}
-
- {#each ELEVENLABS_MODELS as model (model.id)} - - {/each} -
+ {#each ELEVENLABS_FAMILIES as family (family.id)} +
+
+ {family.label} + {family.note} +
+
+ {#each family.models as model (model.id)} + + {/each} +
+
+ {/each} + + providersState.setElevenLabsOptions(providersState.elevenLabsModelId, patch)} + onReset={() => + providersState.resetElevenLabsOptions(providersState.elevenLabsModelId)} + />