diff --git a/.env.example b/.env.example
index 84936d1..539619b 100644
--- a/.env.example
+++ b/.env.example
@@ -15,12 +15,12 @@ VOICEFORGE_DATA_DIR=data
# Where downloaded model checkpoints (Hugging Face cache, etc.) live.
VOICEFORGE_MODELS_DIR=models
-# "cpu", "cuda", or "auto" (auto = use CUDA if available, else CPU).
+# "cpu", "cuda", "mps", or "auto" (auto = CUDA, then Apple MPS, then CPU).
VOICEFORGE_DEVICE=auto
# Comma-separated engine ids enabled for clone/synth in this deployment.
-# Empty = all registered engines. Studio still lists all nine; unconfigured
-# ones show "not in this deployment". Modal: see MODAL_ENABLED_ENGINES in modal_app.py.
+# Empty = established engines only; experimental `voxcpm2` remains disabled.
+# Studio still lists every registered engine and marks deployment state.
# VOICEFORGE_ENABLED_ENGINES=openvoice-v2,f5-tts,xtts-v2,qwen3-tts,chatterbox
# Preload models at container startup (faster first synth; adds to cold-start time).
@@ -49,6 +49,16 @@ VOICEFORGE_PREPROCESS_SAMPLES=true
# VOICEFORGE_COSYVOICE_PYTHON=/opt/cosyvoice-venv/bin/python
# VOICEFORGE_INDEXTTS_PYTHON=/opt/indextts-venv/bin/python
# VOICEFORGE_CHATTERBOX_PYTHON=/opt/chatterbox-venv/bin/python
+# VOICEFORGE_CHATTERBOX_MODEL_DIR=/app/models/chatterbox
+# VOICEFORGE_QWEN3_TTS_MODEL_DIR=/absolute/path/to/models/qwen3-tts
+
+# Experimental providers — explicit local worker + local snapshot only:
+# VOICEFORGE_VOXCPM2_PYTHON=/path/to/voxcpm2-venv/bin/python
+# VOICEFORGE_VOXCPM2_MODEL_DIR=/absolute/path/to/models/voxcpm2
+# VOICEFORGE_REFERENCE_ASR_PROVIDER=qwen3-asr
+# VOICEFORGE_QWEN3_ASR_PYTHON=/path/to/qwen3-asr-venv/bin/python
+# VOICEFORGE_QWEN3_ASR_MODEL_DIR=/absolute/path/to/models/qwen3-asr-0.6b
+# VOICEFORGE_QWEN3_ALIGNER_MODEL_DIR=/absolute/path/to/models/qwen3-forced-aligner-0.6b
# M7 polish — structured logs and optional synth watermark
# VOICEFORGE_LOG_FORMAT=text
diff --git a/Makefile b/Makefile
index 10b58d9..647ccdf 100644
--- a/Makefile
+++ b/Makefile
@@ -12,7 +12,7 @@ start-gpu:
$(COMPOSE) --profile gpu up --build
stop:
- $(COMPOSE) --profile cpu --profile gpu --profile fish down
+ $(COMPOSE) --profile cpu --profile gpu down
logs:
$(COMPOSE) --profile cpu logs -f --tail=200
diff --git a/README.md b/README.md
index c5c4602..f792212 100644
--- a/README.md
+++ b/README.md
@@ -127,6 +127,7 @@ Details: [RESPONSIBLE_USE.md](RESPONSIBLE_USE.md) ·
### Higher-quality / heavier zero-shot
- `f5-tts`, `xtts-v2` (CPML non-commercial), `chatterbox`, `qwen3-tts`
+- `voxcpm2` (experimental, local-worker opt-in; Hindi + English)
### Advanced / externally managed
@@ -139,14 +140,15 @@ Full matrix (CPU/GPU, Docker defaults, licences, verification labels):
| Engine | Clone type | CPU | GPU | Default CPU image | Licence note | Verification |
|--------|------------|-----|-----|-------------------|--------------|--------------|
| `openvoice-v2` | Zero-shot | Yes | Yes | Yes | MIT VC + verify YourTTS base | Verified (CPU e2e) |
-| `f5-tts` | Zero-shot | Slow | Recommended | Yes | Apache-2.0 / CC (upstream) | Verified (CPU e2e) |
+| `f5-tts` | Zero-shot | Slow | Recommended | Yes | **CC-BY-NC pretrained weights** | Verified (CPU e2e) |
| `xtts-v2` | Zero-shot | Slow | Recommended | Yes | **CPML non-commercial** | Verified (CPU e2e) |
-| `chatterbox` | Zero-shot | Worker | Optional | Yes (worker venv) | MIT | Smoke-documented |
-| `qwen3-tts` | Zero-shot | Limited | Recommended | Yes | Apache-2.0 | Smoke-documented |
+| `chatterbox` | Zero-shot | Worker/MPS | Optional | Worker only; local weights required | MIT | Multilingual V3 wiring; inference unverified |
+| `qwen3-tts` | Zero-shot | Limited/MPS experimental | Recommended | Yes; local snapshot required | Apache-2.0 | Smoke-documented |
+| `voxcpm2` | Zero-shot | Experimental MPS | Recommended | No | Apache-2.0 | Disabled-by-default scaffold |
| `rvc` | Trained | No | Required | No (GPU image) | MIT architecture | GPU verification needed |
-| `fish-speech` | Sidecar | Depends | Recommended | External | Check upstream | External sidecar |
+| `fish-speech` | Sidecar | Depends | Recommended | External | Fish Audio Research License | Digest-pinned sidecar required |
| `cosyvoice-3` | Zero-shot | Limited | Recommended | Extra install | Apache-2.0 | GPU verification needed |
-| `indextts-2` | Zero-shot | Limited | Recommended | Extra install | Check upstream | GPU verification needed |
+| `indextts-2` | Zero-shot | Limited | Recommended | Extra install | Custom Bilibili model license | GPU verification needed |
GPU Docker paths were written carefully but **not** run against real NVIDIA
hardware during development — please verify on your host.
@@ -172,7 +174,8 @@ After the first build, `make start-cpu` (or compose `up` without `--build`) is
enough. Helpers: `make stop`, `make logs`, `make smoke-openvoice`.
Compose caps the CPU service at **4 CPUs / 6 GB RAM** by default. Data → `./data`;
-weights → `models-cache` Docker volume.
+weights → `models-cache` Docker volume. The host port is published on
+`127.0.0.1` only.
**First clone:** follow **[docs/FIRST_CLONE.md](docs/FIRST_CLONE.md)**
(sample length, quiet room, expected download time, deletion).
@@ -338,6 +341,8 @@ Issue/PR templates live under `.github/`.
| [FIRST_CLONE.md](docs/FIRST_CLONE.md) | First successful clone |
| [ENGINES.md](docs/ENGINES.md) | Support matrix |
| [ENGINE_LICENSING.md](docs/ENGINE_LICENSING.md) | Licence matrix |
+| [EXPERIMENTAL_STUDIO.md](docs/EXPERIMENTAL_STUDIO.md) | Mac, Docker CPU, Colab/Kaggle profiles and roadmap |
+| [BENCHMARKING.md](docs/BENCHMARKING.md) | English/Hindi/Hinglish benchmark harness |
| [RESPONSIBLE_USE.md](RESPONSIBLE_USE.md) / [CONSENT.md](docs/CONSENT.md) | Safety & consent |
| [WATERMARKING.md](docs/WATERMARKING.md) | Fingerprint limitations |
| [SELF_HOSTING.md](docs/SELF_HOSTING.md) | Deploy securely |
@@ -359,7 +364,9 @@ docker compose -f docker/docker-compose.yml config
```
Unit tests use a `FakeEngine` (no multi-GB downloads). Optional ML extras are
-in `requirements-*.txt` / `pyproject.toml` optional deps.
+in `requirements-*.txt` / `pyproject.toml` optional deps. `GET /v1/providers`
+returns model revisions, license gates, runtime support, and integration state
+without importing an ML SDK or contacting a model hub.
## Repo layout
diff --git a/app/api/providers.py b/app/api/providers.py
new file mode 100644
index 0000000..91d1218
--- /dev/null
+++ b/app/api/providers.py
@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from fastapi import APIRouter, Depends
+
+from app.providers.models import ProviderStatus
+from app.providers.registry import provider_statuses
+from app.security import auth_dependency
+
+router = APIRouter(
+ prefix="/v1/providers",
+ tags=["providers"],
+ dependencies=[Depends(auth_dependency)],
+)
+
+
+@router.get("", response_model=list[ProviderStatus])
+async def get_providers() -> list[ProviderStatus]:
+ """List engine and audio-intelligence manifests without loading models."""
+
+ return provider_statuses()
diff --git a/app/api/synth.py b/app/api/synth.py
index 7b626a9..32013d1 100644
--- a/app/api/synth.py
+++ b/app/api/synth.py
@@ -53,6 +53,7 @@ async def synthesize(
sample_rate=body.sample_rate,
speed=body.speed,
language=body.language or voice.language,
+ style=body.style,
)
async with get_job_limiter():
diff --git a/app/audio_intelligence/__init__.py b/app/audio_intelligence/__init__.py
new file mode 100644
index 0000000..9ca7f30
--- /dev/null
+++ b/app/audio_intelligence/__init__.py
@@ -0,0 +1 @@
+"""Local audio-intelligence provider contracts and adapters."""
diff --git a/app/audio_intelligence/base.py b/app/audio_intelligence/base.py
new file mode 100644
index 0000000..c300ff3
--- /dev/null
+++ b/app/audio_intelligence/base.py
@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Protocol, runtime_checkable
+
+from pydantic import BaseModel, Field
+
+
+class AlignmentSegment(BaseModel):
+ text: str
+ start_seconds: float
+ end_seconds: float
+
+
+class TranscriptionResult(BaseModel):
+ text: str
+ language: str | None = None
+ timestamps: list[AlignmentSegment] = Field(default_factory=list)
+
+
+@runtime_checkable
+class AudioIntelligenceProvider(Protocol):
+ id: str
+
+ def is_ready(self) -> bool: ...
+
+ def transcribe(
+ self,
+ audio_path: Path,
+ *,
+ language: str | None = None,
+ timestamps: bool = False,
+ ) -> TranscriptionResult: ...
+
+ def align(
+ self,
+ audio_path: Path,
+ *,
+ text: str,
+ language: str,
+ ) -> list[AlignmentSegment]: ...
diff --git a/app/audio_intelligence/qwen3_asr.py b/app/audio_intelligence/qwen3_asr.py
new file mode 100644
index 0000000..1cafbf9
--- /dev/null
+++ b/app/audio_intelligence/qwen3_asr.py
@@ -0,0 +1,132 @@
+"""Qwen3-ASR reference transcription through an isolated, local-only worker."""
+
+from __future__ import annotations
+
+import json
+import subprocess
+import tempfile
+from pathlib import Path
+
+from app.audio_intelligence.base import AlignmentSegment, TranscriptionResult
+from app.config import get_settings
+from app.engines.base import EngineError
+from app.engines.subprocess_env import sanitized_subprocess_env, worker_exec_command
+from app.runtime_device import resolve_torch_device
+
+_WORKER_SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "qwen3_asr_worker.py"
+
+
+class Qwen3AsrProvider:
+ id = "qwen3-asr"
+
+ def is_ready(self) -> bool:
+ settings = get_settings()
+ return (
+ settings.qwen3_asr_python is not None
+ and settings.qwen3_asr_python.is_file()
+ and settings.qwen3_asr_model_dir is not None
+ and settings.qwen3_asr_model_dir.is_dir()
+ and _WORKER_SCRIPT.is_file()
+ )
+
+ def _invoke(self, args: list[str]) -> dict:
+ settings = get_settings()
+ python = settings.qwen3_asr_python
+ if python is None or not python.is_file() or not self.is_ready():
+ raise EngineError(
+ "Qwen3-ASR is not configured; set VOICEFORGE_QWEN3_ASR_PYTHON "
+ "and VOICEFORGE_QWEN3_ASR_MODEL_DIR to an isolated worker and "
+ "an explicitly downloaded local snapshot"
+ )
+
+ settings.data_dir.mkdir(parents=True, exist_ok=True)
+ with tempfile.NamedTemporaryFile(
+ prefix="qwen3-asr-",
+ suffix=".json",
+ dir=settings.data_dir,
+ delete=False,
+ ) as tmp:
+ output_path = Path(tmp.name)
+ try:
+ command = worker_exec_command(
+ python,
+ _WORKER_SCRIPT,
+ [*args, "--output-json", str(output_path)],
+ )
+ proc = subprocess.run( # noqa: S603 - argv contains validated local paths only
+ command,
+ check=False,
+ capture_output=True,
+ text=True,
+ timeout=600,
+ env=sanitized_subprocess_env(),
+ )
+ if proc.returncode != 0:
+ detail = (proc.stdout or proc.stderr or "worker failed").strip()
+ raise EngineError(f"Qwen3-ASR worker failed: {detail[-1000:]}")
+ return json.loads(output_path.read_text(encoding="utf-8"))
+ except subprocess.TimeoutExpired as exc:
+ raise EngineError("Qwen3-ASR worker timed out after 600 seconds") from exc
+ finally:
+ output_path.unlink(missing_ok=True)
+
+ def transcribe(
+ self,
+ audio_path: Path,
+ *,
+ language: str | None = None,
+ timestamps: bool = False,
+ ) -> TranscriptionResult:
+ settings = get_settings()
+ if not audio_path.is_file():
+ raise EngineError(f"Reference audio not found: {audio_path}")
+ assert settings.qwen3_asr_model_dir is not None
+ args = [
+ "transcribe",
+ "--audio",
+ str(audio_path),
+ "--model-dir",
+ str(settings.qwen3_asr_model_dir),
+ "--device",
+ resolve_torch_device(settings.device),
+ ]
+ if language:
+ args.extend(["--language", language])
+ if timestamps:
+ aligner_dir = settings.qwen3_aligner_model_dir
+ if aligner_dir is None or not aligner_dir.is_dir():
+ raise EngineError(
+ "Qwen3-ASR timestamps require VOICEFORGE_QWEN3_ALIGNER_MODEL_DIR"
+ )
+ args.extend(["--timestamps", "--aligner-dir", str(aligner_dir)])
+ return TranscriptionResult.model_validate(self._invoke(args))
+
+ def align(
+ self,
+ audio_path: Path,
+ *,
+ text: str,
+ language: str,
+ ) -> list[AlignmentSegment]:
+ settings = get_settings()
+ aligner_dir = settings.qwen3_aligner_model_dir
+ if aligner_dir is None or not aligner_dir.is_dir():
+ raise EngineError(
+ "Qwen3 forced alignment requires VOICEFORGE_QWEN3_ALIGNER_MODEL_DIR"
+ )
+ payload = self._invoke(
+ [
+ "align",
+ "--audio",
+ str(audio_path),
+ "--text",
+ text,
+ "--language",
+ language,
+ "--aligner-dir",
+ str(aligner_dir),
+ "--device",
+ resolve_torch_device(settings.device),
+ ]
+ )
+ return [AlignmentSegment.model_validate(item) for item in payload["timestamps"]]
diff --git a/app/benchmarking.py b/app/benchmarking.py
new file mode 100644
index 0000000..5db38c3
--- /dev/null
+++ b/app/benchmarking.py
@@ -0,0 +1,229 @@
+"""Model-independent contracts for repeatable VoiceForge benchmarks."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import unicodedata
+from collections import defaultdict
+from pathlib import Path
+from typing import Literal
+
+import numpy as np
+import soundfile as sf
+from pydantic import BaseModel, Field
+
+LanguageGroup = Literal["english", "hindi-devanagari", "hindi-romanized", "hinglish"]
+
+
+class BenchmarkCase(BaseModel):
+ id: str = Field(pattern=r"^[a-z0-9][a-z0-9-]+$")
+ language_group: LanguageGroup
+ language: str
+ text: str = Field(min_length=1)
+ emotion: str = "neutral"
+ tags: list[str] = Field(default_factory=list)
+
+
+class BenchmarkPlan(BaseModel):
+ schema_version: str = "voiceforge-benchmark/v1"
+ provider_id: str
+ model_revision: str
+ seed: int
+ hardware: str
+ suite_sha256: str
+ cases: list[BenchmarkCase]
+
+
+class BenchmarkObservation(BaseModel):
+ case_id: str
+ generated_audio: Path
+ hypothesis: str | None = None
+ latency_seconds: float | None = Field(default=None, ge=0)
+ peak_memory_mb: float | None = Field(default=None, ge=0)
+ speaker_similarity: float | None = Field(default=None, ge=-1, le=1)
+ emotion_score: float | None = Field(default=None, ge=1, le=5)
+ notes: str | None = None
+
+
+class CaseScore(BaseModel):
+ case_id: str
+ language_group: LanguageGroup
+ duration_seconds: float
+ peak_dbfs: float | None
+ clipped_fraction: float
+ wer: float | None = None
+ cer: float | None = None
+ latency_seconds: float | None = None
+ real_time_factor: float | None = None
+ peak_memory_mb: float | None = None
+ speaker_similarity: float | None = None
+ emotion_score: float | None = None
+
+
+def load_cases(path: Path) -> list[BenchmarkCase]:
+ cases: list[BenchmarkCase] = []
+ ids: set[str] = set()
+ for line_number, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
+ if not raw.strip() or raw.lstrip().startswith("#"):
+ continue
+ case = BenchmarkCase.model_validate_json(raw)
+ if case.id in ids:
+ raise ValueError(f"Duplicate benchmark id '{case.id}' on line {line_number}")
+ ids.add(case.id)
+ cases.append(case)
+ if not cases:
+ raise ValueError("Benchmark suite contains no cases")
+ return cases
+
+
+def suite_sha256(cases: list[BenchmarkCase]) -> str:
+ canonical = "\n".join(
+ case.model_dump_json(exclude_none=True) for case in sorted(cases, key=lambda item: item.id)
+ )
+ return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
+
+
+def build_plan(
+ *,
+ provider_id: str,
+ model_revision: str,
+ seed: int,
+ hardware: str,
+ cases: list[BenchmarkCase],
+) -> BenchmarkPlan:
+ return BenchmarkPlan(
+ provider_id=provider_id,
+ model_revision=model_revision,
+ seed=seed,
+ hardware=hardware,
+ suite_sha256=suite_sha256(cases),
+ cases=cases,
+ )
+
+
+def _normalize(text: str) -> str:
+ return unicodedata.normalize("NFKC", text).lower().strip()
+
+
+def _word_tokens(text: str) -> list[str]:
+ return re.findall(r"[\w']+", _normalize(text), flags=re.UNICODE)
+
+
+def _characters(text: str) -> list[str]:
+ return [
+ char
+ for char in _normalize(text)
+ if unicodedata.category(char)[0] in {"L", "M", "N"}
+ ]
+
+
+def _error_rate(reference: list[str], hypothesis: list[str]) -> float:
+ if not reference:
+ return 0.0 if not hypothesis else 1.0
+ previous = list(range(len(hypothesis) + 1))
+ for row, ref_item in enumerate(reference, start=1):
+ current = [row]
+ for col, hyp_item in enumerate(hypothesis, start=1):
+ current.append(
+ min(
+ current[-1] + 1,
+ previous[col] + 1,
+ previous[col - 1] + (ref_item != hyp_item),
+ )
+ )
+ previous = current
+ return previous[-1] / len(reference)
+
+
+def word_error_rate(reference: str, hypothesis: str) -> float:
+ return _error_rate(_word_tokens(reference), _word_tokens(hypothesis))
+
+
+def character_error_rate(reference: str, hypothesis: str) -> float:
+ return _error_rate(_characters(reference), _characters(hypothesis))
+
+
+def score_observation(case: BenchmarkCase, observation: BenchmarkObservation) -> CaseScore:
+ if not observation.generated_audio.is_file():
+ raise ValueError(f"Generated audio not found: {observation.generated_audio}")
+ wav, sample_rate = sf.read(
+ str(observation.generated_audio),
+ dtype="float32",
+ always_2d=True,
+ )
+ mono = np.asarray(wav, dtype=np.float32).mean(axis=1)
+ duration = len(mono) / sample_rate if sample_rate else 0.0
+ peak = float(np.max(np.abs(mono))) if len(mono) else 0.0
+ peak_dbfs = float(20 * np.log10(peak)) if peak > 0 else None
+ clipped = float(np.mean(np.abs(mono) >= 0.999)) if len(mono) else 0.0
+ wer = cer = None
+ if observation.hypothesis is not None:
+ wer = word_error_rate(case.text, observation.hypothesis)
+ cer = character_error_rate(case.text, observation.hypothesis)
+ rtf = None
+ if observation.latency_seconds is not None and duration > 0:
+ rtf = observation.latency_seconds / duration
+ return CaseScore(
+ case_id=case.id,
+ language_group=case.language_group,
+ duration_seconds=round(duration, 4),
+ peak_dbfs=round(peak_dbfs, 4) if peak_dbfs is not None else None,
+ clipped_fraction=round(clipped, 6),
+ wer=round(wer, 6) if wer is not None else None,
+ cer=round(cer, 6) if cer is not None else None,
+ latency_seconds=observation.latency_seconds,
+ real_time_factor=round(rtf, 6) if rtf is not None else None,
+ peak_memory_mb=observation.peak_memory_mb,
+ speaker_similarity=observation.speaker_similarity,
+ emotion_score=observation.emotion_score,
+ )
+
+
+def summarize(scores: list[CaseScore]) -> dict:
+ groups: dict[str, list[CaseScore]] = defaultdict(list)
+ for score in scores:
+ groups[score.language_group].append(score)
+
+ def mean(items: list[float | None]) -> float | None:
+ values = [item for item in items if item is not None]
+ return round(sum(values) / len(values), 6) if values else None
+
+ return {
+ "cases_scored": len(scores),
+ "groups": {
+ name: {
+ "count": len(items),
+ "mean_wer": mean([item.wer for item in items]),
+ "mean_cer": mean([item.cer for item in items]),
+ "mean_rtf": mean([item.real_time_factor for item in items]),
+ "mean_speaker_similarity": mean(
+ [item.speaker_similarity for item in items]
+ ),
+ "mean_emotion_score": mean([item.emotion_score for item in items]),
+ }
+ for name, items in sorted(groups.items())
+ },
+ }
+
+
+def load_observations(path: Path) -> list[BenchmarkObservation]:
+ return [
+ BenchmarkObservation.model_validate_json(line)
+ for line in path.read_text(encoding="utf-8").splitlines()
+ if line.strip() and not line.lstrip().startswith("#")
+ ]
+
+
+def write_json(path: Path, payload: BaseModel | dict | list) -> None:
+ if isinstance(payload, BaseModel):
+ data = payload.model_dump(mode="json")
+ elif isinstance(payload, list):
+ data = [
+ item.model_dump(mode="json") if isinstance(item, BaseModel) else item
+ for item in payload
+ ]
+ else:
+ data = payload
+ path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
diff --git a/app/config.py b/app/config.py
index d369f87..e2ea2a2 100644
--- a/app/config.py
+++ b/app/config.py
@@ -20,7 +20,9 @@ class Settings(BaseSettings):
extra="ignore",
)
- host: str = "0.0.0.0"
+ # Local-first default. Containers explicitly bind 0.0.0.0 internally but
+ # Compose publishes the port on 127.0.0.1 only.
+ host: str = "127.0.0.1"
port: int = 8089
# Bearer token required on every /v1/* request when set. Leave unset only
@@ -35,7 +37,7 @@ class Settings(BaseSettings):
data_dir: Path = Path("data")
models_dir: Path = Path("models")
- # "cpu", "cuda", or "auto" (use CUDA if torch reports it's available).
+ # "cpu", "cuda", "mps", or "auto" (CUDA, then Apple MPS, then CPU).
device: str = "auto"
# Upload / request limits — deliberately conservative for a local-first,
@@ -67,6 +69,24 @@ class Settings(BaseSettings):
cosyvoice_python: Path | None = None
indextts_python: Path | None = None
chatterbox_python: Path | None = None
+ # Workers only load explicitly downloaded local directories. Their
+ # revisions are recorded here and by GET /v1/providers.
+ chatterbox_model_dir: Path | None = None
+ chatterbox_model_revision: str = "5bb1f6e"
+ qwen3_tts_model_dir: Path | None = None
+ qwen3_tts_model_revision: str = "fd4b254"
+ voxcpm2_python: Path | None = None
+ voxcpm2_model_dir: Path | None = None
+ voxcpm2_model_revision: str = "9454c2d"
+
+ # Reference transcription backend. "whisper" preserves the existing
+ # in-process path; qwen3-asr requires an explicit worker + local model.
+ reference_asr_provider: str = "whisper"
+ qwen3_asr_python: Path | None = None
+ qwen3_asr_model_dir: Path | None = None
+ qwen3_asr_model_revision: str = "5eb1441"
+ qwen3_aligner_model_dir: Path | None = None
+ qwen3_aligner_model_revision: str = "c7cbfc2"
# Comma-separated engine ids to enable for clone/synth in this deployment.
enabled_engines: str = ""
diff --git a/app/engine_readiness.py b/app/engine_readiness.py
index de3f6e6..a41e6f0 100644
--- a/app/engine_readiness.py
+++ b/app/engine_readiness.py
@@ -30,6 +30,12 @@ def is_engine_ready_cached(engine_id: str) -> bool:
return _ready.get(engine_id, False)
+def peek_engine_ready(engine_id: str) -> bool:
+ """Read an existing readiness result without probing engines."""
+
+ return _ready.get(engine_id, False)
+
+
def get_health_counts() -> tuple[int, int]:
if _counts is None:
refresh_readiness()
diff --git a/app/engines/asr.py b/app/engines/asr.py
index c6f7370..eda7def 100644
--- a/app/engines/asr.py
+++ b/app/engines/asr.py
@@ -53,6 +53,23 @@ def transcribe_reference_audio(
"""Transcribe reference audio without torchcodec (CPU Docker safe)."""
global _asr_processor, _asr_model
+ from app.config import get_settings
+
+ provider_id = get_settings().reference_asr_provider
+ if provider_id == "qwen3-asr":
+ from app.audio_intelligence.qwen3_asr import Qwen3AsrProvider
+
+ result = Qwen3AsrProvider().transcribe(
+ Path(ref_path),
+ language=language,
+ timestamps=False,
+ )
+ return result.text
+ if provider_id != "whisper":
+ raise ValueError(
+ f"Unsupported reference ASR provider '{provider_id}'; expected whisper or qwen3-asr"
+ )
+
wav, sr = sf.read(str(ref_path), dtype="float32", always_2d=False)
if getattr(wav, "ndim", 1) > 1:
wav = wav.mean(axis=1)
diff --git a/app/engines/base.py b/app/engines/base.py
index 8561056..8507119 100644
--- a/app/engines/base.py
+++ b/app/engines/base.py
@@ -33,6 +33,7 @@ class SynthesizeOptions(BaseModel):
sample_rate: int | None = None
speed: float | None = None
language: str | None = None
+ style: str | None = None
class VoiceArtifact(BaseModel):
diff --git a/app/engines/chatterbox.py b/app/engines/chatterbox.py
index 20fcde7..2c63aaa 100644
--- a/app/engines/chatterbox.py
+++ b/app/engines/chatterbox.py
@@ -1,4 +1,4 @@
-"""Chatterbox (Resemble AI) — MIT zero-shot cloning via an isolated worker.
+"""Chatterbox Multilingual V3 — MIT zero-shot cloning via an isolated worker.
``chatterbox-tts`` pins numpy<2 and older torch, which conflict with
coqui-tts in the main VoiceForge venv. Inference runs under
@@ -35,9 +35,8 @@
CHATTERBOX_NATIVE_SAMPLE_RATE = 24000
SUPPORTED_LANGUAGES = [
- "en", "es", "fr", "de", "it", "pt", "pl", "tr", "ru", "nl",
- "cs", "ar", "zh", "ja", "hu", "ko", "hi", "sv", "da", "fi",
- "no", "ro", "el",
+ "ar", "da", "de", "el", "en", "es", "fi", "fr", "he", "hi", "it", "ja",
+ "ko", "ms", "nl", "no", "pl", "pt", "ru", "sv", "sw", "tr", "zh",
]
@@ -53,12 +52,25 @@ def resolve_chatterbox_python() -> Path | None:
def chatterbox_worker_configured() -> bool:
- return resolve_chatterbox_python() is not None and _WORKER_SCRIPT.is_file()
+ return (
+ resolve_chatterbox_python() is not None
+ and resolve_chatterbox_model_dir() is not None
+ and _WORKER_SCRIPT.is_file()
+ )
+
+
+def resolve_chatterbox_model_dir() -> Path | None:
+ settings = get_settings()
+ if settings.chatterbox_model_dir:
+ path = Path(settings.chatterbox_model_dir)
+ return path if path.is_dir() else None
+ default = settings.models_dir / "chatterbox"
+ return default if default.is_dir() else None
class ChatterboxEngine:
id = "chatterbox"
- label = "Chatterbox (Resemble AI)"
+ label = "Chatterbox Multilingual V3"
capabilities = CloneCapabilities(
zero_shot=True,
fine_tunable=False,
@@ -70,17 +82,6 @@ class ChatterboxEngine:
approx_vram_gb=4.0,
)
- def _resolve_device(self) -> str:
- settings = get_settings()
- if settings.device != "auto":
- return settings.device
- try:
- import torch
-
- return "cuda" if torch.cuda.is_available() else "cpu"
- except Exception:
- return "cpu"
-
def is_ready(self) -> bool:
return chatterbox_worker_configured()
@@ -98,8 +99,13 @@ async def create_voice(
raise EngineError(
"Chatterbox worker is not configured — set "
"VOICEFORGE_CHATTERBOX_PYTHON to an interpreter with "
- "chatterbox-tts installed (see requirements-chatterbox.txt / "
- "CPU Docker /opt/chatterbox-venv)"
+ "chatterbox-tts installed and VOICEFORGE_CHATTERBOX_MODEL_DIR "
+ "to an explicitly downloaded local model snapshot"
+ )
+ normalized_language = (language or "en").split("-")[0].lower()
+ if normalized_language not in SUPPORTED_LANGUAGES:
+ raise EngineError(
+ f"Language '{language}' is not supported by Chatterbox Multilingual V3"
)
async def report(msg: str, extra: dict | None = None) -> None:
@@ -149,6 +155,7 @@ async def synthesize(
ref_audio=ref_path,
text=text,
output=out_wav,
+ language=(opts.language or artifact.data.get("language") or "en").split("-")[0],
)
if not out_wav.is_file():
diff --git a/app/engines/chatterbox_daemon.py b/app/engines/chatterbox_daemon.py
index 418353e..ed90e1f 100644
--- a/app/engines/chatterbox_daemon.py
+++ b/app/engines/chatterbox_daemon.py
@@ -10,6 +10,7 @@
from app.config import get_settings
from app.engines.base import EngineError
from app.engines.subprocess_env import sanitized_subprocess_env, worker_exec_command
+from app.runtime_device import resolve_torch_device
logger = logging.getLogger("voiceforge.engines.chatterbox_daemon")
@@ -26,6 +27,15 @@ def resolve_chatterbox_python() -> Path | None:
return default
return None
+
+def resolve_chatterbox_model_dir() -> Path | None:
+ settings = get_settings()
+ if settings.chatterbox_model_dir:
+ path = Path(settings.chatterbox_model_dir)
+ return path if path.is_dir() else None
+ default = settings.models_dir / "chatterbox"
+ return default if default.is_dir() else None
+
_proc: asyncio.subprocess.Process | None = None
_lock: asyncio.Lock | None = None
@@ -38,15 +48,7 @@ def _get_lock() -> asyncio.Lock:
def _resolve_device() -> str:
- settings = get_settings()
- if settings.device != "auto":
- return settings.device
- try:
- import torch
-
- return "cuda" if torch.cuda.is_available() else "cpu"
- except Exception:
- return "cpu"
+ return resolve_torch_device(get_settings().device)
async def _read_line(proc: asyncio.subprocess.Process) -> str:
@@ -81,11 +83,24 @@ async def ensure_chatterbox_daemon() -> None:
return
python = resolve_chatterbox_python()
- if python is None or not _WORKER_SCRIPT.is_file():
+ model_dir = resolve_chatterbox_model_dir()
+ if python is None or model_dir is None or not _WORKER_SCRIPT.is_file():
raise EngineError("Chatterbox worker is not configured")
device = _resolve_device()
- cmd = worker_exec_command(python, _WORKER_SCRIPT, ["serve", "--device", device])
+ cmd = worker_exec_command(
+ python,
+ _WORKER_SCRIPT,
+ [
+ "serve",
+ "--device",
+ device,
+ "--model-dir",
+ str(model_dir),
+ "--t3-model",
+ "v3",
+ ],
+ )
logger.info("Starting Chatterbox daemon: %s", " ".join(cmd))
_proc = await asyncio.create_subprocess_exec(
*cmd,
@@ -102,6 +117,7 @@ async def synthesize_via_daemon(
ref_audio: Path,
text: str,
output: Path,
+ language: str,
) -> None:
global _proc
await ensure_chatterbox_daemon()
@@ -113,6 +129,7 @@ async def synthesize_via_daemon(
"ref_audio": str(ref_audio),
"text": text,
"output": str(output),
+ "language": language,
}
)
async with _get_lock():
diff --git a/app/engines/f5_tts.py b/app/engines/f5_tts.py
index c1aa7ed..b18dc2d 100644
--- a/app/engines/f5_tts.py
+++ b/app/engines/f5_tts.py
@@ -1,6 +1,7 @@
"""F5-TTS engine — zero-shot cloning via the SWivid diffusion-transformer model.
-Apache-2.0 / CC licensed weights (permissive vs XTTS-v2's CPML). Requires a
+The application adapter is MIT; the official pretrained weights are
+CC-BY-NC-4.0. Requires a
transcript of the reference audio (`ref_text`); VoiceForge auto-transcribes
the uploaded sample at voice-creation time and caches it in the artifact.
@@ -54,7 +55,7 @@ class F5TtsEngine:
recommended_sample_seconds=12.0,
languages=SUPPORTED_LANGUAGES,
requires_gpu=False,
- license="Apache-2.0 / CC (model weights) — see SWivid/F5-TTS repo",
+ license="CC-BY-NC-4.0 pretrained weights (F5-TTS code is MIT)",
approx_vram_gb=4.0,
)
diff --git a/app/engines/fish_speech.py b/app/engines/fish_speech.py
index f1fd8fc..ee150d3 100644
--- a/app/engines/fish_speech.py
+++ b/app/engines/fish_speech.py
@@ -32,10 +32,10 @@
logger = logging.getLogger("voiceforge.engines.fish_speech")
-SUPPORTED_LANGUAGES = [
- "en", "zh", "ja", "ko", "de", "fr", "es", "it", "pt", "ru",
- "ar", "hi", "th", "vi", "id", "nl", "pl", "tr",
-]
+# Upstream currently documents 50+ languages but does not publish a stable,
+# exhaustive ISO list on its primary project page. Keep the API conservative
+# rather than claiming Hindi or other languages that were not verified.
+SUPPORTED_LANGUAGES: list[str] = []
def _validate_sidecar_url(url: str) -> str:
@@ -58,7 +58,7 @@ class FishSpeechEngine:
recommended_sample_seconds=15.0,
languages=SUPPORTED_LANGUAGES,
requires_gpu=False,
- license="Check Fish Audio / fish-speech upstream (open weights; not cloud API)",
+ license="Fish Audio Research License (noncommercial/research weights)",
approx_vram_gb=8.0,
)
diff --git a/app/engines/indextts_2.py b/app/engines/indextts_2.py
index 97be95a..a2f65aa 100644
--- a/app/engines/indextts_2.py
+++ b/app/engines/indextts_2.py
@@ -57,7 +57,7 @@ class IndexTts2Engine:
recommended_sample_seconds=8.0,
languages=SUPPORTED_LANGUAGES,
requires_gpu=True,
- license="Check IndexTTS / IndexTeam upstream license before commercial use",
+ license="Bilibili IndexTTS Model License (custom terms; commercial contact)",
approx_vram_gb=8.0,
)
diff --git a/app/engines/qwen3_tts.py b/app/engines/qwen3_tts.py
index 56d6b5c..0993ed7 100644
--- a/app/engines/qwen3_tts.py
+++ b/app/engines/qwen3_tts.py
@@ -9,7 +9,6 @@
import asyncio
import logging
-import os
import shutil
from pathlib import Path
@@ -26,11 +25,14 @@
VoiceArtifact,
)
from app.engines.wav_output import to_wav_bytes
+from app.providers.registry import QWEN3_TTS_REVISION
+from app.runtime_device import resolve_torch_device, transformers_device_map
from app.storage import artifacts_dir
logger = logging.getLogger("voiceforge.engines.qwen3_tts")
QWEN3_MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
+QWEN3_MODEL_REVISION = QWEN3_TTS_REVISION
QWEN3_NATIVE_SAMPLE_RATE = 24000
SUPPORTED_LANGUAGES = [
@@ -77,19 +79,14 @@ def _lock(self) -> asyncio.Lock:
return self._load_lock
def _resolve_device(self) -> str:
- settings = get_settings()
- if settings.device != "auto":
- return settings.device
- try:
- import torch
-
- return "cuda" if torch.cuda.is_available() else "cpu"
- except Exception:
- return "cpu"
+ return resolve_torch_device(get_settings().device)
def is_ready(self) -> bool:
if self._model is not None:
return True
+ model_dir = get_settings().qwen3_tts_model_dir
+ if model_dir is None or not model_dir.is_dir():
+ return False
try:
import qwen_tts # noqa: F401
except Exception:
@@ -107,8 +104,13 @@ async def _ensure_loaded(self):
return self._model
settings = get_settings()
- settings.models_dir.mkdir(parents=True, exist_ok=True)
- os.environ.setdefault("HF_HOME", str(settings.models_dir))
+ model_dir = settings.qwen3_tts_model_dir
+ if model_dir is None or not model_dir.is_dir():
+ raise EngineError(
+ "Qwen3-TTS is not configured; set "
+ "VOICEFORGE_QWEN3_TTS_MODEL_DIR to an explicitly downloaded "
+ "local snapshot"
+ )
def _load():
try:
@@ -120,22 +122,20 @@ def _load():
"'qwen3' extra or use requirements-qwen3.txt."
) from exc
device = self._resolve_device()
- device_map = device if device == "cpu" else "cuda:0"
- # bfloat16 on CPU roughly halves the 1.7B model's resident
- # memory footprint (~7GB fp32 -> ~3.5GB) so it fits comfortably
- # under the container's memory limit alongside the Whisper ASR
- # model; modern CPUs handle bf16 matmuls fine, just slower than
- # fp32 for the tensor-core-optimized case (irrelevant on CPU).
- dtype = torch.bfloat16
+ device_map = transformers_device_map(device)
+ # Keep CPU/MPS on fp32 until upstream documents a stable lower
+ # precision path. CUDA uses bfloat16 as in the official examples.
+ dtype = torch.bfloat16 if device == "cuda" else torch.float32
logger.info(
"Loading Qwen3-TTS (%s) on device_map=%s",
- QWEN3_MODEL,
+ model_dir,
device_map,
)
return Qwen3TTSModel.from_pretrained(
- QWEN3_MODEL,
+ str(model_dir),
device_map=device_map,
dtype=dtype,
+ local_files_only=True,
)
loop = asyncio.get_running_loop()
diff --git a/app/engines/registry.py b/app/engines/registry.py
index 370dfd6..4e145b4 100644
--- a/app/engines/registry.py
+++ b/app/engines/registry.py
@@ -19,6 +19,7 @@
from app.engines.openvoice_v2 import OpenVoiceV2Engine
from app.engines.qwen3_tts import Qwen3TtsEngine
from app.engines.rvc import RvcEngine
+from app.engines.voxcpm2 import VoxCpm2Engine
from app.engines.xtts_v2 import XttsV2Engine
_FACTORIES: dict[str, Callable[[], CloneEngine]] = {
@@ -31,9 +32,11 @@
"fish-speech": FishSpeechEngine,
"cosyvoice-3": CosyVoice3Engine,
"indextts-2": IndexTts2Engine,
+ "voxcpm2": VoxCpm2Engine,
}
_instances: dict[str, CloneEngine] = {}
+_DEFAULT_DISABLED = {"voxcpm2"}
class UnknownEngineError(KeyError):
@@ -45,7 +48,7 @@ def is_engine_enabled(engine_id: str) -> bool:
return False
allowed = get_settings().enabled_engine_ids
if allowed is None:
- return True
+ return engine_id not in _DEFAULT_DISABLED
return engine_id in allowed
diff --git a/app/engines/subprocess_env.py b/app/engines/subprocess_env.py
index 889ac3c..d365208 100644
--- a/app/engines/subprocess_env.py
+++ b/app/engines/subprocess_env.py
@@ -15,6 +15,11 @@
def sanitized_subprocess_env() -> dict[str, str]:
env = os.environ.copy()
env["PYTHONHASHSEED"] = "0"
+ # Inference workers receive local paths and must never contact a model hub.
+ # Explicit setup/download commands are launched separately without this
+ # environment.
+ env["HF_HUB_OFFLINE"] = "1"
+ env["TRANSFORMERS_OFFLINE"] = "1"
return env
diff --git a/app/engines/voxcpm2.py b/app/engines/voxcpm2.py
new file mode 100644
index 0000000..d9f0cba
--- /dev/null
+++ b/app/engines/voxcpm2.py
@@ -0,0 +1,189 @@
+"""VoxCPM2 experimental clone provider through an opt-in isolated worker."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import shutil
+from pathlib import Path
+
+import numpy as np
+import soundfile as sf
+
+from app.config import get_settings
+from app.engines.asr import map_asr_language, pick_longest_sample, transcribe_reference_audio
+from app.engines.base import (
+ CloneCapabilities,
+ EngineError,
+ ProgressFn,
+ SynthesizeOptions,
+ Tier,
+ VoiceArtifact,
+)
+from app.engines.wav_output import to_wav_bytes
+from app.engines.worker_client import run_worker
+from app.runtime_device import resolve_torch_device
+from app.storage import artifacts_dir
+
+logger = logging.getLogger("voiceforge.engines.voxcpm2")
+
+_WORKER_SCRIPT = Path(__file__).resolve().parents[2] / "scripts" / "voxcpm2_worker.py"
+VOXCPM2_NATIVE_SAMPLE_RATE = 48000
+SUPPORTED_LANGUAGES = [
+ "ar", "my", "zh", "da", "nl", "en", "fi", "fr", "de", "el", "he", "hi", "id", "it",
+ "ja", "km", "ko", "lo", "ms", "no", "pl", "pt", "ru", "es", "sw", "sv", "tl", "th",
+ "tr", "vi",
+]
+
+
+def resolve_voxcpm2_python() -> Path | None:
+ settings = get_settings()
+ if settings.voxcpm2_python:
+ path = Path(settings.voxcpm2_python)
+ return path if path.is_file() else None
+ default = Path("/opt/voxcpm2-venv/bin/python")
+ return default if default.is_file() else None
+
+
+def resolve_voxcpm2_model_dir() -> Path | None:
+ settings = get_settings()
+ if settings.voxcpm2_model_dir:
+ path = Path(settings.voxcpm2_model_dir)
+ return path if path.is_dir() else None
+ default = settings.models_dir / "voxcpm2"
+ return default if default.is_dir() else None
+
+
+class VoxCpm2Engine:
+ id = "voxcpm2"
+ label = "VoxCPM2 (experimental, opt-in)"
+ capabilities = CloneCapabilities(
+ zero_shot=True,
+ fine_tunable=False,
+ min_sample_seconds=3.0,
+ recommended_sample_seconds=10.0,
+ languages=SUPPORTED_LANGUAGES,
+ requires_gpu=False,
+ license="Apache-2.0 (VoxCPM2)",
+ approx_vram_gb=8.0,
+ )
+
+ def is_ready(self) -> bool:
+ return (
+ resolve_voxcpm2_python() is not None
+ and resolve_voxcpm2_model_dir() is not None
+ and _WORKER_SCRIPT.is_file()
+ )
+
+ async def create_voice(
+ self,
+ voice_id: str,
+ sample_paths: list[Path],
+ tier: Tier,
+ language: str,
+ on_progress: ProgressFn | None = None,
+ ) -> VoiceArtifact:
+ if not sample_paths:
+ raise EngineError("At least one reference sample is required")
+ normalized_language = (language or "en").split("-")[0].lower()
+ if normalized_language not in SUPPORTED_LANGUAGES:
+ raise EngineError(f"Language '{language}' is not supported by VoxCPM2")
+
+ async def report(message: str, extra: dict | None = None) -> None:
+ if on_progress:
+ await on_progress(message, extra)
+
+ await report("caching_reference")
+ source = pick_longest_sample(sample_paths)
+ out_dir = artifacts_dir(voice_id)
+ out_dir.mkdir(parents=True, exist_ok=True)
+ ref_audio = out_dir / "voxcpm2_ref.wav"
+ shutil.copy2(source, ref_audio)
+
+ ref_text = ""
+ if tier == "high_fidelity":
+ await report("transcribing_reference")
+
+ def _transcribe() -> str:
+ return transcribe_reference_audio(
+ ref_audio,
+ language=map_asr_language(normalized_language),
+ device=resolve_torch_device(get_settings().device),
+ )
+
+ try:
+ ref_text = await asyncio.get_running_loop().run_in_executor(None, _transcribe)
+ except Exception as exc:
+ logger.warning("VoxCPM2 reference transcription failed: %s", exc)
+
+ await report("done")
+ return VoiceArtifact(
+ engine_id=self.id,
+ tier=tier,
+ data={
+ "ref_audio_path": str(ref_audio.relative_to(get_settings().data_dir)),
+ "ref_text": ref_text,
+ "language": normalized_language,
+ },
+ )
+
+ async def synthesize(
+ self,
+ voice_id: str,
+ artifact: VoiceArtifact,
+ text: str,
+ opts: SynthesizeOptions,
+ ) -> bytes:
+ python = resolve_voxcpm2_python()
+ model_dir = resolve_voxcpm2_model_dir()
+ if python is None or model_dir is None:
+ raise EngineError(
+ "VoxCPM2 is not configured; set VOICEFORGE_VOXCPM2_PYTHON and "
+ "VOICEFORGE_VOXCPM2_MODEL_DIR to an isolated worker and local snapshot"
+ )
+ ref_rel = artifact.data.get("ref_audio_path")
+ if not ref_rel:
+ raise EngineError("Voice is missing VoxCPM2 reference audio")
+ ref_path = get_settings().data_dir / ref_rel
+ if not ref_path.is_file():
+ raise EngineError("Cached VoxCPM2 reference audio is missing")
+
+ out_wav = artifacts_dir(voice_id) / "voxcpm2_last.wav"
+ args = [
+ "synthesize",
+ "--model-dir",
+ str(model_dir),
+ "--ref-audio",
+ str(ref_path),
+ "--text",
+ text,
+ "--output",
+ str(out_wav),
+ "--device",
+ resolve_torch_device(get_settings().device),
+ "--seed",
+ "42",
+ ]
+ ref_text = artifact.data.get("ref_text")
+ if ref_text:
+ args.extend(["--ref-text", str(ref_text)])
+ if opts.style:
+ args.extend(["--style", opts.style])
+
+ await run_worker(
+ python=python,
+ script=_WORKER_SCRIPT,
+ args=args,
+ label="voxcpm2",
+ timeout_s=900,
+ )
+ if not out_wav.is_file():
+ raise EngineError("VoxCPM2 worker did not produce output audio")
+ wav, source_rate = sf.read(str(out_wav), dtype="float32", always_2d=False)
+ if getattr(wav, "ndim", 1) > 1:
+ wav = wav.mean(axis=1)
+ return to_wav_bytes(
+ np.asarray(wav, dtype=np.float32),
+ source_rate=int(source_rate) or VOXCPM2_NATIVE_SAMPLE_RATE,
+ target_rate=opts.sample_rate or 44100,
+ )
diff --git a/app/main.py b/app/main.py
index 42dd311..f9ec382 100644
--- a/app/main.py
+++ b/app/main.py
@@ -10,7 +10,7 @@
from fastapi.responses import FileResponse
from fastapi.staticfiles import StaticFiles
-from app.api import engines, events, metrics, synth, voices
+from app.api import engines, events, metrics, providers, synth, voices
from app.config import get_settings
from app.db import init_db
from app.engine_readiness import get_health_counts, refresh_readiness, start_warmup
@@ -28,6 +28,10 @@
"name": "engines",
"description": "List registered cloning engines and their capabilities.",
},
+ {
+ "name": "providers",
+ "description": "List side-effect-free provider manifests and runtime support.",
+ },
{
"name": "voices",
"description": "Create, list, upgrade, and manage cloned voices.",
@@ -78,7 +82,7 @@ async def lifespan(app: FastAPI):
"**Web UI:** open `/` for the Docker-hosted studio.\n\n"
"**Auth:** Bearer token on `/v1/*` when `VOICEFORGE_API_TOKEN` is set.\n\n"
"**Engines:** OpenVoice V2, F5-TTS, XTTS-v2, Chatterbox, Qwen3-TTS, "
- "Fish Speech, CosyVoice 3, IndexTTS2, RVC.\n\n"
+ "VoxCPM2 (opt-in), Fish Speech, CosyVoice 3, IndexTTS2, RVC.\n\n"
"See the project README for licensing, consent, and deployment notes."
),
version="0.3.1",
@@ -99,6 +103,7 @@ async def lifespan(app: FastAPI):
)
app.include_router(engines.router)
+app.include_router(providers.router)
app.include_router(voices.router)
app.include_router(events.router)
app.include_router(synth.router)
diff --git a/app/providers/__init__.py b/app/providers/__init__.py
new file mode 100644
index 0000000..5ed9b41
--- /dev/null
+++ b/app/providers/__init__.py
@@ -0,0 +1 @@
+"""Provider manifests shared by engines and audio-intelligence backends."""
diff --git a/app/providers/models.py b/app/providers/models.py
new file mode 100644
index 0000000..b15f0cb
--- /dev/null
+++ b/app/providers/models.py
@@ -0,0 +1,74 @@
+"""Structured, API-safe metadata for local model providers.
+
+The engine protocol intentionally stays small. These manifests describe the
+operational facts that do not belong in ``CloneCapabilities``: immutable model
+revisions, license gates, integration maturity, and platform-specific runtime
+support.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+ProviderKind = Literal["tts-clone", "voice-conversion", "audio-intelligence", "tts"]
+IntegrationState = Literal["integrated", "experimental", "manifest-only"]
+SupportLevel = Literal["supported", "experimental", "unsupported"]
+
+
+class ModelPin(BaseModel):
+ model_id: str
+ revision: str | None
+ local_path_required: bool = False
+ purpose: str = "inference"
+
+
+class LicenseMetadata(BaseModel):
+ identifier: str
+ commercial_use: bool | None
+ gated: bool = False
+ source_url: str
+ notes: str | None = None
+
+
+class RuntimeSupport(BaseModel):
+ mac_native: SupportLevel
+ docker_cpu: SupportLevel
+ cuda_notebook: SupportLevel
+ preferred: Literal["mac-native", "docker-cpu", "cuda-notebook", "worker"]
+ notes: str | None = None
+
+
+class ProviderCapabilities(BaseModel):
+ languages: list[str] = Field(default_factory=list)
+ text_to_speech: bool = False
+ voice_clone: bool = False
+ voice_design: bool = False
+ expressive_control: bool = False
+ voice_conversion: bool = False
+ transcription: bool = False
+ alignment: bool = False
+ code_switching: bool | None = None
+
+
+class ProviderManifest(BaseModel):
+ id: str
+ label: str
+ kind: ProviderKind
+ integration: IntegrationState
+ default_enabled: bool
+ explicit_opt_in: bool
+ local_only: bool = True
+ capabilities: ProviderCapabilities
+ runtime: RuntimeSupport
+ license: LicenseMetadata
+ models: list[ModelPin] = Field(default_factory=list)
+ dependency_file: str | None = None
+ notes: list[str] = Field(default_factory=list)
+
+
+class ProviderStatus(BaseModel):
+ manifest: ProviderManifest
+ configured: bool
+ ready: bool
diff --git a/app/providers/registry.py b/app/providers/registry.py
new file mode 100644
index 0000000..4cc8cb0
--- /dev/null
+++ b/app/providers/registry.py
@@ -0,0 +1,460 @@
+"""Durable provider manifest registry.
+
+Manifests are deliberately static and side-effect free: listing providers must
+never import an ML SDK, contact a model hub, or start a worker.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from app.config import get_settings
+from app.providers.models import (
+ LicenseMetadata,
+ ModelPin,
+ ProviderCapabilities,
+ ProviderManifest,
+ ProviderStatus,
+ RuntimeSupport,
+)
+
+QWEN3_TTS_REVISION = "fd4b254"
+CHATTERBOX_REVISION = "5bb1f6e"
+QWEN3_ASR_REVISION = "5eb1441"
+QWEN3_ALIGNER_REVISION = "c7cbfc2"
+VOXCPM2_REVISION = "9454c2d"
+
+_CH = ["ar", "da", "de", "el", "en", "es", "fi", "fr", "he", "hi", "it", "ja", "ko",
+ "ms", "nl", "no", "pl", "pt", "ru", "sv", "sw", "tr", "zh"]
+_QWEN_TTS = ["en", "zh", "ja", "ko", "de", "fr", "ru", "pt", "es", "it"]
+_QWEN_ASR = [
+ "ar", "cs", "da", "de", "el", "en", "es", "fa", "fi", "fil", "fr", "hi", "hu", "id",
+ "it", "ja", "ko", "mk", "ms", "nl", "pl", "pt", "ro", "ru", "sv", "th", "tr", "vi",
+ "yue", "zh",
+]
+_QWEN_ALIGN = ["zh", "en", "yue", "fr", "de", "it", "ja", "ko", "pt", "ru", "es"]
+_VOXCPM2 = [
+ "ar", "my", "zh", "da", "nl", "en", "fi", "fr", "de", "el", "he", "hi", "id", "it",
+ "ja", "km", "ko", "lo", "ms", "no", "pl", "pt", "ru", "es", "sw", "sv", "tl", "th",
+ "tr", "vi",
+]
+
+
+def _license(
+ identifier: str,
+ commercial: bool | None,
+ url: str,
+ *,
+ gated: bool = False,
+ notes: str | None = None,
+) -> LicenseMetadata:
+ return LicenseMetadata(
+ identifier=identifier,
+ commercial_use=commercial,
+ gated=gated,
+ source_url=url,
+ notes=notes,
+ )
+
+
+def _runtime(
+ mac: str,
+ docker: str,
+ notebook: str,
+ preferred: str,
+ notes: str | None = None,
+) -> RuntimeSupport:
+ return RuntimeSupport(
+ mac_native=mac,
+ docker_cpu=docker,
+ cuda_notebook=notebook,
+ preferred=preferred,
+ notes=notes,
+ )
+
+
+_MANIFESTS = [
+ ProviderManifest(
+ id="xtts-v2",
+ label="XTTS v2",
+ kind="tts-clone",
+ integration="integrated",
+ default_enabled=True,
+ explicit_opt_in=False,
+ capabilities=ProviderCapabilities(
+ languages=["en", "es", "fr", "de", "it", "pt", "pl", "tr", "ru", "nl", "cs",
+ "ar", "zh", "ja", "hu", "ko", "hi"],
+ text_to_speech=True,
+ voice_clone=True,
+ ),
+ runtime=_runtime("experimental", "supported", "supported", "docker-cpu"),
+ license=_license(
+ "Coqui Public Model License",
+ False,
+ "https://huggingface.co/coqui/XTTS-v2",
+ notes="Model-weight terms are noncommercial/research-oriented.",
+ ),
+ models=[ModelPin(model_id="tts_models/multilingual/multi-dataset/xtts_v2", revision=None)],
+ dependency_file="requirements-xtts.txt",
+ ),
+ ProviderManifest(
+ id="f5-tts",
+ label="F5-TTS v1 Base",
+ kind="tts-clone",
+ integration="integrated",
+ default_enabled=True,
+ explicit_opt_in=False,
+ capabilities=ProviderCapabilities(
+ languages=["en", "zh"], text_to_speech=True, voice_clone=True
+ ),
+ runtime=_runtime("experimental", "supported", "supported", "cuda-notebook"),
+ license=_license(
+ "CC-BY-NC-4.0 weights",
+ False,
+ "https://github.com/SWivid/F5-TTS/blob/main/src/f5_tts/infer/SHARED.md",
+ notes="Code and pretrained-weight licenses differ.",
+ ),
+ models=[ModelPin(model_id="SWivid/F5-TTS", revision=None)],
+ dependency_file="requirements-f5.txt",
+ ),
+ ProviderManifest(
+ id="openvoice-v2",
+ label="OpenVoice V2",
+ kind="tts-clone",
+ integration="integrated",
+ default_enabled=True,
+ explicit_opt_in=False,
+ capabilities=ProviderCapabilities(
+ languages=["en", "es", "fr", "zh", "ja", "ko"],
+ text_to_speech=True,
+ voice_clone=True,
+ voice_conversion=True,
+ ),
+ runtime=_runtime("experimental", "supported", "supported", "docker-cpu"),
+ license=_license(
+ "MIT adapter; base models separate",
+ None,
+ "https://github.com/myshell-ai/OpenVoice",
+ ),
+ dependency_file="requirements-openvoice.txt",
+ ),
+ ProviderManifest(
+ id="rvc",
+ label="RVC",
+ kind="voice-conversion",
+ integration="integrated",
+ default_enabled=True,
+ explicit_opt_in=False,
+ capabilities=ProviderCapabilities(voice_conversion=True),
+ runtime=_runtime("unsupported", "unsupported", "experimental", "worker"),
+ license=_license(
+ "MIT code; checkpoints/data separate",
+ None,
+ "https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI",
+ ),
+ dependency_file="requirements-rvc.txt",
+ ),
+ ProviderManifest(
+ id="chatterbox",
+ label="Chatterbox Multilingual V3",
+ kind="tts-clone",
+ integration="integrated",
+ default_enabled=True,
+ explicit_opt_in=False,
+ capabilities=ProviderCapabilities(
+ languages=_CH,
+ text_to_speech=True,
+ voice_clone=True,
+ expressive_control=True,
+ code_switching=False,
+ ),
+ runtime=_runtime(
+ "supported", "supported", "supported", "worker",
+ "Mac native worker can use MPS; Docker Desktop is CPU-only.",
+ ),
+ license=_license(
+ "MIT",
+ True,
+ "https://github.com/resemble-ai/chatterbox",
+ ),
+ models=[
+ ModelPin(
+ model_id="ResembleAI/chatterbox",
+ revision=CHATTERBOX_REVISION,
+ local_path_required=True,
+ )
+ ],
+ dependency_file="requirements-chatterbox.txt",
+ ),
+ ProviderManifest(
+ id="qwen3-tts",
+ label="Qwen3-TTS 1.7B Base",
+ kind="tts-clone",
+ integration="integrated",
+ default_enabled=True,
+ explicit_opt_in=False,
+ capabilities=ProviderCapabilities(
+ languages=_QWEN_TTS,
+ text_to_speech=True,
+ voice_clone=True,
+ expressive_control=True,
+ code_switching=None,
+ ),
+ runtime=_runtime(
+ "experimental", "experimental", "supported", "cuda-notebook",
+ "MPS mapping is correct but upstream MPS quality/performance remains unverified.",
+ ),
+ license=_license(
+ "Apache-2.0",
+ True,
+ "https://github.com/QwenLM/Qwen3-TTS",
+ ),
+ models=[
+ ModelPin(
+ model_id="Qwen/Qwen3-TTS-12Hz-1.7B-Base",
+ revision=QWEN3_TTS_REVISION,
+ )
+ ],
+ dependency_file="requirements-qwen3.txt",
+ ),
+ ProviderManifest(
+ id="fish-speech",
+ label="Fish Speech S2 sidecar",
+ kind="tts-clone",
+ integration="integrated",
+ default_enabled=True,
+ explicit_opt_in=False,
+ capabilities=ProviderCapabilities(
+ text_to_speech=True,
+ voice_clone=True,
+ expressive_control=True,
+ languages=[],
+ ),
+ runtime=_runtime("unsupported", "experimental", "supported", "worker"),
+ license=_license(
+ "Fish Audio Research License",
+ False,
+ "https://github.com/fishaudio/fish-speech/blob/main/LICENSE",
+ notes=(
+ "Hindi is not advertised here because it was not verified "
+ "from the primary source."
+ ),
+ ),
+ dependency_file="requirements-fish.txt",
+ ),
+ ProviderManifest(
+ id="cosyvoice-3",
+ label="CosyVoice 3",
+ kind="tts-clone",
+ integration="integrated",
+ default_enabled=True,
+ explicit_opt_in=False,
+ capabilities=ProviderCapabilities(
+ languages=["zh", "en", "ja", "ko", "de", "es", "fr", "it", "ru"],
+ text_to_speech=True,
+ voice_clone=True,
+ expressive_control=True,
+ ),
+ runtime=_runtime("unsupported", "experimental", "supported", "worker"),
+ license=_license(
+ "Apache-2.0",
+ True,
+ "https://github.com/QwenAudio/CosyVoice",
+ ),
+ dependency_file="requirements-cosyvoice.txt",
+ ),
+ ProviderManifest(
+ id="indextts-2",
+ label="IndexTTS2",
+ kind="tts-clone",
+ integration="integrated",
+ default_enabled=True,
+ explicit_opt_in=False,
+ capabilities=ProviderCapabilities(
+ languages=["en", "zh"],
+ text_to_speech=True,
+ voice_clone=True,
+ ),
+ runtime=_runtime("unsupported", "unsupported", "supported", "worker"),
+ license=_license(
+ "Bilibili IndexTTS Model License",
+ None,
+ "https://github.com/index-tts/index-tts/blob/main/LICENSE",
+ ),
+ dependency_file="requirements-indextts.txt",
+ ),
+ ProviderManifest(
+ id="qwen3-asr",
+ label="Qwen3-ASR 0.6B + optional Forced Aligner",
+ kind="audio-intelligence",
+ integration="experimental",
+ default_enabled=False,
+ explicit_opt_in=True,
+ capabilities=ProviderCapabilities(
+ languages=_QWEN_ASR,
+ transcription=True,
+ alignment=True,
+ code_switching=None,
+ ),
+ runtime=_runtime(
+ "experimental", "experimental", "supported", "worker",
+ f"Forced alignment is limited to: {', '.join(_QWEN_ALIGN)}; it does not include Hindi.",
+ ),
+ license=_license(
+ "Apache-2.0",
+ True,
+ "https://github.com/QwenLM/Qwen3-ASR",
+ ),
+ models=[
+ ModelPin(
+ model_id="Qwen/Qwen3-ASR-0.6B",
+ revision=QWEN3_ASR_REVISION,
+ local_path_required=True,
+ purpose="transcription",
+ ),
+ ModelPin(
+ model_id="Qwen/Qwen3-ForcedAligner-0.6B",
+ revision=QWEN3_ALIGNER_REVISION,
+ local_path_required=True,
+ purpose="alignment",
+ ),
+ ],
+ dependency_file="requirements-qwen3-asr.txt",
+ notes=["Reference transcription backend only; no cloud API integration."],
+ ),
+ ProviderManifest(
+ id="voxcpm2",
+ label="VoxCPM2",
+ kind="tts-clone",
+ integration="experimental",
+ default_enabled=False,
+ explicit_opt_in=True,
+ capabilities=ProviderCapabilities(
+ languages=_VOXCPM2,
+ text_to_speech=True,
+ voice_clone=True,
+ voice_design=True,
+ expressive_control=True,
+ code_switching=None,
+ ),
+ runtime=_runtime(
+ "experimental", "experimental", "supported", "worker",
+ "Upstream exposes MPS, but compatibility varies; Docker Desktop is CPU-only.",
+ ),
+ license=_license(
+ "Apache-2.0",
+ True,
+ "https://github.com/OpenBMB/VoxCPM",
+ ),
+ models=[
+ ModelPin(
+ model_id="openbmb/VoxCPM2",
+ revision=VOXCPM2_REVISION,
+ local_path_required=True,
+ )
+ ],
+ dependency_file="requirements-voxcpm2.txt",
+ ),
+ ProviderManifest(
+ id="indicf5",
+ label="AI4Bharat IndicF5 (license-gated experiment)",
+ kind="tts-clone",
+ integration="manifest-only",
+ default_enabled=False,
+ explicit_opt_in=True,
+ capabilities=ProviderCapabilities(
+ languages=["as", "bn", "gu", "hi", "kn", "ml", "mr", "or", "pa", "ta", "te"],
+ text_to_speech=True,
+ voice_clone=True,
+ ),
+ runtime=_runtime("unsupported", "unsupported", "experimental", "cuda-notebook"),
+ license=_license(
+ "MIT",
+ True,
+ "https://huggingface.co/ai4bharat/IndicF5",
+ gated=True,
+ notes=(
+ "Access-gated; consent-only voice-cloning terms apply. No runtime "
+ "integration because the upstream example executes remote model code."
+ ),
+ ),
+ models=[ModelPin(model_id="ai4bharat/IndicF5", revision=None, local_path_required=True)],
+ notes=["Manifest only; a reviewed, pinned local worker is required before enablement."],
+ ),
+ ProviderManifest(
+ id="indic-parler-tts",
+ label="AI4Bharat Indic Parler-TTS (license-gated experiment)",
+ kind="tts",
+ integration="manifest-only",
+ default_enabled=False,
+ explicit_opt_in=True,
+ capabilities=ProviderCapabilities(
+ languages=[
+ "as", "bn", "brx", "doi", "en", "gu", "hi", "kn", "kok", "mai", "ml", "mni",
+ "mr", "ne", "or", "sa", "sat", "sd", "ta", "te", "ur",
+ ],
+ text_to_speech=True,
+ voice_design=True,
+ expressive_control=True,
+ ),
+ runtime=_runtime("unsupported", "unsupported", "experimental", "cuda-notebook"),
+ license=_license(
+ "Apache-2.0",
+ True,
+ "https://huggingface.co/ai4bharat/indic-parler-tts",
+ gated=True,
+ notes="Model files require accepting the upstream access gate.",
+ ),
+ models=[
+ ModelPin(
+ model_id="ai4bharat/indic-parler-tts",
+ revision=None,
+ local_path_required=True,
+ )
+ ],
+ notes=["Manifest only; no startup install, remote code, or model access is attempted."],
+ ),
+]
+
+_BY_ID = {manifest.id: manifest for manifest in _MANIFESTS}
+
+
+def list_provider_manifests() -> list[ProviderManifest]:
+ return list(_MANIFESTS)
+
+
+def get_provider_manifest(provider_id: str) -> ProviderManifest:
+ return _BY_ID[provider_id]
+
+
+def _path_ready(path: Path | None) -> bool:
+ return path is not None and path.is_dir()
+
+
+def provider_statuses() -> list[ProviderStatus]:
+ """Return status using filesystem/config checks only."""
+
+ from app.engine_readiness import peek_engine_ready
+ from app.engines.registry import engine_ids, is_engine_enabled
+
+ settings = get_settings()
+ engine_id_set = set(engine_ids())
+ statuses: list[ProviderStatus] = []
+ for manifest in _MANIFESTS:
+ if manifest.id in engine_id_set:
+ configured = is_engine_enabled(manifest.id)
+ ready = configured and peek_engine_ready(manifest.id)
+ elif manifest.id == "qwen3-asr":
+ configured = settings.reference_asr_provider == "qwen3-asr"
+ ready = (
+ configured
+ and settings.qwen3_asr_python is not None
+ and settings.qwen3_asr_python.is_file()
+ and _path_ready(settings.qwen3_asr_model_dir)
+ and (Path(__file__).resolve().parents[2] / "scripts/qwen3_asr_worker.py").is_file()
+ )
+ else:
+ configured = False
+ ready = False
+ statuses.append(ProviderStatus(manifest=manifest, configured=configured, ready=ready))
+ return statuses
diff --git a/app/runtime_device.py b/app/runtime_device.py
new file mode 100644
index 0000000..1269581
--- /dev/null
+++ b/app/runtime_device.py
@@ -0,0 +1,46 @@
+"""Pure device-selection helpers shared by in-process and worker providers."""
+
+from __future__ import annotations
+
+from typing import Any
+
+VALID_DEVICES = {"auto", "cpu", "cuda", "mps"}
+
+
+def resolve_torch_device(requested: str, torch_module: Any | None = None) -> str:
+ """Resolve ``auto`` with CUDA first, then Apple MPS, then CPU.
+
+ Explicit values are preserved so workers can produce a useful SDK-level
+ error. This helper never imports torch unless automatic detection is
+ requested.
+ """
+
+ normalized = (requested or "auto").lower()
+ if normalized not in VALID_DEVICES:
+ expected = sorted(VALID_DEVICES)
+ raise ValueError(f"Unsupported device '{requested}'; expected one of {expected}")
+ if normalized != "auto":
+ return normalized
+
+ if torch_module is None:
+ try:
+ import torch as torch_module
+ except Exception:
+ return "cpu"
+
+ if torch_module.cuda.is_available():
+ return "cuda"
+ mps = getattr(getattr(torch_module, "backends", None), "mps", None)
+ if mps is not None and mps.is_available():
+ return "mps"
+ return "cpu"
+
+
+def transformers_device_map(device: str) -> str:
+ """Return a valid Transformers ``device_map`` target."""
+
+ if device == "cuda":
+ return "cuda:0"
+ if device in {"cpu", "mps"}:
+ return device
+ raise ValueError(f"Device must be resolved before building device_map: {device}")
diff --git a/app/schemas.py b/app/schemas.py
index 91a0733..6253db3 100644
--- a/app/schemas.py
+++ b/app/schemas.py
@@ -55,6 +55,7 @@ class SynthesizeRequest(CamelModel):
sample_rate: int | None = None
speed: float | None = Field(default=None, gt=0.25, le=4.0)
language: str | None = None
+ style: str | None = Field(default=None, max_length=500)
class MetricsSnapshot(CamelModel):
diff --git a/app/static/index.html b/app/static/index.html
index baf35d7..9f7f9a8 100644
--- a/app/static/index.html
+++ b/app/static/index.html
@@ -245,7 +245,7 @@
VoiceForge Studio
- Clone a voice from a short sample and synthesize speech — all inside Docker.
+ Clone a voice from a short sample and synthesize speech in a local runtime.
API docs: /docs