diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 86b923a..2604697 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -13,7 +13,7 @@ - [ ] Commits are signed off (`git commit -s`). See CONTRIBUTING.md. - [ ] `python tools/check_layering.py .` and `python tools/release_check.py .` pass. -- [ ] `pytest` passes in `emet-sdk`, `emet-hal` and `emet-engine`. +- [ ] `pytest` passes in `emet-sdk`, `emet-hal`, `emet-providers` and `emet-engine`. - [ ] New behaviour has a test and a caller. - [ ] Anything taken from a paper or repository is cited at the point of use and in CITATIONS.md. @@ -34,7 +34,7 @@ even when it works. See DESIGN.md section 2. - [ ] The soul names no hardware. - [ ] Every fallback chain still terminates in a voice rung. -- [ ] `emet_sdk` imports nothing internal; `emet_hal` and `emet_engine` - import `emet_sdk` only. +- [ ] `emet_sdk` imports nothing internal; `emet_hal`, `emet_providers` and + `emet_engine` import `emet_sdk` only. - [ ] Memory is not namespaced by body. - [ ] A missing plugin is still distinct from a schema error. diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 128d35f..62b5ab0 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -40,3 +40,13 @@ updates: patterns: ["*"] commit-message: prefix: "build" + + - package-ecosystem: pip + directory: /emet-providers + schedule: + interval: monthly + groups: + python: + patterns: ["*"] + commit-message: + prefix: "build" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d59dd05..6c22174 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,9 +3,9 @@ # Two things are checked, and the second matters more than it looks. # # tests: the 0.1 acceptance criteria, executable. -# layering: emet_sdk imports nothing internal; emet_hal and emet_engine -# import emet_sdk only. The same job runs the release -# invariants and the house-style check. +# layering: emet_sdk imports nothing internal; emet_hal, emet_providers +# and emet_engine import emet_sdk only. The same job runs the +# release invariants and the house-style check. # # The layering check exists because the closed-engine plan used to enforce that # boundary structurally, and an open monorepo does not. See DISTRIBUTION.md §2. @@ -41,7 +41,7 @@ jobs: - name: Check package layering run: python tools/check_layering.py . - # Cross-package invariants no single suite can see: the three packages + # Cross-package invariants no single suite can see: the four packages # agreeing about the version, every discovery group having something # behind it, and no document advertising a version the code is not. - name: Check release invariants @@ -105,6 +105,7 @@ jobs: python -m pip install --upgrade build python -m build --wheel emet-sdk python -m build --wheel emet-hal + python -m build --wheel emet-providers python -m build --wheel emet-engine - name: Data files are actually inside the wheel @@ -131,7 +132,7 @@ jobs: - name: Install the wheels, not the source run: | - python -m pip install emet-sdk/dist/*.whl emet-hal/dist/*.whl emet-engine/dist/*.whl + python -m pip install emet-sdk/dist/*.whl emet-hal/dist/*.whl emet-providers/dist/*.whl emet-engine/dist/*.whl # `cd /tmp` is the point of this step: from here the source checkout is # not on the path, so anything that resolves must be coming out of the @@ -159,14 +160,22 @@ jobs: assert r.has_wake("pocketsphinx"), "wake group did not survive packaging" assert r.has_audio("microphone"), "audio group did not survive packaging" assert r.has_audio_out("speaker"), "audio_out group did not survive packaging" + assert r.has_stt("mock"), "stt group did not survive packaging" + assert r.has_llm("mock"), "llm group did not survive packaging" + assert r.has_tts("mock"), "tts group did not survive packaging" print("discovered:", [f"{g}:{n}" for g, n in r]) # The version is declared once, in pyproject.toml, and read back # through importlib.metadata. From a wheel that resolution path is # different from an editable install, so it is worth checking here # rather than only in the test suite. - import emet_sdk, emet_hal, emet_engine - for dist, mod in (("emet-sdk", emet_sdk), ("emet-hal", emet_hal), ("emet-engine", emet_engine)): + import emet_sdk, emet_hal, emet_providers, emet_engine + for dist, mod in ( + ("emet-sdk", emet_sdk), + ("emet-hal", emet_hal), + ("emet-providers", emet_providers), + ("emet-engine", emet_engine), + ): assert mod.__version__ == version(dist), f"{dist} version drifted" assert mod.__version__ != "0+unknown", f"{dist} reports no version" print("versions:", emet_sdk.__version__) @@ -186,13 +195,15 @@ jobs: python-version: ${{ matrix.python-version }} cache: pip - # emet-hal is installed too, and not only so its own tests run: plugin - # discovery reads entry points, so several SDK tests are meaningless - # unless something is actually registered to discover. + # emet-hal and emet-providers are installed too, and not only so their + # own tests run: plugin discovery reads entry points, so several SDK and + # engine tests are meaningless unless something is actually registered + # to discover. - name: Install packages run: | python -m pip install -e "emet-sdk[dev]" python -m pip install -e "emet-hal[dev]" + python -m pip install -e "emet-providers[dev,deepgram,anthropic,openai]" python -m pip install -e "emet-engine[dev]" # The shipped wake engine, which is an optional extra rather than a hard @@ -213,9 +224,17 @@ jobs: working-directory: emet-hal run: python -m pytest -q - # Runs without a microphone or an acoustic model: the loop is exercised - # through a wav file and the mock detector, both reached by the same - # entry-point discovery the real ones use. + # No network and no key. The mocks need nothing, and every real + # provider is tested against a stand-in server; the live tests skip + # unless a key and EMET_LIVE_TESTS=1 are set. + - name: Test emet-providers + working-directory: emet-providers + run: python -m pytest -q + + # Runs without a microphone, an acoustic model or a network: the loop is + # exercised through a wav file, the mock detector and the mock + # transcriber, all reached by the same entry-point discovery the real + # ones use. - name: Test emet-engine working-directory: emet-engine run: python -m pytest -q @@ -248,3 +267,46 @@ jobs: fi done echo "all invalid fixtures rejected" + + # The same four suites with every optional extra absent: no pocketsphinx, + # no sounddevice, no websockets, no httpx, no piper-tts. A body that only + # ever wakes installs none of them, and every test that needs one has to + # skip rather than fail. This used to be a rule that lived on one laptop + # (shadow the modules by hand before every push); now it is a job. + bare: + name: test (no extras) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + cache: pip + - name: Install the packages and nothing optional + run: | + python -m pip install -e "emet-sdk[dev]" + python -m pip install -e "emet-hal[dev]" + python -m pip install -e "emet-providers[dev]" + python -m pip install -e "emet-engine[dev]" + - name: Every optional extra is really absent + run: | + python - <<'PY' + import importlib.util, sys + present = [m for m in ("pocketsphinx", "sounddevice", "websockets", "httpx", "piper") if importlib.util.find_spec(m)] + if present: + print("optional extras leaked into the bare job:", present) + sys.exit(1) + print("bare: no optional extras installed") + PY + - name: Test emet-sdk + working-directory: emet-sdk + run: python -m pytest -q + - name: Test emet-hal + working-directory: emet-hal + run: python -m pytest -q + - name: Test emet-providers + working-directory: emet-providers + run: python -m pytest -q + - name: Test emet-engine + working-directory: emet-engine + run: python -m pytest -q diff --git a/.gitignore b/.gitignore index 005b8af..5f4f1e1 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ __pycache__/ dist/ build/ .pytest_cache/ + +# Keys are bring-your-own and never enter history, whatever they are called. +keys.env +.env diff --git a/CITATIONS.md b/CITATIONS.md index de55b5e..2e097f0 100644 --- a/CITATIONS.md +++ b/CITATIONS.md @@ -90,6 +90,37 @@ firing. --- +## Piper, and the LJ Speech voice + +**Michael Hansen and the Open Home Foundation.** Piper, a fast local neural +text-to-speech engine. , +**GPL-3.0-or-later** (the package `piper-tts`, 1.8.0 on 2026-09-04). The +licence is the GPL because espeak-ng, the phonemiser, is compiled into the +wheel; the earlier `rhasspy/piper` was MIT and is no longer where releases +come from. Depends on ONNX Runtime (Microsoft, **MIT**). + +The shipped local voice +([`emet_providers/piper.py`](emet-providers/emet_providers/piper.py)), used as +a dependency behind the optional extra `emet-providers[piper]`. Emet imports +it and does not copy, bundle or redistribute it, so the Apache-2.0 terms of +this repository are unaffected and a body installed without the extra +carries no GPL code. Anyone packaging Emet *with* Piper inside one +distribution takes on the GPL's terms for that distribution, and should know +it. Facts verified against PyPI and the repository on 2026-09-15. + +**Keith Ito and Linda Johnson.** The LJ Speech Dataset. +, **public domain**. The reference +soul's voice, `en_US-ljspeech-medium`, is a Piper model trained on it +(model card in `rhasspy/piper-voices` on Hugging Face, read 2026-09-15). It +was chosen over Amy, the earlier reference voice, because Amy's model card +says it is fine-tuned from the Lessac voice, and the Blizzard 2013 Lessac +corpus licence excludes "the development, marketing, commercialisation, sale +or licencing of voice synthesis" products; a default a project promises +people may fork and sell cannot rest on that. The voice files themselves are +downloaded by the owner and are not in this repository. + +--- + ## Adding to this file If a change takes an idea, a finding, a number, or a data format from outside diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 22aca43..4b271d4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -76,6 +76,9 @@ are cheaper to get wrong. - **Drivers** in `emet-hal`: servos, displays, LEDs, sensors, motor drivers. None are written yet; `emet_hal.mock` shows the shape one takes. +- **Providers** in `emet-providers`: speech recognition today, language + models and voices later. `emet_providers.mock` shows the shape, and + `DESIGN.md` section 12.3 says who chooses one. - **Locomotion plugins**: new kinematics. `drive.kinematics` is an open enum precisely so that `legged`, `omni`, and things nobody has thought of can arrive as packages rather than as schema changes. @@ -117,8 +120,8 @@ wrong even if it works: 2. **Every fallback chain terminates in a voice rung.** The validator enforces this. It is what makes "every intent is always satisfiable" mechanical rather than aspirational. -3. **`emet_sdk` imports nothing internal.** `emet_hal` and `emet_engine` import - `emet_sdk` only. CI checks this on every pull request. +3. **`emet_sdk` imports nothing internal.** `emet_hal`, `emet_providers` and + `emet_engine` import `emet_sdk` only. CI checks this on every pull request. 4. **Memory is never namespaced by body.** Experiences travel with the soul; hardware conditions stay with the body. 5. **A missing plugin is not a schema error.** Keep the two failure modes @@ -171,11 +174,11 @@ Both run in CI, so they cannot quietly stop working. ```sh python -m venv .venv -.venv/bin/pip install -e "emet-sdk[dev]" -e "emet-hal[dev]" -e "emet-engine[dev]" +.venv/bin/pip install -e "emet-sdk[dev]" -e "emet-hal[dev]" -e "emet-providers[dev]" -e "emet-engine[dev]" .venv/bin/pip install -e "emet-hal[audio,wake]" # optional: microphone, speaker, wake engine ``` -Install **all three** packages even if you are only touching one. Plugin +Install **all four** packages even if you are only touching one. Plugin discovery reads entry points, so several SDK and engine tests are meaningless unless something is registered to be discovered. The suites pass with the optional extras absent; the tests that need them skip. @@ -188,6 +191,7 @@ python tools/release_check.py . python tools/check_style.py . cd emet-sdk && python -m pytest -q cd ../emet-hal && python -m pytest -q +cd ../emet-providers && python -m pytest -q cd ../emet-engine && python -m pytest -q ``` @@ -198,7 +202,7 @@ a comma, a colon, or two sentences. None of the six words that read as a press release; `tools/check_style.py` lists them, enforces both rules, and runs in CI, so a stray dash fails the build rather than a review. -CI also builds the three wheels and installs them outside the source tree, so a +CI also builds the four wheels and installs them outside the source tree, so a packaging mistake that an editable install hides still fails the pipeline. The exact steps are in `.github/workflows/ci.yml`. diff --git a/DESIGN.md b/DESIGN.md index 4e119f4..10af0be 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -83,7 +83,7 @@ The invariants. If an implementation decision violates one of these, the decisio ## 3. Package layout -**Emet is open source, in full.** One monorepo, three packages, one permissive license. +**Emet is open source, in full.** One monorepo, four packages, one permissive license. The boundary between the packages is a product surface; changes to it are breaking changes. See `CONTRIBUTING.md` for what is open to contribution and what is not. @@ -93,8 +93,10 @@ emet/ one repository, Apache 2.0 throughout schemas/ manifest, soul bundle, motion pack (JSON Schema) emet_sdk/ types.py Intent, Action, CapabilityDescriptor, Pose, Twist, - WakeDescriptor, AudioFormat, AudioSource, AudioSink - plugin.py ActuatorPlugin, SensorPlugin, LocomotionPlugin, WakePlugin ABCs + WakeDescriptor, AudioFormat, AudioSource, AudioSink, + Transcript, Prompt, ReplyDone, VoiceDescriptor + plugin.py ActuatorPlugin, SensorPlugin, LocomotionPlugin, WakePlugin, + TranscriberPlugin, LanguageModelPlugin, VoicePlugin ABCs intents.py the canonical intent vocabulary chains.py fallback chain format + the voice-rung rule discovery.py entry-point plugin discovery @@ -109,6 +111,14 @@ emet/ one repository, Apache 2.0 throughout audio.py microphone and speaker through PortAudio; wav and null ... pca9685, tb6612, gc9a01, ws2812: not yet written + emet-providers/ the plugins that reach a service, or a model on disk (§12.3 to §12.5) + mock.py a transcriber, a language model and a voice that pretend + deepgram.py speech recognition through Deepgram + anthropic.py Claude through the Messages API + openai.py GPT, or any server speaking Chat Completions + piper.py the local voice, through Piper (the default) + deepgram_voice.py the cloud voice, Aura through Deepgram + emet-engine/ the listen loop today (wake, VAD, endpointing, §13); personality synthesis, memory, arbitration, choreography, prompting, safety and consolidation as releases arrive. @@ -116,7 +126,9 @@ emet/ one repository, Apache 2.0 throughout **HAL** is *hardware abstraction layer*: the standard embedded and OS term for the layer separating generic upper software from specific silicon. Emet uses it in Android's sense: the abstraction itself is `emet_sdk.plugin` (the ABCs and capability descriptors), and `emet-hal` is the collection of per-device *implementations* that satisfy it. Spell the acronym out on first use in any document a newcomer might read first; not every contributor arrives from embedded work. -The engine imports the SDK. Plugins import the SDK. The SDK is types and contracts and almost no logic, targeting under ~3,500 lines. The original target of 2,000 predates the wake and audio contracts, which added about 800 lines; the rest is headroom for the speech-to-text seam. It is the only thing both sides must agree on. Enforced in CI by `tools/check_layering.py`, because with one open monorepo the layering is a test rather than a property of how the software is distributed. +**`emet-providers`** is the HAL's counterpart for what a robot borrows from a computer somewhere else, or from a model on its own disk: speech recognition, language models and speech synthesis. A provider is a plugin in exactly the HAL's sense, satisfying a contract in `emet_sdk.plugin` and reaching the engine by name, and it is a separate package because the HAL is named for hardware, a client for a speech service is not hardware, and provider client libraries are heavy and networked in a way a body that only ever wakes should not have to install. + +The engine imports the SDK. Plugins import the SDK. The SDK is types and contracts and almost no logic, and it is about 3,700 lines at 0.4, against a target of under 4,000. The original target of 2,000 predates the wake and audio contracts, which added about 800 lines, and the three provider contracts (speech recognition, the language model, the voice), which added about 900 more; the rest is headroom for the self-model and memory types. It is the only thing both sides must agree on. Enforced in CI by `tools/check_layering.py`, because with one open monorepo the layering is a test rather than a property of how the software is distributed. **Where the line falls when something is arguably logic:** deterministic pure functions over contract data belong in the SDK; anything stateful, scheduled, or personality-bearing belongs in the engine. Chain resolution (§6) is the former, `(chains, descriptors) → binding table` with no state and no I/O, and it lives in the SDK so that a HAL contributor can check where their driver binds without running an engine. The choreographer, at 50Hz and holding motion state, is the latter. @@ -126,9 +138,9 @@ The consequence to hold onto: **anyone may fork Emet, close their fork, and ship **`P0`**: everything public from the first commit, under its final license. Open is a one-way door: once published, it is published, and a permissive release can never be walked back for code already out. -**Layering is enforced by CI, not by a license wall.** An import linter asserts that `emet-sdk` imports nothing internal, `emet-hal` imports only `emet-sdk`, and `emet-engine` imports only `emet-sdk`. Previously this invariant was maintained by the engine being a separate closed artifact; that structural guarantee is now a test, and it must actually run in CI or it will rot. +**Layering is enforced by CI, not by a license wall.** An import linter asserts that `emet-sdk` imports nothing internal, and that `emet-hal`, `emet-providers` and `emet-engine` each import only `emet-sdk`. Previously this invariant was maintained by the engine being a separate closed artifact; that structural guarantee is now a test, and it must actually run in CI or it will rot. -Python import namespaces: `emet_sdk`, `emet_hal`, `emet_engine`. CLI binaries: `emet` and `emet-listen`. Config root: `/etc/emet/`. Soul bundles: `*.emet` directories. +Python import namespaces: `emet_sdk`, `emet_hal`, `emet_providers`, `emet_engine`. CLI binaries: `emet` and `emet-listen`. Config root: `/etc/emet/`. Provider keys: `/etc/emet/keys.env` for the machine, `~/.config/emet/keys.env` for a person, `NAME=value` a line, read into the environment at start-up and never written into a soul or a manifest (§8.1). Soul bundles: `*.emet` directories. --- @@ -180,6 +192,17 @@ audio: # P0 required: this is the hardware floor # plugin. The PHRASE is on the soul (§8.1). params: {} # passed to the engine untouched +models: # P0 optional. The body's say over the soul's + # models block (§8.1), one entry per stage. + stt: # Set provider and the body takes the stage + provider: mock # over, with its own model and key_env; + params: {} # params ride along either way (§12.3). + chat: # OPEN ENUMS: stt against emet.stt, chat + provider: mock # against emet.llm (§12.4), tts against + tts: # emet.tts (§12.5). + provider: mock + micro: {} # RSV reserved, resolved against nothing yet + capabilities: [] # P0 see 4.2 safety: # P0 @@ -367,6 +390,7 @@ Enforced by `emet_sdk.validate`, run at boot and by the CLI: - `drive.kinematics` must be a **string that resolves to an installed locomotion plugin**, not a member of a frozen list. An unrecognized value fails boot with "no locomotion plugin provides `legged`; install one or change `kinematics`", which is a *missing plugin* error, not a *schema* error. This is what keeps the enum open. - Every referenced `driver.plugin` resolves to an installed plugin, or boot fails loudly with the missing package name. Never silently degrade because of a typo: that is a *different* failure from missing hardware, and conflating them costs support hours. - `audio.wake.engine`, `audio.input.source` and `audio.output.sink` resolve to installed plugins, or validation fails. `driver.plugin` may name hardware not yet wired and so only warns outside `--verify-drivers`; these name software that has to exist for the robot to hear or speak at all. +- `models.stt.provider`, `models.chat.provider` and `models.tts.provider`, when a body sets them, resolve to an installed `emet.stt`, `emet.llm` or `emet.tts` plugin on the same terms. The soul's `models` block is never checked against what is installed: a soul is valid on every machine or on none, and that question is answered at boot (§12.3 to §12.5). --- @@ -612,11 +636,13 @@ identity: created: "2026-08-08" # P0 license: null # RSV for the community soul registry -voice: - engine: piper # P0 piper | cloud - model: "en_US-amy-medium" # P0 - rate: 1.0 # P0 - pitch_shift_semitones: 0 # P0 +voice: # P0 how it sounds, whichever voice speaks + rate: 1.0 # P0 a multiplier on speaking speed; a persona + # trait like patience_ms, honoured by every + # shipped voice. Which voice is models.tts. + pitch_shift_semitones: 0 # RSV no shipped voice honours it yet + # (engine and model, the pre-seam fields, + # are accepted for older bundles and ignored) persona: summary: > # P0 the seed a human writes @@ -658,9 +684,15 @@ memory: models: # P0 BYOK chat: {provider: openai, model: "...", key_env: "EMET_OPENAI_KEY"} + # names an installed emet.llm plugin (§12.4) stt: {provider: deepgram, model: "...", key_env: "EMET_DEEPGRAM_KEY"} - tts: {provider: local_piper} - micro: {provider: local, model: "qwen3-0.6b-q4"} # backchannel only + # names an installed emet.stt plugin (§12.3). + tts: {provider: piper, model: "en_US-ljspeech-medium"} + # names an installed emet.tts plugin (§12.5); + # model is the voice. All three resolved at + # boot; a body may take any over with its + # own models block. + micro: {provider: local, model: "qwen3-0.6b-q4"} # backchannel only, RSV ``` #### 8.1.1 Why `name` and `wake_word` are separate fields @@ -884,11 +916,13 @@ so honestly. ### 12.2 Wake and audio plugins -Six entry-point groups in total. Three are built from a manifest capability +Nine entry-point groups in total. Three are built from a manifest capability (`emet.actuators`, `emet.sensors`, `emet.locomotion`). Three are built from the manifest's `audio` block (`emet.wake`, `emet.audio`, `emet.audio_out`), because the hardware floor already guarantees a microphone and a speaker, so there is no -capability to declare, only a choice of what runs on them. +capability to declare, only a choice of what runs on them. The seventh, eighth +and ninth, `emet.stt`, `emet.llm` and `emet.tts`, are built from the soul's +`models` block and the body's together (§12.3 to §12.5). ```python class PocketSphinxWake(WakePlugin): @@ -931,6 +965,239 @@ permissively licensed, and its output feeds turn-taking (§13), which is personality rather than hardware. Adding a category later is a minor version bump; removing one is major. +### 12.3 Speech recognition plugins + +`emet.stt` is the seventh group, the first whose name comes from the soul, and +the first to have existed before anything real implemented it: the mock +shipped, then Deepgram, through the same seam. + +```python +class DeepgramTranscriber(TranscriberPlugin): + provider = "deepgram" # the string matched against models.stt.provider + + def __init__(self, config, fmt): + """The merged provider reference (provider, model, key_env, params) + and the audio format the wake engine already fixed. The key itself is + never in the config: read the environment variable `key_env` names, + in start(), and report unhealthy naming it when it is absent.""" + + def describe(self) -> TranscriberDescriptor: + """The model that answered, whether partials will arrive, and the + rate this instance will actually run at. Boot refuses a rate other + than the wake engine's: one microphone feeds both.""" + + async def feed(self, frame: bytes) -> Transcript | None: + """One frame in; the newest partial out if the text so far changed. + Every partial carries the whole utterance heard so far.""" + + async def finish(self) -> Transcript: + """The turn ended. Flush and return the final, empty if need be.""" +``` + +**Who chooses, and why it is the soul.** `models.stt` on the soul names the +provider, the model, and the environment variable holding the key. Keys are the +owner's (BYOK) and travel with the soul, and a cloud account is not hardware, so +principle 1 holds. The body may take the choice over with its own +`models.stt.provider`, carrying its own `model` and `key_env`: a test rig +running `mock`, or an owner whose key is for a different vendor than the soul's +author had. The override is whole or nothing, because a provider from one +document with a model and a key from the other is a broken reference. Whichever +provider runs, the body's `models.stt.params` ride along untouched: an endpoint, +a timeout, tuning for this deployment. `emet_sdk.models.model_selection` is the +rule in code, one rule for every stage, and the engine uses it rather than +restating it. + +A soul's provider is never checked against what is installed. A soul is valid +on every machine or on none, and a validator that asked would make the same +bundle valid on one laptop and invalid on the next. A body's `models.stt.provider` +is checked, and fails as `missing_plugin` like `audio.wake.engine`. Whether the +soul's provider is present is answered at boot, the way `identity.wake_word` is +answered against a live `WakeDescriptor`. + +**There is no default provider.** The other audio defaults name the hardware +floor, which every body has. Nobody can be assumed to hold a speech recognition +account, so an absent provider means no transcription, and asking for it anyway +(`emet-listen --transcribe`) is an error that names the field to set. Choosing a +vendor quietly would spend somebody's credits without asking. + +**Why a category, before any provider existed.** The same reason wake is one, +arrived at in advance rather than after a shutdown. A vendor's client library +is the easiest possible thing to build a release around, and there are credits +enough at one of them to do so without noticing. The seam was built first so +that the first real call was made through it. Behind the seam, the provider is +one line of a soul; in front of it, it would have been the shape of the engine. +The shipped Deepgram plugin speaks the wire protocol directly and depends on a +WebSocket client alone, for the same reason: a vendor SDK is the vendor's shape +arriving by another door. + +**Streaming is the contract; batch is the degenerate case.** Frames go in as +they are captured. A streaming provider answers with partials while the person +is still talking, each carrying the whole text so far, so a live caption +replaces its line rather than splicing fragments and a retracted word leaves +nothing behind. `finish()` returns the final. A batch provider returns nothing +from `feed()` and does its work in `finish()`, and the engine cannot tell the +two apart except by latency and by `describe().streaming`. + +**The transcriber hears everything after the wake**, lead-in silence included, +rather than only what the energy detector marked as speech. A quiet speaker the +detector missed is still transcribed. A false wake costs a provider a few +seconds of silence, which is the cheaper mistake. + +**Format.** The wake engine fixed the audio format before the transcriber was +built, and one microphone feeds both, so the transcriber receives the format +rather than stating one, and reports in `describe()` the rate it will actually +run at. The engine refuses a mismatch before it opens the microphone. A +recogniser hearing 16 kHz speech at 8 kHz does not fail; it produces nonsense. + +### 12.4 Language model plugins + +`emet.llm` is the eighth group, chosen the way `emet.stt` is chosen, and built +the same way: the contract and a mock first, then two vendors in one batch, +because a seam with one implementation is untested as a seam. + +```python +class AnthropicLanguageModel(LanguageModelPlugin): + provider = "anthropic" # the string matched against models.chat.provider + + def __init__(self, config): + """The merged provider reference: provider, model, key_env, params. + No audio format: a language model never hears the microphone.""" + + def describe(self) -> LanguageModelDescriptor: + """The model this instance will ask for, whether text streams, whether + tools are honoured, and whether the key and the network were there + at start().""" + + async def reply(self, prompt: Prompt) -> AsyncIterator[ReplyEvent]: + """Text as TextDeltas, a ToolCall per completed call, and one + ReplyDone last, on success and on failure alike.""" +``` + +**What the model is handed, and what it is not.** A `Prompt`: a `system` +string the engine assembled, the recent `messages`, and vendor-neutral +`tools`. The persona is the engine's to write into `system`; the self-model +(§7) and the memories the sensitivity floor lets through (§9.2) join it there +in their releases, and the seam does not change when they do. The body enters +the prompt through the self-model, never through a provider: a language model +plugin knows nothing about the robot it speaks for, which is what lets one +soul run on any body with any vendor. + +**Streaming is the contract.** A reply is spoken, and a person waits from the +end of their sentence to the first word of the answer, so `reply()` is an +async iterator and the first `TextDelta` is the number that matters. The +engine measures it (`llm first` in `emet-listen --stats`) beside the wait for +the whole reply. A provider that cannot stream sends one delta and says +`streaming: false`. + +**Tools are how side effects leave the model.** What is said comes back as +text. What is *done*, a memory written with its sensitivity (§9.2), a fact +looked up, comes back as a `ToolCall`, and the engine answers it with a `tool` +message and asks again. The vocabulary of tools is the engine's; the plugin +translates each `ToolSpec` to its vendor's shape and back. Expressive intents +(§5) are a separate question, decided when the conversation loop is built: a +tool round trip ends the text, so an intent that should land on a sentence +cannot travel as a call without stopping the speech it belongs to. + +**Stop reasons are the seam's, in five words.** `end`, `tool`, `length`, +`refusal`, `error`. A vendor's own names map onto them and anything new maps +to `end`, so a vendor adding a reason does not break a robot. `refusal` is +real: a provider's classifier may decline a request the person made in good +faith, and the robot should say that it will not answer rather than say +nothing. `error` carries the text so far, so a dropped connection mid-sentence +loses the rest of the sentence and not the turn. + +**No vendor SDK.** Both shipped plugins speak JSON over HTTPS and read +Server-Sent Events through one small HTTP client, for the reason given in +§12.3: a vendor SDK is the vendor's shape arriving by another door. The +OpenAI plugin speaks Chat Completions rather than the vendor's newer surface +because Chat Completions is what every other server speaks too: a local model, +a gateway, a company proxy. `params.url` points it at any of them, which is how +the offline mode reserved in §14 arrives as a manifest line. + +**What fails loudly, and where.** `start()` reads the key from the variable +the soul names and refuses to report healthy without one; then one cheap +request, a model listing, so a rejected key or an unreachable network is +known at boot. The engine refuses to run a robot that would hear questions +and never answer them, in the terms the owner can act on. + +### 12.5 Speech synthesis plugins + +`emet.tts` is the ninth group, chosen the way the other two are chosen, and +built the same way: the contract and a mock first, then the local voice, then +one cloud voice, because a seam with one implementation is untested as a seam +and a cloud voice is the implementation most likely to pull the engine out of +true. + +```python +class PiperVoice(VoicePlugin): + provider = "piper" # the string matched against models.tts.provider + + def __init__(self, config, voice): + """The merged provider reference (provider, model, key_env, params) + and the soul's `voice` block. `model` is the voice: a Piper model + name, a vendor's voice id. `voice.rate` is how fast the soul speaks, + a persona trait, and the one field every plugin honours.""" + + def describe(self) -> VoiceDescriptor: + """The voice this instance loaded, the sample rate its audio arrives + at, whether chunks arrive before the sentence is done, and whether + the model, the key or the library was there at start().""" + + async def speak(self, text: str) -> AsyncIterator[bytes]: + """One sentence in; mono int16 chunks at describe().sample_rate out, + in order. A failure raises PluginError after the audio so far, and + the engine loses that sentence, not the turn.""" +``` + +**Local by default, and why.** §14 has always placed synthesis locally: it +is the stage that costs the most per turn in the cloud, and the one a robot in +a home should manage with the network down. The shipped default is Piper, a +small neural voice on ONNX Runtime that a Raspberry Pi 5 runs in a tenth of +real time. Its current packaging, `piper-tts` 1.8.0 (PyPI, 2026-09-04), ships +one `cp39-abi3` wheel per platform, aarch64 Linux included, so the Pi needs no +compiler; it is GPL-3.0-or-later, because espeak-ng is compiled into it. Emet +imports it through the optional extra `emet-providers[piper]` and never bundles +it, so a body without the extra carries no GPL code, and `CITATIONS.md` records +the terms. A cloud voice is a line of a soul: the shipped one is Deepgram's +Aura over `POST /v1/speak`, raw linear16 streamed back at a rate the body +chooses, one request per sentence, at $0.030 per thousand characters (pricing +page, 2026-09-15). Deepgram's newer Flux TTS speaks a different protocol +(`/v2/speak`, raw audio over WebSocket only, its own turn and interrupt +messages) and is in the deferred register. + +**Voices are files, and their licences vary.** A Piper voice is a model and a +config the owner downloads once; the plugin looks in `params.path`, +`params.voices_dir`, `~/.local/share/emet/voices` and `/etc/emet/voices`, and +when it finds nothing it names the command that fixes it rather than fetching +sixty megabytes because a soul asked. Each voice on Hugging Face carries its +own model card. The reference soul names `en_US-ljspeech-medium` because LJ +Speech is public domain; the earlier reference voice, Amy, is fine-tuned from +the Blizzard 2013 Lessac corpus, whose licence forbids use in voice synthesis +products (read 2026-09-15), and a shipped default cannot rest on that. + +**Sentences, and why the reply streams.** A voice given one word at a time +loses the sentence's melody; a voice given the whole reply cannot start until +the model has stopped, which on the reference body is two seconds of silence. +So the engine watches the reply's text deltas for sentence ends, hands each +complete sentence to the voice at once, and plays its audio while the model is +still writing the next. Synthesis of sentence two overlaps playback of +sentence one; the voice is asked for one sentence at a time so that any +provider, streaming or not, fits. `emet-listen --stats` reports `voice first` +and `voice done`: the wait from the final transcript to the first sound at the +speaker, measured where a person hears it, and to the last. + +**The voice states the rate, the speaker conforms.** The wake rule (§12.2) +run the other way. A local model produces one sample rate and only one, and +the engine opens the sink at whatever the voice reports (22050 Hz for a Piper +medium voice, 24000 Hz for Aura by default) rather than resampling in +between, which would be a second place for audio to go quietly wrong. The +sink is the side that can convert, through the card's `plug` layer. + +**A sentence lost is a sentence lost.** A voice that fails on one sentence +raises; the engine logs it, counts it, prints it, and goes on with the next. +The text was still printed, and a robot that skipped a sentence is still a +robot that answered. + --- ## 13. Turn-taking @@ -951,9 +1218,9 @@ Four decisions, defaults chosen. All `P0`. Turn-taking is what separates charmin | Wake word | Local | Open plugin category (`emet.wake`). Default is a phonetic spotter, so any phrase works without a trained model. | | VAD, speaker ID | Local | Reflex tier. | | Gaze / DoA / face tracking | Local | Reflex tier. | -| STT | Cloud (streaming) | BYOK. | -| LLM | Cloud (streaming) | BYOK. Speak first sentence as it streams. | -| TTS | Local (Piper) | The expensive cloud stage; keep it local. Premium cloud voice is an opt-in toggle. | +| STT | Cloud (streaming) | BYOK. Open plugin category (`emet.stt`, §12.3); the soul names the provider and the body may take it over. | +| LLM | Cloud (streaming) | BYOK. Open plugin category (`emet.llm`, §12.4); Anthropic and OpenAI ship, and any Chat Completions server is a `url` away. Speak first sentence as it streams. | +| TTS | Local (Piper) | The expensive cloud stage; keep it local. Open plugin category (`emet.tts`, §12.5); Piper ships as the default, Deepgram Aura as the opt-in cloud voice, one line of a soul. | | Backchannel micro-model | Local | Fills the thinking gap. | | Vision understanding | Cloud (VLM), event-triggered | One frame on an explicit trigger. Never streamed. | | Nightly consolidation | Cloud (batch) | `RSV`. Nobody waiting; use the biggest model. | diff --git a/README.md b/README.md index 7017df2..7872deb 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ from a promise someone has to remember into something the software enforces. ```sh python -m venv .venv -.venv/bin/pip install -e "emet-sdk[dev]" -e "emet-hal[dev]" -e "emet-engine[dev]" # Windows: .venv\Scripts\pip +.venv/bin/pip install -e "emet-sdk[dev]" -e "emet-hal[dev]" -e "emet-providers[dev]" -e "emet-engine[dev]" # Windows: .venv\Scripts\pip cd emet-sdk emet validate examples/mock-scout.yaml --verify-drivers @@ -120,6 +120,7 @@ tells you which rungs were skipped and what was wrong with each: |---|---| | `emet-sdk/` | Types, schemas, the intent vocabulary, chain resolution. The contract everything agrees on. | | `emet-hal/` | Drivers and locomotion plugins. Where hardware support goes. | +| `emet-providers/` | The plugins that reach a service, or a model on disk: speech recognition, language models and voices. | | `emet-engine/` | The listen loop: wake, endpointing, audio in and out. Personality, memory and arbitration arrive from 0.4. | ## Documentation diff --git a/RELEASING.md b/RELEASING.md index 0d86702..160bd89 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -115,7 +115,7 @@ that is merely out of date, so re-read: ## 8. Tag -- The version is bumped in all three `pyproject.toml` files and **nowhere +- The version is bumped in all four `pyproject.toml` files and **nowhere else**. `release_check.py` enforces this. - The tag message is written. Commits stay short; the tag carries the detail. - Every commit in the release is signed off, or the DCO check fails the PR. diff --git a/emet-engine/README.md b/emet-engine/README.md index 18b9d4c..4869ebd 100644 --- a/emet-engine/README.md +++ b/emet-engine/README.md @@ -8,10 +8,66 @@ discovery, never by import. See the layering check in `tools/check_layering.py`. ```sh emet-listen path/to/manifest.yaml path/to/soul.yaml emet-listen path/to/manifest.yaml path/to/soul.yaml --replay recording.wav +emet-listen path/to/manifest.yaml path/to/soul.yaml --transcribe +emet-listen path/to/manifest.yaml path/to/soul.yaml --reply +emet-listen path/to/manifest.yaml path/to/soul.yaml --speak ``` Needs `emet-hal[audio,wake]` installed alongside for a microphone and a -detector to exist. +detector to exist, and `emet-providers` for `--transcribe` to have a speech +recognition provider to hand the speech to. The soul names the provider under +`models.stt`; a body may take the choice over under `audio.stt`. The reference +soul names `deepgram`, which needs `emet-providers[deepgram]` and a key +exported as `EMET_DEEPGRAM_KEY`. Without a key, or offline, the body can take +over with `mock`, which reads words out of the bytes it is given, so on a real +microphone give it a line to say: + +```yaml +audio: + stt: + provider: mock + params: {transcript: "testing the seam"} +``` + +Partials print as they arrive, one word a frame from the mock, and the final +prints as `said`. With `--stats`, an `stt final` line reports the wait from +the endpoint to the final transcript, which is the first latency a person +feels. + +Keys are read from a file rather than exported in every shell: put +`NAME=value` lines in `~/.config/emet/keys.env` (or `/etc/emet/keys.env` for +an installed robot, or a file named with `--keys`), `chmod 600` it, and +`emet-listen` loads them before any provider starts. An exported variable +always wins over the file. The header prints which names were loaded and from +where; values are never printed. + +`--reply` goes one step further and implies `--transcribe`: what was said +goes to the language model the soul names under `models.chat`, with the +soul's persona as the system prompt and the run's conversation so far, and +the answer prints as it streams after `reply:`. The reference soul names +`openai`, which needs `emet-providers[openai]` and `EMET_OPENAI_KEY`; +`anthropic` needs its extra and `EMET_ANTHROPIC_KEY`; the mock needs +neither and repeats what it heard. `--stats` adds `llm first` and `llm done`, +the wait from the final transcript to the first word and to the whole reply. + +`--speak` goes the last step and implies `--reply`: the answer is said +through the voice the soul names under `models.tts`, a sentence at a time as +the model writes it, so the first sentence is heard while the second is still +arriving. The reference soul names `piper`, the local voice, which needs +`emet-providers[piper]` and a voice model downloaded once: + +```sh +pip install -e "emet-providers[piper]" +python -m piper.download_voices en_US-ljspeech-medium --data-dir ~/.local/share/emet/voices +``` + +A cloud voice is one line of the soul (`{provider: deepgram, model: +"aura-2-thalia-en", key_env: "EMET_DEEPGRAM_KEY"}`); the mock voice spells +the words into the audio and needs nothing. The sink is opened at whatever +rate the voice reports, 22050 Hz for a Piper medium voice, 24000 Hz for +Aura, so `--speak` and `--echo` exclude each other. `--stats` adds `voice +first` and `voice done`: the wait from the final transcript to the first +sound at the speaker, and to the last. ## Checking that it keeps up diff --git a/emet-engine/emet_engine/__init__.py b/emet-engine/emet_engine/__init__.py index 633c84a..ace1a57 100644 --- a/emet-engine/emet_engine/__init__.py +++ b/emet-engine/emet_engine/__init__.py @@ -4,11 +4,12 @@ Not for tidiness. The engine is where somebody would reach for a concrete servo or a concrete microphone, and one `from emet_hal.differential import ...` would quietly end the claim that the engine holds no hardware knowledge. Every -driver, detector and audio source arrives by name through entry-point -discovery instead. +driver, detector, audio source, transcriber and language model arrives by +name through entry-point discovery instead. -`emet-hal` is therefore not a dependency of this package. It is a dependency of -a working robot, which is a different thing: install it alongside. +`emet-hal` and `emet-providers` are therefore not dependencies of this package. +They are dependencies of a working robot, which is a different thing: install +them alongside. """ from emet_engine.session import EngineError, ListenSession diff --git a/emet-engine/emet_engine/cli.py b/emet-engine/emet_engine/cli.py index 4d7f47c..db442b3 100644 --- a/emet-engine/emet_engine/cli.py +++ b/emet-engine/emet_engine/cli.py @@ -1,8 +1,13 @@ """`emet-listen`: bring a body up and print what it hears. -The 0.3 milestone in one command. It does not understand anything yet: it +The 0.3 milestone in one command, and the steps of 0.4 behind flags. It brings up a microphone and a wake detector, and says so each time the robot -hears its name. Speech recognition arrives in 0.4. +hears its name. With `--transcribe` it also hands what follows to the speech +recognition provider the soul names and prints what was said, partials as +they arrive and then the final. With `--reply` it hands what was said to the +language model the soul names and prints the answer as it streams. With +`--speak` the answer is also said, through the voice the soul names and out +of the speaker, a sentence at a time while the rest is still being written. Useful before that, though. `--replay` points the same path at a recording instead of a microphone, so a wake failure somebody reports can be reproduced @@ -19,6 +24,7 @@ from typing import Any from emet_sdk.discovery import PluginRegistry +from emet_sdk.types import ReplyDone, TextDelta, Transcript, WakeEvent from emet_sdk.validate import ( MissingPluginError, ValidationError, @@ -27,6 +33,7 @@ validate_soul, ) +from emet_engine.keys import load_keys from emet_engine.session import EngineError, ListenSession from emet_engine.turn import EndReason @@ -63,6 +70,16 @@ def _replay(manifest: dict[str, Any], wav: str) -> dict[str, Any]: return manifest +def _on_wake(event: WakeEvent) -> None: + """Printed the moment the name is heard, not when the turn ends: on a live + run that is the difference between feedback and a second of doubt.""" + print(f" heard {event.phrase!r} (confidence {event.confidence:.2f})") + + +def _on_partial(transcript: Transcript) -> None: + print(f" hearing {transcript.text!r}") + + async def _run(args: argparse.Namespace) -> int: registry = PluginRegistry.discover() if not registry: @@ -74,6 +91,16 @@ async def _run(args: argparse.Namespace) -> int: ) return 2 + # Keys before providers: the plugins read the environment in start(). + # Names are printed, values never are. + try: + loaded = load_keys(args.keys) + except (FileNotFoundError, ValueError) as exc: + raise EngineError(str(exc)) from exc + for entry in loaded: + if entry.warning: + print(f" ! {entry.warning}", file=sys.stderr) + manifest = _load(Path(args.manifest), "manifest", registry) soul = _load(Path(args.soul), "soul", registry) if args.replay: @@ -81,25 +108,55 @@ async def _run(args: argparse.Namespace) -> int: if args.echo: # Echo replays captured *input* audio, so the sink has to run at the # input's rate rather than the synthesis rate it would normally use. - # Once there is speech synthesis this goes away: the sink will run at - # whatever the voice produces and nothing will need to match. + # Under --speak the session opens the sink at the voice's rate + # instead, which is why the two flags exclude each other. manifest = copy.deepcopy(manifest) output = manifest.setdefault("audio", {}).setdefault("output", {}) output["sample_rate"] = int( (manifest["audio"].get("input") or {}).get("sample_rate") or 16000 ) - session = ListenSession(manifest, soul, registry=registry) + # Speaking needs a reply to speak, and a reply needs words to reply to, + # so --speak implies --reply implies --transcribe. + reply = args.reply or args.speak + transcribe = args.transcribe or reply + session = ListenSession( + manifest, + soul, + registry=registry, + transcribe=transcribe, + reply=reply, + speak=args.speak, + on_wake=_on_wake, + on_partial=_on_partial if transcribe else None, + ) async with session: assert session.descriptor is not None and session.format is not None - print( - f"listening for {session.phrase!r}\n" - f" engine {session.engine_name}\n" - f" source {session.source_name}\n" - f" audio {session.format.sample_rate} Hz, " - f"{session.format.frame_ms:.0f} ms frames\n" - f" patience {session.patience_ms} ms" - ) + lines = [ + f"listening for {session.phrase!r}", + f" engine {session.engine_name}", + f" source {session.source_name}", + f" audio {session.format.sample_rate} Hz, {session.format.frame_ms:.0f} ms frames", + f" patience {session.patience_ms} ms", + ] + if transcribe: + described = session.stt_descriptor + model = f", model {described.model}" if described and described.model else "" + mode = "streaming" if described and described.streaming else "batch" + lines.append(f" stt {session.stt_name}{model} ({mode})") + if reply: + described_llm = session.llm_descriptor + model = f", model {described_llm.model}" if described_llm and described_llm.model else "" + lines.append(f" llm {session.chat_name}{model}") + if args.speak: + described_tts = session.tts_descriptor + model = f", model {described_tts.model}" if described_tts and described_tts.model else "" + rate = f" ({described_tts.sample_rate} Hz)" if described_tts else "" + lines.append(f" voice {session.tts_name}{model}{rate}") + for entry in loaded: + names = ", ".join(entry.names) or "nothing new" + lines.append(f" keys {names} from {entry.path}") + print("\n".join(lines)) print(" (ctrl-c to stop)\n" if not args.replay else "") heard = 0 @@ -107,11 +164,7 @@ async def _run(args: argparse.Namespace) -> int: try: async for event, utterance in session.turns(): heard += 1 - print(f" heard {event.phrase!r} (confidence {event.confidence:.2f})") if utterance.had_speech: - # 0.4 hands this audio to speech recognition. Until then - # the useful thing to show is that the turn was bounded - # correctly. print( f" then {utterance.duration_ms / 1000:.1f}s of speech, " f"ended on {utterance.reason.value}" @@ -123,6 +176,13 @@ async def _run(args: argparse.Namespace) -> int: print(" the recording ended before anything followed.") else: print(" then nothing. probably a false wake.") + if utterance.transcript is not None: + if utterance.transcript.text: + print(f" said {utterance.transcript.text!r}") + else: + print(" said nothing the provider could make out") + if reply and utterance.transcript is not None and utterance.transcript.text.strip(): + await _print_reply(session, utterance.transcript.text, speak=args.speak) except asyncio.CancelledError: # Ctrl-C. `asyncio.run` answers SIGINT by cancelling this task, and # a microphone never ends on its own, so this is how every live @@ -143,6 +203,30 @@ async def _run(args: argparse.Namespace) -> int: return 0 +async def _print_reply(session: ListenSession, text: str, *, speak: bool = False) -> None: + """Stream the reply to the console as the model produces it, and with + `speak`, out of the speaker as each sentence completes.""" + print(" reply: ", end="", flush=True) + events = session.answer_aloud(text) if speak else session.answer(text) + async for event in events: + if isinstance(event, TextDelta): + print(event.text, end="", flush=True) + elif isinstance(event, ReplyDone): + print() + if event.stop_reason == "error": + print(f" the language model failed: {event.error}") + elif event.stop_reason == "refusal": + print(" (the provider declined to answer)") + elif event.stop_reason == "length": + print(" (cut off at the reply's token cap)") + if speak: + spoken = session.last_spoken + said = [s for s in spoken if s.ok and s.audio_bytes] + print(f" spoke {len(said)} sentence(s)") + for lost in (s for s in spoken if not s.ok): + print(f" the voice could not say {lost.text!r}: {lost.error}") + + def _finish(session: ListenSession, args: argparse.Namespace) -> None: """Report what the run cost, if asked, and always report what it lost.""" if args.stats: @@ -153,11 +237,11 @@ def _finish(session: ListenSession, args: argparse.Namespace) -> None: hint = session.warm_start_hint() if hint: print("\n" + hint) - if session.dropped and args.echo: + if session.dropped and (args.echo or args.reply or args.speak): print( f"\nnote: {session.dropped} frame(s) were dropped while the robot was " - f"speaking. The loop does not read the microphone during playback; " - f"barge-in, in 1.0, is what changes that." + f"speaking or thinking. The loop does not read the microphone during " + f"playback or a reply; barge-in, in 1.0, is what changes that." ) elif session.dropped: print( @@ -179,12 +263,48 @@ def main(argv: list[str] | None = None) -> int: help="read this 16-bit mono wav instead of the microphone, so a wake " "failure can be reproduced away from the room it happened in", ) - parser.add_argument( + playback = parser.add_mutually_exclusive_group() + playback.add_argument( "--echo", action="store_true", - help="play each captured utterance back through the output. There is " - "no speech synthesis yet, so this is what proves the whole duplex path " - "works: audio in, wake, endpoint, audio out", + help="play each captured utterance back through the output: audio in, " + "wake, endpoint, audio out, with no voice in between. Excludes --speak, " + "which opens the output at the voice's rate rather than the input's", + ) + parser.add_argument( + "--transcribe", + action="store_true", + help="hand the speech after each wake to the speech recognition provider " + "the soul names under models.stt (or the body takes over under " + "audio.stt), and print what was said: partials as they arrive, then the " + "final. The reference soul names deepgram, which needs " + "emet-providers[deepgram] and a key in EMET_DEEPGRAM_KEY; `mock` reads " + "words out of the bytes it is given and needs neither", + ) + parser.add_argument( + "--keys", + metavar="FILE", + help="read provider keys from this file, one NAME=value a line, into the " + "environment before anything starts. Without it, /etc/emet/keys.env and " + "~/.config/emet/keys.env are read when they exist. A variable already " + "exported is never overwritten. Values are never printed", + ) + parser.add_argument( + "--reply", + action="store_true", + help="hand what was said to the language model the soul names under " + "models.chat (or the body takes over) and print the reply as it streams. " + "Implies --transcribe", + ) + playback.add_argument( + "--speak", + action="store_true", + help="say the reply through the voice the soul names under models.tts " + "(or the body takes over) and out of the audio sink, a sentence at a " + "time while the rest is still being written; the sink runs at the " + "voice's sample rate. Implies --reply. The reference soul names piper, " + "which needs emet-providers[piper] and a voice downloaded once; " + "`mock` needs neither and makes a sound no one could mistake for speech", ) parser.add_argument( "--stats", diff --git a/emet-engine/emet_engine/keys.py b/emet-engine/emet_engine/keys.py new file mode 100644 index 0000000..7a3510d --- /dev/null +++ b/emet-engine/emet_engine/keys.py @@ -0,0 +1,168 @@ +"""Keys, kept out of the shell history and out of every document. + +Emet is bring-your-own-key. A soul names the environment variable each +provider reads (`key_env`); the value has to come from somewhere, and +"export it in every new shell" is how keys end up in shell history, in chat +logs and in screenshots. `keys.env` is the answer: one file, `NAME=value` a +line, read once at start-up and put into the environment for the plugins to +find. The environment still wins: a variable that is already exported is +never overwritten, so a one-off key for one run works as it always did. + +**Where it lives.** `--keys PATH` on the command line, or the `EMET_KEYS` +variable, names one file and it must exist. Otherwise the two default places +are tried: `~/.config/emet/keys.env` for a person (`XDG_CONFIG_HOME` honoured +when set), and `/etc/emet/keys.env` for a system install (DESIGN.md section 3 +puts the config root at `/etc/emet/`). Both are read when both exist. Nothing +ever overwrites a variable that is already set, so the personal file is read +first and its keys win over the machine's, and both lose to the shell. + +**Format.** The plain subset of dotenv: `NAME=value`, an optional `export ` +in front, `#` comments, blank lines, and a value in single or double quotes +has the quotes removed. No interpolation and no multi-line values: a key is +one token, and a file that needs more than that is not a keys file. + +**Permissions.** The file should be readable by its owner alone. On POSIX a +file that group or others can read is loaded anyway, with a warning naming +it and the `chmod 600` that fixes it. Refusing would strand somebody on their +first evening; a warning they see every run teaches the habit. + +Values never appear in a log, a header line or an error. Names do. +""" + +from __future__ import annotations + +import os +import re +import stat +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import MutableMapping + +__all__ = ["LoadedKeys", "SYSTEM_KEYS", "default_paths", "parse_env", "load_keys", "user_keys_path"] + +#: The machine's keys, for an installed robot. +SYSTEM_KEYS = Path("/etc/emet/keys.env") + +_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +def user_keys_path() -> Path: + """A person's keys: `$XDG_CONFIG_HOME/emet/keys.env`, or `~/.config/emet/keys.env`.""" + base = os.environ.get("XDG_CONFIG_HOME") + root = Path(base) if base else Path.home() / ".config" + return root / "emet" / "keys.env" + + +def default_paths() -> list[Path]: + """Where keys are looked for when nothing names a file. The personal file + first: a variable once set is never overwritten, so first read wins.""" + return [user_keys_path(), SYSTEM_KEYS] + + +@dataclass(frozen=True, slots=True) +class LoadedKeys: + """What one file contributed. Names only, never values.""" + + path: Path + #: Variables this file put into the environment. + names: tuple[str, ...] + #: Variables the file held that the environment already had. Left alone. + kept: tuple[str, ...] + #: A permissions warning, or None. + warning: str | None = None + + +def parse_env(text: str, *, source: str = "keys.env") -> dict[str, str]: + """Parse the dotenv subset described in the module docstring. + + Raises `ValueError` naming the line for anything else: a bad keys file + should stop the robot with a line number, not run without a key and fail + later in a vendor's words. + """ + out: dict[str, str] = {} + for number, raw in enumerate(text.splitlines(), 1): + line = raw.strip() + if not line or line.startswith("#"): + continue + if line.startswith("export "): + line = line[len("export ") :].lstrip() + name, sep, value = line.partition("=") + name = name.strip() + if not sep or not _NAME.match(name): + raise ValueError( + f"{source}:{number}: expected NAME=value, got {raw.strip()!r}. " + f"One key a line, no spaces around the name." + ) + value = value.strip() + if value[:1] in ("'", '"'): + # A quoted value ends at its closing quote; a comment may follow. + end = value.find(value[0], 1) + if end == -1: + raise ValueError(f"{source}:{number}: unclosed quote in the value for {name}") + value = value[1:end] + elif " #" in value: + # An unquoted value ends at a trailing comment. + value = value.split(" #", 1)[0].rstrip() + out[name] = value + return out + + +def _permissions_warning(path: Path) -> str | None: + if sys.platform.startswith("win"): + return None + try: + mode = path.stat().st_mode + except OSError: + return None + if mode & (stat.S_IRGRP | stat.S_IROTH): + return ( + f"{path} is readable by other users on this machine. Keys belong to " + f"you alone: chmod 600 {path}" + ) + return None + + +def load_keys( + path: Path | str | None = None, + *, + environ: MutableMapping[str, str] = os.environ, +) -> list[LoadedKeys]: + """Read the keys file(s) into `environ`, never overwriting what is there. + + With `path`, or `EMET_KEYS` in the environment, that one file is read and + must exist (`FileNotFoundError`). Otherwise every default file that exists + is read, in order. Returns one `LoadedKeys` per file read, so a caller can + say which names came from where without ever saying what they were. + """ + named = path if path is not None else environ.get("EMET_KEYS") + if named: + candidates = [Path(named).expanduser()] + if not candidates[0].is_file(): + raise FileNotFoundError( + f"no keys file at {candidates[0]}. Create it with one NAME=value a " + f"line, or drop --keys to use the default places." + ) + else: + candidates = [p for p in default_paths() if p.is_file()] + + loaded: list[LoadedKeys] = [] + for candidate in candidates: + values = parse_env(candidate.read_text(encoding="utf-8"), source=str(candidate)) + names: list[str] = [] + kept: list[str] = [] + for name, value in values.items(): + if environ.get(name): + kept.append(name) + continue + environ[name] = value + names.append(name) + loaded.append( + LoadedKeys( + path=candidate, + names=tuple(names), + kept=tuple(kept), + warning=_permissions_warning(candidate), + ) + ) + return loaded diff --git a/emet-engine/emet_engine/metrics.py b/emet-engine/emet_engine/metrics.py index 37fcc4c..5534f42 100644 --- a/emet-engine/emet_engine/metrics.py +++ b/emet-engine/emet_engine/metrics.py @@ -57,6 +57,11 @@ def __enter__(self) -> "Stopwatch": def __exit__(self, *exc: object) -> None: self.elapsed_ms = (time.perf_counter() - self._start) * 1000.0 + def peek_ms(self) -> float: + """Milliseconds so far, without stopping. For a first-token mark + inside a block that keeps running.""" + return (time.perf_counter() - self._start) * 1000.0 + @dataclass class SessionStats: @@ -76,11 +81,38 @@ class SessionStats: #: reported an input overflow. A different cause, the same loss, and the #: first thing to check when a live run's clock skew looks like drift. overflows: int = 0 + #: Of `dropped`, the frames lost while the loop was deliberately not + #: reading: playing audio, or waiting on a language model. Those are the + #: loop's own doing until barge-in arrives, and they say nothing about + #: whether it keeps up while listening, so the verdict leaves them out. + dropped_busy: int = 0 process_ms_total: float = 0.0 process_ms_max: float = 0.0 over_budget: int = 0 + #: Milliseconds from the endpoint to the final transcript, one per + #: transcribed turn. The first number a person feels in 0.4: the robot + #: cannot start thinking until this has elapsed, so it is measured on + #: every turn rather than guessed from a vendor's page. + final_ms: list[float] = field(default_factory=list) + + #: Per answered turn: milliseconds from the final transcript to the first + #: word of the reply, and to the whole reply. The first is what a person + #: waits in silence; the second is what a speaker would need to keep up. + reply_first_ms: list[float] = field(default_factory=list) + reply_done_ms: list[float] = field(default_factory=list) + + #: Per spoken reply: milliseconds from the final transcript to the first + #: sound at the sink, and to the last. The first is the silence a person + #: actually sits through, measured where they hear it rather than where + #: the model produced it; the second is how long the robot held the + #: floor. Sentences the voice managed and lost are counted beside them. + speech_first_ms: list[float] = field(default_factory=list) + speech_done_ms: list[float] = field(default_factory=list) + sentences_spoken: int = 0 + sentences_lost: int = 0 + _samples: deque[float] = field(default_factory=lambda: deque(maxlen=SAMPLE_CAP)) #: Set by the first frame, so that loading the acoustic model and opening #: the card are left out. What remains is the card's clock against the @@ -99,6 +131,21 @@ def record_frame(self, process_ms: float) -> None: self.over_budget += 1 self._samples.append(process_ms) + def record_final(self, wait_ms: float) -> None: + self.final_ms.append(wait_ms) + + def record_reply(self, first_ms: float | None, done_ms: float) -> None: + if first_ms is not None: + self.reply_first_ms.append(first_ms) + self.reply_done_ms.append(done_ms) + + def record_speech(self, first_ms: float | None, done_ms: float, *, spoken: int, lost: int) -> None: + if first_ms is not None: + self.speech_first_ms.append(first_ms) + self.speech_done_ms.append(done_ms) + self.sentences_spoken += spoken + self.sentences_lost += lost + # ------------------------------------------------------------ readings @property @@ -148,11 +195,18 @@ def percentile(self, p: float) -> float: def kept_up(self) -> bool: """Whether this run is evidence the loop is viable here. - Three conditions, and all of them matter. Nothing was dropped, no frame - blew the budget, and there is real margin rather than a bare pass. A - run at 0.99 kept up on a quiet machine and will not on a busy one. + Three conditions, and all of them matter. Nothing was dropped while + listening, no frame blew the budget, and there is real margin rather + than a bare pass. A run at 0.99 kept up on a quiet machine and will not + on a busy one. Frames dropped while the robot was speaking or thinking + are counted, printed, and left out of this: the loop chose not to read + the microphone then, and barge-in is what changes that. """ - return self.dropped == 0 and self.over_budget == 0 and self.realtime_factor < 0.5 + return ( + self.dropped - self.dropped_busy == 0 + and self.over_budget == 0 + and self.realtime_factor < 0.5 + ) # -------------------------------------------------------------- report @@ -174,8 +228,40 @@ def report(self, *, live: bool) -> str: f"{self.over_budget} frame(s) over", f" realtime {self.realtime_factor:.4f} " f"({self.headroom:.0f}x faster than realtime)", - f" dropped {self.dropped} (card overflows {self.overflows})", + f" dropped {self.dropped} (card overflows {self.overflows}" + + (f"; {self.dropped_busy} while speaking or thinking" if self.dropped_busy else "") + + ")", ] + if self.final_ms: + lines.append( + f" stt final mean {sum(self.final_ms) / len(self.final_ms):.0f} ms " + f"max {max(self.final_ms):.0f} ({len(self.final_ms)} turn(s), " + f"endpoint to final transcript)" + ) + if self.reply_first_ms: + lines.append( + f" llm first mean {sum(self.reply_first_ms) / len(self.reply_first_ms):.0f} ms " + f"max {max(self.reply_first_ms):.0f} ({len(self.reply_first_ms)} reply(ies), " + f"transcript to first word)" + ) + if self.reply_done_ms: + lines.append( + f" llm done mean {sum(self.reply_done_ms) / len(self.reply_done_ms):.0f} ms " + f"max {max(self.reply_done_ms):.0f} (transcript to whole reply)" + ) + if self.speech_first_ms: + lines.append( + f" voice first mean {sum(self.speech_first_ms) / len(self.speech_first_ms):.0f} ms " + f"max {max(self.speech_first_ms):.0f} ({len(self.speech_first_ms)} reply(ies), " + f"transcript to first sound)" + ) + if self.speech_done_ms: + lost = f", {self.sentences_lost} lost" if self.sentences_lost else "" + lines.append( + f" voice done mean {sum(self.speech_done_ms) / len(self.speech_done_ms):.0f} ms " + f"max {max(self.speech_done_ms):.0f} (transcript to last sound; " + f"{self.sentences_spoken} sentence(s) spoken{lost})" + ) if live: skew = self.wall_ms - self.audio_ms pct = (skew / self.audio_ms * 100.0) if self.audio_ms else 0.0 diff --git a/emet-engine/emet_engine/prompting.py b/emet-engine/emet_engine/prompting.py new file mode 100644 index 0000000..7542e19 --- /dev/null +++ b/emet-engine/emet_engine/prompting.py @@ -0,0 +1,93 @@ +"""What the language model is told, and what it remembers of the conversation. + +The seed of `DESIGN.md` section 7. Today the system prompt is the soul's own +persona and a rule about speaking aloud. The self-model (0.5) and the +memories the sensitivity floor lets through (0.6) join it here, in this file, +and nothing below this line changes when they do: a `Prompt` is what leaves, +whatever went into it. + +The conversation is short on purpose. A robot on a desk is spoken to in +bursts, and a language model is billed on everything it is shown, so only +the most recent turns ride along. Memory, the long kind, is a different +mechanism with its own release. +""" + +from __future__ import annotations + +from typing import Any, Mapping, Sequence + +from emet_sdk.types import Message, Prompt, ToolSpec + +__all__ = ["SPOKEN_ALOUD", "DEFAULT_MAX_TOKENS", "DEFAULT_TURNS", "system_prompt", "Conversation"] + +#: Appended to every system prompt. A reply is heard, so it has to be +#: listenable: short, plain, said rather than laid out. Honesty about limits +#: is principle 6, and it belongs in the prompt before the self-model arrives +#: to make it specific. +SPOKEN_ALOUD = ( + "You are speaking aloud through a small robot's speaker. Answer in one or " + "two short sentences a person can listen to. Plain speech: no lists, no " + "markdown, no emoji, no stage directions. If you cannot do something or do " + "not know, say so plainly." +) + +#: One spoken reply, bounded. Two sentences are well under this; the cap is +#: there so a model that ignores the rule is cut off rather than read a page. +DEFAULT_MAX_TOKENS = 300 + +#: How many recent messages travel with each request. Twelve turns each way. +DEFAULT_TURNS = 24 + + +def system_prompt(soul: Mapping[str, Any]) -> str: + """The soul's persona, then the rule about speaking aloud. + + `persona.system_prompt` is used when the soul wrote one; otherwise the + prompt is built from `identity.name` and `persona.summary`, which every + soul has. The soul names no hardware here and neither does this: the body + enters the prompt in 0.5, through the self-model. + """ + identity = soul.get("identity") or {} + persona = soul.get("persona") or {} + written = str(persona.get("system_prompt") or "").strip() + if not written: + name = str(identity.get("name") or "Emet").strip() + summary = " ".join(str(persona.get("summary") or "").split()) + written = f"You are {name}." + (f" {summary}" if summary else "") + return f"{written}\n\n{SPOKEN_ALOUD}" + + +class Conversation: + """The recent turns, as the model will be shown them. + + `add_user` and `add_assistant` append; `prompt` packages the tail with a + system prompt. Trimming drops the oldest turns in pairs so the history + still begins with the person speaking, which some providers require. + """ + + def __init__(self, *, turns: int = DEFAULT_TURNS) -> None: + self.turns = max(2, turns) + self.messages: list[Message] = [] + + def add_user(self, text: str) -> None: + self.messages.append(Message(role="user", content=text)) + self._trim() + + def add_assistant(self, text: str) -> None: + self.messages.append(Message(role="assistant", content=text)) + self._trim() + + def _trim(self) -> None: + while len(self.messages) > self.turns: + del self.messages[:2] + while self.messages and self.messages[0].role != "user": + del self.messages[0] + + def prompt( + self, + system: str, + *, + tools: Sequence[ToolSpec] = (), + max_tokens: int = DEFAULT_MAX_TOKENS, + ) -> Prompt: + return Prompt(system=system, messages=tuple(self.messages), tools=tuple(tools), max_tokens=max_tokens) diff --git a/emet-engine/emet_engine/session.py b/emet-engine/emet_engine/session.py index 4265fa2..6286b0c 100644 --- a/emet-engine/emet_engine/session.py +++ b/emet-engine/emet_engine/session.py @@ -1,11 +1,16 @@ -"""The listen loop: audio in, wake events out. +"""The listen loop: audio in, wake events out, and since 0.4 the words after. This is the first thing in Emet that is neither a contract nor a driver. It takes a body manifest and a soul bundle, brings up the two pieces of hardware the floor guarantees, and produces an event each time the robot hears its name. +Asked to, it also hands the speech that follows to a recogniser and returns +what was said, hands what was said to a language model and streams what the +robot would answer, and hands the answer, a sentence at a time as it +streams, to a voice and out through the speaker. **It imports `emet_sdk` and nothing else.** Microphones and wake detectors live -in `emet_hal`, and this module never names that package. Both arrive by name +in `emet_hal`, transcribers, language models and voices in `emet_providers`, +and this module never names either package. All of them arrive by name through entry-point discovery, which is the only reason a layering rule that forbids the import and an engine that needs a microphone can both be true. @@ -14,22 +19,44 @@ configured to match. Doing it the other way, opening the microphone at whatever rate the manifest mentions and hoping the detector agrees, is how a robot ends up running perfectly and hearing nothing, because feeding 48 kHz -audio to a 16 kHz model does not raise anything. It just stops working. +audio to a 16 kHz model does not raise anything. It just stops working. The +transcriber is built between the two: it is handed the detector's format, +reports the rate it will actually run at, and a mismatch is refused before the +microphone is ever opened. The output side runs the same rule the other way +round: the voice states the rate its audio arrives at, and the speaker is +opened to match, so a local voice that produces one rate and only one is +never resampled by the engine. **The boot check that has no fallback.** Every other missing piece degrades: a chain that cannot find a head falls through to a light ring, and one that finds nothing still speaks. A robot that cannot hear its own name has no next rung, so `start()` refuses rather than running deaf. That check is the entire reason -`WakeDescriptor.can_detect` exists. +`WakeDescriptor.can_detect` exists. A transcriber that cannot start is refused +on the same terms: what people say would go nowhere, and P0 fails loudly. """ from __future__ import annotations import logging -from typing import Any, AsyncIterator, Mapping +from dataclasses import replace +from typing import Any, AsyncIterator, Callable, Mapping from emet_sdk.discovery import PluginRegistry -from emet_sdk.types import AudioFormat, AudioSink, AudioSource, WakeDescriptor, WakeEvent +from emet_sdk.models import chat_selection, stt_selection, tts_selection +from emet_sdk.types import ( + AudioFormat, + AudioSink, + AudioSource, + LanguageModelDescriptor, + ReplyDone, + ReplyEvent, + TextDelta, + Transcript, + TranscriberDescriptor, + VoiceDescriptor, + WakeDescriptor, + WakeEvent, +) from emet_sdk.validate import ( DEFAULT_AUDIO_SINK, DEFAULT_AUDIO_SOURCE, @@ -37,6 +64,8 @@ ) from emet_engine.metrics import SessionStats, Stopwatch +from emet_engine.prompting import DEFAULT_MAX_TOKENS, Conversation, system_prompt +from emet_engine.speech import Mouth, Sentences, SpokenSentence from emet_engine.turn import DEFAULT_PATIENCE_MS, Endpointer, Utterance __all__ = ["EngineError", "ListenSession"] @@ -58,6 +87,17 @@ class ListenSession: The iterator ends when the source does, which a file always does and a microphone never should. + + `turns()` yields once per turn, after the person has stopped talking. Two + callbacks let a caller show something while the turn is still going: + `on_wake` fires the moment the name is heard, and `on_partial` fires with + each partial transcript while a transcriber is listening. + + `answer(text)` is the next step, taken by the caller after a turn: the + words go to the language model with the persona and the conversation so + far, and the reply streams back as events. The caller decides whether to + print it, speak it, or both. `answer_aloud(text)` is both: the same + events, and each sentence spoken through the voice as it completes. """ def __init__( @@ -66,6 +106,11 @@ def __init__( soul: Mapping[str, Any], *, registry: PluginRegistry | None = None, + transcribe: bool = False, + reply: bool = False, + speak: bool = False, + on_wake: Callable[[WakeEvent], None] | None = None, + on_partial: Callable[[Transcript], None] | None = None, ) -> None: self.manifest = manifest self.soul = soul @@ -81,6 +126,35 @@ def __init__( self.source_name = str(self._input_block.get("source") or DEFAULT_AUDIO_SOURCE) self.sink_name = str(self._output_block.get("sink") or DEFAULT_AUDIO_SINK) + #: The provider reference the soul and the body agree on, or None when + #: neither names one. Resolved here, unconditionally, so that a caller + #: can report what *would* run without asking for it to run. + self.stt = stt_selection(manifest, soul) + self.stt_name: str | None = self.stt["provider"] if self.stt else None + #: Whether to build the transcriber and feed it. Off by default: the + #: 0.3 loop, unchanged, for anyone who only wants to know that the + #: robot heard its name. + self.transcribe = transcribe + self.on_wake = on_wake + self.on_partial = on_partial + + #: The language model the soul and the body agree on, and whether to + #: bring it up. The persona becomes the system prompt here, once; the + #: conversation accumulates across turns for the length of the run. + self.chat = chat_selection(manifest, soul) + self.chat_name: str | None = self.chat["provider"] if self.chat else None + self.reply = reply + self.system_prompt = system_prompt(soul) + self.conversation = Conversation() + + #: The voice the soul and the body agree on, and whether to bring it + #: up. The soul's `voice` block (its speaking rate) reaches the + #: plugin beside the provider reference. + self.tts = tts_selection(manifest, soul) + self.tts_name: str | None = self.tts["provider"] if self.tts else None + self.speak = speak + self._voice_block: Mapping[str, Any] = soul.get("voice") or {} + # Turn-taking is a persona trait, not engine tuning: a reflective soul # waits longer than an eager one, and that difference is the whole # reason the number lives on the soul rather than in this file. @@ -88,10 +162,18 @@ def __init__( self.patience_ms = int(interaction.get("patience_ms") or DEFAULT_PATIENCE_MS) self._wake: Any = None + self._stt: Any = None + self._llm: Any = None + self._tts: Any = None + self._mouth: Mouth | None = None self._audio: AudioSource | None = None self._sink: AudioSink | None = None self.descriptor: WakeDescriptor | None = None + self.stt_descriptor: TranscriberDescriptor | None = None + self.llm_descriptor: LanguageModelDescriptor | None = None + self.tts_descriptor: VoiceDescriptor | None = None self.format: AudioFormat | None = None + self.sink_format: AudioFormat | None = None #: Whether the loop is keeping up. Populated as it runs; see #: `emet_engine.metrics` for why the real-time factor is the number #: that decides whether a body can run Emet at all. @@ -121,18 +203,33 @@ async def start(self) -> None: self.stats.frame_ms = self.format.frame_ms + # Before the microphone: a provider that cannot start should never + # have caused a device to open. + if self.transcribe: + await self._start_stt() + if self.reply: + await self._start_llm() + if self.speak: + await self._start_tts() + source_cls = self.registry.load_audio(self.source_name) self._audio = source_cls(self._input_block, self.format) await self._audio.start() # The output rate is not the input rate and has no reason to be: one is - # what the detector needs, the other is what synthesis produces. - self.sink_format = AudioFormat( - sample_rate=int(self._output_block.get("sample_rate") or 22050) - ) + # what the detector needs, the other is what synthesis produces. With + # a voice running, the voice states it and the sink is opened to match; + # without one, the manifest's figure, or a synthesis rate. + if self.tts_descriptor is not None: + sink_rate = self.tts_descriptor.sample_rate + else: + sink_rate = int(self._output_block.get("sample_rate") or 22050) + self.sink_format = AudioFormat(sample_rate=sink_rate) sink_cls = self.registry.load_audio_out(self.sink_name) self._sink = sink_cls(self._output_block, self.sink_format) await self._sink.start() + if self._tts is not None: + self._mouth = Mouth(self._tts, self._sink) log.info( "listening for %r via %s on %s at %d Hz", @@ -146,6 +243,91 @@ def _build_wake(self) -> Any: wake_cls = self.registry.load_wake(self.engine_name) return wake_cls(self._wake_block, self.phrase) + async def _start_stt(self) -> None: + """Bring up speech recognition, or say precisely why not. + + Three refusals, each in the terms the person can act on: no provider + named anywhere, a provider that would not start (its own health detail + says why: a missing key, a dead network), and a provider that would run + at a rate other than the one the wake engine fixed. + """ + assert self.format is not None + if self.stt is None or self.stt_name is None: + installed = ", ".join(self.registry.stt_names) or "(none)" + raise EngineError( + "speech recognition was asked for and no provider is configured. " + "Name one under `models.stt.provider` in the soul, or take it over " + f"with `audio.stt.provider` in the body. Installed: {installed}." + ) + stt_cls = self.registry.load_stt(self.stt_name) + self._stt = stt_cls(self.stt, self.format) + await self._stt.start() + + self.stt_descriptor = self._stt.describe() + if not self.stt_descriptor.healthy: + raise EngineError(self._mute_message()) + if self.stt_descriptor.sample_rate != self.format.sample_rate: + raise EngineError( + f"the speech recognition provider {self.stt_name!r} would run at " + f"{self.stt_descriptor.sample_rate} Hz and the wake engine fixed " + f"{self.format.sample_rate} Hz. One microphone feeds both, so they " + f"have to agree; a transcriber hearing speech at the wrong speed " + f"does not fail, it produces nonsense. It will not start." + ) + + async def _start_llm(self) -> None: + """Bring up the language model, or say precisely why not.""" + if self.chat is None or self.chat_name is None: + installed = ", ".join(self.registry.llm_names) or "(none)" + raise EngineError( + "a reply was asked for and no language model is configured. Name " + "one under `models.chat.provider` in the soul, or take it over " + f"with `models.chat.provider` in the body. Installed: {installed}." + ) + llm_cls = self.registry.load_llm(self.chat_name) + self._llm = llm_cls(self.chat) + await self._llm.start() + + self.llm_descriptor = self._llm.describe() + if not self.llm_descriptor.healthy: + detail = "" + health = self._llm.health() + if not health.ok and health.detail: + detail = f" {health.detail}" + raise EngineError( + f"the language model {self.chat_name!r} cannot start, so the robot " + f"would hear questions it can never answer. It will not run.{detail}" + ) + + async def _start_tts(self) -> None: + """Bring up the voice, or say precisely why not. + + The plugin's own reason is quoted, because it is the one that names + the fix: a download command for a missing model, a variable for a + missing key, an extra for a missing library. + """ + if self.tts is None or self.tts_name is None: + installed = ", ".join(self.registry.tts_names) or "(none)" + raise EngineError( + "speech was asked for and no voice is configured. Name one under " + "`models.tts.provider` in the soul, or take it over with " + f"`models.tts.provider` in the body. Installed: {installed}." + ) + tts_cls = self.registry.load_tts(self.tts_name) + self._tts = tts_cls(self.tts, self._voice_block) + await self._tts.start() + + self.tts_descriptor = self._tts.describe() + if not self.tts_descriptor.healthy: + detail = "" + health = self._tts.health() + if not health.ok and health.detail: + detail = f" {health.detail}" + raise EngineError( + f"the voice {self.tts_name!r} cannot start, so the robot would " + f"answer and never be heard. It will not run.{detail}" + ) + def _deaf_message(self) -> str: """Say what is wrong in the terms the person can act on. @@ -162,14 +344,36 @@ def _deaf_message(self) -> str: f"nothing to fall back to, so it will not start.{detail}" ) + def _mute_message(self) -> str: + detail = "" + health = self._stt.health() + if not health.ok and health.detail: + detail = f" {health.detail}" + return ( + f"the speech recognition provider {self.stt_name!r} cannot start, so " + f"what people say would go nowhere. It will not run half-deaf.{detail}" + ) + async def stop(self) -> None: """Safe to call twice, and safe if `start()` raised part way through.""" + if self._mouth is not None: + await self._mouth.hush() + self._mouth = None if self._sink is not None: await self._sink.stop() self._sink = None + if self._tts is not None: + await self._tts.shutdown() + self._tts = None if self._audio is not None: await self._audio.stop() self._audio = None + if self._stt is not None: + await self._stt.shutdown() + self._stt = None + if self._llm is not None: + await self._llm.shutdown() + self._llm = None if self._wake is not None: await self._wake.shutdown() self._wake = None @@ -193,19 +397,47 @@ async def say(self, pcm: bytes) -> None: """ if self._sink is None: raise EngineError("session was not started") - await self._sink.play(pcm) + before = self.dropped + try: + await self._sink.play(pcm) + finally: + self._charge_busy(before) async def hush(self) -> None: - """Stop talking immediately, mid-word. + """Stop talking immediately, mid-word, and drop what was queued. Barge-in is built on this: `DESIGN.md` §13 requires that speech during playback interrupts rather than queues. Nothing calls it yet, and the sink contract carries it from the start so that adding barge-in later is engine work rather than a breaking change to every sink. """ - if self._sink is not None: + if self._mouth is not None: + await self._mouth.hush() + elif self._sink is not None: await self._sink.cancel() + async def speak_text(self, text: str) -> list[SpokenSentence]: + """Say a fixed piece of text, sentence by sentence, and wait for it. + + The voice's path without the language model in front of it: a + greeting, a test line, a robot reading something out. Returns what + happened to each sentence. + """ + if self._mouth is None: + raise EngineError("the voice was not started; construct the session with speak=True") + before = self.dropped + sentences = Sentences() + self._mouth.open() + try: + for sentence in sentences.feed(text): + await self._mouth.say(sentence) + rest = sentences.flush() + if rest: + await self._mouth.say(rest) + return await self._mouth.finish() + finally: + self._charge_busy(before) + # ----------------------------------------------------------------- loop async def wakes(self) -> AsyncIterator[WakeEvent]: @@ -239,6 +471,11 @@ async def turns(self) -> AsyncIterator[tuple[WakeEvent, Utterance]]: is deliberate: the phrase often appears inside what somebody then says ("hey emet, what did you mean, hey emet is a silly name"), and a detector still listening would start a second turn inside the first. + + The transcriber, when there is one, is fed every frame after the wake, + lead-in silence included. So a quiet speaker the energy detector missed + is still transcribed, and a false wake costs a provider a few seconds + of silence. Its final transcript rides on the utterance. """ if self._audio is None or self._wake is None or self.format is None: raise EngineError("session was not started") @@ -253,7 +490,7 @@ async def turns(self) -> AsyncIterator[tuple[WakeEvent, Utterance]]: # The source ran out mid-turn. Hand over what was caught # rather than dropping it: a truncated question is still # more useful than silence. - yield pending, endpointer.close() + yield pending, await self._transcribed(endpointer.close()) return if endpointer is None: @@ -265,18 +502,146 @@ async def turns(self) -> AsyncIterator[tuple[WakeEvent, Utterance]]: await self._wake.reset() pending = event endpointer = Endpointer(self.format, patience_ms=self.patience_ms) + if self.on_wake is not None: + self.on_wake(event) continue + partial: Transcript | None = None with Stopwatch() as watch: utterance = endpointer.feed(frame) + if self._stt is not None: + # Inside the timing on purpose. A provider that blocks + # here on the network is loop cost, and the budget line + # should say so rather than flatter it. + partial = await self._stt.feed(frame) self._record(watch.elapsed_ms) + if partial is not None and self.on_partial is not None: + self.on_partial(partial) if utterance is not None: assert pending is not None self.stats.turns += 1 - yield pending, utterance + yield pending, await self._transcribed(utterance) endpointer = None pending = None + async def _transcribed(self, utterance: Utterance) -> Utterance: + """Close the transcriber's utterance and attach what it heard. + + Timed, and kept apart from the per-frame budget: this is the wait + between the person stopping and the words existing, which is the + first latency 0.4 has to answer for. + """ + if self._stt is None: + return utterance + with Stopwatch() as watch: + final = await self._stt.finish() + self.stats.record_final(watch.elapsed_ms) + return replace(utterance, transcript=final) + + # --------------------------------------------------------------- answer + + async def answer(self, text: str) -> AsyncIterator[ReplyEvent]: + """Hand what was said to the language model and stream the reply. + + The caller's step after a turn, on purpose: `turns()` ends when the + person stops talking, and what happens next is the caller's to show + or to speak. The words join the conversation, the persona and the + conversation become the prompt, and the events come back as the + model produces them, `ReplyDone` last. A reply that ended well joins + the conversation too; one that ended in error does not, so the model + is never shown its own failure as something it said. + + The microphone is not read while this runs, the same as `say()`. + Frames dropped meanwhile are the loop's own doing and are reported + as such; barge-in, in 1.0, is what changes it. + """ + if self._llm is None: + raise EngineError("the language model was not started; construct the session with reply=True") + self.conversation.add_user(text) + prompt = self.conversation.prompt(self.system_prompt, max_tokens=DEFAULT_MAX_TOKENS) + + before = self.dropped + first_ms: float | None = None + try: + with Stopwatch() as watch: + async for event in self._llm.reply(prompt): + if isinstance(event, TextDelta) and first_ms is None: + first_ms = watch.peek_ms() + if isinstance(event, ReplyDone): + if event.stop_reason != "error" and event.text.strip(): + self.conversation.add_assistant(event.text.strip()) + yield event + self.stats.record_reply(first_ms, watch.elapsed_ms) + finally: + self._charge_busy(before) + + async def answer_aloud(self, text: str) -> AsyncIterator[ReplyEvent]: + """`answer()`, spoken: the same events, and every sentence said through + the voice as soon as it is complete. + + The reply's text deltas are watched for sentence ends as they pass + through. Each complete sentence goes to the voice at once, and its + audio plays while the model is still writing the next, which is the + whole reason the reply streams. After `ReplyDone` the tail (a model + that stops without a full stop) is spoken too, and this waits for the + last sound before returning, so the caller knows the floor is free. + + A sentence the voice could not say is logged, counted, and skipped; + the rest of the reply is still heard. `stats` gets the two numbers + a person feels: transcript to first sound, transcript to last. + """ + if self._mouth is None: + raise EngineError("the voice was not started; construct the session with speak=True") + sentences = Sentences() + mouth = self._mouth + first_ms: float | None = None + watch = Stopwatch() + + def mark_first() -> None: + nonlocal first_ms + if first_ms is None: + first_ms = watch.peek_ms() + + mouth.on_first_audio = mark_first + mouth.open() + with watch: + try: + async for event in self.answer(text): + if isinstance(event, TextDelta): + for sentence in sentences.feed(event.text): + await mouth.say(sentence) + yield event + # `answer()` charged the frames dropped while the model wrote; + # what follows is the voice's, and is charged below. + after_reply = self.dropped + rest = sentences.flush() + if rest: + await mouth.say(rest) + spoken = await mouth.finish() + except BaseException: + await mouth.hush() + raise + self.stats.record_speech( + first_ms, + watch.elapsed_ms, + spoken=sum(1 for s in spoken if s.ok and s.audio_bytes), + lost=sum(1 for s in spoken if not s.ok), + ) + self._charge_busy(after_reply) + + @property + def last_spoken(self) -> list[SpokenSentence]: + """What happened to each sentence of the most recent spoken reply.""" + return list(self._mouth.spoken) if self._mouth is not None else [] + + def _charge_busy(self, dropped_before: int) -> None: + """Attribute frames dropped during `say()` or `answer()` to the loop's + own choice not to read, so the verdict judges listening alone.""" + lost = self.dropped - dropped_before + if lost > 0: + self.stats.dropped_busy += lost + self.stats.dropped = self.dropped + # -------------------------------------------------------------- honesty def _record(self, process_ms: float) -> None: diff --git a/emet-engine/emet_engine/speech.py b/emet-engine/emet_engine/speech.py new file mode 100644 index 0000000..0fd378c --- /dev/null +++ b/emet-engine/emet_engine/speech.py @@ -0,0 +1,298 @@ +"""Speaking a reply as it is written: sentences out of a stream, audio out +of sentences, sound out of the audio, each stage running while the last one +is still busy. + +A language model streams words. A voice takes a sentence. A speaker takes a +buffer. Nothing above this module wants to know that, so this is where the +three shapes meet: text deltas go in one end and the robot is heard at the +other, and the first sentence is playing while the model is still writing +the second. That overlap is the whole reason `LanguageModelPlugin.reply()` +streams and `VoicePlugin.speak()` takes one sentence rather than a +paragraph. + +**Sentences, not words and not the whole reply.** A voice given one word at +a time produces choppy prosody, because a sentence's melody depends on +knowing where it ends. A voice given the whole reply cannot start until the +model has finished, which on the reference body is a two-second silence a +person hears as the robot not having understood. A sentence is the unit that +sounds right and arrives early, and `Sentences` finds them in a stream of +deltas by the plain rule: a full stop, a question mark or an exclamation +mark, followed by whitespace or the end, closes one. Abbreviations and +decimals are handled by the short list a spoken reply actually contains, +rather than by a parser; a wrong split costs a pause, never a word. + +**Three stages, two queues.** `Mouth.say()` queues a sentence and returns at +once. A synthesis task takes sentences in order and asks the voice for each, +collecting its audio into one buffer per sentence; a playback task takes +those buffers in order and hands each to the sink. So the voice is +synthesising sentence two while sentence one plays, and the engine loop that +called `say()` is back reading the language model. `finish()` waits for +everything queued to be heard; `hush()` throws it all away and stops the +sink mid-word, which is what barge-in will be made of. + +**A sentence lost is a sentence lost.** A voice that fails on one sentence +(the network dropped, the model choked on a symbol) raises `PluginError`, +and the mouth records it, tells the caller through `failures`, and goes on +with the next sentence. The text was still printed, and a robot that +skipped a sentence is still a robot that answered. +""" + +from __future__ import annotations + +import asyncio +import logging +import re +from dataclasses import dataclass +from typing import Any, AsyncIterator, Callable + +from emet_sdk.plugin import PluginError +from emet_sdk.types import AudioSink + +__all__ = ["Sentences", "Mouth", "SpokenSentence", "split_sentences", "speak_stream"] + +log = logging.getLogger("emet_engine.speech") + +#: Text that ends a sentence when followed by whitespace: the three end +#: marks, optionally with a closing quote or bracket after them. +_END = re.compile(r'[.!?]+["\')\]]?(?=\s)') + +#: An ellipsis is a pause unless what follows starts a new sentence. In a +#: stream the next word may not have arrived yet, in which case the +#: decision waits. +_ELLIPSIS = re.compile(r"\.\.\.") +_NEW_SENTENCE = re.compile(r'\s+["\'(]?[A-Z0-9]') + +#: Endings that look like a sentence's and are not. Titles and the +#: abbreviations a spoken reply is likely to contain; a wrong guess here +#: costs a pause at worst. +_NOT_AN_END = re.compile( + r"(?:\b(?:mr|mrs|ms|dr|prof|sr|jr|st|vs|etc|e\.g|i\.e|no|inc|ltd|co)\.|\b[a-z]\.|\d\.)$", + re.IGNORECASE, +) + + +def split_sentences(text: str) -> tuple[list[str], str]: + """The complete sentences at the front of `text`, and the remainder. + + A sentence is complete when its end mark is followed by whitespace. The + remainder is whatever follows the last complete sentence, which may be + the start of the next one or nothing. + """ + out: list[str] = [] + start = 0 + for m in _END.finditer(text): + candidate = text[start : m.end()] + if _NOT_AN_END.search(candidate.rstrip('"\')]')): + continue + if _ELLIPSIS.search(m.group()) and not _NEW_SENTENCE.match(text, m.end()): + continue + sentence = candidate.strip() + if sentence: + out.append(sentence) + start = m.end() + rest = text[start:] + return out, rest.lstrip() if out else rest + + +class Sentences: + """Sentences out of a stream of text deltas. + + `feed()` returns the sentences the delta completed, in order, and keeps + the rest; `flush()` returns whatever is left when the stream ends, which + a model that stops without a full stop always leaves. + """ + + def __init__(self) -> None: + self._buffer = "" + + def feed(self, delta: str) -> list[str]: + self._buffer += delta + done, self._buffer = split_sentences(self._buffer) + return done + + def flush(self) -> str | None: + rest = " ".join(self._buffer.split()) + self._buffer = "" + return rest or None + + @property + def pending(self) -> str: + return self._buffer + + +@dataclass(frozen=True, slots=True) +class SpokenSentence: + """What happened to one sentence: how long its audio was, and whether the + voice managed it.""" + + text: str + audio_bytes: int + error: str | None = None + + @property + def ok(self) -> bool: + return self.error is None + + +@dataclass +class _Job: + text: str + #: Filled in when synthesis finishes, whatever the outcome. + pcm: bytes = b"" + error: str | None = None + + +class Mouth: + """The output side of a turn: sentences in, sound out, in order. + + `voice` is a started `VoicePlugin`; `sink` a started `AudioSink` opened + at the voice's sample rate. `on_first_audio`, when given, is called once + per `open()`, the moment the first buffer reaches the sink: the number a + person waits for, measured where they hear it. + """ + + def __init__( + self, + voice: Any, + sink: AudioSink, + *, + on_first_audio: Callable[[], None] | None = None, + on_spoken: Callable[[SpokenSentence], None] | None = None, + ) -> None: + self.voice = voice + self.sink = sink + self.on_first_audio = on_first_audio + self.on_spoken = on_spoken + self._sentences: asyncio.Queue[_Job | None] | None = None + self._audio: asyncio.Queue[_Job | None] | None = None + self._synth_task: asyncio.Task | None = None + self._play_task: asyncio.Task | None = None + self._first_audio_reported = False + #: Every sentence handed over since `open()`, with its outcome, in + #: the order it was heard. + self.spoken: list[SpokenSentence] = [] + + # ------------------------------------------------------------ lifecycle + + def open(self) -> None: + """Start the two stages. Called once per reply.""" + self._sentences = asyncio.Queue() + self._audio = asyncio.Queue() + self._first_audio_reported = False + self.spoken = [] + self._synth_task = asyncio.create_task(self._synthesise_all()) + self._play_task = asyncio.create_task(self._play_all()) + + async def say(self, text: str) -> None: + """Queue one sentence. Returns at once.""" + if self._sentences is None: + raise RuntimeError("the mouth is not open") + if text.strip(): + self._sentences.put_nowait(_Job(text=text.strip())) + + async def finish(self) -> list[SpokenSentence]: + """Wait until everything queued has been heard, and close.""" + if self._sentences is None: + return list(self.spoken) + self._sentences.put_nowait(None) + if self._synth_task is not None: + await self._synth_task + if self._play_task is not None: + await self._play_task + self._sentences = None + self._audio = None + self._synth_task = None + self._play_task = None + return list(self.spoken) + + async def hush(self) -> None: + """Drop what is queued and stop the sink mid-word.""" + for task in (self._synth_task, self._play_task): + if task is not None and not task.done(): + task.cancel() + for task in (self._synth_task, self._play_task): + if task is not None: + try: + await task + except (asyncio.CancelledError, Exception): # noqa: BLE001 + pass + await self.sink.cancel() + self._sentences = None + self._audio = None + self._synth_task = None + self._play_task = None + + @property + def failures(self) -> list[SpokenSentence]: + return [s for s in self.spoken if not s.ok] + + # ---------------------------------------------------------------- stages + + async def _synthesise_all(self) -> None: + assert self._sentences is not None and self._audio is not None + while True: + job = await self._sentences.get() + if job is None: + self._audio.put_nowait(None) + return + parts: list[bytes] = [] + try: + async for chunk in self.voice.speak(job.text): + parts.append(chunk) + except PluginError as exc: + job.error = str(exc) + log.warning("voice: %s", exc) + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 - a plugin that raises the wrong thing still loses one sentence + job.error = f"{type(exc).__name__}: {exc}" + log.warning("voice: %s", job.error) + job.pcm = b"".join(parts) + self._audio.put_nowait(job) + + async def _play_all(self) -> None: + assert self._audio is not None + while True: + job = await self._audio.get() + if job is None: + return + if job.pcm and job.error is None: + if not self._first_audio_reported and self.on_first_audio is not None: + self._first_audio_reported = True + self.on_first_audio() + try: + await self.sink.play(job.pcm) + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 - the sink failed; say so, keep going + job.error = f"playback failed: {type(exc).__name__}: {exc}" + log.warning("speaker: %s", job.error) + elif job.pcm and job.error is not None: + # Partial audio from a sentence the voice gave up on. Playing + # half a sentence is worse than skipping it. + pass + outcome = SpokenSentence(text=job.text, audio_bytes=len(job.pcm), error=job.error) + self.spoken.append(outcome) + if self.on_spoken is not None: + self.on_spoken(outcome) + + +async def speak_stream( + mouth: Mouth, + deltas: AsyncIterator[str], +) -> list[SpokenSentence]: + """Feed a stream of text deltas through a mouth, sentence by sentence. + + A convenience for callers that hold a plain text stream; the session + does the same by hand so that it can yield the reply's events as they + pass through. + """ + sentences = Sentences() + mouth.open() + async for delta in deltas: + for sentence in sentences.feed(delta): + await mouth.say(sentence) + rest = sentences.flush() + if rest: + await mouth.say(rest) + return await mouth.finish() diff --git a/emet-engine/emet_engine/turn.py b/emet-engine/emet_engine/turn.py index dd6608e..3444289 100644 --- a/emet-engine/emet_engine/turn.py +++ b/emet-engine/emet_engine/turn.py @@ -41,7 +41,7 @@ from dataclasses import dataclass from enum import StrEnum -from emet_sdk.types import AudioFormat +from emet_sdk.types import AudioFormat, Transcript from emet_engine.vad import EnergyVad, VadTuning @@ -69,6 +69,10 @@ class Utterance: audio: bytes duration_ms: float reason: EndReason + #: What speech recognition made of it, when a transcriber was listening. + #: None means nobody asked, which is different from an empty final: that + #: means a provider listened and heard no words it could make out. + transcript: Transcript | None = None @property def had_speech(self) -> bool: diff --git a/emet-engine/tests/conftest.py b/emet-engine/tests/conftest.py new file mode 100644 index 0000000..9aefb18 --- /dev/null +++ b/emet-engine/tests/conftest.py @@ -0,0 +1,23 @@ +"""What every engine test may assume about its surroundings. + +**No keys file.** `emet-listen` reads `~/.config/emet/keys.env` and +`/etc/emet/keys.env` before the providers start, which is right for a person +and wrong for a test: on the first laptop that held real keys, two tests +asserting "no key, so stop before the network" found the keys, started two +vendors' preflights and passed a network call off as a unit test. The +default places are emptied here for every test, and `EMET_KEYS` is unset, so +a test sees only the environment it built. A test that wants a keys file +names one with `--keys`. +""" + +from __future__ import annotations + +import pytest + +from emet_engine import keys + + +@pytest.fixture(autouse=True) +def no_developer_keys(monkeypatch): + monkeypatch.setattr(keys, "default_paths", lambda: []) + monkeypatch.delenv("EMET_KEYS", raising=False) diff --git a/emet-engine/tests/test_cli.py b/emet-engine/tests/test_cli.py index daa7437..9e902a3 100644 --- a/emet-engine/tests/test_cli.py +++ b/emet-engine/tests/test_cli.py @@ -18,6 +18,8 @@ import wave from pathlib import Path +import pytest + from emet_engine import cli from emet_engine.session import ListenSession @@ -58,8 +60,16 @@ def body(path: str) -> dict: } -def soul() -> dict: - return {"identity": {"name": "Emet", "wake_word": PHRASE}} +def soul(chat: dict | None = None, **stt) -> dict: + doc = {"identity": {"name": "Emet", "wake_word": PHRASE}} + models: dict = {} + if stt: + models["stt"] = stt + if chat is not None: + models["chat"] = chat + if models: + doc["models"] = models + return doc def stub_loaders(monkeypatch, manifest: dict, soul_doc: dict) -> None: @@ -69,7 +79,17 @@ def stub_loaders(monkeypatch, manifest: dict, soul_doc: dict) -> None: def args(**overrides) -> argparse.Namespace: - base = {"manifest": "body.yaml", "soul": "soul.yaml", "replay": None, "echo": False, "stats": True} + base = { + "manifest": "body.yaml", + "soul": "soul.yaml", + "replay": None, + "echo": False, + "stats": True, + "transcribe": False, + "reply": False, + "speak": False, + "keys": None, + } base.update(overrides) return argparse.Namespace(**base) @@ -140,3 +160,334 @@ def test_frames_dropped_while_listening_are_a_warning(tmp_path, monkeypatch, cap asyncio.run(cli._run(args())) assert "not keeping up" in capsys.readouterr().out + + +# ------------------------------------------------------------- --transcribe + + +def spoken(*words: bytes) -> bytes: + """A turn: the phrase, then frames that spell words, then silence. + + The mock transcriber reads a frame's leading text; the energy detector + hears those same bytes as loud, so the endpointer captures them as speech + and closes the turn on the silence after. + """ + def frame(payload: bytes = b"") -> bytes: + return payload + bytes(2 * FRAME - len(payload)) + + return frame() + frame(PHRASE.encode()) + b"".join(frame(w) for w in words) + frame() * 20 + + +def test_transcribe_prints_partials_as_they_arrive_and_then_what_was_said( + tmp_path, monkeypatch, capsys +): + wav = write_wav(tmp_path / "said.wav", spoken(b"what", b"time", b"is it")) + stub_loaders(monkeypatch, body(wav), soul(provider="mock")) + + rc = asyncio.run(cli._run(args(transcribe=True))) + + out = capsys.readouterr().out + assert rc == 0 + assert " stt mock (streaming)" in out + assert " hearing 'what time'" in out + assert " said 'what time is it'" in out + assert "stt final" in out, "the time to the final transcript is part of --stats" + # The wake prints when it is heard, before the words that follow it. + assert out.index("heard 'hey emet'") < out.index("hearing 'what'") + + +def test_without_the_flag_nothing_is_transcribed(tmp_path, monkeypatch, capsys): + wav = write_wav(tmp_path / "quiet.wav", spoken(b"what", b"time")) + stub_loaders(monkeypatch, body(wav), soul(provider="mock")) + + asyncio.run(cli._run(args())) + + out = capsys.readouterr().out + assert "said" not in out and "hearing" not in out and "stt" not in out + + +def test_transcribe_with_no_provider_configured_says_which_field_to_set( + tmp_path, monkeypatch, capsys +): + wav = write_wav(tmp_path / "none.wav", saying(1)) + stub_loaders(monkeypatch, body(wav), soul()) + + rc = cli.main(["body.yaml", "soul.yaml", "--transcribe"]) + + err = capsys.readouterr().err + assert rc == 1 + assert "models.stt.provider" in err + assert "audio.stt.provider" in err + + +def test_transcribe_with_an_uninstalled_provider_is_a_missing_plugin( + tmp_path, monkeypatch, capsys +): + """A provider nobody has packaged has to be the plain missing-plugin + message, not a stack trace.""" + wav = write_wav(tmp_path / "dg.wav", saying(1)) + stub_loaders(monkeypatch, body(wav), soul(provider="whisper", key_env="EMET_OPENAI_KEY")) + + rc = cli.main(["body.yaml", "soul.yaml", "--transcribe"]) + + err = capsys.readouterr().err + assert rc == 1 + assert "whisper" in err and "missing_plugin" in err + + +def test_transcribe_with_the_reference_provider_and_no_key_names_the_variable( + tmp_path, monkeypatch, capsys +): + """The reference soul names deepgram. Without the key, the run must stop + before any network is touched and say which variable to export.""" + monkeypatch.delenv("EMET_DEEPGRAM_KEY", raising=False) + wav = write_wav(tmp_path / "nokey.wav", saying(1)) + stub_loaders(monkeypatch, body(wav), soul(provider="deepgram", key_env="EMET_DEEPGRAM_KEY")) + + rc = cli.main(["body.yaml", "soul.yaml", "--transcribe"]) + + err = capsys.readouterr().err + assert rc == 1 + assert "EMET_DEEPGRAM_KEY" in err + assert "will not run" in err + + +def test_a_false_wake_under_transcribe_reports_an_empty_final(tmp_path, monkeypatch, capsys): + wav = write_wav(tmp_path / "false.wav", saying(1)) + stub_loaders(monkeypatch, body(wav), soul(provider="mock")) + + rc = asyncio.run(cli._run(args(transcribe=True))) + + out = capsys.readouterr().out + assert rc == 0 + assert "probably a false wake" in out + assert "said nothing the provider could make out" in out + + +# ------------------------------------------------------------------ --reply + + +def test_reply_prints_the_answer_as_it_streams_and_implies_transcribe(tmp_path, monkeypatch, capsys): + wav = write_wav(tmp_path / "r.wav", spoken(b"what", b"time", b"is it")) + stub_loaders(monkeypatch, body(wav), soul(provider="mock", chat={"provider": "mock"})) + + rc = asyncio.run(cli._run(args(reply=True))) + + out = capsys.readouterr().out + assert rc == 0 + assert " stt mock (streaming)" in out, "--reply implies --transcribe" + assert " llm mock" in out + assert " said 'what time is it'" in out + assert " reply: You said: what time is it" in out + assert "llm first" in out and "llm done" in out + + +def test_reply_with_no_language_model_configured_names_the_field(tmp_path, monkeypatch, capsys): + wav = write_wav(tmp_path / "n.wav", saying(1)) + stub_loaders(monkeypatch, body(wav), soul(provider="mock")) + + rc = cli.main(["body.yaml", "soul.yaml", "--reply"]) + + err = capsys.readouterr().err + assert rc == 1 + assert "models.chat.provider" in err + + +def test_reply_with_the_reference_provider_and_no_key_names_the_variable(tmp_path, monkeypatch, capsys): + monkeypatch.delenv("EMET_OPENAI_KEY", raising=False) + wav = write_wav(tmp_path / "k.wav", saying(1)) + stub_loaders( + monkeypatch, + body(wav), + soul(provider="mock", chat={"provider": "openai", "key_env": "EMET_OPENAI_KEY"}), + ) + + rc = cli.main(["body.yaml", "soul.yaml", "--reply"]) + + err = capsys.readouterr().err + assert rc == 1 + assert "EMET_OPENAI_KEY" in err and "will not run" in err + + +def test_a_false_wake_gets_no_reply(tmp_path, monkeypatch, capsys): + wav = write_wav(tmp_path / "f.wav", saying(1)) + stub_loaders(monkeypatch, body(wav), soul(provider="mock", chat={"provider": "mock"})) + + asyncio.run(cli._run(args(reply=True))) + + out = capsys.readouterr().out + assert "said nothing the provider could make out" in out + assert "reply:" not in out + + +# ------------------------------------------------------------------ --keys + + +def test_keys_are_read_from_a_file_before_the_providers_start(tmp_path, monkeypatch, capsys): + """The BYOK noun: a key file, loaded once, names printed and values not.""" + monkeypatch.delenv("EMET_TEST_KEY", raising=False) + keys = tmp_path / "keys.env" + keys.write_text("EMET_TEST_KEY=hunter2\n", encoding="utf-8") + wav = write_wav(tmp_path / "k.wav", saying(1)) + stub_loaders(monkeypatch, body(wav), soul()) + + rc = cli.main(["body.yaml", "soul.yaml", "--keys", str(keys)]) + + out = capsys.readouterr().out + assert rc == 0 + assert f" keys EMET_TEST_KEY from {keys}" in out + assert "hunter2" not in out + import os + + assert os.environ["EMET_TEST_KEY"] == "hunter2" + + +def test_the_suite_never_sees_a_developers_keys_file(): + """The guard in conftest.py. Without it, a laptop holding real keys turns + every 'no key' test into a live call to a vendor.""" + from emet_engine import keys + + assert keys.default_paths() == [] + + +def test_a_missing_keys_file_stops_the_run_with_its_path(tmp_path, monkeypatch, capsys): + wav = write_wav(tmp_path / "m.wav", saying(1)) + stub_loaders(monkeypatch, body(wav), soul()) + + rc = cli.main(["body.yaml", "soul.yaml", "--keys", str(tmp_path / "absent.env")]) + + err = capsys.readouterr().err + assert rc == 1 + assert "absent.env" in err and "no keys file" in err + + +def test_without_keys_and_without_default_files_the_header_says_nothing_about_keys( + tmp_path, monkeypatch, capsys +): + from emet_engine import keys as keys_module + + monkeypatch.setattr(keys_module, "default_paths", lambda: [tmp_path / "none.env"]) + monkeypatch.delenv("EMET_KEYS", raising=False) + wav = write_wav(tmp_path / "q.wav", saying(1)) + stub_loaders(monkeypatch, body(wav), soul()) + + asyncio.run(cli._run(args(keys=None))) + + assert " keys " not in capsys.readouterr().out + + +# ------------------------------------------------------------------ --speak + + +def talking_soul() -> dict: + return soul(provider="mock", chat={"provider": "mock"}) | {"models": {"stt": {"provider": "mock"}, "chat": {"provider": "mock"}, "tts": {"provider": "mock"}}} + + +def test_speak_says_the_reply_and_implies_reply_and_transcribe(tmp_path, monkeypatch, capsys): + wav = write_wav(tmp_path / "s.wav", spoken(b"what", b"time", b"is it")) + stub_loaders(monkeypatch, body(wav), talking_soul()) + + rc = asyncio.run(cli._run(args(speak=True))) + + out = capsys.readouterr().out + assert rc == 0 + assert " stt mock (streaming)" in out + assert " llm mock" in out + assert " voice mock (16000 Hz)" in out + assert " reply: You said: what time is it" in out + assert " spoke 1 sentence(s)" in out + assert "voice first" in out and "voice done" in out + + +def test_speak_writes_what_was_said_through_the_wav_sink(tmp_path, monkeypatch, capsys): + """The whole 0.4 path on a laptop: wake, words, reply, voice, sink.""" + wav = write_wav(tmp_path / "w.wav", spoken(b"hello", b"there")) + manifest = body(wav) + out = str(tmp_path / "said.wav") + manifest["audio"]["output"] = {"sink": "wav", "device": "none", "params": {"path": out}} + stub_loaders(monkeypatch, manifest, talking_soul()) + + rc = asyncio.run(cli._run(args(speak=True))) + assert rc == 0 + from emet_providers.mock import read_back + + with wave.open(out, "rb") as w: + assert w.getframerate() == 16000 + assert read_back(w.readframes(w.getnframes())) == "You said: hello there" + + +def test_a_lost_sentence_is_printed_with_its_reason(tmp_path, monkeypatch, capsys): + wav = write_wav(tmp_path / "l.wav", spoken(b"hello", b"there")) + manifest = body(wav) + manifest["models"] = { + "chat": {"provider": "mock", "params": {"reply": "Fine. The POISON word. Fine again."}}, + "tts": {"provider": "mock", "params": {"fail_on": "POISON"}}, + } + stub_loaders(monkeypatch, manifest, talking_soul()) + + asyncio.run(cli._run(args(speak=True))) + + out = capsys.readouterr().out + assert " spoke 2 sentence(s)" in out + assert "the voice could not say 'The POISON word.'" in out + + +def test_speak_with_no_voice_configured_names_the_field(tmp_path, monkeypatch, capsys): + wav = write_wav(tmp_path / "n.wav", saying(1)) + stub_loaders(monkeypatch, body(wav), soul(provider="mock", chat={"provider": "mock"})) + + rc = cli.main(["body.yaml", "soul.yaml", "--speak"]) + + err = capsys.readouterr().err + assert rc == 1 + assert "models.tts.provider" in err + + +def test_speak_with_the_reference_voice_and_nothing_installed_says_what_to_do(tmp_path, monkeypatch, capsys): + """The reference soul names piper. Without the extra, or without the + voice model, the run must stop before the microphone opens and say + which command fixes it.""" + import sys + + monkeypatch.setitem(sys.modules, "piper", None) + wav = write_wav(tmp_path / "p.wav", saying(1)) + doc = talking_soul() + doc["models"]["tts"] = {"provider": "piper", "model": "en_US-ljspeech-medium"} + stub_loaders(monkeypatch, body(wav), doc) + + rc = cli.main(["body.yaml", "soul.yaml", "--speak"]) + + err = capsys.readouterr().err + assert rc == 1 + assert "will not run" in err and "emet-providers[piper]" in err + + +def test_speak_with_the_cloud_voice_and_no_key_names_the_variable(tmp_path, monkeypatch, capsys): + monkeypatch.delenv("EMET_DEEPGRAM_KEY", raising=False) + wav = write_wav(tmp_path / "k.wav", saying(1)) + doc = talking_soul() + doc["models"]["tts"] = {"provider": "deepgram", "key_env": "EMET_DEEPGRAM_KEY"} + stub_loaders(monkeypatch, body(wav), doc) + + rc = cli.main(["body.yaml", "soul.yaml", "--speak"]) + + err = capsys.readouterr().err + assert rc == 1 + assert "EMET_DEEPGRAM_KEY" in err and "will not run" in err + + +def test_echo_and_speak_exclude_each_other(capsys): + with pytest.raises(SystemExit) as exc: + cli.main(["body.yaml", "soul.yaml", "--echo", "--speak"]) + assert exc.value.code == 2 + assert "not allowed with" in capsys.readouterr().err + + +def test_a_false_wake_is_not_spoken_to(tmp_path, monkeypatch, capsys): + wav = write_wav(tmp_path / "f.wav", saying(1)) + stub_loaders(monkeypatch, body(wav), talking_soul()) + + asyncio.run(cli._run(args(speak=True))) + + out = capsys.readouterr().out + assert "spoke" not in out and "reply:" not in out diff --git a/emet-engine/tests/test_keys.py b/emet-engine/tests/test_keys.py new file mode 100644 index 0000000..c868c0c --- /dev/null +++ b/emet-engine/tests/test_keys.py @@ -0,0 +1,133 @@ +"""The keys file: bring your own key, once, and keep it out of the shell.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import pytest + +from emet_engine import keys +from emet_engine.keys import LoadedKeys, load_keys, parse_env + + +# ------------------------------------------------------------------ parsing + + +def test_the_plain_dotenv_subset_parses(): + text = """ + # Emet keys. One a line. + EMET_DEEPGRAM_KEY=dg_abc + export EMET_OPENAI_KEY="sk-quoted" + EMET_ANTHROPIC_KEY='sk-single' # trailing comment after quotes is ignored + EMET_LOCAL_KEY=plain # a comment + """ + assert parse_env(text) == { + "EMET_DEEPGRAM_KEY": "dg_abc", + "EMET_OPENAI_KEY": "sk-quoted", + "EMET_ANTHROPIC_KEY": "sk-single", + "EMET_LOCAL_KEY": "plain", + } + + +def test_a_bad_line_names_its_number_and_stops(): + with pytest.raises(ValueError, match=r"keys.env:2"): + parse_env("GOOD=1\nthis is not a key\n") + with pytest.raises(ValueError, match="NAME=value"): + parse_env("9BAD=1") + + +def test_values_are_one_token_and_never_interpolated(): + assert parse_env("A=$HOME/x")["A"] == "$HOME/x" + assert parse_env("B==with=equals")["B"] == "=with=equals" + assert parse_env('C="has # inside" # comment')["C"] == "has # inside" + + +def test_an_unclosed_quote_is_a_named_error(): + with pytest.raises(ValueError, match=r"keys.env:1.*unclosed quote"): + parse_env("A='oops") + + +# ------------------------------------------------------------------ loading + + +def test_a_named_file_is_loaded_and_the_environment_wins(tmp_path): + file = tmp_path / "keys.env" + file.write_text("EMET_ONE=from_file\nEMET_TWO=from_file\n", encoding="utf-8") + environ = {"EMET_TWO": "already exported"} + + (loaded,) = load_keys(file, environ=environ) + + assert environ == {"EMET_ONE": "from_file", "EMET_TWO": "already exported"} + assert loaded == LoadedKeys(path=file, names=("EMET_ONE",), kept=("EMET_TWO",), warning=loaded.warning) + + +def test_a_named_file_that_does_not_exist_is_an_error(tmp_path): + with pytest.raises(FileNotFoundError, match="no keys file"): + load_keys(tmp_path / "nope.env", environ={}) + + +def test_emet_keys_in_the_environment_names_the_file(tmp_path): + file = tmp_path / "elsewhere.env" + file.write_text("EMET_X=1\n", encoding="utf-8") + environ = {"EMET_KEYS": str(file)} + (loaded,) = load_keys(environ=environ) + assert loaded.path == file and environ["EMET_X"] == "1" + + +def test_the_persons_file_is_read_first_so_it_wins_over_the_machines(tmp_path, monkeypatch): + """Nothing overwrites a variable once set, so the order of reading is the + order of precedence: the shell, then the person, then the machine.""" + user = tmp_path / "home" / "keys.env" + system = tmp_path / "etc" / "keys.env" + user.parent.mkdir(parents=True) + system.parent.mkdir(parents=True) + user.write_text("EMET_SHARED=person\n", encoding="utf-8") + system.write_text("EMET_SHARED=machine\nEMET_ONLY_SYSTEM=yes\n", encoding="utf-8") + monkeypatch.setattr(keys, "default_paths", lambda: [user, system]) + + environ: dict[str, str] = {"EMET_ONLY_SYSTEM": "shell"} + loaded = load_keys(environ=environ) + + assert [l.path for l in loaded] == [user, system] + assert environ["EMET_SHARED"] == "person" + assert loaded[1].kept == ("EMET_SHARED", "EMET_ONLY_SYSTEM") + assert environ["EMET_ONLY_SYSTEM"] == "shell" + + +def test_the_real_default_order_is_person_then_machine(monkeypatch): + monkeypatch.undo() # conftest empties default_paths for every test; look at the real one + paths = keys.default_paths() + assert paths == [keys.user_keys_path(), keys.SYSTEM_KEYS] + + +def test_no_file_anywhere_loads_nothing_and_is_not_an_error(tmp_path, monkeypatch): + monkeypatch.setattr(keys, "default_paths", lambda: [tmp_path / "absent.env"]) + assert load_keys(environ={}) == [] + + +def test_the_user_path_honours_xdg_config_home(monkeypatch, tmp_path): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path)) + assert keys.user_keys_path() == tmp_path / "emet" / "keys.env" + monkeypatch.delenv("XDG_CONFIG_HOME") + assert keys.user_keys_path() == Path.home() / ".config" / "emet" / "keys.env" + + +@pytest.mark.skipif(sys.platform.startswith("win"), reason="POSIX file modes") +def test_a_file_others_can_read_is_loaded_with_a_warning(tmp_path): + file = tmp_path / "keys.env" + file.write_text("EMET_K=1\n", encoding="utf-8") + os.chmod(file, 0o644) + (loaded,) = load_keys(file, environ={}) + assert loaded.warning and "chmod 600" in loaded.warning + os.chmod(file, 0o600) + (loaded,) = load_keys(file, environ={}) + assert loaded.warning is None + + +def test_values_never_appear_in_what_is_reported(tmp_path): + file = tmp_path / "keys.env" + file.write_text("EMET_SECRET=hunter2\n", encoding="utf-8") + (loaded,) = load_keys(file, environ={}) + assert "hunter2" not in repr(loaded) diff --git a/emet-engine/tests/test_prompting.py b/emet-engine/tests/test_prompting.py new file mode 100644 index 0000000..8b3e944 --- /dev/null +++ b/emet-engine/tests/test_prompting.py @@ -0,0 +1,66 @@ +"""What the language model is told, and what it remembers.""" + +from __future__ import annotations + +from emet_sdk.types import Message, ToolSpec + +from emet_engine.prompting import ( + DEFAULT_MAX_TOKENS, + DEFAULT_TURNS, + SPOKEN_ALOUD, + Conversation, + system_prompt, +) + + +def test_a_written_persona_prompt_is_used_and_the_speaking_rule_follows_it(): + soul = {"identity": {"name": "Emet"}, "persona": {"system_prompt": "You are Emet. Be honest.\n"}} + text = system_prompt(soul) + assert text.startswith("You are Emet. Be honest.") + assert text.endswith(SPOKEN_ALOUD) + assert "\n\n" in text + + +def test_without_a_written_prompt_the_name_and_summary_make_one(): + soul = {"identity": {"name": "Barnaby"}, "persona": {"summary": "Gentle,\n reflective."}} + text = system_prompt(soul) + assert text.startswith("You are Barnaby. Gentle, reflective.") + assert SPOKEN_ALOUD in text + + +def test_a_bare_soul_still_gets_a_prompt(): + assert system_prompt({}).startswith("You are Emet.") + + +def test_the_speaking_rule_names_no_hardware(): + """Principle 1 at the prompt: the body enters through the self-model in + 0.5, never through this constant.""" + for word in ("servo", "GPIO", "I2C", "driver", "microphone", "camera"): + assert word.lower() not in SPOKEN_ALOUD.lower() + + +def test_a_conversation_alternates_and_packages_its_tail(): + conversation = Conversation() + conversation.add_user("what time is it") + conversation.add_assistant("Nearly noon.") + conversation.add_user("thanks") + prompt = conversation.prompt("sys", tools=(ToolSpec(name="t", description="d"),), max_tokens=99) + assert prompt.system == "sys" + assert [m.role for m in prompt.messages] == ["user", "assistant", "user"] + assert prompt.messages[1] == Message(role="assistant", content="Nearly noon.") + assert prompt.max_tokens == 99 and prompt.tools[0].name == "t" + + +def test_a_conversation_forgets_the_oldest_turns_in_pairs(): + conversation = Conversation(turns=4) + for i in range(4): + conversation.add_user(f"q{i}") + conversation.add_assistant(f"a{i}") + assert [m.content for m in conversation.messages] == ["q2", "a2", "q3", "a3"] + assert conversation.messages[0].role == "user" + + +def test_the_defaults_suit_a_spoken_reply(): + assert DEFAULT_MAX_TOKENS == 300 + assert DEFAULT_TURNS == 24 + assert Conversation().prompt("s").max_tokens == DEFAULT_MAX_TOKENS diff --git a/emet-engine/tests/test_session.py b/emet-engine/tests/test_session.py index 1b9a294..415bdf9 100644 --- a/emet-engine/tests/test_session.py +++ b/emet-engine/tests/test_session.py @@ -69,11 +69,19 @@ def body( } -def soul(wake_word: str | None = PHRASE) -> dict: +def soul(wake_word: str | None = PHRASE, chat: dict | None = None, **stt) -> dict: identity = {"name": "Emet"} if wake_word is not None: identity["wake_word"] = wake_word - return {"identity": identity} + doc: dict = {"identity": identity} + models: dict = {} + if stt: + models["stt"] = stt + if chat is not None: + models["chat"] = chat + if models: + doc["models"] = models + return doc # ------------------------------------------------------------------ startup @@ -366,3 +374,711 @@ async def scenario(): return session.warm_start_hint() assert run(scenario()) is None + + +# ------------------------------------------------------------ transcription + + +def spoken(*words: bytes) -> bytes: + """The phrase, then frames that spell words, then silence. The mock + transcriber reads the words; the energy detector hears the same bytes as + loud, so the endpointer captures them and closes on the silence after.""" + return frame() + frame(PHRASE.encode()) + b"".join(frame(w) for w in words) + frame() * 20 + + +def transcribing(tmp_path, name: str, pcm: bytes, *, soul_doc=None, manifest=None, **kw): + manifest = manifest or body(write_wav(tmp_path / name, pcm)) + return ListenSession(manifest, soul_doc or soul(provider="mock"), transcribe=True, **kw) + + +def test_a_turn_is_transcribed_when_asked(tmp_path): + """0.4's first step end to end: hear the name, feed what follows to a + provider the engine never imported, get the words back on the turn.""" + session = transcribing(tmp_path, "t.wav", spoken(b"what", b"time", b"is it")) + + async def scenario(): + async with session: + return [(e, u) async for e, u in session.turns()] + + turns = run(scenario()) + assert len(turns) == 1 + _, utterance = turns[0] + assert utterance.had_speech + assert utterance.transcript is not None + assert utterance.transcript.final + assert utterance.transcript.text == "what time is it" + assert session.stt_name == "mock" + assert session.stt_descriptor is not None and session.stt_descriptor.streaming + + +def test_partials_arrive_while_the_person_is_still_talking(tmp_path): + """Streaming through the seam. Each partial carries everything so far, + and the last one is what the final confirms.""" + partials = [] + session = transcribing( + tmp_path, "p.wav", spoken(b"what", b"time", b"is it"), on_partial=partials.append + ) + + async def scenario(): + async with session: + return [u async for _, u in session.turns()] + + (utterance,) = run(scenario()) + assert [p.text for p in partials] == ["what", "what time", "what time is it"] + assert all(not p.final for p in partials) + assert utterance.transcript is not None + assert partials[-1].text == utterance.transcript.text + + +def test_the_wake_callback_fires_before_the_turn_ends(tmp_path): + seen: list[str] = [] + session = transcribing( + tmp_path, + "w.wav", + spoken(b"what"), + on_wake=lambda e: seen.append("wake"), + on_partial=lambda t: seen.append("partial"), + ) + + async def scenario(): + async with session: + async for _ in session.turns(): + seen.append("turn") + + run(scenario()) + assert seen == ["wake", "partial", "turn"] + + +def test_without_asking_nothing_is_built_and_the_transcript_is_none(tmp_path): + """The 0.3 loop, unchanged, even for a soul that names a provider. The + selection is still reported so a caller can say what would run.""" + session = ListenSession(body(write_wav(tmp_path / "n.wav", spoken(b"what"))), soul(provider="mock")) + + async def scenario(): + async with session: + assert session._stt is None + return [u async for _, u in session.turns()] + + (utterance,) = run(scenario()) + assert utterance.transcript is None + assert session.stt_name == "mock" + + +def test_asking_with_no_provider_anywhere_names_the_fields_to_set(tmp_path): + session = transcribing(tmp_path, "x.wav", frame(), soul_doc=soul()) + with pytest.raises(EngineError) as exc: + run(session.start()) + message = str(exc.value) + assert "models.stt.provider" in message and "audio.stt.provider" in message + assert "mock" in message # what is installed + run(session.stop()) + + +def test_an_uninstalled_provider_is_a_missing_plugin(tmp_path): + session = transcribing(tmp_path, "d.wav", frame(), soul_doc=soul(provider="whisper")) + with pytest.raises(MissingPluginError): + run(session.start()) + run(session.stop()) + + +def test_a_provider_that_cannot_start_is_refused_with_its_own_reason(tmp_path): + """A dead network or a missing key. The plugin's health detail is more + specific than anything the engine could invent, so it is quoted.""" + manifest = body(write_wav(tmp_path / "f.wav", frame())) + manifest["models"] = {"stt": {"provider": "mock", "params": {"fail_on_start": True}}} + session = transcribing(tmp_path, "f.wav", frame(), manifest=manifest) + with pytest.raises(EngineError) as exc: + run(session.start()) + assert "unreachable" in str(exc.value) + assert "will not run" in str(exc.value) + run(session.stop()) + + +def test_a_missing_key_is_named_at_boot(tmp_path, monkeypatch): + monkeypatch.delenv("EMET_MOCK_KEY", raising=False) + session = transcribing( + tmp_path, + "k.wav", + frame(), + soul_doc=soul(provider="mock", key_env="EMET_MOCK_KEY"), + ) + session.stt["params"]["require_key"] = True + with pytest.raises(EngineError, match="EMET_MOCK_KEY"): + run(session.start()) + run(session.stop()) + + +def test_a_provider_at_the_wrong_rate_is_refused_before_the_microphone_opens(tmp_path): + """One microphone feeds the wake engine and the transcriber. A recogniser + hearing 16 kHz speech at 8 kHz does not fail; it produces nonsense.""" + manifest = body(write_wav(tmp_path / "r.wav", frame())) + manifest["models"] = {"stt": {"provider": "mock", "params": {"sample_rate": 8000}}} + session = transcribing(tmp_path, "r.wav", frame(), manifest=manifest) + with pytest.raises(EngineError) as exc: + run(session.start()) + message = str(exc.value) + assert "8000" in message and "16000" in message + assert session._audio is None, "the microphone was opened before the refusal" + run(session.stop()) + + +def test_the_body_takes_over_from_the_soul(tmp_path): + """The reference soul names deepgram; a mocked rig names mock and runs.""" + manifest = body(write_wav(tmp_path / "o.wav", spoken(b"hello"))) + manifest["models"] = {"stt": {"provider": "mock"}} + session = transcribing( + tmp_path, + "o.wav", + frame(), + manifest=manifest, + soul_doc=soul(provider="deepgram", model="nova-2", key_env="EMET_DEEPGRAM_KEY"), + ) + assert session.stt_name == "mock" + assert session.stt is not None and session.stt["key_env"] is None + + async def scenario(): + async with session: + return [u async for _, u in session.turns()] + + (utterance,) = run(scenario()) + assert utterance.transcript is not None and utterance.transcript.text == "hello" + + +def test_a_scripted_mock_says_its_line_over_real_looking_audio(tmp_path): + """How the seam is exercised on a body: audio the mock cannot read, and a + line it was told to say.""" + manifest = body(write_wav(tmp_path / "s.wav", frame() + frame(PHRASE.encode()) + loud_frame() * 5 + frame() * 20)) + manifest["models"] = {"stt": {"provider": "mock", "params": {"transcript": "testing the seam"}}} + partials = [] + session = transcribing(tmp_path, "s.wav", frame(), manifest=manifest, on_partial=partials.append) + + async def scenario(): + async with session: + return [u async for _, u in session.turns()] + + (utterance,) = run(scenario()) + assert utterance.transcript is not None + assert utterance.transcript.text == "testing the seam" + assert [p.text for p in partials] == ["testing", "testing the", "testing the seam"] + + +def test_a_false_wake_yields_an_empty_final_not_none(tmp_path): + """Nobody spoke, the provider listened, and it says so. None would mean + nobody asked.""" + session = transcribing(tmp_path, "e.wav", frame() + frame(PHRASE.encode()) + frame() * 60) + + async def scenario(): + async with session: + return [u async for _, u in session.turns()] + + (utterance,) = run(scenario()) + assert not utterance.had_speech + assert utterance.transcript is not None + assert utterance.transcript.final and utterance.transcript.text == "" + + +def test_a_source_that_ends_mid_turn_still_hands_over_the_transcript(tmp_path): + pcm = frame() + frame(PHRASE.encode()) + frame(b"cut") + frame(b"off") + session = transcribing(tmp_path, "c.wav", pcm) + + async def scenario(): + async with session: + return [u async for _, u in session.turns()] + + (utterance,) = run(scenario()) + assert utterance.reason.value == "source_ended" + assert utterance.transcript is not None and utterance.transcript.text == "cut off" + + +def test_each_turn_gets_its_own_transcript(tmp_path): + # Two words a turn: the energy detector needs two loud frames in a row + # before it calls it speech, and a turn has to end by silence for the + # next wake to be heard. + session = transcribing(tmp_path, "two.wav", spoken(b"first", b"one") + spoken(b"second", b"two")) + + async def scenario(): + async with session: + return [u.transcript.text async for _, u in session.turns()] + + assert run(scenario()) == ["first one", "second two"] + + +def test_stopping_after_a_failed_transcriber_start_is_safe(tmp_path): + manifest = body(write_wav(tmp_path / "z.wav", frame())) + manifest["models"] = {"stt": {"provider": "mock", "params": {"fail_on_start": True}}} + session = transcribing(tmp_path, "z.wav", frame(), manifest=manifest) + + async def scenario(): + with pytest.raises(EngineError): + await session.start() + await session.stop() + await session.stop() + + run(scenario()) + + +# ----------------------------------------------------------------- answers + + +from emet_sdk.types import ReplyDone, TextDelta # noqa: E402 + + +def answering(tmp_path, name: str, pcm: bytes, *, chat: dict | None = None, manifest=None, **kw): + manifest = manifest or body(write_wav(tmp_path / name, pcm)) + soul_doc = soul(provider="mock", chat=chat if chat is not None else {"provider": "mock"}) + return ListenSession(manifest, soul_doc, transcribe=True, reply=True, **kw) + + +async def drain(events): + return [e async for e in events] + + +def test_a_turn_can_be_answered_by_a_model_the_engine_never_imported(tmp_path): + """0.4's second seam end to end: hear the name, transcribe the words, + hand them to the language model with the persona, stream the reply.""" + session = answering(tmp_path, "a.wav", spoken(b"what", b"time", b"is it")) + + async def scenario(): + async with session: + turns = [(e, u) async for e, u in session.turns()] + (_, utterance) = turns[0] + events = await drain(session.answer(utterance.transcript.text)) + return utterance, events + + utterance, events = run(scenario()) + assert utterance.transcript.text == "what time is it" + deltas = [e for e in events if isinstance(e, TextDelta)] + done = events[-1] + assert isinstance(done, ReplyDone) + assert done.stop_reason == "end" + assert done.text == "You said: what time is it" + assert "".join(d.text for d in deltas) == done.text + assert session.chat_name == "mock" + assert session.llm_descriptor is not None and session.llm_descriptor.healthy + + +def test_the_persona_becomes_the_system_prompt_and_the_conversation_accumulates(tmp_path): + session = answering(tmp_path, "p.wav", frame()) + session.soul["persona"] = {"system_prompt": "You are Emet. Be honest."} + from emet_engine.prompting import system_prompt + + session.system_prompt = system_prompt(session.soul) + + async def scenario(): + async with session: + await drain(session.answer("hello")) + await drain(session.answer("and again")) + return session._llm.prompts + + prompts = run(scenario()) + assert prompts[0].system.startswith("You are Emet. Be honest.") + assert [m.role for m in prompts[1].messages] == ["user", "assistant", "user"] + assert prompts[1].messages[1].content == "You said: hello" + assert prompts[1].messages[2].content == "and again" + + +def test_a_failed_reply_is_returned_and_kept_out_of_the_conversation(tmp_path): + manifest = body(write_wav(tmp_path / "f.wav", frame())) + manifest["models"] = {"chat": {"provider": "mock"}} + session = answering(tmp_path, "f.wav", frame(), manifest=manifest) + + async def scenario(): + async with session: + session._llm._started = False # the vendor went away mid-run + events = await drain(session.answer("hello")) + return events, list(session.conversation.messages) + + events, messages = run(scenario()) + (done,) = events + assert done.stop_reason == "error" and done.error + assert [m.role for m in messages] == ["user"] + + +def test_answering_records_the_two_latencies(tmp_path): + session = answering(tmp_path, "s.wav", frame()) + + async def scenario(): + async with session: + await drain(session.answer("hello")) + return session.stats + + stats = run(scenario()) + assert len(stats.reply_first_ms) == 1 and len(stats.reply_done_ms) == 1 + assert stats.reply_first_ms[0] <= stats.reply_done_ms[0] + assert "llm first" in stats.report(live=False) and "llm done" in stats.report(live=False) + + +def test_asking_for_replies_with_no_language_model_names_the_field(tmp_path): + session = answering(tmp_path, "n.wav", frame(), chat={}) + session.chat = None + session.chat_name = None + with pytest.raises(EngineError) as exc: + run(session.start()) + assert "models.chat.provider" in str(exc.value) + assert "mock" in str(exc.value) + run(session.stop()) + + +def test_an_uninstalled_language_model_is_a_missing_plugin(tmp_path): + session = answering(tmp_path, "u.wav", frame(), chat={"provider": "abacus"}) + with pytest.raises(MissingPluginError): + run(session.start()) + run(session.stop()) + + +def test_a_language_model_that_cannot_start_is_refused_with_its_own_reason(tmp_path): + manifest = body(write_wav(tmp_path / "x.wav", frame())) + manifest["models"] = {"chat": {"provider": "mock", "params": {"fail_on_start": True}}} + session = answering(tmp_path, "x.wav", frame(), manifest=manifest) + with pytest.raises(EngineError) as exc: + run(session.start()) + assert "unreachable" in str(exc.value) and "will not run" in str(exc.value) + assert session._audio is None, "the microphone was opened before the refusal" + run(session.stop()) + + +def test_answering_without_reply_enabled_is_an_error(tmp_path): + session = ListenSession(body(write_wav(tmp_path / "r.wav", frame())), soul()) + + async def scenario(): + async with session: + with pytest.raises(EngineError, match="reply=True"): + await drain(session.answer("hello")) + + run(scenario()) + + +def test_the_body_takes_the_language_model_over_from_the_soul(tmp_path): + manifest = body(write_wav(tmp_path / "o.wav", frame())) + manifest["models"] = {"stt": {"provider": "mock"}, "chat": {"provider": "mock", "params": {"reply": "As you wish."}}} + session = answering( + tmp_path, "o.wav", frame(), manifest=manifest, chat={"provider": "openai", "model": "x", "key_env": "EMET_OPENAI_KEY"} + ) + assert session.chat_name == "mock" + + async def scenario(): + async with session: + return (await drain(session.answer("anything")))[-1].text + + assert run(scenario()) == "As you wish." + + +# ------------------------------------------------------- honest verdicts + + +def test_frames_dropped_while_answering_are_charged_to_the_loops_own_choice(tmp_path): + """The reference body dropped 11 frames during a two-second reply and + the verdict said the loop did not keep up. It kept up fine; it was not + reading, on purpose. Those frames are counted, labelled, and left out of + the verdict.""" + session = answering(tmp_path, "d.wav", frame()) + + async def scenario(): + async with session: + session._audio.dropped = 3 # lost while listening: the loop's fault + real_reply = session._llm.reply + + async def slow_reply(prompt): + session._audio.dropped = 8 # five more, lost while the model thought + async for event in real_reply(prompt): + yield event + + session._llm.reply = slow_reply + await drain(session.answer("hello")) + return session.stats + + stats = run(scenario()) + assert stats.dropped == 8 and stats.dropped_busy == 5 + assert "5 while speaking or thinking" in stats.report(live=False) + assert not stats.kept_up, "three frames were lost while listening, and that still counts" + + +def test_a_run_that_only_dropped_frames_while_busy_kept_up(): + from emet_engine.metrics import SessionStats + + stats = SessionStats() + for _ in range(10): + stats.record_frame(1.0) + stats.dropped = 11 + stats.dropped_busy = 11 + assert stats.kept_up + assert "11 while speaking or thinking" in stats.report(live=False) + + +# ------------------------------------------------------------------ speech + + +from emet_sdk.plugin import PluginError # noqa: E402 + + +def speaking(tmp_path, name: str, pcm: bytes, *, tts: dict | None = None, manifest=None, sink: str = "null", **kw): + manifest = manifest or body(write_wav(tmp_path / name, pcm), sink=sink) + soul_doc = soul(provider="mock", chat={"provider": "mock"}) + soul_doc["models"]["tts"] = tts if tts is not None else {"provider": "mock"} + return ListenSession(manifest, soul_doc, transcribe=True, reply=True, speak=True, **kw) + + +def wav_body(path: str, out: str, **wake_params) -> dict: + doc = body(path, **wake_params) + doc["audio"]["output"] = {"sink": "wav", "device": "none", "params": {"path": out}} + return doc + + +def read_wav(path: str) -> tuple[int, bytes]: + with wave.open(path, "rb") as w: + return w.getframerate(), w.readframes(w.getnframes()) + + +def test_a_turn_can_be_answered_aloud_through_a_voice_the_engine_never_imported(tmp_path): + """0.4's third seam end to end: hear the name, transcribe, reply, and + speak the reply through a voice and a sink, both reached by name. The + wav the sink wrote reads back as the words the model said.""" + out = str(tmp_path / "said.wav") + manifest = wav_body(write_wav(tmp_path / "a.wav", spoken(b"what", b"time", b"is it")), out) + session = speaking(tmp_path, "a.wav", b"", manifest=manifest) + + async def scenario(): + async with session: + turns = [(e, u) async for e, u in session.turns()] + (_, utterance) = turns[0] + events = await drain(session.answer_aloud(utterance.transcript.text)) + return events, session.last_spoken + + events, outcome = run(scenario()) + done = events[-1] + assert isinstance(done, ReplyDone) and done.text == "You said: what time is it" + assert "".join(e.text for e in events if isinstance(e, TextDelta)) == done.text + assert [s.text for s in outcome] == ["You said: what time is it"] and outcome[0].ok + from emet_providers.mock import read_back # a test may name the layer above; the engine may not + + rate, pcm = read_wav(out) + assert read_back(pcm) == "You said: what time is it" + assert rate == 16000, "the sink was opened at the rate the voice stated" + assert session.tts_name == "mock" + assert session.tts_descriptor is not None and session.tts_descriptor.healthy + + +def test_the_voice_states_the_rate_and_the_sink_is_opened_to_match(tmp_path): + manifest = body(write_wav(tmp_path / "r.wav", frame())) + manifest["audio"]["output"]["sample_rate"] = 48000 + manifest["models"] = {"tts": {"provider": "mock", "params": {"sample_rate": 8000}}} + session = speaking(tmp_path, "r.wav", frame(), manifest=manifest) + + async def scenario(): + async with session: + return session.sink_format, session._sink.format + + sink_format, opened = run(scenario()) + assert sink_format.sample_rate == 8000 and opened.sample_rate == 8000 + + +def test_without_a_voice_the_sink_keeps_the_manifests_rate(tmp_path): + manifest = body(write_wav(tmp_path / "m.wav", frame())) + manifest["audio"]["output"]["sample_rate"] = 48000 + session = ListenSession(manifest, soul()) + + async def scenario(): + async with session: + return session.sink_format.sample_rate + + assert run(scenario()) == 48000 + + +def test_the_first_sentence_plays_before_the_reply_is_done(tmp_path): + """The reason everything streams. A slow model writes two sentences; + the first is at the sink before the model has finished the second.""" + marks: list[str] = [] + session = speaking(tmp_path, "s.wav", frame()) + + async def scenario(): + async with session: + real_sink_play = session._sink.play + + async def logged_play(pcm): + marks.append("play") + await real_sink_play(pcm) + + session._sink.play = logged_play + + async def slow_reply(prompt): + # A model's tokens carry their leading space, so the first + # sentence is confirmed by the token that starts the second. + yield TextDelta("First one.") + yield TextDelta(" Second") + await asyncio.sleep(0.05) + yield TextDelta(" one.") + marks.append("model done") + yield ReplyDone(text="First one. Second one.", stop_reason="end", model="mock") + + session._llm.reply = slow_reply + async for event in session.answer_aloud("hello"): + if isinstance(event, ReplyDone): + marks.append("reply done") + return marks, session.last_spoken + + marks, spoken = run(scenario()) + assert marks.index("play") < marks.index("model done") < marks.index("reply done") + assert [s.text for s in spoken] == ["First one.", "Second one."] + assert marks.count("play") == 2, "the second sentence was spoken after the reply ended" + + +def test_a_reply_without_a_full_stop_is_still_spoken(tmp_path): + manifest = body(write_wav(tmp_path / "t.wav", frame())) + manifest["models"] = {"chat": {"provider": "mock", "params": {"reply": "no end mark here"}}, "tts": {"provider": "mock"}} + session = speaking(tmp_path, "t.wav", frame(), manifest=manifest) + + async def scenario(): + async with session: + await drain(session.answer_aloud("hello")) + return session.last_spoken + + (spoken,) = run(scenario()) + assert spoken.text == "no end mark here" and spoken.ok + + +def test_a_sentence_the_voice_loses_is_counted_and_the_rest_is_heard(tmp_path): + manifest = body(write_wav(tmp_path / "l.wav", frame())) + manifest["models"] = { + "chat": {"provider": "mock", "params": {"reply": "Fine. The symbol POISON here. Fine again."}}, + "tts": {"provider": "mock", "params": {"fail_on": "POISON"}}, + } + session = speaking(tmp_path, "l.wav", frame(), manifest=manifest) + + async def scenario(): + async with session: + await drain(session.answer_aloud("hello")) + return session.last_spoken, session.stats + + spoken, stats = run(scenario()) + assert [s.ok for s in spoken] == [True, False, True] + assert stats.sentences_spoken == 2 and stats.sentences_lost == 1 + assert "1 lost" in stats.report(live=False) + + +def test_answering_aloud_records_the_two_latencies_a_person_feels(tmp_path): + session = speaking(tmp_path, "m.wav", frame()) + + async def scenario(): + async with session: + await drain(session.answer_aloud("hello")) + return session.stats + + stats = run(scenario()) + assert len(stats.speech_first_ms) == 1 and len(stats.speech_done_ms) == 1 + assert stats.speech_first_ms[0] <= stats.speech_done_ms[0] + assert len(stats.reply_first_ms) == 1, "the model's own numbers are still recorded" + report = stats.report(live=False) + assert "voice first" in report and "voice done" in report + + +def test_speak_text_says_a_fixed_line(tmp_path): + out = str(tmp_path / "line.wav") + manifest = wav_body(write_wav(tmp_path / "f.wav", frame()), out) + session = speaking(tmp_path, "f.wav", frame(), manifest=manifest) + + async def scenario(): + async with session: + return await session.speak_text("Hello there. I am awake.") + + spoken = run(scenario()) + assert [s.text for s in spoken] == ["Hello there.", "I am awake."] + from emet_providers.mock import read_back + + assert read_back(read_wav(out)[1]) == "Hello there. I am awake." + + +def test_asking_for_speech_with_no_voice_names_the_field(tmp_path): + session = speaking(tmp_path, "n.wav", frame(), tts={}) + session.tts = None + session.tts_name = None + with pytest.raises(EngineError) as exc: + run(session.start()) + assert "models.tts.provider" in str(exc.value) + assert "mock" in str(exc.value) + run(session.stop()) + + +def test_an_uninstalled_voice_is_a_missing_plugin(tmp_path): + session = speaking(tmp_path, "u.wav", frame(), tts={"provider": "gramophone"}) + with pytest.raises(MissingPluginError): + run(session.start()) + run(session.stop()) + + +def test_a_voice_that_cannot_start_is_refused_with_its_own_reason(tmp_path): + manifest = body(write_wav(tmp_path / "x.wav", frame())) + manifest["models"] = {"tts": {"provider": "mock", "params": {"fail_on_start": True}}} + session = speaking(tmp_path, "x.wav", frame(), manifest=manifest) + with pytest.raises(EngineError) as exc: + run(session.start()) + assert "never be heard" in str(exc.value) and "missing" in str(exc.value) + assert session._audio is None, "the microphone was opened before the refusal" + run(session.stop()) + run(session.stop()) + + +def test_the_souls_voice_block_reaches_the_plugin(tmp_path): + session = speaking(tmp_path, "v.wav", frame()) + session.soul["voice"] = {"rate": 1.3} + session._voice_block = session.soul["voice"] + + async def scenario(): + async with session: + return session._tts.rate + + assert run(scenario()) == 1.3 + + +def test_the_body_takes_the_voice_over_from_the_soul(tmp_path): + manifest = body(write_wav(tmp_path / "o.wav", frame())) + manifest["models"] = {"tts": {"provider": "mock", "params": {"sample_rate": 8000}}} + session = speaking( + tmp_path, "o.wav", frame(), manifest=manifest, + tts={"provider": "deepgram", "model": "aura-2-thalia-en", "key_env": "EMET_DEEPGRAM_KEY"}, + ) + assert session.tts_name == "mock" + + async def scenario(): + async with session: + return session.tts_descriptor.sample_rate + + assert run(scenario()) == 8000 + + +def test_speaking_without_speak_enabled_is_an_error(tmp_path): + session = ListenSession(body(write_wav(tmp_path / "w.wav", frame())), soul(provider="mock", chat={"provider": "mock"}), transcribe=True, reply=True) + + async def scenario(): + async with session: + with pytest.raises(EngineError, match="speak=True"): + await drain(session.answer_aloud("hello")) + with pytest.raises(EngineError, match="speak=True"): + await session.speak_text("hello") + + run(scenario()) + + +def test_hush_with_a_voice_drops_the_queue_and_cancels_the_sink(tmp_path): + session = speaking(tmp_path, "h.wav", frame()) + + async def scenario(): + async with session: + session._mouth.open() + await session._mouth.say("One.") + await session.hush() + return session._sink.cancelled + + assert run(scenario()) == 1 + + +def test_without_asking_no_voice_is_built(tmp_path): + soul_doc = soul(provider="mock", chat={"provider": "mock"}) + soul_doc["models"]["tts"] = {"provider": "mock"} + session = ListenSession(body(write_wav(tmp_path / "q.wav", frame())), soul_doc) + + async def scenario(): + async with session: + return session._tts, session._mouth + + assert run(scenario()) == (None, None) + assert session.tts_name == "mock", "the selection is still reported" diff --git a/emet-engine/tests/test_speech.py b/emet-engine/tests/test_speech.py new file mode 100644 index 0000000..589a09a --- /dev/null +++ b/emet-engine/tests/test_speech.py @@ -0,0 +1,320 @@ +"""Speaking a reply as it is written. + +Three things are proved here without a model, a voice or a speaker: that +sentences are found in a stream of deltas where a person would end them, +that the mouth plays them in order while the next is still being made, and +that a sentence the voice loses is one sentence lost and not the turn. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +from emet_sdk.plugin import PluginError, VoicePlugin +from emet_sdk.types import VoiceDescriptor + +from emet_engine.speech import Mouth, Sentences, split_sentences, speak_stream + + +def run(coro): + return asyncio.run(coro) + + +# ------------------------------------------------------------- sentences + + +@pytest.mark.parametrize( + ("text", "done", "rest"), + [ + ("Hello there. How are you? ", ["Hello there.", "How are you?"], ""), + ("Hello there. How are", ["Hello there."], "How are"), + ("No end yet", [], "No end yet"), + ("Really?! Yes. ", ["Really?!", "Yes."], ""), + ('He said "go." Then left. ', ['He said "go."', "Then left."], ""), + ("It is 3.5 metres long. ", ["It is 3.5 metres long."], ""), + ("Ask Dr. Who about it. Fine. ", ["Ask Dr. Who about it.", "Fine."], ""), + ("Bring apples, pears, etc. and go. ", ["Bring apples, pears, etc. and go."], ""), + ("Wait...", [], "Wait..."), + ("Wait... then go. ", ["Wait... then go."], ""), + ("Wait... ", [], "Wait... "), + ("Wait... Then go. ", ["Wait...", "Then go."], ""), + ("e.g. this one. Next. ", ["e.g. this one.", "Next."], ""), + ], +) +def test_sentences_end_where_a_person_would(text, done, rest): + assert split_sentences(text) == (done, rest) + + +def test_a_stream_of_deltas_yields_sentences_as_they_complete(): + s = Sentences() + assert s.feed("It is") == [] + assert s.feed(" noon.") == [] + assert s.feed(" Go") == ["It is noon."] + assert s.pending == "Go" + assert s.feed(" now! And") == ["Go now!"] + assert s.flush() == "And" + assert s.flush() is None + + +def test_flush_normalises_whitespace_in_the_tail(): + s = Sentences() + s.feed(" a tail \n with no end ") + assert s.flush() == "a tail with no end" + + +# ----------------------------------------------------------------- stand-ins + + +class _Voice(VoicePlugin): + """A voice that turns a sentence into its bytes, slowly if asked, and + fails on a word if asked.""" + + provider = "fake" + + def __init__(self, *, delay_ms: float = 0.0, fail_on: str | None = None) -> None: + super().__init__({"provider": "fake"}) + self.delay_ms = delay_ms + self.fail_on = fail_on + self.asked: list[str] = [] + + def describe(self) -> VoiceDescriptor: + return VoiceDescriptor(provider="fake", sample_rate=8000, streaming=True) + + async def speak(self, text: str): + self.asked.append(text) + if self.delay_ms: + await asyncio.sleep(self.delay_ms / 1000.0) + for word in text.split(): + if self.fail_on and self.fail_on in word: + raise PluginError(f"cannot say {word!r}") + yield word.encode() + b"\x00\x00" + + +class _Sink: + """Records what it was handed and when, relative to a shared log.""" + + def __init__(self, log: list[str] | None = None, *, delay_ms: float = 0.0) -> None: + self.format: Any = None + self.played: list[bytes] = [] + self.log = log if log is not None else [] + self.delay_ms = delay_ms + self.cancelled = 0 + + async def start(self) -> None: + pass + + async def play(self, pcm: bytes) -> None: + words = pcm.replace(bytes(1), b" ").decode().strip() + self.log.append(f"play:{words}") + if self.delay_ms: + await asyncio.sleep(self.delay_ms / 1000.0) + self.played.append(pcm) + self.log.append(f"played:{words}") + + async def cancel(self) -> None: + self.cancelled += 1 + + async def stop(self) -> None: + pass + + +# --------------------------------------------------------------------- mouth + + +def test_sentences_are_spoken_in_order_and_the_audio_is_whole(tmp_path): + voice, sink = _Voice(), _Sink() + mouth = Mouth(voice, sink) + + async def scenario(): + mouth.open() + await mouth.say("First one.") + await mouth.say("Second one.") + return await mouth.finish() + + spoken = run(scenario()) + assert voice.asked == ["First one.", "Second one."] + assert sink.played == [b"First\x00\x00one.\x00\x00", b"Second\x00\x00one.\x00\x00"] + assert [s.text for s in spoken] == ["First one.", "Second one."] + assert all(s.ok for s in spoken) + assert spoken[0].audio_bytes == len(sink.played[0]) + + +def test_the_next_sentence_is_synthesised_while_the_last_one_plays(): + """The pipeline. With a slow speaker, the voice is asked for the second + sentence before the first has finished playing.""" + log: list[str] = [] + voice = _Voice() + real_speak = voice.speak + + async def logged_speak(text): + log.append(f"ask:{text}") + async for chunk in real_speak(text): + yield chunk + + voice.speak = logged_speak # type: ignore[method-assign] + sink = _Sink(log, delay_ms=40) + mouth = Mouth(voice, sink) + + async def scenario(): + mouth.open() + await mouth.say("One.") + await mouth.say("Two.") + await mouth.finish() + + run(scenario()) + assert log.index("ask:Two.") < log.index("played:One."), "the second was being made while the first played" + assert log.index("play:One.") < log.index("play:Two.") + assert log[-1] == "played:Two." + + +def test_the_first_sound_is_reported_once_when_it_reaches_the_sink(): + marks: list[str] = [] + sink = _Sink(marks) + mouth = Mouth(_Voice(), sink, on_first_audio=lambda: marks.append("first")) + + async def scenario(): + mouth.open() + await mouth.say("One.") + await mouth.say("Two.") + await mouth.finish() + + run(scenario()) + assert marks == ["first", "play:One.", "played:One.", "play:Two.", "played:Two."] + + +def test_a_sentence_the_voice_cannot_say_is_skipped_and_the_rest_is_heard(): + voice, sink = _Voice(fail_on="Ω"), _Sink() + mouth = Mouth(voice, sink) + + async def scenario(): + mouth.open() + await mouth.say("Fine.") + await mouth.say("The symbol Ω here.") + await mouth.say("Fine again.") + return await mouth.finish() + + spoken = run(scenario()) + assert [s.ok for s in spoken] == [True, False, True] + assert "cannot say" in (spoken[1].error or "") + assert [p.split(b"\x00")[0] for p in sink.played] == [b"Fine.", b"Fine"] + assert mouth.failures == [spoken[1]] + assert spoken[1].audio_bytes > 0, "the audio before the failure is counted, and not played" + + +def test_a_voice_that_raises_the_wrong_thing_still_loses_only_one_sentence(): + voice = _Voice() + + async def broken(text): + raise ValueError("not a PluginError") + yield b"" # pragma: no cover + + voice.speak = broken # type: ignore[method-assign] + mouth = Mouth(voice, _Sink()) + + async def scenario(): + mouth.open() + await mouth.say("One.") + return await mouth.finish() + + (spoken,) = run(scenario()) + assert not spoken.ok and "ValueError" in (spoken.error or "") + + +def test_a_sink_that_fails_marks_the_sentence_and_goes_on(): + sink = _Sink() + + async def bad_play(pcm): + raise RuntimeError("card unplugged") + + sink.play = bad_play # type: ignore[method-assign] + mouth = Mouth(_Voice(), sink) + + async def scenario(): + mouth.open() + await mouth.say("One.") + await mouth.say("Two.") + return await mouth.finish() + + spoken = run(scenario()) + assert [s.ok for s in spoken] == [False, False] + assert "playback failed" in (spoken[0].error or "") + + +def test_blank_sentences_are_not_queued(): + mouth = Mouth(_Voice(), _Sink()) + + async def scenario(): + mouth.open() + await mouth.say(" ") + return await mouth.finish() + + assert run(scenario()) == [] + + +def test_hush_drops_the_queue_and_cancels_the_sink(): + sink = _Sink(delay_ms=200) + mouth = Mouth(_Voice(delay_ms=20), sink) + + async def scenario(): + mouth.open() + await mouth.say("One.") + await mouth.say("Two.") + await mouth.say("Three.") + await asyncio.sleep(0.05) # the first is playing, the rest queued + await mouth.hush() + return sink.played, sink.cancelled + + played, cancelled = run(scenario()) + assert cancelled == 1 + assert len(played) <= 1 + + +def test_saying_before_open_is_an_error(): + mouth = Mouth(_Voice(), _Sink()) + + async def scenario(): + with pytest.raises(RuntimeError, match="not open"): + await mouth.say("One.") + + run(scenario()) + + +def test_finish_without_open_returns_nothing(): + assert run(Mouth(_Voice(), _Sink()).finish()) == [] + + +def test_a_mouth_can_be_opened_again_for_the_next_reply(): + voice, sink = _Voice(), _Sink() + mouth = Mouth(voice, sink) + + async def scenario(): + mouth.open() + await mouth.say("One.") + first = await mouth.finish() + mouth.open() + await mouth.say("Two.") + second = await mouth.finish() + return first, second + + first, second = run(scenario()) + assert [s.text for s in first] == ["One."] and [s.text for s in second] == ["Two."] + assert len(sink.played) == 2 + + +# ----------------------------------------------------------- speak_stream + + +def test_a_text_stream_is_spoken_sentence_by_sentence_with_the_tail_last(): + voice, sink = _Voice(), _Sink() + mouth = Mouth(voice, sink) + + async def deltas(): + for piece in ["It is", " noon.", " Go now", "! And", " then"]: + yield piece + + spoken = run(speak_stream(mouth, deltas())) + assert voice.asked == ["It is noon.", "Go now!", "And then"] + assert [s.text for s in spoken] == voice.asked diff --git a/emet-hal/README.md b/emet-hal/README.md index e4d0607..25127ce 100644 --- a/emet-hal/README.md +++ b/emet-hal/README.md @@ -24,6 +24,9 @@ for years. | `wav` | `emet.audio_out` | Writes what the robot said to a file | | `null` | `emet.audio_out` | Discards audio, so the loop can run silently | +Plugins that reach a service rather than a chip (speech recognition, +language models and voices) live in `emet-providers`, next door. + **No hardware drivers yet.** The locomotion plugins are arithmetic: they turn a desired velocity into per-wheel speeds and touch no GPIO, so they need no robot and are fully testable against the mock. Wake and audio are optional diff --git a/emet-providers/LICENSE b/emet-providers/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/emet-providers/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/emet-providers/README.md b/emet-providers/README.md new file mode 100644 index 0000000..82e78e1 --- /dev/null +++ b/emet-providers/README.md @@ -0,0 +1,181 @@ +# emet-providers + +The plugins that reach a service for [Emet](../DESIGN.md), or a model on the +robot's own disk: speech recognition, language models and voices. + +Apache 2.0. The counterpart to `emet-hal`, which is named for hardware and +holds drivers, locomotion, wake and audio. A provider is a plugin in the same +sense: it satisfies a contract in `emet_sdk.plugin`, registers an entry point, +and reaches the engine by name. It differs in what it talks to, and in what it +drags in: client libraries are heavy and networked, and a body that only ever +wakes should not install them. So they live here. + +## What ships today + +| Entry point | Group | What it is | +|---|---|---| +| `mock` | `emet.stt` | Reads words out of the bytes it is given, one partial a frame, then a final | +| `deepgram` | `emet.stt` | Streaming recognition through Deepgram, `nova-3` unless the soul says otherwise. Extra: `deepgram` | +| `mock` | `emet.llm` | Repeats what it heard, or says a scripted line, one word a delta; can ask for a tool | +| `anthropic` | `emet.llm` | Claude through the Messages API, streamed, `claude-opus-5` unless the soul says otherwise. Extra: `anthropic` | +| `openai` | `emet.llm` | GPT through Chat Completions, streamed, `gpt-5.6-terra` unless the soul says otherwise; `params.url` points it at any compatible server. Extra: `openai` | +| `mock` | `emet.tts` | Spells the words into the audio it produces, a chunk a word; `read_back()` turns the audio into the words again | +| `piper` | `emet.tts` | The local voice, through Piper on ONNX Runtime, `en_US-ljspeech-medium` unless the soul says otherwise; the model is a file downloaded once. Extra: `piper` (GPL-3.0, imported, never bundled) | +| `deepgram` | `emet.tts` | The cloud voice: Aura over Deepgram's speak API, raw linear16 streamed back, `aura-2-thalia-en` unless the soul says otherwise. Extra: `deepgram` | + +In each group the mock shipped first, so that the real providers were written +against the seam rather than the seam around them, and the real providers +came as a pair: a seam with one implementation is untested as a seam. +`DESIGN.md` sections 12.3 to 12.5 have the contracts and the reasoning. + +## The local voice + +Synthesis is local by default. The reference soul names `piper`, and a +voice is two files the owner downloads once into a place the plugin looks +(`~/.local/share/emet/voices`, or `/etc/emet/voices` for an installed robot, +or wherever `models.tts.params.voices_dir` says): + +```sh +pip install -e "emet-providers[piper]" +python -m piper.download_voices en_US-ljspeech-medium --data-dir ~/.local/share/emet/voices +``` + +The plugin never downloads a voice at boot; when the file is missing it +names that command and refuses to run. Each voice on Hugging Face has its +own licence in its model card. LJ Speech is public domain, which is why it +is the reference voice; several others in the catalogue are fine-tuned from +a corpus licensed for research only, and a shipped default cannot rest on +those. `piper-tts` itself is GPL-3.0-or-later (it compiles espeak-ng in), so +it is an optional extra that Emet imports and does not bundle; a body +without the extra carries none of it. + +A cloud voice is one line of the soul, and a Deepgram key serves both the +transcriber and the voice: + +```yaml +models: + tts: {provider: deepgram, model: "aura-2-thalia-en", key_env: "EMET_DEEPGRAM_KEY"} +``` + +## Keys + +Bring your own. The soul names the environment variable per stage (`key_env`; +the reference soul says `EMET_DEEPGRAM_KEY` and `EMET_OPENAI_KEY`, and the +Anthropic plugin reads `EMET_ANTHROPIC_KEY` when a soul names nothing); the +key itself is never written into a soul or a manifest. Put them in a keys +file, once, and `emet-listen` reads it before any provider starts: + +```sh +mkdir -p ~/.config/emet +cat > ~/.config/emet/keys.env <<'EOF' +EMET_DEEPGRAM_KEY=... # from console.deepgram.com +EMET_OPENAI_KEY=... # from platform.openai.com +EMET_ANTHROPIC_KEY=... # from console.anthropic.com +EOF +chmod 600 ~/.config/emet/keys.env +pip install -e "emet-providers[deepgram,openai,anthropic,piper]" +``` + +An installed robot reads `/etc/emet/keys.env` too, and `--keys FILE` names +any other. A variable already exported in the shell always wins over the +file, so a one-off key for one run still works. `keys.env` is in the +repository's `.gitignore`, whatever directory it lands in. + +At boot each plugin makes one cheap request (a connection for the Deepgram +transcriber, a model listing for the language models, one six-character +word for the Deepgram voice), so a missing key, a rejected key (HTTP 401) +or an unreachable network is reported in those words before the first +question, and the robot refuses to run rather than hear questions it can +never answer. Emet never sees a key except to send it in a header. + +A local model works the same way: point the `openai` provider at any server +that speaks Chat Completions and give it whatever key that server wants. + +```yaml +models: + chat: + provider: openai + model: "llama3.2" + key_env: "EMET_LOCAL_KEY" + params: {url: "http://localhost:11434/v1", max_tokens_field: "max_tokens"} +``` + +## How a provider is chosen + +The soul names one per stage, under `models`: + +```yaml +models: + stt: {provider: deepgram, model: "nova-3", key_env: "EMET_DEEPGRAM_KEY"} + chat: {provider: openai, model: "gpt-5.6-terra", key_env: "EMET_OPENAI_KEY"} + tts: {provider: piper, model: "en_US-ljspeech-medium"} +``` + +The key is the owner's and travels with the soul, so the choice is a soul +field: a cloud account is not hardware. The body may take a stage over with +its own `models..provider`, carrying its own `model` and `key_env`, +which is what a test rig with no key does (`examples/mock-scout.yaml` names +`mock` for all three), and it tunes whichever provider runs through +`models..params`. The soul's `voice.rate`, how fast it speaks, rides +along to whichever voice runs: it is a persona trait, like `patience_ms`. The rule in code is `emet_sdk.models.model_selection`, +one rule for every stage. + +A soul's provider is never checked against what is installed, because a soul +must be valid on every machine or on none. A body's is, and an unresolvable +name is a `missing_plugin` error like `audio.wake.engine`. There is no default +provider: nobody can be assumed to hold an account, and choosing one quietly +would spend somebody's credits. + +## Writing one + +Implement `TranscriberPlugin`, `LanguageModelPlugin` or `VoicePlugin` from +`emet_sdk.plugin`, register an entry point, and install the package: + +```toml +[project.entry-points."emet.stt"] +"deepgram" = "your_package.deepgram:DeepgramTranscriber" + +[project.entry-points."emet.llm"] +"gemini" = "your_package.gemini:GeminiLanguageModel" + +[project.entry-points."emet.tts"] +"kokoro" = "your_package.kokoro:KokoroVoice" +``` + +A transcriber has three methods: `describe()` reports the model that +answered, whether partials will arrive, and the sample rate the instance will +actually run at; `feed(frame)` takes one frame and returns the newest partial +if the text so far changed; `finish()` closes the utterance and returns the +final. Every partial carries the whole text heard so far in the utterance, so +a caption replaces its line rather than splicing fragments. + +A language model has two: `describe()`, and `reply(prompt)`, an async +iterator that yields `TextDelta`s as text arrives, a `ToolCall` per completed +call, and one `ReplyDone` last, on success and on failure alike. A failure +the vendor reported or the network caused ends the stream with +`stop_reason="error"` and the text so far; it never raises through the +engine's turn. + +A voice has two as well: `describe()`, which states the sample rate its +audio arrives at (the engine opens the speaker to match, so say the true +one), and `speak(text)`, an async iterator of mono int16 chunks for one +sentence. Raise `PluginError` for a failure the provider reported; the +engine loses that sentence and goes on with the next. + +Two things worth getting right: + +**Read the key in `start()`, from the environment variable `key_env` names, +and report `healthy=False` with the variable's name when it is absent.** The +engine prints that detail and refuses to run. Claiming health and returning +empty transcripts forever is the failure this contract exists to prevent. + +**`feed()` must return promptly.** It runs inside the capture loop. Hand the +frame to the service and report whatever results have already come back; +a provider that waits for the network here drops audio. + +## Develop and test + +```sh +pip install -e ../emet-sdk -e ".[dev]" +pytest +``` diff --git a/emet-providers/emet_providers/__init__.py b/emet-providers/emet_providers/__init__.py new file mode 100644 index 0000000..8724db0 --- /dev/null +++ b/emet-providers/emet_providers/__init__.py @@ -0,0 +1,51 @@ +"""Emet providers: the plugins that reach a service. + +Hardware lives in `emet-hal`. This package is its counterpart for the things +a robot borrows from a computer somewhere else, or from a model on its own +disk: speech recognition, language models and speech synthesis. They are +plugins in exactly the sense drivers are: they satisfy a contract in +`emet_sdk.plugin`, advertise themselves through entry points, and reach the +engine by name. + +Why a package of its own. `emet-hal` is named for hardware, and a client for +a speech service does not belong under that heading. Provider client +libraries are also heavy and networked, and a body that only ever wakes should +not install them. Keeping them here keeps the HAL a HAL. + +Layering, enforced in CI: this package imports `emet_sdk` and nothing else. + +Shipped, speech recognition (`emet.stt`): + + mock a transcriber that reads words out of the bytes it is given + deepgram streaming recognition through Deepgram (extra: deepgram) + +Shipped, language models (`emet.llm`): + + mock repeats what it heard, or says a scripted line, a word a delta + anthropic Claude through the Messages API (extra: anthropic) + openai GPT, or any Chat Completions server (extra: openai) + +Shipped, voices (`emet.tts`): + + mock spells the words into the audio, a chunk a word + piper local synthesis through Piper, the default (extra: piper) + deepgram Aura through Deepgram's speak API (extra: deepgram) + +In each group the mock came first, so that the real providers were written +against the contract rather than the contract against a provider, and the +real providers came as a pair, because a seam with one implementation is +untested as a seam. +""" + +from importlib import metadata as _metadata + +#: Read from the installed distribution rather than written here, so that +#: `pyproject.toml` is the single place this number appears. Two declarations +#: drift silently, and did once: the metadata said one version and the +#: source said another, and nothing compared them. +try: + __version__ = _metadata.version("emet-providers") +except _metadata.PackageNotFoundError: # pragma: no cover - source checkout + # Imported from a tree that was never installed. Say so rather than + # inventing a number that would later be reported as fact. + __version__ = "0+unknown" diff --git a/emet-providers/emet_providers/_http.py b/emet-providers/emet_providers/_http.py new file mode 100644 index 0000000..89a152b --- /dev/null +++ b/emet-providers/emet_providers/_http.py @@ -0,0 +1,107 @@ +"""What the HTTP providers share. + +Anthropic and OpenAI speak JSON over HTTPS and stream replies as Server-Sent +Events, and Deepgram's voice streams raw audio over the same client, so the +client, the event parser and the error reading are written once here. The +vendors differ in headers, body shape and event names, and those stay in +each provider's own file. + +`httpx` is imported lazily, inside `open_client()`, so that a body with no +cloud provider never needs it and the `anthropic`, `openai` and `deepgram` +extras stay optional. Tests inject an `httpx.MockTransport` through +`transport`. +""" + +from __future__ import annotations + +import json +import logging +from typing import Any, AsyncIterator + +__all__ = ["open_client", "sse_events", "api_error"] + +log = logging.getLogger("emet_providers.http") + + +def open_client(base_url: str, *, timeout_s: float, transport: Any = None, extra: str) -> Any: + """An `httpx.AsyncClient`, or a reason there cannot be one. + + Returns the client, or a string naming what to install. Two return types + rather than an exception because the caller reports this through + `health()`, the way every plugin reports a start failure. + """ + try: + import httpx + except ImportError: + return ( + f"the httpx library is not installed. It is an optional dependency: " + f"install `emet-providers[{extra}]`." + ) + return httpx.AsyncClient(base_url=base_url, timeout=timeout_s, transport=transport) + + +async def sse_events(lines: AsyncIterator[str]) -> AsyncIterator[tuple[str, Any]]: + """Parse Server-Sent Events from a stream of lines. + + Yields `(event, data)` per event: `event` is the `event:` field, or the + data's own `type` when the vendor sends none, or an empty string. `data` + is the parsed JSON, or the raw text under `{"raw": ...}` when it is not + JSON (`[DONE]`, for one vendor). Comment lines are skipped, multi-line + `data:` fields are joined, and a final event without a trailing blank + line is still delivered. + """ + event: str | None = None + data_lines: list[str] = [] + + def flush() -> tuple[str, Any] | None: + if not data_lines: + return None + raw = "\n".join(data_lines) + try: + data: Any = json.loads(raw) + except ValueError: + data = {"raw": raw} + name = event or (data.get("type") if isinstance(data, dict) else None) or "" + return str(name), data + + async for line in lines: + if line == "": + item = flush() + event, data_lines = None, [] + if item is not None: + yield item + continue + if line.startswith(":"): + continue + field, _, value = line.partition(":") + if value.startswith(" "): + value = value[1:] + if field == "event": + event = value + elif field == "data": + data_lines.append(value) + item = flush() + if item is not None: + yield item + + +def api_error(status: int, body: bytes | str) -> str: + """One line for a non-200 response, quoting the vendor's message if any. + + Anthropic and OpenAI answer errors with `{"error": {"type"|"code", + "message"}}`; Deepgram with `{"err_code", "err_msg"}`. Anything else is + quoted as text, trimmed. + """ + text = body.decode("utf-8", "replace") if isinstance(body, bytes) else body + try: + data = json.loads(text) + err = data.get("error") if isinstance(data, dict) else None + if isinstance(err, dict): + kind = err.get("type") or err.get("code") or "error" + return f"HTTP {status} {kind}: {err.get('message', '')}".rstrip(": ") + if isinstance(data, dict) and (data.get("err_code") or data.get("err_msg")): + kind = data.get("err_code") or "error" + return f"HTTP {status} {kind}: {data.get('err_msg', '')}".rstrip(": ") + except ValueError: + pass + return f"HTTP {status}: {text.strip()[:200]}" diff --git a/emet-providers/emet_providers/anthropic.py b/emet-providers/emet_providers/anthropic.py new file mode 100644 index 0000000..c504a19 --- /dev/null +++ b/emet-providers/emet_providers/anthropic.py @@ -0,0 +1,370 @@ +"""The language model through Anthropic's Messages API. + +One of two vendors behind the `emet.llm` seam, shipped together so that the +seam is proven by its second implementation rather than shaped by its first. +Nothing above knows which one is running. + +**Protocol, from Anthropic's API reference on 2026-09-15.** `POST +/v1/messages` with `x-api-key`, `anthropic-version: 2023-06-01` and +`stream: true`. The reply is Server-Sent Events: `message_start` (the model +that answered, input tokens), `content_block_start` for each text or +`tool_use` block, `content_block_delta` carrying `text_delta` or +`input_json_delta` pieces, `content_block_stop`, then `message_delta` with the +`stop_reason` and output tokens, then `message_stop`. Stop reasons: `end_turn`, +`stop_sequence`, `tool_use`, `max_tokens`, `refusal`, `pause_turn`. Tool +results go back as `tool_result` blocks in a user turn. `GET /v1/models` lists +what the key may use, and a bad key is HTTP 401 at any endpoint. + +**What is deliberately not sent.** No `temperature`: the current models +reject it. No `thinking` block: Claude Opus 5 thinks adaptively when the +field is omitted, and the raw reasoning is never returned anyway. Effort is +the knob that matters for a speaking robot and it is a param, unset by +default because not every model accepts it. + +**Refusals.** A safety classifier may decline with HTTP 200 and +`stop_reason: "refusal"`. That is reported as `refusal`, and the engine says +something rather than nothing. Anthropic recommends opting into server-side +fallbacks so a declined request is re-run on another model inside the same +call; that is `params.fallbacks`, off unless the owner turns it on, because it +is a beta and it changes which model answers. + +Params, all optional, from the body's `models.chat.params`: + + effort "low" | "medium" | "high" | "xhigh" | "max". Sent as + `output_config.effort`. `low` suits a spoken reply; unset + leaves the model's default. + fallbacks "default" (Anthropic picks a fallback by refusal category, + beta header `server-side-fallback-2026-07-01`) or a list of + model names (beta header `server-side-fallback-2026-06-01`). + url the API root, for a proxy. Default https://api.anthropic.com + timeout_s seconds per request, connect and read. Default 60. + preflight bool, default true. `GET /v1/models` in `start()` to check + the key and the network. + +Dependencies: `httpx` (BSD-3-Clause), through the `anthropic` extra. Not the +vendor SDK: the wire protocol is small, the same client serves both vendors, +and a vendor SDK is the vendor's shape arriving by another door. +""" + +from __future__ import annotations + +import json +import logging +import os +from typing import Any, AsyncIterator, Mapping, Sequence + +from emet_sdk.plugin import LanguageModelPlugin +from emet_sdk.types import ( + Health, + LanguageModelDescriptor, + Message, + Prompt, + ReplyDone, + ReplyEvent, + TextDelta, + ToolCall, + ToolSpec, +) + +from emet_providers._http import api_error, open_client, sse_events + +__all__ = ["AnthropicLanguageModel", "DEFAULT_MODEL", "DEFAULT_KEY_ENV", "DEFAULT_URL", "API_VERSION"] + +log = logging.getLogger("emet_providers.anthropic") + +DEFAULT_URL = "https://api.anthropic.com" +API_VERSION = "2023-06-01" + +#: Anthropic's current flagship, per their model list on 2026-09-15. The soul +#: names another model when it wants one; a soul that says only +#: `provider: anthropic` gets this. +DEFAULT_MODEL = "claude-opus-5" + +DEFAULT_KEY_ENV = "EMET_ANTHROPIC_KEY" + +FALLBACK_BETA_DEFAULT = "server-side-fallback-2026-07-01" +FALLBACK_BETA_ARRAY = "server-side-fallback-2026-06-01" + +#: The vendor's stop reasons, in the seam's words. Anything unlisted is a +#: natural end: it is how new reasons arrive without breaking a robot. +STOP_REASONS: Mapping[str, str] = { + "end_turn": "end", + "stop_sequence": "end", + "pause_turn": "end", + "tool_use": "tool", + "max_tokens": "length", + "refusal": "refusal", +} + + +def _messages(messages: Sequence[Message]) -> list[dict[str, Any]]: + """The seam's messages in Anthropic's shape. + + Tool results ride in a `user` turn as `tool_result` blocks, and several + results for one assistant turn share one user turn, because the API + wants roles to alternate. + """ + out: list[dict[str, Any]] = [] + for m in messages: + if m.role == "user": + out.append({"role": "user", "content": m.content}) + elif m.role == "assistant": + if not m.tool_calls: + out.append({"role": "assistant", "content": m.content}) + continue + blocks: list[dict[str, Any]] = [] + if m.content: + blocks.append({"type": "text", "text": m.content}) + for call in m.tool_calls: + blocks.append( + {"type": "tool_use", "id": call.id, "name": call.name, "input": dict(call.arguments)} + ) + out.append({"role": "assistant", "content": blocks}) + else: # tool + block = {"type": "tool_result", "tool_use_id": m.tool_call_id, "content": m.content} + last = out[-1] if out else None + if ( + last is not None + and last["role"] == "user" + and isinstance(last["content"], list) + and last["content"] + and last["content"][0].get("type") == "tool_result" + ): + last["content"].append(block) + else: + out.append({"role": "user", "content": [block]}) + return out + + +def _tools(tools: Sequence[ToolSpec]) -> list[dict[str, Any]]: + return [ + {"name": t.name, "description": t.description, "input_schema": dict(t.parameters)} + for t in tools + ] + + +class AnthropicLanguageModel(LanguageModelPlugin): + """Claude, streamed over the Messages API.""" + + provider = "anthropic" + + def __init__(self, config: Mapping[str, Any], *, transport: Any = None) -> None: + super().__init__(config) + #: An `httpx` transport, injected by tests. None means the network. + self._transport = transport + self._client: Any = None + self._key: str | None = None + self._started = False + self._fault: str | None = None + #: The last failure inside a reply, for diagnostics. Not a fault. + self.last_error: str | None = None + + self.model_name: str = self.model or DEFAULT_MODEL + self.key_env_name: str = self.key_env or DEFAULT_KEY_ENV + self.url: str = str(self.params.get("url") or DEFAULT_URL).rstrip("/") + self.timeout_s = float(self.params.get("timeout_s", 60.0)) + self.preflight = bool(self.params.get("preflight", True)) + self.effort: str | None = self.params.get("effort") or None + self.fallbacks: Any = self.params.get("fallbacks") or None + + # ------------------------------------------------------------- request + + def headers(self) -> dict[str, str]: + headers = { + "x-api-key": self._key or "", + "anthropic-version": API_VERSION, + "content-type": "application/json", + } + if self.fallbacks == "default": + headers["anthropic-beta"] = FALLBACK_BETA_DEFAULT + elif self.fallbacks: + headers["anthropic-beta"] = FALLBACK_BETA_ARRAY + return headers + + def body(self, prompt: Prompt) -> dict[str, Any]: + body: dict[str, Any] = { + "model": self.model_name, + "max_tokens": prompt.max_tokens, + "stream": True, + "messages": _messages(prompt.messages), + } + if prompt.system: + body["system"] = prompt.system + if prompt.tools: + body["tools"] = _tools(prompt.tools) + if self.effort: + body["output_config"] = {"effort": self.effort} + if self.fallbacks == "default": + body["fallbacks"] = "default" + elif self.fallbacks: + body["fallbacks"] = [{"model": str(m)} for m in self.fallbacks] + return body + + # ------------------------------------------------------------ lifecycle + + async def start(self) -> None: + self._key = os.environ.get(self.key_env_name) or None + if not self._key: + self._fail( + f"no key: the environment variable {self.key_env_name} is not set. " + f"Anthropic keys come from console.anthropic.com; export it in the " + f"shell that runs the robot. The key is never written into a soul " + f"or a manifest." + ) + return + client = open_client(self.url, timeout_s=self.timeout_s, transport=self._transport, extra="anthropic") + if isinstance(client, str): + self._fail(client) + return + self._client = client + if self.preflight: + reason = await self._preflight() + if reason: + self._fail(reason) + return + self._started = True + log.info("anthropic: ready, model %s", self.model_name) + + async def _preflight(self) -> str | None: + try: + response = await self._client.get("/v1/models", headers=self.headers()) + except Exception as exc: # noqa: BLE001 - reported through health, not raised + return ( + f"could not reach {self.url}: {type(exc).__name__}: {exc}. " + f"The language model needs a network; the wake word does not." + ) + if response.status_code == 200: + return None + if response.status_code in (401, 403): + return ( + f"Anthropic rejected the key in {self.key_env_name} " + f"(HTTP {response.status_code}). Check the key." + ) + return f"Anthropic refused the connection: {api_error(response.status_code, response.content)}" + + def _fail(self, reason: str) -> None: + self._fault = reason + log.error("anthropic: %s", reason) + + async def shutdown(self) -> None: + self._started = False + if self._client is not None: + try: + await self._client.aclose() + except Exception: # noqa: BLE001 + pass + self._client = None + + # ------------------------------------------------------------ reporting + + def describe(self) -> LanguageModelDescriptor: + return LanguageModelDescriptor( + provider=self.provider, + model=self.model_name, + streaming=True, + tools=True, + healthy=self._started, + ) + + def health(self) -> Health: + if self._fault: + fault = "no_key" if "no key" in self._fault else "start_failed" + return Health(ok=False, detail=self._fault, faults=(fault,)) + return Health() + + # ---------------------------------------------------------------- reply + + async def reply(self, prompt: Prompt) -> AsyncIterator[ReplyEvent]: + if not self._started or self._client is None: + yield ReplyDone(text="", stop_reason="error", error="the anthropic plugin was not started") + return + + text: list[str] = [] + calls: list[ToolCall] = [] + blocks: dict[int, dict[str, Any]] = {} + stop = "end" + model: str | None = None + input_tokens: int | None = None + output_tokens: int | None = None + error: str | None = None + + try: + async with self._client.stream( + "POST", "/v1/messages", headers=self.headers(), json=self.body(prompt) + ) as response: + if response.status_code != 200: + error = api_error(response.status_code, await response.aread()) + else: + async for event, data in sse_events(response.aiter_lines()): + if not isinstance(data, dict): + continue + if event == "message_start": + message = data.get("message") or {} + model = message.get("model") or model + usage = message.get("usage") or {} + if isinstance(usage.get("input_tokens"), int): + input_tokens = usage["input_tokens"] + elif event == "content_block_start": + block = data.get("content_block") or {} + index = int(data.get("index", 0)) + if block.get("type") == "tool_use": + blocks[index] = { + "type": "tool_use", + "id": str(block.get("id", "")), + "name": str(block.get("name", "")), + "json": "", + } + elif block.get("type") == "text": + blocks[index] = {"type": "text"} + elif event == "content_block_delta": + delta = data.get("delta") or {} + index = int(data.get("index", 0)) + if delta.get("type") == "text_delta": + piece = str(delta.get("text", "")) + if piece: + text.append(piece) + yield TextDelta(piece) + elif delta.get("type") == "input_json_delta": + if index in blocks and blocks[index]["type"] == "tool_use": + blocks[index]["json"] += str(delta.get("partial_json", "")) + elif event == "content_block_stop": + index = int(data.get("index", 0)) + block = blocks.pop(index, None) + if block and block["type"] == "tool_use": + try: + arguments = json.loads(block["json"]) if block["json"] else {} + except ValueError: + arguments = {} + call = ToolCall(id=block["id"], name=block["name"], arguments=arguments) + calls.append(call) + yield call + elif event == "message_delta": + delta = data.get("delta") or {} + reason = delta.get("stop_reason") + if reason: + stop = STOP_REASONS.get(str(reason), "end") + usage = data.get("usage") or {} + if isinstance(usage.get("output_tokens"), int): + output_tokens = usage["output_tokens"] + elif event == "message_stop": + break + elif event == "error": + err = data.get("error") or {} + error = f"{err.get('type', 'error')}: {err.get('message', '')}" + break + except Exception as exc: # noqa: BLE001 - the reply ends with what was heard + error = f"{type(exc).__name__}: {exc}" + + if error: + stop = "error" + self.last_error = error + log.warning("anthropic: %s", error) + yield ReplyDone( + text="".join(text), + stop_reason=stop, + tool_calls=tuple(calls), + model=model or self.model_name, + input_tokens=input_tokens, + output_tokens=output_tokens, + error=error, + ) diff --git a/emet-providers/emet_providers/deepgram.py b/emet-providers/emet_providers/deepgram.py new file mode 100644 index 0000000..2e003ab --- /dev/null +++ b/emet-providers/emet_providers/deepgram.py @@ -0,0 +1,427 @@ +"""Speech recognition through Deepgram's streaming API. + +The first real provider, and the second thing behind the `emet.stt` seam: +the mock shipped first so that this file would be written against the +contract rather than the contract against this file. Nothing above it knows +it exists. A soul says `provider: deepgram`, the engine asks the registry, +and frames flow. + +**Protocol, verified against Deepgram's documentation on 2026-09-12.** +Audio goes to `wss://api.deepgram.com/v1/listen` as binary WebSocket frames +of 16-bit little-endian PCM, with `encoding=linear16` and the `sample_rate` +that parameter requires. The key travels in the handshake as +`Authorization: Token `; a bad key is refused at the handshake with +HTTP 401. Results come back as JSON text frames of `"type": "Results"`, each +carrying one alternative's `transcript` and `confidence` and an `is_final` +flag. A segment is revised through interim messages until one arrives with +`is_final: true`, and the utterance is the finalised segments joined in order +plus the latest interim of the segment still open. `{"type": "CloseStream"}` +makes the server flush what it holds, send the remaining finals, send one +`"type": "Metadata"` message and close. A connection that receives neither +audio nor `{"type": "KeepAlive"}` for ten seconds is closed by the server. + +**One connection per utterance.** The engine feeds this plugin only between +a wake and an endpoint, and Deepgram closes an idle connection after ten +seconds, so a session-long socket would need a keepalive task ticking through +every silence. Opening a socket when the first frame of an utterance arrives +costs a handshake at the start of each turn, and it costs it in the +background: `feed()` queues frames and returns, a task connects and drains +the queue, and nothing in the capture loop waits on the network. The engine +does the endpointing, so Deepgram's `speech_final` is read and ignored. + +**What fails loudly, and where.** `start()` reads the key and refuses to +report healthy without one, naming the variable to set. It then opens and +closes one connection, so a rejected key or an unreachable network is known +at boot, in the terms the owner can act on, rather than at the first +question. A failure during an utterance is logged and the final carries what +was heard before it; a transient network fault is not a broken plugin. + +Params, all optional, from the body's `audio.stt.params`: + + query mapping of extra Deepgram query parameters (`language`, + `keyterm`, `endpointing`, ...), passed through as strings. + `encoding`, `sample_rate` and `channels` are fixed by the + audio format and cannot be overridden here. + timeout_s seconds `finish()` waits for the final after CloseStream. + Default 5. + open_timeout_s + seconds to allow the handshake. Default 10. + preflight bool, default true. Open and close a connection in + `start()` to check the key and the network. + url the WebSocket URL, for a self-hosted deployment. + +Dependencies: `websockets` (BSD-3-Clause), through the `deepgram` extra. +Nothing from Deepgram's SDK; the wire protocol is small and this file speaks +it directly, which is also what keeps the dependency list to one package. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import os +from dataclasses import dataclass, field +from typing import Any, Callable, Mapping +from urllib.parse import urlencode + +from emet_sdk.plugin import TranscriberPlugin +from emet_sdk.types import AudioFormat, Health, Transcript, TranscriberDescriptor + +__all__ = ["DeepgramTranscriber", "DEFAULT_MODEL", "DEFAULT_KEY_ENV", "DEFAULT_URL"] + +log = logging.getLogger("emet_providers.deepgram") + +DEFAULT_URL = "wss://api.deepgram.com/v1/listen" + +#: Deepgram's current general model, verified against their model list on +#: 2026-09-12. `nova-2` and its variants are still served; `flux-general-en` +#: is a different protocol (`/v2/listen`, its own turn events) and is not +#: reachable through this plugin. +DEFAULT_MODEL = "nova-3" + +#: Where the key is read from when the soul names no `key_env`. The name the +#: reference soul uses, so a soul that says only `provider: deepgram` works. +DEFAULT_KEY_ENV = "EMET_DEEPGRAM_KEY" + +#: Query parameters this plugin always sends. `interim_results` is the whole +#: point of a streaming seam; `punctuate` and `smart_format` are what a +#: language model downstream wants to read. +FIXED_QUERY: Mapping[str, str] = { + "encoding": "linear16", + "channels": "1", + "interim_results": "true", + "punctuate": "true", + "smart_format": "true", +} + +#: Keys the body may not override through `params.query`: the audio format +#: fixes them, and a mismatch is the silent failure the descriptor exists to +#: prevent. +FORMAT_KEYS = frozenset({"encoding", "sample_rate", "channels"}) + +_CLOSE_STREAM = json.dumps({"type": "CloseStream"}) + + +@dataclass +class _Utterance: + """Everything one turn's connection accumulates.""" + + queue: asyncio.Queue = field(default_factory=asyncio.Queue) + task: asyncio.Task | None = None + finals: list[str] = field(default_factory=list) + final_confidences: list[float] = field(default_factory=list) + interim: str = "" + interim_confidence: float = 1.0 + reported: str = "" + error: BaseException | None = None + ws: Any = None + + @property + def text(self) -> str: + parts = [*self.finals] + if self.interim: + parts.append(self.interim) + return " ".join(p.strip() for p in parts if p.strip()) + + +def _is_closed(exc: BaseException) -> bool: + """Whether an exception is the connection closing. + + Duck-typed on the `rcvd` and `sent` attributes every + `websockets.exceptions.ConnectionClosed` carries, so this module never + imports the library at module scope and a test can stand in for it. + """ + return hasattr(exc, "rcvd") and hasattr(exc, "sent") + + +def _status_code(exc: BaseException) -> int | None: + """The HTTP status of a rejected handshake, if this exception has one.""" + response = getattr(exc, "response", None) + code = getattr(response, "status_code", None) + return int(code) if isinstance(code, int) else None + + +class DeepgramTranscriber(TranscriberPlugin): + """Streaming recognition through Deepgram, one connection per utterance.""" + + provider = "deepgram" + + def __init__( + self, + config: Mapping[str, Any], + fmt: AudioFormat, + *, + connect: Callable[..., Any] | None = None, + ) -> None: + super().__init__(config, fmt) + #: Something with the shape of `websockets.asyncio.client.connect`: + #: `await connect(url, additional_headers=..., open_timeout=...)` + #: returns a connection with `send`, `recv` and `close`. Injected by + #: tests; found by `start()` otherwise. + self._connect = connect + self._key: str | None = None + self._started = False + self._fault: str | None = None + self._utterance: _Utterance | None = None + #: The last failure inside an utterance, for diagnostics. Not a fault: + #: a dropped connection is a bad moment, not a broken plugin. + self.last_error: str | None = None + + self.model_name: str = self.model or DEFAULT_MODEL + self.key_env_name: str = self.key_env or DEFAULT_KEY_ENV + self.url: str = str(self.params.get("url") or DEFAULT_URL) + self.timeout_s = float(self.params.get("timeout_s", 5.0)) + self.open_timeout_s = float(self.params.get("open_timeout_s", 10.0)) + self.preflight = bool(self.params.get("preflight", True)) + + # ------------------------------------------------------------- request + + def query(self) -> dict[str, str]: + """The query string, as a mapping. Format keys win over `params.query`.""" + extra = { + str(k): str(v).lower() if isinstance(v, bool) else str(v) + for k, v in (self.params.get("query") or {}).items() + if str(k) not in FORMAT_KEYS + } + return { + **FIXED_QUERY, + **extra, + "sample_rate": str(self.format.sample_rate), + "model": self.model_name, + } + + def request_url(self) -> str: + return f"{self.url}?{urlencode(self.query())}" + + def _headers(self) -> dict[str, str]: + return {"Authorization": f"Token {self._key}"} + + # ------------------------------------------------------------ lifecycle + + async def start(self) -> None: + # The key first, then the library: an owner without a key is the + # common case, and the message for it should not depend on which + # extras happen to be installed. + self._key = os.environ.get(self.key_env_name) or None + if not self._key: + self._fail( + f"no key: the environment variable {self.key_env_name} is not set. " + f"Deepgram keys come from console.deepgram.com; export it in the " + f"shell that runs the robot. The key is never written into a soul " + f"or a manifest." + ) + return + + if self._connect is None: + try: + from websockets.asyncio.client import connect + except ImportError: + self._fail( + "the websockets library is not installed. It is an optional " + "dependency: install `emet-providers[deepgram]`." + ) + return + self._connect = connect + + if self.preflight: + reason = await self._preflight() + if reason: + self._fail(reason) + return + + self._started = True + log.info("deepgram: ready, model %s at %d Hz", self.model_name, self.format.sample_rate) + + async def _preflight(self) -> str | None: + """Open one connection and close it. Returns the reason it could not.""" + assert self._connect is not None + try: + ws = await asyncio.wait_for( + self._connect( + self.request_url(), + additional_headers=self._headers(), + open_timeout=self.open_timeout_s, + ), + timeout=self.open_timeout_s + 1.0, + ) + except asyncio.TimeoutError: + return ( + f"could not reach {self.url} within {self.open_timeout_s:.0f} s. " + f"Speech recognition needs a network; the wake word does not." + ) + except Exception as exc: # noqa: BLE001 - every failure here is reported, not raised + code = _status_code(exc) + if code in (401, 403): + return ( + f"Deepgram rejected the key in {self.key_env_name} (HTTP {code}). " + f"Check the key, and that it has the transcription scope." + ) + if code is not None: + return f"Deepgram refused the connection with HTTP {code}: {exc}" + return ( + f"could not reach {self.url}: {type(exc).__name__}: {exc}. " + f"Speech recognition needs a network; the wake word does not." + ) + try: + await ws.send(_CLOSE_STREAM) + try: + await asyncio.wait_for(self._drain(ws), timeout=3.0) + except asyncio.TimeoutError: + pass + finally: + await self._close(ws) + return None + + async def _drain(self, ws: Any) -> None: + """Read until the server closes. Used after CloseStream.""" + while True: + try: + await ws.recv() + except Exception as exc: # noqa: BLE001 + if _is_closed(exc): + return + raise + + async def _close(self, ws: Any) -> None: + try: + await ws.close() + except Exception: # noqa: BLE001 - closing a closed socket is not news + pass + + def _fail(self, reason: str) -> None: + self._fault = reason + log.error("deepgram: %s", reason) + + async def shutdown(self) -> None: + self._started = False + utterance, self._utterance = self._utterance, None + if utterance is not None and utterance.task is not None and not utterance.task.done(): + utterance.task.cancel() + try: + await utterance.task + except (asyncio.CancelledError, Exception): # noqa: BLE001 + pass + + # ------------------------------------------------------------ reporting + + def describe(self) -> TranscriberDescriptor: + return TranscriberDescriptor( + provider=self.provider, + model=self.model_name, + streaming=True, + sample_rate=self.format.sample_rate, + healthy=self._started, + ) + + def health(self) -> Health: + if self._fault: + fault = "no_key" if "no key" in self._fault else "start_failed" + return Health(ok=False, detail=self._fault, faults=(fault,)) + return Health() + + # ---------------------------------------------------------- recognition + + async def feed(self, frame: bytes) -> Transcript | None: + if not self._started: + return None + utterance = self._utterance + if utterance is None: + utterance = self._utterance = _Utterance() + utterance.task = asyncio.create_task(self._pump(utterance)) + if utterance.task is not None and not utterance.task.done(): + utterance.queue.put_nowait(frame) + text = utterance.text + if text == utterance.reported: + return None + utterance.reported = text + return Transcript(text=text, final=False, confidence=utterance.interim_confidence) + + async def finish(self) -> Transcript: + utterance, self._utterance = self._utterance, None + if utterance is None: + # Nothing was fed, so nothing was opened. An empty final is the + # honest answer and costs no round trip. + return Transcript(text="", final=True, confidence=1.0) + + utterance.queue.put_nowait(None) + if utterance.task is not None: + try: + await asyncio.wait_for(utterance.task, timeout=self.timeout_s) + except asyncio.TimeoutError: + utterance.task.cancel() + self.last_error = f"no final within {self.timeout_s:.0f} s of CloseStream" + log.warning("deepgram: %s; returning what was heard", self.last_error) + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 - reported, and the words so far returned + self.last_error = f"{type(exc).__name__}: {exc}" + log.warning("deepgram: %s; returning what was heard", self.last_error) + + confidences = utterance.final_confidences or [utterance.interim_confidence] + confidence = max(0.0, min(1.0, sum(confidences) / len(confidences))) + return Transcript(text=utterance.text, final=True, confidence=confidence) + + async def _pump(self, utterance: _Utterance) -> None: + """Connect, then send frames and receive results until both sides end.""" + assert self._connect is not None + ws = await self._connect( + self.request_url(), + additional_headers=self._headers(), + open_timeout=self.open_timeout_s, + ) + utterance.ws = ws + try: + async with asyncio.TaskGroup() as group: + group.create_task(self._send_all(utterance, ws)) + group.create_task(self._receive_all(utterance, ws)) + finally: + await self._close(ws) + + async def _send_all(self, utterance: _Utterance, ws: Any) -> None: + while True: + frame = await utterance.queue.get() + if frame is None: + # A text frame, as the documentation insists. Binary would be + # taken for audio. + await ws.send(_CLOSE_STREAM) + return + await ws.send(frame) + + async def _receive_all(self, utterance: _Utterance, ws: Any) -> None: + while True: + try: + message = await ws.recv() + except Exception as exc: # noqa: BLE001 + if _is_closed(exc): + return + raise + if isinstance(message, bytes): + continue + try: + data = json.loads(message) + except ValueError: + log.debug("deepgram: ignoring a frame that is not JSON") + continue + kind = data.get("type") + if kind == "Metadata": + # The last thing the server says after CloseStream. + return + if kind != "Results": + continue + alternatives = (data.get("channel") or {}).get("alternatives") or [] + if not alternatives: + continue + best = alternatives[0] + transcript = str(best.get("transcript") or "") + confidence = best.get("confidence") + confidence = float(confidence) if isinstance(confidence, (int, float)) else 1.0 + if data.get("is_final"): + if transcript.strip(): + utterance.finals.append(transcript) + utterance.final_confidences.append(confidence) + utterance.interim = "" + utterance.interim_confidence = 1.0 + else: + utterance.interim = transcript + utterance.interim_confidence = confidence diff --git a/emet-providers/emet_providers/deepgram_voice.py b/emet-providers/emet_providers/deepgram_voice.py new file mode 100644 index 0000000..8eb332e --- /dev/null +++ b/emet-providers/emet_providers/deepgram_voice.py @@ -0,0 +1,337 @@ +"""The cloud voice, through Deepgram's Aura text-to-speech API. + +The opt-in the design reserves (`DESIGN.md` section 14): synthesis is local +by default, and a soul that wants a better voice than a Pi can produce names +this one and pays per character. It ships beside the local voice for the +same reason the language models shipped as a pair: a seam with one +implementation is untested as a seam, and a cloud voice is the shape most +likely to pull the engine out of true if nothing else stood against it. + +**Protocol, verified against Deepgram's API reference on 2026-09-15.** +`POST https://api.deepgram.com/v1/speak` with `Authorization: Token ` +and a JSON body `{"text": ...}`. The voice is the `model` query parameter +(`aura-2-thalia-en` here unless the soul says otherwise; Aura-2 is the +current family, Aura-1 names such as `aura-asteria-en` still answer). +`encoding=linear16` with `container=none` returns raw 16-bit little-endian +mono PCM with no header, at a `sample_rate` of 8000, 16000, 24000 (the +default), 32000 or 48000. `speed` is a multiplier on the speaking rate, +which is where the soul's `voice.rate` goes. The audio is streamed back as +it is produced, so playback can start on the first bytes. Text is limited +to 2000 characters a request (HTTP 413 beyond it); a bad key is HTTP 401 +with `{"err_code": "INVALID_AUTH", "err_msg": "Invalid credentials."}`, +checked live with a bogus key on 2026-09-15. + +**Pricing, from the pricing page on 2026-09-15.** Aura-2 is $0.030 per +thousand characters pay as you go, Aura-1 $0.015. A two-sentence reply is +around 150 characters, so a thousand replies cost about $4.50. Flux TTS, the +conversation-aware family launched 2026-08-12, is $0.045 per thousand and +speaks a different protocol (`/v2/speak`, raw audio over WebSocket only, with +its own turn and interrupt messages); it is in the deferred register rather +than in this file. + +**One request per sentence.** The engine hands over a sentence at a time and +plays each as its audio completes, so a request per sentence is what lets +the first sentence be heard while the language model is still writing the +second. A reply longer than the limit is split at sentence and then word +boundaries and sent as several requests in order. + +**What fails loudly, and where.** `start()` reads the key and refuses to +report healthy without one, naming the variable. It then synthesises one +short word and discards the audio, so a rejected key, an unreachable +network, or a voice name that does not exist is known at boot in the owner's +terms rather than at the first reply. That preflight costs six characters, +which is to say nothing; `params.preflight: false` skips it. A failure +during a sentence raises `PluginError` after the audio so far, and the +engine goes on with the next sentence. + +Params, all optional, from the body's `models.tts.params`: + + sample_rate 8000, 16000, 24000, 32000 or 48000. Default 24000. The + engine opens the speaker at this rate. + query mapping of extra query parameters, passed through as + strings. `model`, `encoding`, `container` and + `sample_rate` are fixed by this plugin and cannot be + overridden here. + url the endpoint, for a self-hosted deployment. + Default https://api.deepgram.com/v1/speak + timeout_s seconds per request, connect and read. Default 30. + preflight bool, default true. + +Dependencies: `httpx` (BSD-3-Clause), through the `deepgram` extra, which +the transcriber next door shares. Nothing from Deepgram's SDK. +""" + +from __future__ import annotations + +import logging +import os +import re +from typing import Any, AsyncIterator, Mapping + +from emet_sdk.plugin import PluginError, VoicePlugin +from emet_sdk.types import Health, VoiceDescriptor + +from emet_providers._http import api_error, open_client + +__all__ = [ + "DeepgramVoice", + "DEFAULT_MODEL", + "DEFAULT_KEY_ENV", + "DEFAULT_URL", + "DEFAULT_SAMPLE_RATE", + "SAMPLE_RATES", + "MAX_CHARS", + "pieces", +] + +log = logging.getLogger("emet_providers.deepgram_voice") + +DEFAULT_URL = "https://api.deepgram.com/v1/speak" + +#: An Aura-2 voice, the current family on Deepgram's model list (2026-09-15). +#: The soul names another when it wants one. +DEFAULT_MODEL = "aura-2-thalia-en" + +#: The same variable the transcriber reads: one Deepgram key serves both. +DEFAULT_KEY_ENV = "EMET_DEEPGRAM_KEY" + +#: What linear16 may be asked for, and the API's own default. +SAMPLE_RATES: tuple[int, ...] = (8000, 16000, 24000, 32000, 48000) +DEFAULT_SAMPLE_RATE = 24000 + +#: Characters a request may carry. Beyond it the API answers HTTP 413. +MAX_CHARS = 2000 + +#: The speed range the streaming reference documents (0.7 to 1.5); a soul +#: asking for more is clamped and told so once. +SPEED_MIN, SPEED_MAX = 0.7, 1.5 + +#: Query keys the body may not override: the format is this plugin's to +#: state, and the engine opens the speaker to match it. +FORMAT_KEYS = frozenset({"model", "encoding", "container", "sample_rate"}) + +#: What the preflight says. Six characters. +PREFLIGHT_TEXT = "Ready." + +_SENTENCE_END = re.compile(r"(?<=[.!?])\s+") + + +def pieces(text: str, limit: int = MAX_CHARS) -> list[str]: + """Split text into request-sized pieces at sentence, then word, boundaries. + + Almost every sentence the engine hands over fits in one piece; this + exists so that the one that does not is spoken in order rather than + refused. + """ + text = " ".join(text.split()) + if not text: + return [] + if len(text) <= limit: + return [text] + out: list[str] = [] + current = "" + for unit in _SENTENCE_END.split(text): + words = unit.split(" ") if len(unit) > limit else [unit] + for word in words: + if not current: + current = word + elif len(current) + 1 + len(word) <= limit: + current = f"{current} {word}" + else: + out.append(current) + current = word + while len(current) > limit: + out.append(current[:limit]) + current = current[limit:] + if current: + out.append(current) + return out + + +class DeepgramVoice(VoicePlugin): + """Aura, streamed over HTTPS, one request per sentence.""" + + provider = "deepgram" + + def __init__( + self, + config: Mapping[str, Any], + voice: Mapping[str, Any] | None = None, + *, + transport: Any = None, + ) -> None: + super().__init__(config, voice) + #: An `httpx` transport, injected by tests. None means the network. + self._transport = transport + self._client: Any = None + self._key: str | None = None + self._started = False + self._fault: str | None = None + #: The last failure inside a sentence, for diagnostics. Not a fault. + self.last_error: str | None = None + + self.model_name: str = self.model or DEFAULT_MODEL + self.key_env_name: str = self.key_env or DEFAULT_KEY_ENV + self.url: str = str(self.params.get("url") or DEFAULT_URL) + self.timeout_s = float(self.params.get("timeout_s", 30.0)) + self.preflight = bool(self.params.get("preflight", True)) + rate = int(self.params.get("sample_rate") or DEFAULT_SAMPLE_RATE) + if rate not in SAMPLE_RATES: + log.warning( + "deepgram voice: sample_rate %d is not one linear16 offers %s; using %d", + rate, + SAMPLE_RATES, + DEFAULT_SAMPLE_RATE, + ) + rate = DEFAULT_SAMPLE_RATE + self.sample_rate: int = rate + speed = self.rate + if not SPEED_MIN <= speed <= SPEED_MAX: + clamped = min(SPEED_MAX, max(SPEED_MIN, speed)) + log.warning("deepgram voice: rate %.2f is outside %.1f to %.1f; using %.2f", speed, SPEED_MIN, SPEED_MAX, clamped) + speed = clamped + self.speed: float = speed + + # ------------------------------------------------------------- request + + def query(self) -> dict[str, str]: + """The query string, as a mapping. Format keys win over `params.query`.""" + extra = { + str(k): str(v).lower() if isinstance(v, bool) else str(v) + for k, v in (self.params.get("query") or {}).items() + if str(k) not in FORMAT_KEYS + } + out = { + **extra, + "model": self.model_name, + "encoding": "linear16", + "container": "none", + "sample_rate": str(self.sample_rate), + } + if self.speed != 1.0: + out["speed"] = f"{self.speed:g}" + return out + + def headers(self) -> dict[str, str]: + return {"Authorization": f"Token {self._key or ''}", "Content-Type": "application/json"} + + # ------------------------------------------------------------ lifecycle + + async def start(self) -> None: + self._key = os.environ.get(self.key_env_name) or None + if not self._key: + self._fail( + f"no key: the environment variable {self.key_env_name} is not set. " + f"Deepgram keys come from console.deepgram.com; put it in the keys " + f"file. The key is never written into a soul or a manifest." + ) + return + client = open_client(self.url, timeout_s=self.timeout_s, transport=self._transport, extra="deepgram") + if isinstance(client, str): + self._fail(client) + return + self._client = client + if self.preflight: + reason = await self._preflight() + if reason: + self._fail(reason) + return + self._started = True + log.info("deepgram voice: ready, %s at %d Hz", self.model_name, self.sample_rate) + + async def _preflight(self) -> str | None: + """Say one word into the void. Returns the reason it could not.""" + try: + response = await self._client.post( + self.url, params=self.query(), headers=self.headers(), json={"text": PREFLIGHT_TEXT} + ) + except Exception as exc: # noqa: BLE001 - reported through health, not raised + return ( + f"could not reach {self.url}: {type(exc).__name__}: {exc}. " + f"A cloud voice needs a network; the local one does not." + ) + if response.status_code == 200: + return None + if response.status_code in (401, 403): + return ( + f"Deepgram rejected the key in {self.key_env_name} " + f"(HTTP {response.status_code}). Check the key, and that it has the " + f"text-to-speech scope." + ) + return ( + f"Deepgram refused to speak as {self.model_name!r}: " + f"{api_error(response.status_code, response.content)}. Check the voice " + f"name against Deepgram's model list." + ) + + def _fail(self, reason: str) -> None: + self._fault = reason + log.error("deepgram voice: %s", reason) + + async def shutdown(self) -> None: + self._started = False + if self._client is not None: + try: + await self._client.aclose() + except Exception: # noqa: BLE001 + pass + self._client = None + + # ------------------------------------------------------------ reporting + + def describe(self) -> VoiceDescriptor: + return VoiceDescriptor( + provider=self.provider, + model=self.model_name, + sample_rate=self.sample_rate, + streaming=True, + healthy=self._started, + ) + + def health(self) -> Health: + if self._fault: + fault = "no_key" if "no key" in self._fault else "start_failed" + return Health(ok=False, detail=self._fault, faults=(fault,)) + return Health() + + # ---------------------------------------------------------------- speak + + async def speak(self, text: str) -> AsyncIterator[bytes]: + if not text.strip(): + return + if not self._started or self._client is None: + raise PluginError("the deepgram voice was not started") + for piece in pieces(text): + async for chunk in self._request(piece): + yield chunk + + async def _request(self, piece: str) -> AsyncIterator[bytes]: + """One request, streamed. Chunks are kept to whole samples: a chunk + boundary that falls between the two bytes of a sample would otherwise + put a click in the audio and a byte in the wrong place.""" + carry = b"" + try: + async with self._client.stream( + "POST", self.url, params=self.query(), headers=self.headers(), json={"text": piece} + ) as response: + if response.status_code != 200: + body = await response.aread() + self.last_error = api_error(response.status_code, body) + log.warning("deepgram voice: %s", self.last_error) + raise PluginError(f"Deepgram could not say {piece!r}: {self.last_error}") + async for raw in response.aiter_bytes(): + data = carry + raw + if len(data) % 2: + data, carry = data[:-1], data[-1:] + else: + carry = b"" + if data: + yield data + except PluginError: + raise + except Exception as exc: # noqa: BLE001 - one sentence lost, reported + self.last_error = f"{type(exc).__name__}: {exc}" + log.warning("deepgram voice: %s", self.last_error) + raise PluginError(f"Deepgram could not say {piece!r}: {self.last_error}") from exc + if carry: + log.debug("deepgram voice: dropped a dangling byte at the end of a sentence") diff --git a/emet-providers/emet_providers/mock.py b/emet-providers/emet_providers/mock.py new file mode 100644 index 0000000..7361675 --- /dev/null +++ b/emet-providers/emet_providers/mock.py @@ -0,0 +1,398 @@ +"""A speech recogniser, a language model and a voice that pretend. + +`MockTranscriber` reads words out of the bytes it is given. A frame that +begins with printable text, followed by zeros, is taken to be somebody saying +that text. It is the same trick `emet_hal.mock.MockWake` plays with the wake +phrase, and it exists for the same reason: the whole loop, wake to transcript, +runs on a laptop with no microphone, no network and no key, and a test writes +what it wants said into the audio and reads it back out. + +It streams. Each frame that adds words produces a partial carrying everything +heard so far in the utterance, and `finish()` returns the same words as the +final. Silence adds nothing. So the shape of the real thing (frames in, a +growing partial, one final) is exercised end to end before any provider +exists, and the engine above cannot tell this from a vendor except by how +little it costs. + +Params, all optional: + + transcript str. Say this instead of reading the audio: one word per + frame as partials, then the whole line as the final. For + a live microphone or a replay of a real recording, whose + bytes spell nothing on purpose and the odd short word by + accident. It says its line after a false wake too, since + it hears no speech to wait for; the engine reports the + turn as a false wake beside it, and a real provider + returns an empty final there. + fail_on_start bool. Pretend the service is unreachable. `start()` + completes and `describe()` reports unhealthy, which is + how a real provider reports a dead network. + require_key bool. Insist that the environment variable named by + `key_env` is set, and report unhealthy naming it when it + is not. The BYOK failure path, without a vendor. + sample_rate int. Report this rate instead of the one given, so a test + can watch the engine refuse a transcriber that would hear + speech at the wrong speed. + +`MockLanguageModel` answers without a model. Given nothing, it repeats the +last thing the person said; given `params.reply`, it says that line. Either +way it streams one word per delta, so a caller that speaks as it reads has +something to read. It records every prompt it was shown, which is how a test +checks what the engine actually told the model. + +Params, all optional: + + reply str. Say this, whatever was asked. + call_tool {"name": ..., "arguments": {...}}. When the prompt offers + tools and the last message is the person's, ask for this + tool instead of answering, and stop with `tool`. Given a + tool result next, say what the tool said. The tool round + trip, without a vendor. + fail_on_start bool. Pretend the service is unreachable. + require_key bool. Insist on the environment variable `key_env` + names. + +`MockVoice` speaks without a voice. The words it is given become the audio: +one chunk per word, each chunk the word's bytes followed by zeros, which is +the transcriber's trick run backwards. `read_back()` turns that audio into +the words again, so a test that records the robot through the `wav` sink +can assert on what was said. It streams a chunk per word, so the engine's +sentence pipeline is exercised end to end with no model and no network. + +Params, all optional: + + sample_rate int. Report this rate, default 16000; the engine opens + the sink at whatever a voice reports. + latency_ms int. Wait this long before the first chunk of each + sentence, the way a real voice would. + fail_on str. A sentence containing this word raises after its + first chunk, which is how a test watches the engine + lose one sentence and keep the next. + fail_on_start bool. Pretend the model is missing. + require_key bool. Insist on the environment variable `key_env` + names, the way a cloud voice would. +""" + +from __future__ import annotations + +import asyncio +import logging +import os +import re +from typing import Any, AsyncIterator, Mapping + +from emet_sdk.plugin import LanguageModelPlugin, PluginError, TranscriberPlugin, VoicePlugin +from emet_sdk.types import ( + AudioFormat, + Health, + LanguageModelDescriptor, + Prompt, + ReplyDone, + ReplyEvent, + TextDelta, + ToolCall, + Transcript, + TranscriberDescriptor, + VoiceDescriptor, +) + +__all__ = ["MockTranscriber", "MockLanguageModel", "MockVoice", "spelled", "read_back", "MIN_CHARS"] + +log = logging.getLogger("emet_providers.mock") + +#: Shortest run of text a frame must begin with to count as words. Real audio +#: is int16, so a printable byte followed by a zero byte is an ordinary small +#: sample; three printable bytes in a row before a zero is rare enough. +MIN_CHARS = 3 + + +def spelled(frame: bytes) -> str: + """The words a frame spells, or an empty string. + + Text is the run of bytes the frame starts with, up to the first zero byte. + Anything shorter than `MIN_CHARS`, or holding a byte outside printable + ASCII, is audio rather than words. + """ + head, _, _ = frame.partition(b"\x00") + if len(head) < MIN_CHARS or any(b < 0x20 or b > 0x7E for b in head): + return "" + return head.decode("ascii").strip() + + +def read_back(pcm: bytes) -> str: + """The words the mock voice wrote into `pcm`, in order. + + The voice writes each word and then at least two zero bytes, so a word + is a run of printable bytes between runs of zeros, whatever its length: + "is" and "it" are words here, where `spelled()` would take them for + audio, because this reads the mock's own output and nothing else. Works + whatever the chunk size, so a wav the sink wrote can be read back + without knowing how it was cut. + """ + words: list[str] = [] + for run in re.split(rb"\x00{2,}", pcm): + run = run.strip(b"\x00") + if run and all(0x20 <= b <= 0x7E for b in run): + words.append(run.decode("ascii").strip()) + return " ".join(w for w in words if w) + + +class MockTranscriber(TranscriberPlugin): + """Speech recognition with no speech, no recognition and no network.""" + + provider = "mock" + + def __init__(self, config: Mapping[str, Any], fmt: AudioFormat) -> None: + super().__init__(config, fmt) + #: Frames fed, across every utterance. Evidence the engine is feeding. + self.frames = 0 + #: Utterances closed with `finish()`. + self.finished = 0 + self._words: list[str] = [] + self._started = False + self._fault: str | None = None + + # ------------------------------------------------------------ lifecycle + + async def start(self) -> None: + if self.params.get("fail_on_start"): + self._fail(str(self.params.get("fault_detail") or "simulated: the service is unreachable")) + return + if self.params.get("require_key"): + if not self.key_env: + self._fail("this provider needs a key and the soul names no key_env") + return + if not os.environ.get(self.key_env): + self._fail(f"no key: the environment variable {self.key_env} is not set") + return + self._started = True + + def _fail(self, reason: str) -> None: + self._fault = reason + log.warning("mock stt: %s", reason) + + async def shutdown(self) -> None: + self._started = False + self._words = [] + + # ------------------------------------------------------------ reporting + + def describe(self) -> TranscriberDescriptor: + return TranscriberDescriptor( + provider=self.provider, + model=self.model, + streaming=True, + sample_rate=int(self.params.get("sample_rate") or self.format.sample_rate), + healthy=self._started, + ) + + def health(self) -> Health: + if self._fault: + return Health(ok=False, detail=self._fault, faults=("start_failed",)) + return Health() + + # ---------------------------------------------------------- recognition + + @property + def _script(self) -> list[str]: + scripted = self.params.get("transcript") + return str(scripted).split() if scripted is not None else [] + + async def feed(self, frame: bytes) -> Transcript | None: + self.frames += 1 + if not self._started: + return None + script = self._script + if script: + # One word a frame, so the partials grow the way a vendor's do. + if len(self._words) >= len(script): + return None + self._words.append(script[len(self._words)]) + else: + words = spelled(frame) + if not words: + return None + self._words.append(words) + return Transcript(text=" ".join(self._words), final=False, confidence=0.5) + + async def finish(self) -> Transcript: + self.finished += 1 + script = self._script + text = " ".join(script) if script else " ".join(self._words) + self._words = [] + return Transcript(text=text, final=True, confidence=1.0) + + +def _words(text: str) -> list[str]: + return text.split() + + +class MockLanguageModel(LanguageModelPlugin): + """A language model with no model in it.""" + + provider = "mock" + + def __init__(self, config: Mapping[str, Any]) -> None: + super().__init__(config) + #: Every prompt this instance was asked to answer, in order. + self.prompts: list[Prompt] = [] + self._started = False + self._fault: str | None = None + + # ------------------------------------------------------------ lifecycle + + async def start(self) -> None: + if self.params.get("fail_on_start"): + self._fail(str(self.params.get("fault_detail") or "simulated: the service is unreachable")) + return + if self.params.get("require_key"): + if not self.key_env: + self._fail("this provider needs a key and the soul names no key_env") + return + if not os.environ.get(self.key_env): + self._fail(f"no key: the environment variable {self.key_env} is not set") + return + self._started = True + + def _fail(self, reason: str) -> None: + self._fault = reason + log.warning("mock llm: %s", reason) + + async def shutdown(self) -> None: + self._started = False + + # ------------------------------------------------------------ reporting + + def describe(self) -> LanguageModelDescriptor: + return LanguageModelDescriptor( + provider=self.provider, + model=self.model, + streaming=True, + tools=True, + healthy=self._started, + ) + + def health(self) -> Health: + if self._fault: + fault = "no_key" if "no key" in self._fault else "start_failed" + return Health(ok=False, detail=self._fault, faults=(fault,)) + return Health() + + # ---------------------------------------------------------------- reply + + async def reply(self, prompt: Prompt) -> AsyncIterator[ReplyEvent]: + self.prompts.append(prompt) + if not self._started: + yield ReplyDone(text="", stop_reason="error", error="the mock language model was not started") + return + + last = prompt.messages[-1] if prompt.messages else None + input_tokens = len(_words(prompt.system)) + sum(len(_words(m.content)) for m in prompt.messages) + + wanted = self.params.get("call_tool") + if wanted and prompt.tools and last is not None and last.role == "user": + call = ToolCall( + id=f"call_{len(self.prompts)}", + name=str((wanted or {}).get("name") or prompt.tools[0].name), + arguments=dict((wanted or {}).get("arguments") or {}), + ) + yield call + yield ReplyDone(text="", stop_reason="tool", tool_calls=(call,), model="mock", input_tokens=input_tokens, output_tokens=0) + return + + if last is not None and last.role == "tool": + text = f"The tool said: {last.content}" + elif self.params.get("reply") is not None: + text = str(self.params["reply"]) + else: + heard = last.content if last is not None else "" + text = f"You said: {heard}" if heard else "You said nothing." + + for i, word in enumerate(_words(text)): + yield TextDelta(word if i == 0 else f" {word}") + yield ReplyDone( + text=" ".join(_words(text)), + stop_reason="end", + model="mock", + input_tokens=input_tokens, + output_tokens=len(_words(text)), + ) + + +class MockVoice(VoicePlugin): + """A voice with no voice in it: the words become the audio.""" + + provider = "mock" + + #: Bytes a word's chunk is padded to. Short, so a sentence is small. + CHUNK_BYTES = 64 + + def __init__(self, config: Mapping[str, Any], voice: Mapping[str, Any] | None = None) -> None: + super().__init__(config, voice) + #: Every sentence this instance was asked to say, in order. + self.said: list[str] = [] + self._started = False + self._fault: str | None = None + + # ------------------------------------------------------------ lifecycle + + async def start(self) -> None: + if self.params.get("fail_on_start"): + self._fail(str(self.params.get("fault_detail") or "simulated: the voice model is missing")) + return + if self.params.get("require_key"): + if not self.key_env: + self._fail("this voice needs a key and the soul names no key_env") + return + if not os.environ.get(self.key_env): + self._fail(f"no key: the environment variable {self.key_env} is not set") + return + self._started = True + + def _fail(self, reason: str) -> None: + self._fault = reason + log.warning("mock tts: %s", reason) + + async def shutdown(self) -> None: + self._started = False + + # ------------------------------------------------------------ reporting + + def describe(self) -> VoiceDescriptor: + return VoiceDescriptor( + provider=self.provider, + model=self.model, + sample_rate=int(self.params.get("sample_rate") or 16000), + streaming=True, + healthy=self._started, + ) + + def health(self) -> Health: + if self._fault: + fault = "no_key" if "no key" in self._fault else "start_failed" + return Health(ok=False, detail=self._fault, faults=(fault,)) + return Health() + + # ---------------------------------------------------------------- speak + + def _chunk(self, word: str) -> bytes: + payload = word.encode("ascii", "replace") + size = max(self.CHUNK_BYTES, len(payload) + 2) + size += size % 2 + return payload + bytes(size - len(payload)) + + async def speak(self, text: str) -> AsyncIterator[bytes]: + words = text.split() + if not words: + return + if not self._started: + raise PluginError("the mock voice was not started") + self.said.append(" ".join(words)) + latency = float(self.params.get("latency_ms") or 0) + if latency: + await asyncio.sleep(latency / 1000.0) + poison = self.params.get("fail_on") + for i, word in enumerate(words): + yield self._chunk(word) + if poison and poison in word: + raise PluginError(f"simulated: the voice failed on {word!r}") diff --git a/emet-providers/emet_providers/openai.py b/emet-providers/emet_providers/openai.py new file mode 100644 index 0000000..403ed89 --- /dev/null +++ b/emet-providers/emet_providers/openai.py @@ -0,0 +1,317 @@ +"""The language model through OpenAI's Chat Completions API. + +The second vendor behind the `emet.llm` seam, shipped beside Anthropic so that +the seam is proven by two implementations. Nothing above knows which one is +running. + +**Why Chat Completions and not the newer Responses API.** Chat Completions is +the shape every other server speaks too: a local model behind Ollama, vLLM or +LM Studio, a gateway such as OpenRouter, a company's own proxy. `params.url` +points this plugin at any of them, which is how the offline mode the design +reserves for later arrives as a manifest line rather than a rewrite. + +**Protocol, current as of 2026-09-15.** `POST {url}/chat/completions` with +`Authorization: Bearer ` and `stream: true`. Each Server-Sent Event is +one `chat.completion.chunk`: `choices[0].delta.content` carries text, +`choices[0].delta.tool_calls[]` carries a call's `id`, `type` and +`function.name` in its first chunk and `function.arguments` in pieces after, +keyed by `index`; `choices[0].finish_reason` closes the choice (`stop`, +`length`, `tool_calls`, `content_filter`); with `stream_options: +{"include_usage": true}` a final chunk with empty `choices` carries `usage`; +the stream ends with `data: [DONE]`. `GET {url}/models` lists what the key may +use. The output cap is `max_completion_tokens`; servers that predate the +rename take `max_tokens`, and `params.max_tokens_field` says which. + +Params, all optional, from the body's `models.chat.params`: + + url the API root. Default https://api.openai.com/v1; any + compatible server works. + timeout_s seconds per request, connect and read. Default 60. + preflight bool, default true. `GET /models` in `start()` to check + the key and the network. + max_tokens_field "max_completion_tokens" (default) or "max_tokens". + organization sent as `OpenAI-Organization` when set. + +Dependencies: `httpx` (BSD-3-Clause), through the `openai` extra. Not the +vendor SDK, for the reason given in `anthropic.py`. +""" + +from __future__ import annotations + +import json +import logging +import os +from typing import Any, AsyncIterator, Mapping, Sequence + +from emet_sdk.plugin import LanguageModelPlugin +from emet_sdk.types import ( + Health, + LanguageModelDescriptor, + Message, + Prompt, + ReplyDone, + ReplyEvent, + TextDelta, + ToolCall, + ToolSpec, +) + +from emet_providers._http import api_error, open_client, sse_events + +__all__ = ["OpenAILanguageModel", "DEFAULT_MODEL", "DEFAULT_KEY_ENV", "DEFAULT_URL"] + +log = logging.getLogger("emet_providers.openai") + +DEFAULT_URL = "https://api.openai.com/v1" + +#: The model OpenAI's list describes as balancing intelligence and cost, on +#: 2026-09-15. A speaking robot wants the answer soon more than it wants the +#: largest model; the soul names another when it disagrees. +DEFAULT_MODEL = "gpt-5.6-terra" + +DEFAULT_KEY_ENV = "EMET_OPENAI_KEY" + +#: The vendor's finish reasons, in the seam's words. +FINISH_REASONS: Mapping[str, str] = { + "stop": "end", + "length": "length", + "tool_calls": "tool", + "function_call": "tool", + "content_filter": "refusal", +} + + +def _messages(system: str, messages: Sequence[Message]) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + if system: + out.append({"role": "system", "content": system}) + for m in messages: + if m.role == "user": + out.append({"role": "user", "content": m.content}) + elif m.role == "assistant": + entry: dict[str, Any] = {"role": "assistant", "content": m.content or None} + if m.tool_calls: + entry["tool_calls"] = [ + { + "id": c.id, + "type": "function", + "function": {"name": c.name, "arguments": json.dumps(dict(c.arguments))}, + } + for c in m.tool_calls + ] + out.append(entry) + else: # tool + out.append({"role": "tool", "tool_call_id": m.tool_call_id, "content": m.content}) + return out + + +def _tools(tools: Sequence[ToolSpec]) -> list[dict[str, Any]]: + return [ + { + "type": "function", + "function": {"name": t.name, "description": t.description, "parameters": dict(t.parameters)}, + } + for t in tools + ] + + +class OpenAILanguageModel(LanguageModelPlugin): + """GPT, or anything that speaks Chat Completions, streamed.""" + + provider = "openai" + + def __init__(self, config: Mapping[str, Any], *, transport: Any = None) -> None: + super().__init__(config) + self._transport = transport + self._client: Any = None + self._key: str | None = None + self._started = False + self._fault: str | None = None + self.last_error: str | None = None + + self.model_name: str = self.model or DEFAULT_MODEL + self.key_env_name: str = self.key_env or DEFAULT_KEY_ENV + self.url: str = str(self.params.get("url") or DEFAULT_URL).rstrip("/") + self.timeout_s = float(self.params.get("timeout_s", 60.0)) + self.preflight = bool(self.params.get("preflight", True)) + self.max_tokens_field: str = str(self.params.get("max_tokens_field") or "max_completion_tokens") + self.organization: str | None = self.params.get("organization") or None + + # ------------------------------------------------------------- request + + def headers(self) -> dict[str, str]: + headers = {"Authorization": f"Bearer {self._key or ''}", "content-type": "application/json"} + if self.organization: + headers["OpenAI-Organization"] = self.organization + return headers + + def body(self, prompt: Prompt) -> dict[str, Any]: + body: dict[str, Any] = { + "model": self.model_name, + "messages": _messages(prompt.system, prompt.messages), + "stream": True, + "stream_options": {"include_usage": True}, + self.max_tokens_field: prompt.max_tokens, + } + if prompt.tools: + body["tools"] = _tools(prompt.tools) + return body + + # ------------------------------------------------------------ lifecycle + + async def start(self) -> None: + self._key = os.environ.get(self.key_env_name) or None + if not self._key: + self._fail( + f"no key: the environment variable {self.key_env_name} is not set. " + f"OpenAI keys come from platform.openai.com; export it in the shell " + f"that runs the robot. The key is never written into a soul or a " + f"manifest." + ) + return + client = open_client(self.url, timeout_s=self.timeout_s, transport=self._transport, extra="openai") + if isinstance(client, str): + self._fail(client) + return + self._client = client + if self.preflight: + reason = await self._preflight() + if reason: + self._fail(reason) + return + self._started = True + log.info("openai: ready, model %s at %s", self.model_name, self.url) + + async def _preflight(self) -> str | None: + try: + response = await self._client.get("/models", headers=self.headers()) + except Exception as exc: # noqa: BLE001 + return ( + f"could not reach {self.url}: {type(exc).__name__}: {exc}. " + f"The language model needs a network; the wake word does not." + ) + if response.status_code == 200: + return None + if response.status_code in (401, 403): + return ( + f"the server at {self.url} rejected the key in {self.key_env_name} " + f"(HTTP {response.status_code}). Check the key." + ) + return f"the server at {self.url} refused the connection: {api_error(response.status_code, response.content)}" + + def _fail(self, reason: str) -> None: + self._fault = reason + log.error("openai: %s", reason) + + async def shutdown(self) -> None: + self._started = False + if self._client is not None: + try: + await self._client.aclose() + except Exception: # noqa: BLE001 + pass + self._client = None + + # ------------------------------------------------------------ reporting + + def describe(self) -> LanguageModelDescriptor: + return LanguageModelDescriptor( + provider=self.provider, + model=self.model_name, + streaming=True, + tools=True, + healthy=self._started, + ) + + def health(self) -> Health: + if self._fault: + fault = "no_key" if "no key" in self._fault else "start_failed" + return Health(ok=False, detail=self._fault, faults=(fault,)) + return Health() + + # ---------------------------------------------------------------- reply + + async def reply(self, prompt: Prompt) -> AsyncIterator[ReplyEvent]: + if not self._started or self._client is None: + yield ReplyDone(text="", stop_reason="error", error="the openai plugin was not started") + return + + text: list[str] = [] + pending: dict[int, dict[str, str]] = {} + stop = "end" + model: str | None = None + input_tokens: int | None = None + output_tokens: int | None = None + error: str | None = None + + try: + async with self._client.stream( + "POST", "/chat/completions", headers=self.headers(), json=self.body(prompt) + ) as response: + if response.status_code != 200: + error = api_error(response.status_code, await response.aread()) + else: + async for _event, data in sse_events(response.aiter_lines()): + if not isinstance(data, dict): + continue + if data.get("raw") == "[DONE]": + break + if "error" in data and "choices" not in data: + err = data.get("error") or {} + error = f"{err.get('type', 'error')}: {err.get('message', '')}" + break + model = data.get("model") or model + usage = data.get("usage") + if isinstance(usage, dict): + if isinstance(usage.get("prompt_tokens"), int): + input_tokens = usage["prompt_tokens"] + if isinstance(usage.get("completion_tokens"), int): + output_tokens = usage["completion_tokens"] + choices = data.get("choices") or [] + if not choices: + continue + choice = choices[0] + delta = choice.get("delta") or {} + piece = delta.get("content") + if isinstance(piece, str) and piece: + text.append(piece) + yield TextDelta(piece) + for call in delta.get("tool_calls") or []: + index = int(call.get("index", 0)) + entry = pending.setdefault(index, {"id": "", "name": "", "arguments": ""}) + if call.get("id"): + entry["id"] = str(call["id"]) + function = call.get("function") or {} + if function.get("name"): + entry["name"] = str(function["name"]) + entry["arguments"] += str(function.get("arguments") or "") + reason = choice.get("finish_reason") + if reason: + stop = FINISH_REASONS.get(str(reason), "end") + except Exception as exc: # noqa: BLE001 - the reply ends with what was heard + error = f"{type(exc).__name__}: {exc}" + + calls: list[ToolCall] = [] + for index in sorted(pending): + entry = pending[index] + try: + arguments = json.loads(entry["arguments"]) if entry["arguments"] else {} + except ValueError: + arguments = {} + call = ToolCall(id=entry["id"] or f"call_{index}", name=entry["name"], arguments=arguments) + calls.append(call) + yield call + + if error: + stop = "error" + self.last_error = error + log.warning("openai: %s", error) + yield ReplyDone( + text="".join(text), + stop_reason=stop, + tool_calls=tuple(calls), + model=model or self.model_name, + input_tokens=input_tokens, + output_tokens=output_tokens, + error=error, + ) diff --git a/emet-providers/emet_providers/piper.py b/emet-providers/emet_providers/piper.py new file mode 100644 index 0000000..d5af197 --- /dev/null +++ b/emet-providers/emet_providers/piper.py @@ -0,0 +1,247 @@ +"""The local voice, through Piper. + +The design keeps synthesis local by default (`DESIGN.md` section 14): it is +the stage that costs the most per turn in the cloud, and the one a robot in a +home should manage with the network down. Piper is a small neural +text-to-speech engine that runs a voice model through ONNX Runtime on a CPU, +fast enough on a Raspberry Pi 5 to answer before a person notices the wait. + +**Packaging, verified against PyPI on 2026-09-15.** `piper-tts` 1.8.0 +(2026-09-04) ships `cp39-abi3` wheels for manylinux aarch64 and x86_64, macOS +and Windows, so one wheel serves Python 3.9 through 3.13 on the Pi with no +compiler. It depends on `onnxruntime` (MIT, 1.30.0 ships cp313 aarch64 +wheels) and `pathvalidate`; espeak-ng, the phonemiser, is compiled into the +wheel. That is also why the licence is **GPL-3.0-or-later**: the project +moved from the MIT-licensed `rhasspy/piper` to `OHF-Voice/piper1-gpl` when it +took espeak-ng in. Emet does not copy or bundle Piper; this file imports it +through the optional extra `emet-providers[piper]`, and a body without the +extra never loads it. `CITATIONS.md` records the licence. + +**Voices are files the owner downloads once.** A voice is a `.onnx` model +and a `.onnx.json` config beside it, from Hugging Face, and the plugin will +not fetch one at boot: a robot that downloaded sixty megabytes because a +soul named a voice would be doing something nobody asked for. It looks in +`params.path`, then `params.voices_dir`, then `$XDG_DATA_HOME/emet/voices` +(`~/.local/share/emet/voices`), then `/etc/emet/voices`, and when it finds +nothing it names the command that fixes it. Each voice carries its own +licence in a `MODEL_CARD` beside the model on Hugging Face; the reference +soul names `en_US-ljspeech-medium` because LJ Speech is public domain. + +**Streaming.** Piper synthesises a sentence at a time and the engine hands it +one sentence at a time, so `speak()` yields one chunk per sentence Piper +finds in the text, after each is complete. `describe().streaming` is +therefore False, honestly: the first byte arrives when the sentence does. +Synthesis is CPU work and runs in a thread, so the event loop keeps reading +the microphone meanwhile. + +**Warm at boot.** Piper loads espeak-ng on the first sentence it is given, +which on a laptop cost the first reply 3.7 s more than the second (measured +2026-09-15: 3844 ms for the first sentence, 155 ms for the next). So +`start()` says one word into the void after loading the model, and the cost +lands at boot, where a person expects to wait, rather than on the first +answer, where they do not. `params.warm: false` skips it. + +Params, all optional, from the body's `models.tts.params`: + + path the `.onnx` file itself, config beside it as `.onnx.json`. + voices_dir a directory holding `.onnx` and `.onnx.json`. + noise_scale, noise_w_scale, volume + Piper's own synthesis knobs, passed through. The soul's + `voice.rate` becomes `length_scale` (its inverse: Piper + counts phoneme length, a soul counts speed). + use_cuda bool, default false. A GPU, where there is one. + warm bool, default true. Synthesise one word at boot so the + phonemiser is loaded before the first reply. + +Dependencies: `piper-tts` (GPL-3.0-or-later) and through it `onnxruntime` +(MIT), both through the `piper` extra. +""" + +from __future__ import annotations + +import asyncio +import logging +import os +from pathlib import Path +from typing import Any, AsyncIterator, Mapping + +from emet_sdk.plugin import PluginError, VoicePlugin +from emet_sdk.types import Health, VoiceDescriptor + +__all__ = ["PiperVoice", "DEFAULT_MODEL", "SYSTEM_VOICES", "WARM_TEXT", "user_voices_dir", "voice_paths", "download_hint"] + +log = logging.getLogger("emet_providers.piper") + +#: The voice a soul gets when it says only `provider: piper`. LJ Speech is +#: public domain (the dataset's own terms, read 2026-09-15), which is what a +#: shipped default has to be; several other voices in the same catalogue are +#: fine-tuned from a corpus licensed for research only. +DEFAULT_MODEL = "en_US-ljspeech-medium" + +#: The machine's voices, beside the machine's keys. +SYSTEM_VOICES = Path("/etc/emet/voices") + +#: Said at boot to load the phonemiser. Never played. +WARM_TEXT = "Ready." + + +def user_voices_dir() -> Path: + """A person's voices: `$XDG_DATA_HOME/emet/voices`, or `~/.local/share/emet/voices`.""" + base = os.environ.get("XDG_DATA_HOME") + root = Path(base) if base else Path.home() / ".local" / "share" + return root / "emet" / "voices" + + +def voice_paths(model: str, params: Mapping[str, Any]) -> list[Path]: + """Where `.onnx` is looked for, in order. `params.path` names the + file itself and wins outright.""" + explicit = params.get("path") + if explicit: + return [Path(str(explicit)).expanduser()] + dirs: list[Path] = [] + if params.get("voices_dir"): + dirs.append(Path(str(params["voices_dir"])).expanduser()) + dirs.extend([user_voices_dir(), SYSTEM_VOICES]) + return [d / f"{model}.onnx" for d in dirs] + + +def download_hint(model: str) -> str: + """The one command that fetches a voice into the place this plugin looks.""" + return ( + f"python -m piper.download_voices {model} --data-dir {user_voices_dir()}" + ) + + +class PiperVoice(VoicePlugin): + """Local synthesis through Piper, one sentence at a time.""" + + provider = "piper" + + def __init__(self, config: Mapping[str, Any], voice: Mapping[str, Any] | None = None) -> None: + super().__init__(config, voice) + self.model_name: str = self.model or DEFAULT_MODEL + self._voice: Any = None + self._synthesis_config: Any = None + self._sample_rate: int = 22050 + self._started = False + self._fault: str | None = None + self._fault_kind: str = "start_failed" + #: Where the model was found, once it was. + self.path: Path | None = None + #: The last failure inside a sentence, for diagnostics. Not a fault. + self.last_error: str | None = None + + # ------------------------------------------------------------ lifecycle + + async def start(self) -> None: + try: + from piper import PiperVoice as _Loader, SynthesisConfig + except ImportError: + self._fail( + "the piper-tts library is not installed. It is an optional " + "dependency: install `emet-providers[piper]`. Piper is GPL-3.0; " + "Emet imports it and does not bundle it.", + kind="no_library", + ) + return + + candidates = voice_paths(self.model_name, self.params) + found = next((p for p in candidates if p.is_file()), None) + if found is None: + looked = ", ".join(str(p) for p in candidates) + self._fail( + f"no voice model for {self.model_name!r}. Looked for {looked}. " + f"Download it once with:\n {download_hint(self.model_name)}\n" + f"and the plugin finds it there on the next boot. A voice model is " + f"never fetched at boot, so that a soul naming one cannot make the " + f"robot download anything unasked.", + kind="no_model", + ) + return + config_path = found.with_name(found.name + ".json") + if not config_path.is_file(): + self._fail( + f"{found} has no config beside it: expected {config_path}. Piper " + f"voices are two files, and the download command fetches both:\n" + f" {download_hint(self.model_name)}", + kind="no_model", + ) + return + + try: + # Loading the ONNX session takes a second on a Pi; keep the loop + # free rather than stall the microphone during boot. + self._voice = await asyncio.to_thread( + _Loader.load, str(found), str(config_path), bool(self.params.get("use_cuda", False)) + ) + except Exception as exc: # noqa: BLE001 - reported through health, not raised + self._fail(f"piper could not load {found}: {type(exc).__name__}: {exc}") + return + + knobs: dict[str, Any] = {"length_scale": 1.0 / self.rate} + for name in ("noise_scale", "noise_w_scale", "volume"): + if self.params.get(name) is not None: + knobs[name] = float(self.params[name]) + self._synthesis_config = SynthesisConfig(**knobs) + self._sample_rate = int(self._voice.config.sample_rate) + self.path = found + + if self.params.get("warm", True): + voice, config = self._voice, self._synthesis_config + try: + await asyncio.to_thread(lambda: list(voice.synthesize(WARM_TEXT, config))) + except Exception as exc: # noqa: BLE001 - a voice that cannot say one word cannot say any + self._fail(f"piper loaded {found} and could not synthesise: {type(exc).__name__}: {exc}") + return + + self._started = True + log.info("piper: ready, voice %s at %d Hz from %s", self.model_name, self._sample_rate, found) + + def _fail(self, reason: str, *, kind: str = "start_failed") -> None: + self._fault = reason + self._fault_kind = kind + log.error("piper: %s", reason) + + async def shutdown(self) -> None: + self._started = False + self._voice = None + + # ------------------------------------------------------------ reporting + + def describe(self) -> VoiceDescriptor: + return VoiceDescriptor( + provider=self.provider, + model=self.model_name, + sample_rate=self._sample_rate, + streaming=False, + healthy=self._started, + ) + + def health(self) -> Health: + if self._fault: + return Health(ok=False, detail=self._fault, faults=(self._fault_kind,)) + return Health() + + # ---------------------------------------------------------------- speak + + async def speak(self, text: str) -> AsyncIterator[bytes]: + if not text.strip(): + return + if not self._started or self._voice is None: + raise PluginError("the piper voice was not started") + + def synthesise() -> list[bytes]: + return [ + chunk.audio_int16_bytes + for chunk in self._voice.synthesize(text, self._synthesis_config) + ] + + try: + chunks = await asyncio.to_thread(synthesise) + except Exception as exc: # noqa: BLE001 - one sentence lost, reported + self.last_error = f"{type(exc).__name__}: {exc}" + log.warning("piper: %s", self.last_error) + raise PluginError(f"piper could not say {text!r}: {self.last_error}") from exc + for chunk in chunks: + if chunk: + yield chunk diff --git a/emet-providers/pyproject.toml b/emet-providers/pyproject.toml new file mode 100644 index 0000000..74fd8d4 --- /dev/null +++ b/emet-providers/pyproject.toml @@ -0,0 +1,66 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "emet-providers" +version = "0.3.0" +description = "Emet providers: speech recognition, language model and voice plugins that reach a service, or a model on disk." +readme = "README.md" +requires-python = ">=3.11" +license = "Apache-2.0" +authors = [{ name = "The Emet Authors" }] +keywords = ["emet", "robotics", "stt", "llm", "tts", "providers"] + +# emet-sdk only, like emet-hal. A provider is a plugin that talks to a service +# instead of a chip; the contract it satisfies is the same kind of thing and +# lives in the same place. The engine never imports this package either: a +# transcriber reaches it by name through the `emet.stt` entry-point group. +dependencies = ["emet-sdk>=0.1"] + +[project.optional-dependencies] +# The Deepgram transcriber speaks the wire protocol itself and needs only a +# WebSocket client; the Deepgram voice streams audio over HTTPS through the +# same client the language models use. Optional because a body with no key, +# or no network, has no use for either. websockets is BSD-3-Clause and ships +# cp311 to cp313 wheels for aarch64 Linux (PyPI, 2026-09-12); 13 is where its +# asyncio client took `additional_headers`. +deepgram = ["websockets>=13", "httpx>=0.27"] +# The two language model providers speak JSON over HTTPS and stream +# Server-Sent Events, so one HTTP client serves both. httpx is BSD-3-Clause, +# pure Python, and ships one wheel for every platform (PyPI, 2026-09-15). +anthropic = ["httpx>=0.27"] +openai = ["httpx>=0.27"] +# The local voice. piper-tts 1.8.0 ships cp39-abi3 wheels for manylinux +# aarch64 and x86_64, macOS and Windows, one wheel for Python 3.9 to 3.13, +# and depends on onnxruntime (MIT, cp313 aarch64 wheels) and pathvalidate +# (PyPI, 2026-09-15). It is GPL-3.0-or-later, because espeak-ng is compiled +# into it: Emet imports it through this extra and never bundles it, so a body +# without the extra carries no GPL code. See CITATIONS.md. +piper = ["piper-tts>=1.8"] +dev = ["pytest>=8.0"] + +# Installing a package is what makes a provider exist. The entry-point name is +# the string a soul puts in `models.stt.provider`, or a body puts in its own +# `models.stt.provider` to take the choice over. +[project.entry-points."emet.stt"] +"mock" = "emet_providers.mock:MockTranscriber" +"deepgram" = "emet_providers.deepgram:DeepgramTranscriber" + +# The entry-point name is the string a soul puts in `models.chat.provider`. +[project.entry-points."emet.llm"] +"mock" = "emet_providers.mock:MockLanguageModel" +"anthropic" = "emet_providers.anthropic:AnthropicLanguageModel" +"openai" = "emet_providers.openai:OpenAILanguageModel" + +# The entry-point name is the string a soul puts in `models.tts.provider`. +[project.entry-points."emet.tts"] +"mock" = "emet_providers.mock:MockVoice" +"piper" = "emet_providers.piper:PiperVoice" +"deepgram" = "emet_providers.deepgram_voice:DeepgramVoice" + +[tool.hatch.build.targets.wheel] +packages = ["emet_providers"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/emet-providers/tests/test_anthropic.py b/emet-providers/tests/test_anthropic.py new file mode 100644 index 0000000..aaae994 --- /dev/null +++ b/emet-providers/tests/test_anthropic.py @@ -0,0 +1,423 @@ +"""The Anthropic language model, against a stand-in for the Messages API. + +No network and no key. An `httpx.MockTransport` answers every request the +way the API would, in the shapes Anthropic's reference gives (verified +2026-09-15), and records what it was sent. What these tests prove is the +plugin's half of the protocol: headers, body, the mapping of the seam's +messages and tools into the vendor's, and how a streamed reply is assembled +back into text, tool calls and a stop reason. The live test beside this file +proves the other half, when a key is present. +""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any + +import pytest + +httpx = pytest.importorskip("httpx", reason="emet-providers[anthropic] is not installed") + +from emet_sdk.plugin import LanguageModelPlugin # noqa: E402 +from emet_sdk.types import Message, Prompt, ReplyDone, TextDelta, ToolCall, ToolSpec # noqa: E402 + +from emet_providers.anthropic import ( # noqa: E402 + API_VERSION, + DEFAULT_KEY_ENV, + DEFAULT_MODEL, + DEFAULT_URL, + FALLBACK_BETA_ARRAY, + FALLBACK_BETA_DEFAULT, + AnthropicLanguageModel, +) + +KEY = "sk-ant-test" + + +def run(coro): + return asyncio.run(coro) + + +async def collect(events) -> list: + return [e async for e in events] + + +# -------------------------------------------------------------- the fake + + +def sse(*events: tuple[str, dict]) -> bytes: + return "".join(f"event: {name}\ndata: {json.dumps(data)}\n\n" for name, data in events).encode() + + +def message_stream( + *pieces: str, + stop: str = "end_turn", + tool: tuple[str, dict] | None = None, + model: str = "claude-opus-5", + close: bool = True, +) -> bytes: + """A whole streamed message in the documented event sequence.""" + events: list[tuple[str, dict]] = [ + ( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": model, + "content": [], + "usage": {"input_tokens": 12, "output_tokens": 1}, + }, + }, + ), + ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}), + ] + for piece in pieces: + events.append( + ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": piece}}) + ) + events.append(("content_block_stop", {"type": "content_block_stop", "index": 0})) + if tool is not None: + name, arguments = tool + encoded = json.dumps(arguments) + events += [ + ( + "content_block_start", + {"type": "content_block_start", "index": 1, "content_block": {"type": "tool_use", "id": "toolu_1", "name": name, "input": {}}}, + ), + ("content_block_delta", {"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": encoded[:6]}}), + ("content_block_delta", {"type": "content_block_delta", "index": 1, "delta": {"type": "input_json_delta", "partial_json": encoded[6:]}}), + ("content_block_stop", {"type": "content_block_stop", "index": 1}), + ] + if close: + events += [ + ("message_delta", {"type": "message_delta", "delta": {"stop_reason": stop, "stop_sequence": None}, "usage": {"output_tokens": 7}}), + ("message_stop", {"type": "message_stop"}), + ] + return sse(*events) + + +class FakeAnthropic: + """Answers `GET /v1/models` and `POST /v1/messages`, recording each request.""" + + def __init__( + self, + stream: bytes = b"", + *, + models_status: int = 200, + reply_status: int = 200, + reply_json: dict | None = None, + raise_on_connect: Exception | None = None, + ) -> None: + self.stream = stream + self.models_status = models_status + self.reply_status = reply_status + self.reply_json = reply_json + self.raise_on_connect = raise_on_connect + self.requests: list[httpx.Request] = [] + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + if self.raise_on_connect is not None: + raise self.raise_on_connect + if request.method == "GET" and request.url.path.endswith("/v1/models"): + if self.models_status == 200: + return httpx.Response(200, json={"data": [{"id": "claude-opus-5"}]}) + return httpx.Response( + self.models_status, + json={"type": "error", "error": {"type": "authentication_error", "message": "invalid x-api-key"}}, + ) + if request.method == "POST" and request.url.path.endswith("/v1/messages"): + if self.reply_status != 200: + return httpx.Response(self.reply_status, json=self.reply_json or {"type": "error", "error": {"type": "api_error", "message": "boom"}}) + return httpx.Response(200, content=self.stream, headers={"content-type": "text/event-stream"}) + return httpx.Response(404, json={"error": {"type": "not_found_error", "message": request.url.path}}) + + @property + def posts(self) -> list[httpx.Request]: + return [r for r in self.requests if r.method == "POST"] + + +def make(fake: FakeAnthropic, *, config: dict | None = None, **params: Any) -> AnthropicLanguageModel: + cfg: dict[str, Any] = {"provider": "anthropic", "params": params} + cfg.update(config or {}) + return AnthropicLanguageModel(cfg, transport=httpx.MockTransport(fake)) + + +def started(fake: FakeAnthropic, monkeypatch, **kw) -> AnthropicLanguageModel: + monkeypatch.setenv(DEFAULT_KEY_ENV, KEY) + llm = make(fake, **kw) + run(llm.start()) + assert llm.describe().healthy, llm.health().detail + return llm + + +def body_of(request: httpx.Request) -> dict: + return json.loads(request.content) + + +def prompt(*texts: str, system: str = "Be brief.", tools: tuple = ()) -> Prompt: + messages = tuple( + Message(role="user" if i % 2 == 0 else "assistant", content=t) for i, t in enumerate(texts) + ) + return Prompt(system=system, messages=messages, tools=tools, max_tokens=300) + + +# --------------------------------------------------------------- request + + +def test_it_is_a_language_model_plugin_registered_as_anthropic(): + llm = make(FakeAnthropic()) + assert isinstance(llm, LanguageModelPlugin) + assert llm.provider == "anthropic" + assert llm.url == DEFAULT_URL + + +def test_the_request_carries_the_documented_headers_and_no_sampling_knobs(monkeypatch): + fake = FakeAnthropic(message_stream("hi")) + llm = started(fake, monkeypatch) + run(collect(llm.reply(prompt("hello")))) + (post,) = fake.posts + assert post.headers["x-api-key"] == KEY + assert post.headers["anthropic-version"] == API_VERSION + assert "anthropic-beta" not in post.headers + body = body_of(post) + assert body["model"] == DEFAULT_MODEL + assert body["max_tokens"] == 300 + assert body["stream"] is True + assert body["system"] == "Be brief." + assert body["messages"] == [{"role": "user", "content": "hello"}] + for forbidden in ("temperature", "top_p", "top_k", "thinking", "tools", "output_config", "fallbacks"): + assert forbidden not in body, forbidden + + +def test_the_souls_model_and_the_bodys_effort_are_sent(monkeypatch): + fake = FakeAnthropic(message_stream("hi")) + llm = started(fake, monkeypatch, config={"model": "claude-sonnet-5"}, effort="low") + run(collect(llm.reply(prompt("hello")))) + body = body_of(fake.posts[0]) + assert body["model"] == "claude-sonnet-5" + assert body["output_config"] == {"effort": "low"} + assert llm.describe().model == "claude-sonnet-5" + + +def test_fallbacks_are_off_unless_asked_and_carry_the_matching_beta_header(monkeypatch): + fake = FakeAnthropic(message_stream("hi")) + llm = started(fake, monkeypatch, fallbacks="default") + run(collect(llm.reply(prompt("hello")))) + post = fake.posts[0] + assert post.headers["anthropic-beta"] == FALLBACK_BETA_DEFAULT + assert body_of(post)["fallbacks"] == "default" + + fake = FakeAnthropic(message_stream("hi")) + llm = started(fake, monkeypatch, fallbacks=["claude-opus-4-8"]) + run(collect(llm.reply(prompt("hello")))) + post = fake.posts[0] + assert post.headers["anthropic-beta"] == FALLBACK_BETA_ARRAY + assert body_of(post)["fallbacks"] == [{"model": "claude-opus-4-8"}] + + +def test_tools_and_tool_traffic_take_the_vendors_shape(monkeypatch): + fake = FakeAnthropic(message_stream("done")) + llm = started(fake, monkeypatch) + remember = ToolSpec(name="remember", description="keep a fact", parameters={"type": "object", "properties": {"fact": {"type": "string"}}, "required": ["fact"]}) + call = ToolCall(id="toolu_1", name="remember", arguments={"fact": "likes tea"}) + messages = ( + Message(role="user", content="remember I like tea"), + Message(role="assistant", content="Noted.", tool_calls=(call,)), + Message(role="tool", content="stored", tool_call_id="toolu_1"), + Message(role="tool", content="also indexed", tool_call_id="toolu_1"), + ) + run(collect(llm.reply(Prompt(system="s", messages=messages, tools=(remember,))))) + body = body_of(fake.posts[0]) + assert body["tools"] == [{"name": "remember", "description": "keep a fact", "input_schema": remember.parameters}] + assert body["messages"] == [ + {"role": "user", "content": "remember I like tea"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Noted."}, + {"type": "tool_use", "id": "toolu_1", "name": "remember", "input": {"fact": "likes tea"}}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "toolu_1", "content": "stored"}, + {"type": "tool_result", "tool_use_id": "toolu_1", "content": "also indexed"}, + ], + }, + ] + + +def test_a_proxy_url_is_honoured(monkeypatch): + fake = FakeAnthropic(message_stream("hi")) + llm = started(fake, monkeypatch, url="https://proxy.local/anthropic/") + run(collect(llm.reply(prompt("hello")))) + assert str(fake.posts[0].url) == "https://proxy.local/anthropic/v1/messages" + + +# ----------------------------------------------------------------- start + + +def test_preflight_lists_models_once_and_nothing_else(monkeypatch): + fake = FakeAnthropic() + started(fake, monkeypatch) + assert [(r.method, r.url.path) for r in fake.requests] == [("GET", "/v1/models")] + assert fake.requests[0].headers["x-api-key"] == KEY + + +def test_preflight_can_be_turned_off(monkeypatch): + fake = FakeAnthropic() + started(fake, monkeypatch, preflight=False) + assert fake.requests == [] + + +def test_no_key_is_unhealthy_names_the_variable_and_sends_nothing(monkeypatch): + monkeypatch.delenv(DEFAULT_KEY_ENV, raising=False) + fake = FakeAnthropic() + llm = make(fake) + run(llm.start()) + assert not llm.describe().healthy + assert DEFAULT_KEY_ENV in (llm.health().detail or "") + assert "no_key" in llm.health().faults + assert fake.requests == [] + + +def test_the_souls_key_env_is_honoured(monkeypatch): + monkeypatch.delenv(DEFAULT_KEY_ENV, raising=False) + monkeypatch.setenv("MY_ANTHROPIC", KEY) + fake = FakeAnthropic() + llm = make(fake, config={"key_env": "MY_ANTHROPIC"}) + run(llm.start()) + assert llm.describe().healthy + assert fake.requests[0].headers["x-api-key"] == KEY + + +def test_a_rejected_key_is_unhealthy_with_the_status(monkeypatch): + monkeypatch.setenv(DEFAULT_KEY_ENV, "wrong") + llm = make(FakeAnthropic(models_status=401)) + run(llm.start()) + detail = llm.health().detail or "" + assert not llm.describe().healthy + assert "401" in detail and DEFAULT_KEY_ENV in detail and "rejected" in detail + + +def test_an_unreachable_network_is_unhealthy_and_says_so(monkeypatch): + monkeypatch.setenv(DEFAULT_KEY_ENV, KEY) + llm = make(FakeAnthropic(raise_on_connect=httpx.ConnectError("name resolution failed"))) + run(llm.start()) + detail = llm.health().detail or "" + assert not llm.describe().healthy + assert "could not reach" in detail and "network" in detail + + +def test_without_httpx_the_hint_names_the_extra(monkeypatch): + import sys + + monkeypatch.setenv(DEFAULT_KEY_ENV, KEY) + monkeypatch.setitem(sys.modules, "httpx", None) + llm = AnthropicLanguageModel({"provider": "anthropic"}) + run(llm.start()) + assert not llm.describe().healthy + assert "emet-providers[anthropic]" in (llm.health().detail or "") + + +# ----------------------------------------------------------------- reply + + +def test_text_streams_as_deltas_and_the_final_carries_everything(monkeypatch): + fake = FakeAnthropic(message_stream("Hello", ", world", ".")) + llm = started(fake, monkeypatch) + events = run(collect(llm.reply(prompt("hi")))) + assert [e.text for e in events if isinstance(e, TextDelta)] == ["Hello", ", world", "."] + done = events[-1] + assert isinstance(done, ReplyDone) + assert done.text == "Hello, world." + assert done.stop_reason == "end" + assert done.model == "claude-opus-5" + assert (done.input_tokens, done.output_tokens) == (12, 7) + assert done.error is None + + +def test_a_tool_call_is_assembled_from_json_pieces_and_stops_with_tool(monkeypatch): + fake = FakeAnthropic(message_stream("Let me note that.", stop="tool_use", tool=("remember", {"fact": "likes tea", "n": 2}))) + llm = started(fake, monkeypatch) + events = run(collect(llm.reply(prompt("remember I like tea")))) + calls = [e for e in events if isinstance(e, ToolCall)] + assert calls == [ToolCall(id="toolu_1", name="remember", arguments={"fact": "likes tea", "n": 2})] + done = events[-1] + assert done.stop_reason == "tool" and done.tool_calls == tuple(calls) + assert done.text == "Let me note that." + + +@pytest.mark.parametrize( + ("vendor", "seam"), + [("end_turn", "end"), ("stop_sequence", "end"), ("max_tokens", "length"), ("refusal", "refusal"), ("pause_turn", "end"), ("something_new", "end")], +) +def test_stop_reasons_are_translated(monkeypatch, vendor, seam): + llm = started(FakeAnthropic(message_stream("x", stop=vendor)), monkeypatch) + assert run(collect(llm.reply(prompt("q"))))[-1].stop_reason == seam + + +def test_an_http_error_ends_the_reply_with_the_vendors_message(monkeypatch): + fake = FakeAnthropic(reply_status=429, reply_json={"type": "error", "error": {"type": "rate_limit_error", "message": "slow down"}}) + llm = started(fake, monkeypatch) + events = run(collect(llm.reply(prompt("q")))) + (done,) = events + assert done.stop_reason == "error" + assert "429" in (done.error or "") and "slow down" in (done.error or "") + assert llm.last_error == done.error + assert llm.describe().healthy, "a failed reply is not a broken plugin" + + +def test_an_error_event_mid_stream_keeps_the_text_so_far(monkeypatch): + events_bytes = sse( + ("message_start", {"type": "message_start", "message": {"model": "claude-opus-5", "usage": {"input_tokens": 3}}}), + ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}), + ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "Half a"}}), + ("error", {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}), + ) + llm = started(FakeAnthropic(events_bytes), monkeypatch) + events = run(collect(llm.reply(prompt("q")))) + done = events[-1] + assert done.text == "Half a" and done.stop_reason == "error" + assert "overloaded_error" in (done.error or "") + + +def test_a_stream_that_ends_early_still_returns_the_text(monkeypatch): + llm = started(FakeAnthropic(message_stream("cut", close=False)), monkeypatch) + done = run(collect(llm.reply(prompt("q"))))[-1] + assert done.text == "cut" and done.stop_reason == "end" + + +def test_unknown_block_types_are_ignored(monkeypatch): + events_bytes = sse( + ("message_start", {"type": "message_start", "message": {"model": "m", "usage": {"input_tokens": 1}}}), + ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "thinking", "thinking": ""}}), + ("content_block_delta", {"type": "content_block_delta", "index": 0, "delta": {"type": "thinking_delta", "thinking": "hmm"}}), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ("content_block_start", {"type": "content_block_start", "index": 1, "content_block": {"type": "text", "text": ""}}), + ("content_block_delta", {"type": "content_block_delta", "index": 1, "delta": {"type": "text_delta", "text": "fine"}}), + ("content_block_stop", {"type": "content_block_stop", "index": 1}), + ("message_delta", {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 1}}), + ("message_stop", {"type": "message_stop"}), + ) + llm = started(FakeAnthropic(events_bytes), monkeypatch) + assert run(collect(llm.reply(prompt("q"))))[-1].text == "fine" + + +def test_replying_before_start_is_an_error_final_not_an_exception(): + llm = make(FakeAnthropic()) + (done,) = run(collect(llm.reply(prompt("q")))) + assert done.stop_reason == "error" and "not started" in (done.error or "") + + +def test_shutdown_closes_the_client_and_reports_unhealthy(monkeypatch): + llm = started(FakeAnthropic(), monkeypatch) + run(llm.shutdown()) + assert not llm.describe().healthy + assert llm._client is None diff --git a/emet-providers/tests/test_deepgram.py b/emet-providers/tests/test_deepgram.py new file mode 100644 index 0000000..c4dba2d --- /dev/null +++ b/emet-providers/tests/test_deepgram.py @@ -0,0 +1,463 @@ +"""The Deepgram transcriber, against a stand-in for Deepgram. + +No network and no key. `FakeServer` answers `connect()` the way the library +would, records every frame it is sent, and replies with scripted messages in +the shapes Deepgram's documentation gives (verified 2026-09-12). What these +tests prove is the plugin's half of the protocol: the URL and header it +sends, the order of frames, how it assembles interims and finals into one +utterance, and what it says when a key is missing, rejected, or the network +is not there. The live test beside this file proves the other half, when a +key is present. +""" + +from __future__ import annotations + +import asyncio +import json +import sys +import types +from typing import Any +from urllib.parse import parse_qs, urlparse + +import pytest + +from emet_sdk.plugin import TranscriberPlugin +from emet_sdk.types import AudioFormat, Transcript + +from emet_providers.deepgram import DEFAULT_KEY_ENV, DEFAULT_MODEL, DEFAULT_URL, DeepgramTranscriber + +FRAME = bytes(2560) +KEY = "dg_test_key" + + +def run(coro): + return asyncio.run(coro) + + +async def settle() -> None: + """Let the pump, the sender and the receiver each take their turns. + + The fake answers a frame the instant it is sent, so without this a + test's next `feed()` could observe two frames' worth of results at once + and the sequence of partials would depend on scheduling order. + """ + for _ in range(8): + await asyncio.sleep(0) + + +# -------------------------------------------------------------- the fake + + +class _Closed(Exception): + """Shaped like websockets' ConnectionClosed: it carries rcvd and sent.""" + + def __init__(self, code: int = 1000) -> None: + super().__init__(f"closed {code}") + self.rcvd = types.SimpleNamespace(code=code, reason="") + self.sent = types.SimpleNamespace(code=code, reason="") + + +class _Rejected(Exception): + """Shaped like websockets' InvalidStatus: it carries response.status_code.""" + + def __init__(self, status_code: int) -> None: + super().__init__(f"server rejected WebSocket connection: HTTP {status_code}") + self.response = types.SimpleNamespace(status_code=status_code) + + +def results(transcript: str, *, final: bool, confidence: float = 0.9) -> dict: + """One Results message, in the documented shape.""" + return { + "type": "Results", + "channel_index": [0, 1], + "duration": 1.0, + "start": 0.0, + "is_final": final, + "speech_final": False, + "channel": { + "alternatives": [ + {"transcript": transcript, "confidence": confidence, "words": []} + ] + }, + } + + +METADATA = {"type": "Metadata", "request_id": "r", "duration": 1.0, "channels": 1} + + +class FakeConnection: + def __init__(self, script: dict, *, close_after_metadata: bool = True) -> None: + #: `script` maps the count of binary frames received so far to the + #: messages the server sends at that moment, and "close" to what it + #: sends after CloseStream. Metadata and the close follow unless told + #: otherwise. + self.script = script + self.close_after_metadata = close_after_metadata + self.sent: list[Any] = [] + self.frames = 0 + self.closed = False + self._inbox: asyncio.Queue = asyncio.Queue() + + def _say(self, message: Any) -> None: + self._inbox.put_nowait(message if isinstance(message, str) else json.dumps(message)) + + async def send(self, data: Any) -> None: + self.sent.append(data) + if isinstance(data, bytes): + self.frames += 1 + for message in self.script.get(self.frames, []): + self._say(message) + return + if json.loads(data).get("type") == "CloseStream": + for message in self.script.get("close", []): + self._say(message) + if self.close_after_metadata: + self._say(METADATA) + self._inbox.put_nowait(_Closed()) + + async def recv(self) -> str: + item = await self._inbox.get() + if isinstance(item, BaseException): + raise item + return item + + async def close(self) -> None: + self.closed = True + + +class FakeServer: + def __init__(self, script: dict | None = None, *, reject: BaseException | None = None, **conn_kw) -> None: + self.script = script or {} + self.reject = reject + self.conn_kw = conn_kw + self.urls: list[str] = [] + self.kwargs: list[dict] = [] + self.connections: list[FakeConnection] = [] + + async def connect(self, url: str, **kwargs) -> FakeConnection: + self.urls.append(url) + self.kwargs.append(kwargs) + if self.reject is not None: + raise self.reject + conn = FakeConnection(self.script, **self.conn_kw) + self.connections.append(conn) + return conn + + +def make(server: FakeServer, *, config: dict | None = None, **params) -> DeepgramTranscriber: + cfg = {"provider": "deepgram", "params": params} + cfg.update(config or {}) + return DeepgramTranscriber(cfg, AudioFormat(), connect=server.connect) + + +def started(server: FakeServer, monkeypatch, **kw) -> DeepgramTranscriber: + monkeypatch.setenv(DEFAULT_KEY_ENV, KEY) + stt = make(server, **kw) + run(stt.start()) + assert stt.describe().healthy, stt.health().detail + return stt + + +def query_of(url: str) -> dict[str, str]: + return {k: v[0] for k, v in parse_qs(urlparse(url).query).items()} + + +# --------------------------------------------------------------- request + + +def test_it_is_a_transcriber_plugin_registered_as_deepgram(): + stt = make(FakeServer()) + assert isinstance(stt, TranscriberPlugin) + assert stt.provider == "deepgram" + + +def test_the_request_says_linear16_at_the_formats_rate_with_interims_on(): + stt = make(FakeServer()) + url = stt.request_url() + assert url.startswith(DEFAULT_URL + "?") + q = query_of(url) + assert q["encoding"] == "linear16" + assert q["sample_rate"] == "16000" + assert q["channels"] == "1" + assert q["interim_results"] == "true" + assert q["punctuate"] == "true" and q["smart_format"] == "true" + assert q["model"] == DEFAULT_MODEL + + +def test_the_souls_model_is_sent_and_reported(): + stt = make(FakeServer(), config={"model": "nova-2"}) + assert query_of(stt.request_url())["model"] == "nova-2" + assert stt.describe().model == "nova-2" + + +def test_body_query_params_ride_along_but_cannot_change_the_format(): + stt = make( + FakeServer(), + query={"language": "en-GB", "keyterm": "Emet", "sample_rate": 8000, "encoding": "mulaw", "vad_events": True}, + ) + q = query_of(stt.request_url()) + assert q["language"] == "en-GB" + assert q["keyterm"] == "Emet" + assert q["vad_events"] == "true" + assert q["sample_rate"] == "16000" + assert q["encoding"] == "linear16" + + +def test_the_url_can_point_at_a_self_hosted_deployment(): + stt = make(FakeServer(), url="wss://deepgram.internal/v1/listen") + assert stt.request_url().startswith("wss://deepgram.internal/v1/listen?") + + +def test_the_key_travels_as_a_token_header(monkeypatch): + server = FakeServer() + started(server, monkeypatch) + assert server.kwargs[0]["additional_headers"] == {"Authorization": f"Token {KEY}"} + assert server.kwargs[0]["open_timeout"] == 10.0 + + +# ---------------------------------------------------------------- start + + +def test_preflight_opens_one_connection_sends_close_stream_and_closes(monkeypatch): + server = FakeServer() + started(server, monkeypatch) + (conn,) = server.connections + assert conn.sent == [json.dumps({"type": "CloseStream"})] + assert conn.closed + + +def test_preflight_can_be_turned_off(monkeypatch): + server = FakeServer() + started(server, monkeypatch, preflight=False) + assert server.connections == [] + + +def test_no_key_is_unhealthy_and_names_the_variable(monkeypatch): + monkeypatch.delenv(DEFAULT_KEY_ENV, raising=False) + server = FakeServer() + stt = make(server) + run(stt.start()) + assert not stt.describe().healthy + assert DEFAULT_KEY_ENV in (stt.health().detail or "") + assert "no_key" in stt.health().faults + assert server.connections == [], "no connection may be attempted without a key" + + +def test_the_souls_key_env_is_honoured(monkeypatch): + monkeypatch.delenv(DEFAULT_KEY_ENV, raising=False) + monkeypatch.setenv("MY_DG", KEY) + server = FakeServer() + stt = make(server, config={"key_env": "MY_DG"}) + run(stt.start()) + assert stt.describe().healthy + assert server.kwargs[0]["additional_headers"]["Authorization"] == f"Token {KEY}" + + +def test_a_rejected_key_is_unhealthy_with_the_status(monkeypatch): + monkeypatch.setenv(DEFAULT_KEY_ENV, "wrong") + stt = make(FakeServer(reject=_Rejected(401))) + run(stt.start()) + detail = stt.health().detail or "" + assert not stt.describe().healthy + assert "401" in detail and DEFAULT_KEY_ENV in detail and "rejected" in detail + + +def test_an_unreachable_network_is_unhealthy_and_says_so(monkeypatch): + monkeypatch.setenv(DEFAULT_KEY_ENV, KEY) + stt = make(FakeServer(reject=OSError("Name or service not known"))) + run(stt.start()) + detail = stt.health().detail or "" + assert not stt.describe().healthy + assert "could not reach" in detail and "network" in detail + + +def test_without_the_websockets_library_the_hint_names_the_extra(monkeypatch): + monkeypatch.setenv(DEFAULT_KEY_ENV, KEY) + monkeypatch.setitem(sys.modules, "websockets", None) + monkeypatch.setitem(sys.modules, "websockets.asyncio", None) + monkeypatch.setitem(sys.modules, "websockets.asyncio.client", None) + stt = DeepgramTranscriber({"provider": "deepgram"}, AudioFormat()) + run(stt.start()) + assert not stt.describe().healthy + assert "emet-providers[deepgram]" in (stt.health().detail or "") + + +# ------------------------------------------------------------- streaming + + +def test_frames_stream_in_order_and_close_stream_is_a_text_frame(monkeypatch): + server = FakeServer() + stt = started(server, monkeypatch) + + async def scenario(): + for i in range(3): + await stt.feed(bytes([i]) * 2560) + return await stt.finish() + + final = run(scenario()) + conn = server.connections[-1] + assert conn.sent[:3] == [bytes([0]) * 2560, bytes([1]) * 2560, bytes([2]) * 2560] + assert conn.sent[3] == json.dumps({"type": "CloseStream"}) + assert isinstance(conn.sent[3], str) + assert conn.closed + assert final == Transcript(text="", final=True, confidence=1.0) + + +def test_interims_become_partials_and_finals_join_into_the_utterance(monkeypatch): + server = FakeServer( + { + 1: [results("what", final=False, confidence=0.4)], + 2: [results("what time", final=False, confidence=0.6)], + 3: [results("What time", final=True, confidence=0.95)], + 4: [results("is it", final=False, confidence=0.5)], + "close": [results("is it in Tokyo?", final=True, confidence=0.97)], + } + ) + stt = started(server, monkeypatch) + + async def scenario(): + partials = [] + # Six feeds for four scripted messages: each feed observes what the + # previous frame's message changed, and the last two observe nothing. + for _ in range(6): + if (t := await stt.feed(FRAME)) is not None: + partials.append(t) + await settle() + return partials, await stt.finish() + + partials, final = run(scenario()) + texts = [p.text for p in partials] + assert texts == ["what", "what time", "What time", "What time is it"] + assert all(not p.final for p in partials) + assert partials[0].confidence == pytest.approx(0.4) + assert final.final + assert final.text == "What time is it in Tokyo?" + assert final.confidence == pytest.approx((0.95 + 0.97) / 2) + + +def test_empty_interims_for_silence_add_nothing(monkeypatch): + server = FakeServer({1: [results("", final=False)], 2: [results("", final=True)]}) + stt = started(server, monkeypatch) + + async def scenario(): + out = [] + for _ in range(3): + out.append(await stt.feed(FRAME)) + await asyncio.sleep(0) + return out, await stt.finish() + + partials, final = run(scenario()) + assert partials == [None, None, None] + assert final.text == "" + + +def test_finish_with_nothing_fed_opens_nothing(monkeypatch): + server = FakeServer() + stt = started(server, monkeypatch) + final = run(stt.finish()) + assert final == Transcript(text="", final=True, confidence=1.0) + assert len(server.connections) == 1, "only the preflight connection" + + +def test_each_utterance_gets_its_own_connection(monkeypatch): + server = FakeServer({"close": [results("one", final=True)]}) + stt = started(server, monkeypatch) + + async def scenario(): + await stt.feed(FRAME) + first = await stt.finish() + await stt.feed(FRAME) + second = await stt.finish() + return first.text, second.text + + assert run(scenario()) == ("one", "one") + assert len(server.connections) == 3, "preflight plus one per utterance" + + +def test_a_server_that_never_answers_gives_back_what_was_heard(monkeypatch): + server = FakeServer({1: [results("half a", final=False)]}, close_after_metadata=False) + stt = started(server, monkeypatch, timeout_s=0.05) + + async def scenario(): + await stt.feed(FRAME) + await asyncio.sleep(0) + return await stt.finish() + + final = run(scenario()) + assert final.final and final.text == "half a" + assert stt.last_error is not None and "CloseStream" in stt.last_error + + +def test_a_connection_that_drops_mid_utterance_is_not_a_broken_plugin(monkeypatch): + class Dropping(FakeConnection): + async def send(self, data): + if isinstance(data, bytes) and self.frames >= 1: + raise OSError("connection reset") + await super().send(data) + + class DroppingServer(FakeServer): + async def connect(self, url, **kwargs): + self.urls.append(url) + conn = Dropping(self.script) + self.connections.append(conn) + return conn + + server = DroppingServer({1: [results("hello", final=True)]}) + stt = started(server, monkeypatch) + + async def scenario(): + await stt.feed(FRAME) + await settle() + await stt.feed(FRAME) + await settle() + return await stt.finish() + + final = run(scenario()) + assert final.text == "hello" + assert stt.last_error is not None + assert stt.describe().healthy, "a transient failure is logged, not a fault" + + +def test_shutdown_mid_utterance_cancels_and_forgets(monkeypatch): + server = FakeServer(close_after_metadata=False) + stt = started(server, monkeypatch) + + async def scenario(): + await stt.feed(FRAME) + await asyncio.sleep(0) + await stt.shutdown() + return stt.describe().healthy, await stt.finish() + + healthy, final = run(scenario()) + assert not healthy + assert final.text == "" + + +def test_unknown_message_types_and_non_json_frames_are_ignored(monkeypatch): + server = FakeServer( + { + 1: [ + {"type": "SpeechStarted", "timestamp": 0.1}, + "this is not json", + {"type": "UtteranceEnd", "last_word_end": 1.2}, + results("fine", final=True), + ] + } + ) + stt = started(server, monkeypatch) + + async def scenario(): + await stt.feed(FRAME) + await asyncio.sleep(0) + return await stt.finish() + + assert run(scenario()).text == "fine" + + +def test_describe_before_start_is_unhealthy_and_after_is_streaming(monkeypatch): + server = FakeServer() + stt = make(server) + assert not stt.describe().healthy + started(server, monkeypatch) + d = make(server).describe() + assert d.streaming and d.sample_rate == 16000 and d.provider == "deepgram" diff --git a/emet-providers/tests/test_deepgram_live.py b/emet-providers/tests/test_deepgram_live.py new file mode 100644 index 0000000..1787ee8 --- /dev/null +++ b/emet-providers/tests/test_deepgram_live.py @@ -0,0 +1,45 @@ +"""One real round trip to Deepgram, when a key is present and asked for. + +Skipped in CI and on any machine without both `EMET_DEEPGRAM_KEY` and +`EMET_LIVE_TESTS=1` in the environment. It sends two seconds of silence, so +the transcript is expected to be empty; what it proves is the half the fake +cannot: that the handshake, the header, the query string and the CloseStream +sequence are what the live service accepts today. Run it by hand before a +release, and whenever Deepgram announces a change. +""" + +from __future__ import annotations + +import asyncio +import os + +import pytest + +from emet_sdk.types import AudioFormat + +from emet_providers.deepgram import DEFAULT_KEY_ENV, DeepgramTranscriber + +pytestmark = pytest.mark.skipif( + not (os.environ.get(DEFAULT_KEY_ENV) and os.environ.get("EMET_LIVE_TESTS") == "1"), + reason=f"set {DEFAULT_KEY_ENV} and EMET_LIVE_TESTS=1 to call Deepgram for real", +) + + +def test_a_live_round_trip_returns_a_final(): + pytest.importorskip("websockets", reason="emet-providers[deepgram] is not installed") + stt = DeepgramTranscriber({"provider": "deepgram"}, AudioFormat()) + + async def scenario(): + await stt.start() + assert stt.describe().healthy, stt.health().detail + for _ in range(25): # two seconds of silence at 80 ms frames + await stt.feed(bytes(2560)) + await asyncio.sleep(0.08) + final = await stt.finish() + await stt.shutdown() + return final + + final = asyncio.run(scenario()) + assert final.final + assert isinstance(final.text, str) + assert stt.last_error is None, stt.last_error diff --git a/emet-providers/tests/test_deepgram_voice.py b/emet-providers/tests/test_deepgram_voice.py new file mode 100644 index 0000000..e745c6e --- /dev/null +++ b/emet-providers/tests/test_deepgram_voice.py @@ -0,0 +1,350 @@ +"""The Deepgram voice, against a stand-in for the speak API. + +No network and no key. An `httpx.MockTransport` answers in the shapes +Deepgram's API reference gives (verified 2026-09-15; the 401 body was +checked against the live endpoint with a bogus key the same day) and +records what it was sent. What these tests prove is the plugin's half of the +protocol: the URL, the query, the header, the body, and what it does with +the bytes that come back and with the errors. The live test beside this +file proves the other half, when a key is present. +""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any, AsyncIterator + +import pytest + +httpx = pytest.importorskip("httpx", reason="emet-providers[deepgram] is not installed") + +from emet_sdk.plugin import PluginError, VoicePlugin # noqa: E402 + +from emet_providers.deepgram_voice import ( # noqa: E402 + DEFAULT_KEY_ENV, + DEFAULT_MODEL, + DEFAULT_SAMPLE_RATE, + DEFAULT_URL, + MAX_CHARS, + PREFLIGHT_TEXT, + DeepgramVoice, + pieces, +) + +KEY = "dg_test_key" + + +def run(coro): + return asyncio.run(coro) + + +async def collect(chunks) -> list[bytes]: + return [c async for c in chunks] + + +# -------------------------------------------------------------- the fake + + +class _Pieces(httpx.AsyncByteStream): + """A response body delivered in the pieces a network would.""" + + def __init__(self, parts: list[bytes]) -> None: + self.parts = parts + + async def __aiter__(self) -> AsyncIterator[bytes]: + for part in self.parts: + yield part + + +class FakeSpeak: + def __init__( + self, + *, + status: int = 200, + parts: list[bytes] | None = None, + audio: bytes | None = None, + error: dict | None = None, + raise_on_connect: Exception | None = None, + fail_after: int | None = None, + ) -> None: + self.status = status + self.parts = parts + self.audio = audio + self.error = error + self.raise_on_connect = raise_on_connect + self.fail_after = fail_after + self.requests: list[httpx.Request] = [] + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + if self.raise_on_connect is not None: + raise self.raise_on_connect + if self.fail_after is not None and len(self.requests) > self.fail_after: + return httpx.Response(500, json={"err_code": "INTERNAL", "err_msg": "boom"}) + if self.status != 200: + return httpx.Response(self.status, json=self.error or {"err_code": "INVALID_AUTH", "err_msg": "Invalid credentials."}) + if self.parts is not None: + return httpx.Response(200, stream=_Pieces(self.parts), headers={"content-type": "audio/l16"}) + text = json.loads(request.content)["text"] + body = self.audio if self.audio is not None else text.encode() + bytes(2) + return httpx.Response(200, content=body, headers={"content-type": "audio/l16"}) + + @property + def bodies(self) -> list[str]: + return [json.loads(r.content)["text"] for r in self.requests] + + +def make(fake: FakeSpeak, *, config: dict | None = None, voice: dict | None = None, **params: Any) -> DeepgramVoice: + cfg: dict[str, Any] = {"provider": "deepgram", "params": params} + cfg.update(config or {}) + return DeepgramVoice(cfg, voice, transport=httpx.MockTransport(fake)) + + +def started(fake: FakeSpeak, monkeypatch, **kw) -> DeepgramVoice: + monkeypatch.setenv(DEFAULT_KEY_ENV, KEY) + tts = make(fake, **kw) + run(tts.start()) + assert tts.describe().healthy, tts.health().detail + return tts + + +# --------------------------------------------------------------- request + + +def test_it_is_a_voice_plugin_registered_as_deepgram(): + tts = make(FakeSpeak()) + assert isinstance(tts, VoicePlugin) + assert tts.provider == "deepgram" + assert tts.url == DEFAULT_URL and tts.model_name == DEFAULT_MODEL + + +def test_the_request_is_the_documented_one(monkeypatch): + fake = FakeSpeak() + tts = started(fake, monkeypatch, preflight=False) + run(collect(tts.speak("Hello there."))) + (request,) = fake.requests + assert request.method == "POST" + assert request.url.scheme == "https" and request.url.host == "api.deepgram.com" + assert request.url.path == "/v1/speak" + assert dict(request.url.params) == { + "model": DEFAULT_MODEL, + "encoding": "linear16", + "container": "none", + "sample_rate": str(DEFAULT_SAMPLE_RATE), + } + assert request.headers["authorization"] == f"Token {KEY}" + assert request.headers["content-type"] == "application/json" + assert fake.bodies == ["Hello there."] + + +def test_the_souls_voice_and_the_bodys_rate_are_sent_and_reported(monkeypatch): + fake = FakeSpeak() + tts = started(fake, monkeypatch, config={"model": "aura-2-apollo-en"}, sample_rate=16000, preflight=False) + run(collect(tts.speak("Hi."))) + params = dict(fake.requests[0].url.params) + assert params["model"] == "aura-2-apollo-en" and params["sample_rate"] == "16000" + d = tts.describe() + assert d.model == "aura-2-apollo-en" and d.sample_rate == 16000 and d.streaming + + +def test_a_rate_linear16_does_not_offer_falls_back_to_the_default(monkeypatch): + tts = make(FakeSpeak(), sample_rate=22050) + assert tts.sample_rate == DEFAULT_SAMPLE_RATE + + +def test_the_souls_speaking_rate_is_the_speed_parameter(monkeypatch): + fake = FakeSpeak() + tts = started(fake, monkeypatch, voice={"rate": 1.2}, preflight=False) + run(collect(tts.speak("Hi."))) + assert dict(fake.requests[0].url.params)["speed"] == "1.2" + + +def test_a_rate_outside_the_range_is_clamped(monkeypatch): + assert make(FakeSpeak(), voice={"rate": 3.0}).speed == 1.5 + assert make(FakeSpeak(), voice={"rate": 0.1}).speed == 0.7 + assert "speed" not in make(FakeSpeak()).query() + + +def test_body_query_params_ride_along_but_cannot_change_the_format(monkeypatch): + fake = FakeSpeak() + tts = started( + fake, + monkeypatch, + preflight=False, + query={"tag": "emet", "mip_opt_out": True, "encoding": "mp3", "container": "wav", "model": "x"}, + ) + run(collect(tts.speak("Hi."))) + params = dict(fake.requests[0].url.params) + assert params["tag"] == "emet" and params["mip_opt_out"] == "true" + assert params["encoding"] == "linear16" and params["container"] == "none" and params["model"] == DEFAULT_MODEL + + +def test_the_url_can_point_at_a_self_hosted_deployment(monkeypatch): + fake = FakeSpeak() + tts = started(fake, monkeypatch, preflight=False, url="https://speak.example.test/v1/speak") + run(collect(tts.speak("Hi."))) + assert str(fake.requests[0].url).startswith("https://speak.example.test/v1/speak?") + + +# -------------------------------------------------------------- lifecycle + + +def test_preflight_says_one_word_and_discards_it(monkeypatch): + fake = FakeSpeak() + tts = started(fake, monkeypatch) + assert fake.bodies == [PREFLIGHT_TEXT] + assert len(PREFLIGHT_TEXT) <= 6, "the preflight is meant to cost nothing" + assert tts.describe().healthy + + +def test_preflight_can_be_turned_off(monkeypatch): + fake = FakeSpeak() + started(fake, monkeypatch, preflight=False) + assert fake.requests == [] + + +def test_no_key_is_unhealthy_names_the_variable_and_sends_nothing(monkeypatch): + monkeypatch.delenv(DEFAULT_KEY_ENV, raising=False) + fake = FakeSpeak() + tts = make(fake) + run(tts.start()) + assert not tts.describe().healthy + assert DEFAULT_KEY_ENV in (tts.health().detail or "") + assert tts.health().faults == ("no_key",) + assert fake.requests == [] + + +def test_the_souls_key_env_is_honoured(monkeypatch): + monkeypatch.delenv(DEFAULT_KEY_ENV, raising=False) + monkeypatch.setenv("EMET_OTHER_KEY", "other") + fake = FakeSpeak() + tts = make(fake, config={"key_env": "EMET_OTHER_KEY"}) + run(tts.start()) + assert tts.describe().healthy + assert fake.requests[0].headers["authorization"] == "Token other" + + +def test_a_rejected_key_is_unhealthy_with_the_status(monkeypatch): + monkeypatch.setenv(DEFAULT_KEY_ENV, "bad") + tts = make(FakeSpeak(status=401)) + run(tts.start()) + assert not tts.describe().healthy + detail = tts.health().detail or "" + assert "rejected the key" in detail and "401" in detail and DEFAULT_KEY_ENV in detail + + +def test_an_unknown_voice_is_refused_at_boot_in_the_vendors_words(monkeypatch): + monkeypatch.setenv(DEFAULT_KEY_ENV, KEY) + tts = make(FakeSpeak(status=400, error={"err_code": "Bad Request", "err_msg": "Model not found"}), config={"model": "aura-2-nobody-en"}) + run(tts.start()) + assert not tts.describe().healthy + detail = tts.health().detail or "" + assert "aura-2-nobody-en" in detail and "Model not found" in detail + + +def test_an_unreachable_network_is_unhealthy_and_says_so(monkeypatch): + monkeypatch.setenv(DEFAULT_KEY_ENV, KEY) + tts = make(FakeSpeak(raise_on_connect=httpx.ConnectError("no route"))) + run(tts.start()) + assert not tts.describe().healthy + assert "could not reach" in (tts.health().detail or "") + + +def test_without_httpx_the_hint_names_the_extra(monkeypatch): + import sys + + monkeypatch.setenv(DEFAULT_KEY_ENV, KEY) + monkeypatch.setitem(sys.modules, "httpx", None) + tts = DeepgramVoice({"provider": "deepgram"}) + run(tts.start()) + assert "emet-providers[deepgram]" in (tts.health().detail or "") + + +def test_shutdown_closes_the_client_and_reports_unhealthy(monkeypatch): + tts = started(FakeSpeak(), monkeypatch) + run(tts.shutdown()) + assert not tts.describe().healthy and tts._client is None + + +# ----------------------------------------------------------------- audio + + +def test_audio_streams_back_in_whole_samples(monkeypatch): + """A chunk boundary between the two bytes of a sample is carried over.""" + fake = FakeSpeak(parts=[b"\x01\x02\x03", b"\x04\x05\x06\x07", b"\x08"]) + tts = started(fake, monkeypatch, preflight=False) + chunks = run(collect(tts.speak("Hi."))) + assert chunks == [b"\x01\x02", b"\x03\x04\x05\x06", b"\x07\x08"] + assert all(len(c) % 2 == 0 for c in chunks) + + +def test_a_dangling_byte_at_the_end_is_dropped_not_played(monkeypatch): + fake = FakeSpeak(parts=[b"\x01\x02\x03"]) + tts = started(fake, monkeypatch, preflight=False) + assert run(collect(tts.speak("Hi."))) == [b"\x01\x02"] + + +def test_blank_text_sends_nothing(monkeypatch): + fake = FakeSpeak() + tts = started(fake, monkeypatch, preflight=False) + assert run(collect(tts.speak(" "))) == [] + assert fake.requests == [] + + +def test_a_long_reply_is_split_at_sentences_and_sent_in_order(monkeypatch): + fake = FakeSpeak() + tts = started(fake, monkeypatch, preflight=False) + first = "A" * (MAX_CHARS - 10) + "." + second = "B" * 50 + "." + run(collect(tts.speak(f"{first} {second}"))) + assert fake.bodies == [first, second] + + +def test_pieces_splits_at_sentences_then_words_then_characters(): + assert pieces("short") == ["short"] + assert pieces(" spaced out ") == ["spaced out"] + assert pieces("") == [] + assert pieces("one two three four", limit=9) == ["one two", "three", "four"] + assert pieces("First one. Second one! Third?", limit=12) == ["First one.", "Second one!", "Third?"] + assert pieces("x" * 25, limit=10) == ["x" * 10, "x" * 10, "x" * 5] + for piece in pieces("word " * 1000): + assert len(piece) <= MAX_CHARS + + +def test_an_http_error_mid_reply_loses_the_sentence_not_the_plugin(monkeypatch): + fake = FakeSpeak(fail_after=1) + tts = started(fake, monkeypatch, preflight=False) + assert run(collect(tts.speak("First."))) == [b"First.\x00\x00"] + with pytest.raises(PluginError) as exc: + run(collect(tts.speak("Second."))) + assert "HTTP 500 INTERNAL: boom" in str(exc.value) + assert tts.last_error == "HTTP 500 INTERNAL: boom" + assert tts.describe().healthy + + +def test_a_network_failure_mid_reply_is_a_plugin_error(monkeypatch): + monkeypatch.setenv(DEFAULT_KEY_ENV, KEY) + tts = make(FakeSpeak(raise_on_connect=httpx.ReadTimeout("slow")), preflight=False) + run(tts.start()) + with pytest.raises(PluginError, match="ReadTimeout"): + run(collect(tts.speak("Hello."))) + + +def test_speaking_before_start_is_an_error(): + tts = make(FakeSpeak()) + + async def scenario(): + with pytest.raises(PluginError, match="not started"): + await collect(tts.speak("Hello.")) + + run(scenario()) + + +def test_the_module_names_its_verification_and_prices(): + from emet_providers import deepgram_voice + + doc = deepgram_voice.__doc__ or "" + assert "2026-09-15" in doc + assert "$0.030" in doc and "2000 characters" in doc + assert "/v2/speak" in doc, "Flux TTS is named as the protocol this file does not speak" diff --git a/emet-providers/tests/test_llm_live.py b/emet-providers/tests/test_llm_live.py new file mode 100644 index 0000000..99484f4 --- /dev/null +++ b/emet-providers/tests/test_llm_live.py @@ -0,0 +1,68 @@ +"""One real reply from each vendor, when a key is present and asked for. + +Skipped in CI and on any machine without `EMET_LIVE_TESTS=1` and the vendor's +key in the environment. Each asks for a single word and spends a few tokens. +What it proves is the half the stand-ins cannot: that the headers, the body +and the stream are what the live service accepts today. Run by hand before a +release, and whenever a vendor announces a change. +""" + +from __future__ import annotations + +import asyncio +import os + +import pytest + +from emet_sdk.types import Message, Prompt, ReplyDone, TextDelta + +LIVE = os.environ.get("EMET_LIVE_TESTS") == "1" + + +def _round_trip(llm): + prompt = Prompt( + system="You are a test. Reply with the single word: ready.", + messages=(Message(role="user", content="Are you there?"),), + max_tokens=64, + ) + + async def scenario(): + await llm.start() + assert llm.describe().healthy, llm.health().detail + events = [e async for e in llm.reply(prompt)] + await llm.shutdown() + return events + + events = asyncio.run(scenario()) + done = events[-1] + assert isinstance(done, ReplyDone) + assert done.stop_reason == "end", done + assert done.text.strip(), done + assert any(isinstance(e, TextDelta) for e in events) + assert done.output_tokens is None or done.output_tokens > 0 + assert llm.last_error is None, llm.last_error + return done + + +@pytest.mark.skipif( + not (LIVE and os.environ.get("EMET_ANTHROPIC_KEY")), + reason="set EMET_ANTHROPIC_KEY and EMET_LIVE_TESTS=1 to call Anthropic for real", +) +def test_anthropic_answers(): + pytest.importorskip("httpx", reason="emet-providers[anthropic] is not installed") + from emet_providers.anthropic import AnthropicLanguageModel + + done = _round_trip(AnthropicLanguageModel({"provider": "anthropic"})) + assert done.model + + +@pytest.mark.skipif( + not (LIVE and os.environ.get("EMET_OPENAI_KEY")), + reason="set EMET_OPENAI_KEY and EMET_LIVE_TESTS=1 to call OpenAI for real", +) +def test_openai_answers(): + pytest.importorskip("httpx", reason="emet-providers[openai] is not installed") + from emet_providers.openai import OpenAILanguageModel + + done = _round_trip(OpenAILanguageModel({"provider": "openai"})) + assert done.model diff --git a/emet-providers/tests/test_mock.py b/emet-providers/tests/test_mock.py new file mode 100644 index 0000000..5f116f8 --- /dev/null +++ b/emet-providers/tests/test_mock.py @@ -0,0 +1,424 @@ +"""The mock transcriber, and through it the transcriber contract. + +No network, no key, no audio anybody could hear. Words are written into +frames as bytes and read back out, which is what makes every test of the +speech path above this a one-liner. +""" + +from __future__ import annotations + +import asyncio + +import pytest + +from emet_sdk.plugin import TranscriberPlugin +from emet_sdk.types import AudioFormat, Transcript, TranscriberDescriptor + +from emet_providers.mock import MIN_CHARS, MockTranscriber, spelled + +FRAME_BYTES = 2560 + + +def run(coro): + return asyncio.run(coro) + + +def frame(payload: bytes = b"") -> bytes: + # bytes(n) rather than an escape: zero bytes without a backslash in sight. + return payload + bytes(FRAME_BYTES - len(payload)) + + +def make(**params) -> MockTranscriber: + return MockTranscriber({"provider": "mock", "params": params}, AudioFormat()) + + +def started(**params) -> MockTranscriber: + stt = make(**params) + run(stt.start()) + return stt + + +# --------------------------------------------------------------- spelling + + +def test_a_frame_that_begins_with_text_spells_it(): + assert spelled(frame(b"what time is it")) == "what time is it" + + +def test_silence_spells_nothing(): + assert spelled(frame()) == "" + + +def test_loud_audio_spells_nothing(): + """int16 samples of a tone. Bytes outside the printable range appear at + once, so this is audio rather than words.""" + from array import array + + pcm = array("h", [4000, -4000] * (FRAME_BYTES // 4)).tobytes() + assert spelled(pcm) == "" + + +def test_a_short_run_is_audio_not_a_word(): + """A printable byte then a zero is an ordinary small sample.""" + assert spelled(frame(b"A")) == "" + assert spelled(frame(b"ab")) == "" + assert len("abc") == MIN_CHARS + assert spelled(frame(b"abc")) == "abc" + + +# ---------------------------------------------------------------- contract + + +def test_the_mock_is_a_transcriber_plugin(): + stt = make() + assert isinstance(stt, TranscriberPlugin) + assert stt.provider == "mock" + assert stt.format == AudioFormat() + + +def test_the_config_is_unpacked_the_way_the_engine_hands_it_over(): + stt = MockTranscriber( + {"provider": "mock", "model": "tiny", "key_env": "EMET_MOCK_KEY", "params": {"x": 1}}, + AudioFormat(), + ) + assert stt.model == "tiny" + assert stt.key_env == "EMET_MOCK_KEY" + assert stt.params == {"x": 1} + + +def test_a_missing_model_and_key_env_are_none_not_strings(): + stt = make() + assert stt.model is None + assert stt.key_env is None + + +def test_describe_reports_the_format_it_was_given_and_that_it_streams(): + stt = started() + d = stt.describe() + assert isinstance(d, TranscriberDescriptor) + assert d.healthy + assert d.streaming + assert d.sample_rate == 16000 + assert d.provider == "mock" + + +# --------------------------------------------------------------- streaming + + +def test_words_arrive_as_growing_partials_then_one_final(): + stt = started() + + async def scenario(): + partials = [] + for f in (frame(b"what"), frame(), frame(b"time"), frame(b"is it")): + if (t := await stt.feed(f)) is not None: + partials.append(t) + final = await stt.finish() + return partials, final + + partials, final = run(scenario()) + assert [p.text for p in partials] == ["what", "what time", "what time is it"] + assert all(not p.final for p in partials) + assert final == Transcript(text="what time is it", final=True, confidence=1.0) + assert stt.frames == 4 + assert stt.finished == 1 + + +def test_silence_alone_gives_an_empty_final(): + """A real answer: they said nothing the provider could make out.""" + stt = started() + + async def scenario(): + assert await stt.feed(frame()) is None + return await stt.finish() + + final = run(scenario()) + assert final.final + assert final.text == "" + + +def test_finish_resets_for_the_next_utterance(): + stt = started() + + async def scenario(): + await stt.feed(frame(b"first")) + first = await stt.finish() + await stt.feed(frame(b"second")) + second = await stt.finish() + return first.text, second.text + + assert run(scenario()) == ("first", "second") + + +def test_a_scripted_transcript_streams_one_word_a_frame(): + """For a live microphone, whose bytes spell nothing: the mock says what it + was told to, and still exercises the partial path.""" + stt = started(transcript="hello there emet") + + async def scenario(): + partials = [await stt.feed(frame()) for _ in range(5)] + final = await stt.finish() + return partials, final + + partials, final = run(scenario()) + texts = [p.text for p in partials if p is not None] + assert texts == ["hello", "hello there", "hello there emet"] + assert partials[3] is None and partials[4] is None + assert final.text == "hello there emet" and final.final + + +# -------------------------------------------------------------- unhealthy + + +def test_an_unreachable_service_is_unhealthy_not_an_exception(): + stt = started(fail_on_start=True) + assert not stt.describe().healthy + health = stt.health() + assert not health.ok + assert "unreachable" in (health.detail or "") + + +def test_an_unhealthy_transcriber_hears_nothing(): + stt = started(fail_on_start=True) + assert run(stt.feed(frame(b"what time is it"))) is None + + +def test_a_missing_key_is_reported_by_the_name_of_the_variable(monkeypatch): + monkeypatch.delenv("EMET_MOCK_KEY", raising=False) + stt = MockTranscriber( + {"provider": "mock", "key_env": "EMET_MOCK_KEY", "params": {"require_key": True}}, + AudioFormat(), + ) + run(stt.start()) + assert not stt.describe().healthy + assert "EMET_MOCK_KEY" in (stt.health().detail or "") + + +def test_a_present_key_is_enough(monkeypatch): + monkeypatch.setenv("EMET_MOCK_KEY", "anything") + stt = MockTranscriber( + {"provider": "mock", "key_env": "EMET_MOCK_KEY", "params": {"require_key": True}}, + AudioFormat(), + ) + run(stt.start()) + assert stt.describe().healthy + + +def test_requiring_a_key_with_no_key_env_named_says_so(): + stt = started(require_key=True) + assert not stt.describe().healthy + assert "key_env" in (stt.health().detail or "") + + +def test_a_rate_override_shows_in_the_descriptor(): + """So a test above can watch the engine refuse a mismatch.""" + stt = started(sample_rate=8000) + assert stt.describe().sample_rate == 8000 + + +def test_shutdown_forgets_the_utterance_in_progress(): + stt = started() + run(stt.feed(frame(b"half a"))) + run(stt.shutdown()) + assert not stt.describe().healthy + assert run(stt.feed(frame(b"thought"))) is None + + +# ----------------------------------------------------------------- version + + +def test_the_reported_version_matches_the_installed_distribution(): + """One declaration, in `pyproject.toml`, read back at import time, like + the other three packages. See their tests for the drift that motivated + it.""" + from importlib.metadata import version + + import emet_providers + + assert emet_providers.__version__ == version("emet-providers") + assert emet_providers.__version__ != "0+unknown", "package is not installed" + + +# --------------------------------------------------------- language model + + +from emet_sdk.plugin import LanguageModelPlugin # noqa: E402 +from emet_sdk.types import Message, Prompt, ReplyDone, TextDelta, ToolCall, ToolSpec # noqa: E402 + +from emet_providers.mock import MockLanguageModel # noqa: E402 + + +def llm(**params) -> MockLanguageModel: + model = MockLanguageModel({"provider": "mock", "params": params}) + run(model.start()) + return model + + +async def collect(events) -> list: + return [e async for e in events] + + +def prompt(*texts: str, tools: tuple = ()) -> Prompt: + messages = tuple(Message(role="user" if i % 2 == 0 else "assistant", content=t) for i, t in enumerate(texts)) + return Prompt(system="Be brief.", messages=messages, tools=tools) + + +def test_the_mock_language_model_is_a_plugin_that_repeats_what_it_heard(): + model = llm() + assert isinstance(model, LanguageModelPlugin) + assert model.describe().healthy and model.describe().streaming and model.describe().tools + events = run(collect(model.reply(prompt("what time is it")))) + assert [e.text for e in events if isinstance(e, TextDelta)] == ["You", " said:", " what", " time", " is", " it"] + done = events[-1] + assert isinstance(done, ReplyDone) + assert done.text == "You said: what time is it" and done.stop_reason == "end" + assert done.model == "mock" and done.output_tokens == 6 + assert model.prompts[0].system == "Be brief." + + +def test_a_scripted_reply_is_said_whatever_was_asked(): + model = llm(reply="As you wish.") + done = run(collect(model.reply(prompt("anything"))))[-1] + assert done.text == "As you wish." + + +def test_the_mock_can_ask_for_a_tool_and_then_report_its_result(): + remember = ToolSpec(name="remember", description="keep a fact") + model = llm(call_tool={"name": "remember", "arguments": {"fact": "likes tea"}}) + first = run(collect(model.reply(prompt("remember I like tea", tools=(remember,))))) + call = first[0] + assert call == ToolCall(id="call_1", name="remember", arguments={"fact": "likes tea"}) + assert first[-1].stop_reason == "tool" and first[-1].tool_calls == (call,) + + followed = Prompt( + system="s", + messages=( + Message(role="user", content="remember I like tea"), + Message(role="assistant", content="", tool_calls=(call,)), + Message(role="tool", content="stored", tool_call_id=call.id), + ), + tools=(remember,), + ) + done = run(collect(model.reply(followed)))[-1] + assert done.text == "The tool said: stored" and done.stop_reason == "end" + + +def test_without_tools_on_offer_the_mock_answers_instead_of_calling(): + model = llm(call_tool={"name": "remember"}) + done = run(collect(model.reply(prompt("hello"))))[-1] + assert done.stop_reason == "end" and done.text == "You said: hello" + + +def test_an_unstarted_or_unreachable_mock_model_says_so(): + unreachable = llm(fail_on_start=True) + assert not unreachable.describe().healthy + (done,) = run(collect(unreachable.reply(prompt("q")))) + assert done.stop_reason == "error" and "not started" in (done.error or "") + + +def test_a_missing_key_is_reported_by_the_name_of_the_variable_for_the_model_too(monkeypatch): + monkeypatch.delenv("EMET_MOCK_KEY", raising=False) + model = MockLanguageModel({"provider": "mock", "key_env": "EMET_MOCK_KEY", "params": {"require_key": True}}) + run(model.start()) + assert not model.describe().healthy + assert "EMET_MOCK_KEY" in (model.health().detail or "") + + +# ------------------------------------------------------------------ voice + + +from emet_sdk.plugin import PluginError, VoicePlugin # noqa: E402 +from emet_sdk.types import VoiceDescriptor # noqa: E402 + +from emet_providers.mock import MockVoice, read_back # noqa: E402 + + +def voice(config: dict | None = None, soul_voice: dict | None = None, **params) -> MockVoice: + cfg: dict = {"provider": "mock", "params": params} + cfg.update(config or {}) + tts = MockVoice(cfg, soul_voice) + run(tts.start()) + return tts + + +async def audio(tts: MockVoice, text: str) -> list[bytes]: + return [c async for c in tts.speak(text)] + + +def test_the_mock_voice_is_a_voice_plugin_that_spells_the_words_into_the_audio(): + tts = voice() + assert isinstance(tts, VoicePlugin) + d = tts.describe() + assert isinstance(d, VoiceDescriptor) + assert d.healthy and d.streaming and d.sample_rate == 16000 and d.provider == "mock" + chunks = run(audio(tts, "what time is it")) + assert len(chunks) == 4, "one chunk a word, so the pipeline sees a stream" + assert all(len(c) % 2 == 0 and len(c) >= MockVoice.CHUNK_BYTES for c in chunks) + assert read_back(b"".join(chunks)) == "what time is it" + assert tts.said == ["what time is it"] + + +def test_read_back_survives_any_chunking_and_keeps_short_words(): + tts = voice() + pcm = b"".join(run(audio(tts, "I am a robot, it is true."))) + assert read_back(pcm) == "I am a robot, it is true." + assert read_back(pcm[:50] + pcm[50:]) == read_back(pcm) + assert read_back(bytes(100)) == "" + + +def test_the_sample_rate_is_whatever_the_body_says(): + assert voice(sample_rate=8000).describe().sample_rate == 8000 + + +def test_blank_text_makes_no_sound(): + tts = voice() + assert run(audio(tts, " ")) == [] + assert tts.said == [] + + +def test_a_latency_is_waited_before_the_first_chunk(): + import time + + tts = voice(latency_ms=30) + before = time.perf_counter() + run(audio(tts, "hi")) + assert time.perf_counter() - before >= 0.025 + + +def test_a_poisoned_word_fails_the_sentence_after_its_first_chunk(): + tts = voice(fail_on="Ω") + + async def scenario(): + got = [] + with pytest.raises(PluginError, match="simulated"): + async for chunk in tts.speak("the symbol Ω here"): + got.append(chunk) + return got + + got = run(scenario()) + assert [read_back(c) for c in got] == ["the", "symbol", "Ω".encode("ascii", "replace").decode()] + assert run(audio(tts, "fine again")), "the plugin is not broken" + + +def test_a_missing_model_or_key_is_unhealthy_with_the_reason(monkeypatch): + assert not voice(fail_on_start=True).describe().healthy + assert "missing" in (voice(fail_on_start=True).health().detail or "") + monkeypatch.delenv("EMET_MOCK_KEY", raising=False) + tts = voice(config={"key_env": "EMET_MOCK_KEY"}, require_key=True) + assert not tts.describe().healthy + assert "EMET_MOCK_KEY" in (tts.health().detail or "") + assert tts.health().faults == ("no_key",) + + +def test_an_unstarted_mock_voice_says_so(): + tts = MockVoice({"provider": "mock"}) + + async def scenario(): + with pytest.raises(PluginError, match="not started"): + await audio(tts, "hello") + + run(scenario()) + + +def test_the_souls_rate_is_carried(): + assert voice(soul_voice={"rate": 1.5}).rate == 1.5 diff --git a/emet-providers/tests/test_openai.py b/emet-providers/tests/test_openai.py new file mode 100644 index 0000000..25b7846 --- /dev/null +++ b/emet-providers/tests/test_openai.py @@ -0,0 +1,313 @@ +"""The OpenAI language model, against a stand-in for Chat Completions. + +No network and no key. An `httpx.MockTransport` answers in the shapes +OpenAI's streaming reference gives (verified 2026-09-15) and records what it +was sent. The same file stands for every server that speaks this protocol, +which is the reason the plugin speaks it. +""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any + +import pytest + +httpx = pytest.importorskip("httpx", reason="emet-providers[openai] is not installed") + +from emet_sdk.plugin import LanguageModelPlugin # noqa: E402 +from emet_sdk.types import Message, Prompt, ReplyDone, TextDelta, ToolCall, ToolSpec # noqa: E402 + +from emet_providers.openai import DEFAULT_KEY_ENV, DEFAULT_MODEL, DEFAULT_URL, OpenAILanguageModel # noqa: E402 + +KEY = "sk-test" + + +def run(coro): + return asyncio.run(coro) + + +async def collect(events) -> list: + return [e async for e in events] + + +# -------------------------------------------------------------- the fake + + +def chunk(delta: dict | None = None, *, finish: str | None = None, usage: dict | None = None, model: str = "gpt-5.6-terra") -> dict: + data: dict[str, Any] = {"id": "chatcmpl-1", "object": "chat.completion.chunk", "created": 1, "model": model, "choices": []} + if delta is not None or finish is not None: + data["choices"] = [{"index": 0, "delta": delta or {}, "finish_reason": finish}] + if usage is not None: + data["usage"] = usage + return data + + +def stream(*chunks: dict, done: bool = True) -> bytes: + text = "".join(f"data: {json.dumps(c)}\n\n" for c in chunks) + if done: + text += "data: [DONE]\n\n" + return text.encode() + + +def reply_stream(*pieces: str, finish: str = "stop", usage: bool = True) -> bytes: + chunks = [chunk({"role": "assistant", "content": ""})] + chunks += [chunk({"content": p}) for p in pieces] + chunks.append(chunk({}, finish=finish)) + if usage: + chunks.append(chunk(usage={"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13})) + return stream(*chunks) + + +class FakeOpenAI: + def __init__( + self, + body: bytes = b"", + *, + models_status: int = 200, + reply_status: int = 200, + reply_json: dict | None = None, + raise_on_connect: Exception | None = None, + ) -> None: + self.body = body + self.models_status = models_status + self.reply_status = reply_status + self.reply_json = reply_json + self.raise_on_connect = raise_on_connect + self.requests: list[httpx.Request] = [] + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + if self.raise_on_connect is not None: + raise self.raise_on_connect + if request.method == "GET" and request.url.path.endswith("/models"): + if self.models_status == 200: + return httpx.Response(200, json={"object": "list", "data": [{"id": "gpt-5.6-terra", "object": "model"}]}) + return httpx.Response(self.models_status, json={"error": {"message": "Incorrect API key provided", "type": "invalid_request_error", "code": "invalid_api_key"}}) + if request.method == "POST" and request.url.path.endswith("/chat/completions"): + if self.reply_status != 200: + return httpx.Response(self.reply_status, json=self.reply_json or {"error": {"message": "boom", "type": "server_error"}}) + return httpx.Response(200, content=self.body, headers={"content-type": "text/event-stream"}) + return httpx.Response(404, json={"error": {"message": request.url.path, "type": "not_found"}}) + + @property + def posts(self) -> list[httpx.Request]: + return [r for r in self.requests if r.method == "POST"] + + +def make(fake: FakeOpenAI, *, config: dict | None = None, **params: Any) -> OpenAILanguageModel: + cfg: dict[str, Any] = {"provider": "openai", "params": params} + cfg.update(config or {}) + return OpenAILanguageModel(cfg, transport=httpx.MockTransport(fake)) + + +def started(fake: FakeOpenAI, monkeypatch, **kw) -> OpenAILanguageModel: + monkeypatch.setenv(DEFAULT_KEY_ENV, KEY) + llm = make(fake, **kw) + run(llm.start()) + assert llm.describe().healthy, llm.health().detail + return llm + + +def body_of(request: httpx.Request) -> dict: + return json.loads(request.content) + + +def prompt(*texts: str, system: str = "Be brief.", tools: tuple = ()) -> Prompt: + messages = tuple(Message(role="user" if i % 2 == 0 else "assistant", content=t) for i, t in enumerate(texts)) + return Prompt(system=system, messages=messages, tools=tools, max_tokens=300) + + +# --------------------------------------------------------------- request + + +def test_it_is_a_language_model_plugin_registered_as_openai(): + llm = make(FakeOpenAI()) + assert isinstance(llm, LanguageModelPlugin) + assert llm.provider == "openai" + assert llm.url == DEFAULT_URL + + +def test_the_request_carries_a_bearer_token_and_the_documented_fields(monkeypatch): + fake = FakeOpenAI(reply_stream("hi")) + llm = started(fake, monkeypatch) + run(collect(llm.reply(prompt("hello")))) + (post,) = fake.posts + assert post.headers["authorization"] == f"Bearer {KEY}" + assert str(post.url) == "https://api.openai.com/v1/chat/completions" + body = body_of(post) + assert body["model"] == DEFAULT_MODEL + assert body["stream"] is True + assert body["stream_options"] == {"include_usage": True} + assert body["max_completion_tokens"] == 300 + assert "max_tokens" not in body + assert body["messages"] == [{"role": "system", "content": "Be brief."}, {"role": "user", "content": "hello"}] + assert "tools" not in body and "temperature" not in body + + +def test_an_older_compatible_server_can_be_given_max_tokens(monkeypatch): + fake = FakeOpenAI(reply_stream("hi")) + llm = started(fake, monkeypatch, url="http://localhost:11434/v1", max_tokens_field="max_tokens") + run(collect(llm.reply(prompt("hello")))) + post = fake.posts[0] + assert str(post.url) == "http://localhost:11434/v1/chat/completions" + body = body_of(post) + assert body["max_tokens"] == 300 and "max_completion_tokens" not in body + + +def test_the_souls_model_and_an_organization_are_sent(monkeypatch): + fake = FakeOpenAI(reply_stream("hi")) + llm = started(fake, monkeypatch, config={"model": "gpt-5.6-luna"}, organization="org-1") + run(collect(llm.reply(prompt("hello")))) + post = fake.posts[0] + assert body_of(post)["model"] == "gpt-5.6-luna" + assert post.headers["openai-organization"] == "org-1" + assert llm.describe().model == "gpt-5.6-luna" + + +def test_tools_and_tool_traffic_take_the_vendors_shape(monkeypatch): + fake = FakeOpenAI(reply_stream("done")) + llm = started(fake, monkeypatch) + remember = ToolSpec(name="remember", description="keep a fact", parameters={"type": "object", "properties": {"fact": {"type": "string"}}}) + call = ToolCall(id="call_1", name="remember", arguments={"fact": "likes tea"}) + messages = ( + Message(role="user", content="remember I like tea"), + Message(role="assistant", content="", tool_calls=(call,)), + Message(role="tool", content="stored", tool_call_id="call_1"), + ) + run(collect(llm.reply(Prompt(system="s", messages=messages, tools=(remember,))))) + body = body_of(fake.posts[0]) + assert body["tools"] == [{"type": "function", "function": {"name": "remember", "description": "keep a fact", "parameters": remember.parameters}}] + assert body["messages"] == [ + {"role": "system", "content": "s"}, + {"role": "user", "content": "remember I like tea"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "remember", "arguments": json.dumps({"fact": "likes tea"})}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "stored"}, + ] + + +# ----------------------------------------------------------------- start + + +def test_preflight_lists_models_once(monkeypatch): + fake = FakeOpenAI() + started(fake, monkeypatch) + assert [(r.method, r.url.path) for r in fake.requests] == [("GET", "/v1/models")] + assert fake.requests[0].headers["authorization"] == f"Bearer {KEY}" + + +def test_no_key_is_unhealthy_names_the_variable_and_sends_nothing(monkeypatch): + monkeypatch.delenv(DEFAULT_KEY_ENV, raising=False) + fake = FakeOpenAI() + llm = make(fake) + run(llm.start()) + assert not llm.describe().healthy + assert DEFAULT_KEY_ENV in (llm.health().detail or "") + assert fake.requests == [] + + +def test_a_rejected_key_is_unhealthy_with_the_status(monkeypatch): + monkeypatch.setenv(DEFAULT_KEY_ENV, "wrong") + llm = make(FakeOpenAI(models_status=401)) + run(llm.start()) + detail = llm.health().detail or "" + assert not llm.describe().healthy + assert "401" in detail and DEFAULT_KEY_ENV in detail and "rejected" in detail + + +def test_an_unreachable_server_is_unhealthy_and_names_it(monkeypatch): + monkeypatch.setenv(DEFAULT_KEY_ENV, KEY) + llm = make(FakeOpenAI(raise_on_connect=httpx.ConnectError("refused")), url="http://localhost:11434/v1") + run(llm.start()) + detail = llm.health().detail or "" + assert not llm.describe().healthy + assert "localhost:11434" in detail and "could not reach" in detail + + +def test_without_httpx_the_hint_names_the_extra(monkeypatch): + import sys + + monkeypatch.setenv(DEFAULT_KEY_ENV, KEY) + monkeypatch.setitem(sys.modules, "httpx", None) + llm = OpenAILanguageModel({"provider": "openai"}) + run(llm.start()) + assert not llm.describe().healthy + assert "emet-providers[openai]" in (llm.health().detail or "") + + +# ----------------------------------------------------------------- reply + + +def test_text_streams_as_deltas_and_the_final_carries_usage(monkeypatch): + llm = started(FakeOpenAI(reply_stream("Hello", ", world", ".")), monkeypatch) + events = run(collect(llm.reply(prompt("hi")))) + assert [e.text for e in events if isinstance(e, TextDelta)] == ["Hello", ", world", "."] + done = events[-1] + assert isinstance(done, ReplyDone) + assert done.text == "Hello, world." and done.stop_reason == "end" + assert done.model == "gpt-5.6-terra" + assert (done.input_tokens, done.output_tokens) == (9, 4) + + +def test_tool_calls_are_accumulated_by_index_across_chunks(monkeypatch): + body = stream( + chunk({"role": "assistant", "content": None}), + chunk({"tool_calls": [{"index": 0, "id": "call_a", "type": "function", "function": {"name": "remember", "arguments": ""}}]}), + chunk({"tool_calls": [{"index": 0, "function": {"arguments": '{"fact": "li'}}]}), + chunk({"tool_calls": [{"index": 1, "id": "call_b", "type": "function", "function": {"name": "lookup", "arguments": '{"q": 1}'}}]}), + chunk({"tool_calls": [{"index": 0, "function": {"arguments": 'kes tea"}'}}]}), + chunk({}, finish="tool_calls"), + chunk(usage={"prompt_tokens": 5, "completion_tokens": 9}), + ) + llm = started(FakeOpenAI(body), monkeypatch) + events = run(collect(llm.reply(prompt("remember I like tea")))) + calls = [e for e in events if isinstance(e, ToolCall)] + assert calls == [ + ToolCall(id="call_a", name="remember", arguments={"fact": "likes tea"}), + ToolCall(id="call_b", name="lookup", arguments={"q": 1}), + ] + done = events[-1] + assert done.stop_reason == "tool" and done.tool_calls == tuple(calls) and done.text == "" + + +@pytest.mark.parametrize( + ("vendor", "seam"), + [("stop", "end"), ("length", "length"), ("tool_calls", "tool"), ("content_filter", "refusal"), ("function_call", "tool"), ("novel", "end")], +) +def test_finish_reasons_are_translated(monkeypatch, vendor, seam): + llm = started(FakeOpenAI(reply_stream("x", finish=vendor)), monkeypatch) + assert run(collect(llm.reply(prompt("q"))))[-1].stop_reason == seam + + +def test_a_stream_without_done_or_usage_still_finishes(monkeypatch): + body = stream(chunk({"content": "ok"}), chunk({}, finish="stop"), done=False) + llm = started(FakeOpenAI(body), monkeypatch) + done = run(collect(llm.reply(prompt("q"))))[-1] + assert done.text == "ok" and done.stop_reason == "end" and done.output_tokens is None + + +def test_an_http_error_ends_the_reply_with_the_vendors_message(monkeypatch): + fake = FakeOpenAI(reply_status=429, reply_json={"error": {"message": "Rate limit reached", "type": "rate_limit_error", "code": "rate_limit_exceeded"}}) + llm = started(fake, monkeypatch) + (done,) = run(collect(llm.reply(prompt("q")))) + assert done.stop_reason == "error" + assert "429" in (done.error or "") and "Rate limit reached" in (done.error or "") + assert llm.describe().healthy + + +def test_an_error_object_in_the_stream_ends_the_reply(monkeypatch): + body = stream(chunk({"content": "Half"}), {"error": {"message": "The server had an error", "type": "server_error"}}) + llm = started(FakeOpenAI(body), monkeypatch) + done = run(collect(llm.reply(prompt("q"))))[-1] + assert done.text == "Half" and done.stop_reason == "error" + assert "server_error" in (done.error or "") + + +def test_replying_before_start_is_an_error_final_not_an_exception(): + (done,) = run(collect(make(FakeOpenAI()).reply(prompt("q")))) + assert done.stop_reason == "error" and "not started" in (done.error or "") diff --git a/emet-providers/tests/test_piper.py b/emet-providers/tests/test_piper.py new file mode 100644 index 0000000..e22cc8e --- /dev/null +++ b/emet-providers/tests/test_piper.py @@ -0,0 +1,335 @@ +"""The Piper voice, against a stand-in for the Piper library. + +No model on disk, no ONNX Runtime, no GPL code imported. A fake `piper` +module is put in `sys.modules` with the two names the plugin uses, +`PiperVoice.load` and `SynthesisConfig`, shaped as `piper-tts` 1.8.0 has +them (read from the wheel, 2026-09-15). What these tests prove is the +plugin's half: where it looks for a voice, what it says when there is none, +how the soul's rate becomes Piper's length scale, and that a sentence comes +back as int16 audio at the model's rate. A run against the real library is +by hand, on a machine that has it. +""" + +from __future__ import annotations + +import asyncio +import sys +import types +from pathlib import Path +from typing import Any + +import pytest + +from emet_sdk.plugin import PluginError, VoicePlugin + +from emet_providers import piper as plugin +from emet_providers.piper import DEFAULT_MODEL, WARM_TEXT, PiperVoice, download_hint, voice_paths + + +def run(coro): + return asyncio.run(coro) + + +async def collect(chunks) -> list[bytes]: + return [c async for c in chunks] + + +# -------------------------------------------------------------- the fake + + +class _Chunk: + def __init__(self, text: str, rate: int) -> None: + self.sample_rate = rate + self.sample_width = 2 + self.sample_channels = 1 + self.audio_int16_bytes = text.encode("ascii", "replace") + bytes(2) + + +class _FakeVoice: + """Shaped like `piper.PiperVoice`: `config.sample_rate`, `synthesize()` + yielding one chunk per sentence.""" + + loaded: list[tuple[str, str | None, bool]] = [] + rate = 22050 + fail_on: str | None = None + + def __init__(self) -> None: + self.config = types.SimpleNamespace(sample_rate=self.rate) + self.calls: list[tuple[str, Any]] = [] + + @staticmethod + def load(model_path, config_path=None, use_cuda=False, **_): + _FakeVoice.loaded.append((str(model_path), str(config_path) if config_path else None, use_cuda)) + return _FakeVoice() + + def synthesize(self, text: str, syn_config=None, include_alignments=False): + self.calls.append((text, syn_config)) + if self.fail_on and self.fail_on in text: + raise RuntimeError("phonemizer choked") + for sentence in [s for s in text.replace("?", ".").replace("!", ".").split(".") if s.strip()]: + yield _Chunk(sentence.strip(), self.rate) + + +class _SynthesisConfig: + def __init__(self, **kw: Any) -> None: + self.kw = kw + + +def fake_piper(monkeypatch, *, rate: int = 22050, fail_on: str | None = None) -> types.ModuleType: + module = types.ModuleType("piper") + _FakeVoice.loaded = [] + _FakeVoice.rate = rate + _FakeVoice.fail_on = fail_on + # A fresh subclass per test, so a test that breaks `load` breaks its own. + module.PiperVoice = type("PiperVoice", (_FakeVoice,), {}) # type: ignore[attr-defined] + module.SynthesisConfig = _SynthesisConfig # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "piper", module) + return module + + +def voice_files(directory: Path, model: str = DEFAULT_MODEL) -> Path: + directory.mkdir(parents=True, exist_ok=True) + onnx = directory / f"{model}.onnx" + onnx.write_bytes(b"not really a model") + (directory / f"{model}.onnx.json").write_text('{"audio": {"sample_rate": 22050}}', encoding="utf-8") + return onnx + + +def make(voices_dir: Path | None = None, *, config: dict | None = None, voice: dict | None = None, **params) -> PiperVoice: + if voices_dir is not None: + params["voices_dir"] = str(voices_dir) + cfg: dict[str, Any] = {"provider": "piper", "params": params} + cfg.update(config or {}) + return PiperVoice(cfg, voice) + + +def started(monkeypatch, tmp_path, **kw) -> PiperVoice: + fake_piper(monkeypatch) + voice_files(tmp_path / "voices") + tts = make(tmp_path / "voices", **kw) + run(tts.start()) + assert tts.describe().healthy, tts.health().detail + return tts + + +# ---------------------------------------------------------------- lookup + + +def test_it_is_a_voice_plugin_registered_as_piper(): + tts = make() + assert isinstance(tts, VoicePlugin) + assert tts.provider == "piper" + assert tts.model_name == DEFAULT_MODEL + + +def test_the_default_voice_is_public_domain_ljspeech(): + """The reference voice has to be one a shipped default may be. Amy, the + earlier choice, is fine-tuned from a corpus licensed for research only.""" + assert DEFAULT_MODEL == "en_US-ljspeech-medium" + + +def test_voices_are_looked_for_in_the_documented_places(monkeypatch, tmp_path): + monkeypatch.setenv("XDG_DATA_HOME", str(tmp_path / "xdg")) + looked = voice_paths("en_US-ljspeech-medium", {}) + assert looked == [ + tmp_path / "xdg" / "emet" / "voices" / "en_US-ljspeech-medium.onnx", + Path("/etc/emet/voices/en_US-ljspeech-medium.onnx"), + ] + with_dir = voice_paths("v", {"voices_dir": str(tmp_path / "mine")}) + assert with_dir[0] == tmp_path / "mine" / "v.onnx" + assert voice_paths("v", {"path": str(tmp_path / "x.onnx")}) == [tmp_path / "x.onnx"] + + +def test_a_missing_model_is_unhealthy_and_names_the_download_command(monkeypatch, tmp_path): + fake_piper(monkeypatch) + tts = make(tmp_path / "empty") + run(tts.start()) + assert not tts.describe().healthy + detail = tts.health().detail or "" + assert "no voice model" in detail + assert DEFAULT_MODEL in detail + assert download_hint(DEFAULT_MODEL) in detail + assert "piper.download_voices" in detail + assert "never fetched at boot" in detail + assert tts.health().faults == ("no_model",) + assert _FakeVoice.loaded == [], "nothing was loaded" + + +def test_a_model_without_its_config_says_so(monkeypatch, tmp_path): + fake_piper(monkeypatch) + directory = tmp_path / "voices" + directory.mkdir() + (directory / f"{DEFAULT_MODEL}.onnx").write_bytes(b"model") + tts = make(directory) + run(tts.start()) + assert not tts.describe().healthy + assert ".onnx.json" in (tts.health().detail or "") + + +def test_without_the_library_the_hint_names_the_extra_and_the_licence(monkeypatch, tmp_path): + monkeypatch.setitem(sys.modules, "piper", None) # import raises ImportError + tts = make(tmp_path) + run(tts.start()) + assert not tts.describe().healthy + detail = tts.health().detail or "" + assert "emet-providers[piper]" in detail + assert "GPL" in detail + assert tts.health().faults == ("no_library",) + + +def test_the_library_is_checked_before_the_disk(monkeypatch, tmp_path): + """An owner without the extra gets the install hint, whatever is on disk.""" + monkeypatch.setitem(sys.modules, "piper", None) + voice_files(tmp_path / "voices") + tts = make(tmp_path / "voices") + run(tts.start()) + assert "emet-providers[piper]" in (tts.health().detail or "") + + +# ----------------------------------------------------------------- loading + + +def test_the_model_and_its_config_are_loaded_from_where_they_were_found(monkeypatch, tmp_path): + tts = started(monkeypatch, tmp_path) + onnx = tmp_path / "voices" / f"{DEFAULT_MODEL}.onnx" + assert _FakeVoice.loaded == [(str(onnx), str(onnx) + ".json", False)] + assert tts.path == onnx + + +def test_an_explicit_path_wins(monkeypatch, tmp_path): + fake_piper(monkeypatch) + onnx = voice_files(tmp_path / "elsewhere", "en_GB-alan-low") + tts = make(path=str(onnx)) + run(tts.start()) + assert tts.describe().healthy + assert tts.path == onnx + assert tts.model_name == DEFAULT_MODEL, "the name is for the report; the file is what was loaded" + + +def test_the_souls_model_is_looked_for_and_reported(monkeypatch, tmp_path): + fake_piper(monkeypatch) + voice_files(tmp_path / "voices", "en_US-ryan-high") + tts = make(tmp_path / "voices", config={"model": "en_US-ryan-high"}) + run(tts.start()) + assert tts.describe().healthy and tts.describe().model == "en_US-ryan-high" + + +def test_describe_reports_the_models_own_sample_rate(monkeypatch, tmp_path): + fake_piper(monkeypatch, rate=16000) + voice_files(tmp_path / "voices") + tts = make(tmp_path / "voices") + run(tts.start()) + d = tts.describe() + assert d.sample_rate == 16000 + assert not d.streaming, "a sentence arrives whole" + assert d.provider == "piper" + + +def test_cuda_is_off_unless_asked(monkeypatch, tmp_path): + fake_piper(monkeypatch) + voice_files(tmp_path / "voices") + tts = make(tmp_path / "voices", use_cuda=True) + run(tts.start()) + assert _FakeVoice.loaded[0][2] is True + + +def test_a_model_that_will_not_load_is_unhealthy_with_the_reason(monkeypatch, tmp_path): + module = fake_piper(monkeypatch) + + def explode(*a, **k): + raise RuntimeError("bad protobuf") + + module.PiperVoice.load = staticmethod(explode) # type: ignore[attr-defined] + voice_files(tmp_path / "voices") + tts = make(tmp_path / "voices") + run(tts.start()) + assert not tts.describe().healthy + assert "bad protobuf" in (tts.health().detail or "") + + +# ----------------------------------------------------------------- speaking + + +def test_a_sentence_comes_back_as_audio_one_chunk_per_sentence(monkeypatch, tmp_path): + tts = started(monkeypatch, tmp_path) + chunks = run(collect(tts.speak("What time is it? It is noon."))) + assert chunks == [b"What time is it\x00\x00", b"It is noon\x00\x00"] + + +def test_the_phonemiser_is_warmed_at_boot_and_can_be_left_cold(monkeypatch, tmp_path): + """The first sentence through a cold Piper cost 3.7 s more than the + second on the laptop (2026-09-15); that wait belongs at boot.""" + tts = started(monkeypatch, tmp_path) + assert [text for text, _ in tts._voice.calls] == [WARM_TEXT] + fake_piper(monkeypatch) + cold = make(tmp_path / "voices", warm=False) + run(cold.start()) + assert cold.describe().healthy and cold._voice.calls == [] + + +def test_a_voice_that_cannot_say_the_warm_word_is_unhealthy(monkeypatch, tmp_path): + fake_piper(monkeypatch, fail_on=WARM_TEXT) + voice_files(tmp_path / "voices") + tts = make(tmp_path / "voices") + run(tts.start()) + assert not tts.describe().healthy + assert "could not synthesise" in (tts.health().detail or "") + + +def test_blank_text_is_not_synthesised(monkeypatch, tmp_path): + tts = started(monkeypatch, tmp_path, warm=False) + assert run(collect(tts.speak(" \n"))) == [] + assert tts._voice.calls == [] + + +def test_the_souls_rate_becomes_the_inverse_length_scale(monkeypatch, tmp_path): + """Piper counts phoneme length, so faster is a smaller number.""" + tts = started(monkeypatch, tmp_path, voice={"rate": 1.25}, warm=False) + run(collect(tts.speak("Hello."))) + (_, config), = tts._voice.calls + assert config.kw["length_scale"] == pytest.approx(0.8) + + +def test_the_bodys_knobs_ride_along(monkeypatch, tmp_path): + tts = started(monkeypatch, tmp_path, noise_scale=0.5, volume=0.8, warm=False) + run(collect(tts.speak("Hello."))) + (_, config), = tts._voice.calls + assert config.kw == {"length_scale": 1.0, "noise_scale": 0.5, "volume": 0.8} + + +def test_a_sentence_the_engine_chokes_on_is_one_sentence_lost(monkeypatch, tmp_path): + fake_piper(monkeypatch, fail_on="Ω") + voice_files(tmp_path / "voices") + tts = make(tmp_path / "voices") + run(tts.start()) + with pytest.raises(PluginError, match="phonemizer choked"): + run(collect(tts.speak("The symbol Ω."))) + assert tts.last_error and "phonemizer choked" in tts.last_error + assert tts.describe().healthy, "a bad sentence is not a broken plugin" + assert run(collect(tts.speak("Fine again."))) == [b"Fine again\x00\x00"] + + +def test_speaking_before_start_is_an_error(): + tts = make() + + async def scenario(): + with pytest.raises(PluginError, match="not started"): + await collect(tts.speak("Hello.")) + + run(scenario()) + + +def test_shutdown_forgets_the_model(monkeypatch, tmp_path): + tts = started(monkeypatch, tmp_path) + run(tts.shutdown()) + assert not tts.describe().healthy + assert tts._voice is None + + +def test_the_module_names_its_licence_and_packaging_facts(): + """The docstring is where the verification lives; a future reader should + find the date and the licence without leaving the file.""" + doc = plugin.__doc__ or "" + assert "GPL-3.0-or-later" in doc + assert "2026-09-15" in doc + assert "abi3" in doc and "aarch64" in doc diff --git a/emet-providers/tests/test_tts_live.py b/emet-providers/tests/test_tts_live.py new file mode 100644 index 0000000..cd1e986 --- /dev/null +++ b/emet-providers/tests/test_tts_live.py @@ -0,0 +1,49 @@ +"""One real sentence from the cloud voice, when a key is present and asked for. + +Skipped in CI and on any machine without `EMET_LIVE_TESTS=1` and the key in +the environment. It spends a few dozen characters. What it proves is the +half the stand-in cannot: that the URL, the query and the header are what +the live service accepts today, and that raw linear16 comes back at the +rate asked for. Nothing is played; the audio is counted and its length +checked against the rate. Run by hand before a release, and whenever the +vendor announces a change. +""" + +from __future__ import annotations + +import asyncio +import os + +import pytest + +LIVE = os.environ.get("EMET_LIVE_TESTS") == "1" + +SENTENCE = "Emet is ready." + + +@pytest.mark.skipif( + not (LIVE and os.environ.get("EMET_DEEPGRAM_KEY")), + reason="set EMET_DEEPGRAM_KEY and EMET_LIVE_TESTS=1 to call Deepgram for real", +) +def test_deepgram_speaks(): + pytest.importorskip("httpx", reason="emet-providers[deepgram] is not installed") + from emet_providers.deepgram_voice import DeepgramVoice + + tts = DeepgramVoice({"provider": "deepgram", "params": {"sample_rate": 24000}}) + + async def scenario(): + await tts.start() + assert tts.describe().healthy, tts.health().detail + chunks = [c async for c in tts.speak(SENTENCE)] + await tts.shutdown() + return chunks + + chunks = asyncio.run(scenario()) + pcm = b"".join(chunks) + assert chunks, "no audio came back" + assert all(len(c) % 2 == 0 for c in chunks) + seconds = len(pcm) / 2 / 24000 + # Three words take about a second to say; a header, or the wrong rate, + # would put this far outside the range. + assert 0.4 < seconds < 4.0, f"{seconds:.2f} s of audio for {SENTENCE!r}" + assert tts.last_error is None, tts.last_error diff --git a/emet-sdk/README.md b/emet-sdk/README.md index 8dae4df..9449ed2 100644 --- a/emet-sdk/README.md +++ b/emet-sdk/README.md @@ -20,11 +20,12 @@ it deliberately contains almost no logic. | | | |---|---| | `schemas/` | Body manifest, soul bundle, and motion pack, as JSON Schema. The **full** surface: every P0 field, every RSV field reserved for later releases, and the reserved V1 capability types. | -| `emet_sdk/types.py` | `Intent`, `Action`, `Pose`, `Twist`, `CapabilityDescriptor`, `LocomotionDescriptor`, `WakeDescriptor`, `AudioFormat`, `AudioSource`, `AudioSink`, `Health`, `Priority`, `Sensitivity`. | +| `emet_sdk/types.py` | `Intent`, `Action`, `Pose`, `Twist`, `CapabilityDescriptor`, `LocomotionDescriptor`, `WakeDescriptor`, `Transcript`, `TranscriberDescriptor`, `Prompt`, `Message`, `ToolSpec`, `ToolCall`, `TextDelta`, `ReplyDone`, `LanguageModelDescriptor`, `VoiceDescriptor`, `AudioFormat`, `AudioSource`, `AudioSink`, `Health`, `Priority`, `Sensitivity`. | | `emet_sdk/intents.py` | The closed intent vocabulary, plus the four names reserved from P0. | | `emet_sdk/chains.py` | Fallback chain format, and the rule that every chain terminates in a voice rung. | -| `emet_sdk/plugin.py` | `CapabilityPlugin` and its subclasses `ActuatorPlugin`, `SensorPlugin`, `LocomotionPlugin`, plus `WakePlugin`: the public contract. | -| `emet_sdk/discovery.py` | Entry-point discovery across six groups. Installing a package is what makes a driver exist. | +| `emet_sdk/plugin.py` | `CapabilityPlugin` and its subclasses `ActuatorPlugin`, `SensorPlugin`, `LocomotionPlugin`, plus `WakePlugin`, `TranscriberPlugin`, `LanguageModelPlugin` and `VoicePlugin`: the public contract. | +| `emet_sdk/discovery.py` | Entry-point discovery across nine groups. Installing a package is what makes a driver exist. | +| `emet_sdk/models.py` | Which provider a body and a soul agree on, per stage: one rule for speech recognition, the language model, the voice, and the stage still reserved. | | `emet_sdk/resolve.py` | Chain resolution: `(chains, descriptors) → binding table`. | | `emet_sdk/validate.py` | Semantic rules and the error taxonomy. | | `emet_sdk/cli.py` | `emet validate`, `emet explain`. | diff --git a/emet-sdk/emet_sdk/__init__.py b/emet-sdk/emet_sdk/__init__.py index a0b25c4..9dc0f5a 100644 --- a/emet-sdk/emet_sdk/__init__.py +++ b/emet-sdk/emet_sdk/__init__.py @@ -6,8 +6,8 @@ from emet_sdk import Intent, Action, CapabilityDescriptor -Layering, enforced in CI: `emet_sdk` imports nothing internal. `emet_hal` -imports `emet_sdk` only. `emet_engine` imports `emet_sdk` only. +Layering, enforced in CI: `emet_sdk` imports nothing internal. `emet_hal`, +`emet_providers` and `emet_engine` import `emet_sdk` only. """ from importlib import metadata as _metadata @@ -16,8 +16,11 @@ ActuatorPlugin, CapabilityPlugin, LocomotionPlugin, + LanguageModelPlugin, PluginError, SensorPlugin, + TranscriberPlugin, + VoicePlugin, WakePlugin, ) from emet_sdk.types import ( @@ -27,15 +30,26 @@ CapabilityDescriptor, Health, Intent, + LanguageModelDescriptor, LocomotionDescriptor, MemoryKind, + Message, Pose, Priority, + Prompt, Reading, + ReplyDone, + ReplyEvent, Sensitivity, Target, TargetKind, + TextDelta, + ToolCall, + ToolSpec, + Transcript, + TranscriberDescriptor, Twist, + VoiceDescriptor, WakeDescriptor, WakeEvent, ) @@ -68,18 +82,32 @@ "PluginError", "Reading", "SensorPlugin", + "TranscriberPlugin", + "LanguageModelPlugin", + "VoicePlugin", "WakePlugin", "CapabilityDescriptor", "Health", "Intent", + "LanguageModelDescriptor", "LocomotionDescriptor", "MemoryKind", + "Message", "Pose", "Priority", + "Prompt", + "ReplyDone", + "ReplyEvent", "Sensitivity", "Target", "TargetKind", + "TextDelta", + "ToolCall", + "ToolSpec", + "Transcript", + "TranscriberDescriptor", "Twist", + "VoiceDescriptor", "WakeDescriptor", "WakeEvent", ] diff --git a/emet-sdk/emet_sdk/discovery.py b/emet-sdk/emet_sdk/discovery.py index b64b359..125086f 100644 --- a/emet-sdk/emet_sdk/discovery.py +++ b/emet-sdk/emet_sdk/discovery.py @@ -5,7 +5,7 @@ engine has no list of known drivers compiled into it; installing a package is what makes a driver exist. -Six groups: +Nine groups: emet.actuators name = the string used in `driver.plugin` emet.sensors name = the string used in `driver.plugin` @@ -13,6 +13,12 @@ emet.wake name = the string used in `audio.wake.engine` emet.audio name = the string used in `audio.input.source` emet.audio_out name = the string used in `audio.output.sink` + emet.stt name = the string used in `models.stt.provider`, on the + soul or on the body + emet.llm name = the string used in `models.chat.provider`, on the + soul or on the body + emet.tts name = the string used in `models.tts.provider`, on the + soul or on the body `emet.audio` exists for a reason the others do not share. The engine may import `emet_sdk` and nothing else, so it cannot reach into `emet_hal` for a @@ -29,6 +35,14 @@ which is what makes those enums genuinely open: `kinematics: legged` is legal today and resolves the moment somebody publishes a package registering `legged`. +`emet.stt`, `emet.llm` and `emet.tts` are the groups whose names come from +the *soul* rather than the body: `models.stt.provider`, `models.chat.provider` +and `models.tts.provider`. Each is an account or a voice the owner brings, +and that travels with the soul, so the soul names it. The body may override +any of them, and a test rig does. Either way the name resolves here, against +what is installed, and a provider nobody has packaged yet is a +`MissingPluginError` on the day it is named, exactly like `legged`. + That openness is not theoretical for wake. Picovoice disabled every free Porcupine access key on 30 June 2026, and every project that had wired one detector in directly stopped waking. Here it would have been one line of a @@ -55,6 +69,9 @@ "GROUP_WAKE", "GROUP_AUDIO", "GROUP_AUDIO_OUT", + "GROUP_STT", + "GROUP_LLM", + "GROUP_TTS", "PluginRegistry", "discover", ] @@ -65,6 +82,9 @@ GROUP_WAKE = "emet.wake" GROUP_AUDIO = "emet.audio" GROUP_AUDIO_OUT = "emet.audio_out" +GROUP_STT = "emet.stt" +GROUP_LLM = "emet.llm" +GROUP_TTS = "emet.tts" def _entry_points(group: str) -> dict[str, EntryPoint]: @@ -87,6 +107,9 @@ def __init__( wake: Mapping[str, EntryPoint] | None = None, audio: Mapping[str, EntryPoint] | None = None, audio_out: Mapping[str, EntryPoint] | None = None, + stt: Mapping[str, EntryPoint] | None = None, + llm: Mapping[str, EntryPoint] | None = None, + tts: Mapping[str, EntryPoint] | None = None, *, verify_drivers: bool = False, ) -> None: @@ -96,6 +119,9 @@ def __init__( self._wake = dict(wake or {}) self._audio = dict(audio or {}) self._audio_out = dict(audio_out or {}) + self._stt = dict(stt or {}) + self._llm = dict(llm or {}) + self._tts = dict(tts or {}) #: When False, an unrecognised *driver* name is reported as a warning #: rather than an error. See `validate` for why the two callers differ: #: linting a manifest for hardware you have not wired yet is a normal @@ -113,6 +139,9 @@ def discover(cls) -> "PluginRegistry": wake=_entry_points(GROUP_WAKE), audio=_entry_points(GROUP_AUDIO), audio_out=_entry_points(GROUP_AUDIO_OUT), + stt=_entry_points(GROUP_STT), + llm=_entry_points(GROUP_LLM), + tts=_entry_points(GROUP_TTS), ) def with_verification(self, verify_drivers: bool) -> "PluginRegistry": @@ -128,6 +157,9 @@ def with_verification(self, verify_drivers: bool) -> "PluginRegistry": wake=self._wake, audio=self._audio, audio_out=self._audio_out, + stt=self._stt, + llm=self._llm, + tts=self._tts, verify_drivers=verify_drivers, ) @@ -148,6 +180,15 @@ def has_audio(self, source: str) -> bool: def has_audio_out(self, sink: str) -> bool: return sink in self._audio_out + def has_stt(self, provider: str) -> bool: + return provider in self._stt + + def has_llm(self, provider: str) -> bool: + return provider in self._llm + + def has_tts(self, provider: str) -> bool: + return provider in self._tts + @property def driver_names(self) -> list[str]: return sorted({*self._actuators, *self._sensors}) @@ -168,6 +209,18 @@ def audio_names(self) -> list[str]: def audio_out_names(self) -> list[str]: return sorted(self._audio_out) + @property + def stt_names(self) -> list[str]: + return sorted(self._stt) + + @property + def llm_names(self) -> list[str]: + return sorted(self._llm) + + @property + def tts_names(self) -> list[str]: + return sorted(self._tts) + def __bool__(self) -> bool: return bool( self._actuators @@ -176,6 +229,9 @@ def __bool__(self) -> bool: or self._wake or self._audio or self._audio_out + or self._stt + or self._llm + or self._tts ) def __iter__(self) -> Iterator[tuple[str, str]]: @@ -192,6 +248,12 @@ def __iter__(self) -> Iterator[tuple[str, str]]: yield ("audio", name) for name in sorted(self._audio_out): yield ("audio_out", name) + for name in sorted(self._stt): + yield ("stt", name) + for name in sorted(self._llm): + yield ("llm", name) + for name in sorted(self._tts): + yield ("tts", name) # ----------------------------------------------------------------- load @@ -241,6 +303,44 @@ def load_audio_out(self, sink: str) -> type: raise _missing(sink, self.audio_out_names, "audio sink") return ep.load() + def load_stt(self, provider: str) -> type: + """Import and return the plugin class for a speech recognition provider. + + Constructed as `cls(config, fmt)`, where `config` is the merged + provider reference from `emet_sdk.models.stt_selection` and `fmt` the + `AudioFormat` the wake engine fixed. Same shape as a source or a sink, + for the same reason: the engine holds a name and a mapping. + """ + ep = self._stt.get(provider) + if ep is None: + raise _missing(provider, self.stt_names, "speech recognition provider") + return ep.load() + + def load_llm(self, provider: str) -> type: + """Import and return the plugin class for a language model provider. + + Constructed as `cls(config)`, where `config` is the merged provider + reference from `emet_sdk.models.chat_selection`. No audio format: a + language model never hears the microphone. + """ + ep = self._llm.get(provider) + if ep is None: + raise _missing(provider, self.llm_names, "language model provider") + return ep.load() + + def load_tts(self, provider: str) -> type: + """Import and return the plugin class for a speech synthesis provider. + + Constructed as `cls(config, voice)`, where `config` is the merged + provider reference from `emet_sdk.models.tts_selection` and `voice` + the soul's `voice` block. The voice states its own sample rate after + `start()`, and the sink is opened to match. + """ + ep = self._tts.get(provider) + if ep is None: + raise _missing(provider, self.tts_names, "speech synthesis provider") + return ep.load() + def _missing(name: str, available: Iterable[str], what: str) -> Exception: # Imported lazily: validate imports discovery, so discovery must not diff --git a/emet-sdk/emet_sdk/models.py b/emet-sdk/emet_sdk/models.py new file mode 100644 index 0000000..d778738 --- /dev/null +++ b/emet-sdk/emet_sdk/models.py @@ -0,0 +1,99 @@ +"""Which model provider a body and a soul agree on. + +The soul's `models` block names the providers it was written for. The keys +are the owner's (BYOK, bring your own keys) and they travel with the soul, so +a provider is a soul field without breaching principle 1: a cloud account is +not hardware. The body has a say too, and the two documents have to be read +together to know what runs. + +Pure functions over contract data, which is why this lives in the SDK rather +than in the engine (`DESIGN.md` section 3): the engine and the validator must +agree on what an absent field means, and two copies of that rule would drift +onto somebody else's robot before anyone noticed. + +One rule for every stage. Speech recognition (`stt`), the language model +(`chat`), speech synthesis (`tts`) and the backchannel micro-model (`micro`) +are chosen the same way, from the same two blocks, so a manifest author learns +it once. +""" + +from __future__ import annotations + +from typing import Any, Mapping + +__all__ = ["STAGES", "model_selection", "stt_selection", "chat_selection", "tts_selection"] + +#: The keys of a `models` block, on the soul and on the body alike. Each names +#: one stage of the conversation and resolves against one entry-point group: +#: `stt` against `emet.stt`, `chat` against `emet.llm`, `tts` against +#: `emet.tts`. `micro` is reserved in the schema and resolves against nothing +#: yet. +STAGES: frozenset[str] = frozenset({"stt", "chat", "tts", "micro"}) + + +def model_selection( + manifest: Mapping[str, Any], + soul: Mapping[str, Any], + stage: str, +) -> dict[str, Any] | None: + """Resolve which provider runs one stage, and with what. + + The rule, in order: + + 1. If the body's `models..provider` is set, the body has taken the + stage over. Its `provider`, `model` and `key_env` are used and the + soul's are ignored, all three together. A provider name from one + document with a model name and a key from another is a broken + reference, so the override is whole or nothing. + 2. Otherwise the soul's `models.` chooses: `provider`, `model`, + `key_env`. + 3. Either way the body's `models..params` ride along untouched. + They are tuning for this deployment (an endpoint, a timeout) and + belong to whichever provider ends up running. + + Returns None when neither document names a provider. There is no default: + every stage is a cloud account nobody can be assumed to have, and quietly + choosing one would spend somebody's credits without asking. + + The returned mapping is what the stage's plugin is constructed from: + `provider`, `model`, `key_env`, `params`. The key itself is never in it. + """ + if stage not in STAGES: + raise ValueError(f"unknown model stage {stage!r}; expected one of {sorted(STAGES)}") + + body = (manifest.get("models") or {}).get(stage) or {} + soul_ref = (soul.get("models") or {}).get(stage) or {} + + chosen = body if body.get("provider") else soul_ref + provider = chosen.get("provider") + if not isinstance(provider, str) or not provider: + return None + + return { + "provider": provider, + "model": chosen.get("model"), + "key_env": chosen.get("key_env"), + "params": dict(body.get("params") or {}), + } + + +def stt_selection(manifest: Mapping[str, Any], soul: Mapping[str, Any]) -> dict[str, Any] | None: + """Which speech recognition provider runs. `model_selection(..., "stt")`.""" + return model_selection(manifest, soul, "stt") + + +def chat_selection(manifest: Mapping[str, Any], soul: Mapping[str, Any]) -> dict[str, Any] | None: + """Which language model runs. `model_selection(..., "chat")`.""" + return model_selection(manifest, soul, "chat") + + +def tts_selection(manifest: Mapping[str, Any], soul: Mapping[str, Any]) -> dict[str, Any] | None: + """Which voice speaks. `model_selection(..., "tts")`. + + The same rule as the other stages, and the same absence of a default, + though the reason differs: the shipped local voice costs nothing per + word, and there is still no default because a voice model is a file + somebody has to have downloaded, and a robot that chose one quietly would + fail at boot naming a file nobody asked for. + """ + return model_selection(manifest, soul, "tts") diff --git a/emet-sdk/emet_sdk/plugin.py b/emet-sdk/emet_sdk/plugin.py index b3853fe..5fb4399 100644 --- a/emet-sdk/emet_sdk/plugin.py +++ b/emet-sdk/emet_sdk/plugin.py @@ -1,6 +1,6 @@ """The plugin contract. Breaking it is a major version bump. -Four categories, and the split is not arbitrary: +Seven categories, and the split is not arbitrary: * **Actuators** receive `Action`s and do something physical. The engine tells them what should happen; how is theirs. @@ -12,9 +12,22 @@ Wheels solve it with arithmetic, treads with the same arithmetic and different slip assumptions, and a legged plugin with a gait generator. The engine above them does not know or care. -* **Wake** plugins listen for the robot's name and nothing else. They are the - only category configured jointly by a body and a soul, and the only one with +* **Wake** plugins listen for the robot's name and nothing else. They are + configured jointly by a body and a soul, and they are the only category with no fallback beneath it. +* **Transcriber** plugins turn the speech after a wake into text. The soul + chooses the provider, because the keys are the owner's and travel with the + soul; the body may take the choice over, and tunes it. The engine feeds + frames in and reads partial and final transcripts out, so a streaming + provider and a batch one satisfy the same contract. +* **Language model** plugins turn the conversation so far into the next + thing the robot says. Chosen the same way as a transcriber. The engine + hands over an assembled prompt and reads the reply as it streams, so a + sentence can be spoken before the paragraph exists. +* **Voice** plugins turn that text into audio. Chosen the same way again. + The engine hands over one sentence at a time and plays the audio as it + arrives, at the rate the voice states, so the first sentence is heard + while the language model is still writing the second. **Why there is no VAD category.** Voice activity detection looks like it belongs beside wake, and does not. It is not a swap point: there is one real @@ -41,15 +54,22 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import Any, ClassVar, Mapping +from typing import Any, AsyncIterator, ClassVar, Mapping from emet_sdk.types import ( Action, + AudioFormat, CapabilityDescriptor, Health, + LanguageModelDescriptor, LocomotionDescriptor, + Prompt, Reading, + ReplyEvent, + Transcript, + TranscriberDescriptor, Twist, + VoiceDescriptor, WakeDescriptor, WakeEvent, ) @@ -61,6 +81,9 @@ "SensorPlugin", "LocomotionPlugin", "WakePlugin", + "TranscriberPlugin", + "LanguageModelPlugin", + "VoicePlugin", "PluginError", ] @@ -80,10 +103,11 @@ class Plugin(ABC): Every category starts, shuts down, and reports health the same way. What a plugin is *constructed from* differs: three categories are built from one - entry in a manifest's `capabilities` list, and wake is built from - `audio.wake` plus a phrase the soul supplies. Construction therefore - belongs to the subclasses, so that no category inherits a signature it has - to contradict. + entry in a manifest's `capabilities` list, wake is built from `audio.wake` + plus a phrase the soul supplies, and a transcriber or a language model + from the provider reference the soul and the body agree on. Construction + therefore belongs to the subclasses, so that no category inherits a + signature it has to contradict. """ async def start(self) -> None: @@ -287,3 +311,265 @@ async def reset(self) -> None: Default is a no-op. Streaming detectors holding a rolling buffer override it so that one utterance cannot trigger twice. """ + + +class TranscriberPlugin(Plugin): + """What turns the speech after a wake into text. + + The seam between Emet and a speech recognition vendor, and it exists + before any vendor does. There are credits enough at one provider to build + the whole of a release against its client library without noticing, and + the result would be an engine that speaks one company's dialect. Behind + this contract, the provider is one line of a soul. + + **Who chooses.** The soul, through `models.stt`: the provider, the model, + and the name of the environment variable holding the key. Keys are the + owner's (BYOK) and travel with the soul, so this is a soul field without + breaching principle 1: a cloud account is not hardware. The body may take + the choice over through `audio.stt.provider` (a test rig running the + mock, an owner whose key is for a different provider), and it tunes + whichever provider runs through `audio.stt.params`. The merged result is + what this constructor receives; `emet_sdk.models.stt_selection` is the + one place the merge is written down. + + **Streaming.** Frames go in one at a time, as they are captured. A + provider that streams answers with partial transcripts while the person + is still talking, and every partial carries the whole text heard so far. + `finish()` closes the utterance and returns the final. A provider that + does not stream returns None from every `feed()` and does its work in + `finish()`. Same contract, and the engine above cannot tell them apart + except by latency and by `describe().streaming`. + + **Format.** The wake engine fixed the audio format before this plugin was + built, and one microphone feeds both, so the transcriber receives the + format rather than stating one. It reports the rate it will actually run + at in `describe()`, and the engine refuses a mismatch rather than letting + speech be heard at the wrong speed. + """ + + #: The string matched against `models.stt.provider` in the soul (or + #: `audio.stt.provider` in the body), and the entry-point name this + #: plugin registers under. + provider: ClassVar[str] = "" + + def __init__(self, config: Mapping[str, Any], fmt: AudioFormat) -> None: + """Receive the merged provider reference and the audio format. + + `config` has the shape of the soul's `models.stt` block plus the + body's `params`: `provider`, `model`, `key_env`, `params`. The key + itself is never in it. A plugin that needs one reads the environment + variable `key_env` names, in `start()`, and reports `healthy=False` + with a detail naming the variable when it is absent. The robot then + fails loudly and in the owner's terms, which is what P0 promises. + """ + self.config: Mapping[str, Any] = config + self.model: str | None = ( + str(config["model"]) if config.get("model") is not None else None + ) + self.key_env: str | None = ( + str(config["key_env"]) if config.get("key_env") is not None else None + ) + self.params: Mapping[str, Any] = dict(config.get("params") or {}) + #: The audio the engine will feed: mono int16 at this rate and frame + #: size, fixed by the wake engine before this plugin was built. + self.format = fmt + + @abstractmethod + def describe(self) -> TranscriberDescriptor: + """Report what this instance can actually do, after `start()`. + + Report narrowly. A provider whose key is missing or whose network is + down says `healthy=False` and puts the reason in `health().detail`; + the boot check prints it. Claiming health and returning empty + transcripts forever is the silent failure this contract exists to + prevent. + """ + + @abstractmethod + async def feed(self, frame: bytes) -> Transcript | None: + """Consume one frame of the utterance. Return a partial if the text + heard so far changed, else None. + + Frames arrive at the rate and size in `self.format`, in order, for one + utterance at a time. Return promptly: this runs inside the capture + loop, and a provider that blocks here on the network drops audio. Hand + the frame off and report whatever results have already come back. + """ + + @abstractmethod + async def finish(self) -> Transcript: + """The turn has ended. Flush, and return the final transcript. + + Always returns one, with `final=True`, even when nothing was heard: + an empty final is a real answer (they said nothing the provider could + make out) and the engine treats it as one. After this the plugin is + ready for the next utterance's first `feed()`. + """ + + +class LanguageModelPlugin(Plugin): + """What turns the conversation so far into the next thing the robot says. + + The second provider seam, built like the first: the contract and a mock + first, then the vendors, so that the engine is written against this + class and never against a vendor's client library. There are credits at + more than one vendor, and this is what keeps the choice one line of a + soul. + + **Who chooses.** The soul, through `models.chat`, on the terms `models.stt` + set: provider, model, and the name of the environment variable holding + the key. The body may take the choice over through its own `models.chat` + and tunes whichever runs through `params`. `emet_sdk.models.chat_selection` + is the rule. + + **What the model does and does not decide.** It receives a `Prompt` the + engine assembled and returns text. The persona is the engine's to write + into `system`, the memories are the engine's to retrieve, and what the + body does while the words are spoken is the engine's to resolve through + the chains. A vendor sees one turn at a time and nothing of the robot. + + **Streaming.** `reply()` is an async iterator. Text arrives as + `TextDelta`s in order, a `ToolCall` arrives once its arguments are + complete, and exactly one `ReplyDone` arrives last, on success and on + failure alike, carrying the whole text and why it stopped. A caller that + speaks as it reads starts on the first sentence; a caller that wants the + text waits for the last event. + + **Tools.** `Prompt.tools` are vendor-neutral specs; the plugin translates + them. When the model stops with `tool`, the caller runs the tools, + appends the assistant turn with its calls and one `tool` message per + result, and calls `reply()` again. The plugin keeps no conversation + state between calls. + """ + + #: The string matched against `models.chat.provider`, and the entry-point + #: name this plugin registers under. + provider: ClassVar[str] = "" + + def __init__(self, config: Mapping[str, Any]) -> None: + """Receive the merged provider reference: `provider`, `model`, + `key_env`, `params`. The key itself is never in it. A plugin that + needs one reads the environment variable `key_env` names, in + `start()`, and reports `healthy=False` naming the variable when it + is absent. + """ + self.config: Mapping[str, Any] = config + self.model: str | None = ( + str(config["model"]) if config.get("model") is not None else None + ) + self.key_env: str | None = ( + str(config["key_env"]) if config.get("key_env") is not None else None + ) + self.params: Mapping[str, Any] = dict(config.get("params") or {}) + + @abstractmethod + def describe(self) -> LanguageModelDescriptor: + """Report what this instance can actually do, after `start()`. + + Report narrowly. A provider whose key is missing or whose network is + down says `healthy=False` and puts the reason in `health().detail`; + the boot check prints it, and the robot refuses to run rather than + hear questions it can never answer. + """ + + @abstractmethod + def reply(self, prompt: Prompt) -> AsyncIterator[ReplyEvent]: + """Generate one reply, as an async iterator of events. + + Yields `TextDelta`s as text is generated, a `ToolCall` per completed + call, and one `ReplyDone` last. Never raises for a failure the + provider reported or the network caused: those end the stream with + `ReplyDone(stop_reason="error", error=...)` and whatever text came + first, so the engine can say something rather than crash mid-turn. + """ + + +class VoicePlugin(Plugin): + """What turns the words the robot will say into audio. + + The third provider seam, built like the other two: the contract and a + mock first, then the local voice, then one cloud voice. `DESIGN.md` + section 14 keeps synthesis local by default, because it is the stage that + costs the most per turn in the cloud and the one a robot in a home should + be able to do with the network down, so the shipped default is a local + model and a cloud voice is a line of a soul. + + **Who chooses.** The soul, through `models.tts`, on the terms `models.stt` + set: the provider, the model (a voice, here: a Piper model name, a + vendor's voice id), and the environment variable holding the key when + there is one. The body may take the choice over through its own + `models.tts` and tunes whichever runs through `params`. + `emet_sdk.models.tts_selection` is the rule. + + **What else the soul says.** How fast it speaks. `voice.rate` on the soul + is a persona trait, like `patience_ms`: a reflective character talks + slower than an eager one, and that is true whichever voice produces the + sound. So the soul's `voice` block reaches this constructor beside the + provider reference, the way `identity.wake_word` reaches a wake plugin. + Principle 1 holds: the soul says how it sounds, never which library, on + what device, at what sample rate. + + **Streaming.** `speak()` takes one sentence and is an async iterator of + audio chunks: mono int16 at `describe().sample_rate`, in order, + concatenable. A provider that streams yields the first chunk before the + sentence is finished; one that does not yields the whole sentence as one + chunk and says `streaming=False`. The engine plays each sentence as its + audio completes and asks for the next while it plays, which is what lets + the first sentence be heard while the reply is still being written. + + **Format.** The voice states the sample rate; the engine opens the sink + to match. A local model produces one rate and only one, and the sink is + the side that can convert. + """ + + #: The string matched against `models.tts.provider`, and the entry-point + #: name this plugin registers under. + provider: ClassVar[str] = "" + + def __init__(self, config: Mapping[str, Any], voice: Mapping[str, Any] | None = None) -> None: + """Receive the merged provider reference and the soul's `voice` block. + + `config` is `provider`, `model`, `key_env`, `params`; the key itself is + never in it, and a plugin that needs one reads the environment + variable `key_env` names, in `start()`, and reports `healthy=False` + naming it when it is absent. `voice` is the soul's `voice` block: + `rate`, a multiplier on speaking speed, is the field every plugin + honours as far as its engine allows; the rest is advisory. + """ + self.config: Mapping[str, Any] = config + self.model: str | None = ( + str(config["model"]) if config.get("model") is not None else None + ) + self.key_env: str | None = ( + str(config["key_env"]) if config.get("key_env") is not None else None + ) + self.params: Mapping[str, Any] = dict(config.get("params") or {}) + self.voice: Mapping[str, Any] = dict(voice or {}) + #: Speaking speed as a multiplier: 1.0 is the voice's own pace, 1.2 is + #: a fifth faster. From the soul; a persona trait. + rate = self.voice.get("rate") + self.rate: float = float(rate) if isinstance(rate, (int, float)) and rate > 0 else 1.0 + + @abstractmethod + def describe(self) -> VoiceDescriptor: + """Report what this instance can actually do, after `start()`. + + Report narrowly. A voice whose model is not on disk, whose key is + missing or whose library is not installed says `healthy=False` and + puts the reason, and the command that fixes it, in `health().detail`; + the boot check prints it, and the robot refuses to run rather than + answer questions nobody will hear. + """ + + @abstractmethod + def speak(self, text: str) -> AsyncIterator[bytes]: + """Turn one sentence into audio, as an async iterator of chunks. + + Each chunk is mono int16 PCM at `describe().sample_rate`, and the + chunks concatenate into the sentence. Empty or blank text yields + nothing. A failure the provider reported or the network caused + raises `PluginError` after whatever audio came first; the engine + treats that as one sentence lost, says so, and goes on with the + next, because a robot that drops a sentence is still a robot that + talks. + """ diff --git a/emet-sdk/emet_sdk/types.py b/emet-sdk/emet_sdk/types.py index 5beaaa5..356e8e9 100644 --- a/emet-sdk/emet_sdk/types.py +++ b/emet-sdk/emet_sdk/types.py @@ -27,6 +27,18 @@ "LocomotionDescriptor", "WakeDescriptor", "WakeEvent", + "Transcript", + "TranscriberDescriptor", + "ToolSpec", + "ToolCall", + "Message", + "Prompt", + "TextDelta", + "ReplyDone", + "ReplyEvent", + "STOP_REASONS", + "LanguageModelDescriptor", + "VoiceDescriptor", "AudioFormat", "AudioSource", "AudioSink", @@ -236,6 +248,237 @@ def __post_init__(self) -> None: raise ValueError(f"confidence must be in [0.0, 1.0], got {self.confidence}") +@dataclass(frozen=True, slots=True) +class Transcript: + """What speech recognition heard, so far or in full. + + A streaming recogniser sends words back while the person is still talking, + and revises them: "what time" becomes "what time is it" becomes "what time + is it in Tokyo". Each revision arrives as a partial carrying the whole text + heard so far in this utterance, and `final=False`. When the turn ends, one + more arrives with `final=True`, and that is the text the engine acts on. + + Cumulative rather than incremental on purpose. A caller showing a live + caption replaces the line; it does not have to splice fragments, and a + provider that reorders or retracts a word cannot leave a stale fragment + behind. A recogniser that does not stream sends no partials and one final. + """ + + text: str + final: bool = False + #: Advisory. Providers that report no calibrated score say 1.0, which is + #: honesty about the absence of a number rather than a claim of certainty. + confidence: float = 1.0 + + def __post_init__(self) -> None: + if not 0.0 <= self.confidence <= 1.0: + raise ValueError(f"confidence must be in [0.0, 1.0], got {self.confidence}") + + +@dataclass(frozen=True, slots=True) +class TranscriberDescriptor: + """What a speech recogniser can actually do, reported after `start()`. + + The boot check reads `healthy` and `sample_rate`. A recogniser is built + with the audio format the wake engine already fixed, because one + microphone feeds both, and it echoes back the rate it will actually run + at. If the two differ the engine refuses to start rather than letting a + 16 kHz stream be heard at 8 kHz, which does not fail, it just produces + nonsense. Same rule as the detector: the consumer states the format, and + a mismatch is loud. + """ + + provider: str + #: The model this instance loaded, if the provider has such a thing. What + #: was asked for is in the config; this is what answered. + model: str | None = None + #: Whether partial transcripts arrive while the person is still speaking. + #: A batch recogniser says False and sends one final per utterance. + streaming: bool = False + sample_rate: int = 16000 + healthy: bool = True + + +@dataclass(frozen=True, slots=True) +class ToolSpec: + """A function the language model may ask to have run. + + Vendor-neutral: `parameters` is a JSON Schema object and each provider + translates it to its own tool format. Tools are how side effects leave + the model (a memory to write, a fact to look up); what is *said* comes + back as text. + """ + + name: str + description: str + parameters: Mapping[str, Any] = field( + default_factory=lambda: {"type": "object", "properties": {}} + ) + + +@dataclass(frozen=True, slots=True) +class ToolCall: + """The model asking for a tool to run, with parsed arguments. + + Streamed once the arguments are complete. The caller runs the tool and + answers with a `Message` of role `tool` carrying this call's `id`. + """ + + id: str + name: str + arguments: Mapping[str, Any] = field(default_factory=dict) + + +#: Roles a `Message` may carry. The system prompt is not a message: it sits +#: on the `Prompt`, because every provider treats it differently. +MESSAGE_ROLES: frozenset[str] = frozenset({"user", "assistant", "tool"}) + + +@dataclass(frozen=True, slots=True) +class Message: + """One turn of the conversation as the model sees it. + + `user` and `assistant` carry text. An `assistant` turn that asked for + tools carries the calls beside its text; a `tool` turn answers one call + and names it. Providers that want tool results in a different wrapper + (a user turn holding result blocks, say) do the wrapping themselves. + """ + + role: str + content: str = "" + tool_calls: tuple[ToolCall, ...] = () + tool_call_id: str | None = None + + def __post_init__(self) -> None: + if self.role not in MESSAGE_ROLES: + raise ValueError(f"role must be one of {sorted(MESSAGE_ROLES)}, got {self.role!r}") + if self.role == "tool" and not self.tool_call_id: + raise ValueError("a tool message must name the call it answers (tool_call_id)") + if self.tool_calls and self.role != "assistant": + raise ValueError("only an assistant message carries tool_calls") + + +@dataclass(frozen=True, slots=True) +class Prompt: + """Everything one request to a language model needs. + + Assembled by the engine: the persona, and in later releases the + self-model and the memories the sensitivity floor lets through, become + `system`; the conversation so far and the words just heard become + `messages`. Nothing about which vendor answers is in here. + + `max_tokens` bounds one reply. A reply is spoken aloud, so the engine + sets this low on purpose; a provider that hits it reports `length`. + """ + + system: str + messages: tuple[Message, ...] + tools: tuple[ToolSpec, ...] = () + max_tokens: int = 1024 + + def __post_init__(self) -> None: + if self.max_tokens < 1: + raise ValueError("max_tokens must be positive") + + +@dataclass(frozen=True, slots=True) +class TextDelta: + """A piece of the reply, as it is generated. Concatenate in order.""" + + text: str + + +#: Why a reply ended, in words every provider maps onto. `end` is the +#: natural finish; `tool` means the model wants tool results before it goes +#: on; `length` means `max_tokens` cut it off; `refusal` means the provider +#: declined to answer; `error` means the reply is incomplete and `error` +#: says why. +STOP_REASONS: frozenset[str] = frozenset({"end", "tool", "length", "refusal", "error"}) + + +@dataclass(frozen=True, slots=True) +class ReplyDone: + """The last event of a reply: the whole text, why it stopped, what it cost. + + Always the final event, on success and on failure alike. A caller that + only wants the text reads it here; a caller that streamed the deltas + reads `stop_reason` and the token counts. + """ + + text: str + stop_reason: str = "end" + tool_calls: tuple[ToolCall, ...] = () + model: str | None = None + input_tokens: int | None = None + output_tokens: int | None = None + error: str | None = None + + def __post_init__(self) -> None: + if self.stop_reason not in STOP_REASONS: + raise ValueError( + f"stop_reason must be one of {sorted(STOP_REASONS)}, got {self.stop_reason!r}" + ) + + +#: What `LanguageModelPlugin.reply()` yields: text as it arrives, a tool call +#: once its arguments are complete, and one `ReplyDone` last. +ReplyEvent = TextDelta | ToolCall | ReplyDone + + +@dataclass(frozen=True, slots=True) +class LanguageModelDescriptor: + """What a language model plugin can actually do, reported after `start()`. + + `healthy` is what the boot check reads: a provider whose key is missing, + rejected, or unreachable says so here and in `health().detail`, and the + engine refuses to run a robot that would hear and never answer. + """ + + provider: str + #: The model this instance will ask for. What answered is on each + #: `ReplyDone`, because a provider may substitute one. + model: str | None = None + #: Whether `TextDelta`s arrive before `ReplyDone`. A provider that does + #: not stream sends the whole text as one delta. + streaming: bool = True + #: Whether `Prompt.tools` will be honoured. A provider without tool use + #: says False and the engine leaves tools out of its prompts. + tools: bool = True + healthy: bool = True + + +@dataclass(frozen=True, slots=True) +class VoiceDescriptor: + """What a speech synthesis plugin can actually do, reported after `start()`. + + The third provider descriptor, and the one the output side is built + from. `sample_rate` is the rate the audio from `speak()` will arrive at, + and the engine opens the sink at that rate rather than asking the voice + to match a card: a local voice produces one rate and only one, and + resampling in the engine would be a second place for audio to go + wrong. The same rule as the wake engine, the other way round: the + detector states the format and the microphone conforms; the voice states + the format and the speaker conforms. + + `healthy` is what the boot check reads. A voice whose model file is + missing, whose key is rejected or whose library is not installed says so + here and in `health().detail`, and the engine refuses to run a robot that + would answer and never be heard. + """ + + provider: str + #: The voice this instance loaded: a Piper model name, a vendor's voice + #: id. What was asked for is in the config; this is what will speak. + model: str | None = None + #: The rate of the audio `speak()` yields. Mono int16, always. + sample_rate: int = 22050 + #: Whether audio arrives in pieces before the sentence is finished. A + #: provider that synthesises a whole sentence at once yields one chunk + #: and says False; the engine cannot tell them apart except by latency. + streaming: bool = False + healthy: bool = True + + #: Audio is int16 throughout. Two bytes a sample, everywhere. SAMPLE_BYTES = 2 diff --git a/emet-sdk/emet_sdk/validate.py b/emet-sdk/emet_sdk/validate.py index 28ee7bb..66e0396 100644 --- a/emet-sdk/emet_sdk/validate.py +++ b/emet-sdk/emet_sdk/validate.py @@ -56,6 +56,9 @@ "DEFAULT_AUDIO_SOURCE", "BUILTIN_AUDIO_OUT", "DEFAULT_AUDIO_SINK", + "BUILTIN_STT", + "BUILTIN_LLM", + "BUILTIN_TTS", "load_yaml", "validate_manifest", "validate_soul", @@ -114,6 +117,24 @@ BUILTIN_AUDIO_OUT: frozenset[str] = frozenset({"speaker", "wav", "null"}) DEFAULT_AUDIO_SINK = "speaker" +#: Speech recognition providers `emet-providers` ships. Documentation, like the +#: others: `models.stt.provider` is an open enum resolved against entry points. +#: +#: There is deliberately no `DEFAULT_STT_PROVIDER`, and no default for any +#: model stage. The other defaults name the hardware floor, which every body +#: has. A model stage is a cloud account with a key the owner brings, and +#: nobody can be assumed to hold one, so an absent provider means "this stage +#: does not run", and asking for it anyway is an error that names the field +#: to set. See `emet_sdk.models.model_selection`. +BUILTIN_STT: frozenset[str] = frozenset({"mock", "deepgram"}) + +#: Language model providers `emet-providers` ships. Same status. +BUILTIN_LLM: frozenset[str] = frozenset({"mock", "anthropic", "openai"}) + +#: Voices `emet-providers` ships. Same status. `piper` is the local default +#: the design asks for; `deepgram` is the cloud voice a soul opts into. +BUILTIN_TTS: frozenset[str] = frozenset({"mock", "piper", "deepgram"}) + # -------------------------------------------------------------------------- # Findings @@ -279,6 +300,7 @@ def validate_manifest( _check_wake_engine(doc, registry, report) _check_audio_source(doc, registry, report) _check_audio_sink(doc, registry, report) + _check_model_providers(doc, registry, report) return report @@ -379,6 +401,50 @@ def _check_audio_sink( ) +#: Which entry-point group each body-side `models..provider` resolves +#: against, and what to call it in a message. A stage absent here (`micro`) +#: is reserved: accepted by the schema, resolved against nothing yet, so a +#: name under it is not checked. +_MODEL_STAGE_GROUPS: Mapping[str, tuple[str, str]] = { + "stt": ("has_stt", "speech recognition provider"), + "chat": ("has_llm", "language model provider"), + "tts": ("has_tts", "speech synthesis provider"), +} + + +def _check_model_providers( + doc: Mapping[str, Any], + registry: PluginRegistry, + report: ValidationReport, +) -> None: + """Resolve the body's `models..provider`, on the wake engine's terms. + + Absent is the common case: most bodies leave every stage to the soul's + `models` block. Those soul fields are deliberately checked neither here + nor in `validate_soul`. A soul is valid on every machine or on none, and + whether the provider it names is installed is a question for boot, the + way `identity.wake_word` is answered against a live descriptor. A body is + one machine, so a body that names a provider names software that has to + be installed on it, and an unresolvable name is an error like + `audio.wake.engine`. + """ + models = doc.get("models") or {} + for stage, (query, what) in _MODEL_STAGE_GROUPS.items(): + provider = (models.get(stage) or {}).get("provider") + if not isinstance(provider, str) or getattr(registry, query)(provider): + continue + names = getattr(registry, f"{query[4:]}_names") + installed = ", ".join(names) or "(none)" + report.error( + "missing_plugin", + f"no {what} provides {provider!r}. Installed: {installed}. " + f"`models.{stage}.provider` is an open enum: this value is legal, " + f"the plugin simply is not installed. Omit it to let the soul's " + f"`models.{stage}` choose.", + f"/models/{stage}/provider", + ) + + def _check_unique_ids(caps: Sequence[Mapping[str, Any]], report: ValidationReport) -> None: seen: dict[str, int] = {} for i, cap in enumerate(caps): diff --git a/emet-sdk/examples/emet-soul.yaml b/emet-sdk/examples/emet-soul.yaml index d6332d9..ae3a308 100644 --- a/emet-sdk/examples/emet-soul.yaml +++ b/emet-sdk/examples/emet-soul.yaml @@ -23,10 +23,11 @@ identity: license: null # RSV voice: - engine: piper - model: "en_US-amy-medium" + # How it sounds, whichever voice produces the sound. `rate` is a persona + # trait like `patience_ms`: 1.0 is the voice's own pace. Which voice speaks + # is `models.tts` below, beside the other providers. rate: 1.0 - pitch_shift_semitones: 0 + pitch_shift_semitones: 0 # RSV persona: summary: > @@ -69,7 +70,13 @@ memory: retention_days: 0 # 0 = forever models: - chat: {provider: openai, model: "gpt-4o", key_env: "EMET_OPENAI_KEY"} - stt: {provider: deepgram, model: "nova-2", key_env: "EMET_DEEPGRAM_KEY"} - tts: {provider: local_piper} + chat: {provider: openai, model: "gpt-5.6-terra", key_env: "EMET_OPENAI_KEY"} + stt: {provider: deepgram, model: "nova-3", key_env: "EMET_DEEPGRAM_KEY"} + # Local by design: synthesis is the stage a robot in a home should manage + # with the network down. The voice is a file the owner downloads once: + # python -m piper.download_voices en_US-ljspeech-medium --data-dir ~/.local/share/emet/voices + # LJ Speech is public domain, which is why it is the reference voice and + # Amy, fine-tuned from a corpus licensed for research only, is not. A cloud + # voice is one line: {provider: deepgram, model: "aura-2-thalia-en", key_env: "EMET_DEEPGRAM_KEY"} + tts: {provider: piper, model: "en_US-ljspeech-medium"} micro: {provider: local, model: "qwen3-0.6b-q4"} diff --git a/emet-sdk/examples/invalid/unknown-llm-provider.yaml b/emet-sdk/examples/invalid/unknown-llm-provider.yaml new file mode 100644 index 0000000..49e92ee --- /dev/null +++ b/emet-sdk/examples/invalid/unknown-llm-provider.yaml @@ -0,0 +1,35 @@ +# INVALID: a language model provider that is not installed, named by the body. +# +# `models.chat.provider` on a body resolves against installed `emet.llm` +# plugins, exactly as `models.stt.provider` resolves against `emet.stt`. The +# value below is legal and does not resolve, and on a body that is an error: +# a body is one machine, and it has named software that has to be installed +# on it. The soul's `models.chat.provider` is never checked this way; a soul +# has to be valid on every machine or on none. +# +# Expected: missing_plugin, /models/chat/provider + +manifest_version: "0.1" + +body: + id: oracle_less + name: "a body whose brain is on order" + scale: desk + power: plugged_in + +audio: + input: + device: "plughw:1,0" + sample_rate: 16000 + channels: 1 + aec: hardware + doa: false + output: + device: "plughw:1,0" + gain_db: -6.0 + +models: + chat: + provider: abacus + +capabilities: [] diff --git a/emet-sdk/examples/invalid/unknown-stt-provider.yaml b/emet-sdk/examples/invalid/unknown-stt-provider.yaml new file mode 100644 index 0000000..ac83442 --- /dev/null +++ b/emet-sdk/examples/invalid/unknown-stt-provider.yaml @@ -0,0 +1,39 @@ +# INVALID: a speech recognition provider that is not installed, named by the body. +# +# `models.stt.provider` on a body follows the rule `audio.wake.engine` follows. +# The value below is legal, since the field is an open enum resolved against +# installed `emet.stt` plugins, but it does not resolve, and on a body that is +# an error rather than a warning: a body is one machine, and it has named +# software that has to be installed on it. +# +# The soul's `models.stt.provider` is the other half of the choice and is +# deliberately never checked this way. A soul has to be valid on every machine +# or on none, so whether the provider it names is installed is answered at +# boot, like whether its wake phrase can be heard. +# +# Expected: missing_plugin, /models/stt/provider + +manifest_version: "0.1" + +body: + id: pigeon_post + name: "a body whose ears report to nobody" + scale: desk + power: plugged_in + +audio: + input: + device: "plughw:1,0" + sample_rate: 16000 + channels: 1 + aec: hardware + doa: false + output: + device: "plughw:1,0" + gain_db: -6.0 + +models: + stt: + provider: carrier_pigeon + +capabilities: [] diff --git a/emet-sdk/examples/invalid/unknown-tts-provider.yaml b/emet-sdk/examples/invalid/unknown-tts-provider.yaml new file mode 100644 index 0000000..dce5d98 --- /dev/null +++ b/emet-sdk/examples/invalid/unknown-tts-provider.yaml @@ -0,0 +1,36 @@ +# INVALID: a voice that is not installed, named by the body. +# +# `models.tts.provider` on a body resolves against installed `emet.tts` +# plugins, exactly as `models.stt.provider` and `models.chat.provider` +# resolve against theirs. The value below is legal and does not resolve, and +# on a body that is an error: a body is one machine, and it has named +# software that has to be installed on it. The soul's `models.tts.provider` +# is never checked this way; a soul has to be valid on every machine or on +# none. +# +# Expected: missing_plugin, /models/tts/provider + +manifest_version: "0.1" + +body: + id: voiceless + name: "a body whose voice is on order" + scale: desk + power: plugged_in + +audio: + input: + device: "plughw:1,0" + sample_rate: 16000 + channels: 1 + aec: hardware + doa: false + output: + device: "plughw:1,0" + gain_db: -6.0 + +models: + tts: + provider: gramophone + +capabilities: [] diff --git a/emet-sdk/examples/mock-scout.yaml b/emet-sdk/examples/mock-scout.yaml index 9f10d08..17725b5 100644 --- a/emet-sdk/examples/mock-scout.yaml +++ b/emet-sdk/examples/mock-scout.yaml @@ -56,6 +56,22 @@ audio: # testable on a laptop with no microphone. engine: mock +# And its understanding, its answers, and its voice. The reference soul names +# real providers under its `models` block; this rig has no key, no network +# and no voice model on disk, so the body takes all three stages over, which +# is what this block is for. The mock transcriber reads words out of the +# bytes it is given, the mock language model repeats them back, and the mock +# voice spells the words into the audio it produces, so +# `emet-listen --transcribe --reply --speak` runs here and a wav sink can be +# read back to check what was said. +models: + stt: + provider: mock + chat: + provider: mock + tts: + provider: mock + capabilities: - id: head type: joint_group diff --git a/emet-sdk/examples/pi-speakerphone.yaml b/emet-sdk/examples/pi-speakerphone.yaml index f992bf9..c91d648 100644 --- a/emet-sdk/examples/pi-speakerphone.yaml +++ b/emet-sdk/examples/pi-speakerphone.yaml @@ -54,9 +54,12 @@ audio: device: "USB" gain_db: -6.0 # Playback runs at this rate except under `--echo`, which plays captured - # 16 kHz audio and switches the sink to 16 kHz for that run. 48 kHz is - # what nearly every USB DAC does natively, so a `--stats` run never waits - # on a rate the card refuses. + # 16 kHz audio and switches the sink to 16 kHz for that run, and under + # `--speak`, where the voice states the rate (22050 Hz for a Piper medium + # voice, 24000 Hz for Deepgram Aura) and the sink is opened to match. + # 48 kHz is what nearly every USB DAC does natively, so a `--stats` run + # never waits on a rate the card refuses; the card converts the others + # through ALSA's plug layer. sample_rate: 48000 wake: engine: pocketsphinx @@ -69,6 +72,26 @@ audio: # params: # cmninit: "43.0,6.0,5.6,8.1,-5.9,-1.9,-8.2,-0.5,-0.9,3.4,7.6,-1.3,-0.4" +# Speech recognition, the language model and the voice. The reference soul +# names real providers under its `models` block; with +# `emet-providers[deepgram,openai,piper]` installed, the keys in +# ~/.config/emet/keys.env (EMET_DEEPGRAM_KEY, EMET_OPENAI_KEY) and the voice +# downloaded once +# (`python -m piper.download_voices en_US-ljspeech-medium --data-dir ~/.local/share/emet/voices`), +# `--transcribe --reply --speak` runs against them and this block stays +# commented. Without keys, or offline, uncomment it and the body takes the +# stages over with the mocks: the transcriber reads nothing out of a real +# microphone and so is given a line to say, the language model repeats what +# it heard, and the mock voice makes a sound no one could mistake for speech. +# models: +# stt: +# provider: mock +# params: {transcript: "testing the seam"} +# chat: +# provider: mock +# tts: +# provider: mock + capabilities: [] safety: diff --git a/emet-sdk/schemas/body-manifest.schema.json b/emet-sdk/schemas/body-manifest.schema.json index cb91b1a..88e4043 100644 --- a/emet-sdk/schemas/body-manifest.schema.json +++ b/emet-sdk/schemas/body-manifest.schema.json @@ -133,6 +133,18 @@ } }, + "models": { + "description": "P0, optional. The body's say over the soul's models block. The soul chooses each stage's provider through its own models block, because the key is the owner's and travels with the soul. A stage named here takes that choice over on this body (provider, with its own model and key_env; the soul's three are ignored together) or tunes whichever provider runs (params, passed through untouched). Each provider resolves against an entry-point group, emet.stt for stt, emet.llm for chat and emet.tts for tts, so these are open enums: an unknown value is a missing plugin, never a schema error. There is no default provider for any stage. micro is RSV: accepted, resolved against nothing yet.", + "type": "object", + "additionalProperties": false, + "properties": { + "stt": { "$ref": "#/$defs/model_override" }, + "chat": { "$ref": "#/$defs/model_override" }, + "tts": { "$ref": "#/$defs/model_override" }, + "micro": { "$ref": "#/$defs/model_override" } + } + }, + "capabilities": { "description": "P0. May be empty: a Pi with a USB speakerphone and nothing else is a valid body, and is milestone one.", "type": "array", @@ -152,6 +164,24 @@ }, "$defs": { + "model_override": { + "description": "The same shape as the soul's model_ref, plus params. Set provider to take the stage over; leave it out to tune the soul's choice with params alone.", + "type": "object", + "additionalProperties": false, + "properties": { + "provider": { "type": "string", "minLength": 1 }, + "model": { "type": "string" }, + "key_env": { + "description": "Name of the environment variable holding the key. The key itself is never written into a manifest.", + "type": "string" + }, + "params": { + "description": "Passed to the provider untouched. Emet never inspects these.", + "type": "object" + } + } + }, + "identifier": { "type": "string", "pattern": "^[a-z][a-z0-9_]*$", diff --git a/emet-sdk/schemas/soul-bundle.schema.json b/emet-sdk/schemas/soul-bundle.schema.json index 88a60b6..7e0e65f 100644 --- a/emet-sdk/schemas/soul-bundle.schema.json +++ b/emet-sdk/schemas/soul-bundle.schema.json @@ -44,13 +44,27 @@ }, "voice": { + "description": "P0. How the soul sounds, whichever voice produces it. rate is a persona trait like patience_ms: a multiplier on speaking speed, 1.0 the voice's own pace, honoured by every shipped voice. engine and model are superseded by models.tts, which names the provider and the voice model; they are accepted for older bundles and ignored. pitch_shift_semitones is RSV: no shipped voice honours it yet.", "type": "object", "additionalProperties": false, "properties": { - "engine": { "enum": ["piper", "cloud"] }, - "model": { "type": "string" }, - "rate": { "type": "number", "exclusiveMinimum": 0 }, - "pitch_shift_semitones": { "type": "number" } + "engine": { + "description": "Superseded by models.tts.provider. Accepted, ignored.", + "enum": ["piper", "cloud"] + }, + "model": { + "description": "Superseded by models.tts.model. Accepted, ignored.", + "type": "string" + }, + "rate": { + "description": "P0. Speaking speed as a multiplier: 1.2 is a fifth faster than the voice's own pace. Handed to whichever voice runs.", + "type": "number", + "exclusiveMinimum": 0 + }, + "pitch_shift_semitones": { + "description": "RSV. Accepted, and no shipped voice honours it yet.", + "type": "number" + } } }, @@ -170,7 +184,7 @@ }, "models": { - "description": "P0. BYOK, bring your own keys. Online-first: in P0 the robot fails loudly and in character when a key or the network is missing.", + "description": "P0. BYOK, bring your own keys. Online-first: in P0 the robot fails loudly and in character when a key or the network is missing. stt resolves against emet.stt, chat against emet.llm, tts against emet.tts (model is the voice: a Piper model name, a vendor's voice id; key_env only for a cloud voice). micro is RSV. None of these is checked against what is installed: a soul is valid on every machine or on none, and the question is answered at boot.", "type": "object", "additionalProperties": false, "properties": { diff --git a/emet-sdk/tests/test_acceptance.py b/emet-sdk/tests/test_acceptance.py index 3cee313..480946a 100644 --- a/emet-sdk/tests/test_acceptance.py +++ b/emet-sdk/tests/test_acceptance.py @@ -75,6 +75,9 @@ def test_shipped_chain_files_are_valid(): ("home-out-of-range.yaml", "home_out_of_range"), ("two-drives.yaml", "multiple_drives"), ("unknown-mount.yaml", "unknown_reference"), + ("unknown-stt-provider.yaml", "missing_plugin"), + ("unknown-llm-provider.yaml", "missing_plugin"), + ("unknown-tts-provider.yaml", "missing_plugin"), ], ) def test_invalid_manifests_are_rejected(fixture: str, expected: str): diff --git a/emet-sdk/tests/test_llm.py b/emet-sdk/tests/test_llm.py new file mode 100644 index 0000000..de333c4 --- /dev/null +++ b/emet-sdk/tests/test_llm.py @@ -0,0 +1,307 @@ +"""The language model seam. + +The second provider seam, built like the first: a contract in the SDK, a +mock behind it, then the vendors. Two vendors ship in the same batch, because +a seam with one implementation is untested as a seam. + +**Note what this file imports.** Only `emet_sdk`. Every plugin it exercises +lives in `emet_providers` and arrives by discovery, which is the constraint +the engine works under. +""" + +from __future__ import annotations + +import asyncio +import sys +from pathlib import Path +from typing import Any, AsyncIterator + +import pytest + +from emet_sdk.discovery import GROUP_LLM, PluginRegistry +from emet_sdk.models import STAGES, chat_selection, model_selection, stt_selection +from emet_sdk.plugin import LanguageModelPlugin +from emet_sdk.types import ( + LanguageModelDescriptor, + Message, + Prompt, + ReplyDone, + TextDelta, + ToolCall, + ToolSpec, +) +from emet_sdk.validate import MissingPluginError, load_yaml, validate_manifest, validate_soul + +EXAMPLES = Path(__file__).resolve().parent.parent / "examples" + + +def run(coro): + return asyncio.run(coro) + + +def codes(report) -> set[str]: + return {f.code for f in report.errors} + + +def manifest(**models: Any) -> dict: + doc = {"audio": {"input": {"device": "x"}, "output": {"device": "y"}}} + if models: + doc["models"] = models + return doc + + +def soul(**models: Any) -> dict: + doc = {"identity": {"name": "Emet", "wake_word": "hey emet"}} + if models: + doc["models"] = models + return doc + + +async def collect(events: AsyncIterator) -> list: + return [e async for e in events] + + +class _FakeBatchModel(LanguageModelPlugin): + """Defined here rather than imported from emet_providers: the SDK's own + tests must not depend on the layer above it. Sends the whole text as one + delta, the way a provider that does not stream would.""" + + provider = "fake" + + def describe(self) -> LanguageModelDescriptor: + return LanguageModelDescriptor(provider=self.provider, model=self.model, streaming=False, tools=False) + + async def reply(self, prompt: Prompt): + last = prompt.messages[-1].content if prompt.messages else "" + yield TextDelta(f"You said {last}.") + yield ReplyDone(text=f"You said {last}.", stop_reason="end", model="fake-1", output_tokens=3) + + +# ---------------------------------------------------------------- discovery + + +def test_llm_is_a_real_entry_point_group(): + registry = PluginRegistry.discover() + assert GROUP_LLM == "emet.llm" + assert {"mock", "anthropic", "openai"} <= set(registry.llm_names), ( + "emet-providers registers three language models; if this fails the " + "package needs installing so its entry points are picked up" + ) + assert registry.has_llm("mock") + + +def test_llm_appears_in_the_registry_listing(): + assert not PluginRegistry(llm={}) + listed = {group for group, _ in PluginRegistry.discover()} + assert "llm" in listed + + +def test_an_uninstalled_provider_is_a_missing_plugin_not_a_schema_error(): + registry = PluginRegistry.discover() + with pytest.raises(MissingPluginError) as exc: + registry.load_llm("gemini") + assert "gemini" in str(exc.value.report) + assert "language model provider" in str(exc.value.report) + + +def test_every_shipped_provider_arrives_without_being_imported(): + registry = PluginRegistry.discover() + for name in ("mock", "anthropic", "openai"): + cls = registry.load_llm(name) + assert cls.__module__.startswith("emet_providers"), cls.__module__ + assert issubclass(cls, LanguageModelPlugin) + assert cls.provider == name + + +def test_the_engine_can_build_and_drive_a_model_it_never_imported(): + """A name from the documents, a class from the registry, `cls(config)`, + a prompt in, a streamed reply out.""" + selection = chat_selection(manifest(), soul(chat={"provider": "mock"})) + assert selection is not None + llm = PluginRegistry.discover().load_llm(selection["provider"])(selection) + prompt = Prompt(system="Be brief.", messages=(Message(role="user", content="what time is it"),)) + + async def scenario(): + await llm.start() + assert llm.describe().healthy + events = await collect(llm.reply(prompt)) + await llm.shutdown() + return events + + events = run(scenario()) + assert isinstance(events[-1], ReplyDone) + assert events[-1].stop_reason == "end" + deltas = [e for e in events if isinstance(e, TextDelta)] + assert deltas and "".join(d.text for d in deltas) == events[-1].text + assert "what time is it" in events[-1].text + + +# ----------------------------------------------------------------- contract + + +def test_a_batch_model_satisfies_the_same_contract(): + llm = _FakeBatchModel({"provider": "fake", "model": "fake-1"}) + prompt = Prompt(system="", messages=(Message(role="user", content="hello"),)) + events = run(collect(llm.reply(prompt))) + assert [type(e) for e in events] == [TextDelta, ReplyDone] + assert events[-1].text == "You said hello." + assert not llm.describe().streaming and not llm.describe().tools + assert llm.model == "fake-1" + + +def test_the_config_is_unpacked_the_way_the_engine_hands_it_over(): + llm = _FakeBatchModel({"provider": "fake", "model": "m", "key_env": "K", "params": {"x": 1}}) + assert (llm.model, llm.key_env, llm.params) == ("m", "K", {"x": 1}) + assert "key" not in {k.lower() for k in llm.config} + + +def test_reply_done_only_speaks_the_seams_stop_reasons(): + for reason in ("end", "tool", "length", "refusal", "error"): + ReplyDone(text="", stop_reason=reason) + with pytest.raises(ValueError): + ReplyDone(text="", stop_reason="max_tokens") + + +def test_messages_carry_roles_the_providers_can_map(): + Message(role="user", content="hi") + Message(role="assistant", content="", tool_calls=(ToolCall(id="c1", name="remember"),)) + Message(role="tool", content="ok", tool_call_id="c1") + with pytest.raises(ValueError): + Message(role="system", content="no: the system prompt lives on the Prompt") + with pytest.raises(ValueError): + Message(role="tool", content="ok") + with pytest.raises(ValueError): + Message(role="user", content="x", tool_calls=(ToolCall(id="c", name="n"),)) + + +def test_a_prompt_bounds_its_reply_and_a_tool_spec_defaults_to_no_parameters(): + prompt = Prompt(system="s", messages=(), tools=(ToolSpec(name="remember", description="keep a fact"),)) + assert prompt.max_tokens == 1024 + assert prompt.tools[0].parameters == {"type": "object", "properties": {}} + with pytest.raises(ValueError): + Prompt(system="s", messages=(), max_tokens=0) + + +def test_a_language_model_is_a_plugin_with_the_shared_lifecycle(): + from emet_sdk.plugin import Plugin + + llm = _FakeBatchModel({"provider": "fake"}) + assert isinstance(llm, Plugin) + assert llm.health().ok + run(llm.start()) + run(llm.shutdown()) + + +# ---------------------------------------------------------------- selection + + +def test_one_rule_for_every_stage(): + assert STAGES == {"stt", "chat", "tts", "micro"} + soul_doc = soul(chat={"provider": "openai", "model": "m", "key_env": "EMET_OPENAI_KEY"}) + assert chat_selection(manifest(), soul_doc) == { + "provider": "openai", + "model": "m", + "key_env": "EMET_OPENAI_KEY", + "params": {}, + } + assert model_selection(manifest(), soul_doc, "chat") == chat_selection(manifest(), soul_doc) + assert stt_selection(manifest(), soul_doc) is None + + +def test_the_body_takes_a_stage_over_whole_or_not_at_all(): + chosen = chat_selection( + manifest(chat={"provider": "mock", "params": {"reply": "hi"}}), + soul(chat={"provider": "openai", "model": "m", "key_env": "EMET_OPENAI_KEY"}), + ) + assert chosen == {"provider": "mock", "model": None, "key_env": None, "params": {"reply": "hi"}} + + +def test_the_body_tunes_the_souls_choice_of_model(): + chosen = chat_selection( + manifest(chat={"params": {"effort": "low"}}), soul(chat={"provider": "anthropic"}) + ) + assert chosen is not None + assert chosen["provider"] == "anthropic" and chosen["params"] == {"effort": "low"} + + +def test_stages_are_independent(): + """Taking speech recognition over says nothing about the language model.""" + chosen = chat_selection(manifest(stt={"provider": "mock"}), soul(chat={"provider": "anthropic"})) + assert chosen is not None and chosen["provider"] == "anthropic" + assert chat_selection(manifest(stt={"provider": "mock"}), soul()) is None + + +def test_an_unknown_stage_is_a_programming_error(): + with pytest.raises(ValueError, match="stage"): + model_selection(manifest(), soul(), "vision") + + +def test_the_reference_soul_names_a_language_model_that_ships(): + """The reference soul is what newcomers copy, so it names a provider that + is installed, with the model that plugin would use unasked.""" + doc = load_yaml(EXAMPLES / "emet-soul.yaml") + chosen = chat_selection(load_yaml(EXAMPLES / "bodiless.yaml"), doc) + assert chosen is not None and chosen["provider"] == "openai" + assert chosen["key_env"] == "EMET_OPENAI_KEY" + cls = PluginRegistry.discover().load_llm("openai") + assert chosen["model"] == getattr(sys.modules[cls.__module__], "DEFAULT_MODEL") + + +# --------------------------------------------------------------- validation + + +def test_a_body_naming_an_uninstalled_language_model_is_an_error(): + doc = load_yaml(EXAMPLES / "bodiless.yaml") + doc["models"] = {"chat": {"provider": "abacus"}} + report = validate_manifest(doc) + assert not report.ok + assert "missing_plugin" in codes(report) + assert any(f.path == "/models/chat/provider" for f in report.errors) + + +def test_the_invalid_fixture_says_the_same(): + report = validate_manifest(load_yaml(EXAMPLES / "invalid" / "unknown-llm-provider.yaml")) + assert not report.ok and "missing_plugin" in codes(report) + + +def test_a_body_naming_shipped_providers_validates_clean(): + doc = load_yaml(EXAMPLES / "bodiless.yaml") + doc["models"] = { + "stt": {"provider": "mock"}, + "chat": {"provider": "anthropic", "params": {"effort": "low"}}, + } + assert validate_manifest(doc).ok + + +def test_the_reserved_stage_is_accepted_and_not_checked(): + """`micro` resolves against nothing yet, so a body may name anything + there and the schema still accepts the shape. `tts` used to be here and + is checked since the voice seam arrived.""" + doc = load_yaml(EXAMPLES / "bodiless.yaml") + doc["models"] = {"micro": {"provider": "nobody_yet", "params": {"x": 1}}} + assert validate_manifest(doc).ok + + +def test_an_unknown_key_under_a_stage_is_a_schema_error(): + doc = load_yaml(EXAMPLES / "bodiless.yaml") + doc["models"] = {"chat": {"provider": "mock", "api_key": "never-in-a-manifest"}} + assert "schema" in codes(validate_manifest(doc)) + + +def test_an_unknown_stage_is_a_schema_error(): + doc = load_yaml(EXAMPLES / "bodiless.yaml") + doc["models"] = {"vision": {"provider": "mock"}} + assert "schema" in codes(validate_manifest(doc)) + + +def test_a_soul_naming_an_uninstalled_language_model_still_validates(): + doc = load_yaml(EXAMPLES / "emet-soul.yaml") + doc["models"]["chat"] = {"provider": "abacus", "key_env": "X"} + assert validate_soul(doc).ok + + +def test_the_mocked_example_body_takes_both_stages_over(): + doc = load_yaml(EXAMPLES / "mock-scout.yaml") + assert doc["models"]["stt"]["provider"] == "mock" + assert doc["models"]["chat"]["provider"] == "mock" + assert validate_manifest(doc).ok diff --git a/emet-sdk/tests/test_stt.py b/emet-sdk/tests/test_stt.py new file mode 100644 index 0000000..e4576a4 --- /dev/null +++ b/emet-sdk/tests/test_stt.py @@ -0,0 +1,332 @@ +"""The speech recognition seam. + +Built before any provider exists, on purpose. There are credits enough at one +vendor to write the whole of 0.4 against its client library without +noticing, and the seam is what makes that vendor one line of a soul instead +of the shape of the engine. + +**Note what this file imports.** Only `emet_sdk`. The one transcriber that +ships lives in `emet_providers`, and it is never named here: it arrives by +discovery, which is the constraint the engine works under, so these tests +fail the way the engine would. +""" + +from __future__ import annotations + +import asyncio +import sys +from pathlib import Path +from typing import Any + +import pytest + +from emet_sdk.discovery import GROUP_STT, PluginRegistry +from emet_sdk.models import stt_selection +from emet_sdk.plugin import TranscriberPlugin +from emet_sdk.types import AudioFormat, Transcript, TranscriberDescriptor +from emet_sdk.validate import MissingPluginError, load_yaml, validate_manifest, validate_soul + +EXAMPLES = Path(__file__).resolve().parent.parent / "examples" +FRAME_BYTES = 2560 + + +def run(coro): + return asyncio.run(coro) + + +def frame(payload: bytes = b"") -> bytes: + return payload + bytes(FRAME_BYTES - len(payload)) + + +def codes(report) -> set[str]: + return {f.code for f in report.errors} + + +def manifest(**stt: Any) -> dict: + doc = {"audio": {"input": {"device": "x"}, "output": {"device": "y"}}} + if stt: + doc["models"] = {"stt": stt} + return doc + + +def soul(**stt: Any) -> dict: + doc = {"identity": {"name": "Emet", "wake_word": "hey emet"}} + if stt: + doc["models"] = {"stt": stt} + return doc + + +class _FakeTranscriber(TranscriberPlugin): + """Defined here rather than imported from emet_providers: the SDK's own + tests must not depend on the layer above it.""" + + provider = "fake" + + def __init__(self, config, fmt): + super().__init__(config, fmt) + self.heard: list[bytes] = [] + + def describe(self) -> TranscriberDescriptor: + return TranscriberDescriptor( + provider=self.provider, + model=self.model, + streaming=False, + sample_rate=self.format.sample_rate, + ) + + async def feed(self, frame: bytes) -> Transcript | None: + self.heard.append(frame) + return None + + async def finish(self) -> Transcript: + n = len(self.heard) + self.heard = [] + return Transcript(text=f"{n} frames", final=True) + + +# ---------------------------------------------------------------- discovery + + +def test_stt_is_a_real_entry_point_group(): + registry = PluginRegistry.discover() + assert GROUP_STT == "emet.stt" + assert "mock" in registry.stt_names, ( + "emet-providers registers a mock transcriber; if this fails the package " + "needs installing so its entry points are picked up" + ) + assert registry.has_stt("mock") + + +def test_stt_appears_in_the_registry_listing(): + assert not PluginRegistry(stt={}) + listed = {group for group, _ in PluginRegistry.discover()} + assert "stt" in listed + + +def test_an_uninstalled_provider_is_a_missing_plugin_not_a_schema_error(): + """`provider: whisper` is a legal value the day before anyone packages + it, exactly as `provider: deepgram` was the day before it shipped.""" + registry = PluginRegistry.discover() + with pytest.raises(MissingPluginError) as exc: + registry.load_stt("whisper") + assert "whisper" in str(exc.value.report) + assert "speech recognition provider" in str(exc.value.report) + + +def test_a_transcriber_arrives_without_being_imported(): + """The crux. This module imports `emet_sdk` only, and still ends up + holding a class defined in `emet_providers`.""" + cls = PluginRegistry.discover().load_stt("mock") + assert cls.__module__.startswith("emet_providers"), cls.__module__ + assert issubclass(cls, TranscriberPlugin) + + +def test_the_engine_can_build_and_drive_a_transcriber_it_never_imported(): + """What the listen loop does, minus the loop: a name from the documents, + a class from the registry, `cls(config, fmt)`, frames in, text out.""" + selection = stt_selection(manifest(), soul(provider="mock")) + assert selection is not None + cls = PluginRegistry.discover().load_stt(selection["provider"]) + stt = cls(selection, AudioFormat()) + + async def scenario(): + await stt.start() + assert stt.describe().healthy + partials = [] + for f in (frame(b"what"), frame(b"time"), frame(), frame(b"is it")): + if (t := await stt.feed(f)) is not None: + partials.append(t) + final = await stt.finish() + await stt.shutdown() + return partials, final + + partials, final = run(scenario()) + assert partials and all(not p.final for p in partials) + assert partials[-1].text == "what time is it" + assert final.final and final.text == "what time is it" + + +# ----------------------------------------------------------------- contract + + +def test_a_batch_transcriber_satisfies_the_same_contract(): + """Returns nothing from `feed()` and does its work in `finish()`. The + engine above cannot tell it from a streaming one except by latency.""" + stt = _FakeTranscriber({"provider": "fake", "model": "m"}, AudioFormat()) + + async def scenario(): + await stt.start() + results = [await stt.feed(frame()) for _ in range(3)] + return results, await stt.finish() + + results, final = run(scenario()) + assert results == [None, None, None] + assert final == Transcript(text="3 frames", final=True) + assert not stt.describe().streaming + assert stt.describe().model == "m" + + +def test_the_format_is_received_not_stated(): + """One microphone feeds the wake engine and the transcriber, so the wake + engine fixes the format and the transcriber is handed it.""" + fmt = AudioFormat(sample_rate=8000, frame_samples=640) + stt = _FakeTranscriber({"provider": "fake"}, fmt) + assert stt.format == fmt + assert stt.describe().sample_rate == 8000 + + +def test_the_key_is_never_in_the_config_only_its_name(): + stt = _FakeTranscriber( + {"provider": "fake", "key_env": "EMET_FAKE_KEY", "params": {"region": "eu"}}, + AudioFormat(), + ) + assert stt.key_env == "EMET_FAKE_KEY" + assert "key" not in {k.lower() for k in stt.config} + assert stt.params == {"region": "eu"} + + +def test_a_transcript_is_partial_by_default_and_bounded(): + assert not Transcript(text="so far").final + Transcript(text="x", confidence=0.0) + Transcript(text="x", confidence=1.0) + with pytest.raises(ValueError): + Transcript(text="x", confidence=1.2) + + +def test_a_transcriber_is_a_plugin_with_the_shared_lifecycle(): + from emet_sdk.plugin import Plugin + + stt = _FakeTranscriber({"provider": "fake"}, AudioFormat()) + assert isinstance(stt, Plugin) + assert stt.health().ok + run(stt.start()) + run(stt.shutdown()) + + +# ---------------------------------------------------------------- selection + + +def test_the_soul_chooses_when_the_body_says_nothing(): + chosen = stt_selection( + manifest(), soul(provider="deepgram", model="nova-2", key_env="EMET_DEEPGRAM_KEY") + ) + assert chosen == { + "provider": "deepgram", + "model": "nova-2", + "key_env": "EMET_DEEPGRAM_KEY", + "params": {}, + } + + +def test_the_body_tunes_the_souls_choice(): + """`audio.stt.params` ride along with whichever provider runs.""" + chosen = stt_selection( + manifest(params={"endpoint": "eu"}), soul(provider="deepgram", model="nova-2") + ) + assert chosen is not None + assert chosen["provider"] == "deepgram" + assert chosen["params"] == {"endpoint": "eu"} + + +def test_the_body_takes_over_whole_or_not_at_all(): + """A provider from one document with a model and key from the other is a + broken reference, so a body override replaces all three together.""" + chosen = stt_selection( + manifest(provider="mock", params={"transcript": "hi"}), + soul(provider="deepgram", model="nova-2", key_env="EMET_DEEPGRAM_KEY"), + ) + assert chosen == {"provider": "mock", "model": None, "key_env": None, "params": {"transcript": "hi"}} + + +def test_a_body_override_may_carry_its_own_model_and_key(): + chosen = stt_selection( + manifest(provider="whisper", model="small", key_env="EMET_OPENAI_KEY"), + soul(provider="deepgram"), + ) + assert chosen is not None + assert (chosen["provider"], chosen["model"], chosen["key_env"]) == ( + "whisper", + "small", + "EMET_OPENAI_KEY", + ) + + +def test_no_provider_anywhere_means_no_transcription_and_no_default(): + assert stt_selection(manifest(), soul()) is None + assert stt_selection(manifest(params={"x": 1}), soul()) is None + assert stt_selection({}, {}) is None + + +def test_the_reference_soul_names_a_provider_that_ships(): + """The reference soul is what newcomers copy, so it names a real + provider, and that provider is installed. Its model is the one the plugin + would use unasked, so the example and the default cannot drift apart.""" + doc = load_yaml(EXAMPLES / "emet-soul.yaml") + chosen = stt_selection(load_yaml(EXAMPLES / "bodiless.yaml"), doc) + assert chosen is not None and chosen["provider"] == "deepgram" + assert chosen["key_env"] == "EMET_DEEPGRAM_KEY" + registry = PluginRegistry.discover() + assert registry.has_stt("deepgram") + cls = registry.load_stt("deepgram") + assert cls.__module__.startswith("emet_providers") + assert chosen["model"] == getattr(sys.modules[cls.__module__], "DEFAULT_MODEL") + + +# --------------------------------------------------------------- validation + + +def test_an_absent_body_block_is_the_common_case(): + report = validate_manifest(load_yaml(EXAMPLES / "bodiless.yaml")) + assert report.ok, report.errors + assert "models" not in load_yaml(EXAMPLES / "bodiless.yaml") + + +def test_a_body_naming_an_uninstalled_provider_is_an_error(): + """Like `wake.engine`, `input.source` and `output.sink`: a body is one + machine, and it has named software that has to be installed on it.""" + doc = load_yaml(EXAMPLES / "bodiless.yaml") + doc["models"] = {"stt": {"provider": "carrier_pigeon"}} + report = validate_manifest(doc) + assert not report.ok + assert "missing_plugin" in codes(report) + assert any(f.path == "/models/stt/provider" for f in report.errors) + + +def test_the_invalid_fixture_says_the_same(): + report = validate_manifest(load_yaml(EXAMPLES / "invalid" / "unknown-stt-provider.yaml")) + assert not report.ok + assert "missing_plugin" in codes(report) + + +def test_a_body_naming_the_shipped_provider_validates_clean(): + doc = load_yaml(EXAMPLES / "bodiless.yaml") + doc["models"] = {"stt": {"provider": "mock", "params": {"transcript": "hello"}}} + assert validate_manifest(doc).ok + + +def test_the_mocked_example_body_takes_speech_recognition_over(): + doc = load_yaml(EXAMPLES / "mock-scout.yaml") + assert doc["models"]["stt"]["provider"] == "mock" + assert validate_manifest(doc).ok + + +def test_a_body_may_carry_params_alone(): + doc = load_yaml(EXAMPLES / "bodiless.yaml") + doc["models"] = {"stt": {"params": {"endpoint": "eu"}}} + assert validate_manifest(doc).ok + + +def test_an_unknown_key_under_the_body_block_is_a_schema_error(): + doc = load_yaml(EXAMPLES / "bodiless.yaml") + doc["models"] = {"stt": {"provider": "mock", "api_key": "never-in-a-manifest"}} + report = validate_manifest(doc) + assert "schema" in codes(report) + + +def test_a_soul_naming_an_uninstalled_provider_still_validates(): + """A soul is valid on every machine or on none. Whether its provider is + installed is a question for boot, like whether its wake phrase can be + heard.""" + doc = load_yaml(EXAMPLES / "emet-soul.yaml") + doc["models"]["stt"] = {"provider": "nobody_ships_this", "key_env": "X"} + assert validate_soul(doc).ok diff --git a/emet-sdk/tests/test_tts.py b/emet-sdk/tests/test_tts.py new file mode 100644 index 0000000..9d33155 --- /dev/null +++ b/emet-sdk/tests/test_tts.py @@ -0,0 +1,297 @@ +"""The speech synthesis seam. + +The third provider seam, built like the other two: a contract in the SDK, a +mock behind it, then the local voice and one cloud voice. The design keeps +synthesis local by default, so the reference soul names the local one. + +**Note what this file imports.** Only `emet_sdk`. Every plugin it exercises +lives in `emet_providers` and arrives by discovery, which is the constraint +the engine works under. +""" + +from __future__ import annotations + +import asyncio +import sys +from pathlib import Path +from typing import Any, AsyncIterator + +import pytest + +from emet_sdk.discovery import GROUP_TTS, PluginRegistry +from emet_sdk.models import STAGES, model_selection, tts_selection +from emet_sdk.plugin import PluginError, VoicePlugin +from emet_sdk.types import VoiceDescriptor +from emet_sdk.validate import ( + BUILTIN_TTS, + MissingPluginError, + load_yaml, + validate_manifest, + validate_soul, +) + +EXAMPLES = Path(__file__).resolve().parent.parent / "examples" + + +def run(coro): + return asyncio.run(coro) + + +def codes(report) -> set[str]: + return {f.code for f in report.errors} + + +def manifest(**models: Any) -> dict: + doc = {"audio": {"input": {"device": "x"}, "output": {"device": "y"}}} + if models: + doc["models"] = models + return doc + + +def soul(voice: dict | None = None, **models: Any) -> dict: + doc: dict = {"identity": {"name": "Emet", "wake_word": "hey emet"}} + if voice is not None: + doc["voice"] = voice + if models: + doc["models"] = models + return doc + + +async def collect(chunks: AsyncIterator[bytes]) -> list[bytes]: + return [c async for c in chunks] + + +class _FakeBatchVoice(VoicePlugin): + """Defined here rather than imported from emet_providers: the SDK's own + tests must not depend on the layer above it. Yields a sentence as one + chunk, the way a voice that does not stream would.""" + + provider = "fake" + + def describe(self) -> VoiceDescriptor: + return VoiceDescriptor(provider=self.provider, model=self.model, sample_rate=16000, streaming=False) + + async def speak(self, text: str): + if not text.strip(): + return + if text == "fail": + yield b"\x01\x00" + raise PluginError("the fake voice lost its breath") + yield bytes(2 * len(text)) + + +# ---------------------------------------------------------------- discovery + + +def test_tts_is_a_real_entry_point_group(): + registry = PluginRegistry.discover() + assert GROUP_TTS == "emet.tts" + assert BUILTIN_TTS <= set(registry.tts_names), ( + "emet-providers registers three voices; if this fails the package " + "needs installing so its entry points are picked up" + ) + assert registry.has_tts("mock") + + +def test_tts_appears_in_the_registry_listing(): + assert not PluginRegistry(tts={}) + listed = {group for group, _ in PluginRegistry.discover()} + assert "tts" in listed + + +def test_an_uninstalled_voice_is_a_missing_plugin_not_a_schema_error(): + registry = PluginRegistry.discover() + with pytest.raises(MissingPluginError) as exc: + registry.load_tts("gramophone") + assert "gramophone" in str(exc.value.report) + assert "speech synthesis provider" in str(exc.value.report) + + +def test_every_shipped_voice_arrives_without_being_imported(): + registry = PluginRegistry.discover() + for name in sorted(BUILTIN_TTS): + cls = registry.load_tts(name) + assert cls.__module__.startswith("emet_providers"), cls.__module__ + assert issubclass(cls, VoicePlugin) + assert cls.provider == name + + +def test_the_engine_can_build_and_drive_a_voice_it_never_imported(): + """A name from the documents, a class from the registry, + `cls(config, voice)`, a sentence in, audio out.""" + selection = tts_selection(manifest(), soul(tts={"provider": "mock"})) + assert selection is not None + voice = PluginRegistry.discover().load_tts(selection["provider"])(selection, {"rate": 1.0}) + + async def scenario(): + await voice.start() + described = voice.describe() + assert described.healthy and described.sample_rate > 0 + chunks = await collect(voice.speak("what time is it")) + await voice.shutdown() + return chunks + + chunks = run(scenario()) + assert chunks and all(isinstance(c, bytes) for c in chunks) + assert all(len(c) % 2 == 0 for c in chunks), "int16 audio has an even number of bytes" + + +# ----------------------------------------------------------------- contract + + +def test_a_batch_voice_satisfies_the_same_contract(): + voice = _FakeBatchVoice({"provider": "fake", "model": "fake-1"}) + chunks = run(collect(voice.speak("hello"))) + assert chunks == [bytes(10)] + assert not voice.describe().streaming + assert voice.model == "fake-1" + + +def test_blank_text_yields_nothing(): + voice = _FakeBatchVoice({"provider": "fake"}) + assert run(collect(voice.speak(" "))) == [] + + +def test_a_failure_mid_sentence_raises_after_the_audio_so_far(): + voice = _FakeBatchVoice({"provider": "fake"}) + + async def scenario(): + got = [] + with pytest.raises(PluginError, match="breath"): + async for chunk in voice.speak("fail"): + got.append(chunk) + return got + + assert run(scenario()) == [b"\x01\x00"] + + +def test_the_config_and_the_souls_voice_block_are_unpacked_as_the_engine_hands_them_over(): + voice = _FakeBatchVoice( + {"provider": "fake", "model": "m", "key_env": "K", "params": {"x": 1}}, + {"rate": 1.25, "pitch_shift_semitones": 2}, + ) + assert (voice.model, voice.key_env, voice.params) == ("m", "K", {"x": 1}) + assert voice.rate == 1.25 + assert voice.voice["pitch_shift_semitones"] == 2 + assert "key" not in {k.lower() for k in voice.config} + + +def test_the_rate_defaults_to_the_voices_own_pace(): + assert _FakeBatchVoice({"provider": "fake"}).rate == 1.0 + assert _FakeBatchVoice({"provider": "fake"}, {}).rate == 1.0 + assert _FakeBatchVoice({"provider": "fake"}, {"rate": 0}).rate == 1.0 + assert _FakeBatchVoice({"provider": "fake"}, {"rate": "fast"}).rate == 1.0 + + +def test_a_voice_descriptor_states_the_rate_the_sink_must_open_at(): + d = VoiceDescriptor(provider="fake") + assert d.sample_rate == 22050 and not d.streaming and d.healthy + assert VoiceDescriptor(provider="fake", sample_rate=24000, streaming=True).sample_rate == 24000 + + +def test_a_voice_is_a_plugin_with_the_shared_lifecycle(): + from emet_sdk.plugin import Plugin + + voice = _FakeBatchVoice({"provider": "fake"}) + assert isinstance(voice, Plugin) + assert voice.health().ok + run(voice.start()) + run(voice.shutdown()) + + +# ---------------------------------------------------------------- selection + + +def test_the_voice_is_chosen_by_the_one_rule(): + assert "tts" in STAGES + soul_doc = soul(tts={"provider": "piper", "model": "en_US-ljspeech-medium"}) + assert tts_selection(manifest(), soul_doc) == { + "provider": "piper", + "model": "en_US-ljspeech-medium", + "key_env": None, + "params": {}, + } + assert model_selection(manifest(), soul_doc, "tts") == tts_selection(manifest(), soul_doc) + + +def test_the_body_takes_the_voice_over_whole_or_not_at_all(): + chosen = tts_selection( + manifest(tts={"provider": "mock", "params": {"sample_rate": 8000}}), + soul(tts={"provider": "deepgram", "model": "aura-2-thalia-en", "key_env": "EMET_DEEPGRAM_KEY"}), + ) + assert chosen == {"provider": "mock", "model": None, "key_env": None, "params": {"sample_rate": 8000}} + + +def test_the_body_tunes_the_souls_voice(): + chosen = tts_selection( + manifest(tts={"params": {"voices_dir": "/etc/emet/voices"}}), + soul(tts={"provider": "piper", "model": "en_US-ljspeech-medium"}), + ) + assert chosen is not None + assert chosen["provider"] == "piper" and chosen["params"] == {"voices_dir": "/etc/emet/voices"} + + +def test_there_is_no_default_voice(): + assert tts_selection(manifest(), soul()) is None + assert tts_selection(manifest(stt={"provider": "mock"}), soul(chat={"provider": "mock"})) is None + + +def test_the_reference_soul_names_a_local_voice_that_ships(): + """The design keeps synthesis local by default, and the reference soul + is what newcomers copy, so it names the local voice, with the model that + plugin would use unasked.""" + doc = load_yaml(EXAMPLES / "emet-soul.yaml") + chosen = tts_selection(load_yaml(EXAMPLES / "bodiless.yaml"), doc) + assert chosen is not None and chosen["provider"] == "piper" + assert chosen["key_env"] is None, "a local voice needs no key" + cls = PluginRegistry.discover().load_tts("piper") + assert chosen["model"] == getattr(sys.modules[cls.__module__], "DEFAULT_MODEL") + + +def test_the_reference_souls_voice_block_carries_the_rate_and_no_engine(): + """`voice.engine` and `voice.model` were the pre-seam way to name a + voice. `models.tts` is the way now, so the reference soul does not set + them; the schema still accepts them for older bundles.""" + doc = load_yaml(EXAMPLES / "emet-soul.yaml") + assert doc["voice"]["rate"] == 1.0 + assert "engine" not in doc["voice"] and "model" not in doc["voice"] + assert validate_soul(doc).ok + doc["voice"]["engine"] = "piper" + doc["voice"]["model"] = "en_US-amy-medium" + assert validate_soul(doc).ok, "older bundles still validate" + + +# --------------------------------------------------------------- validation + + +def test_a_body_naming_an_uninstalled_voice_is_an_error(): + doc = load_yaml(EXAMPLES / "bodiless.yaml") + doc["models"] = {"tts": {"provider": "gramophone"}} + report = validate_manifest(doc) + assert not report.ok + assert "missing_plugin" in codes(report) + assert any(f.path == "/models/tts/provider" for f in report.errors) + + +def test_the_invalid_fixture_says_the_same(): + report = validate_manifest(load_yaml(EXAMPLES / "invalid" / "unknown-tts-provider.yaml")) + assert not report.ok and "missing_plugin" in codes(report) + + +def test_a_body_naming_shipped_voices_validates_clean(): + doc = load_yaml(EXAMPLES / "bodiless.yaml") + for name in sorted(BUILTIN_TTS): + doc["models"] = {"tts": {"provider": name, "params": {"x": 1}}} + assert validate_manifest(doc).ok, name + + +def test_a_soul_naming_an_uninstalled_voice_still_validates(): + doc = load_yaml(EXAMPLES / "emet-soul.yaml") + doc["models"]["tts"] = {"provider": "gramophone"} + assert validate_soul(doc).ok + + +def test_the_mocked_example_body_takes_all_three_stages_over(): + doc = load_yaml(EXAMPLES / "mock-scout.yaml") + assert doc["models"]["tts"]["provider"] == "mock" + assert validate_manifest(doc).ok diff --git a/tools/check_layering.py b/tools/check_layering.py index b9fb0f3..fff6711 100644 --- a/tools/check_layering.py +++ b/tools/check_layering.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 """Assert the package layering that the architecture depends on. - emet_sdk imports nothing internal - emet_hal imports emet_sdk only - emet_engine imports emet_sdk only + emet_sdk imports nothing internal + emet_hal imports emet_sdk only + emet_providers imports emet_sdk only + emet_engine imports emet_sdk only Under the earlier closed-engine plan this was structural: an outside contributor had no engine source to couple to. In a monorepo with everything @@ -34,6 +35,7 @@ ALLOWED: dict[str, frozenset[str]] = { "emet_sdk": frozenset(), "emet_hal": frozenset({"emet_sdk"}), + "emet_providers": frozenset({"emet_sdk"}), "emet_engine": frozenset({"emet_sdk"}), } diff --git a/tools/release_check.py b/tools/release_check.py index 3801472..35de55e 100644 --- a/tools/release_check.py +++ b/tools/release_check.py @@ -10,10 +10,10 @@ that and duplicating it here would produce a slow script people stop running. It checks the *cross-cutting* invariants: - versions all three packages agree + versions all four packages agree plugins every discovery group has a shipped implementation docs nothing advertises a version the code no longer is - hal readme every shipped entry point is named in emet-hal/README.md + readmes every shipped entry point is named in its package's README markers no TODO or FIXME left in shipped source **Why this exists.** 0.3's scope was "audio in/out, wake word, VAD, @@ -36,11 +36,17 @@ import tomllib from pathlib import Path -PACKAGES = ("emet-sdk", "emet-hal", "emet-engine") +PACKAGES = ("emet-sdk", "emet-hal", "emet-providers", "emet-engine") #: Docs that state a version and will lie if they are not updated. A stale #: README is the first thing a newcomer reads and the last thing anybody edits. -VERSIONED_DOCS = ("README.md", "emet-sdk/README.md", "emet-hal/README.md", "emet-engine/README.md") +VERSIONED_DOCS = ( + "README.md", + "emet-sdk/README.md", + "emet-hal/README.md", + "emet-providers/README.md", + "emet-engine/README.md", +) #: Groups the SDK knows how to discover. Each should have at least one shipped #: implementation, or the group is a promise nothing keeps. @@ -51,6 +57,9 @@ "emet.wake", "emet.audio", "emet.audio_out", + "emet.stt", + "emet.llm", + "emet.tts", ) problems: list[str] = [] @@ -78,7 +87,7 @@ def declared_versions(root: Path) -> dict[str, str]: def check_versions_agree(versions: dict[str, str]) -> str | None: - """One repository, one version. Three packages released together that + """One repository, one version. Four packages released together that disagree about which release they are is the kind of thing nobody notices until a bug report quotes two of them.""" distinct = set(versions.values()) @@ -134,26 +143,28 @@ def check_groups(root: Path) -> None: notes.append(f"{total} entry points across {len(provided)} groups") -def check_hal_readme(root: Path) -> None: - """Every entry point emet-hal registers is named in its README. +def check_plugin_readmes(root: Path) -> None: + """Every entry point a package registers is named in that package's README. - 0.3 shipped seven new plugins and the README's "what ships" table kept + 0.3 shipped seven new plugins and emet-hal's "what ships" table kept listing the four from 0.2, through several rounds of "what is left". - A newcomer reads that table before anything else. + A newcomer reads that table before anything else. emet-providers has the + same table and the same exposure, so the rule covers every package. """ - pyproject = root / "emet-hal" / "pyproject.toml" - readme = root / "emet-hal" / "README.md" - if not pyproject.exists() or not readme.exists(): - return - data = tomllib.loads(pyproject.read_text(encoding="utf-8")) - text = readme.read_text(encoding="utf-8") - for group, entries in (data.get("project", {}).get("entry-points") or {}).items(): - for name in entries: - if f"`{name}`" not in text: - problem( - f"emet-hal/README.md does not name the {group} entry point `{name}`. " - f"The 'what ships' table is the first thing a contributor reads." - ) + for pkg in PACKAGES: + pyproject = root / pkg / "pyproject.toml" + readme = root / pkg / "README.md" + if not pyproject.exists() or not readme.exists(): + continue + data = tomllib.loads(pyproject.read_text(encoding="utf-8")) + text = readme.read_text(encoding="utf-8") + for group, entries in (data.get("project", {}).get("entry-points") or {}).items(): + for name in entries: + if f"`{name}`" not in text: + problem( + f"{pkg}/README.md does not name the {group} entry point `{name}`. " + f"The 'what ships' table is the first thing a contributor reads." + ) def check_docs(root: Path, version: str | None) -> None: @@ -201,7 +212,7 @@ def main(argv: list[str]) -> int: check_single_declaration(root) check_groups(root) check_docs(root, version) - check_hal_readme(root) + check_plugin_readmes(root) check_markers(root) for note in notes: diff --git a/tools/tag_release.py b/tools/tag_release.py index 5e5afe6..efa6684 100644 --- a/tools/tag_release.py +++ b/tools/tag_release.py @@ -2,7 +2,7 @@ """Make the release tag the way the project makes release tags. Signed, annotated, from the release-notes file, on a clean master that -matches origin, at the version the three packages declare. Every check here +matches origin, at the version the four packages declare. Every check here is a mistake that has been made or nearly made: unsigned tag v0.3 first went up with `git tag -a`, no `-s`, and showed @@ -33,7 +33,7 @@ import tomllib from pathlib import Path -PACKAGES = ("emet-sdk", "emet-hal", "emet-engine") +PACKAGES = ("emet-sdk", "emet-hal", "emet-providers", "emet-engine") def git(root: Path, *args: str) -> str: @@ -72,7 +72,7 @@ def main(argv: list[str]) -> int: if not first.startswith(f"{series}:"): return fail(f"first line of {notes.name} is {first!r}; expected it to start with {series + ':'!r}") - # The three packages agree with the tag. + # The four packages agree with the tag. for pkg in PACKAGES: data = tomllib.loads((root / pkg / "pyproject.toml").read_text(encoding="utf-8")) declared = data["project"]["version"]