diff --git a/.gitignore b/.gitignore index e571e019..d26e6caa 100644 --- a/.gitignore +++ b/.gitignore @@ -26,5 +26,6 @@ myenv # installation flags and installed libraries installers/i2s_mic_installed.flag installers/i2s_speech_recognition_installed.flag +installers/audio/backups/* speech.wav diff --git a/environments/voice.yml b/environments/voice.yml new file mode 100644 index 00000000..c07181a4 --- /dev/null +++ b/environments/voice.yml @@ -0,0 +1,24 @@ +# Laptop — local development and testing environment with no hardware modules. +# Useful for developing, testing, and running the Discord bot locally. +# Matches modules configured with environment: 'laptop' or without 'robot' restriction. + +# ── Core ────────────────────────────────────────────────────────────────────── + +messaging_service: + enabled: true + +logwrapper: + enabled: true + +speechinput: + enabled: true + config: + wake_word: ['cody','kodi'] # if set, will only process speech that contains this word. Can be used to avoid processing random speech in the environment. + start_on_boot: true + +chatgpt: + enabled: true + +ttsmodule: + enabled: false # need to add credit + diff --git a/install_audio.sh b/install_audio.sh new file mode 100644 index 00000000..3809a516 --- /dev/null +++ b/install_audio.sh @@ -0,0 +1,22 @@ + +sudo python3 -m pip install --upgrade click --break-system-packages + +sudo python3 -m pip install --upgrade setuptools --break-system-packages +sudo python3 -m pip install --upgrade adafruit-python-shell --break-system-packages + +sudo python3 installers/i2samp.py + +# Example aplay -l output on CM5 before googlevoicehat overlay takes effect (first boot after install): +# card 0: vc4hdmi0 [vc4-hdmi-0], device 0: MAI PCM i2s-hifi-0 [MAI PCM i2s-hifi-0] +# card 1: vc4hdmi1 [vc4-hdmi-1], device 0: MAI PCM i2s-hifi-0 [MAI PCM i2s-hifi-0] +# card 2: I2S [Dual I2S], device 0: 1f000a0000.i2s-dit-hifi dit-hifi-0 [...] +# +# After reboot with googlevoicehat overlay active - card numbers can vary between boots! +# card 0: vc4hdmi0 [vc4-hdmi-0], ... +# card 1: vc4hdmi1 [vc4-hdmi-1], ... +# card 2: sndrpigooglevoi [snd_rpi_googlevoicehat_soundcar], device 0: Google voiceHAT SoundCard HiFi ... +# OR (on a subsequent restart): +# card 1: sndrpigooglevoi [snd_rpi_googlevoicehat_soundcar], device 0: Google voiceHAT SoundCard HiFi ... +# card 2: vc4hdmi1 [vc4-hdmi-1], ... +# +# This is why asound.conf must reference the card by NAME (sndrpigooglevoi) not by number. diff --git a/installers/audio/Audio.md b/installers/audio/Audio.md new file mode 100644 index 00000000..94e626af --- /dev/null +++ b/installers/audio/Audio.md @@ -0,0 +1,163 @@ +# I2S Audio — CM5 Carrier Board + +## Hardware + +| Signal | GPIO | Connected to | +|--------|------|--------------| +| BCLK (I2S clock) | 18 | MAX98357A + ICS-43434 | +| LRCLK (word select) | 19 | MAX98357A + ICS-43434 | +| SDO (TX, data out) | 21 | MAX98357A DIN | +| SDI (RX, data in) | 20 | ICS-43434 DATA (both mics) | +| SD_MODE (amp enable) | 16 | MAX98357A SD | + +Two ICS-43434 I2S microphones share the same DATA line (GPIO 20). One has its L/R SELECT pin tied to GND (left channel), the other to 3.3 V (right channel), so they form a stereo pair on a single wire. + +The CM5 I2S controller is the Synopsys DesignWare (`snps,designware-i2s`) on the RP1 chip. It supports **`S32_LE` at `48000 Hz` only**. + +--- + +## Device Tree Overlay + +The kernel learns about hardware connections through a **Device Tree overlay** — a small binary patch applied at boot that tells the kernel which drivers to load and how to wire them together. + +### Files + +| File | Purpose | +|------|---------| +| `installers/audio/cm5-i2s-audio.dts` | Human-readable source describing the hardware | +| `cm5-i2s-audio.dtbo` (compiled from above) | Binary patch applied by the bootloader — installed to `/boot/firmware/overlays/` | + +The DTS source creates a `simple-audio-card` with **two DAI (Digital Audio Interface) links** on the same I2S controller: + +- **DAI link 0** → `invensense,ics43432` codec driver → ALSA PCM **device 0** (capture) +- **DAI link 1** → `maxim,max98357a` codec driver → ALSA PCM **device 1** (playback) + +### Why not `googlevoicehat-soundcard`? + +The `voicehat-codec` driver only defines a speaker output in its DAPM (Dynamic Audio Power Management) routing. DAPM is how the kernel knows which hardware paths to power up for a given operation. Because the voicehat codec had no capture DAPM route, the RP1's I2S RX engine was never enabled — microphone recordings returned silence regardless of wiring. + +The new overlay uses dedicated drivers (`max98357a`, `ics43432`) that have correct DAPM routes for both playback and capture. + +### Installing the overlay + +`install-audio.sh` (run from the `installers/` directory on the device) compiles the DTS and installs it: + +```bash +cd ~/modular-biped/installers +bash install-audio.sh +sudo reboot +``` + +Requires `dtc` (`sudo apt install device-tree-compiler` if missing). + +### `config.txt` entry + +``` +dtoverlay=cm5-i2s-audio +hdmi_ignore_edid_audio=1 +#dtparam=audio=on ← keep disabled +``` + +--- + +## ALSA Configuration (`/etc/asound.conf`) + +After reboot the sound card appears as: + +``` +card 0: cm5i2saudio [cm5-i2s-audio] + device 0: ics43432-hifi (capture) + device 1: MAX98357A HiFi (playback) +``` + +The ALSA card number can change between boots when HDMI devices are present, so always reference the card by **name** (`cm5i2saudio`). + +`asound.conf` defines a software pipeline on top of the hardware: + +- **`pcm.speakerbonnet`** — raw hardware playback (`hw:cm5i2saudio,1`) +- **`pcm.dmixer`** — software mixer so multiple apps can play simultaneously (`type dmix`) +- **`pcm.softvol`** — software volume control on top of dmixer +- **`pcm.mic_plug`** — capture with format conversion (`type plug` over `hw:cm5i2saudio,0`), so apps that request `S16_LE` are transparently converted to the `S32_LE` the hardware requires +- **`pcm.!default`** — `type asym` routes playback and capture independently (dmix/softvol are playback-only; without `asym`, capture would be blocked) + +On Raspberry Pi OS Bookworm, PipeWire is the default audio server and overrides `pcm.!default` for most applications. The `asound.conf` chain is used by CLI tools (`aplay`, `arecord`) that access ALSA directly. + +--- + +## Testing + +**List devices:** +```bash +aplay -l # shows device 1 (MAX98357A, playback) +arecord -l # shows device 0 (ICS-43434, capture) +``` + +**Speaker test (direct hardware, both channels):** +```bash +speaker-test -D hw:cm5i2saudio,1 -c 2 -r 48000 -t sine -l 1 +``` + +**Microphone test (record 3 s, play back):** +```bash +arecord -D hw:cm5i2saudio,0 -f S32_LE -c 2 -r 48000 -d 3 test.wav +aplay -D hw:cm5i2saudio,1 test.wav +``` + +**Via default device (through PipeWire or asound.conf chain):** +```bash +arecord -D default -f S16_LE -c 2 -r 48000 -d 3 /tmp/test.wav +aplay -D default /tmp/test.wav +``` + +--- + +## Speech Recognition (`SpeechInput` module) + +The `SpeechInput` module uses the `speech_recognition` library, which requests audio at **16000 Hz**. The hardware only supports 48000 Hz, so a device that handles resampling is required. + +The module is configured to use the `pulse` device, which is PipeWire's PulseAudio compatibility interface. This is the correct choice because: + +- The direct hardware device (`hw:cm5i2saudio,0`) rejects any sample rate other than 48000 Hz — `speech_recognition`'s 16000 Hz request will fail immediately. +- The `pulse` device routes through PipeWire, which resamples transparently between whatever the application requests and what the hardware needs. + +--- + +## `aplay.service` + +This service was originally used to hold the hardware open to prevent a pop on first playback. **Do not enable it on PipeWire systems** — it holds an exclusive ALSA dmix lock and causes PipeWire to time out. `install-audio.sh` explicitly disables it. The service file is kept in the repo only as a reference for pure-ALSA setups. + +--- + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---------|-------------|-----| +| `aplay -l` shows no `cm5i2saudio` card | Overlay not applied | Check `dmesg \| grep -i "i2s\|max98\|ics43"` for probe errors; confirm `/boot/firmware/overlays/cm5-i2s-audio.dtbo` exists | +| Speaker silent, no errors | SD_MODE GPIO wrong | Verify GPIO 16 is correct for the carrier board; check `pinctrl get 16` | +| Microphone records silence | DAPM not routing capture | Confirm you are using `cm5-i2s-audio` overlay (not `googlevoicehat-soundcard`); check `sudo find /sys/kernel/debug/asoc -name "Capture"` exists | +| `arecord` error: unsupported format | Wrong format for hw device | Use `-f S32_LE` with `hw:cm5i2saudio,0`; use `-f S16_LE` with `default` (plug converts) | +| PipeWire stream timeout | `aplay.service` is running | `sudo systemctl disable --now aplay.service` | + +--- + +## Terms + +**Device Tree** — A data structure the bootloader passes to the Linux kernel describing hardware that cannot be auto-detected (I2S buses, GPIO assignments, codec chip connections, etc.). + +**DTS (Device Tree Source)** — Human-readable text source file (like `cm5-i2s-audio.dts`). + +**DTB (Device Tree Blob)** — The compiled binary form of a DTS, loaded at boot. + +**DTBO (Device Tree Blob Overlay)** — A partial DTB patched on top of the base device tree at boot. Pi overlays are `.dtbo` files in `/boot/firmware/overlays/`. `dtoverlay=name` in `config.txt` loads `name.dtbo`. + +**DAPM (Dynamic Audio Power Management)** — The ASoC (ALSA System on Chip) subsystem that manages which hardware audio paths are powered at any time. A codec driver must define DAPM widgets and routes for both playback and capture; if a capture route is missing, the I2S RX engine is never enabled even if the hardware supports it. + +**DAI (Digital Audio Interface)** — The connection between the CPU-side I2S controller and a codec chip. A sound card can have multiple DAI links (one per codec), each appearing as a separate PCM device in ALSA. + +**dmix** — ALSA plugin that software-mixes multiple playback streams so more than one app can play audio simultaneously. + +**softvol** — ALSA plugin providing software volume control for hardware that has no built-in mixer. + +**asym** — ALSA plugin that routes playback and capture to independent devices, needed when the playback chain (dmix/softvol) cannot handle capture. + +**plug** — ALSA plugin that transparently converts sample format and rate between what an application requests and what the hardware supports (e.g. S16_LE → S32_LE). diff --git a/installers/audio/aplay.service b/installers/audio/aplay.service new file mode 100644 index 00000000..c20ba24c --- /dev/null +++ b/installers/audio/aplay.service @@ -0,0 +1,9 @@ + +[Unit] +Description=Invoke aplay from /dev/zero at system start. + +[Service] +ExecStart=/usr/bin/aplay -D default -t raw -r 48000 -c 2 -f S32_LE /dev/zero + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/installers/audio/archive/i2samp.py b/installers/audio/archive/i2samp.py new file mode 100644 index 00000000..e332ca70 --- /dev/null +++ b/installers/audio/archive/i2samp.py @@ -0,0 +1,148 @@ +import os + +try: + from adafruit_shell import Shell + from clint.textui import colored +except ImportError: + raise RuntimeError("The library 'adafruit_shell' was not found. To install, try typing: sudo pip3 install adafruit-python-shell") + +shell = Shell() + +BLACKLIST = "/etc/modprobe.d/raspi-blacklist.conf" +PRODUCT_NAME = "I2S Amplifier" +OVERLAY = "googlevoicehat-soundcard" + +def driver_loaded(driver_name): + return shell.run_command(f"lsmod | grep -q '{driver_name}'", suppress_message=True) + +def main(): + reboot = False + shell.clear() + if not shell.is_raspberry_pi(): + shell.bail("Non-Raspberry Pi board detected.") + print("\nThis script will install everything needed to use\n" + f"{PRODUCT_NAME}.\n") + print(colored.red("--- Warning ---")) + print("\nAlways be careful when running scripts and commands\n" + "copied from the internet. Ensure they are from a\n" + "trusted source.\n") + if not shell.prompt("Do you wish to continue?"): + print("\nAborting...") + shell.exit() + + print("\nChecking hardware requirements...") + + # Enable I2S overlay + config = shell.get_boot_config() + if config is None: + shell.bail("No Device Tree Detected, not supported") + + print(f"\nAdding Device Tree Entry to {config}") + + if shell.pattern_search(config, f"^dtoverlay={OVERLAY}$"): + print("dtoverlay already active") + else: + shell.write_text_file(config, f"dtoverlay={OVERLAY}") + reboot = True + + if os.path.exists(BLACKLIST): + print("\nCommenting out Blacklist entry in", BLACKLIST) + shell.pattern_replace(BLACKLIST, "^blacklist[[:space:]]*snd_soc_max98357a.*", "#blacklist snd_soc_max98357a") + shell.pattern_replace(BLACKLIST, "^blacklist[[:space:]]*snd_soc_max98357a_i2c.*", "#blacklist snd_soc_max98357a_i2c") + shell.pattern_replace(BLACKLIST, "^blacklist[[:space:]]*snd_soc_max98357a.*", "#blacklist snd_soc_max98357a") + + print("Configuring sound output") + if os.path.exists("/etc/asound.conf"): + if os.path.exists("/etc/asound.conf.old"): + shell.remove("/etc/asound.conf.old") + shell.move("/etc/asound.conf", "/etc/asound.conf.old") + shell.write_text_file("~/asound.conf", +""" +# Use the card name (sndrpigooglevoi) instead of a hardcoded card number. +# The ALSA card number for the I2S device can change between boots when HDMI +# devices are present, so referencing by name ensures the correct device is +# always targeted. + +pcm.speakerbonnet { + type hw + card sndrpigooglevoi +} + +pcm.dmixer { + type dmix + ipc_key 1024 + ipc_perm 0666 + slave { + pcm "speakerbonnet" + period_time 0 + period_size 1024 + buffer_size 8192 + rate 44100 + channels 2 + } +} + +ctl.dmixer { + type hw + card sndrpigooglevoi +} + +pcm.softvol { + type softvol + slave.pcm "dmixer" + control.name "PCM" + control.card sndrpigooglevoi +} + +ctl.softvol { + type hw + card sndrpigooglevoi +} + +# Separate playback and capture defaults using asym so that: +# - aplay / speaker output uses the softvol/dmix chain +# - arecord / microphone input uses the I2S capture device directly +pcm.!default { + type asym + playback.pcm "softvol" + capture.pcm "hw:sndrpigooglevoi" +} +""") + shell.move("~/asound.conf", "/etc/asound.conf") + + + print("Installing aplay systemd unit") + shell.write_text_file("/etc/systemd/system/aplay.service", """ +[Unit] +Description=Invoke aplay from /dev/zero at system start. + +[Service] +ExecStart=/usr/bin/aplay -D default -t raw -r 44100 -c 2 -f S16_LE /dev/zero + +[Install] +WantedBy=multi-user.target""", append=False) + + shell.run_command("sudo systemctl daemon-reload") + shell.run_command("sudo systemctl disable aplay") + print("\nYou can optionally activate '/dev/zero' playback in\n" + "the background at boot. This will remove all\n" + "popping/clicking but does use some processor time.\n\n") + if shell.prompt("Activate '/dev/zero' playback in background? [RECOMMENDED]\n", default="y"): + shell.run_command("sudo systemctl enable aplay") + reboot = True + + if driver_loaded("max98357a"): + print(f"\nWe can now test your {PRODUCT_NAME}") + shell.warn("Set your speakers at a low volume if possible!") + if shell.prompt("Do you wish to test your system now?"): + print("Testing...") + shell.run_command("speaker-test -l5 -c2 -t wav") + print("\n" + colored.green("All done!")) + print(f"\nEnjoy your new {PRODUCT_NAME}!") + if reboot: + shell.prompt_reboot() + +# Main function +if __name__ == "__main__": + shell.require_root() + main() diff --git a/installers/audio/archive/i2smic.py b/installers/audio/archive/i2smic.py new file mode 100644 index 00000000..bed3c188 --- /dev/null +++ b/installers/audio/archive/i2smic.py @@ -0,0 +1,78 @@ +import os + +try: + from adafruit_shell import Shell +except ImportError: + raise RuntimeError("The library 'adafruit_shell' was not found. To install, try typing: sudo pip3 install adafruit-python-shell") + +shell = Shell() + +def main(): + shell.clear() + boot_config = shell.get_boot_config() + print("""This script downloads and installs +I2S microphone support. +""") + if not shell.is_raspberry_pi(): + shell.bail("Non-Raspberry Pi board detected.") + pi_model = shell.get_board_model() + print("{} detected.\n".format(pi_model)) + if pi_model in ("RASPBERRY_PI_ZERO", "RASPBERRY_PI_ZERO_W", "RASPBERRY_PI_B_REV2"): + pimodel_select = 0 + elif pi_model in ("RASPBERRY_PI_2B", "RASPBERRY_PI_3B", "RASPBERRY_PI_CM3_PLUS", "RASPBERRY_PI_3B_PLUS", "RASPBERRY_PI_3A_PLUS", "RASPBERRY_PI_ZERO_2_W"): + pimodel_select = 1 + elif pi_model in ("RASPBERRY_PI_4B", "RASPBERRY_PI_CM4", "RASPBERRY_PI_400", + "RASPBERRY_PI_5B", "RASPBERRY_PI_CM5", "RASPBERRY_PI_5"): + pimodel_select = 2 + else: + shell.bail("Unsupported Pi board detected.") + + auto_load = ( + not shell.argument_exists('noautoload') and + shell.prompt("Auto load module at boot?", force_arg="autoload")) + + print(""" +Installing...""") + + # Get needed packages + shell.run_command("apt-get -y install git raspberrypi-kernel-headers") + + # Clone the repo + shell.run_command("git clone https://github.com/adafruit/Raspberry-Pi-Installer-Scripts.git") + + # Build and install the module + shell.chdir("Raspberry-Pi-Installer-Scripts/i2s_mic_module") + shell.run_command("make clean") + shell.run_command("make") + shell.run_command("make install") + + # Setup auto load at boot if selected + if auto_load: + shell.write_text_file( + "/etc/modules-load.d/snd-i2smic-rpi.conf", + "snd-i2smic-rpi" + ) + shell.write_text_file( + "/etc/modprobe.d/snd-i2smic-rpi.conf", + "options snd-i2smic-rpi rpi_platform_generation={}".format(pimodel_select) + ) + + # Enable I2S overlay + shell.run_command(f"sed -i -e 's/#dtparam=i2s/dtparam=i2s/g' {boot_config}") + + # Done + print("""DONE. + +Settings take effect on next boot. +""") + if not shell.argument_exists('noreboot'): + shell.prompt_reboot(force_arg="reboot") + +# Main function +if __name__ == "__main__": + shell.require_root() + # Probably not necessary for now because the Pi 5 doesn't work anyway + if shell.is_raspberry_pi_os() and shell.is_kernel_userspace_mismatched() and shell.is_pi5_or_newer(): + shell.bail("Unable to proceed on Pi 5 or newer boards with a with a 32-bit OS. Please reinstall with a 64-bit OS.") + shell.check_kernel_userspace_mismatch() + main() diff --git a/installers/install-amp.sh b/installers/audio/archive/install-amp.sh similarity index 100% rename from installers/install-amp.sh rename to installers/audio/archive/install-amp.sh diff --git a/installers/install-mics.sh b/installers/audio/archive/install-mics.sh similarity index 100% rename from installers/install-mics.sh rename to installers/audio/archive/install-mics.sh diff --git a/installers/install-speech-recognition.sh b/installers/audio/archive/install-speech-recognition.sh similarity index 100% rename from installers/install-speech-recognition.sh rename to installers/audio/archive/install-speech-recognition.sh diff --git a/installers/install-spi-tft.sh b/installers/audio/archive/install-spi-tft.sh similarity index 100% rename from installers/install-spi-tft.sh rename to installers/audio/archive/install-spi-tft.sh diff --git a/installers/audio/asound.conf b/installers/audio/asound.conf new file mode 100644 index 00000000..8420ff6f --- /dev/null +++ b/installers/audio/asound.conf @@ -0,0 +1,64 @@ + + +# Card name: cm5i2saudio (ALSA normalises "cm5-i2s-audio" → "cm5i2saudio") +# The custom cm5-i2s-audio overlay creates a simple-audio-card with two DAI links: +# device 1: MAX98357A speaker amplifier (playback only) +# device 0: ICS-43434 stereo mic pair (capture only) +# +# With PipeWire running (default on Bookworm) pcm.!default is overridden by +# PipeWire's ALSA plugin. The definitions below are used by CLI tools +# (aplay, arecord) that access ALSA directly. + +pcm.speakerbonnet { + type hw + card cm5i2saudio + device 1 +} + +pcm.dmixer { + type dmix + ipc_key 1024 + ipc_perm 0666 + slave { + pcm "speakerbonnet" + period_time 0 + period_size 1024 + buffer_size 8192 + rate 48000 + channels 2 + format S32_LE + } +} + +ctl.dmixer { + type hw + card cm5i2saudio +} + +pcm.softvol { + type softvol + slave.pcm "dmixer" + control.name "PCM" + control.card cm5i2saudio +} + +ctl.softvol { + type hw + card cm5i2saudio +} + +# Separate playback and capture defaults using asym so that: +# - aplay / speaker output uses the softvol/dmix chain +# - arecord / microphone input uses the ICS-43434 capture device (hw:X,0) +# plug wrapper allows ALSA to convert format/rate for apps that don't natively +# support S32_LE / 48000 Hz (the only format the DW I2S hardware accepts). +pcm.mic_plug { + type plug + slave.pcm "hw:cm5i2saudio,0" +} + +pcm.!default { + type asym + playback.pcm "softvol" + capture.pcm "mic_plug" +} diff --git a/installers/audio/cm5-i2s-audio.dtbo b/installers/audio/cm5-i2s-audio.dtbo new file mode 100644 index 00000000..5ba8a3f6 Binary files /dev/null and b/installers/audio/cm5-i2s-audio.dtbo differ diff --git a/installers/audio/cm5-i2s-audio.dts b/installers/audio/cm5-i2s-audio.dts new file mode 100644 index 00000000..93602359 --- /dev/null +++ b/installers/audio/cm5-i2s-audio.dts @@ -0,0 +1,94 @@ +// Custom I2S audio overlay for CM5 carrier board +// - MAX98357A speaker amplifier (I2S TX, SD_MODE on GPIO16) +// - 2x ICS-43434 stereo microphones (I2S RX, L/R SELECT pins) +// +// Creates a single simple-audio-card with two DAI links: +// PCM device 0 (hw:cm5-i2s-audio,0): playback via MAX98357A +// PCM device 1 (hw:cm5-i2s-audio,1): capture via ICS-43434 stereo pair +// +// Replaces dtoverlay=googlevoicehat-soundcard in /boot/firmware/config.txt + +/dts-v1/; +/plugin/; + +/ { + compatible = "brcm,bcm2712"; + + // Enable the RP1 I2S controller as clock producer (master) + fragment@0 { + target = <&i2s_clk_producer>; + __overlay__ { + status = "okay"; + }; + }; + + // MAX98357A speaker amplifier node (SD_MODE on GPIO16) + fragment@1 { + target-path = "/"; + __overlay__ { + max98357a_dac: max98357a { + compatible = "maxim,max98357a"; + #sound-dai-cells = <0>; + sdmode-gpios = <&gpio 16 0>; + status = "okay"; + }; + }; + }; + + // ICS-43434 stereo microphone pair (treated as ICS-43432 compatible) + // L/R SELECT: one mic at GND (left), one at 3.3V (right), both on GPIO20 + fragment@2 { + target-path = "/"; + __overlay__ { + ics43434_mics: ics43434-stereo { + compatible = "invensense,ics43432"; + #sound-dai-cells = <0>; + status = "okay"; + }; + }; + }; + + // Sound card: two DAI links share the same I2S controller + // dai-link@0: I2S TX -> MAX98357A (playback, PCM device 0) + // dai-link@1: ICS-43434 -> I2S RX (capture, PCM device 1) + fragment@3 { + target = <&sound>; + __overlay__ { + compatible = "simple-audio-card"; + simple-audio-card,name = "cm5-i2s-audio"; + status = "okay"; + + spk_link: simple-audio-card,dai-link@0 { + format = "i2s"; + bitclock-master = <&spk_cpu>; + frame-master = <&spk_cpu>; + + spk_cpu: cpu { + sound-dai = <&i2s_clk_producer>; + dai-tdm-slot-num = <2>; + dai-tdm-slot-width = <32>; + }; + + spk_codec: codec { + sound-dai = <&max98357a_dac>; + }; + }; + + mic_link: simple-audio-card,dai-link@1 { + format = "i2s"; + bitclock-master = <&mic_cpu>; + frame-master = <&mic_cpu>; + + mic_cpu: cpu { + sound-dai = <&i2s_clk_producer>; + dai-tdm-slot-num = <2>; + dai-tdm-slot-width = <32>; + }; + + mic_codec: codec { + sound-dai = <&ics43434_mics>; + }; + }; + }; + }; +}; diff --git a/installers/audio/config.txt b/installers/audio/config.txt new file mode 100755 index 00000000..96e7c43f --- /dev/null +++ b/installers/audio/config.txt @@ -0,0 +1,59 @@ +# For more options and information see +# http://rptl.io/configtxt +# Some settings may impact device functionality. See link above for details + +# Uncomment some or all of these to enable the optional hardware interfaces +dtparam=i2c_arm=on +dtparam=i2s=on +dtparam=spi=on + +# Enable audio (loads snd_bcm2835) +#dtparam=audio=on + +# Additional overlays and parameters are documented +# /boot/firmware/overlays/README + +# Automatically load overlays for detected cameras +camera_auto_detect=1 + +# Automatically load overlays for detected DSI displays +display_auto_detect=1 + +# Automatically load initramfs files, if found +auto_initramfs=1 + +# Enable DRM VC4 V3D driver +#dtoverlay=vc4-kms-v3d +max_framebuffers=2 + +# Don't have the firmware create an initial video= setting in cmdline.txt. +# Use the kernel's default instead. +disable_fw_kms_setup=1 + +# Run in 64-bit mode +arm_64bit=1 + +# Disable compensation for displays with overscan +disable_overscan=1 + +# Run as fast as firmware / board allows +arm_boost=1 + +[cm4] +# Enable host mode on the 2711 built-in XHCI USB controller. +# This line should be removed if the legacy DWC2 controller is required +# (e.g. for USB device mode) or if USB support is not required. +otg_mode=1 + +[cm5] +dtoverlay=dwc2,dr_mode=host +dtoverlay=imx500-pi5,cam0 + +[all] +dtparam=uart0=on +#dtoverlay=i2s-mmap +#dtoverlay=googlevoicehat-soundcard +#dtoverlay=max98357a +dtoverlay=cm5-i2s-audio +#dtoverlay=dual_i2s +hdmi_ignore_edid_audio=1 diff --git a/installers/audio/raspi-blacklist.conf b/installers/audio/raspi-blacklist.conf new file mode 100644 index 00000000..e69de29b diff --git a/installers/install-audio.sh b/installers/install-audio.sh new file mode 100755 index 00000000..7ec8931f --- /dev/null +++ b/installers/install-audio.sh @@ -0,0 +1,35 @@ +mkdir audio/backups/backup-$(date +%Y-%m-%d-%H-%M-%S) +cd audio/backups/backup-$(date +%Y-%m-%d-%H-%M-%S) +cp /boot/firmware/config.txt config.txt +cp /etc/modprobe.d/raspi-blacklist.conf raspi-blacklist.conf +cp /etc/asound.conf asound.conf +cp /etc/systemd/system/aplay.service aplay.service + +echo "Backed up current audio configuration files to $(pwd)." +echo "Installing audio config." +cd ../../.. + +# Compile and install the custom cm5-i2s-audio device tree overlay. +# This overlay creates a simple-audio-card with proper DAPM routes for both +# the MAX98357A speaker (device 1) and ICS-43434 stereo mics (device 0). +echo "Compiling cm5-i2s-audio device tree overlay..." +if ! command -v dtc &> /dev/null; then + echo "dtc not found — installing device-tree-compiler..." + sudo apt-get install -y device-tree-compiler +fi +dtc -@ -I dts -O dtb -o /tmp/cm5-i2s-audio.dtbo audio/cm5-i2s-audio.dts +sudo cp /tmp/cm5-i2s-audio.dtbo /boot/firmware/overlays/cm5-i2s-audio.dtbo +echo "cm5-i2s-audio.dtbo installed to /boot/firmware/overlays/" + +sudo cp audio/config.txt /boot/firmware/config.txt +sudo cp audio/raspi-blacklist.conf /etc/modprobe.d/raspi-blacklist.conf +sudo cp audio/asound.conf /etc/asound.conf +sudo cp audio/aplay.service /etc/systemd/system/aplay.service + +# Disable aplay.service: it holds the hardware open via ALSA dmix, which +# blocks PipeWire (the default audio server on Bookworm) from accessing the +# device. PipeWire handles device lifecycle and mixing natively. +sudo systemctl disable --now aplay.service 2>/dev/null || true + +echo "Audio configuration files copied. Please reboot the system for changes to take effect." + diff --git a/src/modules/audio/speechinput/config.yml b/src/modules/audio/speechinput/config.yml index b5900864..b1ad46bd 100644 --- a/src/modules/audio/speechinput/config.yml +++ b/src/modules/audio/speechinput/config.yml @@ -1,7 +1,4 @@ speechinput: - config: - device_name: pulse - start_on_boot: false dependencies: unix: - flac @@ -10,6 +7,5 @@ speechinput: python: - SpeechRecognition additional: - - installers/install-mics.sh - - installers/install-speech-recognition.sh + - installers/install-audio.sh class: SpeechInput diff --git a/src/modules/audio/speechinput/speechinput.py b/src/modules/audio/speechinput/speechinput.py index d72c496b..e197f6cb 100644 --- a/src/modules/audio/speechinput/speechinput.py +++ b/src/modules/audio/speechinput/speechinput.py @@ -1,3 +1,5 @@ +import os +import subprocess import speech_recognition as sr from time import sleep from threading import Thread @@ -11,7 +13,7 @@ def __init__(self, **kwargs): self.recognizer = sr.Recognizer() self.recognizer.pause_threshold = 2 - self.device_name = kwargs.get('device_name', 'lp') + self.device_name = kwargs.get('device_name', 'pulse') self.device = self.get_device_index(self.device_name) self.sample_rate = kwargs.get('sample_rate', 16000) @@ -19,6 +21,11 @@ def __init__(self, **kwargs): self.listening = False self.start_on_boot = kwargs.get('start_on_boot', False) + self.wake_word = kwargs.get('wake_word', None) + + self.capture_detections = kwargs.get('capture_detections', False) + self.repeat_captures = kwargs.get('repeat_captures', False) + print('SpeechInput initialized with device ' + str(self.device) + ' and sample rate ' + str(self.sample_rate) + '. Start on boot: ' + str(self.start_on_boot)) def setup_messaging(self): self.subscribe('speech:listen', self.start) @@ -34,6 +41,7 @@ def __del__(self): def start(self): self.listening = True + self.log('Starting speech detection') Thread(target=self.detect, args=()).start() return self @@ -43,6 +51,8 @@ def get_device_index(self, device_name): if name == device_name: print('Mapping mic to index ' + str(index)) return index + # throw exception if device name is not found + raise Exception('Device name ' + device_name + ' not found. Available devices: ' + str(sr.Microphone.list_microphone_names())) def detect(self): """ @@ -59,11 +69,27 @@ def detect(self): audio = self.recognizer.listen(source, timeout=10, phrase_time_limit=15) val = self.recognizer.recognize_google(audio) - #save audio with filename as val substituting any non alphanumeric characters with underscores - filename = str(val).replace(' ', '_').replace('[^a-zA-Z0-9]', '_') - with open("speech_" + filename + ".wav", "wb") as f: - f.write(audio.get_wav_data()) - + # if wake word is set check each word in list against wake word and ignore if not detected + if self.wake_word: + found = False + for word in self.wake_word: + if word.lower() in val.lower(): + found = True + break + if not found: + self.log('Wake word "' + str(self.wake_word) + '" not detected in "' + val + '". Ignoring.') + continue + + if self.capture_detections: + #save audio with filename as val substituting any non alphanumeric characters with underscores + filename = str(val).replace(' ', '_').replace('[^a-zA-Z0-9]', '_') + with open("speech_" + filename + ".wav", "wb") as f: + f.write(audio.get_wav_data()) + if self.repeat_captures: + # play back file via speakers + self.log('Playing back audio') + subprocess.call(['aplay', '-D', 'plughw:cm5i2saudio,1', "speech_" + filename + ".wav"]) + self.log('I heard: ' + str(val)) self.publish('speech', text=val.lower()) self.publish('tts', msg='I heard ' + val.lower()) diff --git a/src/modules/audio/speechinput/tests/test_speechinput.py b/src/modules/audio/speechinput/tests/test_speechinput.py index 5e6f418c..2ded7999 100644 --- a/src/modules/audio/speechinput/tests/test_speechinput.py +++ b/src/modules/audio/speechinput/tests/test_speechinput.py @@ -7,6 +7,7 @@ mock_microphone = MagicMock() mock_speech_recognition.Recognizer.return_value = mock_recognizer mock_speech_recognition.Microphone.return_value = mock_microphone +mock_speech_recognition.Microphone.list_microphone_names.return_value = ['test', 'pulse'] sys.modules['speech_recognition'] = mock_speech_recognition from modules.audio.speechinput.speechinput import SpeechInput diff --git a/src/modules/audio/ttsmodule/config.yml b/src/modules/audio/ttsmodule/config.yml index 36be8926..02a8c6b3 100644 --- a/src/modules/audio/ttsmodule/config.yml +++ b/src/modules/audio/ttsmodule/config.yml @@ -11,5 +11,5 @@ ttsmodule: - ffmpeg - libespeak1 additional: - - installers/install-amp.sh + - installers/install-audio.sh class: TTSModule diff --git a/src/modules/chatgpt/config.yml b/src/modules/chatgpt/config.yml index 756ef472..d056108b 100644 --- a/src/modules/chatgpt/config.yml +++ b/src/modules/chatgpt/config.yml @@ -1,10 +1,7 @@ chatgpt: config: model: gpt-4o-mini - persona: 'You are a helpful assistant robot. You respond with short phrases where - possible. Alternatively, you can respond with the following commands instead - of text if you feel they are appropriate: - animate/head_nod - animate/head_shake - - gpio/laser/on - gpio/laser/off' + persona: 'You are a helpful assistant robot. You respond with short phrases where possible. dependencies: python: - openai