Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 23 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@ GROQ_API_KEY=""
# Get yours at: https://picovoice.ai (free tier available)
PORCUPINE_ACCESS_KEY=""

# The bundled Jasper Porcupine model is Linux-only. Wake-word detection
# defaults to enabled on Linux and disabled elsewhere. Set to false to use
# the overlay text input without initializing microphone or wake-word services.
# WAKE_WORD_ENABLED=false

# ========================
# API Keys (Optional)
# ========================
Expand All @@ -22,6 +27,24 @@ GOOGLE_API_KEY=""
# Get yours at: https://tavily.com
TAVILY_API_KEY=""

# ========================
# LLM Configuration
# ========================

# qwen/qwen3.8-27b supports Groq chat, image input, and local function calls.
LLM_PROVIDER="groq"
LLM_MODEL="qwen/qwen3.8-27b"
LLM_TEMPERATURE="0.8"

# Groq speech-to-text configuration. whisper-large-v3-turbo is the current
# low-latency successor to the retired distil-whisper-large-v3-en.
TRANSCRIPTION_MODEL="whisper-large-v3-turbo"
TRANSCRIPTION_LANGUAGE="en"
TRANSCRIPTION_TIMEOUT="30"

# Optional persistent LangGraph conversation identifier.
JASPER_THREAD_ID="jasper"

# ========================
# Logging
# ========================
Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ SPEAKER_SIMILARITY_THRESHOLD = 0.6 # Voice verification threshold
### LLM Configuration
```bash
# In your .env file (or defaults in config.py)
LLM_MODEL="meta-llama/llama-4-scout-17b-16e-instruct"
LLM_MODEL="qwen/qwen3.8-27b"
LLM_PROVIDER="groq"
LLM_TEMPERATURE=0.8
```
Expand Down
80 changes: 72 additions & 8 deletions main.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,9 @@
import threading

from src.audio.audio_processor import AudioProcessor
from src.audio.listener import Listener
from src.audio.manual_recorder import ManualRecorder, MicrophoneError
from src.audio.ttsplayer import TTSPlayer
from src.config import config
from src.core.assistant import call_agent
from src.core.tools import register_stop_assistant
from src.ui.overlay import app, overlay
Expand All @@ -30,6 +31,7 @@ class DesktopAssistant:
def __init__(self):
"""Initialize the assistant with all required components."""
self.lock = threading.Lock()
self._shutting_down = False
self._setup_components()
self._setup_ui()

Expand All @@ -41,11 +43,22 @@ def _setup_components(self):

# Audio processing
self.audio_processor = AudioProcessor()
self.listener = Listener(tts_player=self.speech, overlay=overlay)
self.manual_recorder = ManualRecorder()
self.listener = None
if config.WAKE_WORD_ENABLED:
# Import lazily: the listener's optional voice-verification stack
# should not be required for text-only operation on macOS.
from src.audio.listener import Listener

self.listener = Listener(tts_player=self.speech, overlay=overlay)
else:
logger.info("Wake-word detection is disabled; use the overlay text input to submit requests.")

def _setup_ui(self):
"""Setup user interface."""
overlay.on_new_message = self.process_query
overlay.on_recording_toggled = self.handle_manual_recording
overlay.exit_callback = self.shutdown
overlay.start()

def process_audio(self, audio):
Expand All @@ -54,8 +67,45 @@ def process_audio(self, audio):
logger.info("Processing audio...")
overlay.put_message("status", "Analyzing voice...", "gold")
transcription = self.audio_processor.process_audio(audio)
if not transcription:
detail = self.audio_processor.last_error or "no speech could be transcribed"
logger.error("Speech transcription did not return text: %s", detail)
overlay.put_message("response", f"Voice input failed: {detail}")
overlay.put_message("status", "Voice transcription failed", "red")
return
self.process_query(transcription)

def handle_manual_recording(self, should_record: bool):
"""Start or stop explicit microphone capture from the overlay control."""
if should_record:
try:
self.manual_recorder.start()
except MicrophoneError as exc:
self._report_microphone_error(exc)
return
overlay.put_message("recording", True)
overlay.put_message("status", "Recording… click Stop voice when finished", "skyblue")
return

try:
audio = self.manual_recorder.stop()
except MicrophoneError as exc:
self._report_microphone_error(exc)
return

overlay.put_message("recording", False)
overlay.put_message("status", "Transcribing voice…", "gold")
self.process_audio(audio)

def _report_microphone_error(self, error: Exception):
message = f"Microphone unavailable: {error}"
if config.PLATFORM == "Darwin":
message += " Allow this terminal or IDE in System Settings > Privacy & Security > Microphone."
logger.error(message)
overlay.put_message("recording", False)
overlay.put_message("response", message)
overlay.put_message("status", "Microphone error", "red")

def process_query(self, query: str):
"""Process the text query."""
overlay.put_message("query", query)
Expand All @@ -69,24 +119,36 @@ def process_query(self, query: str):
self.speech.speak(response)
except Exception as e:
logger.error(f"Error processing query: {e}")
overlay.put_message("response", f"Error: {e}")
overlay.put_message("status", "Error occurred", "red")
self.speech.speak("Sorry, an error occurred while processing your request.")

def start(self):
"""Start the assistant."""
logger.info("🔊 Listening in the background... Say something!")
overlay.put_message("status", "Active", "green")
executor.submit(self.listener.listen, self.process_audio)
if self.listener:
logger.info("🔊 Listening in the background... Say something!")
overlay.put_message("status", "Active", "green")
executor.submit(self.listener.listen, self.process_audio)
else:
logger.info("Wake word is disabled. Use text input or Start voice in the overlay.")
overlay.put_message("status", "Wake word disabled — use text or voice", "gold")

def shutdown(self):
"""Clean shutdown of the assistant."""
if self._shutting_down:
return
self._shutting_down = True
self.speech.speak("Shutting down!")
overlay.put_message("status", "Shutting down...", "red")
logger.info("Shutting down...")

self.listener.stop_listening()
if self.listener:
self.listener.stop_listening()
self.manual_recorder.cancel()
self.speech.shutdown()
executor.shutdown(wait=False, cancel_futures=True)
# Avoid recursing when this shutdown was initiated by the window close.
overlay.exit_callback = None
overlay.close()


Expand All @@ -95,8 +157,10 @@ def main():
assistant = DesktopAssistant()
register_stop_assistant(assistant.shutdown)
assistant.start()
app.exec()
exit_code = app.exec()
logger.info("Qt event loop exited with code %s", exit_code)
return exit_code


if __name__ == "__main__":
main()
sys.exit(main())
11 changes: 9 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
[project]
name = "desktopai"
version = "0.1.0"
description = "An intelligent, voice-activated AI assistant for Linux desktop environments"
description = "An intelligent AI desktop assistant with Linux wake-word support"
readme = "README.md"
license = { text = "MIT" }
requires-python = ">=3.12"
Expand All @@ -11,6 +11,7 @@ classifiers = [
"Development Status :: 3 - Alpha",
"Intended Audience :: End Users/Desktop",
"License :: OSI Approved :: MIT License",
"Operating System :: MacOS",
"Operating System :: POSIX :: Linux",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.12",
Expand All @@ -20,9 +21,13 @@ classifiers = [
]
dependencies = [
"adbutils>=2.9.3",
# Python 3.13 removed audioop; resemblyzer's audio stack still imports it.
"audioop-lts>=0.2.1; python_version >= '3.13'",
"anthropic>=0.51.0",
"colorama>=0.4.6",
"edge-tts>=7.0.2",
# 7.3.0 updates the Edge Read Aloud handshake required after Microsoft's
# legacy endpoint began returning WebSocket 403 responses.
"edge-tts>=7.3.0",
"groq>=0.24.0",
"langchain>=0.3.25",
"langchain-anthropic>=0.3.13",
Expand All @@ -42,6 +47,8 @@ dependencies = [
"resemblyzer>=0.1.4",
"sounddevice>=0.5.2",
"soundfile>=0.13.1",
# webrtcvad imports pkg_resources during installation; retain its provider.
"setuptools<81",
"torch>=2.0.0",
]

Expand Down
9 changes: 8 additions & 1 deletion run.sh
Original file line number Diff line number Diff line change
@@ -1,2 +1,9 @@
#!/usr/bin/env bash
QT_QPA_PLATFORM=xcb uv run python main.py
set -euo pipefail

# xcb is a Linux Qt backend. macOS uses Qt's native Cocoa backend by default.
if [[ "$(uname -s)" == "Linux" ]]; then
export QT_QPA_PLATFORM="${QT_QPA_PLATFORM:-xcb}"
fi

exec uv run python main.py
43 changes: 35 additions & 8 deletions src/audio/audio_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,30 +17,57 @@ class AudioProcessor:
"""Handles audio transcription and processing."""

def __init__(self):
self.groq_client = Groq(api_key=config.GROQ_API_KEY)
self.groq_client = Groq(api_key=config.GROQ_API_KEY, timeout=config.TRANSCRIPTION_TIMEOUT)
self.last_error: str | None = None

def process_audio(self, audio_data):
"""Process audio data and return transcription."""
logger.info("Processing audio...")
self.last_error = None
temp_file = config.TEMP_AUDIO_FILE

try:
# Convert audio bytes to numpy array and save as WAV
# Both sounddevice's manual recorder and the Linux listener produce
# signed 16-bit mono samples. Persist an explicit PCM WAV accepted
# by Groq's transcription endpoint.
audio_array = np.frombuffer(audio_data, dtype=np.int16)
sf.write(temp_file, audio_array, config.SAMPLE_RATE)
if not len(audio_array):
raise ValueError("The recording is empty.")
duration = len(audio_array) / config.SAMPLE_RATE
rms = float(np.sqrt(np.mean(np.square(audio_array.astype(np.float32) / np.iinfo(np.int16).max))))
logger.info(
"Preparing transcription: %.2fs, %d samples, %d bytes, PCM_16 mono/%dHz, rms=%.5f",
duration,
len(audio_array),
audio_array.nbytes,
config.SAMPLE_RATE,
rms,
)
if rms < config.SILENCE_RMS_THRESHOLD:
raise ValueError("The recording is silent or below the configured microphone threshold.")
sf.write(temp_file, audio_array, config.SAMPLE_RATE, subtype="PCM_16")

with open(temp_file, "rb") as f:
payload = f.read()
logger.info(
"Sending %d-byte WAV to Groq transcription model %s.", len(payload), config.TRANSCRIPTION_MODEL
)
transcription = self.groq_client.audio.transcriptions.create(
file=("temp.wav", f.read()),
file=("jasper-recording.wav", payload),
model=config.TRANSCRIPTION_MODEL,
response_format="text",
prompt=config.TRANSCRIPTION_PROMPT,
language=config.TRANSCRIPTION_LANGUAGE,
)
logger.info(f"Transcription: {transcription}")
return transcription
text = getattr(transcription, "text", transcription)
text = str(text).strip()
if not text:
raise ValueError("Groq returned an empty transcription.")
logger.info("Groq transcription completed (%d characters).", len(text))
return text

except Exception as e:
logger.error(f"Error processing audio: {e}")
self.last_error = str(e)
logger.error("Transcription failed: %s", self.last_error)
return None
finally:
if os.path.exists(temp_file):
Expand Down
45 changes: 28 additions & 17 deletions src/audio/listener.py
Original file line number Diff line number Diff line change
@@ -1,32 +1,34 @@
import collections
import os
import struct
import sys
import threading
import time

import numpy as np
import pvporcupine
import sounddevice as sd
import soundfile as sf
import torch
from resemblyzer import VoiceEncoder, preprocess_wav

torch.set_num_threads(1)

from src.config import config # noqa: E402
from src.utils.logger import get_logger # noqa: E402
from src.utils.thread_executor import executor # noqa: E402

model, utils = torch.hub.load(repo_or_dir="snakers4/silero-vad", model="silero_vad")

get_speech_timestamps, save_audio, read_audio, VADIterator, collect_chunks = utils
from src.config import config
from src.utils.logger import get_logger
from src.utils.thread_executor import executor

logger = get_logger()


class Listener:
def __init__(self, tts_player=None, overlay=None):
if not config.WAKE_WORD_ENABLED:
raise RuntimeError(
"Wake-word detection is disabled. Use the overlay text input, or set "
"WAKE_WORD_ENABLED=true only with a Porcupine model for this platform."
)
if config.PLATFORM != "Linux":
raise RuntimeError(
f"No bundled Porcupine wake-word model supports {config.PLATFORM}. "
"Set WAKE_WORD_ENABLED=false and use the overlay text input."
)

# Audio config
self.SAMPLE_RATE = config.SAMPLE_RATE
self.CHANNELS = config.CHANNELS
Expand Down Expand Up @@ -64,6 +66,13 @@ def __init__(self, tts_player=None, overlay=None):

self.confidence_threshold = config.CONFIDENCE_THRESHOLD

# These imports can pull in Python-version-specific audio dependencies
# and the VAD model. They are only needed when wake-word mode is active.
import torch
from resemblyzer import VoiceEncoder, preprocess_wav

torch.set_num_threads(1)
self.vad_model, _ = torch.hub.load(repo_or_dir="snakers4/silero-vad", model="silero_vad")
self.voice_encoder = VoiceEncoder()
self.OWNER_VOICE_FILE = config.OWNER_VOICE_FILE
self.owner_embeddings = None
Expand All @@ -72,11 +81,10 @@ def __init__(self, tts_player=None, overlay=None):
self.owner_embeddings = self.voice_encoder.embed_utterance(preprocess_wav(self.OWNER_VOICE_FILE))
logger.info("✅ Owner voice file found. Loading embeddings...")
else:
logger.error(
"Owner voice file not found. Please record your voice and save it in data/owner.wav\n"
"Run `python setup/record_owner_voice.py` to record your voice."
raise FileNotFoundError(
"Owner voice file not found. Run `python setup/record_owner_voice.py` "
"to save it as data/owner.wav before enabling wake-word detection."
)
sys.exit(1)

def int2float(self, sound):
abs_max = np.abs(sound).max()
Expand Down Expand Up @@ -105,7 +113,10 @@ def record_audio(self, frames=None):
audio_float32 = self.int2float(audio_chunk)
frames.append(audio_chunk)

confidence = model(torch.from_numpy(audio_float32), self.SAMPLE_RATE).item()
# Silero VAD accepts the NumPy buffer directly through Torch.
import torch

confidence = self.vad_model(torch.from_numpy(audio_float32), self.SAMPLE_RATE).item()

if confidence > self.confidence_threshold:
silence_frames = 0
Expand Down
Loading
Loading