diff --git a/.github/workflows/build_server.yml b/.github/workflows/build_server.yml index 21abdefc4..f408814b9 100644 --- a/.github/workflows/build_server.yml +++ b/.github/workflows/build_server.yml @@ -53,12 +53,11 @@ jobs: sudo apt update sudo apt install -y portaudio19-dev libhamlib-dev libhamlib-utils build-essential cmake patchelf - - name: Install MacOS pyAudio + - name: Install MacOS dependencies if: ${{startsWith(matrix.os, 'macos')}} run: | brew install portaudio python -m pip install --upgrade pip - pip3 install pyaudio - name: Install Python dependencies run: | diff --git a/.github/workflows/pip_package.yml b/.github/workflows/pip_package.yml index 3a1e64ad1..9947fb115 100644 --- a/.github/workflows/pip_package.yml +++ b/.github/workflows/pip_package.yml @@ -1,5 +1,8 @@ name: Deploy Python Package -on: [push] +on: + push: + tags: + - "v*" jobs: deploy: @@ -17,16 +20,64 @@ jobs: with: node-version: 24 - - name: Install Linux dependencies - run: | - sudo apt update - sudo apt install -y portaudio19-dev libhamlib-dev libhamlib-utils build-essential cmake patchelf - - name: Install Python dependencies run: | python -m pip install --upgrade pip pip install .[build] + - name: Set package version from tag + env: + RELEASE_TAG: ${{ github.ref_name }} + run: | + python3 - <<'EOF' + import os + import re + import sys + + from packaging.version import InvalidVersion, Version + + tag = os.environ["RELEASE_TAG"] + version = tag.removeprefix("v") + + if not re.fullmatch(r"\d+\.\d+\.\d+(-[0-9A-Za-z.]+)?", version): + print( + f"::error::Tag '{tag}' does not look like a release version. " + f"Use e.g. v1.2.3, v1.2.3-beta, v1.2.3-rc1 or v1.2.3-alpha.1." + ) + sys.exit(1) + + try: + Version(version) + except InvalidVersion: + print( + f"::error::Tag '{tag}' has an unrecognized pre-release suffix " + f"('{version}' is not valid PEP 440). Use a standard suffix such " + f"as -alpha, -alpha.1, -beta, -rc1 or -dev." + ) + sys.exit(1) + + print(f"Releasing version {version} (from tag {tag})") + + path = "freedata_server/constants.py" + with open(path) as f: + content = f.read() + + new_content, count = re.subn( + r'^MODEM_VERSION = .*$', + f'MODEM_VERSION = "{version}"', + content, + count=1, + flags=re.MULTILINE, + ) + if count != 1: + print("::error::Could not find MODEM_VERSION in freedata_server/constants.py") + sys.exit(1) + + with open(path, "w") as f: + f.write(new_content) + EOF + grep "^MODEM_VERSION" freedata_server/constants.py + - name: Build GUI working-directory: freedata_gui run: | @@ -39,7 +90,7 @@ jobs: - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@v1.14.0 - if: startsWith(github.ref, 'refs/tags/v') with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} + skip-existing: true diff --git a/Dockerfile b/Dockerfile index 1f0de20a2..86b2b15bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ ARG HAMLIB_VERSION=4.5.5 ENV HAMLIB_VERSION=${HAMLIB_VERSION} RUN apt-get update && \ - apt-get install --upgrade -y fonts-noto-color-emoji git build-essential cmake portaudio19-dev python3-pyaudio python3-colorama wget && \ + apt-get install --upgrade -y fonts-noto-color-emoji git build-essential cmake portaudio19-dev python3-colorama wget && \ mkdir -p /app/FreeDATA WORKDIR /src diff --git a/freedata_server/codec2.py b/freedata_server/codec2.py index 62bbfb997..a694f4cbd 100644 --- a/freedata_server/codec2.py +++ b/freedata_server/codec2.py @@ -89,20 +89,36 @@ def freedv_get_mode_name_by_value(mode: int) -> str: return FREEDV_MODE(mode).name -# Get the directory of the current script file -script_dir = os.path.dirname(os.path.abspath(__file__)) +# Determine the base directory to search for the codec2 shared library. +# +# In normal (non-frozen) execution this is simply the directory containing +# this script, and that's where "lib/codec2/*" lives relative to +# freedata_server/codec2.py. +# +# When compiled by Nuitka into a standalone binary however, data files added +# via --include-data-dir/--include-data-files (e.g. "lib=lib") are placed +# relative to the *distribution* directory (next to the produced .exe), not +# relative to this module's own (nested) package directory. Using +# os.path.dirname(__file__) in that case points at "/freedata_server" +# while the actual DLL ends up at "/lib/codec2/libcodec2.dll" - a +# sibling directory, not a child - so the glob below never finds it. +# +# Nuitka exposes the correct directory via the compiled-only global +# `__compiled__.containing_dir`, which always points at the distribution +# directory regardless of platform or nesting. See: +# https://nuitka.net/user-documentation/common-issue-solutions.html#standalone-finding-files +try: + script_dir = __compiled__.containing_dir # type: ignore[name-defined] +except NameError: + script_dir = os.path.dirname(os.path.abspath(__file__)) # Use script_dir to construct the paths for file search if sys.platform == "linux": - files = glob.glob(os.path.join(script_dir, "**/*libcodec2*"), recursive=True) - # files.append(os.path.join(script_dir, "libcodec2.so")) + files = glob.glob(os.path.join(script_dir, "**", "*libcodec2*"), recursive=True) elif sys.platform == "darwin": - if hasattr(sys, "_MEIPASS"): - files = glob.glob(os.path.join(getattr(sys, "_MEIPASS"), "**/*libcodec2*"), recursive=True) - else: - files = glob.glob(os.path.join(script_dir, "**/*libcodec2*.dylib"), recursive=True) + files = glob.glob(os.path.join(script_dir, "**", "*libcodec2*.dylib"), recursive=True) elif sys.platform in ["win32", "win64"]: - files = glob.glob(os.path.join(script_dir, "**\\*libcodec2*.dll"), recursive=True) + files = glob.glob(os.path.join(script_dir, "**", "*libcodec2*.dll"), recursive=True) else: files = [] api = None diff --git a/freedata_server/config.py b/freedata_server/config.py index d6f0e8328..c8c9822d2 100644 --- a/freedata_server/config.py +++ b/freedata_server/config.py @@ -1,6 +1,7 @@ import configparser import structlog import json +import os class CONFIG: @@ -296,6 +297,11 @@ def write_to_file(self): data if successful, False otherwise. """ try: + # need to create the directory before writing to it + config_dir = os.path.dirname(self.config_name) + if config_dir: + os.makedirs(config_dir, exist_ok=True) + with open(self.config_name, "w") as configfile: self.parser.write(configfile) self.ctx.config = self.read() diff --git a/freedata_server/constants.py b/freedata_server/constants.py index 781c2bae2..10497a720 100644 --- a/freedata_server/constants.py +++ b/freedata_server/constants.py @@ -1,6 +1,32 @@ # Module for saving some constants +import os +import sys + + +def _default_app_dir() -> str: + """ + Per-user directory for config, database and log file, following each + OS's own convention rather than forcing a single layout everywhere: + - Windows: %APPDATA%\\FreeDATA + - macOS: ~/Library/Application Support/FreeDATA + - Linux: $XDG_CONFIG_HOME/FreeDATA or ~/.config/FreeDATA + Used only when FREEDATA_CONFIG / FREEDATA_DATABASE are not set (e.g. a + plain `pip install freedata` run). Keeping this outside the installed + package directory means it survives package upgrades/reinstalls. + """ + home = os.path.expanduser("~") + if sys.platform == "win32": + base = os.getenv("APPDATA") or home + elif sys.platform == "darwin": + base = os.path.join(home, "Library", "Application Support") + else: + base = os.getenv("XDG_CONFIG_HOME") or os.path.join(home, ".config") + return os.path.join(base, "FreeDATA") + + CONFIG_ENV_VAR = "FREEDATA_CONFIG" DEFAULT_CONFIG_FILE = "config.ini" +DEFAULT_APP_DIR = _default_app_dir() MODEM_VERSION = "0.18.1" API_VERSION = 4 ARQ_PROTOCOL_VERSION = 1 diff --git a/freedata_server/message_system_db_manager.py b/freedata_server/message_system_db_manager.py index 12dcc5f27..cea8f4886 100644 --- a/freedata_server/message_system_db_manager.py +++ b/freedata_server/message_system_db_manager.py @@ -5,7 +5,7 @@ import structlog from freedata_server import helpers import os -from freedata_server.constants import MESSAGE_SYSTEM_DATABASE_VERSION +from freedata_server.constants import MESSAGE_SYSTEM_DATABASE_VERSION, DEFAULT_APP_DIR class DatabaseManager: @@ -42,18 +42,19 @@ def get_database(self): This method determines the database file path based on the environment variable `FREEDATA_DATABASE`. If the variable is set, its value is used as the path. Otherwise, it defaults to - `freedata-messages.db` in the script directory. + `freedata-messages.db` in the per-user app directory + (DEFAULT_APP_DIR), so a plain `pip install` keeps the database + outside the installed package and it survives upgrades. Returns: str: The database file path as a SQLAlchemy URL. """ - script_directory = os.path.dirname(os.path.abspath(__file__)) - if self.DATABASE_ENV_VAR in os.environ: - # db_path = os.getenv(self.DATABASE_ENV_VAR, os.path.join(script_directory, self.DEFAULT_DATABASE_FILE)) db_path = os.getenv(self.DATABASE_ENV_VAR) else: - db_path = os.path.join(script_directory, self.DEFAULT_DATABASE_FILE) + db_path = os.path.join(DEFAULT_APP_DIR, self.DEFAULT_DATABASE_FILE) + + os.makedirs(os.path.dirname(db_path), exist_ok=True) return "sqlite:///" + db_path def initialize_default_values(self): diff --git a/freedata_server/modem.py b/freedata_server/modem.py index 2f203478f..7c5346ddc 100644 --- a/freedata_server/modem.py +++ b/freedata_server/modem.py @@ -6,6 +6,7 @@ """ import queue +import threading import time from freedata_server import codec2 import numpy as np @@ -70,11 +71,45 @@ def __init__(self, ctx) -> None: self.AUDIO_STREAMING_CHUNK_SIZE = 2400 self.audio_out_queue = queue.Queue() + # Size of the block the RX DSP chain runs on, in 48 kHz samples. The input + # stream is opened with blocksize=0 (PortAudio picks the capture size per + # device), so the DSP must not rely on the captured block size: + # rx_audio_processing_worker re-blocks whatever the callback is handed + # to exactly this size, so a device or setting that delivers some other + # size cannot reach the DSP. 4800 (100 ms, 800 samples at 8 kHz) is the + # size the rest of the chain is built around: + # * codec2.resampler.resample48_to_8 asserts len % FDMDV_OS_48 (6) == 0 + # and raises AssertionError otherwise, + # * audio.calculate_fft pads/truncates to 800 samples at 8 kHz, so a + # shorter block is mostly zero padding and the spectrum, channel busy + # detection and audio_dbfs all degrade, + # * enqueue_streaming_audio_chunks zero-pads every block up to + # AUDIO_STREAMING_CHUNK_SIZE, so a shorter block streams mostly + # silence and emits one chunk per block regardless of size, + # * audio.normalize_audio (rx_auto_audio_level, on by default) + # normalizes per block, so a shorter block means a faster, jumpier AGC. + # Lowering this lowers RX latency, but needs those four addressed first. + self.RX_DSP_BLOCK_48K = 4800 + # Make sure our resampler will work assert (self.AUDIO_SAMPLE_RATE / self.modem_sample_rate) == codec2.api.FDMDV_OS_48 # type: ignore self.data_queue_received = queue.Queue() + # RX audio captured by the real-time sounddevice callback is handed to + # rx_audio_processing_worker through this queue, keeping the callback + # minimal (copy + enqueue). Running the DSP in the callback under the GIL + # is what makes it miss its deadline and overflow on slower CPUs. + self.rx_audio_in_queue = queue.Queue(maxsize=100) + self.rx_audio_worker_running = False + self.rx_audio_worker_thread = None + self.rx_audio_dropped_blocks = 0 + # 48 kHz samples that have been captured but do not yet fill a whole + # RX_DSP_BLOCK_48K; they are carried over to the next captured block so the + # sample stream handed to the resampler stays gapless (its filter memory + # depends on that) and always has a valid length. + self.rx_audio_carry_48k = np.empty(0, dtype=np.int16) + self.demodulator = demodulator.Demodulator(self.ctx) self.modulator = modulator.Modulator(self.ctx) @@ -116,6 +151,12 @@ def stop_modem(self): # self.stream = lambda: None # self.stream.active = False # self.stream.stop + # stop the RX audio processing worker before closing the streams + self.rx_audio_worker_running = False + try: + self.rx_audio_in_queue.put_nowait(None) + except queue.Full: + pass self.sd_input_stream.close() self.sd_output_stream.close() except Exception as e: @@ -163,16 +204,41 @@ def init_audio(self): self.resampler = codec2.resampler() # SoundDevice audio input stream + # blocksize=0 lets PortAudio pick the capture block size per device. + # This is deliberate and load-bearing: a fixed blocksize=4800 makes + # some virtual devices (measured on snd-aloop) starve/overflow -- + # a two period ring at 100 ms periods drops audio continuously and + # the modem is deaf. latency=0.2 sets the ring depth explicitly so + # the ring comes out deep (many small periods) on every device + # measured (CM108 hardware and snd-aloop alike). + # The capture block size is decoupled from the DSP block size: + # PortAudio may deliver any block length here (512/1024-class blocks + # are common on real hardware, and are NOT a multiple of codec2's + # FDMDV_OS_48 == 6, which resample48_to_8 asserts on). + # rx_audio_processing_worker re-blocks whatever arrives into exact + # RX_DSP_BLOCK_48K blocks, so no capture size can reach the DSP + # chain short or misaligned; see the note on that constant. self.sd_input_stream = sd.InputStream( channels=1, dtype="int16", callback=self.sd_input_audio_callback, device=in_dev_index, samplerate=self.AUDIO_SAMPLE_RATE, - blocksize=4800, + blocksize=0, + latency=0.2, ) self.sd_input_stream.start() + # process RX audio off the real-time callback thread + self.rx_audio_carry_48k = np.empty(0, dtype=np.int16) # no stale audio across restarts + self.rx_audio_worker_running = True + self.rx_audio_worker_thread = threading.Thread( + target=self.rx_audio_processing_worker, + name="rx_audio_processing_worker", + daemon=True, + ) + self.rx_audio_worker_thread.start() + self.sd_output_stream = sd.OutputStream( channels=1, dtype="int16", @@ -415,34 +481,97 @@ def sd_input_audio_callback(self, indata: np.ndarray, frames: int, time, status) # if status.input_overflow: # self.self.ctx.modem_service.put("restart") return + # Keep this real-time callback minimal: copy the captured block and hand + # it to rx_audio_processing_worker. The DSP (resample, FFT, demod-buffer + # push) runs there so a long GIL hold by another thread cannot stall this + # callback and cause a sounddevice input overflow on slower hardware. try: - audio_48k = np.frombuffer(indata, dtype=np.int16) - audio_8k = self.resampler.resample48_to_8(audio_48k) - - self.enqueue_streaming_audio_chunks(audio_8k, self.ctx.audio_rx_queue) - - if self.ctx.config_manager.config["AUDIO"].get("rx_auto_audio_level"): - audio_8k = audio.normalize_audio(audio_8k) - - audio_8k_level_adjusted = audio.set_audio_volume(audio_8k, self.rx_audio_level) - - if not self.ctx.state_manager.isTransmitting(): - audio.calculate_fft(audio_8k_level_adjusted, self.ctx.modem_fft, self.ctx.state_manager) - - length_audio_8k_level_adjusted = len(audio_8k_level_adjusted) - # Avoid buffer overflow by filling only if buffer for - # selected datachannel mode is not full - index = 0 - for mode in self.demodulator.MODE_DICT: - mode_data = self.demodulator.MODE_DICT[mode] - audiobuffer = mode_data["audio_buffer"] - decode = mode_data["decode"] - index += 1 - if audiobuffer: - if (audiobuffer.nbuffer + length_audio_8k_level_adjusted) > audiobuffer.size: - self.demodulator.buffer_overflow_counter[index] += 1 - self.ctx.event_manager.send_buffer_overflow(self.demodulator.buffer_overflow_counter) - elif decode: - audiobuffer.push(audio_8k_level_adjusted) - except Exception as e: - self.log.warning("[AUDIO EXCEPTION]", status=status, time=time, frames=frames, e=e) + self.rx_audio_in_queue.put_nowait(indata.copy()) + except queue.Full: + # worker is not draining fast enough; drop this block (counted) + self.rx_audio_dropped_blocks += 1 + + def rx_audio_processing_worker(self) -> None: + """Performs all RX audio DSP off the real-time input callback. + + Drains rx_audio_in_queue (raw 48 kHz int16 blocks copied by + sd_input_audio_callback) and runs the resample to 8 kHz, optional + level/FFT processing and the demodulator-buffer push on a normal + worker thread, so the audio callback is never blocked by these + operations (or by the GIL while another thread holds it). + """ + while self.rx_audio_worker_running: + try: + indata = self.rx_audio_in_queue.get(timeout=0.5) + except queue.Empty: + continue + if indata is None: # shutdown sentinel + break + try: + self.process_rx_audio_block(indata) + except Exception as e: + self.log.warning("[AUDIO EXCEPTION]", e=e) + + def process_rx_audio_block(self, indata) -> None: + """Re-blocks one captured audio block and runs the DSP chain on it. + + The input stream is free to hand the callback any block size, so the + captured samples are appended to rx_audio_carry_48k and the DSP chain is + run once per whole RX_DSP_BLOCK_48K available. Anything left over is + carried into the next captured block rather than being processed short: + a short block would fail codec2's "multiple of 6" resampler assertion and + silently degrade the FFT, streaming and AGC paths (see RX_DSP_BLOCK_48K). + + Args: + indata (np.ndarray): One captured 48 kHz int16 block, any length. + """ + captured_48k = np.frombuffer(indata, dtype=np.int16) + self.rx_audio_carry_48k = np.concatenate((self.rx_audio_carry_48k, captured_48k)) + + block = self.RX_DSP_BLOCK_48K + processed = 0 + while len(self.rx_audio_carry_48k) - processed >= block: + self.run_rx_audio_dsp(self.rx_audio_carry_48k[processed : processed + block]) + processed += block + + if processed: + # copy so the carry does not keep the whole concatenated block alive + self.rx_audio_carry_48k = self.rx_audio_carry_48k[processed:].copy() + + def run_rx_audio_dsp(self, audio_48k: np.ndarray) -> None: + """Runs the RX DSP chain on exactly one RX_DSP_BLOCK_48K of audio. + + Resamples to 8 kHz, feeds the audio streaming queue, applies the optional + auto level and the configured RX gain, updates the FFT, and pushes the + result into each decoding demodulator buffer that has room for it. + + Args: + audio_48k (np.ndarray): RX_DSP_BLOCK_48K 48 kHz int16 samples. + """ + audio_8k = self.resampler.resample48_to_8(audio_48k) + + self.enqueue_streaming_audio_chunks(audio_8k, self.ctx.audio_rx_queue) + + if self.ctx.config_manager.config["AUDIO"].get("rx_auto_audio_level"): + audio_8k = audio.normalize_audio(audio_8k) + + audio_8k_level_adjusted = audio.set_audio_volume(audio_8k, self.rx_audio_level) + + if not self.ctx.state_manager.isTransmitting(): + audio.calculate_fft(audio_8k_level_adjusted, self.ctx.modem_fft, self.ctx.state_manager) + + length_audio_8k_level_adjusted = len(audio_8k_level_adjusted) + # Avoid buffer overflow by filling only if buffer for + # selected datachannel mode is not full + index = 0 + for mode in self.demodulator.MODE_DICT: + mode_data = self.demodulator.MODE_DICT[mode] + audiobuffer = mode_data["audio_buffer"] + decode = mode_data["decode"] + index += 1 + if audiobuffer: + if (audiobuffer.nbuffer + length_audio_8k_level_adjusted) > audiobuffer.size: + self.demodulator.buffer_overflow_counter[index] += 1 + self.ctx.event_manager.send_buffer_overflow(self.demodulator.buffer_overflow_counter) + elif decode: + audiobuffer.push(audio_8k_level_adjusted) diff --git a/freedata_server/server.py b/freedata_server/server.py index 2da863389..84da828e9 100644 --- a/freedata_server/server.py +++ b/freedata_server/server.py @@ -1,4 +1,5 @@ import os +import shutil import sys import threading @@ -10,7 +11,7 @@ from fastapi.staticfiles import StaticFiles from freedata_server.log_handler import setup_logging -from freedata_server.constants import CONFIG_ENV_VAR, DEFAULT_CONFIG_FILE, API_VERSION +from freedata_server.constants import CONFIG_ENV_VAR, DEFAULT_CONFIG_FILE, DEFAULT_APP_DIR, API_VERSION from freedata_server.context import AppContext from freedata_server.api.general import router as general_router @@ -27,18 +28,39 @@ # --- Resolve config path FIRST (no logger needed yet) --- def resolve_config_path() -> str: """ - Determine the configuration file to use (env var or default next to this file). - Exits if not found. + Determine the configuration file to use. + + Uses FREEDATA_CONFIG if set, otherwise defaults to a per-user config + directory (DEFAULT_APP_DIR). If no config file exists yet at that + location, a fresh one is bootstrapped from the bundled + config.ini.example template so a plain `pip install freedata` followed + by `freedata` works out of the box without any manual setup. """ - candidate = os.getenv( - CONFIG_ENV_VAR, - os.path.join(os.path.dirname(__file__), DEFAULT_CONFIG_FILE), + candidate = os.path.abspath( + os.getenv( + CONFIG_ENV_VAR, + os.path.join(DEFAULT_APP_DIR, DEFAULT_CONFIG_FILE), + ) ) + if not os.path.exists(candidate): # We cannot log to file yet since we don't know the directory; write to stderr. - sys.stderr.write(f"[FATAL] Config file not found: {candidate}\n") - sys.exit(1) - return os.path.abspath(candidate) + template = os.path.join(os.path.dirname(__file__), "config.ini.example") + try: + os.makedirs(os.path.dirname(candidate), exist_ok=True) + if os.path.isfile(template): + shutil.copyfile(template, candidate) + sys.stderr.write(f"[INFO] No config found - created a default one at: {candidate}\n") + else: + sys.stderr.write( + f"[FATAL] Config file not found and no template available to create one: {candidate}\n" + ) + sys.exit(1) + except OSError as e: + sys.stderr.write(f"[FATAL] Could not create config file at {candidate}: {e}\n") + sys.exit(1) + + return candidate config_file = resolve_config_path() @@ -95,11 +117,15 @@ async def nocache(request: Request, call_next): # Static GUI mounting +# Order matters: prefer paths anchored to this file's location (work no +# matter what the current working directory is) over cwd-relative +# fallbacks kept for backwards compatibility with older layouts. potential_gui_dirs = [ + os.path.join(os.path.dirname(__file__), "gui"), # nuitka standalone build + os.path.join(os.path.dirname(os.path.dirname(__file__)), "freedata_gui", "dist"), # pip install (sibling package) "../freedata_gui/dist", "freedata_gui/dist", "FreeDATA/freedata_gui/dist", - os.path.join(os.path.dirname(__file__), "gui"), ] gui_dir = next((d for d in potential_gui_dirs if os.path.isdir(d)), None) if gui_dir: diff --git a/pyproject.toml b/pyproject.toml index cc64850eb..6685995de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,6 @@ requires-python = ">=3.10" dependencies = [ "numpy", "psutil", - "PyAudio", "pyserial", "sounddevice", "structlog", @@ -78,12 +77,13 @@ nuitka = [ [tool.setuptools.packages.find] where = [ "." ] -exclude = [ - "tools*", +include = [ + "freedata_server*", + "freedata_gui", ] [tool.setuptools.package-data] -freedata_server = [ "lib/**/*" ] +freedata_server = [ "lib/**/*", "config.ini.example" ] freedata_gui = [ "dist/**/*" ] [tool.setuptools.dynamic] diff --git a/requirements.txt b/requirements.txt index 97d2bd0d0..533e64e4f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ numpy psutil -PyAudio pyserial sounddevice structlog diff --git a/tests/test_rx_audio_callback.py b/tests/test_rx_audio_callback.py new file mode 100644 index 000000000..76310b6d2 --- /dev/null +++ b/tests/test_rx_audio_callback.py @@ -0,0 +1,168 @@ +"""Tests for the real-time-safe RX audio callback / worker split. + +modem.RF.sd_input_audio_callback no longer runs the RX DSP inline; it copies the +captured block onto rx_audio_in_queue and returns, and rx_audio_processing_worker +drains that queue and runs the DSP (resample 48->8 kHz, FFT, demod-buffer push). + +The stream is opened with blocksize=0, so the captured block size is whatever +PortAudio has available: device specific, variable, and not a multiple of +anything. rx_audio_processing_worker therefore re-blocks the captured stream to +RX_DSP_BLOCK_48K before any DSP runs on it. + +These tests exercise that split directly with synthetic blocks, so they need no +audio hardware (CI has none). They cover: + - the worker actually performs the relocated DSP on an enqueued block, + - odd capture sizes are re-blocked rather than passed to the DSP short, and + - the callback drops (and counts) instead of blocking when the queue is full, + which is the real-time-safety property the change exists to provide. +""" + +import threading +import time +import unittest + +import numpy as np + +from freedata_server.context import AppContext +from freedata_server import modem, codec2 + +CONFIG = "freedata_server/config.ini.example" + +# A capture size PortAudio really does hand us, and deliberately NOT a multiple of +# codec2's FDMDV_OS_48 (6) -- 512 % 6 == 2. Passing this straight to +# resample48_to_8 trips its "multiple of 6" assertion, which is what made the RX +# chain deaf (every block raising AssertionError) once blocksize=0 was used. +CAPTURE_FRAMES = 512 + + +def _rf(): + """A real RF wired to a real AppContext, without opening audio devices. + + start_modem() would normally create the resampler (and, in TESTMODE, start + the demodulator decode threads); we only need the resampler here, so we set + it directly and leave the demod buffers as None -- the worker's buffer-push + is guarded by `if audiobuffer` and is intentionally not under test. + """ + ctx = AppContext(CONFIG) + ctx.TESTMODE = True + rf = modem.RF(ctx) + rf.resampler = codec2.resampler() + return rf + + +def _block(frames=CAPTURE_FRAMES): + # sounddevice delivers indata as shape (frames, channels); int16 mono here. + return (np.random.randn(frames, 1) * 3000).astype(np.int16) + + +class TestRxAudioCallbackWorkerSplit(unittest.TestCase): + def test_worker_processes_enqueued_blocks(self): + """Blocks handed to the callback are drained and DSP'd by the worker.""" + rf = _rf() + rf.rx_audio_worker_running = True + worker = threading.Thread(target=rf.rx_audio_processing_worker, daemon=True) + worker.start() + try: + # status=None -> the block is enqueued (a truthy status is an + # over/underflow and is dropped by the callback, unchanged by this PR). + # Feed enough captured blocks to complete at least one DSP block. + for _ in range(rf.RX_DSP_BLOCK_48K // CAPTURE_FRAMES + 1): + rf.sd_input_audio_callback(_block(), CAPTURE_FRAMES, None, None) + + # The worker resamples to 8 kHz and feeds enqueue_streaming_audio_chunks, + # which lands on ctx.audio_rx_queue -- our deterministic "DSP ran" signal. + deadline = time.time() + 5 + while rf.ctx.audio_rx_queue.qsize() == 0 and time.time() < deadline: + time.sleep(0.02) + + self.assertGreater(rf.ctx.audio_rx_queue.qsize(), 0, "worker did not process the enqueued RX audio block") + self.assertTrue(rf.rx_audio_in_queue.empty(), "worker should have drained the input queue") + self.assertEqual(rf.rx_audio_dropped_blocks, 0, "no block should be dropped under normal operation") + finally: + rf.rx_audio_worker_running = False + rf.rx_audio_in_queue.put_nowait(None) # release the worker's get() + worker.join(timeout=2) + + def test_callback_drops_and_does_not_block_when_queue_full(self): + """With the worker stalled and the queue full, the callback drops the + block (counted) and returns immediately -- it must never block the + real-time audio thread.""" + rf = _rf() # worker intentionally NOT started -> queue never drains + for _ in range(rf.rx_audio_in_queue.maxsize): + rf.rx_audio_in_queue.put_nowait(object()) + self.assertTrue(rf.rx_audio_in_queue.full()) + + t0 = time.perf_counter() + rf.sd_input_audio_callback(_block(), CAPTURE_FRAMES, None, None) + elapsed = time.perf_counter() - t0 + + self.assertEqual(rf.rx_audio_dropped_blocks, 1, "a full queue must drop the block and count it") + self.assertLess(elapsed, 0.05, "callback must not block on a full queue (real-time safety)") + + +class TestRxAudioReblocking(unittest.TestCase): + """The capture block size must never reach the DSP chain. + + blocksize=0 means PortAudio picks the size, and real devices hand back sizes + that are not multiples of codec2's FDMDV_OS_48 (6). resample48_to_8 asserts on + those, so process_rx_audio_block must accumulate instead of resampling short. + These call process_rx_audio_block directly (no worker thread) so a failure is + a raised exception rather than a swallowed, logged one. + """ + + def test_odd_capture_size_does_not_reach_the_resampler(self): + """A 512-frame capture (512 % 6 == 2) must not raise AssertionError.""" + rf = _rf() + # One short block: not enough for a DSP block, so it is carried, not resampled. + rf.process_rx_audio_block(_block(CAPTURE_FRAMES)) + self.assertEqual(len(rf.rx_audio_carry_48k), CAPTURE_FRAMES) + self.assertEqual(rf.ctx.audio_rx_queue.qsize(), 0, "a partial DSP block must not be processed short") + + def test_carry_reassembles_whole_dsp_blocks(self): + """Odd captures are accumulated into exact RX_DSP_BLOCK_48K blocks.""" + rf = _rf() + processed = [] + rf.run_rx_audio_dsp = lambda audio_48k: processed.append(len(audio_48k)) + + # A spread of sizes a real device might deliver, none a multiple of 6. + sizes = [512, 1024, 441, 512, 2048, 1024, 512, 940, 512, 1024] + for size in sizes: + rf.process_rx_audio_block(_block(size)) + + total = sum(sizes) + self.assertEqual( + processed, + [rf.RX_DSP_BLOCK_48K] * (total // rf.RX_DSP_BLOCK_48K), + "every DSP invocation must get exactly one whole block", + ) + self.assertEqual(len(rf.rx_audio_carry_48k), total % rf.RX_DSP_BLOCK_48K, "remainder must be carried over") + + def test_reblocking_preserves_the_sample_stream(self): + """No sample is dropped, duplicated or reordered by the re-blocking. + + The resampler's filter memory spans blocks, so the stream it sees has to be + the captured stream exactly. + """ + rf = _rf() + seen = [] + rf.run_rx_audio_dsp = lambda audio_48k: seen.append(np.array(audio_48k)) + + sizes = [700, 1300, 512, 4800, 441] + captured = [np.arange(s, dtype=np.int16).reshape(-1, 1) for s in sizes] + for block in captured: + rf.process_rx_audio_block(block) + + expected = np.concatenate([b.reshape(-1) for b in captured]) + got = np.concatenate(seen + [rf.rx_audio_carry_48k]) + np.testing.assert_array_equal(got, expected) + + def test_real_resampler_accepts_every_reblocked_block(self): + """End to end with the real codec2 resampler: odd captures, no assertion.""" + rf = _rf() + for size in (512, 1024, 441, 2048, 512, 1024, 512, 4800): + rf.process_rx_audio_block(_block(size)) # raises AssertionError if short + self.assertGreater(rf.ctx.audio_rx_queue.qsize(), 0, "DSP should have run on the reassembled blocks") + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/Linux/install-freedata-linux.sh b/tools/Linux/install-freedata-linux.sh index f69c0fb9e..da3ab5a13 100755 --- a/tools/Linux/install-freedata-linux.sh +++ b/tools/Linux/install-freedata-linux.sh @@ -44,6 +44,9 @@ # # # Changelog: +# 2.10: 24 Jul 2026 +# Remove python3-pyaudio (unused dependency, FreeDATA uses sounddevice) +# # 2.9: 10 Jan Sep 2026 # Add Ubuntu 24.10 and 25.04 # Change hamlib default version to 4.6.5 @@ -164,7 +167,7 @@ case $osname in "Debian GNU/Linux") case $osversion in "11" | "12" | "13") - sudo apt install --upgrade -y fonts-noto-color-emoji git build-essential cmake python3 portaudio19-dev python3-pyaudio python3-pip python3-colorama python3-venv wget python3-dev + sudo apt install --upgrade -y fonts-noto-color-emoji git build-essential cmake python3 portaudio19-dev python3-pip python3-colorama python3-venv wget python3-dev ;; *) @@ -182,7 +185,7 @@ case $osname in "Ubuntu" | "Linux Mint") case $osversion in "21.3" | "22.04" | "24.04" | "24.10" | "25.04" ) - sudo apt install --upgrade -y fonts-noto-color-emoji git build-essential cmake python3 portaudio19-dev python3-pyaudio python3-pip python3-colorama python3-venv wget python3-dev + sudo apt install --upgrade -y fonts-noto-color-emoji git build-essential cmake python3 portaudio19-dev python3-pip python3-colorama python3-venv wget python3-dev ;; *) @@ -197,7 +200,7 @@ case $osname in "Fedora Linux") case $osversion in "VERSION_ID=40" | "VERSION_ID=41") - sudo dnf install -y git cmake make automake gcc gcc-c++ kernel-devel wget portaudio-devel python3-pyaudio python3-pip python3-colorama python3-virtualenv google-noto-emoji-fonts python3-devel + sudo dnf install -y git cmake make automake gcc gcc-c++ kernel-devel wget portaudio-devel python3-pip python3-colorama python3-virtualenv google-noto-emoji-fonts python3-devel ;; esac ;;