forked from TheDeathDragon/LiveTranslate
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmain.py
More file actions
2343 lines (2072 loc) · 89.8 KB
/
Copy pathmain.py
File metadata and controls
2343 lines (2072 loc) · 89.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
LiveTranslate - Phase 0 Prototype
Real-time audio translation using WASAPI loopback + faster-whisper + LLM.
"""
import sys
import signal
import logging
import threading
import queue
import gc
from concurrent.futures import ThreadPoolExecutor
import yaml
import time
import numpy as np
from pathlib import Path
from datetime import datetime
from model_manager import (
DEFAULT_FUNASR_MODEL,
apply_cache_env,
funasr_display_name,
funasr_supports_padding,
get_missing_models,
is_asr_cached,
ASR_DISPLAY_NAMES,
MODELS_DIR,
local_faster_whisper_display_name,
migrate_funasr_settings,
normalize_asr_engine_selection,
normalize_funasr_model_key,
resolve_custom_whisper_model,
)
# Set cache env BEFORE importing torch so TORCH_HOME is respected
apply_cache_env()
import os
# torch must be imported before PyQt6 to avoid DLL conflicts on Windows
import torch # noqa: F401
from audio_capture import AudioCapture
from vad_processor import VADProcessor
from asr_client import ASRClient, ASRWorkerError, ASRWorkerExited, ASRWorkerTimeout
from translator import Translator, RepetitionError
from transcript_writer import TranscriptWriter
from PyQt6.QtWidgets import QApplication, QSystemTrayIcon, QMenu, QDialog, QMessageBox
from PyQt6.QtGui import (
QAction,
QActionGroup,
QIcon,
QPixmap,
QPainter,
QColor,
QFont,
QFontDatabase,
)
from PyQt6.QtCore import QTimer, Qt
from subtitle_overlay import SubtitleOverlay
from subtitle_window import SubtitleWindow
from log_window import LogWindow
from control_panel import (
ControlPanel,
SETTINGS_FILE,
_load_saved_settings,
_save_settings,
)
from dialogs import (
SetupWizardDialog,
ModelDownloadDialog,
_ModelLoadDialog,
)
from i18n import t, set_lang, LANGUAGES, COMMON_LANG_CODES
_NO_PENDING = object()
def setup_logging():
log_dir = Path(__file__).parent / "logs"
log_dir.mkdir(exist_ok=True)
log_file = log_dir / f"livetrans_{datetime.now():%Y%m%d_%H%M%S}.log"
file_handler = logging.FileHandler(log_file, encoding="utf-8")
file_handler.setLevel(logging.DEBUG)
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setLevel(logging.INFO)
fmt = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(message)s")
file_handler.setFormatter(fmt)
console_handler.setFormatter(fmt)
logging.basicConfig(level=logging.DEBUG, handlers=[file_handler, console_handler])
for noisy in (
"httpcore",
"httpx",
"openai",
"filelock",
"huggingface_hub",
"funasr",
"modelscope",
"onnxruntime",
):
logging.getLogger(noisy).setLevel(logging.WARNING)
logging.info(f"Log file: {log_file}")
# FunASR/ModelScope spam the root logger; suppress after our own init log
logging.getLogger().setLevel(logging.WARNING)
logging.getLogger("LiveTranslate").setLevel(logging.DEBUG)
_logger = logging.getLogger("LiveTranslate")
def _excepthook(exc_type, exc_value, exc_tb):
_logger.critical("Uncaught exception", exc_info=(exc_type, exc_value, exc_tb))
sys.__excepthook__(exc_type, exc_value, exc_tb)
sys.excepthook = _excepthook
def _thread_excepthook(args):
_logger.critical(
f"Uncaught exception in thread {args.thread}",
exc_info=(args.exc_type, args.exc_value, args.exc_traceback),
)
threading.excepthook = _thread_excepthook
return _logger
log = logging.getLogger("LiveTranslate")
def create_app_icon() -> QIcon:
pix = QPixmap(64, 64)
pix.fill(QColor(0, 0, 0, 0))
p = QPainter(pix)
p.setRenderHint(QPainter.RenderHint.Antialiasing)
p.setBrush(QColor(60, 130, 240))
p.setPen(Qt.PenStyle.NoPen)
p.drawRoundedRect(4, 4, 56, 56, 12, 12)
p.setPen(QColor(255, 255, 255))
p.setFont(QFont("Consolas", 28, QFont.Weight.Bold))
p.drawText(pix.rect(), Qt.AlignmentFlag.AlignCenter, "LT")
p.end()
return QIcon(pix)
def load_config():
config_path = Path(__file__).parent / "config.yaml"
with open(config_path, "r", encoding="utf-8") as f:
return yaml.safe_load(f)
class LiveTranslateApp:
def __init__(self, config):
self._config = config
self._running = False
self._paused = False
self._asr_ready = False # True when ASR model is loaded
self._audio = AudioCapture(
device=config["audio"].get("device"),
sample_rate=config["audio"]["sample_rate"],
chunk_duration=config["audio"]["chunk_duration"],
)
self._vad = VADProcessor(
sample_rate=config["audio"]["sample_rate"],
threshold=config["asr"]["vad_threshold"],
min_speech_duration=config["asr"]["min_speech_duration"],
max_speech_duration=config["asr"]["max_speech_duration"],
chunk_duration=config["audio"]["chunk_duration"],
)
self._asr_type = None
self._asr = None
self._asr_signature = None
self._asr_config = None
self._asr_error_count = 0
self._asr_device = config["asr"]["device"]
self._whisper_model_size = config["asr"]["model_size"]
self._funasr_model_key = normalize_funasr_model_key(
config["asr"].get("funasr_model", DEFAULT_FUNASR_MODEL)
)
self._asr_lock = threading.RLock()
self._vad_lock = threading.Lock()
# Settings changed from the Qt thread are deferred here and applied by the
# ASR thread before its next transcribe, so the UI never blocks on the
# worker pipe (which may be busy with an in-flight cross-process call).
# Padding is keyed by engine_type because one settings save updates both
# the funasr and whisper padding and they must not clobber each other.
self._asr_pending_lock = threading.Lock()
self._asr_pending_language = _NO_PENDING
self._asr_pending_padding = {}
# Auto-restart bookkeeping for a worker that dies mid-session. _asr_generation
# is bumped on every (de)activation so a slow background (re)start can detect
# that a newer engine switch superseded it and discard its stale worker.
self._asr_restart_state = None
self._asr_restart_count = 0
self._asr_restart_max = 3
self._asr_generation = 0
self._asr_recycling = False
# Proactively recycle the worker once its RSS grows this far past the
# post-load baseline, to bound native-side (FunASR/CTranslate2) leaks that
# accumulate in the long-lived worker process.
self._asr_worker_baseline_mb = None
self._asr_recycle_delta_mb = 2048
self._target_language = config["translation"]["target_language"]
self._translator = Translator(
api_base=config["translation"]["api_base"],
api_key=config["translation"]["api_key"],
model=config["translation"]["model"],
target_language=self._target_language,
max_tokens=config["translation"]["max_tokens"],
temperature=config["translation"]["temperature"],
streaming=config["translation"]["streaming"],
system_prompt=config["translation"].get("system_prompt"),
)
self._translator.set_context_turns(
config["translation"].get("context_window", 0)
)
self._overlay = None
self._subwin = None
self._panel = None
self._capture_thread = None
self._asr_thread = None
self._asr_queue = queue.Queue(maxsize=16)
self._tl_executor = ThreadPoolExecutor(max_workers=8)
self._transcript = TranscriptWriter(Path(__file__).parent / "transcripts")
# Memory diagnostic state
import psutil
self._mem_proc = psutil.Process(os.getpid())
self._mem_baseline_mb = self._mem_proc.memory_info().rss / 1024 / 1024
self._mem_last_mb = self._mem_baseline_mb
self._mem_asr_call_count = 0
self._mem_periodic_timer = None
# Memory ceiling: warn once when combined RSS (main + ASR worker) exceeds
# threshold. The ASR backend now runs in a worker process and keeps
# native-side workspaces/caches that Python GC cannot always reclaim, so the
# ceiling must include the worker's RSS (see _mem_snapshot).
self._mem_threshold_mb = 4096
self._mem_warned = False
self._mem_warning_callback = None
self._asr_count = 0
self._translate_count = 0
self._total_prompt_tokens = 0
self._total_completion_tokens = 0
self._input_price = 0.0
self._output_price = 0.0
self._msg_id = 0
self._last_original = ""
self._last_msg_id = 0
# Incremental ASR state
self._incremental_enabled = False
self._interim_interval = 2.0
self._interim_pending = ""
self._interim_active = False
self._last_interim_samples = 0
self._last_interim_check_time = 0.0
self._interim_committed_tail = ""
def set_overlay(self, overlay: SubtitleOverlay):
self._overlay = overlay
def set_subtitle_window(self, subwin: SubtitleWindow):
self._subwin = subwin
def set_panel(self, panel: ControlPanel):
self._panel = panel
panel.settings_changed.connect(self._on_settings_changed)
panel.model_changed.connect(self._on_model_changed)
panel.models_list_changed.connect(self._on_models_list_changed)
def _on_models_list_changed(self, models: list, active_idx: int):
if self._overlay:
self._overlay.set_models(models, active_idx)
def _on_settings_changed(self, settings):
self._vad.update_settings(settings)
if "style" in settings and self._overlay:
self._overlay.apply_style(settings["style"])
if "asr_language" in settings:
self._set_asr_language(settings["asr_language"])
if "sensevoice_pad_seconds" in settings:
self._set_asr_padding("funasr", settings["sensevoice_pad_seconds"])
if "whisper_pad_seconds" in settings:
self._set_asr_padding("whisper", settings["whisper_pad_seconds"])
if any(
key in settings
for key in (
"asr_engine",
"asr_device",
"whisper_model_size",
"funasr_model",
"hub",
)
):
self._switch_asr_engine(
settings.get(
"asr_engine",
self._asr_type or self._config["asr"].get("asr_engine", "funasr"),
)
)
if "audio_device" in settings:
old_device = self._audio._device_name
self._audio.set_device(settings["audio_device"])
if old_device != settings.get("audio_device"):
self._vad.flush()
self._vad._reset()
if self._overlay:
self._overlay.update_monitor(0.0, 0.0)
if "mic_device" in settings:
self._audio.set_mic_device(settings["mic_device"])
if "incremental_asr" in settings:
self._incremental_enabled = settings["incremental_asr"]
if "interim_interval" in settings:
self._interim_interval = settings["interim_interval"]
if "target_language" in settings:
self._target_language = settings["target_language"]
if self._overlay:
self._overlay.set_target_language(self._target_language)
if "timeout" in settings and self._translator:
self._translator.set_timeout(settings["timeout"])
if "auto_save_transcript" in settings:
self._transcript.set_enabled(settings["auto_save_transcript"])
def _mark_asr_unavailable(self, reason: str, client=None):
with self._asr_lock:
current = client or self._asr
if client is not None and self._asr is not client:
return
self._asr_ready = False
self._asr = None
self._asr_type = None
self._asr_signature = None
self._asr_config = None
self._asr_error_count = 0
self._asr_restart_state = None
self._asr_worker_baseline_mb = None
self._asr_generation += 1
if current is not None:
try:
current.shutdown()
except Exception:
try:
current.terminate()
except Exception:
pass
log.warning(f"ASR worker unavailable: {reason}")
if self._overlay:
self._overlay.update_asr_device("ASR unavailable")
def _shutdown_asr_worker(self):
with self._asr_lock:
client = self._asr
self._asr = None
self._asr_ready = False
self._asr_type = None
self._asr_signature = None
self._asr_config = None
self._asr_error_count = 0
self._asr_restart_state = None
self._asr_worker_baseline_mb = None
self._asr_generation += 1
if client is not None:
log.info(f"Shutting down ASR worker: pid={client.pid}")
client.shutdown()
def _set_asr_language(self, language: str):
with self._asr_pending_lock:
self._asr_pending_language = language
def _set_asr_padding(self, engine_type: str, pad_seconds):
with self._asr_pending_lock:
self._asr_pending_padding[engine_type] = pad_seconds
def _apply_pending_asr_settings(self, client, asr_type, funasr_key):
"""Apply deferred language/padding on the ASR thread, just before a transcribe.
A pending value is cleared only once delivered; worker-death exceptions
propagate with the pending intact so the restarted worker re-applies it. The
applied value is written back into the restart config so an auto-restart or
recycle does not revert a runtime override to the engine-switch-time value."""
with self._asr_pending_lock:
language = self._asr_pending_language
pad_seconds = self._asr_pending_padding.get(asr_type, _NO_PENDING)
if language is not _NO_PENDING:
try:
client.set_language(language)
except ASRWorkerError as exc:
log.warning(f"ASR language update failed: {exc}")
self._update_restart_config(language=language)
self._clear_pending_language(language)
if pad_seconds is not _NO_PENDING:
if not (asr_type == "funasr" and not funasr_supports_padding(funasr_key)):
try:
client.set_input_padding(pad_seconds)
except ASRWorkerError as exc:
log.warning(f"ASR padding update failed: {exc}")
self._update_restart_config(pad_seconds=pad_seconds)
self._clear_pending_padding(asr_type, pad_seconds)
def _clear_pending_language(self, language):
with self._asr_pending_lock:
if self._asr_pending_language is language:
self._asr_pending_language = _NO_PENDING
def _clear_pending_padding(self, asr_type, pad_seconds):
with self._asr_pending_lock:
if self._asr_pending_padding.get(asr_type) == pad_seconds:
del self._asr_pending_padding[asr_type]
def _update_restart_config(self, **kwargs):
with self._asr_lock:
if self._asr_restart_state and self._asr_restart_state.get("config"):
self._asr_restart_state["config"].update(kwargs)
def _load_engine_client(self, config: dict):
"""Build the ASR backend for a worker config. Local engines run in an isolated
worker subprocess (ASRClient); remote-whisper is a thin in-process HTTP client
that needs no subprocess isolation (no native deps, no GPU model to load)."""
if config.get("engine_type") == "remote-whisper":
from asr_remote import RemoteASREngine
url = config.get("remote_asr_url") or "http://127.0.0.1:8765"
engine = RemoteASREngine(server_url=url)
language = config.get("language")
if language:
engine.set_language(language)
return engine
return self._load_asr_client(config)
def _load_asr_client(self, worker_config: dict) -> ASRClient:
# request_timeout bounds how long a hung worker can stall the realtime path
# before it is killed and auto-restarted. VAD caps segments at a few seconds,
# so 60s is generous for a healthy transcribe yet far below the old 120s.
client = ASRClient(worker_config, request_timeout=60.0)
try:
client.start()
client.wait_ready()
return client
except Exception:
client.shutdown()
raise
def _on_target_language_changed(self, lang: str):
self._target_language = lang
log.info(f"Target language: {lang}")
if self._translator:
self._translator.set_target_language(lang)
if self._panel:
settings = self._panel.get_settings()
settings["target_language"] = lang
from control_panel import _save_settings
_save_settings(settings)
def _on_model_changed(self, model_config: dict):
log.info(
f"Switching translator: {model_config['name']} ({model_config['model']})"
)
prompt = None
if self._panel:
prompt = self._panel.get_settings().get("system_prompt")
if not prompt:
prompt = self._config["translation"].get("system_prompt")
timeout = 10
if self._panel:
timeout = self._panel.get_settings().get("timeout", 10)
self._translator = Translator(
api_base=model_config["api_base"],
api_key=model_config["api_key"],
model=model_config["model"],
target_language=self._target_language,
max_tokens=self._config["translation"]["max_tokens"],
temperature=self._config["translation"]["temperature"],
streaming=model_config.get("streaming", True),
system_prompt=prompt,
proxy=model_config.get("proxy", "none"),
no_system_role=model_config.get("no_system_role", False),
no_think=model_config.get("no_think", True),
json_response=model_config.get("json_response", False),
timeout=timeout,
overrides=model_config.get("overrides"),
extra_body=model_config.get("extra_body"),
)
context_turns = model_config.get(
"context_turns", self._config["translation"].get("context_window", 0)
)
self._translator.set_context_turns(context_turns)
self._input_price = model_config.get("input_price", 0)
self._output_price = model_config.get("output_price", 0)
def _switch_asr_engine(self, engine_type: str):
settings = self._panel.get_settings() if self._panel else {}
engine_type, funasr_model = normalize_asr_engine_selection(
engine_type, settings.get("funasr_model", self._funasr_model_key)
)
device = settings.get("asr_device", self._asr_device)
hub = "ms"
download_proxy = "system"
if self._panel:
hub = settings.get("hub", "ms")
download_proxy = settings.get("download_proxy", "system")
model_size = self._config["asr"]["model_size"]
if self._panel:
model_size = settings.get("whisper_model_size", model_size)
model_path = None
cache_model_key = model_size
if engine_type == "whisper":
model_path = resolve_custom_whisper_model(model_size)
if model_path:
cache_model_key = model_path
elif engine_type == "funasr":
cache_model_key = funasr_model
remote_asr_url = settings.get(
"remote_asr_url",
self._config["asr"].get("remote_asr_url", "http://127.0.0.1:8765"),
)
compute = self._config["asr"]["compute_type"]
if engine_type == "whisper":
signature_model = cache_model_key
elif engine_type == "funasr":
signature_model = funasr_model
elif engine_type == "remote-whisper":
# URL is part of the identity so editing it triggers a reconnect.
signature_model = remote_asr_url
else:
signature_model = engine_type
signature = (engine_type, signature_model, device, hub, compute)
with self._asr_lock:
current_asr = self._asr
current_ready = (
self._asr_ready
and current_asr is not None
and current_asr.status == "ready"
)
if current_ready and self._asr_signature == signature:
return
if not current_ready:
self._asr_ready = False
log.info(f"Switching ASR worker: {self._asr_type} -> {engine_type}")
# Reset interim state for the engine boundary. The active worker is
# stopped before the target worker starts loading.
self._interim_active = False
self._interim_pending = ""
self._last_interim_samples = 0
self._last_interim_check_time = 0.0
self._interim_committed_tail = ""
self._vad.flush()
self._vad._reset()
cached = is_asr_cached(engine_type, cache_model_key, hub)
display_name = ASR_DISPLAY_NAMES.get(engine_type, engine_type)
if engine_type == "whisper":
display_model = (
local_faster_whisper_display_name(model_size)
if model_path
else model_size
) or Path(model_size).name
display_name = f"Whisper {display_model}"
elif engine_type == "funasr":
display_name = funasr_display_name(funasr_model)
parent = (
self._panel if self._panel and self._panel.isVisible() else self._overlay
)
worker_config = {
"engine_type": engine_type,
"funasr_model": funasr_model,
"model_size": cache_model_key,
"device": device,
"compute_type": compute,
"hub": hub,
"language": settings.get(
"asr_language", self._config["asr"].get("language", "auto")
),
"pad_seconds": (
settings.get(
"sensevoice_pad_seconds",
self._config["asr"].get("sensevoice_pad_seconds", 0.5),
)
if engine_type == "funasr"
else settings.get(
"whisper_pad_seconds",
self._config["asr"].get("whisper_pad_seconds", 0.5),
)
if engine_type == "whisper"
else None
),
"download_root": str((MODELS_DIR / "huggingface" / "hub").resolve()),
"display_name": display_name,
"remote_asr_url": remote_asr_url,
}
target_state = {
"type": engine_type,
"signature": signature,
"device": device,
"funasr_model_key": funasr_model
if engine_type == "funasr"
else self._funasr_model_key,
"whisper_model_size": model_size
if engine_type == "whisper"
else self._whisper_model_size,
"config": worker_config,
"display_name": display_name,
"device_label": (
remote_asr_url if engine_type == "remote-whisper" else device
),
}
if not cached:
missing = get_missing_models(engine_type, cache_model_key, hub)
missing = [m for m in missing if m["type"] != "silero-vad"]
if missing:
dlg = ModelDownloadDialog(
missing, hub=hub, proxy=download_proxy, parent=parent
)
if dlg.exec() != QDialog.DialogCode.Accepted:
log.info(f"Download cancelled/failed: {engine_type}")
with self._asr_lock:
self._asr_ready = (
self._asr is not None and self._asr.status == "ready"
)
return
with self._asr_lock:
old_asr = self._asr
old_config = dict(self._asr_config) if self._asr_config else None
old_state = {
"type": self._asr_type,
"signature": self._asr_signature,
"device": self._asr_device,
"funasr_model_key": self._funasr_model_key,
"whisper_model_size": self._whisper_model_size,
"config": old_config,
"display_name": (old_config or {}).get("display_name"),
"device_label": (
(old_config or {}).get("remote_asr_url")
if self._asr_type == "remote-whisper"
else self._asr_device
),
}
self._asr = None
self._asr_ready = False
self._asr_type = None
self._asr_signature = None
self._asr_config = None
self._asr_error_count = 0
self._asr_restart_state = None
self._asr_worker_baseline_mb = None
self._asr_generation += 1
dlg = _ModelLoadDialog(
t("loading_model").format(name=display_name), parent=parent
)
new_asr = [None]
restored_asr = [None]
load_error = [None]
restore_error = [None]
def _load():
if old_asr is not None:
log.info(f"Stopping old ASR worker before switch: pid={old_asr.pid}")
old_asr.shutdown()
self._release_memory_caches()
try:
new_asr[0] = self._load_engine_client(worker_config)
except Exception as e:
load_error[0] = str(e)
# A remote server that is simply down is an expected, user-actionable
# condition, not a bug, so skip the noisy traceback for it.
expected = isinstance(e, ConnectionError)
log.error(
f"Failed to load ASR worker: {e}", exc_info=not expected
)
if old_config:
try:
log.info("Restoring previous ASR worker after switch failure")
restored_asr[0] = self._load_engine_client(old_config)
except Exception as restore_exc:
restore_error[0] = str(restore_exc)
log.error(
f"Failed to restore previous ASR worker: {restore_exc}",
exc_info=True,
)
thread = threading.Thread(target=_load, daemon=True)
thread.start()
poll_timer = QTimer()
def _check():
if not thread.is_alive():
poll_timer.stop()
dlg.accept()
poll_timer.setInterval(100)
poll_timer.timeout.connect(_check)
poll_timer.start()
dlg.exec()
poll_timer.stop()
def _activate_asr(client, state):
with self._asr_lock:
self._asr = client
self._asr_type = state["type"]
self._asr_signature = state["signature"]
self._asr_device = state["device"]
self._asr_config = dict(state["config"]) if state["config"] else None
self._funasr_model_key = state["funasr_model_key"]
self._whisper_model_size = state["whisper_model_size"]
self._asr_ready = True
self._asr_error_count = 0
self._asr_restart_state = dict(state)
self._asr_restart_count = 0
self._asr_worker_baseline_mb = None
self._asr_generation += 1
if new_asr[0] is not None:
_activate_asr(new_asr[0], target_state)
if self._overlay:
self._overlay.update_asr_device(
f"{display_name} [{target_state['device_label']}]"
)
log.info(f"ASR worker ready: {engine_type} on {device}")
return
if restored_asr[0] is not None:
_activate_asr(restored_asr[0], old_state)
restored_name = old_state.get("display_name") or old_state.get("type")
if self._overlay:
self._overlay.update_asr_device(
f"{restored_name} [{old_state.get('device_label', old_state['device'])}]"
)
QMessageBox.warning(
parent,
t("error_title"),
t("error_load_asr").format(
error=(
f"{load_error[0] or 'unknown error'}\n"
f"{t('asr_restore_succeeded')}"
)
),
)
log.info(
f"Previous ASR worker restored: "
f"{old_state.get('type')} on {old_state.get('device')}"
)
return
error = load_error[0] or "unknown error"
if restore_error[0]:
error = (
f"{error}\n"
f"{t('asr_restore_failed').format(error=restore_error[0])}"
)
QMessageBox.warning(
parent,
t("error_title"),
t("error_load_asr").format(error=error),
)
if self._overlay:
self._overlay.update_asr_device("ASR unavailable")
def _mem_snapshot(self) -> dict:
rss_mb = self._mem_proc.memory_info().rss / 1024 / 1024
# The ASR model (and its native-side leak) lives in the worker process now,
# so sample its RSS too; the main process holds only VAD + Qt.
worker_rss_mb = 0.0
client = self._asr
if client is not None and client.pid is not None:
try:
import psutil
worker_rss_mb = (
psutil.Process(client.pid).memory_info().rss / 1024 / 1024
)
except Exception:
worker_rss_mb = 0.0
gpu_alloc_mb = 0.0
gpu_reserved_mb = 0.0
try:
if torch.cuda.is_available():
gpu_alloc_mb = torch.cuda.memory_allocated() / 1024 / 1024
gpu_reserved_mb = torch.cuda.memory_reserved() / 1024 / 1024
except Exception:
pass
msgs = len(self._overlay._messages) if self._overlay else 0
vad_buf = len(self._vad._speech_buffer)
return {
"rss": rss_mb,
"worker_rss": worker_rss_mb,
"total_rss": rss_mb + worker_rss_mb,
"gpu_alloc": gpu_alloc_mb,
"gpu_reserved": gpu_reserved_mb,
"msgs": msgs,
"vad_buf": vad_buf,
}
def _log_mem_after_asr(self, kind: str, audio_seconds: float, asr_ms: float):
self._mem_asr_call_count += 1
snap = self._mem_snapshot()
delta = snap["rss"] - self._mem_last_mb
total_delta = snap["rss"] - self._mem_baseline_mb
self._mem_last_mb = snap["rss"]
log.info(
f"MEM[asr#{self._mem_asr_call_count}:{kind}] RSS={snap['rss']:.1f}MB "
f"(Δ{delta:+.2f} since last, {total_delta:+.1f} since start) "
f"worker_rss={snap['worker_rss']:.0f}MB "
f"GPU(main alloc/reserved)={snap['gpu_alloc']:.0f}/{snap['gpu_reserved']:.0f}MB "
f"audio={audio_seconds:.1f}s asr={asr_ms:.0f}ms "
f"outputs={self._asr_count} msgs={snap['msgs']} vad_buf={snap['vad_buf']}"
)
self._check_memory_threshold(snap["total_rss"])
def _release_memory_caches(self):
gc.collect()
try:
if torch.cuda.is_available():
torch.cuda.empty_cache()
except Exception:
pass
def _run_asr(self, audio: np.ndarray, kind: str, **kwargs):
audio_seconds = len(audio) / 16000
asr_start = time.perf_counter()
# Snapshot the active client under the lock, then release it: the blocking
# cross-process transcribe must not hold _asr_lock, or a slow/hung worker
# would freeze the Qt thread on every settings change. ASRClient serializes
# its own pipe access, and only this (single) ASR thread calls transcribe.
with self._asr_lock:
if not self._asr_ready or self._asr is None:
return None, 0.0
client = self._asr
asr_type = self._asr_type
funasr_key = self._funasr_model_key
try:
self._apply_pending_asr_settings(client, asr_type, funasr_key)
result = client.transcribe(audio, **kwargs)
except (ASRWorkerExited, ASRWorkerTimeout) as exc:
asr_ms = (time.perf_counter() - asr_start) * 1000
self._log_mem_after_asr(f"{kind}:error", audio_seconds, asr_ms)
self._recover_asr_worker(client, str(exc))
raise
except ASRWorkerError as exc:
asr_ms = (time.perf_counter() - asr_start) * 1000
self._log_mem_after_asr(f"{kind}:error", audio_seconds, asr_ms)
fatal = False
with self._asr_lock:
if self._asr is client:
self._asr_error_count += 1
fatal = not exc.recoverable or self._asr_error_count >= 3
if fatal:
self._mark_asr_unavailable(str(exc), client)
raise
except Exception:
asr_ms = (time.perf_counter() - asr_start) * 1000
self._log_mem_after_asr(f"{kind}:error", audio_seconds, asr_ms)
raise
with self._asr_lock:
if self._asr is client:
self._asr_error_count = 0
self._asr_restart_count = 0
asr_ms = (time.perf_counter() - asr_start) * 1000
self._log_mem_after_asr(kind, audio_seconds, asr_ms)
return result, asr_ms
def _start_worker_from_state(self, state: dict, expected_gen: int) -> bool:
"""Load a worker from a saved state and activate it only if no newer engine
switch happened in the meantime (generation guard). Runs on the ASR thread;
the load is intentionally done outside _asr_lock. Returns True on activation."""
try:
client = self._load_engine_client(state["config"])
except Exception as e:
log.error(f"ASR worker (re)start failed: {e}", exc_info=True)
return False
stale = None
with self._asr_lock:
if self._asr_generation != expected_gen or not self._running:
stale = client
else:
self._asr = client
self._asr_type = state["type"]
self._asr_signature = state["signature"]
self._asr_device = state["device"]
self._asr_config = dict(state["config"]) if state["config"] else None
self._funasr_model_key = state["funasr_model_key"]
self._whisper_model_size = state["whisper_model_size"]
self._asr_ready = True
self._asr_error_count = 0
self._asr_restart_state = dict(state)
self._asr_worker_baseline_mb = None
self._asr_generation += 1
if stale is not None:
log.info("Discarding superseded ASR worker (newer switch won the race)")
try:
stale.shutdown()
except Exception:
pass
return False
name = state.get("display_name") or state.get("type")
if self._overlay:
self._overlay.update_asr_device(
f"{name} [{state.get('device_label', state['device'])}]"
)
return True
def _recover_asr_worker(self, dead_client, reason: str):
"""Auto-restart a worker that died mid-session. Without this, a single crash
or transcribe timeout would leave ASR permanently silent for the session."""
with self._asr_lock:
if self._asr is not dead_client:
return # an engine switch already replaced/cleared it
state = dict(self._asr_restart_state) if self._asr_restart_state else None
attempt = self._asr_restart_count + 1
give_up = (
state is None
or not state.get("config")
or attempt > self._asr_restart_max
)
self._asr_restart_count = attempt
self._asr = None
self._asr_ready = False
self._asr_type = None
self._asr_signature = None
self._asr_config = None
self._asr_error_count = 0
self._asr_worker_baseline_mb = None
self._asr_generation += 1
gen = self._asr_generation
try:
dead_client.shutdown()
except Exception:
try:
dead_client.terminate()
except Exception:
pass
if not self._running:
return # shutting down; do not spawn a replacement worker
if give_up:
log.error(
f"ASR worker died and auto-restart gave up after "
f"{self._asr_restart_max} attempts: {reason}"
)
if self._overlay:
self._overlay.update_asr_device("ASR unavailable")
return
log.warning(
f"ASR worker died ({reason}); auto-restart attempt "
f"{attempt}/{self._asr_restart_max}"
)
self._release_memory_caches()
if self._start_worker_from_state(state, gen):
log.info(
f"ASR worker auto-restarted: {state.get('type')} on "
f"{state.get('device')}"
)
elif self._asr is None and self._overlay:
self._overlay.update_asr_device("ASR unavailable")
def _maybe_recycle_asr_worker(self):
"""Recycle the worker once its RSS grows well past the post-load baseline, to
bound native-side leaks that accumulate in the long-lived worker process.
Called from the ASR thread between segments so the reload gap costs no audio
beyond what arrives during it."""
if not self._running:
return
with self._asr_lock:
client = self._asr
if not self._asr_ready or client is None or self._asr_recycling:
return
state = dict(self._asr_restart_state) if self._asr_restart_state else None
if state is None or not state.get("config") or client.pid is None:
return
try:
import psutil
rss = psutil.Process(client.pid).memory_info().rss / 1024 / 1024
except Exception:
return
if self._asr_worker_baseline_mb is None:
self._asr_worker_baseline_mb = rss
return
if rss < self._asr_worker_baseline_mb + self._asr_recycle_delta_mb:
return