From 055740d7784b86c82c963b428cbccaae3f2c2981 Mon Sep 17 00:00:00 2001
From: "google-labs-jules[bot]"
 <161369871+google-labs-jules[bot]@users.noreply.github.com>
Date: Sat, 22 Aug 2026 02:21:16 +0000
Subject: [PATCH] Cache ML models in memory to prevent reload bottleneck

Co-authored-by: benpiper <4343814+benpiper@users.noreply.github.com>
---
 .Jules/bolt.md        |  3 +++
 background_remover.py |  2 ++
 processor.py          | 10 ++++++++--
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/.Jules/bolt.md b/.Jules/bolt.md
index 14295c7..325d797 100644
--- a/.Jules/bolt.md
+++ b/.Jules/bolt.md
@@ -1,3 +1,6 @@
 ## 2024-05-18 - Replacing moviepy with ffprobe/ffmpeg subprocess
 **Learning:** Instantiating `VideoFileClip` from `moviepy.editor` introduces significant performance overhead when used solely to check if a video has an audio stream, or to read the duration and size of the video, as it initializes the full subclip and PyGame dependencies. By using raw `subprocess` calls to `ffprobe` (reading from `-of json`) and `ffmpeg`, we can gather properties and perform extractions in <100ms compared to `moviepy`'s >300ms overhead.
 **Action:** When extracting audio, or querying video metadata like `duration` and `size`, favor `ffprobe -show_entries` rather than instantiating `moviepy` objects if `moviepy` is not otherwise needed. Be sure to handle cleanup of original code using `.close()` if replacing those initializations.
+## 2024-05-19 - Caching ML Models in memory
+**Learning:** Large ML models (like Whisper or RVM) can create a severe performance bottleneck or Out-Of-Memory (OOM) issues if reloaded from disk on every function call. This is particularly problematic in a web service or batch processing pipeline.
+**Action:** Use `@functools.lru_cache(maxsize=1)` on model loading functions to cache the model in memory globally. This guarantees the model is loaded exactly once, significantly reducing latency and memory thrashing.
diff --git a/background_remover.py b/background_remover.py
index a55c856..b8e42f5 100644
--- a/background_remover.py
+++ b/background_remover.py
@@ -3,6 +3,7 @@
 import urllib.request
 from pathlib import Path
 from typing import Optional, Tuple
+import functools
 import torch
 import cv2
 import numpy as np
@@ -64,6 +65,7 @@ def reporthook(count, block_size, total_size):
     return model_path
 
 
+@functools.lru_cache(maxsize=1)
 def load_rvm_model(model_name: str = 'mobilenetv3', device: str = 'auto') -> torch.nn.Module:
     """
     Load RVM model.
diff --git a/processor.py b/processor.py
index 1ff5c1b..c0347c2 100644
--- a/processor.py
+++ b/processor.py
@@ -2,6 +2,7 @@
 import logging
 from typing import List, Tuple, Optional, Callable
 
+import functools
 import whisper
 from pydub import AudioSegment, silence
 
@@ -82,14 +83,19 @@ def detect_filler_words(audio_path: str, model_size: str = "large-v3-turbo", fil
     """
     return detect_filler_words_whisper(audio_path, model_size, filler_words_list)
 
+@functools.lru_cache(maxsize=1)
+def get_whisper_model(model_size: str):
+    """Cached loader for Whisper model to prevent repeated disk loading."""
+    logging.info(f"Loading Whisper model ({model_size})...")
+    return whisper.load_model(model_size)
+
 def detect_filler_words_whisper(audio_path: str, model_size: str = "large-v3-turbo", filler_words_list: List[str] = None) -> List[Tuple[float, float]]:
     """
     Detects filler words using standard Whisper.
     Returns:
         Tuple of (List of (start, end) tuples, transcript string).
     """
-    logging.info(f"Loading Whisper model ({model_size})...")
-    model = whisper.load_model(model_size)
+    model = get_whisper_model(model_size)
     logging.info("Transcribing audio for filler word detection...")
     # We use a prompt to encourage transcribing filler words if possible, though Whisper is trained to remove them.
     # Sometimes standard transcription removes them. We can try to rely on word-level timestamps.