From 54919dbb6c46f8db7faf64257f2272405cb335f0 Mon Sep 17 00:00:00 2001 From: Ryder Casazza Date: Tue, 21 Jul 2026 14:21:34 -0700 Subject: [PATCH 1/2] fix: stop evicting title caches per generic-search page retain_titles pruned the Sonarr/Radarr title caches down to the IDs seen in a single RSS-sync page, wiping entries cached by other pages and by tv/movie searches. That caused repeated lookups against Sonarr/Radarr and a cache-file rewrite on every generic search. The caches only hold small title strings, so they are now left to grow with the set of titles actually searched. --- src/radarr.rs | 81 +------------------------------------------------- src/service.rs | 36 ++-------------------- src/sonarr.rs | 81 +------------------------------------------------- 3 files changed, 5 insertions(+), 193 deletions(-) diff --git a/src/radarr.rs b/src/radarr.rs index 9273433..46eab28 100644 --- a/src/radarr.rs +++ b/src/radarr.rs @@ -1,5 +1,5 @@ use std::{ - collections::{HashMap, HashSet}, + collections::HashMap, io::ErrorKind, path::{Path, PathBuf}, sync::Arc, @@ -114,29 +114,6 @@ impl RadarrClient { Ok(movie) } - pub async fn retain_titles(&self, keep: &HashSet) -> Result<(), RadarrError> { - if keep.is_empty() { - let mut guard = self.cache.write().await; - if guard.is_empty() { - return Ok(()); - } - guard.clear(); - drop(guard); - return self.persist_cache().await; - } - - let mut guard = self.cache.write().await; - let original_len = guard.len(); - guard.retain(|tmdb_id, _| keep.contains(tmdb_id)); - - if guard.len() == original_len { - return Ok(()); - } - - drop(guard); - self.persist_cache().await - } - async fn cached_movie(&self, tmdb_id: i64) -> Option { let guard = self.cache.read().await; guard.get(&tmdb_id).cloned() @@ -349,60 +326,4 @@ mod tests { assert_eq!(entry.year, 2001); } - #[tokio::test] - async fn retain_titles_clears_when_keep_empty() { - let dir = TempDir::new().unwrap(); - let client = make_client(&dir); - - client.store_movie(1, &movie("A", 2001)).await.unwrap(); - client.store_movie(2, &movie("B", 2002)).await.unwrap(); - - client.retain_titles(&HashSet::new()).await.unwrap(); - - assert!(client.cached_movie(1).await.is_none()); - assert!(client.cached_movie(2).await.is_none()); - assert!( - load_cache(&dir.path().join(CACHE_FILENAME)) - .unwrap() - .is_empty() - ); - } - - #[tokio::test] - async fn retain_titles_drops_unlisted_entries() { - let dir = TempDir::new().unwrap(); - let client = make_client(&dir); - - client.store_movie(1, &movie("A", 2001)).await.unwrap(); - client.store_movie(2, &movie("B", 2002)).await.unwrap(); - client.store_movie(3, &movie("C", 2003)).await.unwrap(); - - let keep: HashSet = [1, 3].into_iter().collect(); - client.retain_titles(&keep).await.unwrap(); - - assert!(client.cached_movie(1).await.is_some()); - assert!(client.cached_movie(2).await.is_none()); - assert!(client.cached_movie(3).await.is_some()); - } - - #[tokio::test] - async fn retain_titles_skips_persist_when_unchanged() { - // Superset keep on populated cache: nothing removed, no rewrite expected. - let dir = TempDir::new().unwrap(); - let client = make_client(&dir); - client.store_movie(1, &movie("A", 2001)).await.unwrap(); - - let cache_path = dir.path().join(CACHE_FILENAME); - std::fs::remove_file(&cache_path).unwrap(); - - let keep: HashSet = [1, 2].into_iter().collect(); - client.retain_titles(&keep).await.unwrap(); - assert!(!cache_path.exists()); - - // Empty keep on empty cache: same short-circuit. - let dir = TempDir::new().unwrap(); - let client = make_client(&dir); - client.retain_titles(&HashSet::new()).await.unwrap(); - assert!(!dir.path().join(CACHE_FILENAME).exists()); - } } diff --git a/src/service.rs b/src/service.rs index 6536913..8010876 100644 --- a/src/service.rs +++ b/src/service.rs @@ -1,4 +1,4 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use crate::anilist::{AniListClient, AniListError, MediaFormat}; use crate::config::AppConfig; @@ -384,8 +384,6 @@ impl SearchService { let mut tv_title_cache: HashMap<(i64, u32), String> = HashMap::new(); let mut movie_title_cache: HashMap = HashMap::new(); - let mut active_tvdb_ids: HashSet = HashSet::new(); - let mut active_tmdb_ids: HashSet = HashSet::new(); let mut items = Vec::with_capacity(window.len()); let mut grouped_torrents: HashMap<(String, Vec), Vec> = HashMap::new(); @@ -402,11 +400,7 @@ impl SearchService { match &media.format { format if self.format_allowed(format) && self.sonarr.is_some() => { let title = match self - .resolve_tv_generic_title( - &torrent, - &mut tv_title_cache, - &mut active_tvdb_ids, - ) + .resolve_tv_generic_title(&torrent, &mut tv_title_cache) .await { Ok(Some(title)) => title, @@ -430,11 +424,7 @@ impl SearchService { } MediaFormat::Movie if self.radarr.is_some() => { match self - .resolve_movie_generic_title( - anilist_id, - &mut movie_title_cache, - &mut active_tmdb_ids, - ) + .resolve_movie_generic_title(anilist_id, &mut movie_title_cache) .await { Ok(Some(title)) => { @@ -463,20 +453,6 @@ impl SearchService { items.extend(self.process_torrents(torrents, title, categories)); } - if let Some(sonarr) = &self.sonarr { - sonarr - .retain_titles(&active_tvdb_ids) - .await - .map_err(ServiceError::Sonarr)?; - } - - if let Some(radarr) = &self.radarr { - radarr - .retain_titles(&active_tmdb_ids) - .await - .map_err(ServiceError::Radarr)?; - } - Ok((items, total)) } @@ -484,7 +460,6 @@ impl SearchService { &self, torrent: &Torrent, cache: &mut HashMap<(i64, u32), String>, - active_tvdb_ids: &mut HashSet, ) -> Result, ServiceError> { let Some(anilist_id) = torrent.anilist_id else { return Ok(None); @@ -501,8 +476,6 @@ impl SearchService { } if let Some((tvdb_id, season)) = self.select_tvdb_and_season(&mappings) { - active_tvdb_ids.insert(tvdb_id); - if let Some(existing) = cache.get(&(tvdb_id, season)) { return Ok(Some(existing.clone())); } @@ -519,7 +492,6 @@ impl SearchService { &self, anilist_id: i64, cache: &mut HashMap, - active_tmdb_ids: &mut HashSet, ) -> Result, ServiceError> { let Some(tmdb_id) = self .mappings @@ -531,7 +503,6 @@ impl SearchService { }; if let Some(existing) = cache.get(&tmdb_id) { - active_tmdb_ids.insert(tmdb_id); return Ok(Some(existing.clone())); } @@ -548,7 +519,6 @@ impl SearchService { let formatted = self.format_movie_feed_title(&movie.title, movie.year); cache.insert(tmdb_id, formatted.clone()); - active_tmdb_ids.insert(tmdb_id); Ok(Some(formatted)) } diff --git a/src/sonarr.rs b/src/sonarr.rs index 84fc31e..c0a6e65 100644 --- a/src/sonarr.rs +++ b/src/sonarr.rs @@ -1,5 +1,5 @@ use std::{ - collections::{HashMap, HashSet}, + collections::HashMap, io::ErrorKind, path::{Path, PathBuf}, sync::Arc, @@ -101,29 +101,6 @@ impl SonarrClient { Ok(title) } - pub async fn retain_titles(&self, keep: &HashSet) -> Result<(), SonarrError> { - if keep.is_empty() { - let mut guard = self.cache.write().await; - if guard.is_empty() { - return Ok(()); - } - guard.clear(); - drop(guard); - return self.persist_cache().await; - } - - let mut guard = self.cache.write().await; - let original_len = guard.len(); - guard.retain(|tvdb_id, _| keep.contains(tvdb_id)); - - if guard.len() == original_len { - return Ok(()); - } - - drop(guard); - self.persist_cache().await - } - async fn cached_title(&self, tvdb_id: i64) -> Option { let guard = self.cache.read().await; guard.get(&tvdb_id).cloned() @@ -315,60 +292,4 @@ mod tests { assert_eq!(reloaded.get(&42), Some(&"Naruto".to_string())); } - #[tokio::test] - async fn retain_titles_clears_when_keep_empty() { - let dir = TempDir::new().unwrap(); - let client = make_client(&dir); - - client.store_title(1, "A").await.unwrap(); - client.store_title(2, "B").await.unwrap(); - - client.retain_titles(&HashSet::new()).await.unwrap(); - - assert_eq!(client.cached_title(1).await, None); - assert_eq!(client.cached_title(2).await, None); - assert!( - load_cache(&dir.path().join(CACHE_FILENAME)) - .unwrap() - .is_empty() - ); - } - - #[tokio::test] - async fn retain_titles_drops_unlisted_entries() { - let dir = TempDir::new().unwrap(); - let client = make_client(&dir); - - client.store_title(1, "A").await.unwrap(); - client.store_title(2, "B").await.unwrap(); - client.store_title(3, "C").await.unwrap(); - - let keep: HashSet = [1, 3].into_iter().collect(); - client.retain_titles(&keep).await.unwrap(); - - assert_eq!(client.cached_title(1).await, Some("A".to_string())); - assert_eq!(client.cached_title(2).await, None); - assert_eq!(client.cached_title(3).await, Some("C".to_string())); - } - - #[tokio::test] - async fn retain_titles_skips_persist_when_unchanged() { - // Superset keep on populated cache: nothing removed, no rewrite expected. - let dir = TempDir::new().unwrap(); - let client = make_client(&dir); - client.store_title(1, "A").await.unwrap(); - - let cache_path = dir.path().join(CACHE_FILENAME); - std::fs::remove_file(&cache_path).unwrap(); - - let keep: HashSet = [1, 2].into_iter().collect(); - client.retain_titles(&keep).await.unwrap(); - assert!(!cache_path.exists()); - - // Empty keep on empty cache: same short-circuit. - let dir = TempDir::new().unwrap(); - let client = make_client(&dir); - client.retain_titles(&HashSet::new()).await.unwrap(); - assert!(!dir.path().join(CACHE_FILENAME).exists()); - } } From 10de63843615037dbbd768489b77e27b694e753e Mon Sep 17 00:00:00 2001 From: Ryder Casazza Date: Tue, 21 Jul 2026 14:23:47 -0700 Subject: [PATCH 2/2] perf: resolve generic-search titles concurrently The generic search resolved feed titles one torrent at a time, so a cold cache meant up to a page of serial Sonarr/Radarr round trips. Torrents are now mapped to unique (tvdb, season)/tmdb lookup keys first, resolved concurrently (capped at 8 in-flight lookups), then grouped. Per-request title memoization falls out of the dedup, so the ad-hoc HashMap caches are gone. --- src/radarr.rs | 1 - src/service.rs | 239 +++++++++++++++++++++++++++++-------------------- src/sonarr.rs | 1 - 3 files changed, 142 insertions(+), 99 deletions(-) diff --git a/src/radarr.rs b/src/radarr.rs index 46eab28..17e0644 100644 --- a/src/radarr.rs +++ b/src/radarr.rs @@ -325,5 +325,4 @@ mod tests { assert_eq!(entry.title, "Spirited Away"); assert_eq!(entry.year, 2001); } - } diff --git a/src/service.rs b/src/service.rs index 8010876..d352734 100644 --- a/src/service.rs +++ b/src/service.rs @@ -1,4 +1,5 @@ -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; use crate::anilist::{AniListClient, AniListError, MediaFormat}; use crate::config::AppConfig; @@ -8,11 +9,35 @@ use crate::releases::{ReleasesClient, ReleasesError, Torrent, Tracker}; use crate::sonarr::{SonarrClient, SonarrError}; use crate::torznab::{self, ANIME_CATEGORY, MOVIE_CATEGORY, TorznabItem}; use thiserror::Error; +use tokio::sync::Semaphore; +use tokio::task::JoinSet; use tracing::{debug, info, trace, warn}; /// Upper bound on items returned in a single torznab response. const MAX_RESPONSE_ITEMS: usize = 100; +/// Cap on simultaneous Sonarr/Radarr title lookups during a generic search. +const MAX_CONCURRENT_TITLE_LOOKUPS: usize = 8; + +/// The upstream lookup a torrent's feed title comes from during a generic search. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +enum TitleKey { + Tv { tvdb_id: i64, season: u32 }, + Movie { tmdb_id: i64 }, +} + +fn format_tv_feed_title(series_title: &str, season: u32) -> String { + format!("{series_title} S{season:02} Bluray 1080p remux") +} + +fn format_movie_feed_title(title: &str, year: u32) -> String { + if year == 0 { + format!("{title} Bluray 1080p remux") + } else { + format!("{title} ({year}) Bluray 1080p remux") + } +} + /// Seeder value to use for a torrent based on its tracker and preference. /// A value needs to be 10x greater than the next lower value for Sonarr/Radarr to prioritize it. fn priority_seeders(tracker: Tracker, preferred: bool) -> u32 { @@ -253,7 +278,7 @@ impl SearchService { let total = torrents.len(); let feed_title = match self.radarr.as_ref().unwrap().resolve_name(tmdb_id).await { - Ok(movie) => self.format_movie_feed_title(&movie.title, movie.year), + Ok(movie) => format_movie_feed_title(&movie.title, movie.year), Err(RadarrError::NotFound { .. } | RadarrError::Api { .. }) => { debug!( tmdb_id, @@ -382,11 +407,8 @@ impl SearchService { return Ok((Vec::new(), total)); } - let mut tv_title_cache: HashMap<(i64, u32), String> = HashMap::new(); - let mut movie_title_cache: HashMap = HashMap::new(); - let mut items = Vec::with_capacity(window.len()); - - let mut grouped_torrents: HashMap<(String, Vec), Vec> = HashMap::new(); + // Resolve each torrent to its title lookup key via the in-memory mapping index. + let mut targets: Vec<(Torrent, TitleKey)> = Vec::with_capacity(window.len()); for torrent in window.into_iter() { let Some(anilist_id) = torrent.anilist_id else { @@ -399,16 +421,10 @@ impl SearchService { match &media.format { format if self.format_allowed(format) && self.sonarr.is_some() => { - let title = match self - .resolve_tv_generic_title(&torrent, &mut tv_title_cache) - .await - { - Ok(Some(title)) => title, - Ok(None) => { - debug!(torrent_id = %torrent.id, "no tv title resolved for generic search; skipping"); - continue; - } + let mappings = match self.mappings.resolve_tvdb_mappings(anilist_id).await { + Ok(mappings) => mappings, Err(error) => { + let error = ServiceError::Mapping(error); warn!( torrent_id = %torrent.id, %error, @@ -417,26 +433,22 @@ impl SearchService { continue; } }; - grouped_torrents - .entry((title, self.tv_category_ids())) - .or_default() - .push(torrent); + let Some((tvdb_id, season)) = self.select_tvdb_and_season(&mappings) else { + debug!(torrent_id = %torrent.id, "no tv title resolved for generic search; skipping"); + continue; + }; + targets.push((torrent, TitleKey::Tv { tvdb_id, season })); } MediaFormat::Movie if self.radarr.is_some() => { - match self - .resolve_movie_generic_title(anilist_id, &mut movie_title_cache) - .await - { - Ok(Some(title)) => { - grouped_torrents - .entry((title, self.movie_category_ids())) - .or_default() - .push(torrent); + match self.mappings.resolve_tmdb_id(anilist_id).await { + Ok(Some(tmdb_id)) => { + targets.push((torrent, TitleKey::Movie { tmdb_id })); } Ok(None) => { debug!(torrent_id = %torrent.id, "no movie title resolved for generic search; skipping"); } Err(error) => { + let error = ServiceError::Mapping(error); warn!( torrent_id = %torrent.id, %error, @@ -449,6 +461,27 @@ impl SearchService { } } + // Resolve each unique key against Sonarr/Radarr concurrently. + let titles = self.resolve_titles_concurrently(&targets).await; + + // Group torrents under their resolved feed titles. + let mut items = Vec::with_capacity(targets.len()); + let mut grouped_torrents: HashMap<(String, Vec), Vec> = HashMap::new(); + + for (torrent, key) in targets { + let Some(title) = titles.get(&key) else { + continue; + }; + let categories = match key { + TitleKey::Tv { .. } => self.tv_category_ids(), + TitleKey::Movie { .. } => self.movie_category_ids(), + }; + grouped_torrents + .entry((title.clone(), categories)) + .or_default() + .push(torrent); + } + for ((title, categories), torrents) in grouped_torrents { items.extend(self.process_torrents(torrents, title, categories)); } @@ -456,70 +489,90 @@ impl SearchService { Ok((items, total)) } - pub async fn resolve_tv_generic_title( + /// Resolves the unique title keys in `targets` against Sonarr/Radarr, a + /// bounded number at a time. Failed lookups are logged and omitted. + async fn resolve_titles_concurrently( &self, - torrent: &Torrent, - cache: &mut HashMap<(i64, u32), String>, - ) -> Result, ServiceError> { - let Some(anilist_id) = torrent.anilist_id else { - return Ok(None); - }; - - let mappings = self - .mappings - .resolve_tvdb_mappings(anilist_id) - .await - .map_err(ServiceError::Mapping)?; - - if mappings.is_empty() { - return Ok(None); - } - - if let Some((tvdb_id, season)) = self.select_tvdb_and_season(&mappings) { - if let Some(existing) = cache.get(&(tvdb_id, season)) { - return Ok(Some(existing.clone())); + targets: &[(Torrent, TitleKey)], + ) -> HashMap { + let unique_keys: HashSet = targets.iter().map(|(_, key)| *key).collect(); + let semaphore = Arc::new(Semaphore::new(MAX_CONCURRENT_TITLE_LOOKUPS)); + let mut lookups: JoinSet<(TitleKey, Option)> = JoinSet::new(); + + for key in unique_keys { + let semaphore = semaphore.clone(); + match key { + TitleKey::Tv { tvdb_id, season } => { + let sonarr = self + .sonarr + .as_ref() + .expect("tv title lookups require Sonarr to be enabled") + .clone(); + lookups.spawn(async move { + let _permit = semaphore + .acquire_owned() + .await + .expect("title lookup semaphore is never closed"); + let title = match sonarr.resolve_name(tvdb_id).await { + Ok(series_title) => Some(format_tv_feed_title(&series_title, season)), + Err(error) => { + warn!( + tvdb_id, + %error, + "failed to resolve tv title for generic search; skipping" + ); + None + } + }; + (key, title) + }); + } + TitleKey::Movie { tmdb_id } => { + let radarr = self + .radarr + .as_ref() + .expect("movie title lookups require Radarr to be enabled") + .clone(); + lookups.spawn(async move { + let _permit = semaphore + .acquire_owned() + .await + .expect("title lookup semaphore is never closed"); + let title = match radarr.resolve_name(tmdb_id).await { + Ok(movie) => Some(format_movie_feed_title(&movie.title, movie.year)), + Err(RadarrError::NotFound { .. } | RadarrError::Api { .. }) => { + debug!( + tmdb_id, + "no movie title resolved for generic search; skipping" + ); + None + } + Err(error) => { + warn!( + tmdb_id, + %error, + "failed to resolve movie title for generic search; skipping" + ); + None + } + }; + (key, title) + }); + } } - - let title = self.resolve_feed_title(tvdb_id, season).await?; - cache.insert((tvdb_id, season), title.clone()); - return Ok(Some(title)); } - Ok(None) - } - - pub async fn resolve_movie_generic_title( - &self, - anilist_id: i64, - cache: &mut HashMap, - ) -> Result, ServiceError> { - let Some(tmdb_id) = self - .mappings - .resolve_tmdb_id(anilist_id) - .await - .map_err(ServiceError::Mapping)? - else { - return Ok(None); - }; - - if let Some(existing) = cache.get(&tmdb_id) { - return Ok(Some(existing.clone())); + let mut titles = HashMap::new(); + while let Some(joined) = lookups.join_next().await { + match joined { + Ok((key, Some(title))) => { + titles.insert(key, title); + } + Ok((_, None)) => {} + Err(error) => warn!(%error, "title lookup task failed"), + } } - - let radarr = self - .radarr - .as_ref() - .expect("resolve_movie_generic_title requires Radarr to be enabled"); - - let movie = match radarr.resolve_name(tmdb_id).await { - Ok(movie) => movie, - Err(RadarrError::NotFound { .. } | RadarrError::Api { .. }) => return Ok(None), - Err(err) => return Err(ServiceError::Radarr(err)), - }; - - let formatted = self.format_movie_feed_title(&movie.title, movie.year); - cache.insert(tmdb_id, formatted.clone()); - Ok(Some(formatted)) + titles } pub async fn resolve_feed_title( @@ -537,15 +590,7 @@ impl SearchService { .await .map_err(ServiceError::Sonarr)?; trace!(tvdb_id, %series_title, "resolved series title from sonarr"); - Ok(format!("{series_title} S{season:02} Bluray 1080p remux")) - } - - pub fn format_movie_feed_title(&self, title: &str, year: u32) -> String { - if year == 0 { - format!("{title} Bluray 1080p remux") - } else { - format!("{title} ({year}) Bluray 1080p remux") - } + Ok(format_tv_feed_title(&series_title, season)) } fn format_allowed(&self, format: &MediaFormat) -> bool { diff --git a/src/sonarr.rs b/src/sonarr.rs index c0a6e65..a18e1e0 100644 --- a/src/sonarr.rs +++ b/src/sonarr.rs @@ -291,5 +291,4 @@ mod tests { let reloaded = load_cache(&dir.path().join(CACHE_FILENAME)).unwrap(); assert_eq!(reloaded.get(&42), Some(&"Naruto".to_string())); } - }