From e2d0e2d8d5d6f15d64126769f49fbfad3a3f1228 Mon Sep 17 00:00:00 2001 From: Harshith Mohan <26010946+harshithmohan@users.noreply.github.com> Date: Tue, 1 Sep 2026 19:58:20 +0530 Subject: [PATCH 1/4] fix(anidb): refresh group status during periodic anime updates `AniDB_GroupStatus` was only refreshed by the legacy v1 API, so with a v3-only install the cached release-group progress stopped updating and "released-but-absent" missing-episode counts under-counted. - Enqueue `GetAniDBReleaseGroupStatusJob` for each updated series in `GetUpdatedAniDBAnimeJob`, propagating `ForceRefresh` so scheduled runs keep the `ShouldSkip()` ended-anime gate intact --- .../Scheduling/Jobs/AniDB/GetUpdatedAniDBAnimeJob.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Shoko.Server/Scheduling/Jobs/AniDB/GetUpdatedAniDBAnimeJob.cs b/Shoko.Server/Scheduling/Jobs/AniDB/GetUpdatedAniDBAnimeJob.cs index 65ec62e78..beccd07d6 100644 --- a/Shoko.Server/Scheduling/Jobs/AniDB/GetUpdatedAniDBAnimeJob.cs +++ b/Shoko.Server/Scheduling/Jobs/AniDB/GetUpdatedAniDBAnimeJob.cs @@ -3,9 +3,11 @@ using Microsoft.Extensions.Logging; using Shoko.Abstractions.Metadata.Anidb.Enums; using Shoko.Abstractions.Metadata.Anidb.Services; +using Shoko.QueueProcessor.Abstractions; using Shoko.QueueProcessor.Acquisition.Attributes; using Shoko.QueueProcessor.Builder; using Shoko.QueueProcessor.Concurrency; +using Shoko.QueueProcessor.Scheduling; using Shoko.Server.Models.Internal; using Shoko.Server.Providers.AniDB.Interfaces; using Shoko.Server.Providers.AniDB.Titles; @@ -25,7 +27,7 @@ namespace Shoko.Server.Scheduling.Jobs.AniDB; [AniDBUdpRateLimited] [DisallowConcurrencyGroup(ConcurrencyGroups.AniDB_UDP)] [JobKeyGroup(JobKeyGroup.AniDB)] -public class GetUpdatedAniDBAnimeJob(IRequestFactory requestFactory, IAnidbService anidbService, ISettingsProvider settingsProvider, AniDBTitleHelper titleHelper, AniDB_AnimeRepository anidbAnimeRepository, AniDB_AnimeUpdateRepository anidbAnimeUpdates, AnimeSeriesRepository animeSeries, ScheduledUpdateRepository scheduledUpdates) : BaseJob +public class GetUpdatedAniDBAnimeJob(IRequestFactory requestFactory, IAnidbService anidbService, ISettingsProvider settingsProvider, AniDBTitleHelper titleHelper, AniDB_AnimeRepository anidbAnimeRepository, AniDB_AnimeUpdateRepository anidbAnimeUpdates, AnimeSeriesRepository animeSeries, ScheduledUpdateRepository scheduledUpdates, IQueueScheduler scheduler) : BaseJob { public bool ForceRefresh { get; set; } @@ -144,6 +146,9 @@ public override async Task Execute() await anidbService.ScheduleRefreshOfAnimeByID(animeID, AnidbRefreshMethod.Remote | AnidbRefreshMethod.DeferToRemoteIfUnsuccessful).ConfigureAwait(false); countSeries++; + + await scheduler.StartJob( + c => (c.AnimeID, c.ForceRefresh) = (animeID, ForceRefresh)); } return (response, countAnime, countSeries); From 6ec2e3c23c56790e46b4076095702e5ebd0c4f3d Mon Sep 17 00:00:00 2001 From: Harshith Mohan <26010946+harshithmohan@users.noreply.github.com> Date: Tue, 1 Sep 2026 20:36:58 +0530 Subject: [PATCH 2/4] fix(api): unify missing episode definition behind shared predicate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `/v3/Series` → `Sizes.Missing.Episodes` used an aired-but-absent check while `/v3/MissingEpisodes` and the persisted counts used a released-but-absent check via `AniDB_GroupStatus`, so the same series could report different missing counts. - Added `AnimeEpisodeExtensions.IsMissingEpisode` as the single source of truth: aired, not hidden, no local file, and either no group status data or a group with `CompletionState` complete/finished or `LastEpisodeNumber` past the episode number - Applied the group status gate to `Sizes.Missing.Episodes` in both `GenerateSeriesSizes` and `GenerateGroupSizes` so series and group counts agree; `GenerateSeriesSizes` now requires the per-anime status dictionary so future callers cannot fall back to the divergent aired-but-absent logic - Replaced the non-collecting `GetMissing` raw SQL with cache-based filtering through the shared predicate; the collecting variant stays as raw SQL - Added `AniDB_GroupStatusRepository.GetByAnimeIDs` with batched (1000-per-query) lookups --- Shoko.Server/API/v3/Helpers/ModelHelper.cs | 14 ++- Shoko.Server/API/v3/Models/Shoko/Series.cs | 3 +- .../Extensions/AnimeEpisodeExtensions.cs | 36 +++++++ .../Cached/AnimeEpisodeRepository.cs | 100 +++++++----------- .../Direct/AniDB_GroupStatusRepository.cs | 25 +++++ 5 files changed, 112 insertions(+), 66 deletions(-) create mode 100644 Shoko.Server/Extensions/AnimeEpisodeExtensions.cs diff --git a/Shoko.Server/API/v3/Helpers/ModelHelper.cs b/Shoko.Server/API/v3/Helpers/ModelHelper.cs index 4722f806b..6067138ac 100644 --- a/Shoko.Server/API/v3/Helpers/ModelHelper.cs +++ b/Shoko.Server/API/v3/Helpers/ModelHelper.cs @@ -5,6 +5,8 @@ using Shoko.Abstractions.Metadata.Enums; using Shoko.Abstractions.Video.Enums; using Shoko.Server.API.v3.Models.Common; +using Shoko.Server.Extensions; +using Shoko.Server.Models.AniDB; using Shoko.Server.Models.CrossReference; using Shoko.Server.Models.Shoko; using Shoko.Server.Repositories; @@ -232,7 +234,7 @@ public static int GetTotalEpisodesForType(IEnumerable episodeList, .Count(anidbEpisode => anidbEpisode != null && anidbEpisode.EpisodeType == episodeType); } - public static SeriesSizes GenerateSeriesSizes(IEnumerable episodeList, int userID) + public static SeriesSizes GenerateSeriesSizes(IEnumerable episodeList, int userID, IReadOnlyDictionary> groupStatusesByAnime) { var sizes = new SeriesSizes(); var fileSet = new HashSet(); @@ -325,7 +327,7 @@ public static SeriesSizes GenerateSeriesSizes(IEnumerable episodeL sizes.Watched.Episodes++; } } - else if (anidbEpisode.HasAired) + else if (episode.IsMissingEpisode(groupStatusesByAnime.GetValueOrDefault(anidbEpisode.AnimeID) ?? [])) { sizes.Missing.Episodes++; } @@ -409,7 +411,13 @@ public static SeriesSizes GenerateSeriesSizes(IEnumerable episodeL public static GroupSizes GenerateGroupSizes(IEnumerable seriesList, IEnumerable episodeList, int subGroups, int userID) { - var sizes = new GroupSizes(GenerateSeriesSizes(episodeList, userID)); + var animeIDs = seriesList + .Select(ser => ser.AniDB_ID) + .Where(animeID => animeID > 0) + .Distinct() + .ToList(); + var groupStatusesByAnime = RepoFactory.AniDB_GroupStatus.GetByAnimeIDs(animeIDs); + var sizes = new GroupSizes(GenerateSeriesSizes(episodeList, userID, groupStatusesByAnime)); foreach (var series in seriesList) { var anime = series.AniDB_Anime; diff --git a/Shoko.Server/API/v3/Models/Shoko/Series.cs b/Shoko.Server/API/v3/Models/Shoko/Series.cs index acb798ea6..018cd215a 100644 --- a/Shoko.Server/API/v3/Models/Shoko/Series.cs +++ b/Shoko.Server/API/v3/Models/Shoko/Series.cs @@ -135,7 +135,8 @@ public Series(AnimeSeries ser, int userId = 0, bool randomizeImages = false, Has var userData = RepoFactory.AnimeSeries_User.GetByUserAndSeriesID(userId, ser.AnimeSeriesID); var tmdbMovieXRefs = ser.TmdbMovieCrossReferences; var tmdbShowXRefs = ser.TmdbShowCrossReferences; - var sizes = ModelHelper.GenerateSeriesSizes(allEpisodes, userId); + var groupStatusesByAnime = new Dictionary> { [ser.AniDB_ID] = RepoFactory.AniDB_GroupStatus.GetByAnimeID(ser.AniDB_ID) }; + var sizes = ModelHelper.GenerateSeriesSizes(allEpisodes, userId, groupStatusesByAnime); IDs = new() { ID = ser.AnimeSeriesID, diff --git a/Shoko.Server/Extensions/AnimeEpisodeExtensions.cs b/Shoko.Server/Extensions/AnimeEpisodeExtensions.cs new file mode 100644 index 000000000..4405f7f98 --- /dev/null +++ b/Shoko.Server/Extensions/AnimeEpisodeExtensions.cs @@ -0,0 +1,36 @@ +using System.Collections.Generic; +using System.Linq; +using Shoko.Abstractions.Metadata.Enums; +using Shoko.Server.Models.AniDB; +using Shoko.Server.Models.Shoko; + +namespace Shoko.Server.Extensions; + +/// +/// Extension methods for . +/// +public static class AnimeEpisodeExtensions +{ + /// + /// Determines whether an aired episode with no local files should be counted as missing, + /// based on the cached AniDB group release statuses for its anime. + /// + /// The episode to evaluate. + /// Group statuses already scoped to the episode's anime. An empty list is treated as missing. + /// if the episode is considered missing; otherwise, . + /// + /// This predicate is intended for episodes only; callers must + /// pre-filter by episode type AND file presence before calling it. + /// + public static bool IsMissingEpisode(this AnimeEpisode episode, IReadOnlyList groupStatuses) + { + if (episode.IsHidden) return false; + var anidb = episode.AniDB_Episode; + if (anidb == null) return false; + if (!anidb.HasAired) return false; + + return groupStatuses.Count == 0 || groupStatuses.Any(gs => + gs.CompletionState is (int)GroupCompletionStatus.Complete or (int)GroupCompletionStatus.Finished + || gs.LastEpisodeNumber >= anidb.EpisodeNumber); + } +} diff --git a/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs b/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs index e94b59719..3804a5007 100644 --- a/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs +++ b/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs @@ -6,6 +6,7 @@ using NutzCode.InMemoryIndex; using Shoko.Abstractions.Extensions; using Shoko.Server.Databases; +using Shoko.Server.Extensions; using Shoko.Server.Models.Shoko; using EpisodeType = Shoko.Abstractions.Metadata.Enums.EpisodeType; @@ -215,48 +216,6 @@ public IEnumerable GetWithDuplicateFiles(int? animeID = null) .Select(tuple => tuple.episode!); } - // Group_CompletionStatus.Complete = 3, Group_CompletionStatus.Finished = 5 - // EpisodeType.Episode = 1 - // AirDate = 0 means unknown; kept as a candidate here and resolved exactly via HasAired in GetMissing(). - // LastEpisodeNumber >= EpisodeNumber approximates HasGroupReleasedEpisode. - // GS.GroupID (int) = SRI.GroupID (varchar) relies on implicit int↔string coercion present in SQLite, MySQL, and SQL Server. - private const string MissingEpisodesQuery = @" -SELECT AE.AnimeEpisodeID -FROM AnimeEpisode AE -INNER JOIN AniDB_Episode ADBE ON AE.AniDB_EpisodeID = ADBE.EpisodeID -WHERE AE.IsHidden = 0 - AND ADBE.EpisodeType = 1 - AND (ADBE.AirDate = 0 OR ADBE.AirDate < :currentTime) - AND NOT EXISTS (SELECT 1 FROM CrossRef_File_Episode CFE WHERE CFE.EpisodeID = ADBE.EpisodeID) - AND ( - NOT EXISTS (SELECT 1 FROM AniDB_GroupStatus GS WHERE GS.AnimeID = ADBE.AnimeID) - OR EXISTS ( - SELECT 1 FROM AniDB_GroupStatus GS - WHERE GS.AnimeID = ADBE.AnimeID - AND (GS.CompletionState IN (3, 5) OR GS.LastEpisodeNumber >= ADBE.EpisodeNumber) - ) - ) -"; - - private const string MissingEpisodesWithAnimeQuery = @" -SELECT AE.AnimeEpisodeID -FROM AnimeEpisode AE -INNER JOIN AniDB_Episode ADBE ON AE.AniDB_EpisodeID = ADBE.EpisodeID -WHERE AE.IsHidden = 0 - AND ADBE.EpisodeType = 1 - AND (ADBE.AirDate = 0 OR ADBE.AirDate < :currentTime) - AND ADBE.AnimeID = :animeID - AND NOT EXISTS (SELECT 1 FROM CrossRef_File_Episode CFE WHERE CFE.EpisodeID = ADBE.EpisodeID) - AND ( - NOT EXISTS (SELECT 1 FROM AniDB_GroupStatus GS WHERE GS.AnimeID = ADBE.AnimeID) - OR EXISTS ( - SELECT 1 FROM AniDB_GroupStatus GS - WHERE GS.AnimeID = ADBE.AnimeID - AND (GS.CompletionState IN (3, 5) OR GS.LastEpisodeNumber >= ADBE.EpisodeNumber) - ) - ) -"; - private const string MissingCollectingEpisodesQuery = @" SELECT AE.AnimeEpisodeID FROM AnimeEpisode AE @@ -311,10 +270,10 @@ AND SRI.GroupID IS NOT NULL public IEnumerable GetMissing(bool collecting, int? animeID = null) { var currentTime = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); - using var session = _databaseFactory.SessionFactory.OpenSession(); - IList ids; if (collecting) { + using var session = _databaseFactory.SessionFactory.OpenSession(); + IList ids; if (animeID.HasValue) ids = session.CreateSQLQuery(MissingCollectingEpisodesWithAnimeQuery) .AddScalar("AnimeEpisodeID", NHibernateUtil.Int32) @@ -326,26 +285,43 @@ public IEnumerable GetMissing(bool collecting, int? animeID = null .AddScalar("AnimeEpisodeID", NHibernateUtil.Int32) .SetParameter("currentTime", currentTime) .List(); + + return ids + .Select(GetByID) + .WhereNotNull() + .OrderBy(e => e.AniDB_Episode?.AnimeID) + .ThenBy(e => e.AniDB_Episode?.EpisodeType) + .ThenBy(e => e.AniDB_Episode?.EpisodeNumber); } - else if (animeID.HasValue) - ids = session.CreateSQLQuery(MissingEpisodesWithAnimeQuery) - .AddScalar("AnimeEpisodeID", NHibernateUtil.Int32) - .SetParameter("currentTime", currentTime) - .SetParameter("animeID", animeID.Value) - .List(); - else - ids = session.CreateSQLQuery(MissingEpisodesQuery) - .AddScalar("AnimeEpisodeID", NHibernateUtil.Int32) - .SetParameter("currentTime", currentTime) - .List(); - return ids - .Select(GetByID) - .WhereNotNull() - .Where(e => e.AniDB_Episode is { HasAired: true }) - .OrderBy(e => e.AniDB_Episode?.AnimeID) - .ThenBy(e => e.AniDB_Episode?.EpisodeType) - .ThenBy(e => e.AniDB_Episode?.EpisodeNumber); + // Load AniDB episodes from cache for the non-collecting path + var anidbEpisodes = RepoFactory.AniDB_Episode.GetAll() + .Where(a => a.EpisodeType == EpisodeType.Episode) + .Where(a => a.AirDate == 0 || a.AirDate < currentTime) + .ToDictionary(a => a.EpisodeID); + + // Filter Shoko episodes by valid AniDB episode and optional anime ID + var episodes = GetAll() + .Where(e => !e.IsHidden && e.AniDB_EpisodeID > 0 && anidbEpisodes.ContainsKey(e.AniDB_EpisodeID)) + .ToList(); + + if (animeID.HasValue) + episodes = episodes.Where(e => anidbEpisodes[e.AniDB_EpisodeID]!.AnimeID == animeID.Value).ToList(); + + // Load group statuses and file cross-references in batch + var animeIDs = episodes.Select(e => anidbEpisodes[e.AniDB_EpisodeID]!.AnimeID).Distinct().ToList(); + var groupStatusesByAnime = RepoFactory.AniDB_GroupStatus.GetByAnimeIDs(animeIDs); + var episodeIDsWithFiles = RepoFactory.CrossRef_File_Episode.GetAll() + .Select(f => f.EpisodeID) + .ToHashSet(); + + // Apply the shared missing-episode predicate + return episodes + .Where(e => !episodeIDsWithFiles.Contains(e.AniDB_EpisodeID) && + e.IsMissingEpisode(groupStatusesByAnime.GetValueOrDefault(anidbEpisodes[e.AniDB_EpisodeID]!.AnimeID) ?? [])) + .OrderBy(e => anidbEpisodes[e.AniDB_EpisodeID]?.AnimeID) + .ThenBy(e => anidbEpisodes[e.AniDB_EpisodeID]?.EpisodeType) + .ThenBy(e => anidbEpisodes[e.AniDB_EpisodeID]?.EpisodeNumber); } public IReadOnlyList GetAllWatchedEpisodes(int userid, DateTime? after_date) diff --git a/Shoko.Server/Repositories/Direct/AniDB_GroupStatusRepository.cs b/Shoko.Server/Repositories/Direct/AniDB_GroupStatusRepository.cs index e74c1e218..ee96f7297 100644 --- a/Shoko.Server/Repositories/Direct/AniDB_GroupStatusRepository.cs +++ b/Shoko.Server/Repositories/Direct/AniDB_GroupStatusRepository.cs @@ -3,6 +3,7 @@ using NHibernate.Linq; using Shoko.QueueProcessor.Abstractions; using Shoko.Server.Databases; +using Shoko.Server.Extensions; using Shoko.Server.Models.AniDB; using Shoko.Server.Scheduling.Jobs.Actions; @@ -20,6 +21,30 @@ public List GetByAnimeID(int id) .ToList(); } + /// + /// Gets the cached group release statuses for multiple anime in a single batched query. + /// + /// The AniDB anime IDs to look up. Duplicates are ignored. + /// A dictionary keyed by anime ID containing the group statuses for that anime. Anime without any statuses are omitted from the result. + public Dictionary> GetByAnimeIDs(IEnumerable animeIDs) + { + var idList = animeIDs.Distinct().ToList(); + if (idList.Count == 0) + return new Dictionary>(); + + using var session = _databaseFactory.SessionFactory.OpenStatelessSession(); + var records = idList + .Batch(1000) + .SelectMany(batch => session.Query() + .Where(a => batch.Contains(a.AnimeID)) + .ToList()) + .ToList(); + + return records + .GroupBy(a => a.AnimeID) + .ToDictionary(g => g.Key, g => g.ToList()); + } + public void DeleteForAnime(int animeid) { using var session = _databaseFactory.SessionFactory.OpenStatelessSession(); From f6a6287ea5f88f5457bdfc0055007a0787d93c9a Mon Sep 17 00:00:00 2001 From: Harshith Mohan <26010946+harshithmohan@users.noreply.github.com> Date: Tue, 15 Sep 2026 08:56:24 +0530 Subject: [PATCH 3/4] refactor(api): pass AniDB episode explicitly to missing predicate and fix line length - `IsMissingEpisode` now takes the `AniDB_Episode` as a parameter so `GenerateSeriesSizes` and `GetMissing` reuse the already-resolved AniDB episode instead of re-reading `episode.AniDB_Episode` - wrapped the `GenerateSeriesSizes` signature to respect the 160-char line limit --- Shoko.Server/API/v3/Helpers/ModelHelper.cs | 5 +++-- Shoko.Server/Extensions/AnimeEpisodeExtensions.cs | 4 ++-- Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Shoko.Server/API/v3/Helpers/ModelHelper.cs b/Shoko.Server/API/v3/Helpers/ModelHelper.cs index 6067138ac..7905df55e 100644 --- a/Shoko.Server/API/v3/Helpers/ModelHelper.cs +++ b/Shoko.Server/API/v3/Helpers/ModelHelper.cs @@ -234,7 +234,8 @@ public static int GetTotalEpisodesForType(IEnumerable episodeList, .Count(anidbEpisode => anidbEpisode != null && anidbEpisode.EpisodeType == episodeType); } - public static SeriesSizes GenerateSeriesSizes(IEnumerable episodeList, int userID, IReadOnlyDictionary> groupStatusesByAnime) + public static SeriesSizes GenerateSeriesSizes(IEnumerable episodeList, int userID, + IReadOnlyDictionary> groupStatusesByAnime) { var sizes = new SeriesSizes(); var fileSet = new HashSet(); @@ -327,7 +328,7 @@ public static SeriesSizes GenerateSeriesSizes(IEnumerable episodeL sizes.Watched.Episodes++; } } - else if (episode.IsMissingEpisode(groupStatusesByAnime.GetValueOrDefault(anidbEpisode.AnimeID) ?? [])) + else if (episode.IsMissingEpisode(anidbEpisode, groupStatusesByAnime.GetValueOrDefault(anidbEpisode.AnimeID) ?? [])) { sizes.Missing.Episodes++; } diff --git a/Shoko.Server/Extensions/AnimeEpisodeExtensions.cs b/Shoko.Server/Extensions/AnimeEpisodeExtensions.cs index 4405f7f98..143ef39bc 100644 --- a/Shoko.Server/Extensions/AnimeEpisodeExtensions.cs +++ b/Shoko.Server/Extensions/AnimeEpisodeExtensions.cs @@ -16,16 +16,16 @@ public static class AnimeEpisodeExtensions /// based on the cached AniDB group release statuses for its anime. /// /// The episode to evaluate. + /// The AniDB episode for . /// Group statuses already scoped to the episode's anime. An empty list is treated as missing. /// if the episode is considered missing; otherwise, . /// /// This predicate is intended for episodes only; callers must /// pre-filter by episode type AND file presence before calling it. /// - public static bool IsMissingEpisode(this AnimeEpisode episode, IReadOnlyList groupStatuses) + public static bool IsMissingEpisode(this AnimeEpisode episode, AniDB_Episode anidb, IReadOnlyList groupStatuses) { if (episode.IsHidden) return false; - var anidb = episode.AniDB_Episode; if (anidb == null) return false; if (!anidb.HasAired) return false; diff --git a/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs b/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs index 3804a5007..6a0b47c74 100644 --- a/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs +++ b/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs @@ -318,7 +318,7 @@ public IEnumerable GetMissing(bool collecting, int? animeID = null // Apply the shared missing-episode predicate return episodes .Where(e => !episodeIDsWithFiles.Contains(e.AniDB_EpisodeID) && - e.IsMissingEpisode(groupStatusesByAnime.GetValueOrDefault(anidbEpisodes[e.AniDB_EpisodeID]!.AnimeID) ?? [])) + e.IsMissingEpisode(anidbEpisodes[e.AniDB_EpisodeID], groupStatusesByAnime.GetValueOrDefault(anidbEpisodes[e.AniDB_EpisodeID]!.AnimeID) ?? [])) .OrderBy(e => anidbEpisodes[e.AniDB_EpisodeID]?.AnimeID) .ThenBy(e => anidbEpisodes[e.AniDB_EpisodeID]?.EpisodeType) .ThenBy(e => anidbEpisodes[e.AniDB_EpisodeID]?.EpisodeNumber); From 77bf057fae18a84ec58755b9453de8490770af6f Mon Sep 17 00:00:00 2001 From: Harshith Mohan <26010946+harshithmohan@users.noreply.github.com> Date: Tue, 15 Sep 2026 10:28:51 +0530 Subject: [PATCH 4/4] refactor(anidb): cache AniDB_GroupStatus repository to fix N+1 queries Converted `AniDB_GroupStatusRepository` from a direct repository to a cached one with a `PocoIndex` on `AnimeID`. `GetByAnimeID` and `GetByAnimeIDs` now serve from the in-memory cache, so building Series DTOs and other group-status lookups no longer hit the database once per call. `DeleteForAnime` deletes through the base repository so the cache stays in sync, and still schedules `RefreshAnimeStatsJob` afterwards. --- .../AniDB/AniDB_GroupStatusRepository.cs | 55 +++++++++++++++++ .../Direct/AniDB_GroupStatusRepository.cs | 60 ------------------- .../Repositories/RepositoryStartup.cs | 2 +- Shoko.Server/Services/AnimeSeriesService.cs | 1 + 4 files changed, 57 insertions(+), 61 deletions(-) create mode 100644 Shoko.Server/Repositories/Cached/AniDB/AniDB_GroupStatusRepository.cs delete mode 100644 Shoko.Server/Repositories/Direct/AniDB_GroupStatusRepository.cs diff --git a/Shoko.Server/Repositories/Cached/AniDB/AniDB_GroupStatusRepository.cs b/Shoko.Server/Repositories/Cached/AniDB/AniDB_GroupStatusRepository.cs new file mode 100644 index 000000000..8d64efaba --- /dev/null +++ b/Shoko.Server/Repositories/Cached/AniDB/AniDB_GroupStatusRepository.cs @@ -0,0 +1,55 @@ +using System.Collections.Generic; +using System.Linq; +using NutzCode.InMemoryIndex; +using Shoko.QueueProcessor.Abstractions; +using Shoko.Server.Databases; +using Shoko.Server.Models.AniDB; +using Shoko.Server.Scheduling.Jobs.Actions; + +namespace Shoko.Server.Repositories.Cached.AniDB; + +public class AniDB_GroupStatusRepository : BaseCachedRepository +{ + private PocoIndex? _animeIDs; + + private readonly IQueueScheduler _scheduler; + + protected override int SelectKey(AniDB_GroupStatus entity) + => entity.AniDB_GroupStatusID; + + public override void PopulateIndexes() + => _animeIDs = Cache.CreateIndex(a => a.AnimeID); + + public virtual List GetByAnimeID(int id) + => _animeIDs!.GetMultiple(id); + + /// + /// Gets the cached group release statuses for multiple anime in a single batched lookup. + /// + /// The AniDB anime IDs to look up. Duplicates are ignored. + /// A dictionary keyed by anime ID containing the group statuses for that anime. Anime without any statuses are omitted from the result. + public Dictionary> GetByAnimeIDs(IEnumerable animeIDs) + { + var result = new Dictionary>(); + foreach (var id in animeIDs.Distinct()) + { + var statuses = _animeIDs!.GetMultiple(id); + if (statuses.Count > 0) + result[id] = statuses; + } + + return result; + } + + public void DeleteForAnime(int animeid) + { + Delete(_animeIDs!.GetMultiple(animeid)); + + _scheduler.RunAfterCurrent(j => j.AnimeID = animeid).GetAwaiter().GetResult(); + } + + public AniDB_GroupStatusRepository(DatabaseFactory databaseFactory, IQueueScheduler scheduler) : base(databaseFactory) + { + _scheduler = scheduler; + } +} diff --git a/Shoko.Server/Repositories/Direct/AniDB_GroupStatusRepository.cs b/Shoko.Server/Repositories/Direct/AniDB_GroupStatusRepository.cs deleted file mode 100644 index 356bd73d3..000000000 --- a/Shoko.Server/Repositories/Direct/AniDB_GroupStatusRepository.cs +++ /dev/null @@ -1,60 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using NHibernate.Linq; -using Shoko.QueueProcessor.Abstractions; -using Shoko.Server.Databases; -using Shoko.Server.Extensions; -using Shoko.Server.Models.AniDB; -using Shoko.Server.Scheduling.Jobs.Actions; - -namespace Shoko.Server.Repositories.Direct; - -public class AniDB_GroupStatusRepository : BaseDirectRepository -{ - private readonly IQueueScheduler _scheduler; - - public virtual List GetByAnimeID(int id) - { - using var session = _databaseFactory.SessionFactory.OpenStatelessSession(); - return session.Query() - .Where(a => a.AnimeID == id) - .ToList(); - } - - /// - /// Gets the cached group release statuses for multiple anime in a single batched query. - /// - /// The AniDB anime IDs to look up. Duplicates are ignored. - /// A dictionary keyed by anime ID containing the group statuses for that anime. Anime without any statuses are omitted from the result. - public Dictionary> GetByAnimeIDs(IEnumerable animeIDs) - { - var idList = animeIDs.Distinct().ToList(); - if (idList.Count == 0) - return new Dictionary>(); - - using var session = _databaseFactory.SessionFactory.OpenStatelessSession(); - var records = idList - .Batch(1000) - .SelectMany(batch => session.Query() - .Where(a => batch.Contains(a.AnimeID)) - .ToList()) - .ToList(); - - return records - .GroupBy(a => a.AnimeID) - .ToDictionary(g => g.Key, g => g.ToList()); - } - - public void DeleteForAnime(int animeid) - { - using var session = _databaseFactory.SessionFactory.OpenStatelessSession(); - session.Query().Where(a => a.AnimeID == animeid).Delete(); - - _scheduler.RunAfterCurrent(j => j.AnimeID = animeid).GetAwaiter().GetResult(); - } - - public AniDB_GroupStatusRepository(DatabaseFactory databaseFactory, IQueueScheduler scheduler) : base(databaseFactory) - { - _scheduler = scheduler; - } -} diff --git a/Shoko.Server/Repositories/RepositoryStartup.cs b/Shoko.Server/Repositories/RepositoryStartup.cs index 9589d5069..79cf6be64 100644 --- a/Shoko.Server/Repositories/RepositoryStartup.cs +++ b/Shoko.Server/Repositories/RepositoryStartup.cs @@ -22,7 +22,6 @@ public static IServiceCollection AddRepositories(this IServiceCollection service services.AddDirectRepository(); services.AddDirectRepository(); - services.AddDirectRepository(); services.AddDirectRepository(); services.AddDirectRepository(); services.AddDirectRepository(); @@ -59,6 +58,7 @@ public static IServiceCollection AddRepositories(this IServiceCollection service services.AddCachedRepository(); services.AddCachedRepository(); services.AddCachedRepository(); + services.AddCachedRepository(); services.AddCachedRepository(); services.AddCachedRepository(); services.AddCachedRepository(); diff --git a/Shoko.Server/Services/AnimeSeriesService.cs b/Shoko.Server/Services/AnimeSeriesService.cs index 4f7529948..fd1a158a8 100644 --- a/Shoko.Server/Services/AnimeSeriesService.cs +++ b/Shoko.Server/Services/AnimeSeriesService.cs @@ -21,6 +21,7 @@ using Shoko.Server.Providers.AniDB; using Shoko.Server.Providers.TMDB; using Shoko.Server.Repositories.Cached; +using Shoko.Server.Repositories.Cached.AniDB; using Shoko.Server.Repositories.Direct; using Shoko.Server.Scheduling.Jobs.Actions; using Shoko.Server.Scheduling.Jobs.TMDB;