diff --git a/Shoko.Server/API/v3/Helpers/ModelHelper.cs b/Shoko.Server/API/v3/Helpers/ModelHelper.cs index 4722f806b..7905df55e 100644 --- a/Shoko.Server/API/v3/Helpers/ModelHelper.cs +++ b/Shoko.Server/API/v3/Helpers/ModelHelper.cs @@ -5,6 +5,8 @@ using Shoko.Abstractions.Metadata.Enums; using Shoko.Abstractions.Video.Enums; using Shoko.Server.API.v3.Models.Common; +using Shoko.Server.Extensions; +using Shoko.Server.Models.AniDB; using Shoko.Server.Models.CrossReference; using Shoko.Server.Models.Shoko; using Shoko.Server.Repositories; @@ -232,7 +234,8 @@ public static int GetTotalEpisodesForType(IEnumerable episodeList, .Count(anidbEpisode => anidbEpisode != null && anidbEpisode.EpisodeType == episodeType); } - public static SeriesSizes GenerateSeriesSizes(IEnumerable episodeList, int userID) + public static SeriesSizes GenerateSeriesSizes(IEnumerable episodeList, int userID, + IReadOnlyDictionary> groupStatusesByAnime) { var sizes = new SeriesSizes(); var fileSet = new HashSet(); @@ -325,7 +328,7 @@ public static SeriesSizes GenerateSeriesSizes(IEnumerable episodeL sizes.Watched.Episodes++; } } - else if (anidbEpisode.HasAired) + else if (episode.IsMissingEpisode(anidbEpisode, groupStatusesByAnime.GetValueOrDefault(anidbEpisode.AnimeID) ?? [])) { sizes.Missing.Episodes++; } @@ -409,7 +412,13 @@ public static SeriesSizes GenerateSeriesSizes(IEnumerable episodeL public static GroupSizes GenerateGroupSizes(IEnumerable seriesList, IEnumerable episodeList, int subGroups, int userID) { - var sizes = new GroupSizes(GenerateSeriesSizes(episodeList, userID)); + var animeIDs = seriesList + .Select(ser => ser.AniDB_ID) + .Where(animeID => animeID > 0) + .Distinct() + .ToList(); + var groupStatusesByAnime = RepoFactory.AniDB_GroupStatus.GetByAnimeIDs(animeIDs); + var sizes = new GroupSizes(GenerateSeriesSizes(episodeList, userID, groupStatusesByAnime)); foreach (var series in seriesList) { var anime = series.AniDB_Anime; diff --git a/Shoko.Server/API/v3/Models/Shoko/Series.cs b/Shoko.Server/API/v3/Models/Shoko/Series.cs index 64cbe95d7..1c19d4cfe 100644 --- a/Shoko.Server/API/v3/Models/Shoko/Series.cs +++ b/Shoko.Server/API/v3/Models/Shoko/Series.cs @@ -135,7 +135,8 @@ public Series(AnimeSeries ser, int userId = 0, bool randomizeImages = false, Has var userData = RepoFactory.AnimeSeries_User.GetByUserAndSeriesID(userId, ser.AnimeSeriesID); var tmdbMovieXRefs = ser.TmdbMovieCrossReferences; var tmdbShowXRefs = ser.TmdbShowCrossReferences; - var sizes = ModelHelper.GenerateSeriesSizes(allEpisodes, userId); + var groupStatusesByAnime = new Dictionary> { [ser.AniDB_ID] = RepoFactory.AniDB_GroupStatus.GetByAnimeID(ser.AniDB_ID) }; + var sizes = ModelHelper.GenerateSeriesSizes(allEpisodes, userId, groupStatusesByAnime); IDs = new() { ID = ser.AnimeSeriesID, diff --git a/Shoko.Server/Extensions/AnimeEpisodeExtensions.cs b/Shoko.Server/Extensions/AnimeEpisodeExtensions.cs new file mode 100644 index 000000000..143ef39bc --- /dev/null +++ b/Shoko.Server/Extensions/AnimeEpisodeExtensions.cs @@ -0,0 +1,36 @@ +using System.Collections.Generic; +using System.Linq; +using Shoko.Abstractions.Metadata.Enums; +using Shoko.Server.Models.AniDB; +using Shoko.Server.Models.Shoko; + +namespace Shoko.Server.Extensions; + +/// +/// Extension methods for . +/// +public static class AnimeEpisodeExtensions +{ + /// + /// Determines whether an aired episode with no local files should be counted as missing, + /// based on the cached AniDB group release statuses for its anime. + /// + /// The episode to evaluate. + /// The AniDB episode for . + /// Group statuses already scoped to the episode's anime. An empty list is treated as missing. + /// if the episode is considered missing; otherwise, . + /// + /// This predicate is intended for episodes only; callers must + /// pre-filter by episode type AND file presence before calling it. + /// + public static bool IsMissingEpisode(this AnimeEpisode episode, AniDB_Episode anidb, IReadOnlyList groupStatuses) + { + if (episode.IsHidden) return false; + if (anidb == null) return false; + if (!anidb.HasAired) return false; + + return groupStatuses.Count == 0 || groupStatuses.Any(gs => + gs.CompletionState is (int)GroupCompletionStatus.Complete or (int)GroupCompletionStatus.Finished + || gs.LastEpisodeNumber >= anidb.EpisodeNumber); + } +} diff --git a/Shoko.Server/Repositories/Cached/AniDB/AniDB_GroupStatusRepository.cs b/Shoko.Server/Repositories/Cached/AniDB/AniDB_GroupStatusRepository.cs new file mode 100644 index 000000000..8d64efaba --- /dev/null +++ b/Shoko.Server/Repositories/Cached/AniDB/AniDB_GroupStatusRepository.cs @@ -0,0 +1,55 @@ +using System.Collections.Generic; +using System.Linq; +using NutzCode.InMemoryIndex; +using Shoko.QueueProcessor.Abstractions; +using Shoko.Server.Databases; +using Shoko.Server.Models.AniDB; +using Shoko.Server.Scheduling.Jobs.Actions; + +namespace Shoko.Server.Repositories.Cached.AniDB; + +public class AniDB_GroupStatusRepository : BaseCachedRepository +{ + private PocoIndex? _animeIDs; + + private readonly IQueueScheduler _scheduler; + + protected override int SelectKey(AniDB_GroupStatus entity) + => entity.AniDB_GroupStatusID; + + public override void PopulateIndexes() + => _animeIDs = Cache.CreateIndex(a => a.AnimeID); + + public virtual List GetByAnimeID(int id) + => _animeIDs!.GetMultiple(id); + + /// + /// Gets the cached group release statuses for multiple anime in a single batched lookup. + /// + /// The AniDB anime IDs to look up. Duplicates are ignored. + /// A dictionary keyed by anime ID containing the group statuses for that anime. Anime without any statuses are omitted from the result. + public Dictionary> GetByAnimeIDs(IEnumerable animeIDs) + { + var result = new Dictionary>(); + foreach (var id in animeIDs.Distinct()) + { + var statuses = _animeIDs!.GetMultiple(id); + if (statuses.Count > 0) + result[id] = statuses; + } + + return result; + } + + public void DeleteForAnime(int animeid) + { + Delete(_animeIDs!.GetMultiple(animeid)); + + _scheduler.RunAfterCurrent(j => j.AnimeID = animeid).GetAwaiter().GetResult(); + } + + public AniDB_GroupStatusRepository(DatabaseFactory databaseFactory, IQueueScheduler scheduler) : base(databaseFactory) + { + _scheduler = scheduler; + } +} diff --git a/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs b/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs index e94b59719..6a0b47c74 100644 --- a/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs +++ b/Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs @@ -6,6 +6,7 @@ using NutzCode.InMemoryIndex; using Shoko.Abstractions.Extensions; using Shoko.Server.Databases; +using Shoko.Server.Extensions; using Shoko.Server.Models.Shoko; using EpisodeType = Shoko.Abstractions.Metadata.Enums.EpisodeType; @@ -215,48 +216,6 @@ public IEnumerable GetWithDuplicateFiles(int? animeID = null) .Select(tuple => tuple.episode!); } - // Group_CompletionStatus.Complete = 3, Group_CompletionStatus.Finished = 5 - // EpisodeType.Episode = 1 - // AirDate = 0 means unknown; kept as a candidate here and resolved exactly via HasAired in GetMissing(). - // LastEpisodeNumber >= EpisodeNumber approximates HasGroupReleasedEpisode. - // GS.GroupID (int) = SRI.GroupID (varchar) relies on implicit int↔string coercion present in SQLite, MySQL, and SQL Server. - private const string MissingEpisodesQuery = @" -SELECT AE.AnimeEpisodeID -FROM AnimeEpisode AE -INNER JOIN AniDB_Episode ADBE ON AE.AniDB_EpisodeID = ADBE.EpisodeID -WHERE AE.IsHidden = 0 - AND ADBE.EpisodeType = 1 - AND (ADBE.AirDate = 0 OR ADBE.AirDate < :currentTime) - AND NOT EXISTS (SELECT 1 FROM CrossRef_File_Episode CFE WHERE CFE.EpisodeID = ADBE.EpisodeID) - AND ( - NOT EXISTS (SELECT 1 FROM AniDB_GroupStatus GS WHERE GS.AnimeID = ADBE.AnimeID) - OR EXISTS ( - SELECT 1 FROM AniDB_GroupStatus GS - WHERE GS.AnimeID = ADBE.AnimeID - AND (GS.CompletionState IN (3, 5) OR GS.LastEpisodeNumber >= ADBE.EpisodeNumber) - ) - ) -"; - - private const string MissingEpisodesWithAnimeQuery = @" -SELECT AE.AnimeEpisodeID -FROM AnimeEpisode AE -INNER JOIN AniDB_Episode ADBE ON AE.AniDB_EpisodeID = ADBE.EpisodeID -WHERE AE.IsHidden = 0 - AND ADBE.EpisodeType = 1 - AND (ADBE.AirDate = 0 OR ADBE.AirDate < :currentTime) - AND ADBE.AnimeID = :animeID - AND NOT EXISTS (SELECT 1 FROM CrossRef_File_Episode CFE WHERE CFE.EpisodeID = ADBE.EpisodeID) - AND ( - NOT EXISTS (SELECT 1 FROM AniDB_GroupStatus GS WHERE GS.AnimeID = ADBE.AnimeID) - OR EXISTS ( - SELECT 1 FROM AniDB_GroupStatus GS - WHERE GS.AnimeID = ADBE.AnimeID - AND (GS.CompletionState IN (3, 5) OR GS.LastEpisodeNumber >= ADBE.EpisodeNumber) - ) - ) -"; - private const string MissingCollectingEpisodesQuery = @" SELECT AE.AnimeEpisodeID FROM AnimeEpisode AE @@ -311,10 +270,10 @@ AND SRI.GroupID IS NOT NULL public IEnumerable GetMissing(bool collecting, int? animeID = null) { var currentTime = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); - using var session = _databaseFactory.SessionFactory.OpenSession(); - IList ids; if (collecting) { + using var session = _databaseFactory.SessionFactory.OpenSession(); + IList ids; if (animeID.HasValue) ids = session.CreateSQLQuery(MissingCollectingEpisodesWithAnimeQuery) .AddScalar("AnimeEpisodeID", NHibernateUtil.Int32) @@ -326,26 +285,43 @@ public IEnumerable GetMissing(bool collecting, int? animeID = null .AddScalar("AnimeEpisodeID", NHibernateUtil.Int32) .SetParameter("currentTime", currentTime) .List(); + + return ids + .Select(GetByID) + .WhereNotNull() + .OrderBy(e => e.AniDB_Episode?.AnimeID) + .ThenBy(e => e.AniDB_Episode?.EpisodeType) + .ThenBy(e => e.AniDB_Episode?.EpisodeNumber); } - else if (animeID.HasValue) - ids = session.CreateSQLQuery(MissingEpisodesWithAnimeQuery) - .AddScalar("AnimeEpisodeID", NHibernateUtil.Int32) - .SetParameter("currentTime", currentTime) - .SetParameter("animeID", animeID.Value) - .List(); - else - ids = session.CreateSQLQuery(MissingEpisodesQuery) - .AddScalar("AnimeEpisodeID", NHibernateUtil.Int32) - .SetParameter("currentTime", currentTime) - .List(); - return ids - .Select(GetByID) - .WhereNotNull() - .Where(e => e.AniDB_Episode is { HasAired: true }) - .OrderBy(e => e.AniDB_Episode?.AnimeID) - .ThenBy(e => e.AniDB_Episode?.EpisodeType) - .ThenBy(e => e.AniDB_Episode?.EpisodeNumber); + // Load AniDB episodes from cache for the non-collecting path + var anidbEpisodes = RepoFactory.AniDB_Episode.GetAll() + .Where(a => a.EpisodeType == EpisodeType.Episode) + .Where(a => a.AirDate == 0 || a.AirDate < currentTime) + .ToDictionary(a => a.EpisodeID); + + // Filter Shoko episodes by valid AniDB episode and optional anime ID + var episodes = GetAll() + .Where(e => !e.IsHidden && e.AniDB_EpisodeID > 0 && anidbEpisodes.ContainsKey(e.AniDB_EpisodeID)) + .ToList(); + + if (animeID.HasValue) + episodes = episodes.Where(e => anidbEpisodes[e.AniDB_EpisodeID]!.AnimeID == animeID.Value).ToList(); + + // Load group statuses and file cross-references in batch + var animeIDs = episodes.Select(e => anidbEpisodes[e.AniDB_EpisodeID]!.AnimeID).Distinct().ToList(); + var groupStatusesByAnime = RepoFactory.AniDB_GroupStatus.GetByAnimeIDs(animeIDs); + var episodeIDsWithFiles = RepoFactory.CrossRef_File_Episode.GetAll() + .Select(f => f.EpisodeID) + .ToHashSet(); + + // Apply the shared missing-episode predicate + return episodes + .Where(e => !episodeIDsWithFiles.Contains(e.AniDB_EpisodeID) && + e.IsMissingEpisode(anidbEpisodes[e.AniDB_EpisodeID], groupStatusesByAnime.GetValueOrDefault(anidbEpisodes[e.AniDB_EpisodeID]!.AnimeID) ?? [])) + .OrderBy(e => anidbEpisodes[e.AniDB_EpisodeID]?.AnimeID) + .ThenBy(e => anidbEpisodes[e.AniDB_EpisodeID]?.EpisodeType) + .ThenBy(e => anidbEpisodes[e.AniDB_EpisodeID]?.EpisodeNumber); } public IReadOnlyList GetAllWatchedEpisodes(int userid, DateTime? after_date) diff --git a/Shoko.Server/Repositories/Direct/AniDB_GroupStatusRepository.cs b/Shoko.Server/Repositories/Direct/AniDB_GroupStatusRepository.cs deleted file mode 100644 index 2814d4972..000000000 --- a/Shoko.Server/Repositories/Direct/AniDB_GroupStatusRepository.cs +++ /dev/null @@ -1,35 +0,0 @@ -using System.Collections.Generic; -using System.Linq; -using NHibernate.Linq; -using Shoko.QueueProcessor.Abstractions; -using Shoko.Server.Databases; -using Shoko.Server.Models.AniDB; -using Shoko.Server.Scheduling.Jobs.Actions; - -namespace Shoko.Server.Repositories.Direct; - -public class AniDB_GroupStatusRepository : BaseDirectRepository -{ - private readonly IQueueScheduler _scheduler; - - public virtual List GetByAnimeID(int id) - { - using var session = _databaseFactory.SessionFactory.OpenStatelessSession(); - return session.Query() - .Where(a => a.AnimeID == id) - .ToList(); - } - - public void DeleteForAnime(int animeid) - { - using var session = _databaseFactory.SessionFactory.OpenStatelessSession(); - session.Query().Where(a => a.AnimeID == animeid).Delete(); - - _scheduler.RunAfterCurrent(j => j.AnimeID = animeid).GetAwaiter().GetResult(); - } - - public AniDB_GroupStatusRepository(DatabaseFactory databaseFactory, IQueueScheduler scheduler) : base(databaseFactory) - { - _scheduler = scheduler; - } -} diff --git a/Shoko.Server/Repositories/RepositoryStartup.cs b/Shoko.Server/Repositories/RepositoryStartup.cs index 9589d5069..79cf6be64 100644 --- a/Shoko.Server/Repositories/RepositoryStartup.cs +++ b/Shoko.Server/Repositories/RepositoryStartup.cs @@ -22,7 +22,6 @@ public static IServiceCollection AddRepositories(this IServiceCollection service services.AddDirectRepository(); services.AddDirectRepository(); - services.AddDirectRepository(); services.AddDirectRepository(); services.AddDirectRepository(); services.AddDirectRepository(); @@ -59,6 +58,7 @@ public static IServiceCollection AddRepositories(this IServiceCollection service services.AddCachedRepository(); services.AddCachedRepository(); services.AddCachedRepository(); + services.AddCachedRepository(); services.AddCachedRepository(); services.AddCachedRepository(); services.AddCachedRepository(); diff --git a/Shoko.Server/Scheduling/Jobs/AniDB/GetUpdatedAniDBAnimeJob.cs b/Shoko.Server/Scheduling/Jobs/AniDB/GetUpdatedAniDBAnimeJob.cs index 596cfb5e3..f1ccfd9ac 100644 --- a/Shoko.Server/Scheduling/Jobs/AniDB/GetUpdatedAniDBAnimeJob.cs +++ b/Shoko.Server/Scheduling/Jobs/AniDB/GetUpdatedAniDBAnimeJob.cs @@ -3,9 +3,11 @@ using Microsoft.Extensions.Logging; using Shoko.Abstractions.Metadata.Anidb.Enums; using Shoko.Abstractions.Metadata.Anidb.Services; +using Shoko.QueueProcessor.Abstractions; using Shoko.QueueProcessor.Acquisition.Attributes; using Shoko.QueueProcessor.Builder; using Shoko.QueueProcessor.Concurrency; +using Shoko.QueueProcessor.Scheduling; using Shoko.Server.Models.Internal; using Shoko.Server.Providers.AniDB.Interfaces; using Shoko.Server.Providers.AniDB.Titles; @@ -25,7 +27,7 @@ namespace Shoko.Server.Scheduling.Jobs.AniDB; [AniDBUdpRateLimited] [DisallowConcurrencyGroup(ConcurrencyGroups.AniDB_UDP)] [JobKeyGroup(JobKeyGroup.AniDB)] -public class GetUpdatedAniDBAnimeJob(IRequestFactory requestFactory, IAnidbService anidbService, ISettingsProvider settingsProvider, AniDBTitleHelper titleHelper, AniDB_AnimeRepository anidbAnimeRepository, AniDB_AnimeUpdateRepository anidbAnimeUpdates, AnimeSeriesRepository animeSeries, ScheduledUpdateRepository scheduledUpdates) : BaseJob +public class GetUpdatedAniDBAnimeJob(IRequestFactory requestFactory, IAnidbService anidbService, ISettingsProvider settingsProvider, AniDBTitleHelper titleHelper, AniDB_AnimeRepository anidbAnimeRepository, AniDB_AnimeUpdateRepository anidbAnimeUpdates, AnimeSeriesRepository animeSeries, ScheduledUpdateRepository scheduledUpdates, IQueueScheduler scheduler) : BaseJob { public bool ForceRefresh { get; set; } @@ -144,6 +146,9 @@ public override async Task Execute() await anidbService.ScheduleRefreshOfAnimeByID(animeID, AnidbRefreshMethod.Remote | AnidbRefreshMethod.DeferToRemoteIfUnsuccessful).ConfigureAwait(false); countSeries++; + + await scheduler.StartJob( + c => (c.AnimeID, c.ForceRefresh) = (animeID, ForceRefresh)); } return (response, countAnime, countSeries); diff --git a/Shoko.Server/Services/AnimeSeriesService.cs b/Shoko.Server/Services/AnimeSeriesService.cs index 4f7529948..fd1a158a8 100644 --- a/Shoko.Server/Services/AnimeSeriesService.cs +++ b/Shoko.Server/Services/AnimeSeriesService.cs @@ -21,6 +21,7 @@ using Shoko.Server.Providers.AniDB; using Shoko.Server.Providers.TMDB; using Shoko.Server.Repositories.Cached; +using Shoko.Server.Repositories.Cached.AniDB; using Shoko.Server.Repositories.Direct; using Shoko.Server.Scheduling.Jobs.Actions; using Shoko.Server.Scheduling.Jobs.TMDB;