Unify missing episode definition and refresh AniDB group status periodically

Shoko.Server/API/v3/Helpers/ModelHelper.cs

@@ -5,6 +5,8 @@
 using Shoko.Abstractions.Metadata.Enums;
 using Shoko.Abstractions.Video.Enums;
 using Shoko.Server.API.v3.Models.Common;
+using Shoko.Server.Extensions;
+using Shoko.Server.Models.AniDB;
 using Shoko.Server.Models.CrossReference;
 using Shoko.Server.Models.Shoko;
 using Shoko.Server.Repositories;
@@ -232,7 +234,7 @@
             .Count(anidbEpisode => anidbEpisode != null && anidbEpisode.EpisodeType == episodeType);
     }
 
-    public static SeriesSizes GenerateSeriesSizes(IEnumerable<AnimeEpisode> episodeList, int userID)
+    public static SeriesSizes GenerateSeriesSizes(IEnumerable<AnimeEpisode> episodeList, int userID, IReadOnlyDictionary<int, List<AniDB_GroupStatus>> groupStatusesByAnime)
     {
         var sizes = new SeriesSizes();
         var fileSet = new HashSet<int>();
@@ -325,7 +327,7 @@
                             sizes.Watched.Episodes++;
                         }
                     }
-                    else if (anidbEpisode.HasAired)
+                    else if (episode.IsMissingEpisode(groupStatusesByAnime.GetValueOrDefault(anidbEpisode.AnimeID) ?? []))
                     {
                         sizes.Missing.Episodes++;
                     }
@@ -409,7 +411,13 @@
     public static GroupSizes GenerateGroupSizes(IEnumerable<AnimeSeries> seriesList, IEnumerable<AnimeEpisode> episodeList,
         int subGroups, int userID)
     {
-        var sizes = new GroupSizes(GenerateSeriesSizes(episodeList, userID));
+        var animeIDs = seriesList
+            .Select(ser => ser.AniDB_ID)
+            .Where(animeID => animeID > 0)
+            .Distinct()
+            .ToList();
+        var groupStatusesByAnime = RepoFactory.AniDB_GroupStatus.GetByAnimeIDs(animeIDs);
+        var sizes = new GroupSizes(GenerateSeriesSizes(episodeList, userID, groupStatusesByAnime));
         foreach (var series in seriesList)
         {
             var anime = series.AniDB_Anime;

Shoko.Server/API/v3/Models/Shoko/Series.cs

@@ -135,7 +135,8 @@ public Series(AnimeSeries ser, int userId = 0, bool randomizeImages = false, Has
         var userData = RepoFactory.AnimeSeries_User.GetByUserAndSeriesID(userId, ser.AnimeSeriesID);
         var tmdbMovieXRefs = ser.TmdbMovieCrossReferences;
         var tmdbShowXRefs = ser.TmdbShowCrossReferences;
-        var sizes = ModelHelper.GenerateSeriesSizes(allEpisodes, userId);
+        var groupStatusesByAnime = new Dictionary<int, List<AniDB_GroupStatus>> { [ser.AniDB_ID] = RepoFactory.AniDB_GroupStatus.GetByAnimeID(ser.AniDB_ID) };
+        var sizes = ModelHelper.GenerateSeriesSizes(allEpisodes, userId, groupStatusesByAnime);
         IDs = new()
         {
             ID = ser.AnimeSeriesID,

Shoko.Server/Extensions/AnimeEpisodeExtensions.cs

@@ -0,0 +1,36 @@
+using System.Collections.Generic;
+using System.Linq;
+using Shoko.Abstractions.Metadata.Enums;
+using Shoko.Server.Models.AniDB;
+using Shoko.Server.Models.Shoko;
+
+namespace Shoko.Server.Extensions;
+
+/// <summary>
+/// Extension methods for <see cref="AnimeEpisode"/>.
+/// </summary>
+public static class AnimeEpisodeExtensions
+{
+    /// <summary>
+    /// Determines whether an aired episode with no local files should be counted as missing,
+    /// based on the cached AniDB group release statuses for its anime.
+    /// </summary>
+    /// <param name="episode">The episode to evaluate.</param>
+    /// <param name="groupStatuses">Group statuses already scoped to the episode's anime. An empty list is treated as missing.</param>
+    /// <returns><see langword="true"/> if the episode is considered missing; otherwise, <see langword="false"/>.</returns>
+    /// <remarks>
+    /// This predicate is intended for <see cref="EpisodeType.Episode"/> episodes only; callers must
+    /// pre-filter by episode type AND file presence before calling it.
+    /// </remarks>
+    public static bool IsMissingEpisode(this AnimeEpisode episode, IReadOnlyList<AniDB_GroupStatus> groupStatuses)
+    {
+        if (episode.IsHidden) return false;
+        var anidb = episode.AniDB_Episode;
+        if (anidb == null) return false;
+        if (!anidb.HasAired) return false;
+
+        return groupStatuses.Count == 0 || groupStatuses.Any(gs =>
+            gs.CompletionState is (int)GroupCompletionStatus.Complete or (int)GroupCompletionStatus.Finished
+            || gs.LastEpisodeNumber >= anidb.EpisodeNumber);
+    }
+}

Shoko.Server/Repositories/Cached/AnimeEpisodeRepository.cs

@@ -6,6 +6,7 @@
 using NutzCode.InMemoryIndex;
 using Shoko.Abstractions.Extensions;
 using Shoko.Server.Databases;
+using Shoko.Server.Extensions;
 using Shoko.Server.Models.Shoko;
 
 using EpisodeType = Shoko.Abstractions.Metadata.Enums.EpisodeType;
@@ -215,48 +216,6 @@
             .Select(tuple => tuple.episode!);
     }
 
-    // Group_CompletionStatus.Complete = 3, Group_CompletionStatus.Finished = 5
-    // EpisodeType.Episode = 1
-    // AirDate = 0 means unknown; kept as a candidate here and resolved exactly via HasAired in GetMissing().
-    // LastEpisodeNumber >= EpisodeNumber approximates HasGroupReleasedEpisode.
-    // GS.GroupID (int) = SRI.GroupID (varchar) relies on implicit int↔string coercion present in SQLite, MySQL, and SQL Server.
-    private const string MissingEpisodesQuery = @"
-SELECT AE.AnimeEpisodeID
-FROM AnimeEpisode AE
-INNER JOIN AniDB_Episode ADBE ON AE.AniDB_EpisodeID = ADBE.EpisodeID
-WHERE AE.IsHidden = 0
-  AND ADBE.EpisodeType = 1
-  AND (ADBE.AirDate = 0 OR ADBE.AirDate < :currentTime)
-  AND NOT EXISTS (SELECT 1 FROM CrossRef_File_Episode CFE WHERE CFE.EpisodeID = ADBE.EpisodeID)
-  AND (
-      NOT EXISTS (SELECT 1 FROM AniDB_GroupStatus GS WHERE GS.AnimeID = ADBE.AnimeID)
-      OR EXISTS (
-          SELECT 1 FROM AniDB_GroupStatus GS
-          WHERE GS.AnimeID = ADBE.AnimeID
-            AND (GS.CompletionState IN (3, 5) OR GS.LastEpisodeNumber >= ADBE.EpisodeNumber)
-      )
-  )
-";
-
-    private const string MissingEpisodesWithAnimeQuery = @"
-SELECT AE.AnimeEpisodeID
-FROM AnimeEpisode AE
-INNER JOIN AniDB_Episode ADBE ON AE.AniDB_EpisodeID = ADBE.EpisodeID
-WHERE AE.IsHidden = 0
-  AND ADBE.EpisodeType = 1
-  AND (ADBE.AirDate = 0 OR ADBE.AirDate < :currentTime)
-  AND ADBE.AnimeID = :animeID
-  AND NOT EXISTS (SELECT 1 FROM CrossRef_File_Episode CFE WHERE CFE.EpisodeID = ADBE.EpisodeID)
-  AND (
-      NOT EXISTS (SELECT 1 FROM AniDB_GroupStatus GS WHERE GS.AnimeID = ADBE.AnimeID)
-      OR EXISTS (
-          SELECT 1 FROM AniDB_GroupStatus GS
-          WHERE GS.AnimeID = ADBE.AnimeID
-            AND (GS.CompletionState IN (3, 5) OR GS.LastEpisodeNumber >= ADBE.EpisodeNumber)
-      )
-  )
-";
-
     private const string MissingCollectingEpisodesQuery = @"
 SELECT AE.AnimeEpisodeID
 FROM AnimeEpisode AE
@@ -311,10 +270,10 @@
     public IEnumerable<AnimeEpisode> GetMissing(bool collecting, int? animeID = null)
     {
         var currentTime = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
-        using var session = _databaseFactory.SessionFactory.OpenSession();
-        IList<int> ids;
         if (collecting)
         {
+            using var session = _databaseFactory.SessionFactory.OpenSession();
+            IList<int> ids;
             if (animeID.HasValue)
                 ids = session.CreateSQLQuery(MissingCollectingEpisodesWithAnimeQuery)
                     .AddScalar("AnimeEpisodeID", NHibernateUtil.Int32)
@@ -326,26 +285,43 @@
                     .AddScalar("AnimeEpisodeID", NHibernateUtil.Int32)
                     .SetParameter("currentTime", currentTime)
                     .List<int>();
+
+            return ids
+                .Select(GetByID)
+                .WhereNotNull()
+                .OrderBy(e => e.AniDB_Episode?.AnimeID)
+                .ThenBy(e => e.AniDB_Episode?.EpisodeType)
+                .ThenBy(e => e.AniDB_Episode?.EpisodeNumber);
         }
-        else if (animeID.HasValue)
-            ids = session.CreateSQLQuery(MissingEpisodesWithAnimeQuery)
-                .AddScalar("AnimeEpisodeID", NHibernateUtil.Int32)
-                .SetParameter("currentTime", currentTime)
-                .SetParameter("animeID", animeID.Value)
-                .List<int>();
-        else
-            ids = session.CreateSQLQuery(MissingEpisodesQuery)
-                .AddScalar("AnimeEpisodeID", NHibernateUtil.Int32)
-                .SetParameter("currentTime", currentTime)
-                .List<int>();
 
-        return ids
-            .Select(GetByID)
-            .WhereNotNull()
-            .Where(e => e.AniDB_Episode is { HasAired: true })
-            .OrderBy(e => e.AniDB_Episode?.AnimeID)
-            .ThenBy(e => e.AniDB_Episode?.EpisodeType)
-            .ThenBy(e => e.AniDB_Episode?.EpisodeNumber);
+        // Load AniDB episodes from cache for the non-collecting path
+        var anidbEpisodes = RepoFactory.AniDB_Episode.GetAll()
+            .Where(a => a.EpisodeType == EpisodeType.Episode)
+            .Where(a => a.AirDate == 0 || a.AirDate < currentTime)
+            .ToDictionary(a => a.EpisodeID);
+
+        // Filter Shoko episodes by valid AniDB episode and optional anime ID
+        var episodes = GetAll()
+            .Where(e => !e.IsHidden && e.AniDB_EpisodeID > 0 && anidbEpisodes.ContainsKey(e.AniDB_EpisodeID))
+            .ToList();
+
+        if (animeID.HasValue)
+            episodes = episodes.Where(e => anidbEpisodes[e.AniDB_EpisodeID]!.AnimeID == animeID.Value).ToList();
+
+        // Load group statuses and file cross-references in batch
+        var animeIDs = episodes.Select(e => anidbEpisodes[e.AniDB_EpisodeID]!.AnimeID).Distinct().ToList();
+        var groupStatusesByAnime = RepoFactory.AniDB_GroupStatus.GetByAnimeIDs(animeIDs);
+        var episodeIDsWithFiles = RepoFactory.CrossRef_File_Episode.GetAll()
+            .Select(f => f.EpisodeID)
+            .ToHashSet();
+
+        // Apply the shared missing-episode predicate
+        return episodes
+            .Where(e => !episodeIDsWithFiles.Contains(e.AniDB_EpisodeID) &&
+                e.IsMissingEpisode(groupStatusesByAnime.GetValueOrDefault(anidbEpisodes[e.AniDB_EpisodeID]!.AnimeID) ?? []))
+            .OrderBy(e => anidbEpisodes[e.AniDB_EpisodeID]?.AnimeID)
+            .ThenBy(e => anidbEpisodes[e.AniDB_EpisodeID]?.EpisodeType)
+            .ThenBy(e => anidbEpisodes[e.AniDB_EpisodeID]?.EpisodeNumber);
     }
 
     public IReadOnlyList<AnimeEpisode> GetAllWatchedEpisodes(int userid, DateTime? after_date)

Shoko.Server/Repositories/Direct/AniDB_GroupStatusRepository.cs

@@ -3,6 +3,7 @@
 using NHibernate.Linq;
 using Shoko.QueueProcessor.Abstractions;
 using Shoko.Server.Databases;
+using Shoko.Server.Extensions;
 using Shoko.Server.Models.AniDB;
 using Shoko.Server.Scheduling.Jobs.Actions;
 
@@ -20,6 +21,30 @@ public virtual List<AniDB_GroupStatus> GetByAnimeID(int id)
             .ToList();
     }
 
+    /// <summary>
+    /// Gets the cached group release statuses for multiple anime in a single batched query.
+    /// </summary>
+    /// <param name="animeIDs">The AniDB anime IDs to look up. Duplicates are ignored.</param>
+    /// <returns>A dictionary keyed by anime ID containing the group statuses for that anime. Anime without any statuses are omitted from the result.</returns>
+    public Dictionary<int, List<AniDB_GroupStatus>> GetByAnimeIDs(IEnumerable<int> animeIDs)
+    {
+        var idList = animeIDs.Distinct().ToList();
+        if (idList.Count == 0)
+            return new Dictionary<int, List<AniDB_GroupStatus>>();
+
+        using var session = _databaseFactory.SessionFactory.OpenStatelessSession();
+        var records = idList
+            .Batch(1000)
+            .SelectMany(batch => session.Query<AniDB_GroupStatus>()
+                .Where(a => batch.Contains(a.AnimeID))
+                .ToList())
+            .ToList();
+
+        return records
+            .GroupBy(a => a.AnimeID)
+            .ToDictionary(g => g.Key, g => g.ToList());
+    }
+
     public void DeleteForAnime(int animeid)
     {
         using var session = _databaseFactory.SessionFactory.OpenStatelessSession();

Shoko.Server/Scheduling/Jobs/AniDB/GetUpdatedAniDBAnimeJob.cs

@@ -3,9 +3,11 @@
 using Microsoft.Extensions.Logging;
 using Shoko.Abstractions.Metadata.Anidb.Enums;
 using Shoko.Abstractions.Metadata.Anidb.Services;
+using Shoko.QueueProcessor.Abstractions;
 using Shoko.QueueProcessor.Acquisition.Attributes;
 using Shoko.QueueProcessor.Builder;
 using Shoko.QueueProcessor.Concurrency;
+using Shoko.QueueProcessor.Scheduling;
 using Shoko.Server.Models.Internal;
 using Shoko.Server.Providers.AniDB.Interfaces;
 using Shoko.Server.Providers.AniDB.Titles;
@@ -25,7 +27,7 @@
 [AniDBUdpRateLimited]
 [DisallowConcurrencyGroup(ConcurrencyGroups.AniDB_UDP)]
 [JobKeyGroup(JobKeyGroup.AniDB)]
-public class GetUpdatedAniDBAnimeJob(IRequestFactory requestFactory, IAnidbService anidbService, ISettingsProvider settingsProvider, AniDBTitleHelper titleHelper, AniDB_AnimeRepository anidbAnimeRepository, AniDB_AnimeUpdateRepository anidbAnimeUpdates, AnimeSeriesRepository animeSeries, ScheduledUpdateRepository scheduledUpdates) : BaseJob
+public class GetUpdatedAniDBAnimeJob(IRequestFactory requestFactory, IAnidbService anidbService, ISettingsProvider settingsProvider, AniDBTitleHelper titleHelper, AniDB_AnimeRepository anidbAnimeRepository, AniDB_AnimeUpdateRepository anidbAnimeUpdates, AnimeSeriesRepository animeSeries, ScheduledUpdateRepository scheduledUpdates, IQueueScheduler scheduler) : BaseJob
 {
     public bool ForceRefresh { get; set; }
 
@@ -144,6 +146,9 @@
 
             await anidbService.ScheduleRefreshOfAnimeByID(animeID, AnidbRefreshMethod.Remote | AnidbRefreshMethod.DeferToRemoteIfUnsuccessful).ConfigureAwait(false);
             countSeries++;
+
+            await scheduler.StartJob<GetAniDBReleaseGroupStatusJob>(
+                c => (c.AnimeID, c.ForceRefresh) = (animeID, ForceRefresh));
         }
 
         return (response, countAnime, countSeries);