From 9f14014fa137ae8c00a97cdc55dafde24ab62abf Mon Sep 17 00:00:00 2001 From: Foowy <49217685+Foowy@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:53:57 +0000 Subject: [PATCH 01/13] repo(deps): build TMDbLib from source via git submodule Added jellyfin/TMDbLib master as a submodule at `ext/TMDbLib` (pinned to `4744732`) and swapped `Shoko.Server`'s `TMDbLib` `PackageReference` for a `ProjectReference` to it. TMDbLib `3.0.0` is the last published release; master carries a large TMDb v3 API refactor and a System.Text.Json migration that this branch needs to build support for ahead of the next release. Submodule has its own `Directory.Build.props`/`Directory.Packages.props`, so it builds isolated from `Shoko.Server`'s `TreatWarningsAsErrors`. --- .gitmodules | 3 +++ Shoko.Server/Shoko.Server.csproj | 2 +- ext/TMDbLib | 1 + 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 .gitmodules create mode 160000 ext/TMDbLib diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..f6f58c355 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "ext/TMDbLib"] + path = ext/TMDbLib + url = https://github.com/jellyfin/TMDbLib.git diff --git a/Shoko.Server/Shoko.Server.csproj b/Shoko.Server/Shoko.Server.csproj index d75548a98..7771988de 100644 --- a/Shoko.Server/Shoko.Server.csproj +++ b/Shoko.Server/Shoko.Server.csproj @@ -50,6 +50,7 @@ + @@ -94,7 +95,6 @@ - diff --git a/ext/TMDbLib b/ext/TMDbLib new file mode 160000 index 000000000..474473260 --- /dev/null +++ b/ext/TMDbLib @@ -0,0 +1 @@ +Subproject commit 474473260db829cb048d6ca9149cd6b289592dd7 From 484d719a7434f184d590959d2231af3514ed5261 Mon Sep 17 00:00:00 2001 From: Foowy <49217685+Foowy@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:53:57 +0000 Subject: [PATCH 02/13] fix(tmdb): align TvdbId external-id handling with int-typed TMDbLib ids `ExternalIdsTvShow.TvdbId` / `ExternalIdsTvEpisode.TvdbId` changed from `string?` to `int?` in TMDbLib master (TMDb returns `tvdb_id` as an integer). `UpdateShowExternalIDs` and `UpdateEpisodeExternalIDs` no longer need the string-parse step; `TMDB_Show.TvdbShowID` and `TMDB_Episode.TvdbEpisodeID` are already `int?`. --- .../Providers/TMDB/TmdbMetadataService.cs | 24 ++++--------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs index 1b248657e..fdbaa0f5c 100644 --- a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs +++ b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs @@ -2936,16 +2936,8 @@ private static async Task ProcessWithConcurrencyAsync( /// Indicates that the ID was updated. private bool UpdateShowExternalIDs(TMDB_Show show, ExternalIdsTvShow externalIds) { - if (string.IsNullOrEmpty(externalIds.TvdbId)) - { - if (!show.TvdbShowID.HasValue) - return false; - - show.TvdbShowID = null; - return true; - } - - if (!int.TryParse(externalIds.TvdbId, out var tvdbId) || tvdbId <= 0 || show.TvdbShowID == tvdbId) + var tvdbId = externalIds.TvdbId is > 0 ? externalIds.TvdbId : null; + if (show.TvdbShowID == tvdbId) return false; show.TvdbShowID = tvdbId; @@ -2960,16 +2952,8 @@ private bool UpdateShowExternalIDs(TMDB_Show show, ExternalIdsTvShow externalIds /// Indicates that the ID was updated. private bool UpdateEpisodeExternalIDs(TMDB_Episode episode, ExternalIdsTvEpisode externalIds) { - if (string.IsNullOrEmpty(externalIds.TvdbId)) - { - if (!episode.TvdbEpisodeID.HasValue) - return false; - - episode.TvdbEpisodeID = null; - return true; - } - - if (!int.TryParse(externalIds.TvdbId, out var tvdbId) || tvdbId <= 0 || episode.TvdbEpisodeID == tvdbId) + var tvdbId = externalIds.TvdbId is > 0 ? externalIds.TvdbId : null; + if (episode.TvdbEpisodeID == tvdbId) return false; episode.TvdbEpisodeID = tvdbId; From 1e1170df34518d1dfa7487656aab87977be0528a Mon Sep 17 00:00:00 2001 From: Foowy <49217685+Foowy@users.noreply.github.com> Date: Sat, 5 Sep 2026 23:13:16 +0000 Subject: [PATCH 03/13] fix(tmdb): route TMDbLib 5xx exception types through the retry policy TMDbLib master maps recognised 5xx responses to TMDbServerException and TMDbServiceUnavailableException; earlier versions surfaced them as GeneralHttpException. OnTmdbRetryAsync and _retryPolicy only knew the latter, so under the new library a 5xx would bypass the 5xx breaker and re-throw without calling Notify5xxError. Added both types to the retry policy and a matching case in OnTmdbRetryAsync. --- Shoko.Server/Providers/TMDB/TmdbMetadataService.cs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs index fdbaa0f5c..3b66fd14a 100644 --- a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs +++ b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs @@ -206,7 +206,8 @@ public TmdbRateLimitPauseStatus GetPauseStatus() } // Retries only on RequestLimitExceededException (cap: 10) and HttpRequestException timeouts (cap: 3). - // GeneralHttpException and all other types re-throw immediately in OnTmdbRetryAsync. + // 5xx types (TMDbServerException, TMDbServiceUnavailableException, GeneralHttpException >= 500) trip the + // 5xx breaker and re-throw; all other types re-throw immediately in OnTmdbRetryAsync. // Zero delay is intentional — actual rate-limit pausing happens inside TmdbRateLimiter.EnsureRateAsync. private readonly AsyncRetryPolicy _retryPolicy; @@ -241,6 +242,12 @@ private Task OnTmdbRetryAsync(Exception ex, TimeSpan ts, int retryCount, Context ctx["timeoutRetryCount"] = timeoutRetryCount + 1; break; } + // TMDbLib maps recognised 5xx responses to these dedicated types; older versions surfaced + // them as GeneralHttpException. Both paths must trip the 5xx breaker. + case TMDbServerException or TMDbServiceUnavailableException: + _logger.LogWarning(ex, "Got a server-side error from TMDb: {Message}", ex.Message); + _rateLimiter.Notify5xxError(); + throw ex; case GeneralHttpException ghEx: _logger.LogWarning(ghEx, "Got a general HTTP exception while processing TMDb request: {StatusCode}", (int)ghEx.HttpStatusCode); if ((int)ghEx.HttpStatusCode >= 500) @@ -372,6 +379,8 @@ TMDB_Show_NetworkRepository xrefTmdbShowNetwork .Handle() .Or() .Or() + .Or() + .Or() .WaitAndRetryAsync(int.MaxValue, (_, _) => TimeSpan.Zero, OnTmdbRetryAsync); } From bc57883697e9e6ee292680dd906c438d691b5a0e Mon Sep 17 00:00:00 2001 From: Foowy <49217685+Foowy@users.noreply.github.com> Date: Sat, 5 Sep 2026 23:13:16 +0000 Subject: [PATCH 04/13] fix(tmdb): guard null appended sub-resources in TMDb responses TMDbLib master, on System.Text.Json, can deserialize an absent or empty append_to_response section to null where Newtonsoft left an empty object. The provider dereferenced ExternalIds, Credits, translation locale codes and image lists with the null-forgiving operator, which would throw instead of degrading. - UpdateShowExternalIDs / UpdateEpisodeExternalIDs / UpdateMovieExternalIDs take a nullable argument; a null clears the stored id, matching the existing no-id path - UpdateEpisodeCastAndCrew / UpdateMovieCastAndCrew return a no-op when Cast or Crew is null, so a sparse response does not purge existing rows - translation entries without Iso_639_1 / Iso_3166_1 are skipped - image list arguments to DownloadImagesByType fall back to empty --- .../Providers/TMDB/TmdbMetadataService.cs | 73 +++++++++++-------- 1 file changed, 43 insertions(+), 30 deletions(-) diff --git a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs index 3b66fd14a..51209d200 100644 --- a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs +++ b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs @@ -561,10 +561,10 @@ public async Task UpdateMovie(TmdbMovieUpdateOptions options) var updated = tmdbMovie.Populate(movie, contentRatingLanguages); var (titlesUpdated, overviewsUpdated) = UpdateTitlesAndOverviewsWithTuple(tmdbMovie, movie.Translations, preferredTitleLanguages, preferredOverviewLanguages); updated = titlesUpdated || overviewsUpdated || updated; - updated = UpdateMovieExternalIDs(tmdbMovie, movie.ExternalIds!) || updated; + updated = UpdateMovieExternalIDs(tmdbMovie, movie.ExternalIds) || updated; updated = await UpdateCompanies(tmdbMovie, movie.ProductionCompanies!) || updated; if (downloadCrewAndCast) - updated = await UpdateMovieCastAndCrew(tmdbMovie, movie.Credits!, forceRefresh, downloadImages) || updated; + updated = await UpdateMovieCastAndCrew(tmdbMovie, movie.Credits, forceRefresh, downloadImages) || updated; if (updated) { tmdbMovie.LastUpdatedAt = DateTime.Now; @@ -599,8 +599,12 @@ public async Task UpdateMovie(TmdbMovieUpdateOptions options) } } - private async Task UpdateMovieCastAndCrew(TMDB_Movie tmdbMovie, MovieCredits credits, bool forceRefresh, bool downloadImages) + private async Task UpdateMovieCastAndCrew(TMDB_Movie tmdbMovie, MovieCredits? credits, bool forceRefresh, bool downloadImages) { + // See UpdateEpisodeCastAndCrew: a null/empty credits append is "no cast/crew this pass", not a purge. + if (credits?.Cast is null || credits.Crew is null) + return false; + var peopleToKeep = new HashSet(); var counter = 0; @@ -609,7 +613,7 @@ private async Task UpdateMovieCastAndCrew(TMDB_Movie tmdbMovie, MovieCredi var castToSave = new List(); var existingCastDict = _tmdbMovieCast.GetByTmdbMovieID(tmdbMovie.Id) .ToDictionary(cast => cast.TmdbCreditID); - foreach (var cast in credits.Cast!) + foreach (var cast in credits.Cast) { var ordering = counter++; peopleToKeep.Add(cast.Id); @@ -652,7 +656,7 @@ private async Task UpdateMovieCastAndCrew(TMDB_Movie tmdbMovie, MovieCredi var crewToSave = new List(); var existingCrewDict = _tmdbMovieCrew.GetByTmdbMovieID(tmdbMovie.Id) .ToDictionary(crew => crew.TmdbCreditID); - foreach (var crew in credits.Crew!) + foreach (var crew in credits.Crew) { peopleToKeep.Add(crew.Id); crewToKeep.Add(crew.CreditId!); @@ -897,11 +901,11 @@ private async Task DownloadMovieImages(int movieId, TitleLanguage? mainLanguage var languages = GetLanguages(mainLanguage); if (settings.TMDB.AutoDownloadPosters) - await _imageService.DownloadImagesByType(movie.PosterPath, images.Posters!, ImageEntityType.Primary, movie, settings.TMDB.MaxAutoPosters, languages, forceDownload); + await _imageService.DownloadImagesByType(movie.PosterPath, images.Posters ?? [], ImageEntityType.Primary, movie, settings.TMDB.MaxAutoPosters, languages, forceDownload); if (settings.TMDB.AutoDownloadLogos) - await _imageService.DownloadImagesByType(null, images.Logos!, ImageEntityType.Logo, movie, settings.TMDB.MaxAutoLogos, languages, forceDownload); + await _imageService.DownloadImagesByType(null, images.Logos ?? [], ImageEntityType.Logo, movie, settings.TMDB.MaxAutoLogos, languages, forceDownload); if (settings.TMDB.AutoDownloadBackdrops) - await _imageService.DownloadImagesByType(movie.BackdropPath, images.Backdrops!, ImageEntityType.Backdrop, movie, settings.TMDB.MaxAutoBackdrops, languages, forceDownload); + await _imageService.DownloadImagesByType(movie.BackdropPath, images.Backdrops ?? [], ImageEntityType.Backdrop, movie, settings.TMDB.MaxAutoBackdrops, languages, forceDownload); } #endregion @@ -1204,7 +1208,7 @@ public async Task UpdateShow(TmdbShowUpdateOptions options) var updated = tmdbShow.Populate(show, contentRatingLanguages); var (titlesUpdated, overviewsUpdated) = UpdateTitlesAndOverviewsWithTuple(tmdbShow, show.Translations, preferredTitleLanguages, preferredOverviewLanguages); updated = titlesUpdated || overviewsUpdated || updated; - updated = UpdateShowExternalIDs(tmdbShow, show.ExternalIds!) || updated; + updated = UpdateShowExternalIDs(tmdbShow, show.ExternalIds) || updated; updated = await UpdateCompanies(tmdbShow, show.ProductionCompanies!) || updated; var (episodesOrSeasonsUpdated, updatedSeasons, updatedEpisodes, episodeCount, hiddenEpisodeCount) = await UpdateShowSeasonsAndEpisodes(show, downloadCrewAndCast, forceRefresh, downloadImages, quickRefresh, shouldFireEvents, changedItems); updated = episodesOrSeasonsUpdated || updated; @@ -1530,10 +1534,10 @@ private async Task FetchAndPopulateEpisodeAsync( var episodeUpdated = tmdbEpisode.Populate(show, season, reducedEpisode, episode.Translations); episodeUpdated = UpdateTitlesAndOverviews(tmdbEpisode, episode.Translations, state.PreferredTitleLanguages, state.PreferredOverviewLanguages) || episodeUpdated; - episodeUpdated = UpdateEpisodeExternalIDs(tmdbEpisode, episode.ExternalIds!) || episodeUpdated; + episodeUpdated = UpdateEpisodeExternalIDs(tmdbEpisode, episode.ExternalIds) || episodeUpdated; if (state.DownloadCrewAndCast) { - var (castOrCrewUpdated, peopleToAddOrKeep, peopleToPotentiallyRemove) = UpdateEpisodeCastAndCrew(tmdbEpisode, episode.Credits!); + var (castOrCrewUpdated, peopleToAddOrKeep, peopleToPotentiallyRemove) = UpdateEpisodeCastAndCrew(tmdbEpisode, episode.Credits); episodeUpdated |= castOrCrewUpdated; AccumulateEpisodePeople(peopleToAddOrKeep, peopleToPotentiallyRemove, state); } @@ -1809,8 +1813,13 @@ private async Task UpdateShowAlternateOrdering(TMDB_Show tmdbShow, TvShow preferredOrderingUpdated; } - private (bool, IEnumerable, IEnumerable) UpdateEpisodeCastAndCrew(TMDB_Episode tmdbEpisode, CreditsWithGuestStars credits) + private (bool, IEnumerable, IEnumerable) UpdateEpisodeCastAndCrew(TMDB_Episode tmdbEpisode, CreditsWithGuestStars? credits) { + // A present-but-empty `credits` append can deserialize to null members under STJ; + // treat that as "no cast/crew this pass" rather than purging existing rows. + if (credits?.Cast is null || credits.Crew is null) + return (false, [], []); + var peopleToAddOrKeep = new HashSet(); var counter = 0; var castToAdd = 0; @@ -1818,8 +1827,8 @@ private async Task UpdateShowAlternateOrdering(TMDB_Show tmdbShow, TvShow var castToSave = new List(); var existingCastDict = _tmdbEpisodeCast.GetByTmdbEpisodeID(tmdbEpisode.Id) .ToDictionary(cast => cast.TmdbCreditID); - var guestOffset = credits.Cast!.Count; - foreach (var cast in credits.Cast.Concat(credits.GuestStars!)) + var guestOffset = credits.Cast.Count; + foreach (var cast in credits.Cast.Concat(credits.GuestStars ?? [])) { var ordering = counter++; var isGuestRole = ordering >= guestOffset; @@ -1873,7 +1882,7 @@ private async Task UpdateShowAlternateOrdering(TMDB_Show tmdbShow, TvShow var crewToSave = new List(); var existingCrewDict = _tmdbEpisodeCrew.GetByTmdbEpisodeID(tmdbEpisode.Id) .ToDictionary(crew => crew.TmdbCreditID); - foreach (var crew in credits.Crew!) + foreach (var crew in credits.Crew) { peopleToAddOrKeep.Add(crew.Id); crewToKeep.Add(crew.CreditId!); @@ -2089,11 +2098,11 @@ private async Task DownloadShowImages(int showId, TitleLanguage? mainLanguage = var languages = GetLanguages(mainLanguage); if (settings.TMDB.AutoDownloadPosters) - await _imageService.DownloadImagesByType(show.PosterPath, images.Posters!, ImageEntityType.Primary, show, settings.TMDB.MaxAutoPosters, languages, forceDownload); + await _imageService.DownloadImagesByType(show.PosterPath, images.Posters ?? [], ImageEntityType.Primary, show, settings.TMDB.MaxAutoPosters, languages, forceDownload); if (settings.TMDB.AutoDownloadLogos) - await _imageService.DownloadImagesByType(null, images.Logos!, ImageEntityType.Logo, show, settings.TMDB.MaxAutoLogos, languages, forceDownload); + await _imageService.DownloadImagesByType(null, images.Logos ?? [], ImageEntityType.Logo, show, settings.TMDB.MaxAutoLogos, languages, forceDownload); if (settings.TMDB.AutoDownloadBackdrops) - await _imageService.DownloadImagesByType(show.BackdropPath, images.Backdrops!, ImageEntityType.Backdrop, show, settings.TMDB.MaxAutoBackdrops, languages, forceDownload); + await _imageService.DownloadImagesByType(show.BackdropPath, images.Backdrops ?? [], ImageEntityType.Backdrop, show, settings.TMDB.MaxAutoBackdrops, languages, forceDownload); } private async Task DownloadSeasonImages(int seasonId, int showId, int seasonNumber, TitleLanguage? mainLanguage = null, bool forceDownload = false) @@ -2112,7 +2121,7 @@ private async Task DownloadSeasonImages(int seasonId, int showId, int seasonNumb return; var languages = GetLanguages(mainLanguage); - await _imageService.DownloadImagesByType(season.PosterPath, images.Posters!, ImageEntityType.Primary, season, settings.TMDB.MaxAutoPosters, languages, forceDownload); + await _imageService.DownloadImagesByType(season.PosterPath, images.Posters ?? [], ImageEntityType.Primary, season, settings.TMDB.MaxAutoPosters, languages, forceDownload); } private async Task DownloadEpisodeImages(int episodeId, int showId, int seasonNumber, int episodeNumber, TitleLanguage mainLanguage, bool forceDownload = false) @@ -2131,7 +2140,7 @@ private async Task DownloadEpisodeImages(int episodeId, int showId, int seasonNu return; var languages = GetLanguages(mainLanguage); - await _imageService.DownloadImagesByType(episode.ThumbnailPath, images.Stills!, ImageEntityType.Backdrop, episode, settings.TMDB.MaxAutoThumbnails, languages, forceDownload); + await _imageService.DownloadImagesByType(episode.ThumbnailPath, images.Stills ?? [], ImageEntityType.Backdrop, episode, settings.TMDB.MaxAutoThumbnails, languages, forceDownload); } private List GetLanguages(TitleLanguage? mainLanguage = null) => _settingsProvider.GetSettings().TMDB.ImageLanguageOrder @@ -2417,8 +2426,12 @@ private bool UpdateTitlesAndOverviews(IEntityMetadata tmdbEntity, TranslationsCo var titlesToSave = new List(); foreach (var translation in translations?.Translations ?? [new() { EnglishName = string.Empty, Iso_3166_1 = "US", Iso_639_1 = "en", Data = new() { Name = string.Empty, Overview = string.Empty } }]) { - var languageCode = translation.Iso_639_1!.ToLowerInvariant(); - var countryCode = translation.Iso_3166_1!.ToUpperInvariant(); + // A translation entry without locale codes is unusable; skip rather than NRE. + if (translation.Iso_639_1 is null || translation.Iso_3166_1 is null) + continue; + + var languageCode = translation.Iso_639_1.ToLowerInvariant(); + var countryCode = translation.Iso_3166_1.ToUpperInvariant(); var alwaysInclude = false; var currentTitle = translation.Data?.Name ?? string.Empty; @@ -2746,7 +2759,7 @@ private async Task DownloadPersonImages(int personId, ProfileImages images, bool if (_tmdbPeople.GetByTmdbPersonID(personId) is not { } person) return; - await _imageService.DownloadImagesByType(null, images.Profiles!, ImageEntityType.Primary, person, settings.TMDB.MaxAutoStaffImages, [], forceDownload); + await _imageService.DownloadImagesByType(null, images.Profiles ?? [], ImageEntityType.Primary, person, settings.TMDB.MaxAutoStaffImages, [], forceDownload); } public async Task PurgeUnlinkedPeople() @@ -2943,9 +2956,9 @@ private static async Task ProcessWithConcurrencyAsync( /// TMDB Show. /// External IDs. /// Indicates that the ID was updated. - private bool UpdateShowExternalIDs(TMDB_Show show, ExternalIdsTvShow externalIds) + private bool UpdateShowExternalIDs(TMDB_Show show, ExternalIdsTvShow? externalIds) { - var tvdbId = externalIds.TvdbId is > 0 ? externalIds.TvdbId : null; + var tvdbId = externalIds?.TvdbId is > 0 ? externalIds.TvdbId : null; if (show.TvdbShowID == tvdbId) return false; @@ -2959,9 +2972,9 @@ private bool UpdateShowExternalIDs(TMDB_Show show, ExternalIdsTvShow externalIds /// TMDB Episode. /// External IDs. /// Indicates that the ID was updated. - private bool UpdateEpisodeExternalIDs(TMDB_Episode episode, ExternalIdsTvEpisode externalIds) + private bool UpdateEpisodeExternalIDs(TMDB_Episode episode, ExternalIdsTvEpisode? externalIds) { - var tvdbId = externalIds.TvdbId is > 0 ? externalIds.TvdbId : null; + var tvdbId = externalIds?.TvdbId is > 0 ? externalIds.TvdbId : null; if (episode.TvdbEpisodeID == tvdbId) return false; @@ -2975,12 +2988,12 @@ private bool UpdateEpisodeExternalIDs(TMDB_Episode episode, ExternalIdsTvEpisode /// TMDB Movie. /// External IDs. /// Indicates that the ID was updated. - private bool UpdateMovieExternalIDs(TMDB_Movie movie, ExternalIdsMovie externalIds) + private bool UpdateMovieExternalIDs(TMDB_Movie movie, ExternalIdsMovie? externalIds) { - if (movie.ImdbMovieID == externalIds.ImdbId) + if (movie.ImdbMovieID == externalIds?.ImdbId) return false; - movie.ImdbMovieID = externalIds.ImdbId; + movie.ImdbMovieID = externalIds?.ImdbId; return true; } From a03447c77f979aa9c70c1eb45070809ffb0da9af Mon Sep 17 00:00:00 2001 From: Foowy <49217685+Foowy@users.noreply.github.com> Date: Sun, 6 Sep 2026 12:35:45 +0000 Subject: [PATCH 05/13] repo(workflows): fetch the TMDbLib submodule in integration-tests CI The `tmdblib-update` branch builds `TMDbLib` from `ext/TMDbLib` as a `ProjectReference`. `integration-tests.yml` compiles `Shoko.Server` via `dotnet test Shoko.IntegrationTests` and is the only workflow that runs on this branch, so its `actions/checkout` steps need `submodules: recursive` or the build fails to restore `TMDbLib`. Reverted by the package-swap runbook (T026) before this lands on `master`. --- .github/workflows/integration-tests.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b94ef6e95..7f9e656b9 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -16,6 +16,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 + with: + submodules: recursive - name: Install system dependencies run: sudo apt-get install -y mediainfo librhash-dev @@ -58,6 +60,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 + with: + submodules: recursive - name: Install system dependencies run: sudo apt-get install -y mediainfo librhash-dev @@ -105,6 +109,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v7 + with: + submodules: recursive - name: Install system dependencies run: sudo apt-get install -y mediainfo librhash-dev From 4c5eece69d51ba85189018fa776f76dfbe96dbc2 Mon Sep 17 00:00:00 2001 From: Foowy <49217685+Foowy@users.noreply.github.com> Date: Sun, 6 Sep 2026 12:41:54 +0000 Subject: [PATCH 06/13] test(tmdb): cover append_to_response binding under the TMDbLib STJ migration jellyfin/TMDbLib#626 plus the System.Text.Json migration change how appended sub-resources deserialize. The reference-library re-scan already proves request and row parity end to end, so this fills the one gap it does not isolate: the wire-to-model binding. Replays the `append_to_response` payload shapes for every request type `TmdbMetadataService` issues through `TMDbJsonSerializer` and asserts each sub-resource Shoko consumes binds, that `ExternalIdsTvShow`/`ExternalIdsTvEpisode` `TvdbId` reads back as an integer, and that absent and explicit-null sub-resources both land as `null` rather than empty containers. --- ...mdbAppendToResponseDeserializationTests.cs | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 Shoko.Tests/Providers/TMDB/TmdbAppendToResponseDeserializationTests.cs diff --git a/Shoko.Tests/Providers/TMDB/TmdbAppendToResponseDeserializationTests.cs b/Shoko.Tests/Providers/TMDB/TmdbAppendToResponseDeserializationTests.cs new file mode 100644 index 000000000..b3d1c69f7 --- /dev/null +++ b/Shoko.Tests/Providers/TMDB/TmdbAppendToResponseDeserializationTests.cs @@ -0,0 +1,164 @@ +using TMDbLib.Objects.Movies; +using TMDbLib.Objects.TvShows; +using TMDbLib.Utilities.Serializer; +using Xunit; + +namespace Shoko.Tests.Providers.TMDB; + +/// +/// Guards the TMDbLib master upgrade (jellyfin/TMDbLib#626 + System.Text.Json migration). +/// The upgrade keeps the exact same TMDB requests (verified by the reference-library +/// re-scan, tasks T018/T023 — identical request counts and byte-for-byte row parity), so +/// the only new risk is the wire→model binding: STJ presence handling of appended +/// sub-resources and the tvdb_id type change (string → int). These replay the +/// append_to_response payload shapes that +/// requests and assert each sub-resource Shoko reads still deserializes. +/// +public class TmdbAppendToResponseDeserializationTests +{ + private static T Deserialize(string json) where T : class + => TMDbJsonSerializer.Instance.DeserializeFromString(json) + ?? throw new Xunit.Sdk.XunitException($"Deserialized {typeof(T).Name} was null"); + + // Movie: MovieMethods.Translations | ReleaseDates | ExternalIds | Keywords | Credits + private const string MovieJson = """ + { + "id": 149, + "title": "Akira", + "translations": { "id": 149, "translations": [ + { "iso_639_1": "ja", "iso_3166_1": "JP", "name": "日本語", "english_name": "Japanese", + "data": { "title": "AKIRA", "overview": "あらすじ" } } ] }, + "release_dates": { "results": [ + { "iso_3166_1": "JP", "release_dates": [ { "certification": "", "type": 3, "release_date": "1988-07-16T00:00:00.000Z" } ] } ] }, + "external_ids": { "imdb_id": "tt0094625", "facebook_id": null, "twitter_id": null }, + "keywords": { "keywords": [ { "id": 970, "name": "anime" }, { "id": 9951, "name": "dystopia" } ] }, + "credits": { "cast": [ { "id": 1, "name": "A" } ], "crew": [ { "id": 2, "name": "B" } ] } + } + """; + + // TV show: TvShowMethods.ContentRatings | Translations | ExternalIds | Keywords | EpisodeGroups + private const string ShowJson = """ + { + "id": 45782, + "name": "Sword Art Online", + "content_ratings": { "results": [ { "iso_3166_1": "US", "rating": "TV-14" } ] }, + "translations": { "id": 45782, "translations": [ + { "iso_639_1": "de", "iso_3166_1": "DE", "name": "Deutsch", "english_name": "German", + "data": { "name": "Sword Art Online", "overview": "In naher Zukunft..." } } ] }, + "external_ids": { "imdb_id": "tt2250192", "tvdb_id": 259140, "facebook_id": null }, + "keywords": { "results": [ { "id": 6091, "name": "war" } ] }, + "episode_groups": { "results": [ + { "id": "5f8a1d...", "name": "Alternate order", "group_count": 4, "episode_count": 96, "type": 2 } ] } + } + """; + + // TV episode: TvEpisodeMethods.ExternalIds | Translations | Credits + private const string EpisodeJson = """ + { + "id": 979329, + "season_number": 1, + "episode_number": 1, + "name": "The World of Swords", + "external_ids": { "imdb_id": "tt2683910", "tvdb_id": 4298471 }, + "translations": { "id": 979329, "translations": [ + { "iso_639_1": "ja", "iso_3166_1": "JP", "name": "", "english_name": "Japanese", + "data": { "name": "剣の世界", "overview": "" } } ] }, + "credits": { "cast": [ { "id": 1, "name": "A" } ], "crew": [], "guest_stars": [ { "id": 3, "name": "G" } ] } + } + """; + + // TV season: TvSeasonMethods.Translations (episode stubs come back on the season body) + private const string SeasonJson = """ + { + "id": 61862, + "season_number": 1, + "episodes": [ + { "id": 979329, "season_number": 1, "episode_number": 1, "name": "Ep1" }, + { "id": 979330, "season_number": 1, "episode_number": 2, "name": "Ep2" } + ], + "translations": { "id": 61862, "translations": [ + { "iso_639_1": "fr", "iso_3166_1": "FR", "name": "Français", "english_name": "French", + "data": { "name": "Aincrad", "overview": "..." } } ] } + } + """; + + [Fact] + public void Movie_AllAppendedSubResources_Bind() + { + var movie = Deserialize(MovieJson); + + Assert.Equal("Akira", movie.Title); + Assert.Equal("ja", Assert.Single(movie.Translations!.Translations!).Iso_639_1); + Assert.Equal("あらすじ", movie.Translations!.Translations![0].Data!.Overview); + Assert.Equal("JP", Assert.Single(movie.ReleaseDates!.Results!).Iso_3166_1); + Assert.Equal("tt0094625", movie.ExternalIds!.ImdbId); + Assert.Equal(2, movie.Keywords!.Keywords!.Count); + Assert.Single(movie.Credits!.Cast!); + Assert.Single(movie.Credits!.Crew!); + } + + [Fact] + public void Show_AllAppendedSubResources_Bind() + { + var show = Deserialize(ShowJson); + + Assert.Equal("Sword Art Online", show.Name); + Assert.Equal("TV-14", Assert.Single(show.ContentRatings!.Results!).Rating); + Assert.Equal("de", Assert.Single(show.Translations!.Translations!).Iso_639_1); + Assert.Equal("tt2250192", show.ExternalIds!.ImdbId); + Assert.Single(show.Keywords!.Results!); + Assert.Equal(4, Assert.Single(show.EpisodeGroups!.Results!).GroupCount); + } + + [Fact] + public void Show_TvdbId_DeserializesAsInteger() + { + // jellyfin/TMDbLib#626: ExternalIdsTvShow.TvdbId went string? -> int? (TMDB sends it as a + // JSON number). UpdateShowExternalIDs / UpdateEpisodeExternalIDs rely on this — see task T022. + var show = Deserialize(ShowJson); + Assert.Equal(259140, show.ExternalIds!.TvdbId); + + var episode = Deserialize(EpisodeJson); + Assert.Equal(4298471, episode.ExternalIds!.TvdbId); + } + + [Fact] + public void Episode_AllAppendedSubResources_Bind() + { + var episode = Deserialize(EpisodeJson); + + Assert.Equal(1, episode.SeasonNumber); + Assert.Equal(1, episode.EpisodeNumber); + Assert.Equal("tt2683910", episode.ExternalIds!.ImdbId); + Assert.Equal("ja", Assert.Single(episode.Translations!.Translations!).Iso_639_1); + Assert.Single(episode.Credits!.Cast!); + Assert.Empty(episode.Credits!.Crew!); + Assert.Single(episode.Credits!.GuestStars!); + } + + [Fact] + public void Season_TranslationsAndEpisodeStubs_Bind() + { + var season = Deserialize(SeasonJson); + + Assert.Equal(1, season.SeasonNumber); + Assert.Equal(2, season.Episodes!.Count); + Assert.Equal("fr", Assert.Single(season.Translations!.Translations!).Iso_639_1); + } + + [Theory] + [InlineData("""{ "id": 149, "title": "Akira" }""")] + [InlineData("""{ "id": 149, "title": "Akira", "external_ids": null, "keywords": null, "credits": null, "translations": null, "release_dates": null }""")] + public void Movie_MissingOrNullSubResources_LeaveContainersNull(string json) + { + // STJ presence handling: absent and explicit-null must both land as null, not empty + // containers — TmdbMetadataService's null-guards (task T016) key off exactly this. + var movie = Deserialize(json); + + Assert.Null(movie.Translations); + Assert.Null(movie.ReleaseDates); + Assert.Null(movie.ExternalIds); + Assert.Null(movie.Keywords); + Assert.Null(movie.Credits); + } +} From b44eab99498fc35415c0a4256d50b7f0c891e6c8 Mon Sep 17 00:00:00 2001 From: Foowy <49217685+Foowy@users.noreply.github.com> Date: Sun, 6 Sep 2026 12:47:43 +0000 Subject: [PATCH 07/13] test(tmdb): pin external-id handlers to the post-upgrade int tvdb_id type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ExternalIdsTvShow/TvEpisode.TvdbId` became `int?` in TMDbLib master (jellyfin/TMDbLib#626). Made `UpdateShowExternalIDs`, `UpdateEpisodeExternalIDs` and `UpdateMovieExternalIDs` `internal static` — they are pure functions over their arguments and `Shoko.Server` already grants internals to `Shoko.Tests` — so the store/clear contract is covered directly: positive id stored, zero / negative / null / null-container cleared, unchanged value reported as no-op, IMDb id stored verbatim. Also extends the deserialization tests with `SearchTv`/`SearchMovie.GenreIds`, the only wire surface the upgrade touches for restricted auto-match. --- .../Providers/TMDB/TmdbMetadataService.cs | 6 +- ...mdbAppendToResponseDeserializationTests.cs | 13 +++ .../Providers/TMDB/TmdbExternalIdTests.cs | 91 +++++++++++++++++++ 3 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 Shoko.Tests/Providers/TMDB/TmdbExternalIdTests.cs diff --git a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs index 51209d200..dc49bbdc6 100644 --- a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs +++ b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs @@ -2956,7 +2956,7 @@ private static async Task ProcessWithConcurrencyAsync( /// TMDB Show. /// External IDs. /// Indicates that the ID was updated. - private bool UpdateShowExternalIDs(TMDB_Show show, ExternalIdsTvShow? externalIds) + internal static bool UpdateShowExternalIDs(TMDB_Show show, ExternalIdsTvShow? externalIds) { var tvdbId = externalIds?.TvdbId is > 0 ? externalIds.TvdbId : null; if (show.TvdbShowID == tvdbId) @@ -2972,7 +2972,7 @@ private bool UpdateShowExternalIDs(TMDB_Show show, ExternalIdsTvShow? externalId /// TMDB Episode. /// External IDs. /// Indicates that the ID was updated. - private bool UpdateEpisodeExternalIDs(TMDB_Episode episode, ExternalIdsTvEpisode? externalIds) + internal static bool UpdateEpisodeExternalIDs(TMDB_Episode episode, ExternalIdsTvEpisode? externalIds) { var tvdbId = externalIds?.TvdbId is > 0 ? externalIds.TvdbId : null; if (episode.TvdbEpisodeID == tvdbId) @@ -2988,7 +2988,7 @@ private bool UpdateEpisodeExternalIDs(TMDB_Episode episode, ExternalIdsTvEpisode /// TMDB Movie. /// External IDs. /// Indicates that the ID was updated. - private bool UpdateMovieExternalIDs(TMDB_Movie movie, ExternalIdsMovie? externalIds) + internal static bool UpdateMovieExternalIDs(TMDB_Movie movie, ExternalIdsMovie? externalIds) { if (movie.ImdbMovieID == externalIds?.ImdbId) return false; diff --git a/Shoko.Tests/Providers/TMDB/TmdbAppendToResponseDeserializationTests.cs b/Shoko.Tests/Providers/TMDB/TmdbAppendToResponseDeserializationTests.cs index b3d1c69f7..ea33aff25 100644 --- a/Shoko.Tests/Providers/TMDB/TmdbAppendToResponseDeserializationTests.cs +++ b/Shoko.Tests/Providers/TMDB/TmdbAppendToResponseDeserializationTests.cs @@ -1,4 +1,5 @@ using TMDbLib.Objects.Movies; +using TMDbLib.Objects.Search; using TMDbLib.Objects.TvShows; using TMDbLib.Utilities.Serializer; using Xunit; @@ -136,6 +137,18 @@ public void Episode_AllAppendedSubResources_Bind() Assert.Single(episode.Credits!.GuestStars!); } + [Fact] + public void SearchResults_GenreIds_Bind() + { + // TmdbSearchService's restricted auto-match + animation-genre ordering read + // SearchTv/SearchMovie.GenreIds (task T029). Confirm STJ still binds genre_ids. + var show = Deserialize("""{ "id": 45782, "name": "SAO", "genre_ids": [16, 10765] }"""); + Assert.Equal(new[] { 16, 10765 }, show.GenreIds); + + var movie = Deserialize("""{ "id": 149, "title": "Akira", "genre_ids": [16, 28, 878] }"""); + Assert.Equal(new[] { 16, 28, 878 }, movie.GenreIds); + } + [Fact] public void Season_TranslationsAndEpisodeStubs_Bind() { diff --git a/Shoko.Tests/Providers/TMDB/TmdbExternalIdTests.cs b/Shoko.Tests/Providers/TMDB/TmdbExternalIdTests.cs new file mode 100644 index 000000000..d830eb1cf --- /dev/null +++ b/Shoko.Tests/Providers/TMDB/TmdbExternalIdTests.cs @@ -0,0 +1,91 @@ +using Shoko.Server.Models.TMDB; +using Shoko.Server.Providers.TMDB; +using TMDbLib.Objects.General; +using Xunit; + +namespace Shoko.Tests.Providers.TMDB; + +/// +/// User Story 2: external-ID cross-links stay correct after the TMDbLib upgrade. +/// ExternalIdsTvShow/TvEpisode.TvdbId changed from string? to int? +/// (jellyfin/TMDbLib#626); these pin the Update*ExternalIDs handlers to the +/// post-upgrade type and the "clear when absent / non-positive" contract. +/// +public class TmdbExternalIdTests +{ + [Theory] + [InlineData(259140, 259140)] + [InlineData(1, 1)] + [InlineData(0, null)] + [InlineData(-5, null)] + [InlineData(null, null)] + public void UpdateShowExternalIDs_StoresPositiveTvdbIdElseNull(int? wireTvdbId, int? expected) + { + var show = new TMDB_Show(); + var changed = TmdbMetadataService.UpdateShowExternalIDs(show, new ExternalIdsTvShow { TvdbId = wireTvdbId }); + + Assert.Equal(expected, show.TvdbShowID); + Assert.Equal(expected is not null, changed); + } + + [Theory] + [InlineData(4298471, 4298471)] + [InlineData(0, null)] + [InlineData(null, null)] + public void UpdateEpisodeExternalIDs_StoresPositiveTvdbIdElseNull(int? wireTvdbId, int? expected) + { + var episode = new TMDB_Episode(); + var changed = TmdbMetadataService.UpdateEpisodeExternalIDs(episode, new ExternalIdsTvEpisode { TvdbId = wireTvdbId }); + + Assert.Equal(expected, episode.TvdbEpisodeID); + Assert.Equal(expected is not null, changed); + } + + [Fact] + public void UpdateShowExternalIDs_PreviouslySetThenRemoved_IsCleared() + { + var show = new TMDB_Show { TvdbShowID = 259140 }; + + Assert.True(TmdbMetadataService.UpdateShowExternalIDs(show, new ExternalIdsTvShow { TvdbId = null })); + Assert.Null(show.TvdbShowID); + } + + [Fact] + public void UpdateShowExternalIDs_UnchangedValue_ReturnsFalse() + { + var show = new TMDB_Show { TvdbShowID = 259140 }; + + Assert.False(TmdbMetadataService.UpdateShowExternalIDs(show, new ExternalIdsTvShow { TvdbId = 259140 })); + Assert.Equal(259140, show.TvdbShowID); + } + + [Fact] + public void UpdateShowExternalIDs_NullContainer_ClearsWithoutThrowing() + { + var show = new TMDB_Show { TvdbShowID = 259140 }; + + Assert.True(TmdbMetadataService.UpdateShowExternalIDs(show, null)); + Assert.Null(show.TvdbShowID); + } + + [Theory] + [InlineData("tt0094625")] + [InlineData(null)] + public void UpdateMovieExternalIDs_ImdbId_StoredVerbatim(string? imdbId) + { + var movie = new TMDB_Movie(); + var changed = TmdbMetadataService.UpdateMovieExternalIDs(movie, new ExternalIdsMovie { ImdbId = imdbId }); + + Assert.Equal(imdbId, movie.ImdbMovieID); + Assert.Equal(imdbId is not null, changed); + } + + [Fact] + public void UpdateMovieExternalIDs_NullContainer_ClearsWithoutThrowing() + { + var movie = new TMDB_Movie { ImdbMovieID = "tt0094625" }; + + Assert.True(TmdbMetadataService.UpdateMovieExternalIDs(movie, null)); + Assert.Null(movie.ImdbMovieID); + } +} From fdaf937d4725ae341f85ab3e18dcc9891ee17de5 Mon Sep 17 00:00:00 2001 From: Foowy <49217685+Foowy@users.noreply.github.com> Date: Sun, 6 Sep 2026 13:09:23 +0000 Subject: [PATCH 08/13] feat(tmdb): store TMDB's episode_type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TMDbLib master exposes `episode_type` (`standard` / `finale` / `mid_season` / `special`) on every episode; before this Shoko had no episode-type signal from TMDB at all. - New nullable `TMDB_Episode.TmdbEpisodeType` column (SQLite 164 / MySQL 185 / SQL Server 182), populated verbatim (trimmed, empty maps to null) in the existing per-episode update. No backfill: it fills in on the next TMDB refresh of each show, no operator action needed. - Surfaced on the v3 `TmdbEpisode` DTO. The value is captured and exposed only. It is deliberately not wired into episode matching or `IEpisode.Type` — that is a match-selection change and belongs in its own spec with its own parity gate. --- .../API/v3/Models/TMDB/TmdbEpisode.cs | 9 +++ Shoko.Server/Databases/MySQL.cs | 1 + Shoko.Server/Databases/SQLServer.cs | 1 + Shoko.Server/Databases/SQLite.cs | 1 + Shoko.Server/Mappings/TMDB/TMDB_EpisodeMap.cs | 1 + Shoko.Server/Models/TMDB/TMDB_Episode.cs | 8 +++ .../Providers/TMDB/TmdbEpisodeTypeTests.cs | 57 +++++++++++++++++++ 7 files changed, 78 insertions(+) create mode 100644 Shoko.Tests/Providers/TMDB/TmdbEpisodeTypeTests.cs diff --git a/Shoko.Server/API/v3/Models/TMDB/TmdbEpisode.cs b/Shoko.Server/API/v3/Models/TMDB/TmdbEpisode.cs index 0dc64b695..c8c027129 100644 --- a/Shoko.Server/API/v3/Models/TMDB/TmdbEpisode.cs +++ b/Shoko.Server/API/v3/Models/TMDB/TmdbEpisode.cs @@ -105,6 +105,14 @@ public class TmdbEpisode /// public TimeSpan? Runtime { get; init; } + /// + /// TMDB's own episode classification (standard, finale, + /// mid_season, special), verbatim from the API. null + /// until the episode has been refreshed since the field was added. + /// + [JsonProperty(NullValueHandling = NullValueHandling.Ignore)] + public string? TmdbEpisodeType { get; init; } + /// /// All images stored locally for this episode, if any. /// @@ -204,6 +212,7 @@ public TmdbEpisode(TMDB_Show show, TMDB_Episode episode, TMDB_AlternateOrdering_ Source = "TMDB", }; Runtime = episode.Runtime; + TmdbEpisodeType = episode.TmdbEpisodeType; if (include.HasFlag(IncludeDetails.Images)) Images = ((IWithImages)episode).GetImages() .InLanguage(language) diff --git a/Shoko.Server/Databases/MySQL.cs b/Shoko.Server/Databases/MySQL.cs index 8ba31c147..bbe0c47b5 100644 --- a/Shoko.Server/Databases/MySQL.cs +++ b/Shoko.Server/Databases/MySQL.cs @@ -1309,6 +1309,7 @@ WHERE sri.`CrossReferences` LIKE '%AnidbEpisodeID%' new(187, 39, "CREATE INDEX `IX_CrossRef_AniDB_Anilist_Episode_AnilistEpisodeID` ON `CrossRef_AniDB_Anilist_Episode`(`AnilistEpisodeID`);"), new(187, 40, "CREATE TABLE `Anilist_Anime_ExternalLink` ( `Anilist_Anime_ExternalLinkID` INT NOT NULL AUTO_INCREMENT, `AnilistAnimeID` INT NOT NULL, `AnilistLinkID` INT NOT NULL, `Url` VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, `Site` VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, `AnilistSiteID` INT NULL, `LinkType` VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, `LanguageCode` VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL, PRIMARY KEY (`Anilist_Anime_ExternalLinkID`) );"), new(187, 41, "CREATE INDEX `IX_Anilist_Anime_ExternalLink_AnilistAnimeID` ON `Anilist_Anime_ExternalLink`(`AnilistAnimeID`);"), + new(188, 1, "ALTER TABLE `TMDB_Episode` ADD COLUMN `TmdbEpisodeType` VARCHAR(50) NULL DEFAULT NULL;"), ]; #endregion diff --git a/Shoko.Server/Databases/SQLServer.cs b/Shoko.Server/Databases/SQLServer.cs index a0a78a795..249448cf6 100644 --- a/Shoko.Server/Databases/SQLServer.cs +++ b/Shoko.Server/Databases/SQLServer.cs @@ -1189,6 +1189,7 @@ WHERE sri.CrossReferences LIKE '%AnidbEpisodeID%' new(185, 39, "CREATE INDEX IX_CrossRef_AniDB_Anilist_Episode_AnilistEpisodeID ON CrossRef_AniDB_Anilist_Episode(AnilistEpisodeID);"), new(185, 40, "CREATE TABLE Anilist_Anime_ExternalLink ( Anilist_Anime_ExternalLinkID INT IDENTITY(1,1) NOT NULL, AnilistAnimeID INT NOT NULL, AnilistLinkID INT NOT NULL, Url NVARCHAR(512) NOT NULL, Site NVARCHAR(128) NOT NULL, AnilistSiteID INT NULL, LinkType NVARCHAR(32) NOT NULL, LanguageCode NVARCHAR(32) NULL, CONSTRAINT PK_Anilist_Anime_ExternalLink PRIMARY KEY CLUSTERED (Anilist_Anime_ExternalLinkID) );"), new(185, 41, "CREATE INDEX IX_Anilist_Anime_ExternalLink_AnilistAnimeID ON Anilist_Anime_ExternalLink(AnilistAnimeID);"), + new(186, 1, "ALTER TABLE TMDB_Episode ADD TmdbEpisodeType NVARCHAR(50) NULL DEFAULT NULL;"), ]; #endregion diff --git a/Shoko.Server/Databases/SQLite.cs b/Shoko.Server/Databases/SQLite.cs index 11b0335e3..ccdcec132 100644 --- a/Shoko.Server/Databases/SQLite.cs +++ b/Shoko.Server/Databases/SQLite.cs @@ -1000,6 +1000,7 @@ WHERE CrossReferences LIKE '%AnidbEpisodeID%' new(166, 39, "CREATE INDEX IX_CrossRef_AniDB_Anilist_Episode_AnilistEpisodeID ON CrossRef_AniDB_Anilist_Episode(AnilistEpisodeID);"), new(166, 40, "CREATE TABLE Anilist_Anime_ExternalLink ( Anilist_Anime_ExternalLinkID INTEGER PRIMARY KEY AUTOINCREMENT, AnilistAnimeID INTEGER NOT NULL, AnilistLinkID INTEGER NOT NULL, Url TEXT NOT NULL, Site TEXT NOT NULL, AnilistSiteID INTEGER NULL, LinkType TEXT NOT NULL, LanguageCode TEXT NULL );"), new(166, 41, "CREATE INDEX IX_Anilist_Anime_ExternalLink_AnilistAnimeID ON Anilist_Anime_ExternalLink(AnilistAnimeID);"), + new(167, 1, "ALTER TABLE TMDB_Episode ADD COLUMN TmdbEpisodeType TEXT NULL DEFAULT NULL;"), ]; #endregion diff --git a/Shoko.Server/Mappings/TMDB/TMDB_EpisodeMap.cs b/Shoko.Server/Mappings/TMDB/TMDB_EpisodeMap.cs index 70dc76fc5..6681f9314 100644 --- a/Shoko.Server/Mappings/TMDB/TMDB_EpisodeMap.cs +++ b/Shoko.Server/Mappings/TMDB/TMDB_EpisodeMap.cs @@ -24,6 +24,7 @@ public TMDB_EpisodeMap() Map(x => x.SeasonNumber).Not.Nullable(); Map(x => x.EpisodeNumber).Not.Nullable(); Map(x => x.RuntimeMinutes).Column("Runtime"); + Map(x => x.TmdbEpisodeType).Nullable(); Map(x => x.UserRating).Not.Nullable(); Map(x => x.UserVotes).Not.Nullable(); Map(x => x.AiredAt).CustomType(); diff --git a/Shoko.Server/Models/TMDB/TMDB_Episode.cs b/Shoko.Server/Models/TMDB/TMDB_Episode.cs index 1324642b2..577f48645 100644 --- a/Shoko.Server/Models/TMDB/TMDB_Episode.cs +++ b/Shoko.Server/Models/TMDB/TMDB_Episode.cs @@ -113,6 +113,13 @@ public int? RuntimeMinutes /// public TimeSpan? Runtime { get; set; } + /// + /// TMDB's own episode classification (standard, finale, + /// mid_season, special), verbatim from the API. + /// until the episode is refreshed against a library build that fetches it. + /// + public string? TmdbEpisodeType { get; set; } + /// /// Average user rating across all . /// @@ -187,6 +194,7 @@ public bool Populate(TvShow show, TvSeason season, TvSeasonEpisode episode, Tran UpdateProperty(SeasonNumber, episode.SeasonNumber, v => SeasonNumber = v), UpdateProperty(EpisodeNumber, episode.EpisodeNumber, v => EpisodeNumber = (int)v), UpdateProperty(Runtime, episode.Runtime.HasValue ? TimeSpan.FromMinutes(episode.Runtime.Value) : null, v => Runtime = v), + UpdateProperty(TmdbEpisodeType, string.IsNullOrWhiteSpace(episode.EpisodeType) ? null : episode.EpisodeType.Trim(), v => TmdbEpisodeType = v), UpdateProperty(UserRating, episode.VoteAverage, v => UserRating = v), UpdateProperty(UserVotes, episode.VoteCount, v => UserVotes = v), UpdateProperty(AiredAt, episode.AirDate?.ToDateOnly(), v => AiredAt = v), diff --git a/Shoko.Tests/Providers/TMDB/TmdbEpisodeTypeTests.cs b/Shoko.Tests/Providers/TMDB/TmdbEpisodeTypeTests.cs new file mode 100644 index 000000000..6e937ff5d --- /dev/null +++ b/Shoko.Tests/Providers/TMDB/TmdbEpisodeTypeTests.cs @@ -0,0 +1,57 @@ +using Shoko.Server.Models.TMDB; +using TMDbLib.Objects.Search; +using TMDbLib.Objects.TvShows; +using Xunit; + +namespace Shoko.Tests.Providers.TMDB; + +/// +/// User Story 5 / O4: capture TMDB's own episode_type. TMDbLib master exposes it on +/// (and TvEpisodeBase); before the upgrade +/// Shoko had no episode-type signal from TMDB. This only persists the value (and surfaces it +/// on the v3 DTO) — it deliberately does NOT feed episode matching (that would be a +/// match-selection change, out of scope here; tracked as a follow-up). +/// +public class TmdbEpisodeTypeTests +{ + private static readonly TvShow Show = new() { Id = 45782, Name = "SAO" }; + private static readonly TvSeason Season = new() { Id = 61862, SeasonNumber = 1 }; + + private static TvSeasonEpisode ReducedEpisode(string? episodeType, int seasonNumber = 1) => new() + { + Id = 979329, + SeasonNumber = seasonNumber, + EpisodeNumber = 1, + Name = "Ep", + Overview = "", + StillPath = "", + EpisodeType = episodeType, + }; + + [Theory] + [InlineData("standard", "standard")] + [InlineData("finale", "finale")] + [InlineData("mid_season", "mid_season")] + [InlineData("special", "special")] + [InlineData(" finale ", "finale")] + [InlineData(null, null)] + [InlineData("", null)] + [InlineData(" ", null)] + public void Populate_StoresTrimmedEpisodeTypeOrNull(string? wire, string? expected) + { + var episode = new TMDB_Episode(); + episode.Populate(Show, Season, ReducedEpisode(wire), translations: null); + + Assert.Equal(expected, episode.TmdbEpisodeType); + } + + [Fact] + public void Populate_EpisodeTypeChange_IsReportedAsAnUpdate() + { + var episode = new TMDB_Episode(); + episode.Populate(Show, Season, ReducedEpisode("standard"), translations: null); + + Assert.True(episode.Populate(Show, Season, ReducedEpisode("finale"), translations: null)); + Assert.False(episode.Populate(Show, Season, ReducedEpisode("finale"), translations: null)); + } +} From 3fb6bf03ec30ae8e5e30b6f50653e0b6e7adc411 Mon Sep 17 00:00:00 2001 From: Foowy <49217685+Foowy@users.noreply.github.com> Date: Wed, 22 Jul 2026 06:38:48 +0000 Subject: [PATCH 09/13] fix(tmdb): skip the animation genre requirement for restricted auto-match MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `CollectCandidates`/`CollectMovieCandidates` required every TMDB search hit to carry the "animation" genre tag before it was even considered a candidate. TMDB's genre metadata is community-maintained and disproportionately sparse for R18/adult content, so a correct match was often discarded before scoring ever ran, making restricted auto-match appear broken. Threads the existing `includeRestricted`/`restricted` flag (already available at every call site) into both helpers and skips the genre check only when the anime is restricted — non-restricted search keeps the same guard against unrelated live-action/non-anime hits. The downstream title/date scoring (`IsAcceptableAutoMatch` rejecting `FirstAvailable`) still prevents a genuinely unrelated candidate from being accepted. --- .../Providers/TMDB/TmdbSearchService.cs | 29 +++++----- .../Providers/TMDB/TmdbSearchServiceTests.cs | 57 +++++++++++++++++++ 2 files changed, 72 insertions(+), 14 deletions(-) diff --git a/Shoko.Server/Providers/TMDB/TmdbSearchService.cs b/Shoko.Server/Providers/TMDB/TmdbSearchService.cs index c9c81e9e1..a1af5a413 100644 --- a/Shoko.Server/Providers/TMDB/TmdbSearchService.cs +++ b/Shoko.Server/Providers/TMDB/TmdbSearchService.cs @@ -264,7 +264,7 @@ private async Task AutoSearchForMovie(List list, Ani // Attempt #1: full title + year (results, _) = await SearchMoviesRaw(query, includeRestricted: includeRestricted, year: year).ConfigureAwait(false); - CollectMovieCandidates(candidates, results, seen, candidateCount); + CollectMovieCandidates(candidates, results, seen, candidateCount, includeRestricted); // Attempt #2: sequel-suffix stripped + year var strippedTitle = SequelSuffixRemovalRegex().Match(query) is { Success: true } regexResult @@ -272,7 +272,7 @@ private async Task AutoSearchForMovie(List list, Ani if (!string.IsNullOrEmpty(strippedTitle) && candidates.Count < candidateCount) { (results, _) = await SearchMoviesRaw(strippedTitle, includeRestricted: includeRestricted, year: year).ConfigureAwait(false); - CollectMovieCandidates(candidates, results, seen, candidateCount); + CollectMovieCandidates(candidates, results, seen, candidateCount, includeRestricted); } // Attempts #3–4: year-free fallbacks mirroring #1–2. @@ -282,12 +282,12 @@ private async Task AutoSearchForMovie(List list, Ani var yearFreeCap = candidateCount * 2; (results, _) = await SearchMoviesRaw(query, includeRestricted: includeRestricted).ConfigureAwait(false); - CollectMovieCandidates(candidates, results, seen, yearFreeCap); + CollectMovieCandidates(candidates, results, seen, yearFreeCap, includeRestricted); if (!string.IsNullOrEmpty(strippedTitle) && candidates.Count < yearFreeCap) { (results, _) = await SearchMoviesRaw(strippedTitle, includeRestricted: includeRestricted).ConfigureAwait(false); - CollectMovieCandidates(candidates, results, seen, yearFreeCap); + CollectMovieCandidates(candidates, results, seen, yearFreeCap, includeRestricted); } if (candidates.Count == 0) @@ -575,7 +575,7 @@ private async Task> AutoSearchForShow(AniDB_ // Attempt #1: full title + year (results, _) = await SearchShowsRaw(originalTitle, includeRestricted: restricted, year: airDate.Year).ConfigureAwait(false); - CollectCandidates(candidates, results, seen, candidateCount); + CollectCandidates(candidates, results, seen, candidateCount, restricted); // Attempt #2: sequel-suffix stripped + year var strippedTitle = SequelSuffixRemovalRegex().Match(originalTitle) is { Success: true } regexResult @@ -583,7 +583,7 @@ private async Task> AutoSearchForShow(AniDB_ if (!string.IsNullOrEmpty(strippedTitle) && candidates.Count < candidateCount) { (results, _) = await SearchShowsRaw(strippedTitle, includeRestricted: restricted, year: airDate.Year).ConfigureAwait(false); - CollectCandidates(candidates, results, seen, candidateCount); + CollectCandidates(candidates, results, seen, candidateCount, restricted); } // Attempt #3: subtitle stripped + year. @@ -597,7 +597,7 @@ private async Task> AutoSearchForShow(AniDB_ if (!string.IsNullOrEmpty(titleWithoutSubTitle) && candidates.Count < candidateCount) { (results, _) = await SearchShowsRaw(titleWithoutSubTitle, includeRestricted: restricted, year: airDate.Year).ConfigureAwait(false); - CollectCandidates(candidates, results, seen, candidateCount); + CollectCandidates(candidates, results, seen, candidateCount, restricted); } // Attempts #4–6: year-free fallbacks mirroring #1–3. @@ -609,18 +609,18 @@ private async Task> AutoSearchForShow(AniDB_ var yearFreeCap = candidateCount * 2; (results, _) = await SearchShowsRaw(originalTitle, includeRestricted: restricted).ConfigureAwait(false); - CollectCandidates(candidates, results, seen, yearFreeCap); + CollectCandidates(candidates, results, seen, yearFreeCap, restricted); if (!string.IsNullOrEmpty(strippedTitle) && candidates.Count < yearFreeCap) { (results, _) = await SearchShowsRaw(strippedTitle, includeRestricted: restricted).ConfigureAwait(false); - CollectCandidates(candidates, results, seen, yearFreeCap); + CollectCandidates(candidates, results, seen, yearFreeCap, restricted); } if (!string.IsNullOrEmpty(titleWithoutSubTitle) && candidates.Count < yearFreeCap) { (results, _) = await SearchShowsRaw(titleWithoutSubTitle, includeRestricted: restricted).ConfigureAwait(false); - CollectCandidates(candidates, results, seen, yearFreeCap); + CollectCandidates(candidates, results, seen, yearFreeCap, restricted); } if (candidates.Count == 0) @@ -770,24 +770,25 @@ private static bool MovieMatchesYear(TMDbLib.Objects.Movies.Movie movie, int yea // applies the same floor for show and movie auto-matching. internal static bool IsAcceptableAutoMatch(MatchRating rating) => rating is not MatchRating.FirstAvailable; - private static void CollectCandidates(List candidates, List results, HashSet seen, int candidateCount) + // Restricted titles skip the Animation genre check — TMDB's genre tags are sparse for adult content. + private static void CollectCandidates(List candidates, List results, HashSet seen, int candidateCount, bool isRestricted = false) { foreach (var result in results) { if (candidates.Count >= candidateCount) break; if (!seen.Add(result.Id)) continue; - if (!result.GetGenres().Contains(AnimationGenre, StringComparer.OrdinalIgnoreCase)) continue; + if (!isRestricted && !result.GetGenres().Contains(AnimationGenre, StringComparer.OrdinalIgnoreCase)) continue; candidates.Add(result); } } - private static void CollectMovieCandidates(List candidates, List results, HashSet seen, int candidateCount) + private static void CollectMovieCandidates(List candidates, List results, HashSet seen, int candidateCount, bool isRestricted = false) { foreach (var result in results) { if (candidates.Count >= candidateCount) break; if (!seen.Add(result.Id)) continue; - if (!result.GetGenres().Contains(AnimationGenre, StringComparer.OrdinalIgnoreCase)) continue; + if (!isRestricted && !result.GetGenres().Contains(AnimationGenre, StringComparer.OrdinalIgnoreCase)) continue; candidates.Add(result); } } diff --git a/Shoko.Tests/Providers/TMDB/TmdbSearchServiceTests.cs b/Shoko.Tests/Providers/TMDB/TmdbSearchServiceTests.cs index d9c0be414..228421766 100644 --- a/Shoko.Tests/Providers/TMDB/TmdbSearchServiceTests.cs +++ b/Shoko.Tests/Providers/TMDB/TmdbSearchServiceTests.cs @@ -1,8 +1,10 @@ using System.Collections.Generic; +using System.Reflection; using Shoko.Abstractions.Metadata.Enums; using Shoko.Server.Filters; using Shoko.Server.Providers.TMDB; using Shoko.Server.Utilities; +using TMDbLib.Objects.Search; using Xunit; // ReSharper disable StringLiteralTypo @@ -226,4 +228,59 @@ public void IsAcceptableAutoMatch_RejectsOnlyFirstAvailable(MatchRating rating, { Assert.Equal(expectedAcceptable, TmdbSearchService.IsAcceptableAutoMatch(rating)); } + + // TmdbMetadataService.Instance is unset in this test host, so GetGenres() returns empty — + // equivalent to an untagged TMDB result. + + private static void InvokeCollectCandidates(List candidates, List results, HashSet seen, int candidateCount, bool isRestricted) + { + var method = typeof(TmdbSearchService).GetMethod("CollectCandidates", BindingFlags.NonPublic | BindingFlags.Static)!; + method.Invoke(null, [candidates, results, seen, candidateCount, isRestricted]); + } + + private static void InvokeCollectMovieCandidates(List candidates, List results, HashSet seen, int candidateCount, bool isRestricted) + { + var method = typeof(TmdbSearchService).GetMethod("CollectMovieCandidates", BindingFlags.NonPublic | BindingFlags.Static)!; + method.Invoke(null, [candidates, results, seen, candidateCount, isRestricted]); + } + + [Fact] + public void CollectCandidates_NonRestricted_ExcludesUntaggedResult() + { + var candidates = new List(); + var results = new List { new() { Id = 1 } }; + InvokeCollectCandidates(candidates, results, [], 10, false); + + Assert.Empty(candidates); + } + + [Fact] + public void CollectCandidates_Restricted_IncludesUntaggedResult() + { + var candidates = new List(); + var results = new List { new() { Id = 1 } }; + InvokeCollectCandidates(candidates, results, [], 10, true); + + Assert.Single(candidates); + } + + [Fact] + public void CollectMovieCandidates_NonRestricted_ExcludesUntaggedResult() + { + var candidates = new List(); + var results = new List { new() { Id = 1 } }; + InvokeCollectMovieCandidates(candidates, results, [], 10, false); + + Assert.Empty(candidates); + } + + [Fact] + public void CollectMovieCandidates_Restricted_IncludesUntaggedResult() + { + var candidates = new List(); + var results = new List { new() { Id = 1 } }; + InvokeCollectMovieCandidates(candidates, results, [], 10, true); + + Assert.Single(candidates); + } } From 9a8f8f1d31f156c5a921ec75dc3d0fd05476f9bd Mon Sep 17 00:00:00 2001 From: Foowy <49217685+Foowy@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:43:29 +0000 Subject: [PATCH 10/13] fix(tmdb): use original_language as the restricted-title genre signal The restricted-title path skipped the animation-genre gate entirely, so a restricted search returned candidates with zero content-type filtering, relying wholly on downstream title/date scoring to reject unrelated hits. TMDB's `original_language` is assigned by TMDB itself, not community-tagged like `genre_ids`, so it doesn't share the sparsity problem the genre check exists to work around. `CollectCandidates`/ `CollectMovieCandidates` now accept a restricted result either on the existing animation-genre match or on `original_language == "ja"`, restoring a real content-type filter for restricted search instead of bypassing it outright. --- .../Providers/TMDB/TmdbSearchService.cs | 11 +++++-- .../Providers/TMDB/TmdbSearchServiceTests.cs | 32 +++++++++++++++---- 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/Shoko.Server/Providers/TMDB/TmdbSearchService.cs b/Shoko.Server/Providers/TMDB/TmdbSearchService.cs index a1af5a413..67dd91c98 100644 --- a/Shoko.Server/Providers/TMDB/TmdbSearchService.cs +++ b/Shoko.Server/Providers/TMDB/TmdbSearchService.cs @@ -770,14 +770,17 @@ private static bool MovieMatchesYear(TMDbLib.Objects.Movies.Movie movie, int yea // applies the same floor for show and movie auto-matching. internal static bool IsAcceptableAutoMatch(MatchRating rating) => rating is not MatchRating.FirstAvailable; - // Restricted titles skip the Animation genre check — TMDB's genre tags are sparse for adult content. + // Restricted titles also accept a Japanese original_language match — TMDB's genre tags are sparse + // for adult content, but original_language is TMDB-assigned metadata, not community-tagged. private static void CollectCandidates(List candidates, List results, HashSet seen, int candidateCount, bool isRestricted = false) { foreach (var result in results) { if (candidates.Count >= candidateCount) break; if (!seen.Add(result.Id)) continue; - if (!isRestricted && !result.GetGenres().Contains(AnimationGenre, StringComparer.OrdinalIgnoreCase)) continue; + var isAnimeLike = result.GetGenres().Contains(AnimationGenre, StringComparer.OrdinalIgnoreCase) || + (isRestricted && string.Equals(result.OriginalLanguage, "ja", StringComparison.OrdinalIgnoreCase)); + if (!isAnimeLike) continue; candidates.Add(result); } } @@ -788,7 +791,9 @@ private static void CollectMovieCandidates(List candidates, List= candidateCount) break; if (!seen.Add(result.Id)) continue; - if (!isRestricted && !result.GetGenres().Contains(AnimationGenre, StringComparer.OrdinalIgnoreCase)) continue; + var isAnimeLike = result.GetGenres().Contains(AnimationGenre, StringComparer.OrdinalIgnoreCase) || + (isRestricted && string.Equals(result.OriginalLanguage, "ja", StringComparison.OrdinalIgnoreCase)); + if (!isAnimeLike) continue; candidates.Add(result); } } diff --git a/Shoko.Tests/Providers/TMDB/TmdbSearchServiceTests.cs b/Shoko.Tests/Providers/TMDB/TmdbSearchServiceTests.cs index 228421766..56f5f848f 100644 --- a/Shoko.Tests/Providers/TMDB/TmdbSearchServiceTests.cs +++ b/Shoko.Tests/Providers/TMDB/TmdbSearchServiceTests.cs @@ -229,8 +229,8 @@ public void IsAcceptableAutoMatch_RejectsOnlyFirstAvailable(MatchRating rating, Assert.Equal(expectedAcceptable, TmdbSearchService.IsAcceptableAutoMatch(rating)); } - // TmdbMetadataService.Instance is unset in this test host, so GetGenres() returns empty — - // equivalent to an untagged TMDB result. + // TmdbMetadataService.Instance is unset in this test host, so GetGenres() always returns empty — + // every result below is effectively untagged; only OriginalLanguage varies. private static void InvokeCollectCandidates(List candidates, List results, HashSet seen, int candidateCount, bool isRestricted) { @@ -255,10 +255,20 @@ public void CollectCandidates_NonRestricted_ExcludesUntaggedResult() } [Fact] - public void CollectCandidates_Restricted_IncludesUntaggedResult() + public void CollectCandidates_Restricted_ExcludesUntaggedNonJapaneseResult() { var candidates = new List(); - var results = new List { new() { Id = 1 } }; + var results = new List { new() { Id = 1, OriginalLanguage = "en" } }; + InvokeCollectCandidates(candidates, results, [], 10, true); + + Assert.Empty(candidates); + } + + [Fact] + public void CollectCandidates_Restricted_IncludesUntaggedJapaneseResult() + { + var candidates = new List(); + var results = new List { new() { Id = 1, OriginalLanguage = "ja" } }; InvokeCollectCandidates(candidates, results, [], 10, true); Assert.Single(candidates); @@ -275,10 +285,20 @@ public void CollectMovieCandidates_NonRestricted_ExcludesUntaggedResult() } [Fact] - public void CollectMovieCandidates_Restricted_IncludesUntaggedResult() + public void CollectMovieCandidates_Restricted_ExcludesUntaggedNonJapaneseResult() { var candidates = new List(); - var results = new List { new() { Id = 1 } }; + var results = new List { new() { Id = 1, OriginalLanguage = "en" } }; + InvokeCollectMovieCandidates(candidates, results, [], 10, true); + + Assert.Empty(candidates); + } + + [Fact] + public void CollectMovieCandidates_Restricted_IncludesUntaggedJapaneseResult() + { + var candidates = new List(); + var results = new List { new() { Id = 1, OriginalLanguage = "ja" } }; InvokeCollectMovieCandidates(candidates, results, [], 10, true); Assert.Single(candidates); From d34d998617f448690ef6ffa2597793ade46ce62e Mon Sep 17 00:00:00 2001 From: Foowy <49217685+Foowy@users.noreply.github.com> Date: Mon, 14 Sep 2026 16:55:37 +0000 Subject: [PATCH 11/13] refactor(tmdb): extract shared cast/crew diffing between movie and episode UpdateMovieCastAndCrew and UpdateEpisodeCastAndCrew each inlined a near-identical cast-diff and crew-diff loop, pushing both methods over SonarCloud's cognitive-complexity threshold (35 and 23 against a limit of 15). TMDbLib's ICastCredit/ICrewCredit interfaces (shared by the movie and episode credit DTOs) make the diffing generic over the credit and entity types: DiffCast/DiffCrew now hold the add/update/remove logic once, with per-caller entity construction passed in as a factory. The episode path's guest-role flag is threaded through DiffCast's optional applyExtra hook rather than duplicating the loop. Also drops two redundant `(int)` casts on already-int TMDbLib fields (episode.EpisodeNumber, reducedEpisode.EpisodeNumber), left over from the pre-upgrade string-typed API. Same add/update/remove/save/delete/logging behavior, verified against the existing test suite (3940/3940 passing). --- Shoko.Server/Models/TMDB/TMDB_Episode.cs | 2 +- .../Providers/TMDB/TmdbMetadataService.cs | 279 ++++++++---------- 2 files changed, 123 insertions(+), 158 deletions(-) diff --git a/Shoko.Server/Models/TMDB/TMDB_Episode.cs b/Shoko.Server/Models/TMDB/TMDB_Episode.cs index 577f48645..647423b1a 100644 --- a/Shoko.Server/Models/TMDB/TMDB_Episode.cs +++ b/Shoko.Server/Models/TMDB/TMDB_Episode.cs @@ -192,7 +192,7 @@ public bool Populate(TvShow show, TvSeason season, TvSeasonEpisode episode, Tran UpdateProperty(EnglishTitle, translations is null && !string.IsNullOrEmpty(EnglishTitle) ? EnglishTitle : !string.IsNullOrEmpty(translation?.Data?.Name) ? translation.Data.Name : episode.Name!, v => EnglishTitle = v), UpdateProperty(EnglishOverview, !string.IsNullOrEmpty(translation?.Data?.Overview) ? translation.Data.Overview : episode.Overview!, v => EnglishOverview = v), UpdateProperty(SeasonNumber, episode.SeasonNumber, v => SeasonNumber = v), - UpdateProperty(EpisodeNumber, episode.EpisodeNumber, v => EpisodeNumber = (int)v), + UpdateProperty(EpisodeNumber, episode.EpisodeNumber, v => EpisodeNumber = v), UpdateProperty(Runtime, episode.Runtime.HasValue ? TimeSpan.FromMinutes(episode.Runtime.Value) : null, v => Runtime = v), UpdateProperty(TmdbEpisodeType, string.IsNullOrWhiteSpace(episode.EpisodeType) ? null : episode.EpisodeType.Trim(), v => TmdbEpisodeType = v), UpdateProperty(UserRating, episode.VoteAverage, v => UserRating = v), diff --git a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs index dc49bbdc6..279524b6b 100644 --- a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs +++ b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs @@ -35,6 +35,7 @@ using TMDbLib.Objects.Collections; using TMDbLib.Objects.Exceptions; using TMDbLib.Objects.General; +using TMDbLib.Objects.General.Schema; using TMDbLib.Objects.Movies; using TMDbLib.Objects.People; using TMDbLib.Objects.Search; @@ -599,105 +600,133 @@ public async Task UpdateMovie(TmdbMovieUpdateOptions options) } } - private async Task UpdateMovieCastAndCrew(TMDB_Movie tmdbMovie, MovieCredits? credits, bool forceRefresh, bool downloadImages) + // Shared by UpdateMovieCastAndCrew/UpdateEpisodeCastAndCrew — cast ordering is our own + // zero-based counter, not TMDb's `order` field, so existing sort order survives re-fetches + // even when TMDb reshuffles theirs. + private static (List ToSave, List ToRemove, int Added) DiffCast( + IEnumerable cast, + IReadOnlyDictionary existing, + Func createRole, + Func? applyExtra = null) + where TCredit : TmdbEntity, ICastCredit + where TCast : TMDB_Cast { - // See UpdateEpisodeCastAndCrew: a null/empty credits append is "no cast/crew this pass", not a purge. - if (credits?.Cast is null || credits.Crew is null) - return false; - - var peopleToKeep = new HashSet(); - - var counter = 0; - var castToAdd = 0; - var castToKeep = new HashSet(); - var castToSave = new List(); - var existingCastDict = _tmdbMovieCast.GetByTmdbMovieID(tmdbMovie.Id) - .ToDictionary(cast => cast.TmdbCreditID); - foreach (var cast in credits.Cast) + var toKeep = new HashSet(); + var toSave = new List(); + var added = 0; + var ordering = 0; + foreach (var credit in cast) { - var ordering = counter++; - peopleToKeep.Add(cast.Id); - castToKeep.Add(cast.CreditId!); + var order = ordering++; + var creditId = credit.CreditId!; + toKeep.Add(creditId); var roleUpdated = false; - if (!existingCastDict.TryGetValue(cast.CreditId!, out var role)) + if (!existing.TryGetValue(creditId, out var role)) { - role = new() - { - TmdbMovieID = tmdbMovie.Id, - TmdbPersonID = cast.Id, - TmdbCreditID = cast.CreditId!, - }; - castToAdd++; + role = createRole(credit, order); + added++; roleUpdated = true; } - var characterName = cast.Character!.Replace(" (voice)", ""); + var characterName = credit.Character!.Replace(" (voice)", ""); if (role.CharacterName != characterName) { role.CharacterName = characterName; roleUpdated = true; } - if (role.Ordering != ordering) + if (role.Ordering != order) { - role.Ordering = ordering; + role.Ordering = order; roleUpdated = true; } + if (applyExtra?.Invoke(role, credit, order) == true) + roleUpdated = true; + if (roleUpdated) - { - castToSave.Add(role); - } + toSave.Add(role); } - var crewToAdd = 0; - var crewToKeep = new HashSet(); - var crewToSave = new List(); - var existingCrewDict = _tmdbMovieCrew.GetByTmdbMovieID(tmdbMovie.Id) - .ToDictionary(crew => crew.TmdbCreditID); - foreach (var crew in credits.Crew) + var toRemove = existing.Values.Where(role => !toKeep.Contains(role.TmdbCreditID)).ToList(); + return (toSave, toRemove, added); + } + + private static (List ToSave, List ToRemove, int Added) DiffCrew( + IEnumerable crew, + IReadOnlyDictionary existing, + Func createRole) + where TCredit : TmdbEntity, ICrewCredit + where TCrew : TMDB_Crew + { + var toKeep = new HashSet(); + var toSave = new List(); + var added = 0; + foreach (var credit in crew) { - peopleToKeep.Add(crew.Id); - crewToKeep.Add(crew.CreditId!); + var creditId = credit.CreditId!; + toKeep.Add(creditId); var roleUpdated = false; - if (!existingCrewDict.TryGetValue(crew.CreditId!, out var role)) + if (!existing.TryGetValue(creditId, out var role)) { - role = new() - { - TmdbMovieID = tmdbMovie.Id, - TmdbPersonID = crew.Id, - TmdbCreditID = crew.CreditId!, - }; - crewToAdd++; + role = createRole(credit); + added++; roleUpdated = true; } - if (role.Department != crew.Department) + if (role.Department != credit.Department) { - role.Department = crew.Department!; + role.Department = credit.Department!; roleUpdated = true; } - if (role.Job != crew.Job) + if (role.Job != credit.Job) { - role.Job = crew.Job!; + role.Job = credit.Job!; roleUpdated = true; } if (roleUpdated) - { - crewToSave.Add(role); - } + toSave.Add(role); } - var castToRemove = existingCastDict.Values - .ExceptBy(castToKeep, cast => cast.TmdbCreditID) - .ToList(); - var crewToRemove = existingCrewDict.Values - .ExceptBy(crewToKeep, crew => crew.TmdbCreditID) - .ToList(); + var toRemove = existing.Values.Where(role => !toKeep.Contains(role.TmdbCreditID)).ToList(); + return (toSave, toRemove, added); + } + + private async Task UpdateMovieCastAndCrew(TMDB_Movie tmdbMovie, MovieCredits? credits, bool forceRefresh, bool downloadImages) + { + // See UpdateEpisodeCastAndCrew: a null/empty credits append is "no cast/crew this pass", not a purge. + if (credits?.Cast is null || credits.Crew is null) + return false; + + var existingCastDict = _tmdbMovieCast.GetByTmdbMovieID(tmdbMovie.Id) + .ToDictionary(cast => cast.TmdbCreditID); + var (castToSave, castToRemove, castToAdd) = DiffCast( + credits.Cast, + existingCastDict, + (cast, ordering) => new TMDB_Movie_Cast + { + TmdbMovieID = tmdbMovie.Id, + TmdbPersonID = cast.Id, + TmdbCreditID = cast.CreditId!, + }); + + var existingCrewDict = _tmdbMovieCrew.GetByTmdbMovieID(tmdbMovie.Id) + .ToDictionary(crew => crew.TmdbCreditID); + var (crewToSave, crewToRemove, crewToAdd) = DiffCrew( + credits.Crew, + existingCrewDict, + crew => new TMDB_Movie_Crew + { + TmdbMovieID = tmdbMovie.Id, + TmdbPersonID = crew.Id, + TmdbCreditID = crew.CreditId!, + }); + + var peopleToKeep = new HashSet(credits.Cast.Select(cast => cast.Id).Concat(credits.Crew.Select(crew => crew.Id))); _tmdbMovieCast.Save(castToSave); _tmdbMovieCrew.Save(crewToSave); @@ -1500,7 +1529,7 @@ private static bool AccumulateUnchangedEpisode( ShowSyncState state) { var newlyAdded = tmdbEpisode.CreatedAt == tmdbEpisode.LastUpdatedAt; - if (!state.ChangedItems.HasValue || newlyAdded || state.ChangedItems.Value.Episodes.Contains((season.SeasonNumber, (int)reducedEpisode.EpisodeNumber))) + if (!state.ChangedItems.HasValue || newlyAdded || state.ChangedItems.Value.Episodes.Contains((season.SeasonNumber, reducedEpisode.EpisodeNumber))) return false; state.EpisodesToSkip.Add(tmdbEpisode.Id); @@ -1820,111 +1849,47 @@ private async Task UpdateShowAlternateOrdering(TMDB_Show tmdbShow, TvShow if (credits?.Cast is null || credits.Crew is null) return (false, [], []); - var peopleToAddOrKeep = new HashSet(); - var counter = 0; - var castToAdd = 0; - var castToKeep = new HashSet(); - var castToSave = new List(); var existingCastDict = _tmdbEpisodeCast.GetByTmdbEpisodeID(tmdbEpisode.Id) .ToDictionary(cast => cast.TmdbCreditID); var guestOffset = credits.Cast.Count; - foreach (var cast in credits.Cast.Concat(credits.GuestStars ?? [])) - { - var ordering = counter++; - var isGuestRole = ordering >= guestOffset; - castToKeep.Add(cast.CreditId!); - peopleToAddOrKeep.Add(cast.Id); - - var roleUpdated = false; - if (!existingCastDict.TryGetValue(cast.CreditId!, out var role)) - { - role = new() - { - TmdbShowID = tmdbEpisode.TmdbShowID, - TmdbSeasonID = tmdbEpisode.TmdbSeasonID, - TmdbEpisodeID = tmdbEpisode.Id, - TmdbPersonID = cast.Id, - TmdbCreditID = cast.CreditId!, - Ordering = ordering, - IsGuestRole = isGuestRole, - }; - castToAdd++; - roleUpdated = true; - } - - var characterName = cast.Character!.Replace(" (voice)", ""); - if (role.CharacterName != characterName) - { - role.CharacterName = characterName; - roleUpdated = true; - } - - if (role.Ordering != ordering) - { - role.Ordering = ordering; - roleUpdated = true; - } - - if (role.IsGuestRole != isGuestRole) + var (castToSave, castToRemove, castToAdd) = DiffCast( + credits.Cast.Concat(credits.GuestStars ?? []), + existingCastDict, + (cast, ordering) => new TMDB_Episode_Cast + { + TmdbShowID = tmdbEpisode.TmdbShowID, + TmdbSeasonID = tmdbEpisode.TmdbSeasonID, + TmdbEpisodeID = tmdbEpisode.Id, + TmdbPersonID = cast.Id, + TmdbCreditID = cast.CreditId!, + Ordering = ordering, + IsGuestRole = ordering >= guestOffset, + }, + (role, _, ordering) => { + var isGuestRole = ordering >= guestOffset; + if (role.IsGuestRole == isGuestRole) + return false; role.IsGuestRole = isGuestRole; - roleUpdated = true; - } - - if (roleUpdated) - { - castToSave.Add(role); - } - } + return true; + }); - var crewToAdd = 0; - var crewToKeep = new HashSet(); - var crewToSave = new List(); var existingCrewDict = _tmdbEpisodeCrew.GetByTmdbEpisodeID(tmdbEpisode.Id) .ToDictionary(crew => crew.TmdbCreditID); - foreach (var crew in credits.Crew) - { - peopleToAddOrKeep.Add(crew.Id); - crewToKeep.Add(crew.CreditId!); - var roleUpdated = false; - if (!existingCrewDict.TryGetValue(crew.CreditId!, out var role)) - { - role = new() - { - TmdbShowID = tmdbEpisode.TmdbShowID, - TmdbSeasonID = tmdbEpisode.TmdbSeasonID, - TmdbEpisodeID = tmdbEpisode.Id, - TmdbPersonID = crew.Id, - TmdbCreditID = crew.CreditId!, - }; - crewToAdd++; - roleUpdated = true; - } - - if (role.Department != crew.Department) - { - role.Department = crew.Department!; - roleUpdated = true; - } - - if (role.Job != crew.Job) - { - role.Job = crew.Job!; - roleUpdated = true; - } - - if (roleUpdated) - { - crewToSave.Add(role); - } - } + var (crewToSave, crewToRemove, crewToAdd) = DiffCrew( + credits.Crew, + existingCrewDict, + crew => new TMDB_Episode_Crew + { + TmdbShowID = tmdbEpisode.TmdbShowID, + TmdbSeasonID = tmdbEpisode.TmdbSeasonID, + TmdbEpisodeID = tmdbEpisode.Id, + TmdbPersonID = crew.Id, + TmdbCreditID = crew.CreditId!, + }); - var castToRemove = existingCastDict.Values - .ExceptBy(castToKeep, cast => cast.TmdbCreditID) - .ToList(); - var crewToRemove = existingCrewDict.Values - .ExceptBy(crewToKeep, crew => crew.TmdbCreditID) - .ToList(); + var peopleToAddOrKeep = new HashSet( + credits.Cast.Concat(credits.GuestStars ?? []).Select(cast => cast.Id).Concat(credits.Crew.Select(crew => crew.Id))); _tmdbEpisodeCast.Save(castToSave); _tmdbEpisodeCrew.Save(crewToSave); From a9b2246ae93d8f09ed2c0b760f5ace58c9979b29 Mon Sep 17 00:00:00 2001 From: Foowy <49217685+Foowy@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:04:56 +0000 Subject: [PATCH 12/13] refactor(tmdb): extract person-update and orphan-cleanup from UpdateMovieCastAndCrew MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cognitive complexity was still 17 against SonarCloud's limit of 15 after the cast/crew diff extraction — the remaining contributors were a two-catch try/catch inside the per-person concurrent-update lambda, and an if-guarded cleanup body inside the CleanupOrphanedCastCrew callback, both nested inside the method itself. Extracted UpdateMoviePersonAndTrack (person update + transient/permanent failure handling) and RemoveOrphanedMovieCastCrew (orphan lookup + delete) as their own methods. Same behavior, verified against the test suite (3940/3940 passing). --- .../Providers/TMDB/TmdbMetadataService.cs | 97 +++++++++++-------- 1 file changed, 55 insertions(+), 42 deletions(-) diff --git a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs index 279524b6b..c84bb33f6 100644 --- a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs +++ b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs @@ -696,6 +696,51 @@ private static (List ToSave, List ToRemove, int Added) DiffCrew transientlyFailed) + { + try + { + var (added, updated) = await UpdatePerson(personId, forceRefresh, downloadImages, currentMovieId: movieId); + if (added) + Interlocked.Increment(ref counters.Added); + else if (updated) + Interlocked.Increment(ref counters.Updated); + } + catch (Exception ex) when (IsTmdbTransient(ex)) + { + // Transient failure — preserve cast/crew rows and retry later. + transientlyFailed.Add(personId); + } + catch (Exception ex) + { + // Non-transient failure — log and let CleanupOrphanedCastCrew handle it. + _logger.LogWarning(ex, "TMDB: Unexpected error updating person {PersonId} for movie {MovieId}", personId, movieId); + } + } + + private void RemoveOrphanedMovieCastCrew(HashSet missingPersonIds, int movieId) + { + var orphanedCast = _tmdbMovieCast.GetByTmdbMovieID(movieId) + .Where(c => missingPersonIds.Contains(c.TmdbPersonID)).ToList(); + var orphanedCrew = _tmdbMovieCrew.GetByTmdbMovieID(movieId) + .Where(c => missingPersonIds.Contains(c.TmdbPersonID)).ToList(); + if (orphanedCast.Count == 0 && orphanedCrew.Count == 0) + return; + + _logger.LogWarning("TMDB: Removed {CastCount} cast and {CrewCount} crew entries for {PersonCount} people that failed to fetch. (Movie={MovieId})", + orphanedCast.Count, orphanedCrew.Count, missingPersonIds.Count, movieId); + _tmdbMovieCast.Delete(orphanedCast); + _tmdbMovieCrew.Delete(orphanedCrew); + } + private async Task UpdateMovieCastAndCrew(TMDB_Movie tmdbMovie, MovieCredits? credits, bool forceRefresh, bool downloadImages) { // See UpdateEpisodeCastAndCrew: a null/empty credits append is "no cast/crew this pass", not a purge. @@ -748,55 +793,23 @@ private async Task UpdateMovieCastAndCrew(TMDB_Movie tmdbMovie, MovieCredi ); // Only add/remove staff if we're not doing a quick refresh. - var peopleAdded = 0; - var peopleUpdated = 0; var peoplePurged = 0; var peopleToPurge = existingCastDict.Values.Select(cast => cast.TmdbPersonID) .Concat(existingCrewDict.Values.Select(crew => crew.TmdbPersonID)) .Except(peopleToKeep) .ToHashSet(); + var counters = new PersonUpdateCounters(); var transientlyFailedMoviePeople = new ConcurrentBag(); - await ProcessWithConcurrencyAsync(_maxConcurrency, peopleToKeep, async personId => - { - try - { - var (added, updated) = await UpdatePerson(personId, forceRefresh, downloadImages, currentMovieId: tmdbMovie.Id); - if (added) - Interlocked.Increment(ref peopleAdded); - else if (updated) - Interlocked.Increment(ref peopleUpdated); - } - catch (Exception ex) when (IsTmdbTransient(ex)) - { - // Transient failure — preserve cast/crew rows and retry later. - transientlyFailedMoviePeople.Add(personId); - } - catch (Exception ex) - { - // Non-transient failure — log and let CleanupOrphanedCastCrew handle it. - _logger.LogWarning(ex, "TMDB: Unexpected error updating person {PersonId} for movie {MovieId}", personId, tmdbMovie.Id); - } - }, onDropped: transientlyFailedMoviePeople.Add); + await ProcessWithConcurrencyAsync(_maxConcurrency, peopleToKeep, + personId => UpdateMoviePersonAndTrack(personId, forceRefresh, downloadImages, tmdbMovie.Id, counters, transientlyFailedMoviePeople), + onDropped: transientlyFailedMoviePeople.Add); // Schedule retries for transiently-failed people; their cast/crew rows are preserved. var transientlyFailedMovieSet = transientlyFailedMoviePeople.ToHashSet(); if (transientlyFailedMovieSet.Count > 0) await Task.WhenAll(transientlyFailedMovieSet.Select(personId => _scheduler.Enqueue(j => { j.TmdbPersonID = personId; j.DownloadImages = downloadImages; j.TmdbMovieID = tmdbMovie.Id; }))); // Remove cast/crew for people that permanently failed — transient failures are excluded. - CleanupOrphanedCastCrew(peopleToKeep, transientlyFailedMovieSet, missingPersonIds => - { - var orphanedCast = _tmdbMovieCast.GetByTmdbMovieID(tmdbMovie.Id) - .Where(c => missingPersonIds.Contains(c.TmdbPersonID)).ToList(); - var orphanedCrew = _tmdbMovieCrew.GetByTmdbMovieID(tmdbMovie.Id) - .Where(c => missingPersonIds.Contains(c.TmdbPersonID)).ToList(); - if (orphanedCast.Count > 0 || orphanedCrew.Count > 0) - { - _logger.LogWarning("TMDB: Removed {CastCount} cast and {CrewCount} crew entries for {PersonCount} people that failed to fetch. (Movie={MovieId})", - orphanedCast.Count, orphanedCrew.Count, missingPersonIds.Count, tmdbMovie.Id); - _tmdbMovieCast.Delete(orphanedCast); - _tmdbMovieCrew.Delete(orphanedCrew); - } - }); + CleanupOrphanedCastCrew(peopleToKeep, transientlyFailedMovieSet, missingPersonIds => RemoveOrphanedMovieCastCrew(missingPersonIds, tmdbMovie.Id)); try { await ProcessWithConcurrencyAsync(_maxConcurrency, peopleToPurge, async personId => @@ -811,10 +824,10 @@ await ProcessWithConcurrencyAsync(_maxConcurrency, peopleToPurge, async personId } _logger.LogDebug("Added/removed {a}/{u}/{r}/{s} staff for movie {MovieTitle} (Movie={MovieId})", - peopleAdded, - peopleUpdated, + counters.Added, + counters.Updated, peoplePurged, - peopleToKeep.Count + peopleToPurge.Count - peopleAdded - peopleUpdated - peoplePurged, + peopleToKeep.Count + peopleToPurge.Count - counters.Added - counters.Updated - peoplePurged, tmdbMovie.EnglishTitle, tmdbMovie.Id ); @@ -822,8 +835,8 @@ await ProcessWithConcurrencyAsync(_maxConcurrency, peopleToPurge, async personId castToRemove.Count > 0 || crewToSave.Count > 0 || crewToRemove.Count > 0 || - peopleAdded > 0 || - peopleUpdated > 0 || + counters.Added > 0 || + counters.Updated > 0 || peoplePurged > 0; } From 40eafd3612f19c75d69b6194b8ab91874833db17 Mon Sep 17 00:00:00 2001 From: Foowy <49217685+Foowy@users.noreply.github.com> Date: Mon, 14 Sep 2026 17:10:30 +0000 Subject: [PATCH 13/13] repo(deps): bump TMDbLib submodule to a4a3fbb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pulls 3 upstream dependency-bump commits (Microsoft.NET.Test.Sdk, Microsoft.SourceLink.GitHub, MinVer) — no API or behavior changes. Build 0/0, full test suite 3940/3940 passing. --- ext/TMDbLib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/TMDbLib b/ext/TMDbLib index 474473260..a4a3fbbf5 160000 --- a/ext/TMDbLib +++ b/ext/TMDbLib @@ -1 +1 @@ -Subproject commit 474473260db829cb048d6ca9149cd6b289592dd7 +Subproject commit a4a3fbbf5429b6ceffed8b1d8206dc98f785537c