diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
index b94ef6e95..7f9e656b9 100644
--- a/.github/workflows/integration-tests.yml
+++ b/.github/workflows/integration-tests.yml
@@ -16,6 +16,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v7
+ with:
+ submodules: recursive
- name: Install system dependencies
run: sudo apt-get install -y mediainfo librhash-dev
@@ -58,6 +60,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v7
+ with:
+ submodules: recursive
- name: Install system dependencies
run: sudo apt-get install -y mediainfo librhash-dev
@@ -105,6 +109,8 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v7
+ with:
+ submodules: recursive
- name: Install system dependencies
run: sudo apt-get install -y mediainfo librhash-dev
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 000000000..f6f58c355
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "ext/TMDbLib"]
+ path = ext/TMDbLib
+ url = https://github.com/jellyfin/TMDbLib.git
diff --git a/Shoko.Server/API/v3/Models/TMDB/TmdbEpisode.cs b/Shoko.Server/API/v3/Models/TMDB/TmdbEpisode.cs
index 0dc64b695..c8c027129 100644
--- a/Shoko.Server/API/v3/Models/TMDB/TmdbEpisode.cs
+++ b/Shoko.Server/API/v3/Models/TMDB/TmdbEpisode.cs
@@ -105,6 +105,14 @@ public class TmdbEpisode
///
public TimeSpan? Runtime { get; init; }
+ ///
+ /// TMDB's own episode classification (standard, finale,
+ /// mid_season, special), verbatim from the API. null
+ /// until the episode has been refreshed since the field was added.
+ ///
+ [JsonProperty(NullValueHandling = NullValueHandling.Ignore)]
+ public string? TmdbEpisodeType { get; init; }
+
///
/// All images stored locally for this episode, if any.
///
@@ -204,6 +212,7 @@ public TmdbEpisode(TMDB_Show show, TMDB_Episode episode, TMDB_AlternateOrdering_
Source = "TMDB",
};
Runtime = episode.Runtime;
+ TmdbEpisodeType = episode.TmdbEpisodeType;
if (include.HasFlag(IncludeDetails.Images))
Images = ((IWithImages)episode).GetImages()
.InLanguage(language)
diff --git a/Shoko.Server/Databases/MySQL.cs b/Shoko.Server/Databases/MySQL.cs
index 8ba31c147..bbe0c47b5 100644
--- a/Shoko.Server/Databases/MySQL.cs
+++ b/Shoko.Server/Databases/MySQL.cs
@@ -1309,6 +1309,7 @@ WHERE sri.`CrossReferences` LIKE '%AnidbEpisodeID%'
new(187, 39, "CREATE INDEX `IX_CrossRef_AniDB_Anilist_Episode_AnilistEpisodeID` ON `CrossRef_AniDB_Anilist_Episode`(`AnilistEpisodeID`);"),
new(187, 40, "CREATE TABLE `Anilist_Anime_ExternalLink` ( `Anilist_Anime_ExternalLinkID` INT NOT NULL AUTO_INCREMENT, `AnilistAnimeID` INT NOT NULL, `AnilistLinkID` INT NOT NULL, `Url` VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, `Site` VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, `AnilistSiteID` INT NULL, `LinkType` VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL, `LanguageCode` VARCHAR(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL, PRIMARY KEY (`Anilist_Anime_ExternalLinkID`) );"),
new(187, 41, "CREATE INDEX `IX_Anilist_Anime_ExternalLink_AnilistAnimeID` ON `Anilist_Anime_ExternalLink`(`AnilistAnimeID`);"),
+ new(188, 1, "ALTER TABLE `TMDB_Episode` ADD COLUMN `TmdbEpisodeType` VARCHAR(50) NULL DEFAULT NULL;"),
];
#endregion
diff --git a/Shoko.Server/Databases/SQLServer.cs b/Shoko.Server/Databases/SQLServer.cs
index 1653336bf..7667ef8f8 100644
--- a/Shoko.Server/Databases/SQLServer.cs
+++ b/Shoko.Server/Databases/SQLServer.cs
@@ -1191,6 +1191,7 @@ WHERE sri.CrossReferences LIKE '%AnidbEpisodeID%'
new(185, 39, "CREATE INDEX IX_CrossRef_AniDB_Anilist_Episode_AnilistEpisodeID ON CrossRef_AniDB_Anilist_Episode(AnilistEpisodeID);"),
new(185, 40, "CREATE TABLE Anilist_Anime_ExternalLink ( Anilist_Anime_ExternalLinkID INT IDENTITY(1,1) NOT NULL, AnilistAnimeID INT NOT NULL, AnilistLinkID INT NOT NULL, Url NVARCHAR(512) NOT NULL, Site NVARCHAR(128) NOT NULL, AnilistSiteID INT NULL, LinkType NVARCHAR(32) NOT NULL, LanguageCode NVARCHAR(32) NULL, CONSTRAINT PK_Anilist_Anime_ExternalLink PRIMARY KEY CLUSTERED (Anilist_Anime_ExternalLinkID) );"),
new(185, 41, "CREATE INDEX IX_Anilist_Anime_ExternalLink_AnilistAnimeID ON Anilist_Anime_ExternalLink(AnilistAnimeID);"),
+ new(186, 1, "ALTER TABLE TMDB_Episode ADD TmdbEpisodeType NVARCHAR(50) NULL DEFAULT NULL;"),
];
#endregion
diff --git a/Shoko.Server/Databases/SQLite.cs b/Shoko.Server/Databases/SQLite.cs
index 743e21806..995bb5ec4 100644
--- a/Shoko.Server/Databases/SQLite.cs
+++ b/Shoko.Server/Databases/SQLite.cs
@@ -1000,6 +1000,7 @@ WHERE CrossReferences LIKE '%AnidbEpisodeID%'
new(166, 39, "CREATE INDEX IX_CrossRef_AniDB_Anilist_Episode_AnilistEpisodeID ON CrossRef_AniDB_Anilist_Episode(AnilistEpisodeID);"),
new(166, 40, "CREATE TABLE Anilist_Anime_ExternalLink ( Anilist_Anime_ExternalLinkID INTEGER PRIMARY KEY AUTOINCREMENT, AnilistAnimeID INTEGER NOT NULL, AnilistLinkID INTEGER NOT NULL, Url TEXT NOT NULL, Site TEXT NOT NULL, AnilistSiteID INTEGER NULL, LinkType TEXT NOT NULL, LanguageCode TEXT NULL );"),
new(166, 41, "CREATE INDEX IX_Anilist_Anime_ExternalLink_AnilistAnimeID ON Anilist_Anime_ExternalLink(AnilistAnimeID);"),
+ new(167, 1, "ALTER TABLE TMDB_Episode ADD COLUMN TmdbEpisodeType TEXT NULL DEFAULT NULL;"),
];
#endregion
diff --git a/Shoko.Server/Mappings/TMDB/TMDB_EpisodeMap.cs b/Shoko.Server/Mappings/TMDB/TMDB_EpisodeMap.cs
index 70dc76fc5..6681f9314 100644
--- a/Shoko.Server/Mappings/TMDB/TMDB_EpisodeMap.cs
+++ b/Shoko.Server/Mappings/TMDB/TMDB_EpisodeMap.cs
@@ -24,6 +24,7 @@ public TMDB_EpisodeMap()
Map(x => x.SeasonNumber).Not.Nullable();
Map(x => x.EpisodeNumber).Not.Nullable();
Map(x => x.RuntimeMinutes).Column("Runtime");
+ Map(x => x.TmdbEpisodeType).Nullable();
Map(x => x.UserRating).Not.Nullable();
Map(x => x.UserVotes).Not.Nullable();
Map(x => x.AiredAt).CustomType();
diff --git a/Shoko.Server/Models/TMDB/TMDB_Episode.cs b/Shoko.Server/Models/TMDB/TMDB_Episode.cs
index 1324642b2..647423b1a 100644
--- a/Shoko.Server/Models/TMDB/TMDB_Episode.cs
+++ b/Shoko.Server/Models/TMDB/TMDB_Episode.cs
@@ -113,6 +113,13 @@ public int? RuntimeMinutes
///
public TimeSpan? Runtime { get; set; }
+ ///
+ /// TMDB's own episode classification (standard, finale,
+ /// mid_season, special), verbatim from the API.
+ /// until the episode is refreshed against a library build that fetches it.
+ ///
+ public string? TmdbEpisodeType { get; set; }
+
///
/// Average user rating across all .
///
@@ -185,8 +192,9 @@ public bool Populate(TvShow show, TvSeason season, TvSeasonEpisode episode, Tran
UpdateProperty(EnglishTitle, translations is null && !string.IsNullOrEmpty(EnglishTitle) ? EnglishTitle : !string.IsNullOrEmpty(translation?.Data?.Name) ? translation.Data.Name : episode.Name!, v => EnglishTitle = v),
UpdateProperty(EnglishOverview, !string.IsNullOrEmpty(translation?.Data?.Overview) ? translation.Data.Overview : episode.Overview!, v => EnglishOverview = v),
UpdateProperty(SeasonNumber, episode.SeasonNumber, v => SeasonNumber = v),
- UpdateProperty(EpisodeNumber, episode.EpisodeNumber, v => EpisodeNumber = (int)v),
+ UpdateProperty(EpisodeNumber, episode.EpisodeNumber, v => EpisodeNumber = v),
UpdateProperty(Runtime, episode.Runtime.HasValue ? TimeSpan.FromMinutes(episode.Runtime.Value) : null, v => Runtime = v),
+ UpdateProperty(TmdbEpisodeType, string.IsNullOrWhiteSpace(episode.EpisodeType) ? null : episode.EpisodeType.Trim(), v => TmdbEpisodeType = v),
UpdateProperty(UserRating, episode.VoteAverage, v => UserRating = v),
UpdateProperty(UserVotes, episode.VoteCount, v => UserVotes = v),
UpdateProperty(AiredAt, episode.AirDate?.ToDateOnly(), v => AiredAt = v),
diff --git a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs
index 1b248657e..c84bb33f6 100644
--- a/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs
+++ b/Shoko.Server/Providers/TMDB/TmdbMetadataService.cs
@@ -35,6 +35,7 @@
using TMDbLib.Objects.Collections;
using TMDbLib.Objects.Exceptions;
using TMDbLib.Objects.General;
+using TMDbLib.Objects.General.Schema;
using TMDbLib.Objects.Movies;
using TMDbLib.Objects.People;
using TMDbLib.Objects.Search;
@@ -206,7 +207,8 @@ public TmdbRateLimitPauseStatus GetPauseStatus()
}
// Retries only on RequestLimitExceededException (cap: 10) and HttpRequestException timeouts (cap: 3).
- // GeneralHttpException and all other types re-throw immediately in OnTmdbRetryAsync.
+ // 5xx types (TMDbServerException, TMDbServiceUnavailableException, GeneralHttpException >= 500) trip the
+ // 5xx breaker and re-throw; all other types re-throw immediately in OnTmdbRetryAsync.
// Zero delay is intentional — actual rate-limit pausing happens inside TmdbRateLimiter.EnsureRateAsync.
private readonly AsyncRetryPolicy _retryPolicy;
@@ -241,6 +243,12 @@ private Task OnTmdbRetryAsync(Exception ex, TimeSpan ts, int retryCount, Context
ctx["timeoutRetryCount"] = timeoutRetryCount + 1;
break;
}
+ // TMDbLib maps recognised 5xx responses to these dedicated types; older versions surfaced
+ // them as GeneralHttpException. Both paths must trip the 5xx breaker.
+ case TMDbServerException or TMDbServiceUnavailableException:
+ _logger.LogWarning(ex, "Got a server-side error from TMDb: {Message}", ex.Message);
+ _rateLimiter.Notify5xxError();
+ throw ex;
case GeneralHttpException ghEx:
_logger.LogWarning(ghEx, "Got a general HTTP exception while processing TMDb request: {StatusCode}", (int)ghEx.HttpStatusCode);
if ((int)ghEx.HttpStatusCode >= 500)
@@ -372,6 +380,8 @@ TMDB_Show_NetworkRepository xrefTmdbShowNetwork
.Handle()
.Or()
.Or()
+ .Or()
+ .Or()
.WaitAndRetryAsync(int.MaxValue, (_, _) => TimeSpan.Zero, OnTmdbRetryAsync);
}
@@ -552,10 +562,10 @@ public async Task UpdateMovie(TmdbMovieUpdateOptions options)
var updated = tmdbMovie.Populate(movie, contentRatingLanguages);
var (titlesUpdated, overviewsUpdated) = UpdateTitlesAndOverviewsWithTuple(tmdbMovie, movie.Translations, preferredTitleLanguages, preferredOverviewLanguages);
updated = titlesUpdated || overviewsUpdated || updated;
- updated = UpdateMovieExternalIDs(tmdbMovie, movie.ExternalIds!) || updated;
+ updated = UpdateMovieExternalIDs(tmdbMovie, movie.ExternalIds) || updated;
updated = await UpdateCompanies(tmdbMovie, movie.ProductionCompanies!) || updated;
if (downloadCrewAndCast)
- updated = await UpdateMovieCastAndCrew(tmdbMovie, movie.Credits!, forceRefresh, downloadImages) || updated;
+ updated = await UpdateMovieCastAndCrew(tmdbMovie, movie.Credits, forceRefresh, downloadImages) || updated;
if (updated)
{
tmdbMovie.LastUpdatedAt = DateTime.Now;
@@ -590,101 +600,178 @@ public async Task UpdateMovie(TmdbMovieUpdateOptions options)
}
}
- private async Task UpdateMovieCastAndCrew(TMDB_Movie tmdbMovie, MovieCredits credits, bool forceRefresh, bool downloadImages)
+ // Shared by UpdateMovieCastAndCrew/UpdateEpisodeCastAndCrew — cast ordering is our own
+ // zero-based counter, not TMDb's `order` field, so existing sort order survives re-fetches
+ // even when TMDb reshuffles theirs.
+ private static (List ToSave, List ToRemove, int Added) DiffCast(
+ IEnumerable cast,
+ IReadOnlyDictionary existing,
+ Func createRole,
+ Func? applyExtra = null)
+ where TCredit : TmdbEntity, ICastCredit
+ where TCast : TMDB_Cast
{
- var peopleToKeep = new HashSet();
-
- var counter = 0;
- var castToAdd = 0;
- var castToKeep = new HashSet();
- var castToSave = new List();
- var existingCastDict = _tmdbMovieCast.GetByTmdbMovieID(tmdbMovie.Id)
- .ToDictionary(cast => cast.TmdbCreditID);
- foreach (var cast in credits.Cast!)
+ var toKeep = new HashSet();
+ var toSave = new List();
+ var added = 0;
+ var ordering = 0;
+ foreach (var credit in cast)
{
- var ordering = counter++;
- peopleToKeep.Add(cast.Id);
- castToKeep.Add(cast.CreditId!);
+ var order = ordering++;
+ var creditId = credit.CreditId!;
+ toKeep.Add(creditId);
var roleUpdated = false;
- if (!existingCastDict.TryGetValue(cast.CreditId!, out var role))
+ if (!existing.TryGetValue(creditId, out var role))
{
- role = new()
- {
- TmdbMovieID = tmdbMovie.Id,
- TmdbPersonID = cast.Id,
- TmdbCreditID = cast.CreditId!,
- };
- castToAdd++;
+ role = createRole(credit, order);
+ added++;
roleUpdated = true;
}
- var characterName = cast.Character!.Replace(" (voice)", "");
+ var characterName = credit.Character!.Replace(" (voice)", "");
if (role.CharacterName != characterName)
{
role.CharacterName = characterName;
roleUpdated = true;
}
- if (role.Ordering != ordering)
+ if (role.Ordering != order)
{
- role.Ordering = ordering;
+ role.Ordering = order;
roleUpdated = true;
}
+ if (applyExtra?.Invoke(role, credit, order) == true)
+ roleUpdated = true;
+
if (roleUpdated)
- {
- castToSave.Add(role);
- }
+ toSave.Add(role);
}
- var crewToAdd = 0;
- var crewToKeep = new HashSet();
- var crewToSave = new List();
- var existingCrewDict = _tmdbMovieCrew.GetByTmdbMovieID(tmdbMovie.Id)
- .ToDictionary(crew => crew.TmdbCreditID);
- foreach (var crew in credits.Crew!)
+ var toRemove = existing.Values.Where(role => !toKeep.Contains(role.TmdbCreditID)).ToList();
+ return (toSave, toRemove, added);
+ }
+
+ private static (List ToSave, List ToRemove, int Added) DiffCrew(
+ IEnumerable crew,
+ IReadOnlyDictionary existing,
+ Func createRole)
+ where TCredit : TmdbEntity, ICrewCredit
+ where TCrew : TMDB_Crew
+ {
+ var toKeep = new HashSet();
+ var toSave = new List();
+ var added = 0;
+ foreach (var credit in crew)
{
- peopleToKeep.Add(crew.Id);
- crewToKeep.Add(crew.CreditId!);
+ var creditId = credit.CreditId!;
+ toKeep.Add(creditId);
var roleUpdated = false;
- if (!existingCrewDict.TryGetValue(crew.CreditId!, out var role))
+ if (!existing.TryGetValue(creditId, out var role))
{
- role = new()
- {
- TmdbMovieID = tmdbMovie.Id,
- TmdbPersonID = crew.Id,
- TmdbCreditID = crew.CreditId!,
- };
- crewToAdd++;
+ role = createRole(credit);
+ added++;
roleUpdated = true;
}
- if (role.Department != crew.Department)
+ if (role.Department != credit.Department)
{
- role.Department = crew.Department!;
+ role.Department = credit.Department!;
roleUpdated = true;
}
- if (role.Job != crew.Job)
+ if (role.Job != credit.Job)
{
- role.Job = crew.Job!;
+ role.Job = credit.Job!;
roleUpdated = true;
}
if (roleUpdated)
- {
- crewToSave.Add(role);
- }
+ toSave.Add(role);
}
- var castToRemove = existingCastDict.Values
- .ExceptBy(castToKeep, cast => cast.TmdbCreditID)
- .ToList();
- var crewToRemove = existingCrewDict.Values
- .ExceptBy(crewToKeep, crew => crew.TmdbCreditID)
- .ToList();
+ var toRemove = existing.Values.Where(role => !toKeep.Contains(role.TmdbCreditID)).ToList();
+ return (toSave, toRemove, added);
+ }
+
+ // Mutable via Interlocked from concurrent UpdateMoviePersonAndTrack calls; a plain int can't be
+ // captured by-ref across the async lambda ProcessWithConcurrencyAsync schedules per person.
+ private sealed class PersonUpdateCounters
+ {
+ public int Added;
+ public int Updated;
+ }
+
+ private async Task UpdateMoviePersonAndTrack(int personId, bool forceRefresh, bool downloadImages, int movieId, PersonUpdateCounters counters, ConcurrentBag transientlyFailed)
+ {
+ try
+ {
+ var (added, updated) = await UpdatePerson(personId, forceRefresh, downloadImages, currentMovieId: movieId);
+ if (added)
+ Interlocked.Increment(ref counters.Added);
+ else if (updated)
+ Interlocked.Increment(ref counters.Updated);
+ }
+ catch (Exception ex) when (IsTmdbTransient(ex))
+ {
+ // Transient failure — preserve cast/crew rows and retry later.
+ transientlyFailed.Add(personId);
+ }
+ catch (Exception ex)
+ {
+ // Non-transient failure — log and let CleanupOrphanedCastCrew handle it.
+ _logger.LogWarning(ex, "TMDB: Unexpected error updating person {PersonId} for movie {MovieId}", personId, movieId);
+ }
+ }
+
+ private void RemoveOrphanedMovieCastCrew(HashSet missingPersonIds, int movieId)
+ {
+ var orphanedCast = _tmdbMovieCast.GetByTmdbMovieID(movieId)
+ .Where(c => missingPersonIds.Contains(c.TmdbPersonID)).ToList();
+ var orphanedCrew = _tmdbMovieCrew.GetByTmdbMovieID(movieId)
+ .Where(c => missingPersonIds.Contains(c.TmdbPersonID)).ToList();
+ if (orphanedCast.Count == 0 && orphanedCrew.Count == 0)
+ return;
+
+ _logger.LogWarning("TMDB: Removed {CastCount} cast and {CrewCount} crew entries for {PersonCount} people that failed to fetch. (Movie={MovieId})",
+ orphanedCast.Count, orphanedCrew.Count, missingPersonIds.Count, movieId);
+ _tmdbMovieCast.Delete(orphanedCast);
+ _tmdbMovieCrew.Delete(orphanedCrew);
+ }
+
+ private async Task UpdateMovieCastAndCrew(TMDB_Movie tmdbMovie, MovieCredits? credits, bool forceRefresh, bool downloadImages)
+ {
+ // See UpdateEpisodeCastAndCrew: a null/empty credits append is "no cast/crew this pass", not a purge.
+ if (credits?.Cast is null || credits.Crew is null)
+ return false;
+
+ var existingCastDict = _tmdbMovieCast.GetByTmdbMovieID(tmdbMovie.Id)
+ .ToDictionary(cast => cast.TmdbCreditID);
+ var (castToSave, castToRemove, castToAdd) = DiffCast(
+ credits.Cast,
+ existingCastDict,
+ (cast, ordering) => new TMDB_Movie_Cast
+ {
+ TmdbMovieID = tmdbMovie.Id,
+ TmdbPersonID = cast.Id,
+ TmdbCreditID = cast.CreditId!,
+ });
+
+ var existingCrewDict = _tmdbMovieCrew.GetByTmdbMovieID(tmdbMovie.Id)
+ .ToDictionary(crew => crew.TmdbCreditID);
+ var (crewToSave, crewToRemove, crewToAdd) = DiffCrew(
+ credits.Crew,
+ existingCrewDict,
+ crew => new TMDB_Movie_Crew
+ {
+ TmdbMovieID = tmdbMovie.Id,
+ TmdbPersonID = crew.Id,
+ TmdbCreditID = crew.CreditId!,
+ });
+
+ var peopleToKeep = new HashSet(credits.Cast.Select(cast => cast.Id).Concat(credits.Crew.Select(crew => crew.Id)));
_tmdbMovieCast.Save(castToSave);
_tmdbMovieCrew.Save(crewToSave);
@@ -706,55 +793,23 @@ private async Task UpdateMovieCastAndCrew(TMDB_Movie tmdbMovie, MovieCredi
);
// Only add/remove staff if we're not doing a quick refresh.
- var peopleAdded = 0;
- var peopleUpdated = 0;
var peoplePurged = 0;
var peopleToPurge = existingCastDict.Values.Select(cast => cast.TmdbPersonID)
.Concat(existingCrewDict.Values.Select(crew => crew.TmdbPersonID))
.Except(peopleToKeep)
.ToHashSet();
+ var counters = new PersonUpdateCounters();
var transientlyFailedMoviePeople = new ConcurrentBag();
- await ProcessWithConcurrencyAsync(_maxConcurrency, peopleToKeep, async personId =>
- {
- try
- {
- var (added, updated) = await UpdatePerson(personId, forceRefresh, downloadImages, currentMovieId: tmdbMovie.Id);
- if (added)
- Interlocked.Increment(ref peopleAdded);
- else if (updated)
- Interlocked.Increment(ref peopleUpdated);
- }
- catch (Exception ex) when (IsTmdbTransient(ex))
- {
- // Transient failure — preserve cast/crew rows and retry later.
- transientlyFailedMoviePeople.Add(personId);
- }
- catch (Exception ex)
- {
- // Non-transient failure — log and let CleanupOrphanedCastCrew handle it.
- _logger.LogWarning(ex, "TMDB: Unexpected error updating person {PersonId} for movie {MovieId}", personId, tmdbMovie.Id);
- }
- }, onDropped: transientlyFailedMoviePeople.Add);
+ await ProcessWithConcurrencyAsync(_maxConcurrency, peopleToKeep,
+ personId => UpdateMoviePersonAndTrack(personId, forceRefresh, downloadImages, tmdbMovie.Id, counters, transientlyFailedMoviePeople),
+ onDropped: transientlyFailedMoviePeople.Add);
// Schedule retries for transiently-failed people; their cast/crew rows are preserved.
var transientlyFailedMovieSet = transientlyFailedMoviePeople.ToHashSet();
if (transientlyFailedMovieSet.Count > 0)
await Task.WhenAll(transientlyFailedMovieSet.Select(personId =>
_scheduler.Enqueue(j => { j.TmdbPersonID = personId; j.DownloadImages = downloadImages; j.TmdbMovieID = tmdbMovie.Id; })));
// Remove cast/crew for people that permanently failed — transient failures are excluded.
- CleanupOrphanedCastCrew(peopleToKeep, transientlyFailedMovieSet, missingPersonIds =>
- {
- var orphanedCast = _tmdbMovieCast.GetByTmdbMovieID(tmdbMovie.Id)
- .Where(c => missingPersonIds.Contains(c.TmdbPersonID)).ToList();
- var orphanedCrew = _tmdbMovieCrew.GetByTmdbMovieID(tmdbMovie.Id)
- .Where(c => missingPersonIds.Contains(c.TmdbPersonID)).ToList();
- if (orphanedCast.Count > 0 || orphanedCrew.Count > 0)
- {
- _logger.LogWarning("TMDB: Removed {CastCount} cast and {CrewCount} crew entries for {PersonCount} people that failed to fetch. (Movie={MovieId})",
- orphanedCast.Count, orphanedCrew.Count, missingPersonIds.Count, tmdbMovie.Id);
- _tmdbMovieCast.Delete(orphanedCast);
- _tmdbMovieCrew.Delete(orphanedCrew);
- }
- });
+ CleanupOrphanedCastCrew(peopleToKeep, transientlyFailedMovieSet, missingPersonIds => RemoveOrphanedMovieCastCrew(missingPersonIds, tmdbMovie.Id));
try
{
await ProcessWithConcurrencyAsync(_maxConcurrency, peopleToPurge, async personId =>
@@ -769,10 +824,10 @@ await ProcessWithConcurrencyAsync(_maxConcurrency, peopleToPurge, async personId
}
_logger.LogDebug("Added/removed {a}/{u}/{r}/{s} staff for movie {MovieTitle} (Movie={MovieId})",
- peopleAdded,
- peopleUpdated,
+ counters.Added,
+ counters.Updated,
peoplePurged,
- peopleToKeep.Count + peopleToPurge.Count - peopleAdded - peopleUpdated - peoplePurged,
+ peopleToKeep.Count + peopleToPurge.Count - counters.Added - counters.Updated - peoplePurged,
tmdbMovie.EnglishTitle,
tmdbMovie.Id
);
@@ -780,8 +835,8 @@ await ProcessWithConcurrencyAsync(_maxConcurrency, peopleToPurge, async personId
castToRemove.Count > 0 ||
crewToSave.Count > 0 ||
crewToRemove.Count > 0 ||
- peopleAdded > 0 ||
- peopleUpdated > 0 ||
+ counters.Added > 0 ||
+ counters.Updated > 0 ||
peoplePurged > 0;
}
@@ -888,11 +943,11 @@ private async Task DownloadMovieImages(int movieId, TitleLanguage? mainLanguage
var languages = GetLanguages(mainLanguage);
if (settings.TMDB.AutoDownloadPosters)
- await _imageService.DownloadImagesByType(movie.PosterPath, images.Posters!, ImageEntityType.Primary, movie, settings.TMDB.MaxAutoPosters, languages, forceDownload);
+ await _imageService.DownloadImagesByType(movie.PosterPath, images.Posters ?? [], ImageEntityType.Primary, movie, settings.TMDB.MaxAutoPosters, languages, forceDownload);
if (settings.TMDB.AutoDownloadLogos)
- await _imageService.DownloadImagesByType(null, images.Logos!, ImageEntityType.Logo, movie, settings.TMDB.MaxAutoLogos, languages, forceDownload);
+ await _imageService.DownloadImagesByType(null, images.Logos ?? [], ImageEntityType.Logo, movie, settings.TMDB.MaxAutoLogos, languages, forceDownload);
if (settings.TMDB.AutoDownloadBackdrops)
- await _imageService.DownloadImagesByType(movie.BackdropPath, images.Backdrops!, ImageEntityType.Backdrop, movie, settings.TMDB.MaxAutoBackdrops, languages, forceDownload);
+ await _imageService.DownloadImagesByType(movie.BackdropPath, images.Backdrops ?? [], ImageEntityType.Backdrop, movie, settings.TMDB.MaxAutoBackdrops, languages, forceDownload);
}
#endregion
@@ -1195,7 +1250,7 @@ public async Task UpdateShow(TmdbShowUpdateOptions options)
var updated = tmdbShow.Populate(show, contentRatingLanguages);
var (titlesUpdated, overviewsUpdated) = UpdateTitlesAndOverviewsWithTuple(tmdbShow, show.Translations, preferredTitleLanguages, preferredOverviewLanguages);
updated = titlesUpdated || overviewsUpdated || updated;
- updated = UpdateShowExternalIDs(tmdbShow, show.ExternalIds!) || updated;
+ updated = UpdateShowExternalIDs(tmdbShow, show.ExternalIds) || updated;
updated = await UpdateCompanies(tmdbShow, show.ProductionCompanies!) || updated;
var (episodesOrSeasonsUpdated, updatedSeasons, updatedEpisodes, episodeCount, hiddenEpisodeCount) = await UpdateShowSeasonsAndEpisodes(show, downloadCrewAndCast, forceRefresh, downloadImages, quickRefresh, shouldFireEvents, changedItems);
updated = episodesOrSeasonsUpdated || updated;
@@ -1487,7 +1542,7 @@ private static bool AccumulateUnchangedEpisode(
ShowSyncState state)
{
var newlyAdded = tmdbEpisode.CreatedAt == tmdbEpisode.LastUpdatedAt;
- if (!state.ChangedItems.HasValue || newlyAdded || state.ChangedItems.Value.Episodes.Contains((season.SeasonNumber, (int)reducedEpisode.EpisodeNumber)))
+ if (!state.ChangedItems.HasValue || newlyAdded || state.ChangedItems.Value.Episodes.Contains((season.SeasonNumber, reducedEpisode.EpisodeNumber)))
return false;
state.EpisodesToSkip.Add(tmdbEpisode.Id);
@@ -1521,10 +1576,10 @@ private async Task FetchAndPopulateEpisodeAsync(
var episodeUpdated = tmdbEpisode.Populate(show, season, reducedEpisode, episode.Translations);
episodeUpdated = UpdateTitlesAndOverviews(tmdbEpisode, episode.Translations, state.PreferredTitleLanguages, state.PreferredOverviewLanguages) || episodeUpdated;
- episodeUpdated = UpdateEpisodeExternalIDs(tmdbEpisode, episode.ExternalIds!) || episodeUpdated;
+ episodeUpdated = UpdateEpisodeExternalIDs(tmdbEpisode, episode.ExternalIds) || episodeUpdated;
if (state.DownloadCrewAndCast)
{
- var (castOrCrewUpdated, peopleToAddOrKeep, peopleToPotentiallyRemove) = UpdateEpisodeCastAndCrew(tmdbEpisode, episode.Credits!);
+ var (castOrCrewUpdated, peopleToAddOrKeep, peopleToPotentiallyRemove) = UpdateEpisodeCastAndCrew(tmdbEpisode, episode.Credits);
episodeUpdated |= castOrCrewUpdated;
AccumulateEpisodePeople(peopleToAddOrKeep, peopleToPotentiallyRemove, state);
}
@@ -1800,113 +1855,54 @@ private async Task UpdateShowAlternateOrdering(TMDB_Show tmdbShow, TvShow
preferredOrderingUpdated;
}
- private (bool, IEnumerable, IEnumerable) UpdateEpisodeCastAndCrew(TMDB_Episode tmdbEpisode, CreditsWithGuestStars credits)
+ private (bool, IEnumerable, IEnumerable) UpdateEpisodeCastAndCrew(TMDB_Episode tmdbEpisode, CreditsWithGuestStars? credits)
{
- var peopleToAddOrKeep = new HashSet();
- var counter = 0;
- var castToAdd = 0;
- var castToKeep = new HashSet();
- var castToSave = new List();
+ // A present-but-empty `credits` append can deserialize to null members under STJ;
+ // treat that as "no cast/crew this pass" rather than purging existing rows.
+ if (credits?.Cast is null || credits.Crew is null)
+ return (false, [], []);
+
var existingCastDict = _tmdbEpisodeCast.GetByTmdbEpisodeID(tmdbEpisode.Id)
.ToDictionary(cast => cast.TmdbCreditID);
- var guestOffset = credits.Cast!.Count;
- foreach (var cast in credits.Cast.Concat(credits.GuestStars!))
- {
- var ordering = counter++;
- var isGuestRole = ordering >= guestOffset;
- castToKeep.Add(cast.CreditId!);
- peopleToAddOrKeep.Add(cast.Id);
-
- var roleUpdated = false;
- if (!existingCastDict.TryGetValue(cast.CreditId!, out var role))
- {
- role = new()
- {
- TmdbShowID = tmdbEpisode.TmdbShowID,
- TmdbSeasonID = tmdbEpisode.TmdbSeasonID,
- TmdbEpisodeID = tmdbEpisode.Id,
- TmdbPersonID = cast.Id,
- TmdbCreditID = cast.CreditId!,
- Ordering = ordering,
- IsGuestRole = isGuestRole,
- };
- castToAdd++;
- roleUpdated = true;
- }
-
- var characterName = cast.Character!.Replace(" (voice)", "");
- if (role.CharacterName != characterName)
- {
- role.CharacterName = characterName;
- roleUpdated = true;
- }
-
- if (role.Ordering != ordering)
- {
- role.Ordering = ordering;
- roleUpdated = true;
- }
-
- if (role.IsGuestRole != isGuestRole)
+ var guestOffset = credits.Cast.Count;
+ var (castToSave, castToRemove, castToAdd) = DiffCast(
+ credits.Cast.Concat(credits.GuestStars ?? []),
+ existingCastDict,
+ (cast, ordering) => new TMDB_Episode_Cast
+ {
+ TmdbShowID = tmdbEpisode.TmdbShowID,
+ TmdbSeasonID = tmdbEpisode.TmdbSeasonID,
+ TmdbEpisodeID = tmdbEpisode.Id,
+ TmdbPersonID = cast.Id,
+ TmdbCreditID = cast.CreditId!,
+ Ordering = ordering,
+ IsGuestRole = ordering >= guestOffset,
+ },
+ (role, _, ordering) =>
{
+ var isGuestRole = ordering >= guestOffset;
+ if (role.IsGuestRole == isGuestRole)
+ return false;
role.IsGuestRole = isGuestRole;
- roleUpdated = true;
- }
-
- if (roleUpdated)
- {
- castToSave.Add(role);
- }
- }
+ return true;
+ });
- var crewToAdd = 0;
- var crewToKeep = new HashSet();
- var crewToSave = new List();
var existingCrewDict = _tmdbEpisodeCrew.GetByTmdbEpisodeID(tmdbEpisode.Id)
.ToDictionary(crew => crew.TmdbCreditID);
- foreach (var crew in credits.Crew!)
- {
- peopleToAddOrKeep.Add(crew.Id);
- crewToKeep.Add(crew.CreditId!);
- var roleUpdated = false;
- if (!existingCrewDict.TryGetValue(crew.CreditId!, out var role))
- {
- role = new()
- {
- TmdbShowID = tmdbEpisode.TmdbShowID,
- TmdbSeasonID = tmdbEpisode.TmdbSeasonID,
- TmdbEpisodeID = tmdbEpisode.Id,
- TmdbPersonID = crew.Id,
- TmdbCreditID = crew.CreditId!,
- };
- crewToAdd++;
- roleUpdated = true;
- }
-
- if (role.Department != crew.Department)
- {
- role.Department = crew.Department!;
- roleUpdated = true;
- }
-
- if (role.Job != crew.Job)
- {
- role.Job = crew.Job!;
- roleUpdated = true;
- }
-
- if (roleUpdated)
- {
- crewToSave.Add(role);
- }
- }
+ var (crewToSave, crewToRemove, crewToAdd) = DiffCrew(
+ credits.Crew,
+ existingCrewDict,
+ crew => new TMDB_Episode_Crew
+ {
+ TmdbShowID = tmdbEpisode.TmdbShowID,
+ TmdbSeasonID = tmdbEpisode.TmdbSeasonID,
+ TmdbEpisodeID = tmdbEpisode.Id,
+ TmdbPersonID = crew.Id,
+ TmdbCreditID = crew.CreditId!,
+ });
- var castToRemove = existingCastDict.Values
- .ExceptBy(castToKeep, cast => cast.TmdbCreditID)
- .ToList();
- var crewToRemove = existingCrewDict.Values
- .ExceptBy(crewToKeep, crew => crew.TmdbCreditID)
- .ToList();
+ var peopleToAddOrKeep = new HashSet(
+ credits.Cast.Concat(credits.GuestStars ?? []).Select(cast => cast.Id).Concat(credits.Crew.Select(crew => crew.Id)));
_tmdbEpisodeCast.Save(castToSave);
_tmdbEpisodeCrew.Save(crewToSave);
@@ -2080,11 +2076,11 @@ private async Task DownloadShowImages(int showId, TitleLanguage? mainLanguage =
var languages = GetLanguages(mainLanguage);
if (settings.TMDB.AutoDownloadPosters)
- await _imageService.DownloadImagesByType(show.PosterPath, images.Posters!, ImageEntityType.Primary, show, settings.TMDB.MaxAutoPosters, languages, forceDownload);
+ await _imageService.DownloadImagesByType(show.PosterPath, images.Posters ?? [], ImageEntityType.Primary, show, settings.TMDB.MaxAutoPosters, languages, forceDownload);
if (settings.TMDB.AutoDownloadLogos)
- await _imageService.DownloadImagesByType(null, images.Logos!, ImageEntityType.Logo, show, settings.TMDB.MaxAutoLogos, languages, forceDownload);
+ await _imageService.DownloadImagesByType(null, images.Logos ?? [], ImageEntityType.Logo, show, settings.TMDB.MaxAutoLogos, languages, forceDownload);
if (settings.TMDB.AutoDownloadBackdrops)
- await _imageService.DownloadImagesByType(show.BackdropPath, images.Backdrops!, ImageEntityType.Backdrop, show, settings.TMDB.MaxAutoBackdrops, languages, forceDownload);
+ await _imageService.DownloadImagesByType(show.BackdropPath, images.Backdrops ?? [], ImageEntityType.Backdrop, show, settings.TMDB.MaxAutoBackdrops, languages, forceDownload);
}
private async Task DownloadSeasonImages(int seasonId, int showId, int seasonNumber, TitleLanguage? mainLanguage = null, bool forceDownload = false)
@@ -2103,7 +2099,7 @@ private async Task DownloadSeasonImages(int seasonId, int showId, int seasonNumb
return;
var languages = GetLanguages(mainLanguage);
- await _imageService.DownloadImagesByType(season.PosterPath, images.Posters!, ImageEntityType.Primary, season, settings.TMDB.MaxAutoPosters, languages, forceDownload);
+ await _imageService.DownloadImagesByType(season.PosterPath, images.Posters ?? [], ImageEntityType.Primary, season, settings.TMDB.MaxAutoPosters, languages, forceDownload);
}
private async Task DownloadEpisodeImages(int episodeId, int showId, int seasonNumber, int episodeNumber, TitleLanguage mainLanguage, bool forceDownload = false)
@@ -2122,7 +2118,7 @@ private async Task DownloadEpisodeImages(int episodeId, int showId, int seasonNu
return;
var languages = GetLanguages(mainLanguage);
- await _imageService.DownloadImagesByType(episode.ThumbnailPath, images.Stills!, ImageEntityType.Backdrop, episode, settings.TMDB.MaxAutoThumbnails, languages, forceDownload);
+ await _imageService.DownloadImagesByType(episode.ThumbnailPath, images.Stills ?? [], ImageEntityType.Backdrop, episode, settings.TMDB.MaxAutoThumbnails, languages, forceDownload);
}
private List GetLanguages(TitleLanguage? mainLanguage = null) => _settingsProvider.GetSettings().TMDB.ImageLanguageOrder
@@ -2408,8 +2404,12 @@ private bool UpdateTitlesAndOverviews(IEntityMetadata tmdbEntity, TranslationsCo
var titlesToSave = new List();
foreach (var translation in translations?.Translations ?? [new() { EnglishName = string.Empty, Iso_3166_1 = "US", Iso_639_1 = "en", Data = new() { Name = string.Empty, Overview = string.Empty } }])
{
- var languageCode = translation.Iso_639_1!.ToLowerInvariant();
- var countryCode = translation.Iso_3166_1!.ToUpperInvariant();
+ // A translation entry without locale codes is unusable; skip rather than NRE.
+ if (translation.Iso_639_1 is null || translation.Iso_3166_1 is null)
+ continue;
+
+ var languageCode = translation.Iso_639_1.ToLowerInvariant();
+ var countryCode = translation.Iso_3166_1.ToUpperInvariant();
var alwaysInclude = false;
var currentTitle = translation.Data?.Name ?? string.Empty;
@@ -2737,7 +2737,7 @@ private async Task DownloadPersonImages(int personId, ProfileImages images, bool
if (_tmdbPeople.GetByTmdbPersonID(personId) is not { } person)
return;
- await _imageService.DownloadImagesByType(null, images.Profiles!, ImageEntityType.Primary, person, settings.TMDB.MaxAutoStaffImages, [], forceDownload);
+ await _imageService.DownloadImagesByType(null, images.Profiles ?? [], ImageEntityType.Primary, person, settings.TMDB.MaxAutoStaffImages, [], forceDownload);
}
public async Task PurgeUnlinkedPeople()
@@ -2934,18 +2934,10 @@ private static async Task ProcessWithConcurrencyAsync(
/// TMDB Show.
/// External IDs.
/// Indicates that the ID was updated.
- private bool UpdateShowExternalIDs(TMDB_Show show, ExternalIdsTvShow externalIds)
+ internal static bool UpdateShowExternalIDs(TMDB_Show show, ExternalIdsTvShow? externalIds)
{
- if (string.IsNullOrEmpty(externalIds.TvdbId))
- {
- if (!show.TvdbShowID.HasValue)
- return false;
-
- show.TvdbShowID = null;
- return true;
- }
-
- if (!int.TryParse(externalIds.TvdbId, out var tvdbId) || tvdbId <= 0 || show.TvdbShowID == tvdbId)
+ var tvdbId = externalIds?.TvdbId is > 0 ? externalIds.TvdbId : null;
+ if (show.TvdbShowID == tvdbId)
return false;
show.TvdbShowID = tvdbId;
@@ -2958,18 +2950,10 @@ private bool UpdateShowExternalIDs(TMDB_Show show, ExternalIdsTvShow externalIds
/// TMDB Episode.
/// External IDs.
/// Indicates that the ID was updated.
- private bool UpdateEpisodeExternalIDs(TMDB_Episode episode, ExternalIdsTvEpisode externalIds)
+ internal static bool UpdateEpisodeExternalIDs(TMDB_Episode episode, ExternalIdsTvEpisode? externalIds)
{
- if (string.IsNullOrEmpty(externalIds.TvdbId))
- {
- if (!episode.TvdbEpisodeID.HasValue)
- return false;
-
- episode.TvdbEpisodeID = null;
- return true;
- }
-
- if (!int.TryParse(externalIds.TvdbId, out var tvdbId) || tvdbId <= 0 || episode.TvdbEpisodeID == tvdbId)
+ var tvdbId = externalIds?.TvdbId is > 0 ? externalIds.TvdbId : null;
+ if (episode.TvdbEpisodeID == tvdbId)
return false;
episode.TvdbEpisodeID = tvdbId;
@@ -2982,12 +2966,12 @@ private bool UpdateEpisodeExternalIDs(TMDB_Episode episode, ExternalIdsTvEpisode
/// TMDB Movie.
/// External IDs.
/// Indicates that the ID was updated.
- private bool UpdateMovieExternalIDs(TMDB_Movie movie, ExternalIdsMovie externalIds)
+ internal static bool UpdateMovieExternalIDs(TMDB_Movie movie, ExternalIdsMovie? externalIds)
{
- if (movie.ImdbMovieID == externalIds.ImdbId)
+ if (movie.ImdbMovieID == externalIds?.ImdbId)
return false;
- movie.ImdbMovieID = externalIds.ImdbId;
+ movie.ImdbMovieID = externalIds?.ImdbId;
return true;
}
diff --git a/Shoko.Server/Providers/TMDB/TmdbSearchService.cs b/Shoko.Server/Providers/TMDB/TmdbSearchService.cs
index c9c81e9e1..67dd91c98 100644
--- a/Shoko.Server/Providers/TMDB/TmdbSearchService.cs
+++ b/Shoko.Server/Providers/TMDB/TmdbSearchService.cs
@@ -264,7 +264,7 @@ private async Task AutoSearchForMovie(List list, Ani
// Attempt #1: full title + year
(results, _) = await SearchMoviesRaw(query, includeRestricted: includeRestricted, year: year).ConfigureAwait(false);
- CollectMovieCandidates(candidates, results, seen, candidateCount);
+ CollectMovieCandidates(candidates, results, seen, candidateCount, includeRestricted);
// Attempt #2: sequel-suffix stripped + year
var strippedTitle = SequelSuffixRemovalRegex().Match(query) is { Success: true } regexResult
@@ -272,7 +272,7 @@ private async Task AutoSearchForMovie(List list, Ani
if (!string.IsNullOrEmpty(strippedTitle) && candidates.Count < candidateCount)
{
(results, _) = await SearchMoviesRaw(strippedTitle, includeRestricted: includeRestricted, year: year).ConfigureAwait(false);
- CollectMovieCandidates(candidates, results, seen, candidateCount);
+ CollectMovieCandidates(candidates, results, seen, candidateCount, includeRestricted);
}
// Attempts #3–4: year-free fallbacks mirroring #1–2.
@@ -282,12 +282,12 @@ private async Task AutoSearchForMovie(List list, Ani
var yearFreeCap = candidateCount * 2;
(results, _) = await SearchMoviesRaw(query, includeRestricted: includeRestricted).ConfigureAwait(false);
- CollectMovieCandidates(candidates, results, seen, yearFreeCap);
+ CollectMovieCandidates(candidates, results, seen, yearFreeCap, includeRestricted);
if (!string.IsNullOrEmpty(strippedTitle) && candidates.Count < yearFreeCap)
{
(results, _) = await SearchMoviesRaw(strippedTitle, includeRestricted: includeRestricted).ConfigureAwait(false);
- CollectMovieCandidates(candidates, results, seen, yearFreeCap);
+ CollectMovieCandidates(candidates, results, seen, yearFreeCap, includeRestricted);
}
if (candidates.Count == 0)
@@ -575,7 +575,7 @@ private async Task> AutoSearchForShow(AniDB_
// Attempt #1: full title + year
(results, _) = await SearchShowsRaw(originalTitle, includeRestricted: restricted, year: airDate.Year).ConfigureAwait(false);
- CollectCandidates(candidates, results, seen, candidateCount);
+ CollectCandidates(candidates, results, seen, candidateCount, restricted);
// Attempt #2: sequel-suffix stripped + year
var strippedTitle = SequelSuffixRemovalRegex().Match(originalTitle) is { Success: true } regexResult
@@ -583,7 +583,7 @@ private async Task> AutoSearchForShow(AniDB_
if (!string.IsNullOrEmpty(strippedTitle) && candidates.Count < candidateCount)
{
(results, _) = await SearchShowsRaw(strippedTitle, includeRestricted: restricted, year: airDate.Year).ConfigureAwait(false);
- CollectCandidates(candidates, results, seen, candidateCount);
+ CollectCandidates(candidates, results, seen, candidateCount, restricted);
}
// Attempt #3: subtitle stripped + year.
@@ -597,7 +597,7 @@ private async Task> AutoSearchForShow(AniDB_
if (!string.IsNullOrEmpty(titleWithoutSubTitle) && candidates.Count < candidateCount)
{
(results, _) = await SearchShowsRaw(titleWithoutSubTitle, includeRestricted: restricted, year: airDate.Year).ConfigureAwait(false);
- CollectCandidates(candidates, results, seen, candidateCount);
+ CollectCandidates(candidates, results, seen, candidateCount, restricted);
}
// Attempts #4–6: year-free fallbacks mirroring #1–3.
@@ -609,18 +609,18 @@ private async Task> AutoSearchForShow(AniDB_
var yearFreeCap = candidateCount * 2;
(results, _) = await SearchShowsRaw(originalTitle, includeRestricted: restricted).ConfigureAwait(false);
- CollectCandidates(candidates, results, seen, yearFreeCap);
+ CollectCandidates(candidates, results, seen, yearFreeCap, restricted);
if (!string.IsNullOrEmpty(strippedTitle) && candidates.Count < yearFreeCap)
{
(results, _) = await SearchShowsRaw(strippedTitle, includeRestricted: restricted).ConfigureAwait(false);
- CollectCandidates(candidates, results, seen, yearFreeCap);
+ CollectCandidates(candidates, results, seen, yearFreeCap, restricted);
}
if (!string.IsNullOrEmpty(titleWithoutSubTitle) && candidates.Count < yearFreeCap)
{
(results, _) = await SearchShowsRaw(titleWithoutSubTitle, includeRestricted: restricted).ConfigureAwait(false);
- CollectCandidates(candidates, results, seen, yearFreeCap);
+ CollectCandidates(candidates, results, seen, yearFreeCap, restricted);
}
if (candidates.Count == 0)
@@ -770,24 +770,30 @@ private static bool MovieMatchesYear(TMDbLib.Objects.Movies.Movie movie, int yea
// applies the same floor for show and movie auto-matching.
internal static bool IsAcceptableAutoMatch(MatchRating rating) => rating is not MatchRating.FirstAvailable;
- private static void CollectCandidates(List candidates, List results, HashSet seen, int candidateCount)
+ // Restricted titles also accept a Japanese original_language match — TMDB's genre tags are sparse
+ // for adult content, but original_language is TMDB-assigned metadata, not community-tagged.
+ private static void CollectCandidates(List candidates, List results, HashSet seen, int candidateCount, bool isRestricted = false)
{
foreach (var result in results)
{
if (candidates.Count >= candidateCount) break;
if (!seen.Add(result.Id)) continue;
- if (!result.GetGenres().Contains(AnimationGenre, StringComparer.OrdinalIgnoreCase)) continue;
+ var isAnimeLike = result.GetGenres().Contains(AnimationGenre, StringComparer.OrdinalIgnoreCase) ||
+ (isRestricted && string.Equals(result.OriginalLanguage, "ja", StringComparison.OrdinalIgnoreCase));
+ if (!isAnimeLike) continue;
candidates.Add(result);
}
}
- private static void CollectMovieCandidates(List candidates, List results, HashSet seen, int candidateCount)
+ private static void CollectMovieCandidates(List candidates, List results, HashSet seen, int candidateCount, bool isRestricted = false)
{
foreach (var result in results)
{
if (candidates.Count >= candidateCount) break;
if (!seen.Add(result.Id)) continue;
- if (!result.GetGenres().Contains(AnimationGenre, StringComparer.OrdinalIgnoreCase)) continue;
+ var isAnimeLike = result.GetGenres().Contains(AnimationGenre, StringComparer.OrdinalIgnoreCase) ||
+ (isRestricted && string.Equals(result.OriginalLanguage, "ja", StringComparison.OrdinalIgnoreCase));
+ if (!isAnimeLike) continue;
candidates.Add(result);
}
}
diff --git a/Shoko.Server/Shoko.Server.csproj b/Shoko.Server/Shoko.Server.csproj
index d75548a98..7771988de 100644
--- a/Shoko.Server/Shoko.Server.csproj
+++ b/Shoko.Server/Shoko.Server.csproj
@@ -50,6 +50,7 @@
+
@@ -94,7 +95,6 @@
-
diff --git a/Shoko.Tests/Providers/TMDB/TmdbAppendToResponseDeserializationTests.cs b/Shoko.Tests/Providers/TMDB/TmdbAppendToResponseDeserializationTests.cs
new file mode 100644
index 000000000..ea33aff25
--- /dev/null
+++ b/Shoko.Tests/Providers/TMDB/TmdbAppendToResponseDeserializationTests.cs
@@ -0,0 +1,177 @@
+using TMDbLib.Objects.Movies;
+using TMDbLib.Objects.Search;
+using TMDbLib.Objects.TvShows;
+using TMDbLib.Utilities.Serializer;
+using Xunit;
+
+namespace Shoko.Tests.Providers.TMDB;
+
+///
+/// Guards the TMDbLib master upgrade (jellyfin/TMDbLib#626 + System.Text.Json migration).
+/// The upgrade keeps the exact same TMDB requests (verified by the reference-library
+/// re-scan, tasks T018/T023 — identical request counts and byte-for-byte row parity), so
+/// the only new risk is the wire→model binding: STJ presence handling of appended
+/// sub-resources and the tvdb_id type change (string → int). These replay the
+/// append_to_response payload shapes that
+/// requests and assert each sub-resource Shoko reads still deserializes.
+///
+public class TmdbAppendToResponseDeserializationTests
+{
+ private static T Deserialize(string json) where T : class
+ => TMDbJsonSerializer.Instance.DeserializeFromString(json)
+ ?? throw new Xunit.Sdk.XunitException($"Deserialized {typeof(T).Name} was null");
+
+ // Movie: MovieMethods.Translations | ReleaseDates | ExternalIds | Keywords | Credits
+ private const string MovieJson = """
+ {
+ "id": 149,
+ "title": "Akira",
+ "translations": { "id": 149, "translations": [
+ { "iso_639_1": "ja", "iso_3166_1": "JP", "name": "日本語", "english_name": "Japanese",
+ "data": { "title": "AKIRA", "overview": "あらすじ" } } ] },
+ "release_dates": { "results": [
+ { "iso_3166_1": "JP", "release_dates": [ { "certification": "", "type": 3, "release_date": "1988-07-16T00:00:00.000Z" } ] } ] },
+ "external_ids": { "imdb_id": "tt0094625", "facebook_id": null, "twitter_id": null },
+ "keywords": { "keywords": [ { "id": 970, "name": "anime" }, { "id": 9951, "name": "dystopia" } ] },
+ "credits": { "cast": [ { "id": 1, "name": "A" } ], "crew": [ { "id": 2, "name": "B" } ] }
+ }
+ """;
+
+ // TV show: TvShowMethods.ContentRatings | Translations | ExternalIds | Keywords | EpisodeGroups
+ private const string ShowJson = """
+ {
+ "id": 45782,
+ "name": "Sword Art Online",
+ "content_ratings": { "results": [ { "iso_3166_1": "US", "rating": "TV-14" } ] },
+ "translations": { "id": 45782, "translations": [
+ { "iso_639_1": "de", "iso_3166_1": "DE", "name": "Deutsch", "english_name": "German",
+ "data": { "name": "Sword Art Online", "overview": "In naher Zukunft..." } } ] },
+ "external_ids": { "imdb_id": "tt2250192", "tvdb_id": 259140, "facebook_id": null },
+ "keywords": { "results": [ { "id": 6091, "name": "war" } ] },
+ "episode_groups": { "results": [
+ { "id": "5f8a1d...", "name": "Alternate order", "group_count": 4, "episode_count": 96, "type": 2 } ] }
+ }
+ """;
+
+ // TV episode: TvEpisodeMethods.ExternalIds | Translations | Credits
+ private const string EpisodeJson = """
+ {
+ "id": 979329,
+ "season_number": 1,
+ "episode_number": 1,
+ "name": "The World of Swords",
+ "external_ids": { "imdb_id": "tt2683910", "tvdb_id": 4298471 },
+ "translations": { "id": 979329, "translations": [
+ { "iso_639_1": "ja", "iso_3166_1": "JP", "name": "", "english_name": "Japanese",
+ "data": { "name": "剣の世界", "overview": "" } } ] },
+ "credits": { "cast": [ { "id": 1, "name": "A" } ], "crew": [], "guest_stars": [ { "id": 3, "name": "G" } ] }
+ }
+ """;
+
+ // TV season: TvSeasonMethods.Translations (episode stubs come back on the season body)
+ private const string SeasonJson = """
+ {
+ "id": 61862,
+ "season_number": 1,
+ "episodes": [
+ { "id": 979329, "season_number": 1, "episode_number": 1, "name": "Ep1" },
+ { "id": 979330, "season_number": 1, "episode_number": 2, "name": "Ep2" }
+ ],
+ "translations": { "id": 61862, "translations": [
+ { "iso_639_1": "fr", "iso_3166_1": "FR", "name": "Français", "english_name": "French",
+ "data": { "name": "Aincrad", "overview": "..." } } ] }
+ }
+ """;
+
+ [Fact]
+ public void Movie_AllAppendedSubResources_Bind()
+ {
+ var movie = Deserialize(MovieJson);
+
+ Assert.Equal("Akira", movie.Title);
+ Assert.Equal("ja", Assert.Single(movie.Translations!.Translations!).Iso_639_1);
+ Assert.Equal("あらすじ", movie.Translations!.Translations![0].Data!.Overview);
+ Assert.Equal("JP", Assert.Single(movie.ReleaseDates!.Results!).Iso_3166_1);
+ Assert.Equal("tt0094625", movie.ExternalIds!.ImdbId);
+ Assert.Equal(2, movie.Keywords!.Keywords!.Count);
+ Assert.Single(movie.Credits!.Cast!);
+ Assert.Single(movie.Credits!.Crew!);
+ }
+
+ [Fact]
+ public void Show_AllAppendedSubResources_Bind()
+ {
+ var show = Deserialize(ShowJson);
+
+ Assert.Equal("Sword Art Online", show.Name);
+ Assert.Equal("TV-14", Assert.Single(show.ContentRatings!.Results!).Rating);
+ Assert.Equal("de", Assert.Single(show.Translations!.Translations!).Iso_639_1);
+ Assert.Equal("tt2250192", show.ExternalIds!.ImdbId);
+ Assert.Single(show.Keywords!.Results!);
+ Assert.Equal(4, Assert.Single(show.EpisodeGroups!.Results!).GroupCount);
+ }
+
+ [Fact]
+ public void Show_TvdbId_DeserializesAsInteger()
+ {
+ // jellyfin/TMDbLib#626: ExternalIdsTvShow.TvdbId went string? -> int? (TMDB sends it as a
+ // JSON number). UpdateShowExternalIDs / UpdateEpisodeExternalIDs rely on this — see task T022.
+ var show = Deserialize(ShowJson);
+ Assert.Equal(259140, show.ExternalIds!.TvdbId);
+
+ var episode = Deserialize(EpisodeJson);
+ Assert.Equal(4298471, episode.ExternalIds!.TvdbId);
+ }
+
+ [Fact]
+ public void Episode_AllAppendedSubResources_Bind()
+ {
+ var episode = Deserialize(EpisodeJson);
+
+ Assert.Equal(1, episode.SeasonNumber);
+ Assert.Equal(1, episode.EpisodeNumber);
+ Assert.Equal("tt2683910", episode.ExternalIds!.ImdbId);
+ Assert.Equal("ja", Assert.Single(episode.Translations!.Translations!).Iso_639_1);
+ Assert.Single(episode.Credits!.Cast!);
+ Assert.Empty(episode.Credits!.Crew!);
+ Assert.Single(episode.Credits!.GuestStars!);
+ }
+
+ [Fact]
+ public void SearchResults_GenreIds_Bind()
+ {
+ // TmdbSearchService's restricted auto-match + animation-genre ordering read
+ // SearchTv/SearchMovie.GenreIds (task T029). Confirm STJ still binds genre_ids.
+ var show = Deserialize("""{ "id": 45782, "name": "SAO", "genre_ids": [16, 10765] }""");
+ Assert.Equal(new[] { 16, 10765 }, show.GenreIds);
+
+ var movie = Deserialize("""{ "id": 149, "title": "Akira", "genre_ids": [16, 28, 878] }""");
+ Assert.Equal(new[] { 16, 28, 878 }, movie.GenreIds);
+ }
+
+ [Fact]
+ public void Season_TranslationsAndEpisodeStubs_Bind()
+ {
+ var season = Deserialize(SeasonJson);
+
+ Assert.Equal(1, season.SeasonNumber);
+ Assert.Equal(2, season.Episodes!.Count);
+ Assert.Equal("fr", Assert.Single(season.Translations!.Translations!).Iso_639_1);
+ }
+
+ [Theory]
+ [InlineData("""{ "id": 149, "title": "Akira" }""")]
+ [InlineData("""{ "id": 149, "title": "Akira", "external_ids": null, "keywords": null, "credits": null, "translations": null, "release_dates": null }""")]
+ public void Movie_MissingOrNullSubResources_LeaveContainersNull(string json)
+ {
+ // STJ presence handling: absent and explicit-null must both land as null, not empty
+ // containers — TmdbMetadataService's null-guards (task T016) key off exactly this.
+ var movie = Deserialize(json);
+
+ Assert.Null(movie.Translations);
+ Assert.Null(movie.ReleaseDates);
+ Assert.Null(movie.ExternalIds);
+ Assert.Null(movie.Keywords);
+ Assert.Null(movie.Credits);
+ }
+}
diff --git a/Shoko.Tests/Providers/TMDB/TmdbEpisodeTypeTests.cs b/Shoko.Tests/Providers/TMDB/TmdbEpisodeTypeTests.cs
new file mode 100644
index 000000000..6e937ff5d
--- /dev/null
+++ b/Shoko.Tests/Providers/TMDB/TmdbEpisodeTypeTests.cs
@@ -0,0 +1,57 @@
+using Shoko.Server.Models.TMDB;
+using TMDbLib.Objects.Search;
+using TMDbLib.Objects.TvShows;
+using Xunit;
+
+namespace Shoko.Tests.Providers.TMDB;
+
+///
+/// User Story 5 / O4: capture TMDB's own episode_type. TMDbLib master exposes it on
+/// (and TvEpisodeBase); before the upgrade
+/// Shoko had no episode-type signal from TMDB. This only persists the value (and surfaces it
+/// on the v3 DTO) — it deliberately does NOT feed episode matching (that would be a
+/// match-selection change, out of scope here; tracked as a follow-up).
+///
+public class TmdbEpisodeTypeTests
+{
+ private static readonly TvShow Show = new() { Id = 45782, Name = "SAO" };
+ private static readonly TvSeason Season = new() { Id = 61862, SeasonNumber = 1 };
+
+ private static TvSeasonEpisode ReducedEpisode(string? episodeType, int seasonNumber = 1) => new()
+ {
+ Id = 979329,
+ SeasonNumber = seasonNumber,
+ EpisodeNumber = 1,
+ Name = "Ep",
+ Overview = "",
+ StillPath = "",
+ EpisodeType = episodeType,
+ };
+
+ [Theory]
+ [InlineData("standard", "standard")]
+ [InlineData("finale", "finale")]
+ [InlineData("mid_season", "mid_season")]
+ [InlineData("special", "special")]
+ [InlineData(" finale ", "finale")]
+ [InlineData(null, null)]
+ [InlineData("", null)]
+ [InlineData(" ", null)]
+ public void Populate_StoresTrimmedEpisodeTypeOrNull(string? wire, string? expected)
+ {
+ var episode = new TMDB_Episode();
+ episode.Populate(Show, Season, ReducedEpisode(wire), translations: null);
+
+ Assert.Equal(expected, episode.TmdbEpisodeType);
+ }
+
+ [Fact]
+ public void Populate_EpisodeTypeChange_IsReportedAsAnUpdate()
+ {
+ var episode = new TMDB_Episode();
+ episode.Populate(Show, Season, ReducedEpisode("standard"), translations: null);
+
+ Assert.True(episode.Populate(Show, Season, ReducedEpisode("finale"), translations: null));
+ Assert.False(episode.Populate(Show, Season, ReducedEpisode("finale"), translations: null));
+ }
+}
diff --git a/Shoko.Tests/Providers/TMDB/TmdbExternalIdTests.cs b/Shoko.Tests/Providers/TMDB/TmdbExternalIdTests.cs
new file mode 100644
index 000000000..d830eb1cf
--- /dev/null
+++ b/Shoko.Tests/Providers/TMDB/TmdbExternalIdTests.cs
@@ -0,0 +1,91 @@
+using Shoko.Server.Models.TMDB;
+using Shoko.Server.Providers.TMDB;
+using TMDbLib.Objects.General;
+using Xunit;
+
+namespace Shoko.Tests.Providers.TMDB;
+
+///
+/// User Story 2: external-ID cross-links stay correct after the TMDbLib upgrade.
+/// ExternalIdsTvShow/TvEpisode.TvdbId changed from string? to int?
+/// (jellyfin/TMDbLib#626); these pin the Update*ExternalIDs handlers to the
+/// post-upgrade type and the "clear when absent / non-positive" contract.
+///
+public class TmdbExternalIdTests
+{
+ [Theory]
+ [InlineData(259140, 259140)]
+ [InlineData(1, 1)]
+ [InlineData(0, null)]
+ [InlineData(-5, null)]
+ [InlineData(null, null)]
+ public void UpdateShowExternalIDs_StoresPositiveTvdbIdElseNull(int? wireTvdbId, int? expected)
+ {
+ var show = new TMDB_Show();
+ var changed = TmdbMetadataService.UpdateShowExternalIDs(show, new ExternalIdsTvShow { TvdbId = wireTvdbId });
+
+ Assert.Equal(expected, show.TvdbShowID);
+ Assert.Equal(expected is not null, changed);
+ }
+
+ [Theory]
+ [InlineData(4298471, 4298471)]
+ [InlineData(0, null)]
+ [InlineData(null, null)]
+ public void UpdateEpisodeExternalIDs_StoresPositiveTvdbIdElseNull(int? wireTvdbId, int? expected)
+ {
+ var episode = new TMDB_Episode();
+ var changed = TmdbMetadataService.UpdateEpisodeExternalIDs(episode, new ExternalIdsTvEpisode { TvdbId = wireTvdbId });
+
+ Assert.Equal(expected, episode.TvdbEpisodeID);
+ Assert.Equal(expected is not null, changed);
+ }
+
+ [Fact]
+ public void UpdateShowExternalIDs_PreviouslySetThenRemoved_IsCleared()
+ {
+ var show = new TMDB_Show { TvdbShowID = 259140 };
+
+ Assert.True(TmdbMetadataService.UpdateShowExternalIDs(show, new ExternalIdsTvShow { TvdbId = null }));
+ Assert.Null(show.TvdbShowID);
+ }
+
+ [Fact]
+ public void UpdateShowExternalIDs_UnchangedValue_ReturnsFalse()
+ {
+ var show = new TMDB_Show { TvdbShowID = 259140 };
+
+ Assert.False(TmdbMetadataService.UpdateShowExternalIDs(show, new ExternalIdsTvShow { TvdbId = 259140 }));
+ Assert.Equal(259140, show.TvdbShowID);
+ }
+
+ [Fact]
+ public void UpdateShowExternalIDs_NullContainer_ClearsWithoutThrowing()
+ {
+ var show = new TMDB_Show { TvdbShowID = 259140 };
+
+ Assert.True(TmdbMetadataService.UpdateShowExternalIDs(show, null));
+ Assert.Null(show.TvdbShowID);
+ }
+
+ [Theory]
+ [InlineData("tt0094625")]
+ [InlineData(null)]
+ public void UpdateMovieExternalIDs_ImdbId_StoredVerbatim(string? imdbId)
+ {
+ var movie = new TMDB_Movie();
+ var changed = TmdbMetadataService.UpdateMovieExternalIDs(movie, new ExternalIdsMovie { ImdbId = imdbId });
+
+ Assert.Equal(imdbId, movie.ImdbMovieID);
+ Assert.Equal(imdbId is not null, changed);
+ }
+
+ [Fact]
+ public void UpdateMovieExternalIDs_NullContainer_ClearsWithoutThrowing()
+ {
+ var movie = new TMDB_Movie { ImdbMovieID = "tt0094625" };
+
+ Assert.True(TmdbMetadataService.UpdateMovieExternalIDs(movie, null));
+ Assert.Null(movie.ImdbMovieID);
+ }
+}
diff --git a/Shoko.Tests/Providers/TMDB/TmdbSearchServiceTests.cs b/Shoko.Tests/Providers/TMDB/TmdbSearchServiceTests.cs
index d9c0be414..56f5f848f 100644
--- a/Shoko.Tests/Providers/TMDB/TmdbSearchServiceTests.cs
+++ b/Shoko.Tests/Providers/TMDB/TmdbSearchServiceTests.cs
@@ -1,8 +1,10 @@
using System.Collections.Generic;
+using System.Reflection;
using Shoko.Abstractions.Metadata.Enums;
using Shoko.Server.Filters;
using Shoko.Server.Providers.TMDB;
using Shoko.Server.Utilities;
+using TMDbLib.Objects.Search;
using Xunit;
// ReSharper disable StringLiteralTypo
@@ -226,4 +228,79 @@ public void IsAcceptableAutoMatch_RejectsOnlyFirstAvailable(MatchRating rating,
{
Assert.Equal(expectedAcceptable, TmdbSearchService.IsAcceptableAutoMatch(rating));
}
+
+ // TmdbMetadataService.Instance is unset in this test host, so GetGenres() always returns empty —
+ // every result below is effectively untagged; only OriginalLanguage varies.
+
+ private static void InvokeCollectCandidates(List candidates, List results, HashSet seen, int candidateCount, bool isRestricted)
+ {
+ var method = typeof(TmdbSearchService).GetMethod("CollectCandidates", BindingFlags.NonPublic | BindingFlags.Static)!;
+ method.Invoke(null, [candidates, results, seen, candidateCount, isRestricted]);
+ }
+
+ private static void InvokeCollectMovieCandidates(List candidates, List results, HashSet seen, int candidateCount, bool isRestricted)
+ {
+ var method = typeof(TmdbSearchService).GetMethod("CollectMovieCandidates", BindingFlags.NonPublic | BindingFlags.Static)!;
+ method.Invoke(null, [candidates, results, seen, candidateCount, isRestricted]);
+ }
+
+ [Fact]
+ public void CollectCandidates_NonRestricted_ExcludesUntaggedResult()
+ {
+ var candidates = new List();
+ var results = new List { new() { Id = 1 } };
+ InvokeCollectCandidates(candidates, results, [], 10, false);
+
+ Assert.Empty(candidates);
+ }
+
+ [Fact]
+ public void CollectCandidates_Restricted_ExcludesUntaggedNonJapaneseResult()
+ {
+ var candidates = new List();
+ var results = new List { new() { Id = 1, OriginalLanguage = "en" } };
+ InvokeCollectCandidates(candidates, results, [], 10, true);
+
+ Assert.Empty(candidates);
+ }
+
+ [Fact]
+ public void CollectCandidates_Restricted_IncludesUntaggedJapaneseResult()
+ {
+ var candidates = new List();
+ var results = new List { new() { Id = 1, OriginalLanguage = "ja" } };
+ InvokeCollectCandidates(candidates, results, [], 10, true);
+
+ Assert.Single(candidates);
+ }
+
+ [Fact]
+ public void CollectMovieCandidates_NonRestricted_ExcludesUntaggedResult()
+ {
+ var candidates = new List();
+ var results = new List { new() { Id = 1 } };
+ InvokeCollectMovieCandidates(candidates, results, [], 10, false);
+
+ Assert.Empty(candidates);
+ }
+
+ [Fact]
+ public void CollectMovieCandidates_Restricted_ExcludesUntaggedNonJapaneseResult()
+ {
+ var candidates = new List();
+ var results = new List { new() { Id = 1, OriginalLanguage = "en" } };
+ InvokeCollectMovieCandidates(candidates, results, [], 10, true);
+
+ Assert.Empty(candidates);
+ }
+
+ [Fact]
+ public void CollectMovieCandidates_Restricted_IncludesUntaggedJapaneseResult()
+ {
+ var candidates = new List();
+ var results = new List { new() { Id = 1, OriginalLanguage = "ja" } };
+ InvokeCollectMovieCandidates(candidates, results, [], 10, true);
+
+ Assert.Single(candidates);
+ }
}
diff --git a/ext/TMDbLib b/ext/TMDbLib
new file mode 160000
index 000000000..a4a3fbbf5
--- /dev/null
+++ b/ext/TMDbLib
@@ -0,0 +1 @@
+Subproject commit a4a3fbbf5429b6ceffed8b1d8206dc98f785537c