diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 3362deb8e0..b94ef6e95a 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -28,8 +28,15 @@ jobs: - name: Run integration tests env: DB_TYPE: SQLite + SHOKO_SCHEMA_DIR: ${{ github.workspace }}/schema-dumps run: dotnet test Shoko.IntegrationTests/Shoko.IntegrationTests.csproj -c Release --logger "console;verbosity=normal" + - name: Upload schema dump + uses: actions/upload-artifact@v7 + with: + name: schema-SQLite + path: schema-dumps/schema-SQLite.json + test-mysql: runs-on: ubuntu-latest name: Integration Tests — MySQL (MariaDB) @@ -67,8 +74,15 @@ jobs: DB_USER: root DB_PASS: root DB_NAME: shoko + SHOKO_SCHEMA_DIR: ${{ github.workspace }}/schema-dumps run: dotnet test Shoko.IntegrationTests/Shoko.IntegrationTests.csproj -c Release --logger "console;verbosity=normal" + - name: Upload schema dump + uses: actions/upload-artifact@v7 + with: + name: schema-MySQL + path: schema-dumps/schema-MySQL.json + test-mssql: runs-on: ubuntu-latest name: Integration Tests — SQL Server @@ -107,4 +121,42 @@ jobs: DB_USER: sa DB_PASS: "ShokoTest1!" DB_NAME: shoko + SHOKO_SCHEMA_DIR: ${{ github.workspace }}/schema-dumps run: dotnet test Shoko.IntegrationTests/Shoko.IntegrationTests.csproj -c Release --logger "console;verbosity=normal" + + - name: Upload schema dump + uses: actions/upload-artifact@v7 + with: + name: schema-SQLServer + path: schema-dumps/schema-SQLServer.json + + schema-parity: + runs-on: ubuntu-latest + name: Schema Parity — SQLite vs MySQL vs SQL Server + needs: [test-sqlite, test-mysql, test-mssql] + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup .NET + uses: actions/setup-dotnet@v6 + with: + dotnet-version: '10.x' + + # Each backend job published the schema its own migration produced; the comparison needs all + # three side by side, and skips rather than passes if any is missing. + - name: Download schema dumps + uses: actions/download-artifact@v8 + with: + pattern: schema-* + merge-multiple: true + path: schema-dumps + + - name: Compare the three schemas + env: + SHOKO_SCHEMA_DIR: ${{ github.workspace }}/schema-dumps + run: >- + dotnet test Shoko.Tests/Shoko.Tests.csproj -c Release + --filter "FullyQualifiedName~SchemaTypeParityTests" + --logger "console;verbosity=normal" diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 0000000000..1ea50040a1 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,31 @@ +name: Unit Tests + +on: + push: + branches: + - master + pull_request: + branches: + - master + +jobs: + test: + runs-on: ubuntu-latest + name: Unit Tests + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Setup .NET + uses: actions/setup-dotnet@v6 + with: + dotnet-version: '10.x' + + # These suites are self-contained: no database, no network, and no native + # dependencies, so they need none of the setup the integration tests do. + - name: Run unit tests + run: dotnet test Shoko.Tests/Shoko.Tests.csproj -c Release --logger "console;verbosity=normal" + + - name: Run queue processor tests + run: dotnet test Shoko.QueueProcessor.Tests/Shoko.QueueProcessor.Tests.csproj -c Release --logger "console;verbosity=normal" diff --git a/CLAUDE.md b/CLAUDE.md index 1bde12609d..41dd7d9130 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -232,11 +232,42 @@ Plugin controllers are registered via `AddPluginControllers` during API setup. ### Testing -- **Framework**: xUnit 2.7.0 with `Xunit.DependencyInjection` 9.1.0 for DI in tests -- **Mocking**: Moq 4.20.70 -- **Coverage**: coverlet 6.0.2 -- **Test SDK**: Microsoft.NET.Test.Sdk 17.9.0 -- Unit tests in `Shoko.Tests/`, integration tests in `Shoko.IntegrationTests/` +- **Framework**: xUnit v3 (`xunit.v3`) — there is no `Xunit.DependencyInjection`; test classes take + their dependencies through fixtures or build them directly +- **Mocking**: Moq +- **Coverage**: coverlet +- **Test SDK**: Microsoft.NET.Test.Sdk + +**Where a test belongs** + +| Project | Scope | +|---------|-------| +| `Shoko.Tests` | Unit tests. No database, no network, no DI container. | +| `Shoko.QueueProcessor.Tests` | The EF Core job queue, against in-memory SQLite. | +| `Shoko.IntegrationTests` | Full server bootstrap against a real database, run in CI over SQLite, MySQL and SQL Server (selected by `DB_TYPE`). | +| `Shoko.TestData` | Shared JSON fixtures consumed by tests and benchmarks. | + +Prefer the cheapest option that can actually exercise the behaviour: plain unit tests first, then the +cache-backed repositories described below, and a real database only when persistence itself is the +subject. + +**Testing code that reads `RepoFactory`** + +Domain models resolve their navigation properties through the `RepoFactory` statics, which normally +forces a database. `Shoko.Tests/Infrastructure/` avoids that: + +- `CachedRepo.Build(keySelector, entities)` returns a **real** repository whose + rows live in an in-memory `PocoCache`. Read paths, including each repository's own indexes, run + exactly as in production. `Save`/`Delete` are not supported — mock those instead. +- `RepoFactoryScope` installs repositories into the `RepoFactory` statics and restores them on + dispose. Its `With(...)` overload builds and installs in one step. + +Those statics are process-global, so every test using `RepoFactoryScope` must be annotated +`[Collection(nameof(RepoFactoryCollection))]`, which serialises them while the rest of the suite +keeps running in parallel. + +Note that `ISystemService.StaticServices` is **write-once per process** — it throws on a second +assignment. Nothing in `Shoko.Tests` sets it, and new tests should keep it that way. ### Database Migrations diff --git a/Shoko.Benchmarks/Program.cs b/Shoko.Benchmarks/Program.cs index b11aa4831f..41471ed358 100644 --- a/Shoko.Benchmarks/Program.cs +++ b/Shoko.Benchmarks/Program.cs @@ -1,4 +1,4 @@ using BenchmarkDotNet.Running; using Benchmarks; -BenchmarkRunner.Run(); +BenchmarkSwitcher.FromTypes([typeof(AniDB_AnimeBenchmarks), typeof(TagFilterBenchmarks)]).RunAll(); diff --git a/Shoko.Benchmarks/TagFilterBenchmarks.cs b/Shoko.Benchmarks/TagFilterBenchmarks.cs new file mode 100644 index 0000000000..ad0460a543 --- /dev/null +++ b/Shoko.Benchmarks/TagFilterBenchmarks.cs @@ -0,0 +1,28 @@ +using BenchmarkDotNet.Attributes; +using Shoko.Server; + +namespace Benchmarks; + +/// +/// Tag filtering runs over every tag of every anime, so its throughput is worth watching. +/// +/// +/// Was a [Fact] in Shoko.Tests asserting an average under 2000ms. That is a benchmark, +/// not a test, and on a shared CI runner it measured 2911ms and failed the suite. +/// +[BenchmarkCategory("TagFilter")] +public class TagFilterBenchmarks +{ + private const TagFilter.Filter Filters = + TagFilter.Filter.Genre | TagFilter.Filter.AnidbInternal | TagFilter.Filter.Programming | TagFilter.Filter.Misc; + + private static readonly string[] _tags = + [ + "comedy", "Comedy", "horror", "18 restricted", "large breasts", "japan", "violence", "action", "romance", + "school life", "seinen", "shounen", "asia", "contemporary fantasy", "earth", "afterlife", "alien", + "angst", "ecchi", "gore", "themes", "elements", "origin", "setting", "manga", "new", "ugly", + ]; + + [Benchmark] + public List ProcessTags() => TagFilter.String.ProcessTags(Filters, _tags); +} diff --git a/Shoko.IntegrationTests/DatabaseCollection.cs b/Shoko.IntegrationTests/DatabaseCollection.cs new file mode 100644 index 0000000000..e0a4bc35a8 --- /dev/null +++ b/Shoko.IntegrationTests/DatabaseCollection.cs @@ -0,0 +1,17 @@ +using Xunit; + +namespace Shoko.IntegrationTests; + +/// +/// Shares one server bootstrap across every test class in the collection. +/// +/// +/// ISystemService.StaticServices is write-once per process, so a second +/// throws. A class fixture is one instance per class; this is +/// one for the run. +/// +[CollectionDefinition(Name)] +public class DatabaseCollection : ICollectionFixture +{ + public const string Name = "Database"; +} diff --git a/Shoko.IntegrationTests/DatabaseMigrationFixture.cs b/Shoko.IntegrationTests/DatabaseMigrationFixture.cs index 9f5dd54312..f4f6d0ac09 100644 --- a/Shoko.IntegrationTests/DatabaseMigrationFixture.cs +++ b/Shoko.IntegrationTests/DatabaseMigrationFixture.cs @@ -1,7 +1,10 @@ using System; +using System.Data; using System.IO; using System.Threading; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Shoko.Server.Databases; using Shoko.Server.Services; using Shoko.Server.Settings; @@ -25,6 +28,22 @@ public sealed class DatabaseMigrationFixture : IDisposable public string? FailureMessage { get; private set; } + /// The backend this run migrated, as selected by DB_TYPE. + public string Backend { get; } = Environment.GetEnvironmentVariable("DB_TYPE") is { Length: > 0 } type ? type : "SQLite"; + + /// + /// An open connection to the migrated database, for reading its catalog. The caller owns it. + /// + public IDbConnection OpenConnection() + { + var session = _host!.Services.GetRequiredService().SessionFactory.OpenSession(); + var connection = session.Connection; + if (connection.State is not ConnectionState.Open) + connection.Open(); + + return connection; + } + private readonly string _tempDir; private readonly IHost? _host; diff --git a/Shoko.IntegrationTests/DatabaseMigrationTests.cs b/Shoko.IntegrationTests/DatabaseMigrationTests.cs index 3eb8017681..1d24bbdd20 100644 --- a/Shoko.IntegrationTests/DatabaseMigrationTests.cs +++ b/Shoko.IntegrationTests/DatabaseMigrationTests.cs @@ -6,8 +6,8 @@ namespace Shoko.IntegrationTests; /// Verifies that all database migrations run without error against the backend /// configured via environment variables (see ). /// -[Collection("Database")] -public class DatabaseMigrationTests : IClassFixture +[Collection(DatabaseCollection.Name)] +public class DatabaseMigrationTests { private readonly DatabaseMigrationFixture _fixture; diff --git a/Shoko.IntegrationTests/SchemaSnapshotTests.cs b/Shoko.IntegrationTests/SchemaSnapshotTests.cs new file mode 100644 index 0000000000..9d9fc7973b --- /dev/null +++ b/Shoko.IntegrationTests/SchemaSnapshotTests.cs @@ -0,0 +1,39 @@ +using System; +using System.IO; +using System.Text.Json; +using Shoko.TestData.Schema; +using Xunit; + +namespace Shoko.IntegrationTests; + +/// +/// Records the schema of the database this run migrated, for the cross-backend comparison in +/// Shoko.Tests to pick up. +/// +/// +/// Written to the directory named by , which each CI job +/// publishes for a later job to compare. Unset, this only checks the schema can be read. +/// +[Collection(DatabaseCollection.Name)] +public class SchemaSnapshotTests(DatabaseMigrationFixture fixture) +{ + [Fact] + public void TheMigratedSchemaIsRecorded() + { + Assert.True(fixture.Success, fixture.FailureMessage); + + using var connection = fixture.OpenConnection(); + var schema = SchemaSnapshot.Read(connection, fixture.Backend); + + // A near-empty dump would make every column look agreed downstream. + Assert.True(schema.Tables.Count > 60, $"{fixture.Backend}: only {schema.Tables.Count} tables."); + + if (Environment.GetEnvironmentVariable(SchemaDumps.DirectoryVariable) is not { Length: > 0 } directory) + return; + + Directory.CreateDirectory(directory); + File.WriteAllText( + Path.Combine(directory, SchemaDumps.FileNameFor(fixture.Backend)), + JsonSerializer.Serialize(schema.Tables, new JsonSerializerOptions { WriteIndented = true })); + } +} diff --git a/Shoko.IntegrationTests/Shoko.IntegrationTests.csproj b/Shoko.IntegrationTests/Shoko.IntegrationTests.csproj index ee40c1cd45..b13b9b3878 100644 --- a/Shoko.IntegrationTests/Shoko.IntegrationTests.csproj +++ b/Shoko.IntegrationTests/Shoko.IntegrationTests.csproj @@ -27,6 +27,7 @@ + diff --git a/Shoko.QueueProcessor.Tests/PersistenceBufferTests.cs b/Shoko.QueueProcessor.Tests/PersistenceBufferTests.cs index 56f0567d29..a2feb806fb 100644 --- a/Shoko.QueueProcessor.Tests/PersistenceBufferTests.cs +++ b/Shoko.QueueProcessor.Tests/PersistenceBufferTests.cs @@ -168,16 +168,22 @@ public async Task OnEnqueue_MaxBatchReached_TriggersImmediateFlush() var (buffer, repo) = Make(flushIntervalMs: 60_000, maxBatch: 3); var insertedCount = 0; + // The flush is fire-and-forget, so the test waits to be told it happened rather than sleeping + // for a period a loaded machine can overrun. + var flushed = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); repo.Setup(r => r.InsertBatchAsync(It.IsAny>(), It.IsAny())) - .Callback, CancellationToken>((jobs, _) => insertedCount += jobs.Count) + .Callback, CancellationToken>((jobs, _) => + { + insertedCount += jobs.Count; + flushed.TrySetResult(); + }) .Returns(Task.CompletedTask); buffer.OnEnqueue(FakeJob()); buffer.OnEnqueue(FakeJob()); buffer.OnEnqueue(FakeJob()); // this triggers the force flush - // Give the async flush a moment (it's fire-and-forget) - await Task.Delay(100, TestContext.Current.CancellationToken); + await flushed.Task.WaitAsync(TimeSpan.FromSeconds(30), TestContext.Current.CancellationToken); Assert.Equal(3, insertedCount); diff --git a/Shoko.QueueProcessor/Acquisition/Filters/NetworkRequiredAcquisitionFilter.cs b/Shoko.QueueProcessor/Acquisition/Filters/NetworkRequiredAcquisitionFilter.cs index 132ed5c9e3..39ebf771e5 100644 --- a/Shoko.QueueProcessor/Acquisition/Filters/NetworkRequiredAcquisitionFilter.cs +++ b/Shoko.QueueProcessor/Acquisition/Filters/NetworkRequiredAcquisitionFilter.cs @@ -19,7 +19,10 @@ public NetworkRequiredAcquisitionFilter(IConnectivityService connectivityService _connectivityService.NetworkAvailabilityChanged += OnNetworkAvailabilityChanged; // Use OfType() rather than IsDefined so that subclasses // of NetworkRequiredAttribute (e.g. AniDBHttpRateLimitedAttribute) are also matched. + // Skipping runtime-emitted assemblies: `GetTypes()` throws on one still being written to, + // and no job type is ever emitted at runtime. _types = AppDomain.CurrentDomain.GetAssemblies() + .Where(a => !a.IsDynamic) .SelectMany(a => a.GetTypes()) .Where(a => typeof(IQueueJob).IsAssignableFrom(a) && !a.IsAbstract && a.GetCustomAttributes(inherit: true).OfType().Any()) diff --git a/Shoko.Server.sln b/Shoko.Server.sln index f08e77f1ff..1ed40ae39a 100644 --- a/Shoko.Server.sln +++ b/Shoko.Server.sln @@ -282,7 +282,9 @@ Global {88A3A583-5844-4888-80F8-B5757EC7939E}.Debug|x86.ActiveCfg = Debug|Any CPU {88A3A583-5844-4888-80F8-B5757EC7939E}.Debug|x86.Build.0 = Debug|Any CPU {88A3A583-5844-4888-80F8-B5757EC7939E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {88A3A583-5844-4888-80F8-B5757EC7939E}.Release|Any CPU.Build.0 = Release|Any CPU {88A3A583-5844-4888-80F8-B5757EC7939E}.Release|x64.ActiveCfg = Release|Any CPU + {88A3A583-5844-4888-80F8-B5757EC7939E}.Release|x64.Build.0 = Release|Any CPU {88A3A583-5844-4888-80F8-B5757EC7939E}.Release|x86.ActiveCfg = Release|Any CPU {88A3A583-5844-4888-80F8-B5757EC7939E}.Release|x86.Build.0 = Release|Any CPU {88A3A583-5844-4888-80F8-B5757EC7939E}.ApiLogging|Any CPU.ActiveCfg = Debug|Any CPU diff --git a/Shoko.Server/API/v3/Helpers/FilterFactory.cs b/Shoko.Server/API/v3/Helpers/FilterFactory.cs index 42a650e97e..750d6450b5 100644 --- a/Shoko.Server/API/v3/Helpers/FilterFactory.cs +++ b/Shoko.Server/API/v3/Helpers/FilterFactory.cs @@ -13,6 +13,7 @@ using Shoko.Server.Models.Shoko; using Shoko.Server.Repositories; using Shoko.Server.Server; +using Shoko.Server.Utilities; namespace Shoko.Server.API.v3.Helpers; @@ -25,7 +26,7 @@ public class FilterFactory static FilterFactory() { - var allTypes = AppDomain.CurrentDomain.GetAssemblies().SelectMany(a => a.GetTypes()).ToList(); + var allTypes = ReflectionUtils.ScannableAssemblies().SelectMany(a => a.GetTypes()).ToList(); s_expressionTypes = allTypes .Where(a => a != typeof(FilterExpression) && !a.IsGenericType && typeof(FilterExpression).IsAssignableFrom(a) && !typeof(SortingExpression).IsAssignableFrom(a)) diff --git a/Shoko.Server/Databases/MySQL.cs b/Shoko.Server/Databases/MySQL.cs index 52b554497d..0718293958 100644 --- a/Shoko.Server/Databases/MySQL.cs +++ b/Shoko.Server/Databases/MySQL.cs @@ -1145,6 +1145,129 @@ FROM JSON_TABLE(sri.`CrossReferences`, '$[*]' COLUMNS (val JSON PATH '$')) AS x WHERE sri.`CrossReferences` LIKE '%AnidbEpisodeID%' """), new(184, 1, "ALTER TABLE `VideoLocal` DROP COLUMN `MyListID`;"), + + // These back non-nullable model properties, so a null could never have been read into one. + // Rows are filled first, since a stored null would fail the alter. + // Lost when `MySQLFixUTF8` and `MySQLFixUTF8MB4` rebuilt every text column with `MODIFY`, + // which replaces the whole definition and drops anything left unstated. For the same reason + // each `MODIFY` has to restate the collation the column already has: the ten that v170 made + // `utf8mb4_bin` — hashes, paths and tokens, compared case-sensitively — keep it here. + new(185, 1, "UPDATE `AniDB_Anime` SET `AllTags` = '' WHERE `AllTags` IS NULL; ALTER TABLE `AniDB_Anime` MODIFY COLUMN `AllTags` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 2, "UPDATE `AniDB_Anime` SET `AllTitles` = '' WHERE `AllTitles` IS NULL; ALTER TABLE `AniDB_Anime` MODIFY COLUMN `AllTitles` varchar(1500) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 3, "UPDATE `AniDB_Anime` SET `Description` = '' WHERE `Description` IS NULL; ALTER TABLE `AniDB_Anime` MODIFY COLUMN `Description` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 4, "UPDATE `AniDB_Anime` SET `MainTitle` = '' WHERE `MainTitle` IS NULL; ALTER TABLE `AniDB_Anime` MODIFY COLUMN `MainTitle` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 5, "UPDATE `AniDB_Anime_Character` SET `Appearance` = '' WHERE `Appearance` IS NULL; ALTER TABLE `AniDB_Anime_Character` MODIFY COLUMN `Appearance` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 6, "UPDATE `AniDB_Anime_Relation` SET `RelationType` = '' WHERE `RelationType` IS NULL; ALTER TABLE `AniDB_Anime_Relation` MODIFY COLUMN `RelationType` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 7, "UPDATE `AniDB_Anime_Staff` SET `Role` = '' WHERE `Role` IS NULL; ALTER TABLE `AniDB_Anime_Staff` MODIFY COLUMN `Role` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 8, "UPDATE `AniDB_Anime_Title` SET `Language` = '' WHERE `Language` IS NULL; ALTER TABLE `AniDB_Anime_Title` MODIFY COLUMN `Language` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 9, "UPDATE `AniDB_Anime_Title` SET `Title` = '' WHERE `Title` IS NULL; ALTER TABLE `AniDB_Anime_Title` MODIFY COLUMN `Title` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 10, "UPDATE `AniDB_Anime_Title` SET `TitleType` = '' WHERE `TitleType` IS NULL; ALTER TABLE `AniDB_Anime_Title` MODIFY COLUMN `TitleType` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 11, "UPDATE `AniDB_Character` SET `Description` = '' WHERE `Description` IS NULL; ALTER TABLE `AniDB_Character` MODIFY COLUMN `Description` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 12, "UPDATE `AniDB_Character` SET `ImagePath` = '' WHERE `ImagePath` IS NULL; ALTER TABLE `AniDB_Character` MODIFY COLUMN `ImagePath` varchar(20) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 13, "UPDATE `AniDB_Character` SET `Name` = '' WHERE `Name` IS NULL; ALTER TABLE `AniDB_Character` MODIFY COLUMN `Name` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 14, "UPDATE `AniDB_Character` SET `OriginalName` = '' WHERE `OriginalName` IS NULL; ALTER TABLE `AniDB_Character` MODIFY COLUMN `OriginalName` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 15, "UPDATE `AniDB_Creator` SET `Name` = '' WHERE `Name` IS NULL; ALTER TABLE `AniDB_Creator` MODIFY COLUMN `Name` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 16, "UPDATE `AniDB_Episode` SET `Description` = '' WHERE `Description` IS NULL; ALTER TABLE `AniDB_Episode` MODIFY COLUMN `Description` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 17, "UPDATE `AniDB_Episode_Title` SET `Language` = '' WHERE `Language` IS NULL; ALTER TABLE `AniDB_Episode_Title` MODIFY COLUMN `Language` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 18, "UPDATE `AniDB_GroupStatus` SET `EpisodeRange` = '' WHERE `EpisodeRange` IS NULL; ALTER TABLE `AniDB_GroupStatus` MODIFY COLUMN `EpisodeRange` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 19, "UPDATE `AniDB_GroupStatus` SET `GroupName` = '' WHERE `GroupName` IS NULL; ALTER TABLE `AniDB_GroupStatus` MODIFY COLUMN `GroupName` longtext CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 20, "UPDATE `AniDB_GroupStatus` SET `Rating` = 0 WHERE `Rating` IS NULL; ALTER TABLE `AniDB_GroupStatus` MODIFY COLUMN `Rating` decimal(6,2) NOT NULL;"), + new(185, 21, "UPDATE `AniDB_Message` SET `Body` = '' WHERE `Body` IS NULL; ALTER TABLE `AniDB_Message` MODIFY COLUMN `Body` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 22, "UPDATE `AniDB_Message` SET `FromUserName` = '' WHERE `FromUserName` IS NULL; ALTER TABLE `AniDB_Message` MODIFY COLUMN `FromUserName` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 23, "UPDATE `AniDB_Message` SET `Title` = '' WHERE `Title` IS NULL; ALTER TABLE `AniDB_Message` MODIFY COLUMN `Title` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 24, "UPDATE `AniDB_Tag` SET `TagDescription` = '' WHERE `TagDescription` IS NULL; ALTER TABLE `AniDB_Tag` MODIFY COLUMN `TagDescription` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 25, "UPDATE `AniDB_Tag` SET `TagName` = '' WHERE `TagName` IS NULL; ALTER TABLE `AniDB_Tag` MODIFY COLUMN `TagName` varchar(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 26, "UPDATE `AnimeGroup` SET `GroupName` = '' WHERE `GroupName` IS NULL; ALTER TABLE `AnimeGroup` MODIFY COLUMN `GroupName` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 27, "UPDATE `AuthTokens` SET `DeviceName` = '' WHERE `DeviceName` IS NULL; ALTER TABLE `AuthTokens` MODIFY COLUMN `DeviceName` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 28, "UPDATE `AuthTokens` SET `Token` = '' WHERE `Token` IS NULL; ALTER TABLE `AuthTokens` MODIFY COLUMN `Token` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL;"), + new(185, 29, "UPDATE `FileNameHash` SET `FileName` = '' WHERE `FileName` IS NULL; ALTER TABLE `FileNameHash` MODIFY COLUMN `FileName` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL;"), + new(185, 30, "UPDATE `FileNameHash` SET `Hash` = '' WHERE `Hash` IS NULL; ALTER TABLE `FileNameHash` MODIFY COLUMN `Hash` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL;"), + new(185, 31, "UPDATE `ImportFolder` SET `ImportFolderLocation` = '' WHERE `ImportFolderLocation` IS NULL; ALTER TABLE `ImportFolder` MODIFY COLUMN `ImportFolderLocation` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL;"), + new(185, 32, "UPDATE `ImportFolder` SET `ImportFolderName` = '' WHERE `ImportFolderName` IS NULL; ALTER TABLE `ImportFolder` MODIFY COLUMN `ImportFolderName` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 33, "UPDATE `Scan` SET `ImportFolders` = '' WHERE `ImportFolders` IS NULL; ALTER TABLE `Scan` MODIFY COLUMN `ImportFolders` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 34, "UPDATE `ScanFile` SET `FullName` = '' WHERE `FullName` IS NULL; ALTER TABLE `ScanFile` MODIFY COLUMN `FullName` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 35, "UPDATE `ScanFile` SET `Hash` = '' WHERE `Hash` IS NULL; ALTER TABLE `ScanFile` MODIFY COLUMN `Hash` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 36, "UPDATE `ScheduledUpdate` SET `UpdateDetails` = '' WHERE `UpdateDetails` IS NULL; ALTER TABLE `ScheduledUpdate` MODIFY COLUMN `UpdateDetails` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 37, "UPDATE `StoredReleaseInfo` SET `ED2K` = '' WHERE `ED2K` IS NULL; ALTER TABLE `StoredReleaseInfo` MODIFY COLUMN `ED2K` varchar(40) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL;"), + new(185, 38, "UPDATE `StoredReleaseInfo_MatchAttempt` SET `ED2K` = '' WHERE `ED2K` IS NULL; ALTER TABLE `StoredReleaseInfo_MatchAttempt` MODIFY COLUMN `ED2K` varchar(40) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL;"), + new(185, 39, "UPDATE `TMDB_AlternateOrdering` SET `EnglishOverview` = '' WHERE `EnglishOverview` IS NULL; ALTER TABLE `TMDB_AlternateOrdering` MODIFY COLUMN `EnglishOverview` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 40, "UPDATE `TMDB_AlternateOrdering` SET `EnglishTitle` = '' WHERE `EnglishTitle` IS NULL; ALTER TABLE `TMDB_AlternateOrdering` MODIFY COLUMN `EnglishTitle` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 41, "UPDATE `TMDB_AlternateOrdering` SET `TmdbEpisodeGroupCollectionID` = '' WHERE `TmdbEpisodeGroupCollectionID` IS NULL; ALTER TABLE `TMDB_AlternateOrdering` MODIFY COLUMN `TmdbEpisodeGroupCollectionID` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 42, "UPDATE `TMDB_AlternateOrdering_Episode` SET `TmdbEpisodeGroupCollectionID` = '' WHERE `TmdbEpisodeGroupCollectionID` IS NULL; ALTER TABLE `TMDB_AlternateOrdering_Episode` MODIFY COLUMN `TmdbEpisodeGroupCollectionID` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 43, "UPDATE `TMDB_AlternateOrdering_Episode` SET `TmdbEpisodeGroupID` = '' WHERE `TmdbEpisodeGroupID` IS NULL; ALTER TABLE `TMDB_AlternateOrdering_Episode` MODIFY COLUMN `TmdbEpisodeGroupID` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 44, "UPDATE `TMDB_AlternateOrdering_Season` SET `EnglishTitle` = '' WHERE `EnglishTitle` IS NULL; ALTER TABLE `TMDB_AlternateOrdering_Season` MODIFY COLUMN `EnglishTitle` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 45, "UPDATE `TMDB_AlternateOrdering_Season` SET `TmdbEpisodeGroupCollectionID` = '' WHERE `TmdbEpisodeGroupCollectionID` IS NULL; ALTER TABLE `TMDB_AlternateOrdering_Season` MODIFY COLUMN `TmdbEpisodeGroupCollectionID` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 46, "UPDATE `TMDB_AlternateOrdering_Season` SET `TmdbEpisodeGroupID` = '' WHERE `TmdbEpisodeGroupID` IS NULL; ALTER TABLE `TMDB_AlternateOrdering_Season` MODIFY COLUMN `TmdbEpisodeGroupID` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 47, "UPDATE `TMDB_Collection` SET `EnglishOverview` = '' WHERE `EnglishOverview` IS NULL; ALTER TABLE `TMDB_Collection` MODIFY COLUMN `EnglishOverview` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 48, "UPDATE `TMDB_Collection` SET `EnglishTitle` = '' WHERE `EnglishTitle` IS NULL; ALTER TABLE `TMDB_Collection` MODIFY COLUMN `EnglishTitle` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 49, "UPDATE `TMDB_Company` SET `CountryOfOrigin` = '' WHERE `CountryOfOrigin` IS NULL; ALTER TABLE `TMDB_Company` MODIFY COLUMN `CountryOfOrigin` varchar(3) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 50, "UPDATE `TMDB_Company` SET `Name` = '' WHERE `Name` IS NULL; ALTER TABLE `TMDB_Company` MODIFY COLUMN `Name` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 51, "UPDATE `TMDB_Episode` SET `EnglishOverview` = '' WHERE `EnglishOverview` IS NULL; ALTER TABLE `TMDB_Episode` MODIFY COLUMN `EnglishOverview` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 52, "UPDATE `TMDB_Episode` SET `EnglishTitle` = '' WHERE `EnglishTitle` IS NULL; ALTER TABLE `TMDB_Episode` MODIFY COLUMN `EnglishTitle` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 53, "UPDATE `TMDB_Episode_Cast` SET `CharacterName` = '' WHERE `CharacterName` IS NULL; ALTER TABLE `TMDB_Episode_Cast` MODIFY COLUMN `CharacterName` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 54, "UPDATE `TMDB_Episode_Cast` SET `TmdbCreditID` = '' WHERE `TmdbCreditID` IS NULL; ALTER TABLE `TMDB_Episode_Cast` MODIFY COLUMN `TmdbCreditID` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 55, "UPDATE `TMDB_Episode_Crew` SET `Department` = '' WHERE `Department` IS NULL; ALTER TABLE `TMDB_Episode_Crew` MODIFY COLUMN `Department` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 56, "UPDATE `TMDB_Episode_Crew` SET `Job` = '' WHERE `Job` IS NULL; ALTER TABLE `TMDB_Episode_Crew` MODIFY COLUMN `Job` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 57, "UPDATE `TMDB_Episode_Crew` SET `TmdbCreditID` = '' WHERE `TmdbCreditID` IS NULL; ALTER TABLE `TMDB_Episode_Crew` MODIFY COLUMN `TmdbCreditID` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 58, "UPDATE `TMDB_Movie` SET `ContentRatings` = '' WHERE `ContentRatings` IS NULL; ALTER TABLE `TMDB_Movie` MODIFY COLUMN `ContentRatings` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 59, "UPDATE `TMDB_Movie` SET `EnglishOverview` = '' WHERE `EnglishOverview` IS NULL; ALTER TABLE `TMDB_Movie` MODIFY COLUMN `EnglishOverview` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 60, "UPDATE `TMDB_Movie` SET `EnglishTitle` = '' WHERE `EnglishTitle` IS NULL; ALTER TABLE `TMDB_Movie` MODIFY COLUMN `EnglishTitle` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 61, "UPDATE `TMDB_Movie` SET `OriginalLanguageCode` = '' WHERE `OriginalLanguageCode` IS NULL; ALTER TABLE `TMDB_Movie` MODIFY COLUMN `OriginalLanguageCode` varchar(5) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 62, "UPDATE `TMDB_Movie` SET `OriginalTitle` = '' WHERE `OriginalTitle` IS NULL; ALTER TABLE `TMDB_Movie` MODIFY COLUMN `OriginalTitle` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 63, "UPDATE `TMDB_Movie_Cast` SET `CharacterName` = '' WHERE `CharacterName` IS NULL; ALTER TABLE `TMDB_Movie_Cast` MODIFY COLUMN `CharacterName` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 64, "UPDATE `TMDB_Movie_Cast` SET `TmdbCreditID` = '' WHERE `TmdbCreditID` IS NULL; ALTER TABLE `TMDB_Movie_Cast` MODIFY COLUMN `TmdbCreditID` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 65, "UPDATE `TMDB_Movie_Crew` SET `Department` = '' WHERE `Department` IS NULL; ALTER TABLE `TMDB_Movie_Crew` MODIFY COLUMN `Department` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 66, "UPDATE `TMDB_Movie_Crew` SET `Job` = '' WHERE `Job` IS NULL; ALTER TABLE `TMDB_Movie_Crew` MODIFY COLUMN `Job` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 67, "UPDATE `TMDB_Movie_Crew` SET `TmdbCreditID` = '' WHERE `TmdbCreditID` IS NULL; ALTER TABLE `TMDB_Movie_Crew` MODIFY COLUMN `TmdbCreditID` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 68, "UPDATE `TMDB_Network` SET `CountryOfOrigin` = '' WHERE `CountryOfOrigin` IS NULL; ALTER TABLE `TMDB_Network` MODIFY COLUMN `CountryOfOrigin` varchar(3) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 69, "UPDATE `TMDB_Network` SET `Name` = '' WHERE `Name` IS NULL; ALTER TABLE `TMDB_Network` MODIFY COLUMN `Name` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 70, "UPDATE `TMDB_Overview` SET `CountryCode` = '' WHERE `CountryCode` IS NULL; ALTER TABLE `TMDB_Overview` MODIFY COLUMN `CountryCode` varchar(5) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 71, "UPDATE `TMDB_Overview` SET `LanguageCode` = '' WHERE `LanguageCode` IS NULL; ALTER TABLE `TMDB_Overview` MODIFY COLUMN `LanguageCode` varchar(5) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 72, "UPDATE `TMDB_Overview` SET `Value` = '' WHERE `Value` IS NULL; ALTER TABLE `TMDB_Overview` MODIFY COLUMN `Value` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 73, "UPDATE `TMDB_Person` SET `Aliases` = '' WHERE `Aliases` IS NULL; ALTER TABLE `TMDB_Person` MODIFY COLUMN `Aliases` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 74, "UPDATE `TMDB_Person` SET `EnglishBiography` = '' WHERE `EnglishBiography` IS NULL; ALTER TABLE `TMDB_Person` MODIFY COLUMN `EnglishBiography` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 75, "UPDATE `TMDB_Person` SET `EnglishName` = '' WHERE `EnglishName` IS NULL; ALTER TABLE `TMDB_Person` MODIFY COLUMN `EnglishName` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 76, "UPDATE `TMDB_Season` SET `EnglishOverview` = '' WHERE `EnglishOverview` IS NULL; ALTER TABLE `TMDB_Season` MODIFY COLUMN `EnglishOverview` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 77, "UPDATE `TMDB_Season` SET `EnglishTitle` = '' WHERE `EnglishTitle` IS NULL; ALTER TABLE `TMDB_Season` MODIFY COLUMN `EnglishTitle` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 78, "UPDATE `TMDB_Show` SET `ContentRatings` = '' WHERE `ContentRatings` IS NULL; ALTER TABLE `TMDB_Show` MODIFY COLUMN `ContentRatings` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 79, "UPDATE `TMDB_Show` SET `EnglishOverview` = '' WHERE `EnglishOverview` IS NULL; ALTER TABLE `TMDB_Show` MODIFY COLUMN `EnglishOverview` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 80, "UPDATE `TMDB_Show` SET `EnglishTitle` = '' WHERE `EnglishTitle` IS NULL; ALTER TABLE `TMDB_Show` MODIFY COLUMN `EnglishTitle` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 81, "UPDATE `TMDB_Show` SET `OriginalLanguageCode` = '' WHERE `OriginalLanguageCode` IS NULL; ALTER TABLE `TMDB_Show` MODIFY COLUMN `OriginalLanguageCode` varchar(5) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 82, "UPDATE `TMDB_Show` SET `OriginalTitle` = '' WHERE `OriginalTitle` IS NULL; ALTER TABLE `TMDB_Show` MODIFY COLUMN `OriginalTitle` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 83, "UPDATE `TMDB_Title` SET `CountryCode` = '' WHERE `CountryCode` IS NULL; ALTER TABLE `TMDB_Title` MODIFY COLUMN `CountryCode` varchar(5) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 84, "UPDATE `TMDB_Title` SET `LanguageCode` = '' WHERE `LanguageCode` IS NULL; ALTER TABLE `TMDB_Title` MODIFY COLUMN `LanguageCode` varchar(5) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 85, "UPDATE `TMDB_Title` SET `Value` = '' WHERE `Value` IS NULL; ALTER TABLE `TMDB_Title` MODIFY COLUMN `Value` varchar(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 86, "UPDATE `VideoLocal` SET `FileName` = '' WHERE `FileName` IS NULL; ALTER TABLE `VideoLocal` MODIFY COLUMN `FileName` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 87, "UPDATE `VideoLocal` SET `Hash` = '' WHERE `Hash` IS NULL; ALTER TABLE `VideoLocal` MODIFY COLUMN `Hash` varchar(50) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL;"), + new(185, 88, "UPDATE `VideoLocal_HashDigest` SET `Type` = '' WHERE `Type` IS NULL; ALTER TABLE `VideoLocal_HashDigest` MODIFY COLUMN `Type` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL;"), + new(185, 89, "UPDATE `VideoLocal_HashDigest` SET `Value` = '' WHERE `Value` IS NULL; ALTER TABLE `VideoLocal_HashDigest` MODIFY COLUMN `Value` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL;"), + new(185, 90, "UPDATE `VideoLocal_Place` SET `FilePath` = '' WHERE `FilePath` IS NULL; ALTER TABLE `VideoLocal_Place` MODIFY COLUMN `FilePath` text CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL;"), + + // Only damaged when the database was created outside Shoko: it then keeps the server's + // default collation, so `MySQLFixUTF8MB4` finds these too. Shoko's own `CREATE DATABASE` + // already uses utf8mb4_unicode_ci and leaves them alone. + new(185, 91, "UPDATE `AniDB_Episode` SET `Rating` = '' WHERE `Rating` IS NULL; ALTER TABLE `AniDB_Episode` MODIFY COLUMN `Rating` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 92, "UPDATE `AniDB_Episode` SET `Votes` = '' WHERE `Votes` IS NULL; ALTER TABLE `AniDB_Episode` MODIFY COLUMN `Votes` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 93, "UPDATE `AnimeEpisode_User` SET `UserTags` = '' WHERE `UserTags` IS NULL; ALTER TABLE `AnimeEpisode_User` MODIFY COLUMN `UserTags` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 94, "UPDATE `AnimeSeries_User` SET `UserTags` = '' WHERE `UserTags` IS NULL; ALTER TABLE `AnimeSeries_User` MODIFY COLUMN `UserTags` text CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 95, "UPDATE `Versions` SET `VersionType` = '' WHERE `VersionType` IS NULL; ALTER TABLE `Versions` MODIFY COLUMN `VersionType` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(185, 96, "UPDATE `Versions` SET `VersionValue` = '' WHERE `VersionValue` IS NULL; ALTER TABLE `Versions` MODIFY COLUMN `VersionValue` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + + // Widths SQL Server already declares and MySQL left as `text`, either from the start or from + // `MySQLFixUTF8` widening everything it touched. Nothing longer can have reached these on + // SQL Server, so the bound is what the data already is; each value is trimmed to it first, + // since anything longer would fail the alter. + new(186, 1, "UPDATE `AniDB_Anime_Relation` SET `RelationType` = LEFT(`RelationType`, 100) WHERE CHAR_LENGTH(`RelationType`) > 100; ALTER TABLE `AniDB_Anime_Relation` MODIFY COLUMN `RelationType` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(186, 2, "UPDATE `AnimeEpisode` SET `EpisodeNameOverride` = LEFT(`EpisodeNameOverride`, 500) WHERE CHAR_LENGTH(`EpisodeNameOverride`) > 500; ALTER TABLE `AnimeEpisode` MODIFY COLUMN `EpisodeNameOverride` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL;"), + // `AirsOn` is a `DayOfWeek`, which Fluent NHibernate stores by name: `Wednesday` is the longest at nine. + new(186, 3, "UPDATE `AnimeSeries` SET `AirsOn` = LEFT(`AirsOn`, 10) WHERE CHAR_LENGTH(`AirsOn`) > 10; ALTER TABLE `AnimeSeries` MODIFY COLUMN `AirsOn` varchar(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL;"), + new(186, 4, "UPDATE `AnimeSeries` SET `SeriesNameOverride` = LEFT(`SeriesNameOverride`, 500) WHERE CHAR_LENGTH(`SeriesNameOverride`) > 500; ALTER TABLE `AnimeSeries` MODIFY COLUMN `SeriesNameOverride` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL;"), + new(186, 5, "UPDATE `CustomTag` SET `TagName` = LEFT(`TagName`, 500) WHERE CHAR_LENGTH(`TagName`) > 500; ALTER TABLE `CustomTag` MODIFY COLUMN `TagName` varchar(500) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL;"), + // The index has to go first: its 255-character prefix no longer fits the column, and a bounded + // column does not need one. + new(186, 6, "UPDATE `FilterPreset` SET `Name` = LEFT(`Name`, 250) WHERE CHAR_LENGTH(`Name`) > 250; ALTER TABLE `FilterPreset` DROP INDEX `IX_FilterPreset_Name`; ALTER TABLE `FilterPreset` MODIFY COLUMN `Name` varchar(250) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL; ALTER TABLE `FilterPreset` ADD INDEX `IX_FilterPreset_Name` (`Name`);"), + new(186, 7, "UPDATE `ScanFile` SET `Hash` = LEFT(`Hash`, 100) WHERE CHAR_LENGTH(`Hash`) > 100; ALTER TABLE `ScanFile` MODIFY COLUMN `Hash` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL;"), + new(186, 8, "UPDATE `ScanFile` SET `HashResult` = LEFT(`HashResult`, 100) WHERE CHAR_LENGTH(`HashResult`) > 100; ALTER TABLE `ScanFile` MODIFY COLUMN `HashResult` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL;"), ]; #endregion @@ -1264,6 +1387,13 @@ private static void DropAniDBUniqueIndex() cmd.ExecuteScalar(); } + /// + /// A , so on a database migrating in one pass this + /// runs after every patch, v170's case-sensitive columns included. utf8mb4_bin is therefore + /// left alone: it is already utf8mb4, and converting it would undo the case-sensitivity that hashes + /// and paths are compared with — on a fresh install only, since a database that ran this long ago + /// never runs it again. + /// private static void MySQLFixUTF8() { var settings = ISettingsProvider.Instance.GetSettings(); @@ -1271,7 +1401,7 @@ private static void MySQLFixUTF8() "SELECT `TABLE_SCHEMA`, `TABLE_NAME`, `COLUMN_NAME`, `DATA_TYPE`, `CHARACTER_MAXIMUM_LENGTH` " + "FROM information_schema.COLUMNS " + $"WHERE table_schema = '{settings.Database.Schema}' " + - "AND collation_name != 'utf8mb4_unicode_ci'"; + "AND collation_name NOT IN ('utf8mb4_unicode_ci', 'utf8mb4_bin')"; using var conn = new MySqlConnection(ConnectionString); var mySQL = (MySQL)ISystemService.StaticServices.GetRequiredService().Instance!; conn.Open(); diff --git a/Shoko.Server/Databases/NHIbernate/NHibernateDependencyInjector.cs b/Shoko.Server/Databases/NHIbernate/NHibernateDependencyInjector.cs index 22f7d04346..038eade619 100644 --- a/Shoko.Server/Databases/NHIbernate/NHibernateDependencyInjector.cs +++ b/Shoko.Server/Databases/NHIbernate/NHibernateDependencyInjector.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.DependencyInjection; using NHibernate; using NHibernate.Type; +using Shoko.Server.Utilities; namespace Shoko.Server.Databases.NHibernate; @@ -48,7 +49,7 @@ public static void RegisterPostInitializationCallback(Func new ConcurrentDictionary( - AppDomain.CurrentDomain.GetAssemblies().SelectMany(a => a.GetTypes()).DistinctBy(a => a.FullName).ToDictionary(a => a.FullName!, a => a))); + ReflectionUtils.ScannableAssemblies().SelectMany(a => a.GetTypes()).DistinctBy(a => a.FullName).ToDictionary(a => a.FullName!, a => a))); if (!allTypes.TryGetValue(clazz, out var type)) return null; if (!s_typeHasValidConstructors.TryGetValue(clazz, out var hasParameters)) { diff --git a/Shoko.Server/Databases/NHIbernate/SimpleNameSerializationBinder.cs b/Shoko.Server/Databases/NHIbernate/SimpleNameSerializationBinder.cs index 19f8eeadb3..fa5a4c750d 100644 --- a/Shoko.Server/Databases/NHIbernate/SimpleNameSerializationBinder.cs +++ b/Shoko.Server/Databases/NHIbernate/SimpleNameSerializationBinder.cs @@ -2,6 +2,7 @@ using System.Linq; using Newtonsoft.Json.Serialization; using NLog; +using Shoko.Server.Utilities; namespace Shoko.Server.Databases.NHibernate; @@ -25,7 +26,7 @@ public override void BindToName( public override Type BindToType(string? assemblyName, string typeName) { var name = typeName.Split('.').LastOrDefault(); - var types = AppDomain.CurrentDomain.GetAssemblies().SelectMany(a => a.GetTypes()) + var types = ReflectionUtils.ScannableAssemblies().SelectMany(a => a.GetTypes()) .Where(a => a.Name.Equals(name) && (_baseType == null || _baseType.IsAssignableFrom(a))).ToArray(); if (types.Length > 1) _logger.Warn($"SimpleNameSerializationBinder found multiple types that match {name}"); return types.FirstOrDefault()!; diff --git a/Shoko.Server/Databases/NHIbernate/StringListConverter.cs b/Shoko.Server/Databases/NHIbernate/StringListConverter.cs index 39557c7cc3..28cd214192 100644 --- a/Shoko.Server/Databases/NHIbernate/StringListConverter.cs +++ b/Shoko.Server/Databases/NHIbernate/StringListConverter.cs @@ -63,8 +63,9 @@ public object DeepCopy(object value) public object Disassemble(object value) => DeepCopy(value); + // Hashed on the serialized form, to match the Equals above. public int GetHashCode(object x) - => x == null ? base.GetHashCode() : x.GetHashCode(); + => x == null ? base.GetHashCode() : ConvertTo(null, null, x, typeof(string))?.GetHashCode() ?? 0; public bool IsMutable => true; @@ -85,7 +86,15 @@ public SqlType[] SqlTypes => new[] { NHibernateUtil.String.SqlType }; bool IUserType.Equals(object x, object y) - => ReferenceEquals(x, y) || (x != null && y != null && x.Equals(y)); + { + if (ReferenceEquals(x, y)) + return true; + if (x is null || y is null) + return false; + + // Compare what would be written, not the list references. + return Equals(ConvertTo(null, null, x, typeof(string)), ConvertTo(null, null, y, typeof(string))); + } #endregion } diff --git a/Shoko.Server/Databases/NHIbernate/TmdbContentRatingConverter.cs b/Shoko.Server/Databases/NHIbernate/TmdbContentRatingConverter.cs index af5d5e4bd4..3d56b66a65 100644 --- a/Shoko.Server/Databases/NHIbernate/TmdbContentRatingConverter.cs +++ b/Shoko.Server/Databases/NHIbernate/TmdbContentRatingConverter.cs @@ -64,8 +64,9 @@ public object DeepCopy(object value) public object Disassemble(object value) => DeepCopy(value); + // Hashed on the serialized form, to match the Equals above. public int GetHashCode(object x) - => x == null ? base.GetHashCode() : x.GetHashCode(); + => x == null ? base.GetHashCode() : ConvertTo(null, null, x, typeof(string))?.GetHashCode() ?? 0; public bool IsMutable => true; @@ -86,7 +87,15 @@ public SqlType[] SqlTypes => new[] { NHibernateUtil.String.SqlType }; bool IUserType.Equals(object x, object y) - => ReferenceEquals(x, y) || (x != null && y != null && x.Equals(y)); + { + if (ReferenceEquals(x, y)) + return true; + if (x is null || y is null) + return false; + + // Compare what would be written, not the list references. + return Equals(ConvertTo(null, null, x, typeof(string)), ConvertTo(null, null, y, typeof(string))); + } #endregion } diff --git a/Shoko.Server/Databases/NHIbernate/TmdbProductionCountryConverter.cs b/Shoko.Server/Databases/NHIbernate/TmdbProductionCountryConverter.cs index 6747171d03..523ec71a0a 100644 --- a/Shoko.Server/Databases/NHIbernate/TmdbProductionCountryConverter.cs +++ b/Shoko.Server/Databases/NHIbernate/TmdbProductionCountryConverter.cs @@ -64,8 +64,9 @@ public object DeepCopy(object value) public object Disassemble(object value) => DeepCopy(value); + // Hashed on the serialized form, to match the Equals above. public int GetHashCode(object x) - => x == null ? base.GetHashCode() : x.GetHashCode(); + => x == null ? base.GetHashCode() : ConvertTo(null, null, x, typeof(string))?.GetHashCode() ?? 0; public bool IsMutable => true; @@ -86,7 +87,15 @@ public SqlType[] SqlTypes => new[] { NHibernateUtil.String.SqlType }; bool IUserType.Equals(object x, object y) - => ReferenceEquals(x, y) || (x != null && y != null && x.Equals(y)); + { + if (ReferenceEquals(x, y)) + return true; + if (x is null || y is null) + return false; + + // Compare what would be written, not the list references. + return Equals(ConvertTo(null, null, x, typeof(string)), ConvertTo(null, null, y, typeof(string))); + } #endregion } diff --git a/Shoko.Server/Databases/NHIbernate/TypeStringConverter.cs b/Shoko.Server/Databases/NHIbernate/TypeStringConverter.cs index ade7de4d3f..c8813bff98 100644 --- a/Shoko.Server/Databases/NHIbernate/TypeStringConverter.cs +++ b/Shoko.Server/Databases/NHIbernate/TypeStringConverter.cs @@ -9,6 +9,7 @@ using NHibernate.Engine; using NHibernate.SqlTypes; using NHibernate.UserTypes; +using Shoko.Server.Utilities; namespace Shoko.Server.Databases.NHibernate; @@ -28,7 +29,7 @@ public override bool CanConvertTo(ITypeDescriptorContext? context, Type? destina object value) { var s = value as string ?? throw new ArgumentException("Can only convert from string"); - return Type.GetType(s) ?? AppDomain.CurrentDomain.GetAssemblies().SelectMany(a => a.GetTypes()).FirstOrDefault(a => a.Name.Equals(s) || Equals(a.FullName, s)); + return Type.GetType(s) ?? ReflectionUtils.ScannableAssemblies().SelectMany(a => a.GetTypes()).FirstOrDefault(a => a.Name.Equals(s) || Equals(a.FullName, s)); } /// diff --git a/Shoko.Server/Databases/SQLServer.cs b/Shoko.Server/Databases/SQLServer.cs index 84287a36d3..c5affdc368 100644 --- a/Shoko.Server/Databases/SQLServer.cs +++ b/Shoko.Server/Databases/SQLServer.cs @@ -1098,6 +1098,56 @@ WHERE sri.CrossReferences LIKE '%AnidbEpisodeID%' new(180, 37, "ALTER TABLE TMDB_Title ADD CONSTRAINT PK_TMDB_Title PRIMARY KEY CLUSTERED (TMDB_TitleID);"), new(180, 38, "ALTER TABLE VideoLocal_HashDigest ADD CONSTRAINT PK_VideoLocal_HashDigest PRIMARY KEY CLUSTERED (VideoLocal_HashDigestID);"), new(181, 1, DropVideoLocalMylistID), + // Seven tables were created before the version 180 sweep but missed by it, leaving them + // without a primary key on SQL Server while SQLite and MySQL both declare one. Every one of + // them keys off an IDENTITY column, so the values are already unique and non-null. + new(182, 1, "ALTER TABLE AniDB_Anime_PreferredImage ADD CONSTRAINT PK_AniDB_Anime_PreferredImage PRIMARY KEY CLUSTERED (AniDB_Anime_PreferredImageID);"), + new(182, 2, "ALTER TABLE AniDB_Episode_PreferredImage ADD CONSTRAINT PK_AniDB_Episode_PreferredImage PRIMARY KEY CLUSTERED (AniDB_Episode_PreferredImageID);"), + new(182, 3, "ALTER TABLE AniDB_FileUpdate ADD CONSTRAINT PK_AniDB_FileUpdate PRIMARY KEY CLUSTERED (AniDB_FileUpdateID);"), + new(182, 4, "ALTER TABLE AuthTokens ADD CONSTRAINT PK_AuthTokens PRIMARY KEY CLUSTERED (AuthID);"), + new(182, 5, "ALTER TABLE ShokoImage_Entity ADD CONSTRAINT PK_ShokoImage_Entity PRIMARY KEY CLUSTERED (ID);"), + new(182, 6, "ALTER TABLE TMDB_Image ADD CONSTRAINT PK_TMDB_Image PRIMARY KEY CLUSTERED (TMDB_ImageID);"), + new(182, 7, "ALTER TABLE TMDB_Image_Entity ADD CONSTRAINT PK_TMDB_Image_Entity PRIMARY KEY CLUSTERED (TMDB_Image_EntityID);"), + + // These back non-nullable model properties, so a null could never have been read into one. + // Rows are filled first, since a stored null would fail the alter. + new(183, 1, "UPDATE AniDB_Creator SET LastUpdatedAt = '0001-01-01T00:00:00' WHERE LastUpdatedAt IS NULL; ALTER TABLE AniDB_Creator ALTER COLUMN LastUpdatedAt datetime2 NOT NULL;"), + new(183, 2, "UPDATE AniDB_GroupStatus SET EpisodeRange = '' WHERE EpisodeRange IS NULL; ALTER TABLE AniDB_GroupStatus ALTER COLUMN EpisodeRange nvarchar(max) NOT NULL;"), + new(183, 3, "UPDATE AniDB_GroupStatus SET GroupName = '' WHERE GroupName IS NULL; ALTER TABLE AniDB_GroupStatus ALTER COLUMN GroupName nvarchar(max) NOT NULL;"), + new(183, 4, "UPDATE AniDB_GroupStatus SET Rating = 0 WHERE Rating IS NULL; ALTER TABLE AniDB_GroupStatus ALTER COLUMN Rating decimal(6, 2) NOT NULL;"), + new(183, 5, "UPDATE AniDB_Message SET Body = '' WHERE Body IS NULL; ALTER TABLE AniDB_Message ALTER COLUMN Body nvarchar(max) NOT NULL;"), + new(183, 6, "UPDATE AniDB_Message SET FromUserName = '' WHERE FromUserName IS NULL; ALTER TABLE AniDB_Message ALTER COLUMN FromUserName nvarchar(100) NOT NULL;"), + new(183, 7, "UPDATE AniDB_Message SET Title = '' WHERE Title IS NULL; ALTER TABLE AniDB_Message ALTER COLUMN Title nvarchar(max) NOT NULL;"), + new(183, 8, "UPDATE TMDB_AlternateOrdering SET CreatedAt = '0001-01-01T00:00:00' WHERE CreatedAt IS NULL; ALTER TABLE TMDB_AlternateOrdering ALTER COLUMN CreatedAt datetime2 NOT NULL;"), + new(183, 9, "UPDATE TMDB_AlternateOrdering SET LastUpdatedAt = '0001-01-01T00:00:00' WHERE LastUpdatedAt IS NULL; ALTER TABLE TMDB_AlternateOrdering ALTER COLUMN LastUpdatedAt datetime2 NOT NULL;"), + new(183, 10, "UPDATE TMDB_AlternateOrdering_Episode SET CreatedAt = '0001-01-01T00:00:00' WHERE CreatedAt IS NULL; ALTER TABLE TMDB_AlternateOrdering_Episode ALTER COLUMN CreatedAt datetime2 NOT NULL;"), + new(183, 11, "UPDATE TMDB_AlternateOrdering_Episode SET LastUpdatedAt = '0001-01-01T00:00:00' WHERE LastUpdatedAt IS NULL; ALTER TABLE TMDB_AlternateOrdering_Episode ALTER COLUMN LastUpdatedAt datetime2 NOT NULL;"), + new(183, 12, "UPDATE TMDB_AlternateOrdering_Season SET CreatedAt = '0001-01-01T00:00:00' WHERE CreatedAt IS NULL; ALTER TABLE TMDB_AlternateOrdering_Season ALTER COLUMN CreatedAt datetime2 NOT NULL;"), + new(183, 13, "UPDATE TMDB_AlternateOrdering_Season SET LastUpdatedAt = '0001-01-01T00:00:00' WHERE LastUpdatedAt IS NULL; ALTER TABLE TMDB_AlternateOrdering_Season ALTER COLUMN LastUpdatedAt datetime2 NOT NULL;"), + new(183, 14, "UPDATE TMDB_Collection SET CreatedAt = '0001-01-01T00:00:00' WHERE CreatedAt IS NULL; ALTER TABLE TMDB_Collection ALTER COLUMN CreatedAt datetime2 NOT NULL;"), + new(183, 15, "UPDATE TMDB_Collection SET LastUpdatedAt = '0001-01-01T00:00:00' WHERE LastUpdatedAt IS NULL; ALTER TABLE TMDB_Collection ALTER COLUMN LastUpdatedAt datetime2 NOT NULL;"), + new(183, 16, "UPDATE TMDB_Episode SET CreatedAt = '0001-01-01T00:00:00' WHERE CreatedAt IS NULL; ALTER TABLE TMDB_Episode ALTER COLUMN CreatedAt datetime2 NOT NULL;"), + new(183, 17, "UPDATE TMDB_Episode SET LastUpdatedAt = '0001-01-01T00:00:00' WHERE LastUpdatedAt IS NULL; ALTER TABLE TMDB_Episode ALTER COLUMN LastUpdatedAt datetime2 NOT NULL;"), + new(183, 18, "UPDATE TMDB_Movie SET CreatedAt = '0001-01-01T00:00:00' WHERE CreatedAt IS NULL; ALTER TABLE TMDB_Movie ALTER COLUMN CreatedAt datetime2 NOT NULL;"), + new(183, 19, "UPDATE TMDB_Movie SET LastUpdatedAt = '0001-01-01T00:00:00' WHERE LastUpdatedAt IS NULL; ALTER TABLE TMDB_Movie ALTER COLUMN LastUpdatedAt datetime2 NOT NULL;"), + new(183, 20, "UPDATE TMDB_Person SET CreatedAt = '0001-01-01T00:00:00' WHERE CreatedAt IS NULL; ALTER TABLE TMDB_Person ALTER COLUMN CreatedAt datetime2 NOT NULL;"), + new(183, 21, "UPDATE TMDB_Person SET LastUpdatedAt = '0001-01-01T00:00:00' WHERE LastUpdatedAt IS NULL; ALTER TABLE TMDB_Person ALTER COLUMN LastUpdatedAt datetime2 NOT NULL;"), + new(183, 22, "UPDATE TMDB_Season SET CreatedAt = '0001-01-01T00:00:00' WHERE CreatedAt IS NULL; ALTER TABLE TMDB_Season ALTER COLUMN CreatedAt datetime2 NOT NULL;"), + new(183, 23, "UPDATE TMDB_Season SET LastUpdatedAt = '0001-01-01T00:00:00' WHERE LastUpdatedAt IS NULL; ALTER TABLE TMDB_Season ALTER COLUMN LastUpdatedAt datetime2 NOT NULL;"), + new(183, 24, "UPDATE TMDB_Show SET CreatedAt = '0001-01-01T00:00:00' WHERE CreatedAt IS NULL; ALTER TABLE TMDB_Show ALTER COLUMN CreatedAt datetime2 NOT NULL;"), + new(183, 25, "UPDATE TMDB_Show SET LastUpdatedAt = '0001-01-01T00:00:00' WHERE LastUpdatedAt IS NULL; ALTER TABLE TMDB_Show ALTER COLUMN LastUpdatedAt datetime2 NOT NULL;"), + + // Widths MySQL already declares and SQL Server left at MAX. Nothing longer can have reached + // these on MySQL, so the bound is what the data already is; each value is trimmed to it + // first, since anything longer would fail the alter. Nullability and collation have to be + // restated, because `ALTER COLUMN` drops whatever it does not name. + new(184, 1, "UPDATE AniDB_Episode SET Rating = LEFT(Rating, 200) WHERE LEN(Rating) > 200; ALTER TABLE AniDB_Episode ALTER COLUMN Rating varchar(200) NOT NULL;"), + new(184, 2, "UPDATE AniDB_Episode SET Votes = LEFT(Votes, 200) WHERE LEN(Votes) > 200; ALTER TABLE AniDB_Episode ALTER COLUMN Votes varchar(200) NOT NULL;"), + new(184, 3, "UPDATE AnimeSeries SET DefaultAudioLanguage = LEFT(DefaultAudioLanguage, 50) WHERE LEN(DefaultAudioLanguage) > 50; ALTER TABLE AnimeSeries ALTER COLUMN DefaultAudioLanguage varchar(50) NULL;"), + new(184, 4, "UPDATE AnimeSeries SET DefaultSubtitleLanguage = LEFT(DefaultSubtitleLanguage, 50) WHERE LEN(DefaultSubtitleLanguage) > 50; ALTER TABLE AnimeSeries ALTER COLUMN DefaultSubtitleLanguage varchar(50) NULL;"), + new(184, 5, "UPDATE ImportFolder SET ImportFolderLocation = LEFT(ImportFolderLocation, 500) WHERE LEN(ImportFolderLocation) > 500; ALTER TABLE ImportFolder ALTER COLUMN ImportFolderLocation nvarchar(500) COLLATE SQL_Latin1_General_CP1_CS_AS NOT NULL;"), + new(184, 6, "UPDATE ImportFolder SET ImportFolderName = LEFT(ImportFolderName, 500) WHERE LEN(ImportFolderName) > 500; ALTER TABLE ImportFolder ALTER COLUMN ImportFolderName nvarchar(500) NOT NULL;"), + new(184, 7, "UPDATE TMDB_Person SET PlaceOfBirth = LEFT(PlaceOfBirth, 128) WHERE LEN(PlaceOfBirth) > 128; ALTER TABLE TMDB_Person ALTER COLUMN PlaceOfBirth nvarchar(128) NULL;"), ]; #endregion diff --git a/Shoko.Server/Databases/SQLite.cs b/Shoko.Server/Databases/SQLite.cs index 5643710546..9574d0f8aa 100644 --- a/Shoko.Server/Databases/SQLite.cs +++ b/Shoko.Server/Databases/SQLite.cs @@ -1,3 +1,4 @@ +using System.Text.RegularExpressions; using System; using System.Collections.Generic; using System.IO; @@ -945,6 +946,19 @@ FROM json_each(CrossReferences) AS x WHERE CrossReferences LIKE '%AnidbEpisodeID%' """), new(163, 1, "ALTER TABLE VideoLocal DROP COLUMN MyListID;"), + + // Both back non-nullable model properties. SQLite cannot tighten in place, so each table is + // rebuilt. + new(164, 1, MakeAniDB_Anime_TitleTitleNotNull), + new(164, 2, MakeVideoLocalDateTimeCreatedNotNull), + + // Six columns SQLite declares as a different type than the other two backends do. SQLite + // cannot retype in place either, so each table is rebuilt. + new(165, 1, RetypeAniDB_AnimeDates), + new(165, 2, RetypeAnimeEpisode_UserUserTags), + new(165, 3, RetypeAnimeSeries_UserUserTags), + new(165, 4, RetypeTMDB_EpisodeRuntime), + new(165, 5, RetypeTMDB_MovieRuntime), ]; #endregion @@ -1117,6 +1131,70 @@ CharDescription TEXT NOT NULL return new Tuple(true, null); } + private static Tuple MakeAniDB_Anime_TitleTitleNotNull(object connection) + => MakeColumnNotNull(connection, "AniDB_Anime_Title", "Title", "''"); + + private static Tuple MakeVideoLocalDateTimeCreatedNotNull(object connection) + // DateTimeUpdated is always set and is the closest thing to a creation time on hand. + => MakeColumnNotNull(connection, "VideoLocal", "DateTimeCreated", "DateTimeUpdated"); + + private static Tuple MakeColumnNotNull(object connection, string tableName, string columnName, string fillExpression) + { + try + { + var factory = (SQLite)ISystemService.StaticServices.GetRequiredService().Instance!; + var db = (SqliteConnection)connection; + factory.Execute(db, $"UPDATE {tableName} SET {columnName} = {fillExpression} WHERE {columnName} IS NULL;"); + factory.Alter(db, tableName, factory.NotNullVariantOf(db, tableName, columnName), factory.RecreateIndexesOf(db, tableName)); + } + catch (Exception e) + { + return new Tuple(false, e.ToString()); + } + + return new Tuple(true, null); + } + + private static Tuple RetypeAniDB_AnimeDates(object connection) + // MySQL v171 and SQL Server v167 moved both to `varchar(10)` when they became a + // `PartialDateOnly`; SQLite kept the `DATETIME` it was created as, and rows written before + // that still carry a time of day the other two dropped. + => ChangeColumnTypes(connection, "AniDB_Anime", [("AirDate", "varchar(10)"), ("EndDate", "varchar(10)")], + "UPDATE AniDB_Anime SET AirDate = substr(AirDate, 1, 10) WHERE length(AirDate) > 10; UPDATE AniDB_Anime SET EndDate = substr(EndDate, 1, 10) WHERE length(EndDate) > 10;"); + + private static Tuple RetypeAnimeEpisode_UserUserTags(object connection) + // Added by an `ALTER TABLE ... ADD COLUMN` that named no type at all, so it has BLOB affinity. + => ChangeColumnTypes(connection, "AnimeEpisode_User", [("UserTags", "TEXT")]); + + private static Tuple RetypeAnimeSeries_UserUserTags(object connection) + => ChangeColumnTypes(connection, "AnimeSeries_User", [("UserTags", "TEXT")]); + + private static Tuple RetypeTMDB_EpisodeRuntime(object connection) + // `Runtime` maps `RuntimeMinutes`, an `int?`. The rebuild's own affinity converts the values. + => ChangeColumnTypes(connection, "TMDB_Episode", [("Runtime", "INTEGER")]); + + private static Tuple RetypeTMDB_MovieRuntime(object connection) + => ChangeColumnTypes(connection, "TMDB_Movie", [("Runtime", "INTEGER")]); + + private static Tuple ChangeColumnTypes(object connection, string tableName, IReadOnlyList<(string Column, string Type)> columns, string? fillCommand = null) + { + try + { + var factory = (SQLite)ISystemService.StaticServices.GetRequiredService().Instance!; + var db = (SqliteConnection)connection; + if (fillCommand is not null) + factory.Execute(db, fillCommand); + + factory.Alter(db, tableName, factory.RetypedVariantOf(db, tableName, columns), factory.RecreateIndexesOf(db, tableName)); + } + catch (Exception e) + { + return new Tuple(false, e.ToString()); + } + + return new Tuple(true, null); + } + private static Tuple AlterAniDB_GroupStatus(object connection) { try @@ -1561,6 +1639,68 @@ private void DropColumns(SqliteConnection db, string tableName, IReadOnlyList + /// The table's own CREATE TABLE, with made NOT NULL. + /// + /// + /// Patched from what the database reports, not written out here: a + /// runs after every other command, so a database + /// migrating in one pass still has columns one migrating from an older version dropped long ago. + /// + private string NotNullVariantOf(SqliteConnection db, string tableName, string columnName) + => NotNullVariantOf((string)ExecuteReader(db, $"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = '{tableName}';")[0][0], columnName); + + /// + internal static string NotNullVariantOf(string createCommand, string columnName) + { + // A definition runs between commas, but its type may bracket a comma of its own: decimal(6,2). + var definition = new Regex( + $@"(?<=[(,]\s*)(?{Regex.Escape(columnName)})(?(?:\s+[^\s,()]+|\s*\([^()]*\))*?)(?\s+(?:NOT\s+)?NULL)?(?=\s*[,)])", + RegexOptions.IgnoreCase); + var patched = definition.Replace(createCommand, match => $"{match.Groups["name"].Value}{match.Groups["type"].Value} NOT NULL", 1); + if (patched == createCommand && !definition.IsMatch(createCommand)) + throw new InvalidOperationException($"Could not find a definition for `{columnName}` in: {createCommand}"); + + return patched; + } + + /// + /// Everything that may follow a column's type in a definition. The type is whatever runs between + /// the name and the first of these. + /// + private const string ColumnConstraints = "CONSTRAINT|PRIMARY|NOT|NULL|UNIQUE|CHECK|DEFAULT|COLLATE|REFERENCES|GENERATED|AS"; + + /// + /// The table's own CREATE TABLE, with each column given the type named for it. + private string RetypedVariantOf(SqliteConnection db, string tableName, IReadOnlyList<(string Column, string Type)> columns) + => columns.Aggregate( + (string)ExecuteReader(db, $"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = '{tableName}';")[0][0], + (createCommand, column) => RetypedVariantOf(createCommand, column.Column, column.Type)); + + /// + internal static string RetypedVariantOf(string createCommand, string columnName, string type) + { + // The type may be several words (UNSIGNED BIG INT), bracket a comma (decimal(6,2)), or be + // absent entirely, as it is for a column added by an ALTER TABLE that named none. + var definition = new Regex( + $@"(?<=[(,]\s*)(?{Regex.Escape(columnName)})(?:\s+(?!(?:{ColumnConstraints})\b)[^\s,()]+|\s*\([^()]*\))*(?=\s*[,)]|\s+(?:{ColumnConstraints})\b)", + RegexOptions.IgnoreCase); + var patched = definition.Replace(createCommand, match => $"{match.Groups["name"].Value} {type}", 1); + if (patched == createCommand && !definition.IsMatch(createCommand)) + throw new InvalidOperationException($"Could not find a definition for `{columnName}` in: {createCommand}"); + + return patched; + } + + /// + /// Commands to drop and recreate each of the table's indexes. The rename carries them along, names + /// and all, so each name has to be freed before it can be reused. + /// + private List RecreateIndexesOf(SqliteConnection db, string tableName) + => ExecuteReader(db, $"SELECT name, sql FROM sqlite_master WHERE type = 'index' AND tbl_name = '{tableName}' AND sql IS NOT NULL;") + .SelectMany(row => new[] { $"DROP INDEX IF EXISTS {(string)row[0]};", $"{(string)row[1]};" }) + .ToList(); + private void Alter(SqliteConnection db, string tableName, string createCommand, IReadOnlyList? indexCommands = null) { indexCommands ??= []; diff --git a/Shoko.Server/Filters/ExpressionDiscovery.cs b/Shoko.Server/Filters/ExpressionDiscovery.cs index f4166c142d..5f62a75740 100644 --- a/Shoko.Server/Filters/ExpressionDiscovery.cs +++ b/Shoko.Server/Filters/ExpressionDiscovery.cs @@ -7,13 +7,14 @@ using Shoko.Abstractions.Filtering.Expressions.Info; using Shoko.Abstractions.Filtering.Sorting; using Shoko.Server.Repositories; +using Shoko.Server.Utilities; namespace Shoko.Server.Filters; internal static class ExpressionDiscovery { public static IReadOnlyList GetExpressionHelp(FilterExpressionGroup? group = null) - => AppDomain.CurrentDomain.GetAssemblies() + => ReflectionUtils.ScannableAssemblies() .SelectMany(a => a.GetTypes()) .Where(a => a != typeof(FilterExpression) && !a.IsAbstract && !a.IsGenericType && @@ -206,7 +207,7 @@ private static (string[]? Parameters, string[]? SecondParameters, string[][]? Pa }; public static IReadOnlyList GetSortingExpressionHelp() - => AppDomain.CurrentDomain.GetAssemblies() + => ReflectionUtils.ScannableAssemblies() .SelectMany(a => a.GetTypes()) .Where(a => a != typeof(FilterExpression) && !a.IsAbstract && !a.IsGenericType && typeof(SortingExpression).IsAssignableFrom(a) diff --git a/Shoko.Server/MediaInfo/Subtitles/SubtitleHelper.cs b/Shoko.Server/MediaInfo/Subtitles/SubtitleHelper.cs index a70348c49b..8bb5522677 100644 --- a/Shoko.Server/MediaInfo/Subtitles/SubtitleHelper.cs +++ b/Shoko.Server/MediaInfo/Subtitles/SubtitleHelper.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using Shoko.Server.Extensions; +using Shoko.Server.Utilities; // ReSharper disable StringLiteralTypo // ReSharper disable InconsistentNaming @@ -43,7 +44,7 @@ private static List InitImplementations() { try { - return AppDomain.CurrentDomain.GetAssemblies() + return ReflectionUtils.ScannableAssemblies() .SelectMany(x => x.GetTypes()) .Where(x => typeof(ISubtitles).IsAssignableFrom(x) && !x.IsInterface && !x.IsAbstract) .Select(type => (ISubtitles)Activator.CreateInstance(type)!) diff --git a/Shoko.Server/Models/Shoko/VideoLocal_Place.cs b/Shoko.Server/Models/Shoko/VideoLocal_Place.cs index 4a10cca363..45ff52a6aa 100644 --- a/Shoko.Server/Models/Shoko/VideoLocal_Place.cs +++ b/Shoko.Server/Models/Shoko/VideoLocal_Place.cs @@ -39,7 +39,7 @@ public class VideoLocal_Place : IVideoFile /// is nullable, meaning it can have a value of null if the unique /// identifier cannot be obtained or the file does not exist. /// - public long? OnDiskUniqueID => ISystemService.StaticServices.GetRequiredService().GetVideoFileUID(Path); + public long? OnDiskUniqueID => ISystemService.StaticServices.GetRequiredService().GetVideoFileUID(Path); private string _relativePath = string.Empty; @@ -86,7 +86,7 @@ public string? Path [MemberNotNullWhen(true, nameof(RelativePath))] [MemberNotNullWhen(true, nameof(FileInfo))] public bool IsAvailable - => ISystemService.StaticServices.GetRequiredService().FileExists(Path); + => ISystemService.StaticServices.GetRequiredService().FileExists(Path); /// /// Helper to get the file name from the relative path. @@ -110,7 +110,7 @@ public ShokoManagedFolder? ManagedFolder /// Helper to get the for the file location if it exists. /// public FileInfo? FileInfo - => ISystemService.StaticServices.GetRequiredService().GetFileInfo(Path); + => ISystemService.StaticServices.GetRequiredService().GetFileInfo(Path); #endregion @@ -136,7 +136,7 @@ public FileInfo? FileInfo if (string.IsNullOrEmpty(filePath)) return null; - var fileSystemHelpers = ISystemService.StaticServices.GetRequiredService(); + var fileSystemHelpers = ISystemService.StaticServices.GetRequiredService(); if (!fileSystemHelpers.FileExists(filePath)) return null; diff --git a/Shoko.Server/Providers/AniDB/AniDBStartup.cs b/Shoko.Server/Providers/AniDB/AniDBStartup.cs index 70c09400ef..46eab93e7d 100644 --- a/Shoko.Server/Providers/AniDB/AniDBStartup.cs +++ b/Shoko.Server/Providers/AniDB/AniDBStartup.cs @@ -10,6 +10,7 @@ using Shoko.Server.Providers.AniDB.Titles; using Shoko.Server.Providers.AniDB.UDP; using Shoko.Server.Settings; +using Shoko.Server.Utilities; namespace Shoko.Server.Providers.AniDB; @@ -21,6 +22,7 @@ public static IServiceCollection AddAniDB(this IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -29,13 +31,13 @@ public static IServiceCollection AddAniDB(this IServiceCollection services) // Register Requests var requestType = typeof(IRequest); - var types = AppDomain.CurrentDomain.GetAssemblies() + var types = ReflectionUtils.ScannableAssemblies() .SelectMany(s => s.GetTypes()) .Where(p => requestType.IsAssignableFrom(p) && !p.IsAbstract && p.IsClass); /* Possibly negate the need for IRequest (non-generic) var requestType = typeof(IRequest<>); - var types = AppDomain.CurrentDomain.GetAssemblies() + var types = ReflectionUtils.ScannableAssemblies() .SelectMany(s => s.GetTypes()) .Where(p => p.IsGenericType && requestType.IsAssignableFrom(p.GetGenericTypeDefinition()) && !p.IsAbstract && p.IsClass) */ diff --git a/Shoko.Server/Providers/AniDB/Interfaces/IAniDBSocketHandlerFactory.cs b/Shoko.Server/Providers/AniDB/Interfaces/IAniDBSocketHandlerFactory.cs new file mode 100644 index 0000000000..bad6d4b30b --- /dev/null +++ b/Shoko.Server/Providers/AniDB/Interfaces/IAniDBSocketHandlerFactory.cs @@ -0,0 +1,17 @@ +namespace Shoko.Server.Providers.AniDB.Interfaces; + +/// +/// Creates the UDP socket the AniDB connection handler talks through. +/// +/// +/// Exists so the connection handler can be exercised without opening a socket: the protocol +/// handling around the socket — session state, ban detection, response codes — is where the logic +/// lives, and none of it should need the network to test. +/// +public interface IAniDBSocketHandlerFactory +{ + /// + /// Creates a socket handler bound to the given server and local port. + /// + IAniDBSocketHandler Create(string host, ushort serverPort, ushort clientPort); +} diff --git a/Shoko.Server/Providers/AniDB/UDP/AniDBSocketHandlerFactory.cs b/Shoko.Server/Providers/AniDB/UDP/AniDBSocketHandlerFactory.cs new file mode 100644 index 0000000000..3e5e956b64 --- /dev/null +++ b/Shoko.Server/Providers/AniDB/UDP/AniDBSocketHandlerFactory.cs @@ -0,0 +1,12 @@ +using Microsoft.Extensions.Logging; +using Shoko.Server.Providers.AniDB.Interfaces; + +namespace Shoko.Server.Providers.AniDB.UDP; + +/// +public class AniDBSocketHandlerFactory(ILoggerFactory loggerFactory) : IAniDBSocketHandlerFactory +{ + /// + public IAniDBSocketHandler Create(string host, ushort serverPort, ushort clientPort) + => new AniDBSocketHandler(loggerFactory, host, serverPort, clientPort); +} diff --git a/Shoko.Server/Providers/AniDB/UDP/AniDBUDPConnectionHandler.cs b/Shoko.Server/Providers/AniDB/UDP/AniDBUDPConnectionHandler.cs index 12725d74b0..8b64ab3dc7 100644 --- a/Shoko.Server/Providers/AniDB/UDP/AniDBUDPConnectionHandler.cs +++ b/Shoko.Server/Providers/AniDB/UDP/AniDBUDPConnectionHandler.cs @@ -34,7 +34,8 @@ public partial class AniDBUDPConnectionHandler : ConnectionHandler, IUDPConnecti private readonly IRequestFactory _requestFactory; private readonly UDPRateLimiter _rateLimiter; private readonly IConnectivityService _connectivityService; - private AniDBSocketHandler? _socketHandler; + private readonly IAniDBSocketHandlerFactory _socketHandlerFactory; + private IAniDBSocketHandler? _socketHandler; private readonly object _socketHandlerLock = new(); // IDK Rider said to use a GeneratedRegex attribute private static readonly Regex s_logMask = GetLogRegex(); @@ -122,9 +123,10 @@ public override bool IsBanned public bool IsNetworkAvailable { private set; get; } - public AniDBUDPConnectionHandler(IRequestFactory requestFactory, ILoggerFactory loggerFactory, ISettingsProvider settings, UDPRateLimiter rateLimiter, IConnectivityService connectivityService) : + public AniDBUDPConnectionHandler(IRequestFactory requestFactory, ILoggerFactory loggerFactory, ISettingsProvider settings, UDPRateLimiter rateLimiter, IConnectivityService connectivityService, IAniDBSocketHandlerFactory socketHandlerFactory) : base(loggerFactory) { + _socketHandlerFactory = socketHandlerFactory; _requestFactory = requestFactory; _rateLimiter = rateLimiter; _connectivityService = connectivityService; @@ -179,7 +181,7 @@ private void InitInternal() _socketHandler = null; } - _socketHandler = new AniDBSocketHandler(_loggerFactory, settings.AniDb.UDPServerAddress, settings.AniDb.UDPServerPort, settings.AniDb.ClientPort); + _socketHandler = _socketHandlerFactory.Create(settings.AniDb.UDPServerAddress, settings.AniDb.UDPServerPort, settings.AniDb.ClientPort); IsNetworkAvailable = _socketHandler.TryConnection(); } diff --git a/Shoko.Server/Repositories/Cached/AnimeSeriesRepository.cs b/Shoko.Server/Repositories/Cached/AnimeSeriesRepository.cs index 5dd79a216b..666d73e494 100644 --- a/Shoko.Server/Repositories/Cached/AnimeSeriesRepository.cs +++ b/Shoko.Server/Repositories/Cached/AnimeSeriesRepository.cs @@ -125,7 +125,7 @@ public override void Save(AnimeSeries obj) Save(obj, true); } - public void Save(AnimeSeries obj, bool updateGroups, bool alsoupdateepisodes = false) + public virtual void Save(AnimeSeries obj, bool updateGroups, bool alsoupdateepisodes = false) { var animeID = obj.AniDB_Anime?.MainTitle ?? obj.AniDB_ID.ToString(); logger.Trace($"Saving Series {animeID}"); diff --git a/Shoko.Server/Repositories/Direct/AniDB_GroupStatusRepository.cs b/Shoko.Server/Repositories/Direct/AniDB_GroupStatusRepository.cs index e74c1e2184..2814d49723 100644 --- a/Shoko.Server/Repositories/Direct/AniDB_GroupStatusRepository.cs +++ b/Shoko.Server/Repositories/Direct/AniDB_GroupStatusRepository.cs @@ -12,7 +12,7 @@ public class AniDB_GroupStatusRepository : BaseDirectRepository GetByAnimeID(int id) + public virtual List GetByAnimeID(int id) { using var session = _databaseFactory.SessionFactory.OpenStatelessSession(); return session.Query() diff --git a/Shoko.Server/Scheduling/Acquisition/Filters/AniDBHttpRateLimitedAcquisitionFilter.cs b/Shoko.Server/Scheduling/Acquisition/Filters/AniDBHttpRateLimitedAcquisitionFilter.cs index 9aeeeee4d7..65f065ce30 100644 --- a/Shoko.Server/Scheduling/Acquisition/Filters/AniDBHttpRateLimitedAcquisitionFilter.cs +++ b/Shoko.Server/Scheduling/Acquisition/Filters/AniDBHttpRateLimitedAcquisitionFilter.cs @@ -5,6 +5,7 @@ using Shoko.Server.Providers.AniDB; using Shoko.Server.Providers.AniDB.Interfaces; using Shoko.Server.Scheduling.Acquisition.Attributes; +using Shoko.Server.Utilities; namespace Shoko.Server.Scheduling.Acquisition.Filters; @@ -17,7 +18,7 @@ public AniDBHttpRateLimitedAcquisitionFilter(IHttpConnectionHandler connectionHa { _connectionHandler = connectionHandler; _connectionHandler.AniDBStateUpdate += OnAniDBStateUpdate; - _types = AppDomain.CurrentDomain.GetAssemblies() + _types = ReflectionUtils.ScannableAssemblies() .SelectMany(a => a.GetTypes()) .Where(a => typeof(IQueueJob).IsAssignableFrom(a) && !a.IsAbstract && a.GetCustomAttributes(inherit: true).OfType().Any()) diff --git a/Shoko.Server/Scheduling/Acquisition/Filters/AniDBUdpRateLimitedAcquisitionFilter.cs b/Shoko.Server/Scheduling/Acquisition/Filters/AniDBUdpRateLimitedAcquisitionFilter.cs index cfe22fbb4e..a142217d76 100644 --- a/Shoko.Server/Scheduling/Acquisition/Filters/AniDBUdpRateLimitedAcquisitionFilter.cs +++ b/Shoko.Server/Scheduling/Acquisition/Filters/AniDBUdpRateLimitedAcquisitionFilter.cs @@ -6,6 +6,7 @@ using Shoko.Server.Providers.AniDB; using Shoko.Server.Providers.AniDB.Interfaces; using Shoko.Server.Scheduling.Acquisition.Attributes; +using Shoko.Server.Utilities; namespace Shoko.Server.Scheduling.Acquisition.Filters; @@ -21,7 +22,7 @@ public AniDBUdpRateLimitedAcquisitionFilter(IUDPConnectionHandler connectionHand _systemService = systemService; _connectionHandler.AniDBStateUpdate += OnAniDBStateUpdate; _systemService.AboutToStart += OnProvidersReady; - _types = AppDomain.CurrentDomain.GetAssemblies() + _types = ReflectionUtils.ScannableAssemblies() .SelectMany(a => a.GetTypes()) .Where(a => typeof(IQueueJob).IsAssignableFrom(a) && !a.IsAbstract && a.GetCustomAttributes(inherit: true).OfType().Any()) diff --git a/Shoko.Server/Scheduling/Acquisition/Filters/DatabaseRequiredAcquisitionFilter.cs b/Shoko.Server/Scheduling/Acquisition/Filters/DatabaseRequiredAcquisitionFilter.cs index 068b1a74e4..0f46435e5a 100644 --- a/Shoko.Server/Scheduling/Acquisition/Filters/DatabaseRequiredAcquisitionFilter.cs +++ b/Shoko.Server/Scheduling/Acquisition/Filters/DatabaseRequiredAcquisitionFilter.cs @@ -4,6 +4,7 @@ using Shoko.Abstractions.Core.Services; using Shoko.QueueProcessor.Abstractions; using Shoko.QueueProcessor.Acquisition.Attributes; +using Shoko.Server.Utilities; namespace Shoko.Server.Scheduling.Acquisition.Filters; @@ -16,7 +17,7 @@ public DatabaseRequiredAcquisitionFilter(ISystemService systemService) { _systemService = systemService; _systemService.DatabaseBlockedChanged += ServerOnDBSetupCompleted; - _types = AppDomain.CurrentDomain.GetAssemblies() + _types = ReflectionUtils.ScannableAssemblies() .SelectMany(a => a.GetTypes()) .Where(a => typeof(IQueueJob).IsAssignableFrom(a) && !a.IsAbstract && a.IsDefined(typeof(DatabaseRequiredAttribute), true)) diff --git a/Shoko.Server/Scheduling/Acquisition/Filters/TmdbApiRateLimitedAcquisitionFilter.cs b/Shoko.Server/Scheduling/Acquisition/Filters/TmdbApiRateLimitedAcquisitionFilter.cs index 6da3f25060..3df2aa58c0 100644 --- a/Shoko.Server/Scheduling/Acquisition/Filters/TmdbApiRateLimitedAcquisitionFilter.cs +++ b/Shoko.Server/Scheduling/Acquisition/Filters/TmdbApiRateLimitedAcquisitionFilter.cs @@ -4,6 +4,7 @@ using Shoko.QueueProcessor.Abstractions; using Shoko.Server.Providers.TMDB; using Shoko.Server.Scheduling.Acquisition.Attributes; +using Shoko.Server.Utilities; #nullable enable namespace Shoko.Server.Scheduling.Acquisition.Filters; @@ -17,7 +18,7 @@ public TmdbApiRateLimitedAcquisitionFilter(TmdbRateLimiter rateLimiter) { _rateLimiter = rateLimiter; _rateLimiter.PauseStateChanged += OnPauseStateChanged; - _types = AppDomain.CurrentDomain.GetAssemblies() + _types = ReflectionUtils.ScannableAssemblies() .SelectMany(a => a.GetTypes()) .Where(a => typeof(IQueueJob).IsAssignableFrom(a) && !a.IsAbstract && a.GetCustomAttributes(inherit: true).OfType().Any()) diff --git a/Shoko.Server/Services/FileSystemHelpers.cs b/Shoko.Server/Services/FileSystemHelpers.cs index 1c760f6d36..2fde279ce6 100644 --- a/Shoko.Server/Services/FileSystemHelpers.cs +++ b/Shoko.Server/Services/FileSystemHelpers.cs @@ -15,7 +15,7 @@ #pragma warning disable CS0618 namespace Shoko.Server.Services; -public class FileSystemHelpers +public class FileSystemHelpers : IFileSystemHelpers { private readonly ILogger _logger; diff --git a/Shoko.Server/Services/FileSystemWatcher/RecoveringFileSystemWatcher.cs b/Shoko.Server/Services/FileSystemWatcher/RecoveringFileSystemWatcher.cs index 4ae5070601..d238aa6659 100644 --- a/Shoko.Server/Services/FileSystemWatcher/RecoveringFileSystemWatcher.cs +++ b/Shoko.Server/Services/FileSystemWatcher/RecoveringFileSystemWatcher.cs @@ -35,7 +35,7 @@ public class RecoveringFileSystemWatcher : IDisposable private readonly TimeSpan _directoryFailedRetryInterval = TimeSpan.FromSeconds(5); private readonly TimeSpan _directoryRetryInterval = TimeSpan.FromMinutes(5); private readonly ILogger _logger; - private readonly FileSystemHelpers _fileSystemHelpers; + private readonly IFileSystemHelpers _fileSystemHelpers; private readonly IReadOnlyCollection _filters; private readonly IReadOnlyCollection _pathExclusions; private readonly ConcurrentDictionary _buffer = new(); @@ -52,7 +52,7 @@ public class RecoveringFileSystemWatcher : IDisposable public event EventHandler? FileDeleted; public FileSystemWatcherLockOptions Options { get; set; } = new(); - public RecoveringFileSystemWatcher(string path, IReadOnlyCollection filters, IReadOnlyCollection pathExclusions, FileSystemHelpers fileSystemHelpers) + public RecoveringFileSystemWatcher(string path, IReadOnlyCollection filters, IReadOnlyCollection pathExclusions, IFileSystemHelpers fileSystemHelpers) { if (path == null) throw new ArgumentException(nameof(path) + " cannot be null"); if (!Directory.Exists(path)) throw new ArgumentException(nameof(path) + $" must be a directory that exists: {path}"); diff --git a/Shoko.Server/Services/FileWatcherService.cs b/Shoko.Server/Services/FileWatcherService.cs index 71a7720c85..6223b85e6b 100644 --- a/Shoko.Server/Services/FileWatcherService.cs +++ b/Shoko.Server/Services/FileWatcherService.cs @@ -32,7 +32,7 @@ public class FileWatcherService private readonly ShokoManagedFolderRepository _managedFolders; - private readonly FileSystemHelpers _fileSystemHelpers; + private readonly IFileSystemHelpers _fileSystemHelpers; private List? _videoExtensions; @@ -40,7 +40,7 @@ public class FileWatcherService private IVideoService? _videoService; - public FileWatcherService(ILogger logger, ConfigurationProvider settingsProvider, ShokoManagedFolderRepository managedFolders, FileSystemHelpers fileSystemHelpers) + public FileWatcherService(ILogger logger, ConfigurationProvider settingsProvider, ShokoManagedFolderRepository managedFolders, IFileSystemHelpers fileSystemHelpers) { _logger = logger; _settingsProvider = settingsProvider; diff --git a/Shoko.Server/Services/IFileSystemHelpers.cs b/Shoko.Server/Services/IFileSystemHelpers.cs new file mode 100644 index 0000000000..ba0bbbfe87 --- /dev/null +++ b/Shoko.Server/Services/IFileSystemHelpers.cs @@ -0,0 +1,82 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading; + +namespace Shoko.Server.Services; + +/// +/// Long-path-safe file system access. +/// +/// +/// Every file system call the server makes goes through this so that Windows' path length limit is +/// handled in one place. It is an interface so that callers which move or delete a user's files — +/// principally — can be exercised without touching a real disk. +/// +public interface IFileSystemHelpers +{ + /// + /// Checks whether a file exists, treating a null or empty path as absent. + /// + bool FileExists(string? path); + + /// + /// Checks whether a directory exists, treating a null or empty path as absent. + /// + bool DirectoryExists(string? path); + + /// + /// Deletes the file at the given path. + /// + void DeleteFile(string path); + + /// + /// Deletes the directory at the given path. + /// + void DeleteDirectory(string path, bool recursive = false); + + /// + /// Creates the directory at the given path, including any missing parents. + /// + void CreateDirectory(string path); + + /// + /// Moves a file from one path to another. + /// + void MoveFile(string sourcePath, string destinationPath); + + /// + /// Opens the file at the given path for reading. + /// + FileStream OpenRead(string path); + + /// + /// Gets the file size, or -1 if the file does not exist. + /// + long GetFileSize(string path); + + /// + /// Gets a for the path, or null if the file does not exist. + /// + FileInfo? GetFileInfo(string? path); + + /// + /// Lists directory paths beneath the given directory. + /// + string[] GetDirectoryPaths(string directoryPath, bool recursive = false, Func? filter = null, CancellationToken cancellationToken = default); + + /// + /// Lists file paths beneath the given directory. + /// + string[] GetFilePaths(string directoryPath, bool recursive = false, IEnumerable? extensions = null, Func? filter = null, CancellationToken cancellationToken = default); + + /// + /// Builds a predicate matching paths against the given extensions and filter. + /// + Func GetPathValidator(IEnumerable? extensions, Func? filter); + + /// + /// Gets the inode number (Unix) or file ID (Windows) for a file, or null if it cannot be obtained. + /// + long? GetVideoFileUID(string? path); +} diff --git a/Shoko.Server/Services/SystemService.cs b/Shoko.Server/Services/SystemService.cs index 7306f4f55d..3a01be44fd 100644 --- a/Shoko.Server/Services/SystemService.cs +++ b/Shoko.Server/Services/SystemService.cs @@ -400,6 +400,7 @@ public void ConfigureServices(IServiceCollection services) services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(sp => sp.GetRequiredService()); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/Shoko.Server/Services/VideoRelocationService.cs b/Shoko.Server/Services/VideoRelocationService.cs index 11a7ad4a42..c8e83b6bac 100644 --- a/Shoko.Server/Services/VideoRelocationService.cs +++ b/Shoko.Server/Services/VideoRelocationService.cs @@ -49,7 +49,7 @@ public class VideoRelocationService( StoredRelocationPresetRepository storedRelocationPresetRepository, FileNameHashRepository fileNameHash, ShokoManagedFolderRepository managedFolders, - FileSystemHelpers fileSystemHelpers + IFileSystemHelpers fileSystemHelpers ) : IVideoRelocationService, IRelocationPresetManager { private Dictionary _relocationProviderInfos = []; diff --git a/Shoko.Server/Services/VideoService.cs b/Shoko.Server/Services/VideoService.cs index 22fd7971e1..5f39eb53b2 100644 --- a/Shoko.Server/Services/VideoService.cs +++ b/Shoko.Server/Services/VideoService.cs @@ -70,7 +70,7 @@ public class VideoService : IVideoService private readonly DatabaseFactory _databaseFactory; - private readonly FileSystemHelpers _fileSystemHelpers; + private readonly IFileSystemHelpers _fileSystemHelpers; /// public event EventHandler? VideoFileDetected; @@ -111,7 +111,7 @@ public VideoService( IMylistService mylistService, ISettingsProvider settingsProvider, DatabaseFactory databaseFactory, - FileSystemHelpers fileSystemHelpers + IFileSystemHelpers fileSystemHelpers ) { _logger = logger; diff --git a/Shoko.Server/Utilities/ReflectionUtils.cs b/Shoko.Server/Utilities/ReflectionUtils.cs index 384103039f..3cb10d960c 100644 --- a/Shoko.Server/Utilities/ReflectionUtils.cs +++ b/Shoko.Server/Utilities/ReflectionUtils.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using System.Linq; using System.Linq.Expressions; using System.Reflection; @@ -6,6 +8,17 @@ namespace Shoko.Server.Utilities; internal class ReflectionUtils { + /// + /// The loaded assemblies worth scanning for a type Shoko defines. + /// + /// + /// Skips assemblies emitted at runtime: throws + /// on one that is still being written to, and nothing + /// found this way is ever emitted at runtime. + /// + public static IEnumerable ScannableAssemblies() + => AppDomain.CurrentDomain.GetAssemblies().Where(assembly => !assembly.IsDynamic); + public delegate T ObjectActivator(params object[] args); public delegate T ObjectMethodActivator(object instance, params object[] args); diff --git a/Shoko.Tests/PartialDateOnlyConverter.cs b/Shoko.TestData/PartialDateOnlyConverter.cs similarity index 97% rename from Shoko.Tests/PartialDateOnlyConverter.cs rename to Shoko.TestData/PartialDateOnlyConverter.cs index 8af6f4d088..13116cc09d 100644 --- a/Shoko.Tests/PartialDateOnlyConverter.cs +++ b/Shoko.TestData/PartialDateOnlyConverter.cs @@ -2,7 +2,7 @@ using Newtonsoft.Json; using Shoko.Abstractions.Metadata; -namespace Shoko.Tests; +namespace Shoko.TestData; public class PartialDateOnlyConverter : JsonConverter { diff --git a/Shoko.TestData/Schema/README.md b/Shoko.TestData/Schema/README.md new file mode 100644 index 0000000000..be9d9cbd6c --- /dev/null +++ b/Shoko.TestData/Schema/README.md @@ -0,0 +1,43 @@ +# Cross-backend schema comparison + +The three supported backends each keep their own hand-written DDL in `Shoko.Server/Databases/`, and +nothing forces them to agree. `SchemaTypeParityTests` in `Shoko.Tests` does: same tables, same +columns, same type, width and nullability for every column. + +It compares schemas read from the catalogs of real databases — one per backend, migrated from empty. +Nothing is committed, because a recorded schema is a copy that can quietly fall behind the migrations +it claims to describe. The dumps are produced at runtime instead: + +- `Shoko.IntegrationTests` → `SchemaSnapshotTests` migrates a database and writes + `schema-.json` into the directory named by `SHOKO_SCHEMA_DIR`. +- CI runs that once per backend, publishes each dump, then runs the comparison over all three. +- Without all three dumps the comparison has nothing to compare and skips, rather than passing. + +The DDL is not simply replayed instead, because a replay cannot see the whole migration: MySQL +performs some of its through `PREPARE stmt FROM @sqlstmt`, and every backend has migrations written +in C# rather than SQL. Only the migrated database knows the real answer. + +## Running it locally + +`scripts/compare_schemas.sh` does the whole thing: starts MariaDB and SQL Server in Docker, migrates +all three backends, and runs the comparison. + +```bash +scripts/compare_schemas.sh +``` + +To do it by hand, migrate each backend into the same directory and then point the comparison at it: + +```bash +export SHOKO_SCHEMA_DIR=/tmp/shoko-schemas +DB_TYPE=SQLite dotnet test Shoko.IntegrationTests/Shoko.IntegrationTests.csproj -c Release --filter SchemaSnapshotTests +DB_TYPE=MySQL DB_HOST=127.0.0.1 DB_USER=root DB_PASS=root DB_NAME=shoko \ + dotnet test Shoko.IntegrationTests/Shoko.IntegrationTests.csproj -c Release --filter SchemaSnapshotTests +DB_TYPE=SQLServer DB_HOST=127.0.0.1 DB_USER=sa DB_PASS='ShokoTest1!' DB_NAME=shoko \ + dotnet test Shoko.IntegrationTests/Shoko.IntegrationTests.csproj -c Release --filter SchemaSnapshotTests + +dotnet test Shoko.Tests/Shoko.Tests.csproj -c Release --filter SchemaTypeParityTests +``` + +Each backend must start from an empty database, or the dump describes a schema nobody will ever +migrate into. `mediainfo` and `librhash-dev` are needed for the server to boot. diff --git a/Shoko.TestData/Schema/SchemaDumps.cs b/Shoko.TestData/Schema/SchemaDumps.cs new file mode 100644 index 0000000000..c6942fb2fc --- /dev/null +++ b/Shoko.TestData/Schema/SchemaDumps.cs @@ -0,0 +1,56 @@ +using System.Text.Json; + +namespace Shoko.TestData.Schema; + +/// +/// Loads the per-backend schema dumps written by Shoko.IntegrationTests, from the directory +/// named by . +/// +/// +/// Nothing is committed — each dump comes from migrating a real database of that backend from empty, +/// which CI does one job per backend, publishing the dumps for the comparison job to collect. +/// +public static class SchemaDumps +{ + /// Environment variable naming the directory holding the dumps. + public const string DirectoryVariable = "SHOKO_SCHEMA_DIR"; + + public static readonly string[] Backends = ["SQLite", "MySQL", "SQLServer"]; + + private static string? Directory => Environment.GetEnvironmentVariable(DirectoryVariable) is { Length: > 0 } directory ? directory : null; + + /// The file a dump for is written to and read from. + public static string FileNameFor(string backend) => $"schema-{backend}.json"; + + /// Why the dumps cannot be compared, or when they can. + public static string? Unavailable() + { + if (Directory is not { } directory) + return $"{DirectoryVariable} is not set."; + + var missing = Backends.Where(backend => !File.Exists(Path.Combine(directory, FileNameFor(backend)))).ToArray(); + + return missing.Length is 0 ? null : $"No schema dump for {string.Join(" or ", missing)} in '{directory}'."; + } + + /// The dumped schema of , keyed by table then column. + public static IReadOnlyDictionary> For(string backend) + => _schemas.GetOrAdd(backend, Read); + + private static readonly System.Collections.Concurrent.ConcurrentDictionary>> _schemas = + new(StringComparer.OrdinalIgnoreCase); + + private static IReadOnlyDictionary> Read(string backend) + { + var path = Path.Combine(Directory ?? throw new InvalidOperationException($"{DirectoryVariable} is not set."), FileNameFor(backend)); + using var stream = File.OpenRead(path); + var tables = JsonSerializer.Deserialize>>(stream) + ?? throw new InvalidOperationException($"'{path}' deserialized to null."); + + return tables.ToDictionary( + table => table.Key, + table => (IReadOnlyDictionary)table.Value.ToDictionary( + column => column.Key, column => column.Value, StringComparer.OrdinalIgnoreCase), + StringComparer.OrdinalIgnoreCase); + } +} diff --git a/Shoko.TestData/Schema/SchemaSnapshot.cs b/Shoko.TestData/Schema/SchemaSnapshot.cs new file mode 100644 index 0000000000..9360b399b8 --- /dev/null +++ b/Shoko.TestData/Schema/SchemaSnapshot.cs @@ -0,0 +1,194 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Globalization; +using System.Linq; + +namespace Shoko.TestData.Schema; + +/// +/// One column as the database itself reports it, reduced to the parts that are meaningful across +/// all three backends. +/// +/// Column name, as declared. +/// Backend-neutral type family; see . +/// +/// "500", "6,2", "max", or where the backend declares no +/// size — SQLite uses type affinity, so most of its columns report . +/// +/// Whether the column accepts nulls. +/// Whether the column takes part in the primary key. +public sealed record ColumnSnapshot(string Name, string Family, string? Size, bool Nullable, bool PrimaryKey); + +/// +/// The live schema of a migrated database, read from the backend's own catalog. +/// +/// +/// Read from the catalog, not replayed from the DDL: MySQL migrates some columns through +/// PREPARE stmt FROM @sqlstmt and every backend has migrations written in C#, neither of which +/// a text replay can see. +/// +public sealed class SchemaSnapshot +{ + private const string Sqlite = "SQLite"; + + + public SortedDictionary> Tables { get; init; } = new(StringComparer.OrdinalIgnoreCase); + + /// Tables Shoko does not own, and which therefore have no cross-backend meaning. + private static readonly string[] _ignoredTables = ["sysdiagrams", "database_firewall_rules", "trace_xe_action_map", "trace_xe_event_map"]; + + public static SchemaSnapshot Read(IDbConnection connection, string backend) + { + var snapshot = new SchemaSnapshot(); + foreach (var (table, column, type, size, nullable, primaryKey) in Rows(connection, backend)) + { + if (_ignoredTables.Contains(table, StringComparer.OrdinalIgnoreCase)) + continue; + + if (!snapshot.Tables.TryGetValue(table, out var columns)) + snapshot.Tables[table] = columns = new SortedDictionary(StringComparer.OrdinalIgnoreCase); + + // A key column is never nullable in practice, whatever the catalog says: SQLite reports + // `INTEGER PRIMARY KEY` as nullable because it is a rowid alias, which would otherwise + // make every table's identity column look like a divergence. + columns[column] = new ColumnSnapshot(column, FamilyOf(type), size, nullable && !primaryKey, primaryKey); + } + + return snapshot; + } + + private static IEnumerable<(string Table, string Column, string Type, string? Size, bool Nullable, bool PrimaryKey)> Rows(IDbConnection connection, string backend) + => backend switch + { + "SQLite" => SqliteRows(connection), + "MySQL" => CatalogRows(connection, MySqlQuery), + "SQLServer" => CatalogRows(connection, SqlServerQuery), + _ => throw new ArgumentOutOfRangeException(nameof(backend), backend, "Unknown backend."), + }; + + private const string MySqlQuery = """ + SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE, CHARACTER_MAXIMUM_LENGTH, NUMERIC_PRECISION, NUMERIC_SCALE, IS_NULLABLE, + CASE WHEN COLUMN_KEY = 'PRI' THEN 1 ELSE 0 END + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + """; + + private const string SqlServerQuery = """ + SELECT c.TABLE_NAME, c.COLUMN_NAME, c.DATA_TYPE, c.CHARACTER_MAXIMUM_LENGTH, c.NUMERIC_PRECISION, c.NUMERIC_SCALE, c.IS_NULLABLE, + CASE WHEN k.COLUMN_NAME IS NULL THEN 0 ELSE 1 END + FROM INFORMATION_SCHEMA.COLUMNS c + JOIN INFORMATION_SCHEMA.TABLES t ON t.TABLE_NAME = c.TABLE_NAME AND t.TABLE_SCHEMA = c.TABLE_SCHEMA + LEFT JOIN ( + SELECT ku.TABLE_SCHEMA, ku.TABLE_NAME, ku.COLUMN_NAME + FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE ku + JOIN INFORMATION_SCHEMA.TABLE_CONSTRAINTS tc + ON tc.CONSTRAINT_NAME = ku.CONSTRAINT_NAME AND tc.CONSTRAINT_SCHEMA = ku.CONSTRAINT_SCHEMA + WHERE tc.CONSTRAINT_TYPE = 'PRIMARY KEY' + ) k ON k.TABLE_SCHEMA = c.TABLE_SCHEMA AND k.TABLE_NAME = c.TABLE_NAME AND k.COLUMN_NAME = c.COLUMN_NAME + WHERE t.TABLE_TYPE = 'BASE TABLE' + """; + + private static IEnumerable<(string, string, string, string?, bool, bool)> CatalogRows(IDbConnection connection, string sql) + { + using var command = connection.CreateCommand(); + command.CommandText = sql; + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + var type = reader.GetString(2); + var length = reader.IsDBNull(3) ? (long?)null : Convert.ToInt64(reader.GetValue(3), CultureInfo.InvariantCulture); + var precision = reader.IsDBNull(4) ? (int?)null : Convert.ToInt32(reader.GetValue(4), CultureInfo.InvariantCulture); + var scale = reader.IsDBNull(5) ? (int?)null : Convert.ToInt32(reader.GetValue(5), CultureInfo.InvariantCulture); + + yield return (reader.GetString(0), reader.GetString(1), type, SizeOf(type, length, precision, scale), + reader.GetString(6).Equals("YES", StringComparison.OrdinalIgnoreCase), + Convert.ToInt32(reader.GetValue(7), CultureInfo.InvariantCulture) == 1); + } + } + + private static IEnumerable<(string, string, string, string?, bool, bool)> SqliteRows(IDbConnection connection) + { + using var command = connection.CreateCommand(); + command.CommandText = """ + SELECT m.name, p.name, p.type, p."notnull", p.pk + FROM sqlite_master m JOIN pragma_table_info(m.name) p + WHERE m.type = 'table' AND m.name NOT LIKE 'sqlite_%' + """; + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + // SQLite stores the declared type verbatim ("TEXT", "nvarchar(128)"), so the size, where + // one was declared at all, has to come out of that string. + var declared = reader.GetString(2); + var open = declared.IndexOf('('); + var type = open < 0 ? declared : declared[..open]; + var size = open < 0 ? null : declared[(open + 1)..].TrimEnd(')').Replace(" ", string.Empty); + + yield return (reader.GetString(0), reader.GetString(1), type.Trim(), Normalize(size), + Convert.ToInt32(reader.GetValue(3), CultureInfo.InvariantCulture) == 0, + Convert.ToInt32(reader.GetValue(4), CultureInfo.InvariantCulture) > 0); + } + } + + private static string? SizeOf(string type, long? length, int? precision, int? scale) + { + if (FamilyOf(type) is "decimal") + return precision is null ? null : $"{precision},{scale ?? 0}"; + + if (length is null) + return null; + + // SQL Server reports -1 for the MAX types; MySQL gives `text` and friends their true byte + // ceiling. Both mean "unbounded" and neither is a width anyone chose. + return length < 0 || length >= 65535 ? "max" : length.Value.ToString(CultureInfo.InvariantCulture); + } + + private static string? Normalize(string? size) + => size is null ? null : size.Equals("max", StringComparison.OrdinalIgnoreCase) ? "max" : size; + + /// + /// Reduces a dialect type name to a backend-neutral family, grouped by what the column is for + /// rather than how it is stored — SQLite has no boolean or GUID type, and MySQL has no GUID type. + /// + public static string FamilyOf(string type) => type.Trim().ToLowerInvariant() switch + { + "int" or "integer" or "smallint" or "mediumint" or "tinyint" or "bit" or "bool" or "boolean" => "integer", + "bigint" => "bigint", + "text" or "varchar" or "nvarchar" or "char" or "nchar" or "longtext" or "mediumtext" or "tinytext" or "ntext" or "uniqueidentifier" => "text", + "date" => "date", + "datetime" or "datetime2" or "smalldatetime" or "timestamp" => "datetime", + "time" => "time", + "decimal" or "numeric" or "real" or "float" or "double" or "money" => "decimal", + "blob" or "longblob" or "mediumblob" or "varbinary" or "binary" or "image" => "binary", + var other => other, + }; + + /// + /// Families SQLite has no separate type for. Its INTEGER is already a variable-width signed + /// 64-bit value, so it has no BIGINT to declare. + /// + private static readonly Dictionary _sqliteCannotDistinguish = new(StringComparer.Ordinal) + { + ["bigint"] = "integer", + }; + + /// + /// Whether the families observed for one column are the same type. The backends with the full type + /// system are held to each other exactly; only SQLite is compared after collapsing. + /// + public static bool FamiliesAgree(IReadOnlyDictionary observed) + { + var precise = observed.Where(entry => entry.Key is not Sqlite).Select(entry => entry.Value).Distinct().ToArray(); + if (precise.Length > 1) + return false; + + if (!observed.TryGetValue(Sqlite, out var sqlite) || precise.Length is 0) + return true; + + return AsSqliteWouldDeclareIt(precise[0]) == AsSqliteWouldDeclareIt(sqlite); + } + + private static string AsSqliteWouldDeclareIt(string family) + => _sqliteCannotDistinguish.TryGetValue(family, out var collapsed) ? collapsed : family; +} diff --git a/Shoko.TestData/TestData.cs b/Shoko.TestData/TestData.cs index 59b5704cdb..55040eb339 100644 --- a/Shoko.TestData/TestData.cs +++ b/Shoko.TestData/TestData.cs @@ -13,15 +13,15 @@ public static class TestData using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream(ResourceName); using var reader = new StreamReader(stream!); var jsonString = reader.ReadToEnd(); - return JsonConvert.DeserializeObject(jsonString)!; + return JsonConvert.DeserializeObject(jsonString, new PartialDateOnlyConverter())!; }); public static Lazy> CrossRef_File_Episode { get; } = new(() => { - const string ResourceName = "Shoko.TestData.AniDB_Anime.json"; + const string ResourceName = "Shoko.TestData.CrossRef_File_Episode.json"; using var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream(ResourceName); using var reader = new StreamReader(stream!); var jsonString = reader.ReadToEnd(); - return JsonConvert.DeserializeObject(jsonString)!; + return JsonConvert.DeserializeObject(jsonString, new PartialDateOnlyConverter())!; }); } diff --git a/Shoko.Tests/API/ModelHelperEpisodeInputTests.cs b/Shoko.Tests/API/ModelHelperEpisodeInputTests.cs new file mode 100644 index 0000000000..7f2c58c393 --- /dev/null +++ b/Shoko.Tests/API/ModelHelperEpisodeInputTests.cs @@ -0,0 +1,100 @@ +using Shoko.Abstractions.Metadata.Enums; +using Shoko.Server.API.v3.Helpers; +using Xunit; + +namespace Shoko.Tests.API; + +/// +/// Covers , which parses the +/// type-prefixed episode identifiers ("S3", "C1", …) that the v3 API accepts in range parameters. +/// A misparse silently addresses the wrong episode rather than reporting an error. +/// +public class ModelHelperEpisodeInputTests +{ + [Theory] + [InlineData("1", 1)] + [InlineData("0", 0)] + [InlineData("26", 26)] + [InlineData("-5", -5)] + public void PlainNumbers_ParseWithNoEpisodeType(string input, int expected) + { + var (number, type, error) = ModelHelper.GetEpisodeNumberAndTypeFromInput(input); + + Assert.Equal(expected, number); + Assert.Null(type); + Assert.Null(error); + } + + [Theory] + [InlineData("S3", 3, EpisodeType.Special)] + [InlineData("C1", 1, EpisodeType.Credits)] + [InlineData("T2", 2, EpisodeType.Trailer)] + [InlineData("P4", 4, EpisodeType.Parody)] + [InlineData("O5", 5, EpisodeType.Other)] + [InlineData("E6", 6, EpisodeType.Episode)] + public void TypePrefixes_MapToTheMatchingEpisodeType(string input, int expectedNumber, EpisodeType expectedType) + { + var (number, type, error) = ModelHelper.GetEpisodeNumberAndTypeFromInput(input); + + Assert.Equal(expectedNumber, number); + Assert.Equal(expectedType, type); + Assert.Null(error); + } + + [Fact] + public void TypePrefixes_AreCaseSensitive() + { + // Lower case is not accepted; it is reported rather than silently treated as a special. + var (number, type, error) = ModelHelper.GetEpisodeNumberAndTypeFromInput("s3"); + + Assert.Equal(0, number); + Assert.Null(type); + Assert.NotNull(error); + Assert.Contains("Unknown episode type", error); + } + + [Fact] + public void AnUnrecognisedPrefix_IsReportedAsAnUnknownType() + { + var (number, type, error) = ModelHelper.GetEpisodeNumberAndTypeFromInput("X1"); + + Assert.Equal(0, number); + Assert.Null(type); + Assert.Contains("Unknown episode type 'X'", error); + } + + [Theory] + [InlineData("SS")] + [InlineData("Sabc")] + [InlineData("S")] + public void ANonNumericRemainder_IsReportedAsAParseFailure(string input) + { + var (number, type, error) = ModelHelper.GetEpisodeNumberAndTypeFromInput(input); + + Assert.Equal(0, number); + Assert.Null(type); + Assert.Contains("Unable to parse an int", error); + } + + [Fact(Skip = "Possible bug - Needs investigation")] + public void AnEmptyInputIsReportedRatherThanThrowing() + { + // `input[0]` runs without a length check, so an empty string throws + // IndexOutOfRangeException instead of returning the error tuple every other bad input + // gets. Reachable from the v3 range parameters, where it surfaces as a 500. + var (number, type, error) = ModelHelper.GetEpisodeNumberAndTypeFromInput(string.Empty); + + Assert.Equal(0, number); + Assert.Null(type); + Assert.NotNull(error); + } + + [Fact] + public void TheParseFailureIsReportedBeforeTheUnknownTypeFailure() + { + // "XX" is both an unknown type and an unparseable number; the number check runs first. + var (_, _, error) = ModelHelper.GetEpisodeNumberAndTypeFromInput("XX"); + + Assert.Contains("Unable to parse an int", error); + } +} diff --git a/Shoko.Tests/Databases/SchemaParityTests.cs b/Shoko.Tests/Databases/SchemaParityTests.cs new file mode 100644 index 0000000000..8121b2db07 --- /dev/null +++ b/Shoko.Tests/Databases/SchemaParityTests.cs @@ -0,0 +1,199 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text.RegularExpressions; +using Shoko.Server.Databases; +using Shoko.Tests.Infrastructure; +using Xunit; + +namespace Shoko.Tests.Databases; + +/// +/// Compares the hand-written DDL of the three database backends against each other. +/// +/// +/// Each backend carries its own copy of the schema as an ordered list of raw SQL statements, so the +/// three can drift apart without anything failing until a user on that backend hits it — +/// add missing primary keys on SQL Server for 38 tables was exactly that, and is not +/// reachable from a test that only exercises SQLite. This replays each backend's statements into a +/// logical schema and compares the results, without touching a database. +/// +/// Tables and primary keys only, from the DDL as written. Columns, types, widths and nullability are +/// compared by , which reads a real migrated database instead — a +/// replay cannot see the migrations written in C#, nor those MySQL runs through +/// PREPARE stmt FROM @sqlstmt. +/// +public class SchemaParityTests +{ + private static readonly string[] s_commandListFields = + ["_createVersionTable", "_updateVersionTable", "_createTables", "_patchCommands"]; + + private sealed class Schema + { + public Dictionary TablesWithPrimaryKey { get; } = new(StringComparer.OrdinalIgnoreCase); + + public IReadOnlyCollection Tables => TablesWithPrimaryKey.Keys; + } + + private static readonly Regex s_createTable = new( + @"CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?[\[`""]?(?\w+)[\]`""]?", RegexOptions.IgnoreCase); + + private static readonly Regex s_dropTable = new( + @"DROP\s+TABLE\s+(?:IF\s+EXISTS\s+)?[\[`""]?(?\w+)[\]`""]?", RegexOptions.IgnoreCase); + + private static readonly Regex s_alterTable = new( + @"ALTER\s+TABLE\s+[\[`""]?(?\w+)[\]`""]?", RegexOptions.IgnoreCase); + + private static readonly Regex s_renameTo = new( + @"ALTER\s+TABLE\s+[\[`""]?(?\w+)[\]`""]?\s+RENAME\s+TO\s+[\[`""]?(?\w+)[\]`""]?", RegexOptions.IgnoreCase); + + private static readonly Regex s_renameTable = new( + @"RENAME\s+TABLE\s+[\[`""]?(?\w+)[\]`""]?\s+TO\s+[\[`""]?(?\w+)[\]`""]?", RegexOptions.IgnoreCase); + + private static readonly Regex s_spRename = new( + @"sp_rename\s+'(?[^']+)'\s*,\s*'(?[^']+)'", RegexOptions.IgnoreCase); + + /// Replays a backend's DDL in order into a logical schema. + private static Schema Build(IDatabase database) + { + var schema = new Schema(); + foreach (var statement in Statements(database)) + { + if (s_spRename.Match(statement) is { Success: true } spRename) + { + Rename(schema, spRename.Groups["from"].Value, spRename.Groups["to"].Value); + continue; + } + + if (s_renameTable.Match(statement) is { Success: true } renameTable) + { + Rename(schema, renameTable.Groups["from"].Value, renameTable.Groups["to"].Value); + continue; + } + + if (s_renameTo.Match(statement) is { Success: true } renameTo) + { + Rename(schema, renameTo.Groups["from"].Value, renameTo.Groups["to"].Value); + continue; + } + + if (s_dropTable.Match(statement) is { Success: true } drop) + { + schema.TablesWithPrimaryKey.Remove(drop.Groups["name"].Value); + continue; + } + + if (s_createTable.Match(statement) is { Success: true } create) + { + schema.TablesWithPrimaryKey[create.Groups["name"].Value] = + statement.Contains("PRIMARY KEY", StringComparison.OrdinalIgnoreCase); + continue; + } + + if (statement.Contains("PRIMARY KEY", StringComparison.OrdinalIgnoreCase) && + s_alterTable.Match(statement) is { Success: true } alter && + schema.TablesWithPrimaryKey.ContainsKey(alter.Groups["name"].Value)) + schema.TablesWithPrimaryKey[alter.Groups["name"].Value] = true; + } + + return schema; + } + + private static void Rename(Schema schema, string from, string to) + { + if (!schema.TablesWithPrimaryKey.Remove(from, out var hadPrimaryKey)) + return; + + schema.TablesWithPrimaryKey[to] = hadPrimaryKey; + } + + private static IEnumerable Statements(IDatabase database) + => s_commandListFields + .Select(name => database.GetType().GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)) + .Where(field => field is not null) + .SelectMany(field => (IEnumerable)field!.GetValue(database)!) + .Where(command => command.Type is DatabaseCommandType.NormalCommand && command.Command is not null) + .SelectMany(command => command.Command!.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)); + + public static TheoryData Backends() => new("SQLite", "MySQL", "SQLServer"); + + private static IDatabase Instantiate(string backend) + { + // MySQL reads the settings singleton while initialising its DDL fields. + StubSettingsProvider.Install(); + return Create(backend); + } + + private static IDatabase Create(string backend) => backend switch + { + "SQLite" => new SQLite(null!), + "MySQL" => new MySQL(null!), + "SQLServer" => new SQLServer(null!), + _ => throw new ArgumentOutOfRangeException(nameof(backend)), + }; + + #region Discovery + + [Theory] + [MemberData(nameof(Backends))] + public void TheDdlIsDiscovered(string backend) + { + // Guards against the private command lists being renamed, which would otherwise turn every + // assertion below into a comparison of two empty schemas. + var statements = Statements(Instantiate(backend)).ToArray(); + + Assert.True(statements.Length > 100, $"{backend}: only found {statements.Length} statements."); + } + + #endregion + + #region Parity + + /// + /// SQLite alone drops this from a coded migration rather than plain SQL, so a static replay + /// cannot see it go. Excluded for SQLite only — MySQL and SQL Server both issue a plain + /// `DROP TABLE`, and if either stopped doing so the comparison should notice. + /// + private static readonly string[] s_droppedByCodeInSqlite = ["Language"]; + + private static HashSet TablesOf(string backend) + { + var tables = Build(Instantiate(backend)).Tables.ToHashSet(StringComparer.OrdinalIgnoreCase); + if (backend is "SQLite") + tables.ExceptWith(s_droppedByCodeInSqlite); + + return tables; + } + + [Theory] + [InlineData("MySQL")] + [InlineData("SQLServer")] + public void EveryBackendDefinesTheSameTablesAsSqlite(string backend) + { + var sqlite = TablesOf("SQLite"); + var other = TablesOf(backend); + + // Each backend keeps its own copy of the schema, so one can gain or lose a table without + // anything failing until a user on that backend hits it. + Assert.Equal(string.Empty, string.Join(", ", sqlite.Except(other).Order())); + Assert.Equal(string.Empty, string.Join(", ", other.Except(sqlite).Order())); + } + + [Theory] + [MemberData(nameof(Backends))] + public void EveryTableDeclaresAPrimaryKey(string backend) + { + var missing = Build(Instantiate(backend)).TablesWithPrimaryKey + .Where(entry => !entry.Value) + .Select(entry => entry.Key) + .Order(StringComparer.OrdinalIgnoreCase) + .ToArray(); + + // NHibernate needs an identifier for every mapped entity, and a table without a primary key + // also silently permits duplicate rows. + Assert.Equal(string.Empty, string.Join(", ", missing)); + } + + #endregion +} diff --git a/Shoko.Tests/Databases/SchemaTypeMappingTests.cs b/Shoko.Tests/Databases/SchemaTypeMappingTests.cs new file mode 100644 index 0000000000..59badc4d39 --- /dev/null +++ b/Shoko.Tests/Databases/SchemaTypeMappingTests.cs @@ -0,0 +1,103 @@ +using System.Collections.Generic; +using Shoko.TestData.Schema; +using Xunit; + +namespace Shoko.Tests.Databases; + +/// +/// The type mapping compares through. +/// +/// +/// The reduction to a family decides what counts as a divergence: fold two together and a real +/// difference stops being reported. Needs no dumps, so this runs on every pull request. +/// +public class SchemaTypeMappingTests +{ + #region Families + + [Theory] + // The same intent, spelled by SQLite, MySQL and SQL Server in turn. + [InlineData("integer", "int", "int")] + [InlineData("text", "varchar", "nvarchar")] + [InlineData("text", "longtext", "nvarchar")] + [InlineData("datetime", "datetime", "datetime2")] + [InlineData("date", "date", "date")] + [InlineData("real", "decimal", "decimal")] + [InlineData("blob", "blob", "varbinary")] + [InlineData("uniqueidentifier", "char", "uniqueidentifier")] + public void TheSameIntentReducesToTheSameFamily(string sqlite, string mySql, string sqlServer) + { + var family = SchemaSnapshot.FamilyOf(sqlite); + + Assert.Equal(family, SchemaSnapshot.FamilyOf(mySql)); + Assert.Equal(family, SchemaSnapshot.FamilyOf(sqlServer)); + } + + [Theory] + // Types that must stay apart, or a column silently changing between them goes unreported. + [InlineData("int", "varchar")] + [InlineData("int", "datetime")] + [InlineData("int", "bigint")] + [InlineData("date", "datetime")] + [InlineData("decimal", "int")] + [InlineData("varbinary", "nvarchar")] + public void DifferentIntentsReduceToDifferentFamilies(string one, string other) + => Assert.NotEqual(SchemaSnapshot.FamilyOf(one), SchemaSnapshot.FamilyOf(other)); + + [Fact] + public void TheDialectSpellingIsIgnored() + { + // Casing and padding come straight from the catalog. + Assert.Equal(SchemaSnapshot.FamilyOf("int"), SchemaSnapshot.FamilyOf(" INT ")); + Assert.Equal(SchemaSnapshot.FamilyOf("nvarchar"), SchemaSnapshot.FamilyOf("NVarChar")); + } + + [Fact] + public void AnUnrecognisedTypeKeepsItsOwnName() + { + // Folding it into an existing family would make it compare equal to an unrelated type. + Assert.Equal("hyperloop", SchemaSnapshot.FamilyOf("HyperLoop")); + Assert.NotEqual(SchemaSnapshot.FamilyOf("int"), SchemaSnapshot.FamilyOf("hyperloop")); + } + + #endregion + + #region What SQLite is excused + + [Fact] + public void SqliteMayDeclareIntegerWhereTheOthersDeclareBigint() + // Its INTEGER already holds 64 bits; there is no BIGINT to require. + => Assert.True(Observed(sqlite: "integer", mySql: "bigint", sqlServer: "bigint")); + + [Fact] + public void TheOtherBackendsMayNotDisagreeWithEachOther() + // Both of these have a BIGINT and can say so. + => Assert.False(Observed(sqlite: "integer", mySql: "integer", sqlServer: "bigint")); + + [Theory] + [InlineData("text", "integer", "integer")] + [InlineData("datetime", "text", "text")] + [InlineData("", "text", "text")] + public void SqliteIsNotExcusedATypeItCouldHaveDeclared(string sqlite, string mySql, string sqlServer) + => Assert.False(Observed(sqlite, mySql, sqlServer)); + + [Fact] + public void AColumnMissingFromABackendDoesNotCountAsAgreement() + { + // Absence is the column comparison's business; the backends that do have it must still agree. + Assert.True(Observed(sqlite: "integer", mySql: null, sqlServer: null)); + Assert.False(Observed(sqlite: null, mySql: "integer", sqlServer: "text")); + } + + private static bool Observed(string? sqlite, string? mySql, string? sqlServer) + { + var observed = new Dictionary(); + if (sqlite is not null) observed["SQLite"] = SchemaSnapshot.FamilyOf(sqlite); + if (mySql is not null) observed["MySQL"] = SchemaSnapshot.FamilyOf(mySql); + if (sqlServer is not null) observed["SQLServer"] = SchemaSnapshot.FamilyOf(sqlServer); + + return SchemaSnapshot.FamiliesAgree(observed); + } + + #endregion +} diff --git a/Shoko.Tests/Databases/SchemaTypeParityTests.cs b/Shoko.Tests/Databases/SchemaTypeParityTests.cs new file mode 100644 index 0000000000..9d6e527a64 --- /dev/null +++ b/Shoko.Tests/Databases/SchemaTypeParityTests.cs @@ -0,0 +1,166 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Shoko.TestData.Schema; +using Xunit; + +namespace Shoko.Tests.Databases; + +/// +/// Holds the three supported backends to one schema: the same tables, the same columns, and the same +/// data type, width and nullability for each column. +/// +/// +/// Each backend keeps its own hand-written DDL, so nothing forces the three to agree and no test on a +/// single backend can see it drift — add missing primary keys on SQL Server for 38 tables was +/// this. The schemas come from Shoko.IntegrationTests migrating a real database of each +/// backend; without all three there is nothing to compare and these skip. See +/// Shoko.TestData/Schema/README.md for running it locally. +/// +/// SQLite declares no widths, so it takes no part in the width comparison. +/// +public class SchemaTypeParityTests +{ + private const string Sqlite = "SQLite"; + + /// Skips rather than passing when there is nothing to compare. + private static void RequireDumps() + { + if (SchemaDumps.Unavailable() is { } reason) + Assert.Skip($"{reason} Nothing to compare."); + } + + public static TheoryData Backends() + { + var data = new TheoryData(); + foreach (var backend in SchemaDumps.Backends) + data.Add(backend); + + return data; + } + + #region Discovery + + [Theory] + [MemberData(nameof(Backends))] + public void EachSchemaDumpIsLoaded(string backend) + { + RequireDumps(); + // A dump that arrived near-empty would make every column below look agreed. + var schema = SchemaDumps.For(backend); + + Assert.True(schema.Count > 60, $"{backend}: only {schema.Count} tables."); + Assert.True(schema.Values.Sum(table => table.Count) > 600, $"{backend}: only {schema.Values.Sum(table => table.Count)} columns."); + } + + #endregion + + #region Shape + + [Theory] + [MemberData(nameof(Backends))] + public void EveryBackendDefinesTheSameTables(string backend) + { + RequireDumps(); + var expected = SchemaDumps.For(Sqlite).Keys; + var actual = SchemaDumps.For(backend).Keys; + + Report($"{backend} does not define the same tables as {Sqlite}", + expected.Except(actual, StringComparer.OrdinalIgnoreCase).Select(table => $"{table}: missing") + .Concat(actual.Except(expected, StringComparer.OrdinalIgnoreCase).Select(table => $"{table}: unexpected"))); + } + + [Theory] + [MemberData(nameof(Backends))] + public void EveryBackendDefinesTheSameColumns(string backend) + { + RequireDumps(); + var missing = new List(); + var extra = new List(); + foreach (var (table, columns) in SchemaDumps.For(Sqlite)) + { + if (!SchemaDumps.For(backend).TryGetValue(table, out var other)) + continue; + + missing.AddRange(columns.Keys.Except(other.Keys, StringComparer.OrdinalIgnoreCase).Select(column => $"{table}.{column}")); + extra.AddRange(other.Keys.Except(columns.Keys, StringComparer.OrdinalIgnoreCase).Select(column => $"{table}.{column}")); + } + + Report($"{backend} does not define the same columns as {Sqlite}", + missing.Select(column => $"{column}: missing").Concat(extra.Select(column => $"{column}: unexpected"))); + } + + #endregion + + #region Types + + [Fact] + public void EveryColumnHasTheSameTypeFamilyOnEveryBackend() + { + RequireDumps(); + AssertAgreement("Columns whose type family differs between backends", column => column.Family, SchemaSnapshot.FamiliesAgree); + } + + [Fact] + public void EveryColumnDeclaresTheSameWidthWhereItDeclaresOne() + { + RequireDumps(); + AssertAgreement("Columns declared at different widths", column => column.Size, observed => observed.Values.Distinct().Count() is 1); + } + + [Fact] + public void EveryColumnHasTheSameNullabilityOnEveryBackend() + { + RequireDumps(); + AssertAgreement("Columns whose nullability differs between backends", column => column.Nullable, observed => observed.Values.Distinct().Count() is 1); + } + + #endregion + + #region Comparison + + private static void AssertAgreement(string what, Func facet, Func, bool> agrees) + { + var divergent = new List(); + foreach (var (table, columns) in SchemaDumps.For(Sqlite)) + { + foreach (var column in columns.Keys) + { + var observed = Observe(table, column, facet); + if (observed.Count > 1 && !agrees(observed)) + divergent.Add($"{table}.{column}: {string.Join(", ", observed.Select(entry => $"{entry.Key}={entry.Value}"))}"); + } + } + + Report(what, divergent); + } + + /// + /// Fails with every divergence listed. Not Assert.Equal against an empty string, which + /// xUnit truncates — the whole list has to be readable from a CI log. + /// + private static void Report(string what, IEnumerable divergent) + { + var lines = divergent.Order(StringComparer.OrdinalIgnoreCase).ToArray(); + + Assert.True(lines.Length is 0, $"{what} ({lines.Length}):\n {string.Join("\n ", lines)}"); + } + + private static Dictionary Observe(string table, string column, Func facet) + { + var observed = new Dictionary(); + foreach (var backend in SchemaDumps.Backends) + { + if (!SchemaDumps.For(backend).TryGetValue(table, out var columns) || !columns.TryGetValue(column, out var snapshot)) + continue; + + // Absent, not disagreeing: SQLite reports no width at all. + if (facet(snapshot) is { } value) + observed[backend] = value; + } + + return observed; + } + + #endregion +} diff --git a/Shoko.Tests/Databases/SimpleNameSerializationBinderTests.cs b/Shoko.Tests/Databases/SimpleNameSerializationBinderTests.cs new file mode 100644 index 0000000000..9de00a38fd --- /dev/null +++ b/Shoko.Tests/Databases/SimpleNameSerializationBinderTests.cs @@ -0,0 +1,29 @@ +using System; +using System.Threading; +using Shoko.Abstractions.Filtering.Expressions; +using Shoko.Abstractions.Filtering.Expressions.Logic.Expressions; +using Shoko.Server.Databases.NHibernate; +using Xunit; + +namespace Shoko.Tests.Databases; + +/// +/// , which resolves a stored filter's type from its short +/// name by scanning every loaded assembly. +/// +/// +/// Anything it throws is swallowed by FilterExpressionConverter's error handler, which returns +/// a null expression, so a failure here silently blanks a saved filter rather than reporting. The +/// scan it runs over is covered by . +/// +public class SimpleNameSerializationBinderTests +{ + [Fact] + public void ATypeIsFoundByItsShortName() + => Assert.Equal(typeof(AndExpression), + new SimpleNameSerializationBinder(typeof(FilterExpression)).BindToType(null, typeof(AndExpression).FullName!)); + + [Fact] + public void ATypeOutsideTheBaseTypeIsNotReturned() + => Assert.Null(new SimpleNameSerializationBinder(typeof(FilterExpression)).BindToType(null, typeof(string).FullName!)); +} diff --git a/Shoko.Tests/Databases/SqliteNotNullVariantTests.cs b/Shoko.Tests/Databases/SqliteNotNullVariantTests.cs new file mode 100644 index 0000000000..58cb764394 --- /dev/null +++ b/Shoko.Tests/Databases/SqliteNotNullVariantTests.cs @@ -0,0 +1,172 @@ +using System; +using Shoko.Server.Databases; +using Xunit; + +namespace Shoko.Tests.Databases; + +/// +/// , which rewrites a table's own +/// CREATE TABLE to tighten one column. +/// +/// +/// SQLite cannot alter a column in place, so the migration rebuilds the table around a patched +/// CREATE TABLE. Getting the patch wrong loses a column, a default or the primary key, and the +/// shapes it must cope with differ by upgrade path — see +/// . +/// +public class SqliteNotNullVariantTests +{ + // As a database migrating in one pass still has it: a PostDatabaseFix drops these later. + private const string VideoLocalBeforeTheHashColumnsAreDropped = """ + CREATE TABLE VideoLocal ( + VideoLocalID INTEGER PRIMARY KEY AUTOINCREMENT, + Hash TEXT NOT NULL, + CRC32 TEXT NULL, MD5 TEXT NULL, + SHA1 TEXT NULL, + FileSize INTEGER NOT NULL, + DateTimeUpdated DATETIME NOT NULL, + FileName TEXT NOT NULL DEFAULT '', + DateTimeCreated DATETIME NULL, + MediaBlob BLOB NULL + ) + """; + + // As a database that migrated earlier already has it. + private const string VideoLocalAfterTheHashColumnsAreDropped = """ + CREATE TABLE VideoLocal ( + VideoLocalID INTEGER PRIMARY KEY AUTOINCREMENT, + Hash TEXT NOT NULL, + FileSize INTEGER NOT NULL, + DateTimeUpdated DATETIME NOT NULL, + FileName TEXT NOT NULL DEFAULT '', + DateTimeCreated DATETIME NULL, + MediaBlob BLOB NULL + ) + """; + + private const string AniDBAnimeTitle = + "CREATE TABLE AniDB_Anime_Title ( AniDB_Anime_TitleID INTEGER PRIMARY KEY AUTOINCREMENT, AnimeID INTEGER NOT NULL, TitleType TEXT NOT NULL, Language TEXT NOT NULL, Title TEXT NULL )"; + + #region The two columns the migration tightens + + [Theory] + [InlineData(VideoLocalBeforeTheHashColumnsAreDropped)] + [InlineData(VideoLocalAfterTheHashColumnsAreDropped)] + public void TheColumnIsTightenedWhicheverShapeTheTableIsIn(string createCommand) + { + var patched = SQLite.NotNullVariantOf(createCommand, "DateTimeCreated"); + + Assert.Contains("DateTimeCreated DATETIME NOT NULL", patched); + Assert.DoesNotContain("DateTimeCreated DATETIME NULL", patched); + } + + [Fact] + public void TheTitleColumnIsTightened() + { + var patched = SQLite.NotNullVariantOf(AniDBAnimeTitle, "Title"); + + Assert.Contains("Title TEXT NOT NULL )", patched); + } + + #endregion + + #region Everything else is left alone + + [Theory] + [InlineData(VideoLocalBeforeTheHashColumnsAreDropped, "DateTimeCreated")] + [InlineData(VideoLocalAfterTheHashColumnsAreDropped, "DateTimeCreated")] + [InlineData(AniDBAnimeTitle, "Title")] + public void EveryOtherColumnSurvivesUnchanged(string createCommand, string columnName) + { + var before = Columns(createCommand); + var after = Columns(SQLite.NotNullVariantOf(createCommand, columnName)); + + // The rebuild copies column by column, so a lost column loses its data with it. + Assert.Equal(before.Length, after.Length); + for (var i = 0; i < before.Length; i++) + { + if (before[i].StartsWith(columnName + " ", StringComparison.Ordinal)) + continue; + + Assert.Equal(before[i], after[i]); + } + } + + [Fact] + public void ThePrimaryKeyAndDefaultsAreKept() + { + var patched = SQLite.NotNullVariantOf(VideoLocalAfterTheHashColumnsAreDropped, "DateTimeCreated"); + + Assert.Contains("VideoLocalID INTEGER PRIMARY KEY AUTOINCREMENT", patched); + Assert.Contains("FileName TEXT NOT NULL DEFAULT ''", patched); + } + + [Fact] + public void AColumnWhoseNameContainsAnotherIsNotConfusedForIt() + { + var patched = SQLite.NotNullVariantOf( + "CREATE TABLE T ( DateTimeCreatedRaw TEXT NULL, DateTimeCreated DATETIME NULL, XDateTimeCreated TEXT NULL )", + "DateTimeCreated"); + + Assert.Contains("DateTimeCreatedRaw TEXT NULL", patched); + Assert.Contains("XDateTimeCreated TEXT NULL", patched); + Assert.Contains("DateTimeCreated DATETIME NOT NULL", patched); + } + + #endregion + + #region Shapes it still has to handle + + [Fact] + public void AColumnStatingNeitherNullNorNotNullIsStillTightened() + // SQLite treats an unstated column as nullable. + => Assert.Contains("LastAVDumped DATETIME NOT NULL", + SQLite.NotNullVariantOf("CREATE TABLE T ( Hash TEXT NOT NULL, LastAVDumped DATETIME )", "LastAVDumped")); + + [Fact] + public void ASizedTypeKeepsItsSize() + // The size contains a comma, which is also what separates columns. + => Assert.Contains("Rating decimal(6,2) NOT NULL", + SQLite.NotNullVariantOf("CREATE TABLE T ( Rating decimal(6,2) NULL, Votes INTEGER NOT NULL )", "Rating")); + + [Fact] + public void AColumnThatIsAlreadyNotNullIsLeftAsItIs() + { + const string createCommand = "CREATE TABLE T ( Hash TEXT NOT NULL, Votes INTEGER NOT NULL )"; + + Assert.Equal(createCommand, SQLite.NotNullVariantOf(createCommand, "Hash")); + } + + [Fact] + public void AColumnThatIsNotThereIsAnError() + // Returning it unchanged would rebuild the table untightened and report success. + => Assert.Throws( + () => SQLite.NotNullVariantOf("CREATE TABLE T ( Hash TEXT NOT NULL )", "Nonexistent")); + + #endregion + + private static string[] Columns(string createCommand) + { + var body = createCommand[(createCommand.IndexOf('(') + 1)..createCommand.LastIndexOf(')')]; + var columns = new System.Collections.Generic.List(); + var depth = 0; + var current = new System.Text.StringBuilder(); + foreach (var character in body) + { + if (character is '(') depth++; + if (character is ')') depth--; + if (character is ',' && depth is 0) + { + columns.Add(current.ToString().Trim()); + current.Clear(); + continue; + } + + current.Append(character); + } + + columns.Add(current.ToString().Trim()); + + return [.. columns]; + } +} diff --git a/Shoko.Tests/Databases/SqliteRetypedVariantTests.cs b/Shoko.Tests/Databases/SqliteRetypedVariantTests.cs new file mode 100644 index 0000000000..da5f9d00a9 --- /dev/null +++ b/Shoko.Tests/Databases/SqliteRetypedVariantTests.cs @@ -0,0 +1,163 @@ +using System; +using Shoko.Server.Databases; +using Xunit; + +namespace Shoko.Tests.Databases; + +/// +/// , which rewrites a table's own +/// CREATE TABLE to give one column a different type. +/// +/// +/// SQLite cannot retype a column in place, so the migration rebuilds the table around a patched +/// CREATE TABLE. Only the type may be replaced: the constraints that follow it carry the +/// nullability, the default and the primary key, and dropping one of those loses data or rejects +/// rows the table used to accept. +/// +public class SqliteRetypedVariantTests +{ + private const string AniDBAnime = + "CREATE TABLE AniDB_Anime ( AniDB_AnimeID INTEGER PRIMARY KEY AUTOINCREMENT, AnimeID INTEGER NOT NULL, AirDate DATETIME NULL, EndDate DATETIME NULL, MainTitle TEXT NOT NULL )"; + + // As an ALTER TABLE that named no type leaves it. + private const string AnimeSeriesUser = + "CREATE TABLE AnimeSeries_User ( AnimeSeries_UserID INTEGER PRIMARY KEY AUTOINCREMENT, JMMUserID INTEGER NOT NULL, WatchedDate DATETIME, UserTags NOT NULL DEFAULT '' )"; + + private const string TmdbEpisode = + "CREATE TABLE TMDB_Episode ( TMDB_EpisodeID INTEGER PRIMARY KEY AUTOINCREMENT, EpisodeNumber INTEGER NOT NULL, Runtime TEXT NULL, UserRating REAL NOT NULL )"; + + #region The columns the migration retypes + + [Fact] + public void ADeclaredTypeIsReplaced() + => Assert.Contains("AirDate varchar(10) NULL", SQLite.RetypedVariantOf(AniDBAnime, "AirDate", "varchar(10)")); + + [Fact] + public void AColumnWithNoTypeAtAllIsGivenOne() + { + var patched = SQLite.RetypedVariantOf(AnimeSeriesUser, "UserTags", "TEXT"); + + Assert.Contains("UserTags TEXT NOT NULL DEFAULT ''", patched); + } + + [Fact] + public void ATextColumnBecomesAnIntegerOne() + => Assert.Contains("Runtime INTEGER NULL", SQLite.RetypedVariantOf(TmdbEpisode, "Runtime", "INTEGER")); + + [Fact] + public void RetypingTwiceRetypesBothColumns() + { + var patched = SQLite.RetypedVariantOf(SQLite.RetypedVariantOf(AniDBAnime, "AirDate", "varchar(10)"), "EndDate", "varchar(10)"); + + Assert.Contains("AirDate varchar(10) NULL", patched); + Assert.Contains("EndDate varchar(10) NULL", patched); + } + + #endregion + + #region Everything else is left alone + + [Theory] + [InlineData(AniDBAnime, "AirDate", "varchar(10)")] + [InlineData(AnimeSeriesUser, "UserTags", "TEXT")] + [InlineData(TmdbEpisode, "Runtime", "INTEGER")] + public void EveryOtherColumnSurvivesUnchanged(string createCommand, string columnName, string type) + { + var before = Columns(createCommand); + var after = Columns(SQLite.RetypedVariantOf(createCommand, columnName, type)); + + // The rebuild copies column by column, so a lost column loses its data with it. + Assert.Equal(before.Length, after.Length); + for (var i = 0; i < before.Length; i++) + { + if (before[i].StartsWith(columnName + " ", StringComparison.Ordinal)) + continue; + + Assert.Equal(before[i], after[i]); + } + } + + [Fact] + public void ThePrimaryKeyAndConstraintsAreKept() + { + var patched = SQLite.RetypedVariantOf(AnimeSeriesUser, "UserTags", "TEXT"); + + Assert.Contains("AnimeSeries_UserID INTEGER PRIMARY KEY AUTOINCREMENT", patched); + Assert.Contains("NOT NULL DEFAULT ''", patched); + } + + [Fact] + public void AColumnWhoseNameContainsAnotherIsNotConfusedForIt() + { + var patched = SQLite.RetypedVariantOf( + "CREATE TABLE T ( AirDateRaw TEXT NULL, AirDate DATETIME NULL, LatestEpisodeAirDate DATETIME NULL )", + "AirDate", + "varchar(10)"); + + Assert.Contains("AirDateRaw TEXT NULL", patched); + Assert.Contains("LatestEpisodeAirDate DATETIME NULL", patched); + Assert.Contains("AirDate varchar(10) NULL", patched); + } + + #endregion + + #region Shapes it still has to handle + + [Fact] + public void ASizedTypeIsReplacedWhole() + // The size contains a comma, which is also what separates columns. + => Assert.Contains("Rating INTEGER NOT NULL", + SQLite.RetypedVariantOf("CREATE TABLE T ( Rating decimal(6,2) NOT NULL, Votes INTEGER NOT NULL )", "Rating", "INTEGER")); + + [Fact] + public void AMultiWordTypeIsReplacedWhole() + => Assert.Contains("FileSize INTEGER NOT NULL", + SQLite.RetypedVariantOf("CREATE TABLE T ( FileSize UNSIGNED BIG INT NOT NULL )", "FileSize", "INTEGER")); + + [Fact] + public void AColumnStatingNoNullabilityKeepsStatingNone() + // SQLite treats an unstated column as nullable, and saying NULL here would be a change. + => Assert.Contains("LastAVDumped varchar(10) )", + SQLite.RetypedVariantOf("CREATE TABLE T ( Hash TEXT NOT NULL, LastAVDumped DATETIME )", "LastAVDumped", "varchar(10)")); + + [Fact] + public void AColumnAlreadyOfThatTypeIsLeftAsItIs() + { + const string createCommand = "CREATE TABLE T ( Hash TEXT NOT NULL, Votes INTEGER NOT NULL )"; + + Assert.Equal(createCommand, SQLite.RetypedVariantOf(createCommand, "Hash", "TEXT")); + } + + [Fact] + public void AColumnThatIsNotThereIsAnError() + // Returning it unchanged would rebuild the table unretyped and report success. + => Assert.Throws( + () => SQLite.RetypedVariantOf("CREATE TABLE T ( Hash TEXT NOT NULL )", "Nonexistent", "INTEGER")); + + #endregion + + private static string[] Columns(string createCommand) + { + var body = createCommand[(createCommand.IndexOf('(') + 1)..createCommand.LastIndexOf(')')]; + var columns = new System.Collections.Generic.List(); + var depth = 0; + var current = new System.Text.StringBuilder(); + foreach (var character in body) + { + if (character is '(') depth++; + if (character is ')') depth--; + if (character is ',' && depth is 0) + { + columns.Add(current.ToString().Trim()); + current.Clear(); + continue; + } + + current.Append(character); + } + + columns.Add(current.ToString().Trim()); + + return [.. columns]; + } +} diff --git a/Shoko.Tests/Databases/UserTypeConverterTests.cs b/Shoko.Tests/Databases/UserTypeConverterTests.cs new file mode 100644 index 0000000000..669f417b53 --- /dev/null +++ b/Shoko.Tests/Databases/UserTypeConverterTests.cs @@ -0,0 +1,301 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Newtonsoft.Json.Linq; +using NHibernate.UserTypes; +using Shoko.Abstractions.Metadata; +using Shoko.Server.Databases.NHibernate; +using Shoko.Server.MediaInfo; +using Shoko.Abstractions.Metadata.Enums; +using Shoko.Server.Models.TMDB; +using Xunit; + +namespace Shoko.Tests.Databases; + +/// +/// Covers the converters, which sit between the entity properties and the +/// database columns. +/// +/// +/// Nothing else stands between a stored column and the value handed back to the application, so a +/// converter that loses information corrupts data with no error anywhere. The existing MediaInfo +/// test covers raw MessagePack, but not the converter NHibernate actually calls, and not the +/// Equals that decides whether a change is written back at all. +/// +public class UserTypeConverterTests +{ + private static readonly Type[] s_converterTypes = + [ + .. typeof(StringListConverter).Assembly + .GetTypes() + .Where(t => t is { IsClass: true, IsAbstract: false, IsGenericTypeDefinition: false, IsPublic: true }) + .Where(t => typeof(IUserType).IsAssignableFrom(t)) + .Where(t => t.GetConstructor(Type.EmptyTypes) is not null) + .OrderBy(t => t.FullName, StringComparer.Ordinal), + // Closed over MediaContainer because reflection never yields an open generic, which quietly + // left the converter for VideoLocal.MediaInfo out of every theory below. + typeof(MessagePackConverter), + ]; + + private static IUserType Resolve(string fullName) + => (IUserType)Activator.CreateInstance(s_converterTypes.Single(t => t.FullName == fullName))!; + + public static TheoryData AllConverters() + { + var data = new TheoryData(); + foreach (var type in s_converterTypes) + data.Add(type.FullName!); + + return data; + } + + #region The contract every converter owes NHibernate + + [Fact] + public void TheConvertersAreDiscovered() + => Assert.True(s_converterTypes.Length >= 10, $"Only found {s_converterTypes.Length} converters."); + + [Theory] + [MemberData(nameof(AllConverters))] + public void EveryConverterDeclaresItsColumnTypes(string fullName) + { + var converter = Resolve(fullName); + + // NHibernate needs at least one column type to build the mapping at all. + Assert.NotEmpty(converter.SqlTypes); + Assert.All(converter.SqlTypes, Assert.NotNull); + } + + public static TheoryData EqualButDistinctValues() => new() + { + { typeof(StringListConverter).FullName!, new List { "a", "b" }, new List { "a", "b" } }, + { typeof(TmdbContentRatingConverter).FullName!, ContentRatings(), ContentRatings() }, + { typeof(TmdbProductionCountryConverter).FullName!, ProductionCountries(), ProductionCountries() }, + { typeof(TitleTypeConverter).FullName!, TitleType.Main, TitleType.Main }, + { typeof(TitleLanguageConverter).FullName!, TitleLanguage.English, TitleLanguage.English }, + { typeof(MessagePackConverter).FullName!, new MediaContainer(), new MediaContainer() }, + { typeof(PartialDateOnlyConverter).FullName!, new PartialDateOnly(2024, 5, 1), new PartialDateOnly(2024, 5, 1) }, + { typeof(DateOnlyConverter).FullName!, new DateOnly(2024, 5, 1), new DateOnly(2024, 5, 1) }, + { + typeof(JTokenDictionaryConverter).FullName!, + new Dictionary { ["a"] = JToken.FromObject(1) }, + new Dictionary { ["a"] = JToken.FromObject(1) } + }, + }; + + private static List ContentRatings() => [new("US", "PG-13")]; + + private static List ProductionCountries() => [new("US", "United States")]; + + public static TheoryData UnequalValues() => new() + { + { typeof(StringListConverter).FullName!, new List { "a", "b" }, new List { "a", "c" } }, + { typeof(StringListConverter).FullName!, new List { "a" }, new List { "a", "b" } }, + { typeof(TmdbContentRatingConverter).FullName!, ContentRatings(), new List { new("GB", "12A") } }, + { typeof(PartialDateOnlyConverter).FullName!, new PartialDateOnly(2024, 5, 1), new PartialDateOnly(2024, 5, 2) }, + { typeof(DateOnlyConverter).FullName!, new DateOnly(2024, 5, 1), new DateOnly(2024, 5, 2) }, + { + typeof(JTokenDictionaryConverter).FullName!, + new Dictionary { ["a"] = JToken.FromObject(1) }, + new Dictionary { ["a"] = JToken.FromObject(2) } + }, + { + typeof(JTokenDictionaryConverter).FullName!, + new Dictionary { ["a"] = JToken.FromObject(1) }, + new Dictionary { ["a"] = JToken.FromObject(1), ["b"] = JToken.FromObject(2) } + }, + { typeof(TmdbProductionCountryConverter).FullName!, ProductionCountries(), new List { new("GB", "United Kingdom") } }, + { typeof(TitleTypeConverter).FullName!, TitleType.Main, TitleType.Official }, + { typeof(TitleLanguageConverter).FullName!, TitleLanguage.English, TitleLanguage.Japanese }, + { typeof(MessagePackConverter).FullName!, new MediaContainer(), new MediaContainer { media = new() } }, + }; + + [Fact] + public void EveryConverterCheckedForEqualityIsAlsoCheckedForInequality() + { + // A converter listed only among the equal values looks covered while still passing with a + // stuck-true Equals, which is the failure this pair of theories exists to catch. + var equal = EqualButDistinctValues().Select(row => row.Data.Item1).ToHashSet(StringComparer.Ordinal); + var unequal = UnequalValues().Select(row => row.Data.Item1).ToHashSet(StringComparer.Ordinal); + + Assert.Equal(string.Empty, string.Join(", ", equal.Except(unequal).Order(StringComparer.Ordinal))); + } + + [Theory] + [MemberData(nameof(UnequalValues))] + public void DifferentValuesDoNotCompareEqual(string fullName, object left, object right) + { + // Without this direction a converter could return true unconditionally and satisfy every + // other assertion here, while NHibernate quietly stopped writing the column back. + Assert.False(Resolve(fullName).Equals(left, right), $"{fullName} reports two different values as the same."); + } + + [Theory] + [MemberData(nameof(EqualButDistinctValues))] + public void EqualValuesCompareEqualEvenWhenTheyAreNotTheSameInstance(string fullName, object left, object right) + { + var converter = Resolve(fullName); + + // This is the comparison NHibernate runs on every flush to decide whether a property + // changed. NullSafeGet hands back a fresh instance on every load, so a converter that can + // only compare by reference reports every untouched row as dirty and rewrites it forever. + Assert.NotSame(left, right); + Assert.True(converter.Equals(left, right), $"{fullName} reports two equal values as different."); + } + + [Theory] + [MemberData(nameof(AllConverters))] + public void EveryConverterTreatsAValueAsEqualToItself(string fullName) + { + var converter = Resolve(fullName); + var value = new object(); + + Assert.True(converter.Equals(value, value)); + } + + #endregion + + #region MessagePack + + [Fact] + public void MediaInfoSurvivesAConverterRoundTrip() + { + var converter = new MessagePackConverter(); + var original = new MediaContainer(); + + var stored = converter.ConvertTo(null, CultureInfo.InvariantCulture, original, typeof(byte[])); + + var bytes = Assert.IsType(stored); + Assert.NotEmpty(bytes); + Assert.IsType(converter.ConvertFrom(null, CultureInfo.InvariantCulture, bytes)); + } + + [Fact] + public void MessagePackConverter_StoresNullAsNull() + => Assert.Null(new MessagePackConverter().ConvertTo(null, CultureInfo.InvariantCulture, null, typeof(byte[]))); + + [Fact] + public void MessagePackConverter_OnlyAcceptsBytesBack() + => Assert.Throws( + () => new MessagePackConverter().ConvertFrom(null, CultureInfo.InvariantCulture, "not bytes")); + + #endregion + + #region String lists + + [Fact] + public void AStringListSurvivesARoundTrip() + { + var converter = new StringListConverter(); + var original = new List { "alpha", "beta", "gamma" }; + + var stored = converter.ConvertTo(null, CultureInfo.InvariantCulture, original, typeof(string)); + var restored = converter.ConvertFrom(null, CultureInfo.InvariantCulture, stored); + + Assert.Equal(original, Assert.IsType>(restored)); + } + + [Fact] + public void AnEmptyStringListStoresAsAnEmptyString() + => Assert.Equal(string.Empty, new StringListConverter().ConvertTo(null, CultureInfo.InvariantCulture, new List(), typeof(string))); + + [Fact] + public void ANullStringListReadsBackAsEmpty() + => Assert.Empty(Assert.IsType>(new StringListConverter().ConvertFrom(null, CultureInfo.InvariantCulture, null))); + + [Fact] + public void AStringListEntryContainingTheSeparatorIsSplitApart() + { + var converter = new StringListConverter(); + + var stored = converter.ConvertTo(null, CultureInfo.InvariantCulture, new List { "a|||b" }, typeof(string)); + var restored = Assert.IsType>(converter.ConvertFrom(null, CultureInfo.InvariantCulture, stored)); + + // The list is delimited by "|||" with no escaping, so a value containing it comes back as + // two entries. Pinned so the limitation is visible rather than discovered in a user's data. + Assert.Equal(["a", "b"], restored); + } + + #endregion + + #region Dates + + [Fact] + public void APartialDateSurvivesARoundTrip() + { + var converter = new PartialDateOnlyConverter(); + var original = new PartialDateOnly(2024, 5, 1); + + var stored = converter.ConvertTo(null, CultureInfo.InvariantCulture, original, typeof(string)); + var restored = converter.ConvertFrom(null, CultureInfo.InvariantCulture, stored); + + Assert.Equal(original, Assert.IsType(restored)); + } + + [Fact] + public void AYearOnlyPartialDateKeepsItsMissingParts() + { + var converter = new PartialDateOnlyConverter(); + var original = new PartialDateOnly(2024); + + var stored = converter.ConvertTo(null, CultureInfo.InvariantCulture, original, typeof(string)); + var restored = Assert.IsType(converter.ConvertFrom(null, CultureInfo.InvariantCulture, stored)); + + // AniDB supplies plenty of year-only dates; filling in a month and day would invent data. + Assert.Equal(2024, restored.Year); + Assert.Null(restored.Month); + Assert.Null(restored.Day); + } + + [Fact] + public void ANullPartialDateStaysNull() + => Assert.Null(new PartialDateOnlyConverter().ConvertFrom(null, CultureInfo.InvariantCulture, null)); + + [Fact] + public void ADateOnlySurvivesARoundTripThroughADateTime() + { + var converter = new DateOnlyConverter(); + var original = new DateOnly(2024, 5, 1); + + var restored = converter.ConvertFrom(null, CultureInfo.InvariantCulture, original.ToDateTime(TimeOnly.MinValue)); + + Assert.Equal(original, Assert.IsType(restored)); + } + + #endregion + + #region Types and JSON + + [Fact] + public void ATypeSurvivesARoundTrip() + { + var converter = new TypeStringConverter(); + + var stored = converter.ConvertTo(null, CultureInfo.InvariantCulture, typeof(StringListConverter), typeof(string)); + var restored = converter.ConvertFrom(null, CultureInfo.InvariantCulture, stored!); + + Assert.Same(typeof(StringListConverter), restored); + } + + [Fact] + public void AnUnknownTypeNameReadsBackAsNull() + => Assert.Null(new TypeStringConverter().ConvertFrom(null, CultureInfo.InvariantCulture, "Nothing.Called.This")); + + [Fact] + public void AJTokenDictionarySurvivesARoundTrip() + { + var converter = new JTokenDictionaryConverter(); + var original = new Dictionary { ["a"] = JToken.FromObject(1), ["b"] = JToken.FromObject("two") }; + + var stored = converter.ConvertTo(null, CultureInfo.InvariantCulture, original, typeof(string)); + var restored = converter.ConvertFrom(null, CultureInfo.InvariantCulture, stored!); + + var dictionary = Assert.IsAssignableFrom>(restored); + Assert.Equal(2, dictionary.Count); + Assert.True(JToken.DeepEquals(original["a"], dictionary["a"])); + Assert.True(JToken.DeepEquals(original["b"], dictionary["b"])); + } + + #endregion +} diff --git a/Shoko.Tests/FilterTests.cs b/Shoko.Tests/FilterTests.cs index d5fb3d02bb..514852cb5e 100644 --- a/Shoko.Tests/FilterTests.cs +++ b/Shoko.Tests/FilterTests.cs @@ -8,6 +8,7 @@ using Shoko.Abstractions.Filtering.Expressions.Logic.Expressions; using Shoko.Abstractions.Filtering.Expressions.Selectors.DateSelectors; using Shoko.Abstractions.Filtering.Expressions.User; +using Shoko.TestData; using Xunit; namespace Shoko.Tests; diff --git a/Shoko.Tests/Filters/FilterExpressionSerializationTests.cs b/Shoko.Tests/Filters/FilterExpressionSerializationTests.cs new file mode 100644 index 0000000000..e930323a52 --- /dev/null +++ b/Shoko.Tests/Filters/FilterExpressionSerializationTests.cs @@ -0,0 +1,185 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using Shoko.Abstractions.Filtering.Expressions; +using Shoko.Abstractions.Filtering.Expressions.Info; +using Shoko.Abstractions.Filtering.Expressions.Logic.Expressions; +using Shoko.Abstractions.Filtering.Expressions.User; +using Shoko.Abstractions.Filtering.Sorting; +using Shoko.Server.Databases.NHibernate; +using Xunit; + +namespace Shoko.Tests.Filters; + +/// +/// Guards the persistence format of saved filters. FilterPreset.Expression and +/// FilterPreset.SortingExpression are stored as JSON written by +/// , which records each node by its simple class +/// name via . +/// +/// +/// That makes the format quietly fragile in two directions, and both fail silently: +/// returns for a name +/// it cannot resolve, and swallows the +/// resulting error, so a renamed or removed expression turns a user's filter into a broken one with +/// nothing logged at the call site. Where two types share a simple name the binder picks the first +/// match it happens to find, which can bind a saved filter to the wrong expression entirely. +/// +public class FilterExpressionSerializationTests +{ + /// Every concrete node that can legitimately appear in a stored filter. + private static readonly Type[] s_expressionTypes = + [ + .. typeof(FilterExpression).Assembly + .GetTypes() + .Where(t => t is { IsClass: true, IsAbstract: false, IsGenericTypeDefinition: false, IsPublic: true }) + .Where(t => !t.Name.Contains('<')) + .Where(typeof(FilterExpression).IsAssignableFrom) + .OrderBy(t => t.FullName, StringComparer.Ordinal), + ]; + + private static Type Resolve(string fullName) + => s_expressionTypes.Single(t => t.FullName == fullName); + + public static TheoryData AllExpressions() + { + var data = new TheoryData(); + foreach (var type in s_expressionTypes) + data.Add(type.FullName!); + + return data; + } + + public static TheoryData ConstructibleExpressions() + { + var data = new TheoryData(); + foreach (var type in s_expressionTypes.Where(t => t.GetConstructor(Type.EmptyTypes) is not null)) + data.Add(type.FullName!); + + return data; + } + + #region Discovery + + [Fact] + public void TheExpressionTypesAreDiscovered() + { + // Guards the theories below from silently becoming empty if the assembly or hierarchy moves. + Assert.True(s_expressionTypes.Length > 250, $"Only found {s_expressionTypes.Length} expression types."); + Assert.Contains(s_expressionTypes, t => typeof(SortingExpression).IsAssignableFrom(t)); + } + + [Fact] + public void TheSortingSelectorsAreIncluded() + { + // SortingExpression derives from FilterExpression, so the sorting column shares this + // binder. If that hierarchy changes, saved sort orders stop resolving. + var selectors = s_expressionTypes.Where(t => typeof(SortingExpression).IsAssignableFrom(t)).ToArray(); + + Assert.True(selectors.Length > 50, $"Only found {selectors.Length} sorting selectors."); + } + + #endregion + + #region Name binding + + [Fact] + public void NoTwoExpressionsShareASimpleName() + { + var collisions = s_expressionTypes + .GroupBy(t => t.Name, StringComparer.Ordinal) + .Where(g => g.Count() > 1) + .Select(g => $"{g.Key}: {string.Join(", ", g.Select(t => t.FullName))}") + .ToArray(); + + // The binder resolves by simple name and takes the first match, so a collision would bind + // stored filters to an arbitrary one of the two types. + Assert.Empty(collisions); + } + + [Theory] + [MemberData(nameof(AllExpressions))] + public void EveryExpressionBindsBackToItsOwnType(string fullName) + { + var type = Resolve(fullName); + var binder = new SimpleNameSerializationBinder(typeof(FilterExpression)); + + binder.BindToName(type, out _, out var typeName); + + Assert.NotNull(typeName); + Assert.Same(type, binder.BindToType(assemblyName: null, typeName!)); + } + + [Fact] + public void AnUnrecognisedNameDoesNotBind() + { + var binder = new SimpleNameSerializationBinder(typeof(FilterExpression)); + + // This is what a renamed or deleted expression looks like on load. It resolves to nothing, + // and the converter turns that into a silently broken filter rather than an error. + Assert.Null(binder.BindToType(assemblyName: null, "AnExpressionThatNoLongerExists")); + } + + [Fact] + public void ATypeOutsideTheExpressionHierarchyDoesNotBind() + { + var binder = new SimpleNameSerializationBinder(typeof(FilterExpression)); + + Assert.Null(binder.BindToType(assemblyName: null, nameof(String))); + } + + #endregion + + #region Round trip + + [Theory] + [MemberData(nameof(ConstructibleExpressions))] + public void EveryConstructibleExpressionSurvivesARoundTrip(string fullName) + { + var type = Resolve(fullName); + var converter = new FilterExpressionConverter(); + var original = Activator.CreateInstance(type)!; + + var json = converter.ConvertTo(null, CultureInfo.InvariantCulture, original, typeof(string)); + Assert.NotNull(json); + + var restored = converter.ConvertFrom(null, CultureInfo.InvariantCulture, json!); + + Assert.NotNull(restored); + Assert.IsType(type, restored); + } + + [Fact] + public void ANestedExpressionTreeSurvivesARoundTrip() + { + var converter = new FilterExpressionConverter(); + var original = new AndExpression(new HasWatchedEpisodesExpression(), new NotExpression(new HasTagExpression("comedy"))); + + var json = converter.ConvertTo(null, CultureInfo.InvariantCulture, original, typeof(string)); + var restored = Assert.IsType(converter.ConvertFrom(null, CultureInfo.InvariantCulture, json!)); + + // The tree, not just the root, has to come back intact. + Assert.IsType(restored.Left); + var not = Assert.IsType(restored.Right); + Assert.Equal("comedy", Assert.IsType(not.Left).Parameter); + } + + [Fact] + public void ConvertTo_WritesOnlyTheSimpleTypeName() + { + var converter = new FilterExpressionConverter(); + + var json = (string)converter.ConvertTo(null, CultureInfo.InvariantCulture, new HasWatchedEpisodesExpression(), typeof(string))!; + + // Assembly-qualified names would tie stored filters to an assembly version. + Assert.Contains("\"$type\": \"HasWatchedEpisodesExpression\"", json.Replace("\"$type\":\"", "\"$type\": \"")); + Assert.DoesNotContain("Shoko.Abstractions,", json); + } + + [Fact] + public void ConvertTo_ReturnsNullForANullExpression() + => Assert.Null(new FilterExpressionConverter().ConvertTo(null, CultureInfo.InvariantCulture, null, typeof(string))); + + #endregion +} diff --git a/Shoko.Tests/Filters/SortingSelectorTests.cs b/Shoko.Tests/Filters/SortingSelectorTests.cs new file mode 100644 index 0000000000..ce6a3e93e6 --- /dev/null +++ b/Shoko.Tests/Filters/SortingSelectorTests.cs @@ -0,0 +1,237 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Shoko.Abstractions.Filtering.Sorting; +using Shoko.Abstractions.Filtering.Sorting.Selectors; +using Shoko.Abstractions.Metadata; +using Shoko.Tests.Infrastructure; +using Xunit; + +namespace Shoko.Tests.Filters; + +/// +/// Covers the sorting selectors, which had no tests of any kind. Each one projects a filterable to +/// the value the collection is ordered by, so a selector that throws, returns null, or quietly +/// disagrees with its own TimeDependent/UserDependent flags produces a wrong or +/// broken sort order for the user. +/// +public class SortingSelectorTests +{ + private static readonly Type[] s_selectorTypes = + [ + .. typeof(SortingExpression).Assembly + .GetTypes() + .Where(t => t is { IsClass: true, IsAbstract: false, IsGenericTypeDefinition: false, IsPublic: true }) + .Where(typeof(SortingExpression).IsAssignableFrom) + .OrderBy(t => t.FullName, StringComparer.Ordinal), + ]; + + private static readonly TestFilterable s_filterable = FilterableFactory.CreatePopulated(); + + private static readonly TestFilterableUserInfo s_userInfo = FilterableFactory.CreatePopulated(); + + private static SortingExpression Create(string fullName) + => (SortingExpression)Activator.CreateInstance(s_selectorTypes.Single(t => t.FullName == fullName))!; + + public static TheoryData AllSelectors() + { + var data = new TheoryData(); + foreach (var type in s_selectorTypes.Where(t => t.GetConstructor(Type.EmptyTypes) is not null)) + data.Add(type.FullName!); + + return data; + } + + #region Discovery + + [Fact] + public void TheSelectorsAreDiscovered() + { + // Guards the theories below from silently emptying out. + Assert.True(s_selectorTypes.Length > 50, $"Only found {s_selectorTypes.Length} sorting selectors."); + } + + [Fact] + public void EverySelectorCanBeConstructedWithoutArguments() + { + // Stored sort orders are rebuilt by the JSON deserialiser, which needs a parameterless ctor. + var missing = s_selectorTypes.Where(t => t.GetConstructor(Type.EmptyTypes) is null).Select(t => t.FullName).ToArray(); + + Assert.Empty(missing); + } + + #endregion + + #region Contract held by every selector + + [Theory] + [MemberData(nameof(AllSelectors))] + public void EverySelectorProducesSomethingComparable(string fullName) + { + // The result is what the collection is ordered by, so it has to be comparable. + Assert.IsAssignableFrom(Create(fullName).Evaluate(s_filterable, s_userInfo, s_date)); + } + + /// + /// Every selector paired with the property it reads. Generated from the selector sources, then + /// held here so a selector that starts reading a different field fails. + /// + public static TheoryData SelectorProperties() => new() + { + { "AddedDateSortingSelector", "filterable", "AddedDate", "" }, + { "AirDateSortingSelector", "filterable", "AirDate", "ToDateTime" }, + { "AudioLanguageCountSortingSelector", "filterable", "AudioLanguages.Count", "" }, + { "AverageAniDBRatingSortingSelector", "filterable", "AverageAniDBRating", "" }, + { "BluRaySourceCountSortingSelector", "filterable", "FileSourceCounts.BluRay", "" }, + { "CameraSourceCountSortingSelector", "filterable", "FileSourceCounts.Camera", "" }, + { "CreditsEpisodesCountSortingSelector", "filterable", "EpisodeCounts.Credits", "" }, + { "CustomTagCountSortingSelector", "filterable", "CustomTags.Count", "" }, + { "DescriptionSortingSelector", "filterable", "Description", "" }, + { "DvdSourceCountSortingSelector", "filterable", "FileSourceCounts.DVD", "" }, + { "EpisodeCountSortingSelector", "filterable", "EpisodeCount", "" }, + { "FilmSourceCountSortingSelector", "filterable", "FileSourceCounts.Film", "" }, + { "GroupIDSortingSelector", "filterable", "GroupID", "" }, + { "HiddenEpisodesSortingSelector", "filterable", "HiddenEpisodes", "" }, + { "HighestAniDBRatingSortingSelector", "filterable", "HighestAniDBRating", "" }, + { "HighestUserRatingSortingSelector", "userInfo", "HighestUserRating", "" }, + { "LaserDiscSourceCountSortingSelector", "filterable", "FileSourceCounts.LaserDisc", "" }, + { "LastAddedDateSortingSelector", "filterable", "LastAddedDate", "" }, + { "LastAirDateSortingSelector", "filterable", "LastAirDate", "ToDateTime" }, + { "LastWatchedDateSortingSelector", "userInfo", "LastWatchedDate", "" }, + { "LocalCreditsEpisodesCountSortingSelector", "filterable", "LocalEpisodeCounts.Credits", "" }, + { "LocalEpisodesCountSortingSelector", "filterable", "LocalEpisodeCounts.Episodes", "" }, + { "LocalOthersEpisodesCountSortingSelector", "filterable", "LocalEpisodeCounts.Others", "" }, + { "LocalParodiesEpisodesCountSortingSelector", "filterable", "LocalEpisodeCounts.Parodies", "" }, + { "LocalSpecialEpisodesCountSortingSelector", "filterable", "LocalEpisodeCounts.Specials", "" }, + { "LocalTrailersEpisodesCountSortingSelector", "filterable", "LocalEpisodeCounts.Trailers", "" }, + { "LowestAniDBRatingSortingSelector", "filterable", "LowestAniDBRating", "" }, + { "LowestUserRatingSortingSelector", "userInfo", "LowestUserRating", "" }, + { "MainNameSortingSelector", "filterable", "MainName", "" }, + { "MissingCreditsEpisodesCountSortingSelector", "filterable", "MissingEpisodeCounts.Credits", "" }, + { "MissingEpisodeCollectingCountSortingSelector", "filterable", "MissingEpisodesCollecting", "" }, + { "MissingEpisodeCountSortingSelector", "filterable", "MissingEpisodes", "" }, + { "MissingEpisodesCountSortingSelector", "filterable", "MissingEpisodeCounts.Episodes", "" }, + { "MissingOthersEpisodesCountSortingSelector", "filterable", "MissingEpisodeCounts.Others", "" }, + { "MissingParodiesEpisodesCountSortingSelector", "filterable", "MissingEpisodeCounts.Parodies", "" }, + { "MissingSpecialEpisodesCountSortingSelector", "filterable", "MissingEpisodeCounts.Specials", "" }, + { "MissingTrailersEpisodesCountSortingSelector", "filterable", "MissingEpisodeCounts.Trailers", "" }, + { "NameSortingSelector", "filterable", "Name", "" }, + { "OriginalNameSortingSelector", "filterable", "OriginalName", "" }, + { "OtherSourceCountSortingSelector", "filterable", "FileSourceCounts.Other", "" }, + { "OthersEpisodesCountSortingSelector", "filterable", "EpisodeCounts.Others", "" }, + { "ParodiesEpisodesCountSortingSelector", "filterable", "EpisodeCounts.Parodies", "" }, + { "SeriesCountSortingSelector", "filterable", "SeriesCount", "" }, + { "SeriesPermanentVoteCountSortingSelector", "userInfo", "SeriesPermanentVoteCount", "" }, + { "SeriesTemporaryVoteCountSortingSelector", "userInfo", "SeriesTemporaryVoteCount", "" }, + { "SeriesVoteCountSortingSelector", "userInfo", "SeriesVoteCount", "" }, + { "SortNameSortingSelector", "filterable", "SortName", "" }, + { "SortingNameSortingSelector", "filterable", "SortName", "" }, + { "SpecialEpisodesCountSortingSelector", "filterable", "EpisodeCounts.Specials", "" }, + { "SubtitleLanguageCountSortingSelector", "filterable", "SubtitleLanguages.Count", "" }, + { "TopLevelGroupIDSortingSelector", "filterable", "TopLevelGroupID", "" }, + { "TotalEpisodeCountSortingSelector", "filterable", "TotalEpisodeCount", "" }, + { "TrailersEpisodesCountSortingSelector", "filterable", "EpisodeCounts.Trailers", "" }, + { "TvSourceCountSortingSelector", "filterable", "FileSourceCounts.TV", "" }, + { "UnairedCreditsEpisodesCountSortingSelector", "filterable", "UnairedEpisodeCounts.Credits", "" }, + { "UnairedEpisodesCountSortingSelector", "filterable", "UnairedEpisodeCounts.Episodes", "" }, + { "UnairedOthersEpisodesCountSortingSelector", "filterable", "UnairedEpisodeCounts.Others", "" }, + { "UnairedParodiesEpisodesCountSortingSelector", "filterable", "UnairedEpisodeCounts.Parodies", "" }, + { "UnairedSpecialEpisodesCountSortingSelector", "filterable", "UnairedEpisodeCounts.Specials", "" }, + { "UnairedTrailersEpisodesCountSortingSelector", "filterable", "UnairedEpisodeCounts.Trailers", "" }, + { "UnknownSourceCountSortingSelector", "filterable", "FileSourceCounts.Unknown", "" }, + { "UnwatchedEpisodeCountSortingSelector", "userInfo", "UnwatchedEpisodes", "" }, + { "UserTagCountSortingSelector", "userInfo", "UserTags.Count", "" }, + { "VcdSourceCountSortingSelector", "filterable", "FileSourceCounts.VCD", "" }, + { "VhsSourceCountSortingSelector", "filterable", "FileSourceCounts.VHS", "" }, + { "WatchedCreditsEpisodesCountSortingSelector", "userInfo", "WatchedEpisodeCounts.Credits", "" }, + { "WatchedDateSortingSelector", "userInfo", "WatchedDate", "" }, + { "WatchedEpisodeCountSortingSelector", "userInfo", "WatchedEpisodes", "" }, + { "WatchedEpisodesCountSortingSelector", "userInfo", "WatchedEpisodeCounts.Episodes", "" }, + { "WatchedOthersEpisodesCountSortingSelector", "userInfo", "WatchedEpisodeCounts.Others", "" }, + { "WatchedParodiesEpisodesCountSortingSelector", "userInfo", "WatchedEpisodeCounts.Parodies", "" }, + { "WatchedSpecialEpisodesCountSortingSelector", "userInfo", "WatchedEpisodeCounts.Specials", "" }, + { "WatchedTrailersEpisodesCountSortingSelector", "userInfo", "WatchedEpisodeCounts.Trailers", "" }, + { "WebSourceCountSortingSelector", "filterable", "FileSourceCounts.Web", "" }, + }; + + [Fact] + public void EverySelectorIsInThePropertyTable() + { + // Without this a selector added tomorrow would be covered only by "returns something + // comparable", which any non-null return satisfies. + var tabled = SelectorProperties().Select(row => row.Data.Item1).ToHashSet(StringComparer.Ordinal); + var missing = s_selectorTypes.Select(t => t.Name) + // Not a plain property read; its scoring is exercised separately. + .Except(["FuzzyNameRelevanceSortingSelector"], StringComparer.Ordinal) + .Except(tabled, StringComparer.Ordinal) + .Order(StringComparer.Ordinal); + + Assert.Equal(string.Empty, string.Join(", ", missing)); + } + + [Fact] + public void TheTestDataTellsEverySelectorApart() + { + // If two selectors reading different fields resolve to the same value on the double, the + // theory below cannot tell one from the other and both assertions become decoration. + // Selectors that genuinely read the same field are expected to agree. + var byValue = new Dictionary>(); + foreach (var row in SelectorProperties()) + { + var (_, source, path, _) = row.Data; + var expected = Resolve(source == "userInfo" ? s_userInfo : s_filterable, path); + byValue.TryAdd($"{expected}", []); + byValue[$"{expected}"].Add($"{source}.{path}"); + } + + var collisions = byValue.Where(entry => entry.Value.Count > 1) + .Select(entry => $"{entry.Key}: {string.Join(", ", entry.Value.Order())}"); + + Assert.Equal(string.Empty, string.Join(" | ", collisions)); + } + + [Theory] + [MemberData(nameof(SelectorProperties))] + public void EverySelectorReadsTheFieldItIsNamedFor(string selector, string source, string path, string transform) + { + var type = s_selectorTypes.Single(t => t.Name == selector); + var instance = (SortingExpression)Activator.CreateInstance(type)!; + object root = source == "userInfo" ? s_userInfo : s_filterable; + + var expected = Resolve(root, path); + if (transform == "ToDateTime") + expected = ((PartialDateOnly)expected!).ToDateTime(); + + Assert.Equal(expected, instance.Evaluate(s_filterable, s_userInfo, s_date)); + } + + /// Walks a dotted property path off the populated double. + private static object? Resolve(object root, string path) + { + var current = root; + foreach (var part in path.Split('.')) + { + var property = current!.GetType().GetProperty(part) + ?? throw new InvalidOperationException($"No property '{part}' on {current.GetType().Name}."); + current = property.GetValue(current); + } + + return current; + } + + #endregion + + #region Defaults + + private static readonly DateTime s_date = new(2020, 1, 2, 3, 4, 5, DateTimeKind.Utc); + + [Fact] + public void Descending_DefaultsToAscending() + => Assert.False(new AddedDateSortingSelector().Descending); + + [Fact] + public void Next_DefaultsToNoFurtherSort() + => Assert.Null(new AddedDateSortingSelector().Next); + + #endregion +} diff --git a/Shoko.Tests/Infrastructure/AniDBTestDoubles.cs b/Shoko.Tests/Infrastructure/AniDBTestDoubles.cs new file mode 100644 index 0000000000..7f8a5e6c9a --- /dev/null +++ b/Shoko.Tests/Infrastructure/AniDBTestDoubles.cs @@ -0,0 +1,142 @@ +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Shoko.Abstractions.Config; +using Shoko.Abstractions.Config.Services; +using Shoko.Server.Providers.AniDB.HTTP; +using Shoko.Server.Providers.AniDB.Interfaces; +using Shoko.Server.Providers.AniDB.UDP; +using Shoko.Server.Settings; + +namespace Shoko.Tests.Infrastructure; + +/// +/// Test doubles for the AniDB connection layer. +/// +/// +/// Nothing here opens a socket or resolves a host. The HTTP side is driven through a stub +/// and the UDP side through , +/// so the protocol handling can be exercised with no network of any kind. +/// +public static class AniDBTestDoubles +{ + /// + /// Builds a over the given settings, with the rate + /// limits zeroed so tests are not paced by the real AniDB throttle. + /// + public static ConfigurationProvider Configuration(ServerSettings? settings = null) + { + settings ??= new ServerSettings(); + settings.AniDb.HTTPRateLimit.BaseRateInSeconds = 0; + settings.AniDb.UDPRateLimit.BaseRateInSeconds = 0; + + var info = (ConfigurationInfo)System.Runtime.CompilerServices.RuntimeHelpers.GetUninitializedObject(typeof(ConfigurationInfo)); + var service = new Mock(); + service.Setup(s => s.GetConfigurationInfo()).Returns(info); + service.Setup(s => s.Load(It.IsAny(), It.IsAny())).Returns(settings); + + return new ConfigurationProvider(service.Object); + } + + public static HttpRateLimiter HttpRateLimiter(ServerSettings? settings = null) + => new(NullLogger.Instance, Configuration(settings)); + + public static UDPRateLimiter UdpRateLimiter(ServerSettings? settings = null) + => new(NullLogger.Instance, Configuration(settings)); + + /// + /// An that answers from a queue instead of the network, and + /// records what it was asked for. + /// + public sealed class StubHttpMessageHandler : HttpMessageHandler + { + private readonly Queue> _responses = new(); + + public List Requests { get; } = []; + + public int CallCount => Requests.Count; + + public StubHttpMessageHandler Respond(HttpStatusCode status, string body) + { + _responses.Enqueue(_ => new HttpResponseMessage(status) { Content = new StringContent(body) }); + return this; + } + + public StubHttpMessageHandler Throw(Exception exception) + { + _responses.Enqueue(_ => throw exception); + return this; + } + + protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + Requests.Add(request); + if (_responses.Count == 0) + throw new InvalidOperationException($"No canned response for {request.RequestUri}."); + + return Task.FromResult(_responses.Dequeue()(request)); + } + } + + /// + /// Builds an whose clients are backed by + /// . + /// + public static IHttpClientFactory HttpClientFactory(HttpMessageHandler handler) + { + var factory = new Mock(); + factory.Setup(f => f.CreateClient(It.IsAny())).Returns(() => new HttpClient(handler, disposeHandler: false)); + return factory.Object; + } + + /// + /// A socket handler that replays canned payloads instead of talking to AniDB. + /// + public sealed class StubSocketHandler : IAniDBSocketHandler + { + private readonly Queue _responses = new(); + + public bool IsConnected { get; set; } = true; + + public bool ConnectionAttempted { get; private set; } + + public List Sent { get; } = []; + + public StubSocketHandler Respond(byte[] payload) + { + _responses.Enqueue(payload); + return this; + } + + public byte[] Send(byte[] payload) + { + Sent.Add(payload); + if (_responses.Count == 0) + throw new InvalidOperationException("No canned response for this UDP call."); + + return _responses.Dequeue(); + } + + public bool TryConnection() + { + ConnectionAttempted = true; + return IsConnected; + } + + public void Dispose() { } + + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + } + + public static IAniDBSocketHandlerFactory SocketHandlerFactory(IAniDBSocketHandler handler) + { + var factory = new Mock(); + factory.Setup(f => f.Create(It.IsAny(), It.IsAny(), It.IsAny())).Returns(handler); + return factory.Object; + } +} diff --git a/Shoko.Tests/Infrastructure/CachedRepo.cs b/Shoko.Tests/Infrastructure/CachedRepo.cs new file mode 100644 index 0000000000..203e6342fd --- /dev/null +++ b/Shoko.Tests/Infrastructure/CachedRepo.cs @@ -0,0 +1,76 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Moq; +using NutzCode.InMemoryIndex; +using Shoko.Server.Repositories; + +namespace Shoko.Tests.Infrastructure; + +/// +/// Builds a real cached repository whose rows live in an in-memory +/// instead of a database. +/// +/// +/// This is deliberately not a mock. is a public field +/// and is public, so seeding them directly gives a +/// repository whose real read paths — including the secondary indexes each repository builds for +/// itself — execute exactly as they do in production. Only Save/Delete reach for the +/// (null) database factory, so a test must not call them; use a mock when a write needs observing. +/// +public static class CachedRepo +{ + public static TRepo Build(Func keySelector, params TEntity[] entities) + where TRepo : BaseCachedRepository + where TEntity : class, new() + where TKey : notnull + => Build(keySelector, (IEnumerable)entities); + + public static TRepo Build(Func keySelector, IEnumerable? entities) + where TRepo : BaseCachedRepository + where TEntity : class, new() + where TKey : notnull + { + // Every cached repository stores its constructor arguments without dereferencing them, so + // nulls are safe for the read-only use this harness supports. + var constructor = typeof(TRepo).GetConstructors() + .OrderBy(c => c.GetParameters().Length) + .First(); + var repository = (TRepo)constructor.Invoke(new object?[constructor.GetParameters().Length]); + + repository.Cache = new PocoCache(entities ?? [], keySelector); + repository.PopulateIndexes(); + + return repository; + } + + /// + /// Builds a cache-backed repository that also accepts writes, returning the mock so a test can + /// verify them. + /// + /// + /// A partial mock with CallBase keeps every real read path intact and replaces only the + /// virtual Save/Delete members, whose real implementations would go to the + /// database. Writes land in the same the reads come from, + /// so a saved entity is visible to a subsequent lookup exactly as it would be in production. + /// + public static Mock BuildWritable(Func keySelector, IEnumerable? entities = null) + where TRepo : BaseCachedRepository + where TEntity : class, new() + where TKey : notnull + { + var constructor = typeof(TRepo).GetConstructors() + .OrderBy(c => c.GetParameters().Length) + .First(); + var mock = new Mock(new object[constructor.GetParameters().Length]) { CallBase = true }; + var repository = mock.Object; + + repository.Cache = new PocoCache(entities ?? [], keySelector); + repository.PopulateIndexes(); + + mock.Setup(r => r.Save(It.IsAny())).Callback(entity => repository.Cache.Update(entity)); + mock.Setup(r => r.Delete(It.IsAny())).Callback(entity => repository.Cache.Remove(entity)); + + return mock; + } +} diff --git a/Shoko.Tests/Infrastructure/FilterableFactory.cs b/Shoko.Tests/Infrastructure/FilterableFactory.cs new file mode 100644 index 0000000000..c50efbe0c8 --- /dev/null +++ b/Shoko.Tests/Infrastructure/FilterableFactory.cs @@ -0,0 +1,146 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Shoko.Abstractions.Metadata; + +namespace Shoko.Tests.Infrastructure; + +/// +/// Builds fully populated filterable test doubles by reflection. +/// +/// +/// The filterable interfaces carry well over a hundred members, and a hand-written initializer +/// would silently stop being complete the moment one is added — leaving anything that reads the new +/// member dereferencing null in tests that look like they cover it. Populating by reflection keeps +/// every member non-null as the interface grows. +/// +public static class FilterableFactory +{ + private static readonly DateTime s_date = new(2020, 1, 2, 3, 4, 5, DateTimeKind.Utc); + + /// + /// Creates with every writable property set to a non-null value that + /// is distinct from its siblings. + /// + /// + /// Values are seeded from the property name rather than its type. Giving every int the + /// same sample would make any test that reads one property indistinguishable from one that + /// reads another of the same type — a selector could return the wrong field and still satisfy + /// its assertion. + /// + public static T CreatePopulated() where T : new() + => (T)Populate(new T(), string.Empty); + + private static object Populate(object instance, string prefix) + { + foreach (var property in instance.GetType().GetProperties(BindingFlags.Public | BindingFlags.Instance)) + { + if (!property.CanWrite) + continue; + + if (SampleFor(property.PropertyType, prefix + property.Name) is { } value) + property.SetValue(instance, value); + } + + return instance; + } + + /// A small stable seed derived from the property name, so runs are reproducible. + private static int SeedFor(string name) + { + var seed = 0; + foreach (var character in name) + seed = ((seed * 31) + character) & 0x7FFFFF; + + return seed; + } + + private static object? SampleFor(Type type, string name) + { + var underlying = Nullable.GetUnderlyingType(type); + if (underlying is not null) + return SampleFor(underlying, name); + + var seed = SeedFor(name); + + if (type == typeof(string)) return $"sample-{name}"; + if (type == typeof(bool)) return true; + if (type == typeof(DateTime)) return s_date.AddSeconds(seed % 100000); + if (type == typeof(DateOnly)) return DateOnly.FromDateTime(s_date.AddDays(seed % 1000)); + if (type == typeof(TimeSpan)) return TimeSpan.FromSeconds(1 + (seed % 10000)); + // A default PartialDateOnly has Year 0, which cannot be converted to a DateOnly, so give it + // a real date rather than letting the struct default through. + if (type == typeof(PartialDateOnly)) return new PartialDateOnly(1990 + (seed % 30), 1 + (seed % 12), 1 + (seed % 28)); + if (type.IsEnum) + { + var values = Enum.GetValues(type); + return values.Length == 0 ? null : values.GetValue(seed % values.Length); + } + // Offset clear of the collection-size range below, so a scalar can never coincide with a + // ".Count" read and make two selectors indistinguishable. + // A wide range so two properties are very unlikely to share a value, and offset clear of + // the collection-size range below so a scalar never coincides with a ".Count" read. + // TheTestDataTellsEverySelectorApart is the backstop if a collision ever does occur. + if (type.IsPrimitive || type == typeof(decimal)) return Convert.ChangeType(1000 + (seed % 1000003), type); + + if (type.IsGenericType) + { + var definition = type.GetGenericTypeDefinition(); + var arguments = type.GetGenericArguments(); + + if (definition == typeof(IReadOnlySet<>) || definition == typeof(ISet<>) || definition == typeof(HashSet<>)) + return BuildSet(arguments[0], name); + + if (definition == typeof(IReadOnlyDictionary<,>) || definition == typeof(IDictionary<,>) || definition == typeof(Dictionary<,>)) + return BuildDictionary(arguments[0], arguments[1], name); + + if (definition == typeof(IReadOnlyList<>) || definition == typeof(IList<>) || definition == typeof(List<>) || definition == typeof(IEnumerable<>)) + return BuildList(arguments[0], name); + } + + // Value types (including tuples) always have a default; reference types need a constructor. + if (type.IsValueType) + return Activator.CreateInstance(type); + + if (type.GetConstructor(Type.EmptyTypes) is null) + return null; + + // Populate one level down as well, so two properties of the same nested type do not come + // back identical either. + return Populate(Activator.CreateInstance(type)!, name + "."); + } + + private static object BuildSet(Type elementType, string name) + { + // Sized from the name as well as filled from it: a set built with one element every time + // gives every ".Count" read the same answer, which makes anything reading one + // indistinguishable from anything reading another. + var set = (IList)Activator.CreateInstance(typeof(List<>).MakeGenericType(elementType))!; + var count = 1 + (SeedFor(name) % 97); + for (var index = 0; index < count; index++) + if (SampleFor(elementType, $"{name}[{index}]") is { } element) + set.Add(element); + + return Activator.CreateInstance(typeof(HashSet<>).MakeGenericType(elementType), set)!; + } + + private static object BuildList(Type elementType, string name) + { + var list = (IList)Activator.CreateInstance(typeof(List<>).MakeGenericType(elementType))!; + if (SampleFor(elementType, name) is { } element) + list.Add(element); + + return list; + } + + private static object BuildDictionary(Type keyType, Type valueType, string name) + { + var dictionary = (IDictionary)Activator.CreateInstance(typeof(Dictionary<,>).MakeGenericType(keyType, valueType))!; + if (SampleFor(keyType, name + ".key") is { } key && SampleFor(valueType, name + ".value") is { } value) + dictionary[key] = value; + + return dictionary; + } +} diff --git a/Shoko.Tests/Infrastructure/RepoFactoryScope.cs b/Shoko.Tests/Infrastructure/RepoFactoryScope.cs new file mode 100644 index 0000000000..79c157cee9 --- /dev/null +++ b/Shoko.Tests/Infrastructure/RepoFactoryScope.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using Shoko.Server.Repositories; +using Xunit; + +namespace Shoko.Tests.Infrastructure; + +/// +/// Temporarily installs repositories into 's static fields, restoring the +/// previous values on dispose. +/// +/// +/// The domain models resolve their navigation properties through these statics (for example +/// AnimeSeries.AniDB_Anime and AnimeEpisode.VideoLocals), so populating them is what +/// lets a test exercise real model and service code with no database. The fields are plain +/// assignable statics — unlike ISystemService.StaticServices, which is write-once — but they +/// are process-global, so every test using this type must join +/// to keep those mutations serialised. +/// +public sealed class RepoFactoryScope : IDisposable +{ + private static readonly FieldInfo[] s_fields = typeof(RepoFactory) + .GetFields(BindingFlags.Public | BindingFlags.Static); + + private readonly List<(FieldInfo Field, object? Previous)> _saved = []; + + /// + /// Installs an already-built repository into the matching field. + /// + public RepoFactoryScope Set(TRepo repository) where TRepo : class + { + var field = s_fields.Single(f => f.FieldType == typeof(TRepo)); + _saved.Add((field, field.GetValue(null))); + field.SetValue(null, repository); + return this; + } + + /// + /// Builds a cache-backed repository from and installs it. + /// + public RepoFactoryScope With(Func keySelector, IEnumerable? entities = null) + where TRepo : BaseCachedRepository + where TEntity : class, new() + where TKey : notnull + => Set(CachedRepo.Build(keySelector, entities)); + + public void Dispose() + { + // Restore in reverse so repeated Set calls for one field unwind to the original value. + for (var i = _saved.Count - 1; i >= 0; i--) + _saved[i].Field.SetValue(null, _saved[i].Previous); + + _saved.Clear(); + } +} + +/// +/// Serialises every test that mutates the process-global statics. Tests +/// outside this collection keep running in parallel. +/// +[CollectionDefinition(nameof(RepoFactoryCollection), DisableParallelization = true)] +public sealed class RepoFactoryCollection; diff --git a/Shoko.Tests/Infrastructure/StubSettingsProvider.cs b/Shoko.Tests/Infrastructure/StubSettingsProvider.cs new file mode 100644 index 0000000000..83506c2701 --- /dev/null +++ b/Shoko.Tests/Infrastructure/StubSettingsProvider.cs @@ -0,0 +1,52 @@ +using System.Threading; +using Shoko.Server.Settings; + +namespace Shoko.Tests.Infrastructure; + +/// +/// A minimal over a plain . +/// +/// +/// Some server code reads the settings singleton from a field initializer — the MySQL backend +/// builds part of its DDL that way — so it has to be installed before those types are constructed. +/// The static is freely assignable, unlike ISystemService.StaticServices, but it is still +/// process-global, so leaves an existing provider alone. +/// +public sealed class StubSettingsProvider(ServerSettings settings) : ISettingsProvider +{ + public ServerSettings Settings { get; } = settings; + + private static readonly Lock _installLock = new(); + + /// + /// Installs a stub provider unless something has already set one. + /// + /// + /// Locked because the check and the assignment are two steps against one process-global static. + /// Two classes installing at once would otherwise each see it unset and each install their own, + /// and the loser's settings — which a caller has already started configuring — would be + /// discarded out from under it. + /// + public static void Install() + { + lock (_installLock) + { + try + { + _ = ISettingsProvider.Instance; + } + catch + { + ISettingsProvider.Instance = new StubSettingsProvider(new ServerSettings()); + } + } + } + + public IServerSettings GetSettings(bool copy = false) => Settings; + + public void SaveSettings(IServerSettings settings) { } + + public void SaveSettings() { } + + public void DebugSettingsToLog() { } +} diff --git a/Shoko.Tests/Models/AnimeEpisodeTitleTests.cs b/Shoko.Tests/Models/AnimeEpisodeTitleTests.cs new file mode 100644 index 0000000000..6f79bd5fe3 --- /dev/null +++ b/Shoko.Tests/Models/AnimeEpisodeTitleTests.cs @@ -0,0 +1,90 @@ +using Shoko.Abstractions.Metadata.Enums; +using Shoko.Server.Models.AniDB; +using Shoko.Server.Models.Shoko; +using Shoko.Server.Repositories.Cached.AniDB; +using Shoko.Tests.Infrastructure; +using Xunit; + +namespace Shoko.Tests.Models; + +/// +/// Covers , which every episode falls back to when the user +/// has set no override and no preferred title matches. It reads through +/// RepoFactory.AniDB_Episode_Title, so these run against real repositories seeded from +/// memory rather than a database. +/// +[Collection(nameof(RepoFactoryCollection))] +public class AnimeEpisodeTitleTests +{ + private static AniDB_Episode_Title Title(int id, int episodeID, string title, TitleLanguage language) + => new() { AniDB_Episode_TitleID = id, AniDB_EpisodeID = episodeID, Title = title, Language = language }; + + private static RepoFactoryScope ScopeWith(params AniDB_Episode_Title[] titles) + => new RepoFactoryScope() + .With(t => t.AniDB_Episode_TitleID, titles); + + [Fact] + public void DefaultTitle_UsesTheEnglishTitleWhenOneExists() + { + using var scope = ScopeWith(Title(1, 100, "The English One", TitleLanguage.English)); + + var episode = new AnimeEpisode { AniDB_EpisodeID = 100 }; + + Assert.Equal("The English One", episode.DefaultTitle.Value); + Assert.Equal(TitleLanguage.English, episode.DefaultTitle.Language); + } + + [Fact] + public void DefaultTitle_IgnoresTitlesInOtherLanguages() + { + using var scope = ScopeWith( + Title(1, 100, "Nihongo", TitleLanguage.Japanese), + Title(2, 100, "The English One", TitleLanguage.English)); + + var episode = new AnimeEpisode { AniDB_EpisodeID = 100 }; + + Assert.Equal("The English One", episode.DefaultTitle.Value); + } + + [Fact] + public void DefaultTitle_IgnoresTitlesBelongingToOtherEpisodes() + { + using var scope = ScopeWith(Title(1, 999, "Someone Else's Title", TitleLanguage.English)); + + var episode = new AnimeEpisode { AniDB_EpisodeID = 100 }; + + Assert.Equal("", episode.DefaultTitle.Value); + } + + [Fact] + public void DefaultTitle_FallsBackToAPlaceholderNamingTheEpisode() + { + using var scope = ScopeWith(); + + var episode = new AnimeEpisode { AniDB_EpisodeID = 100 }; + + Assert.Equal("", episode.DefaultTitle.Value); + Assert.Equal(TitleLanguage.Unknown, episode.DefaultTitle.Language); + Assert.Equal(DataSource.None, episode.DefaultTitle.Source); + } + + [Fact] + public void DefaultTitle_IsResolvedOnceAndReused() + { + using var scope = ScopeWith(Title(1, 100, "The English One", TitleLanguage.English)); + + var episode = new AnimeEpisode { AniDB_EpisodeID = 100 }; + + Assert.Same(episode.DefaultTitle, episode.DefaultTitle); + } + + [Fact] + public void Title_PrefersTheUserOverrideOverAnyStoredTitle() + { + using var scope = ScopeWith(Title(1, 100, "The English One", TitleLanguage.English)); + + var episode = new AnimeEpisode { AniDB_EpisodeID = 100, EpisodeNameOverride = "What The User Called It" }; + + Assert.Equal("What The User Called It", episode.Title); + } +} diff --git a/Shoko.Tests/Providers/AniDB/Connection/AniDBHttpConnectionTests.cs b/Shoko.Tests/Providers/AniDB/Connection/AniDBHttpConnectionTests.cs new file mode 100644 index 0000000000..95779c016f --- /dev/null +++ b/Shoko.Tests/Providers/AniDB/Connection/AniDBHttpConnectionTests.cs @@ -0,0 +1,206 @@ +using System; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Shoko.Server.Providers.AniDB; +using Shoko.Server.Providers.AniDB.HTTP; +using Shoko.Server.Settings; +using Shoko.Tests.Infrastructure; +using Xunit; + +namespace Shoko.Tests.Providers.AniDB.Connection; + +/// +/// Covers — the ban detection and request gating around +/// every AniDB HTTP call. +/// +/// +/// AniDB bans clients that misbehave, and a ban costs the user a day of metadata. The handler is +/// what is supposed to notice one and stop talking, so it is worth knowing it does. Every request +/// here is answered by a stub ; no connection is ever opened. +/// +public class AniDBHttpConnectionTests +{ + private const string BannedBody = "banned"; + + private static (AniDBHttpConnectionHandler Handler, AniDBTestDoubles.StubHttpMessageHandler Http) Create() + { + StubSettingsProvider.Install(); + // The transport below is a stub, so nothing is ever dialed; pointing the base address at an + // RFC 2606 reserved host makes that plain rather than implicit. + ISettingsProvider.Instance.GetSettings().AniDb.HTTPServerUrl = "http://anidb.invalid"; + var http = new AniDBTestDoubles.StubHttpMessageHandler(); + var handler = new AniDBHttpConnectionHandler( + NullLoggerFactory.Instance, + AniDBTestDoubles.HttpRateLimiter(), + AniDBTestDoubles.HttpClientFactory(http)); + + return (handler, http); + } + + #region Successful calls + + [Fact] + public async Task ASuccessfulCallReturnsTheBody() + { + var (handler, http) = Create(); + http.Respond(HttpStatusCode.OK, ""); + + var response = await handler.GetHttp("httpapi?request=anime&aid=1"); + + Assert.Equal("", response.Response); + Assert.Equal(HttpStatusCode.OK, response.Code); + } + + [Fact] + public async Task TheRequestGoesToTheConfiguredServer() + { + var (handler, http) = Create(); + http.Respond(HttpStatusCode.OK, ""); + + await handler.GetHttp("httpapi?request=anime&aid=1"); + + // Asserted whole, including the host: checking only the relative part would pass even if + // the handler ignored the configured server entirely. + var request = Assert.Single(http.Requests); + Assert.Equal("http://anidb.invalid/httpapi?request=anime&aid=1", request.RequestUri!.ToString()); + } + + [Fact] + public async Task AnEmptyBodyIsNotMistakenForABan() + { + var (handler, http) = Create(); + http.Respond(HttpStatusCode.OK, string.Empty); + + var response = await handler.GetHttp("httpapi"); + + Assert.Equal(string.Empty, response.Response); + Assert.False(handler.IsBanned); + } + + [Fact] + public async Task AnOrdinaryBodyMentioningBannedElsewhereIsNotABan() + { + var (handler, http) = Create(); + // The marker is the element `>banned<`, not the word appearing in content. + http.Respond(HttpStatusCode.OK, "The Banned Ones"); + + await handler.GetHttp("httpapi"); + + Assert.False(handler.IsBanned); + } + + #endregion + + #region Ban handling + + [Fact] + public async Task ABannedResponseThrows() + { + var (handler, http) = Create(); + http.Respond(HttpStatusCode.OK, BannedBody); + + await Assert.ThrowsAsync(() => handler.GetHttp("httpapi")); + } + + [Fact] + public async Task ABannedResponseMarksTheConnectionBanned() + { + var (handler, http) = Create(); + http.Respond(HttpStatusCode.OK, BannedBody); + + await Assert.ThrowsAsync(() => handler.GetHttp("httpapi")); + + Assert.True(handler.IsBanned); + Assert.NotNull(handler.BanTime); + } + + [Fact] + public async Task TheBanMarkerIsMatchedRegardlessOfCase() + { + var (handler, http) = Create(); + http.Respond(HttpStatusCode.OK, "BANNED"); + + await Assert.ThrowsAsync(() => handler.GetHttp("httpapi")); + Assert.True(handler.IsBanned); + } + + [Fact] + public async Task AFurtherCallWhileBannedNeverReachesTheNetwork() + { + var (handler, http) = Create(); + http.Respond(HttpStatusCode.OK, BannedBody); + await Assert.ThrowsAsync(() => handler.GetHttp("httpapi")); + + await Assert.ThrowsAsync(() => handler.GetHttp("httpapi")); + + // Talking to AniDB while banned is what extends the ban, so the second call must not go out. + Assert.Equal(1, http.CallCount); + } + + [Fact] + public async Task ABannedCallCanBeForcedThrough() + { + var (handler, http) = Create(); + http.Respond(HttpStatusCode.OK, BannedBody); + await Assert.ThrowsAsync(() => handler.GetHttp("httpapi")); + + http.Respond(HttpStatusCode.OK, ""); + var response = await handler.GetHttp("httpapi", force: true); + + Assert.Equal("", response.Response); + Assert.Equal(2, http.CallCount); + } + + [Fact] + public async Task TheBanExpiryIsTwelveHoursAfterItStarted() + { + var (handler, http) = Create(); + http.Respond(HttpStatusCode.OK, BannedBody); + + var exception = await Assert.ThrowsAsync(() => handler.GetHttp("httpapi")); + + Assert.Equal(12, handler.BanTimerResetLength); + Assert.Equal(handler.BanTime!.Value.AddHours(12), exception.BanExpires); + Assert.Equal(UpdateType.HTTPBan, exception.BanType); + } + + #endregion + + #region Failures + + [Fact] + public async Task AServerErrorIsSurfacedRatherThanSwallowed() + { + var (handler, http) = Create(); + http.Respond(HttpStatusCode.InternalServerError, "boom"); + + await Assert.ThrowsAsync(() => handler.GetHttp("httpapi")); + Assert.False(handler.IsBanned); + } + + [Fact] + public async Task ATransportFailureIsSurfaced() + { + var (handler, http) = Create(); + http.Throw(new HttpRequestException("no route to host")); + + await Assert.ThrowsAsync(() => handler.GetHttp("httpapi")); + } + + [Fact] + public async Task AFailedCallLeavesTheConnectionUsable() + { + var (handler, http) = Create(); + http.Throw(new HttpRequestException("transient")); + await Assert.ThrowsAsync(() => handler.GetHttp("httpapi")); + + http.Respond(HttpStatusCode.OK, ""); + var response = await handler.GetHttp("httpapi"); + + Assert.Equal("", response.Response); + } + + #endregion +} diff --git a/Shoko.Tests/Providers/AniDB/Connection/AniDBUdpConnectionTests.cs b/Shoko.Tests/Providers/AniDB/Connection/AniDBUdpConnectionTests.cs new file mode 100644 index 0000000000..b65acaa264 --- /dev/null +++ b/Shoko.Tests/Providers/AniDB/Connection/AniDBUdpConnectionTests.cs @@ -0,0 +1,257 @@ +using System; +using System.Linq; +using System.Net.Sockets; +using System.Text; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Shoko.Abstractions.Connectivity.Services; +using Shoko.Abstractions.Connectivity.Enums; +using Shoko.Server.Providers.AniDB; +using Shoko.Server.Providers.AniDB.Interfaces; +using Shoko.Server.Providers.AniDB.UDP; +using Shoko.Server.Settings; +using Shoko.Tests.Infrastructure; +using Xunit; + +namespace Shoko.Tests.Providers.AniDB.Connection; + +/// +/// Covers — the encoding, ban detection and gating around +/// every AniDB UDP call. +/// +/// +/// The socket is supplied through and replaced here by a +/// stub that replays canned payloads, so nothing binds a port or sends a datagram. The interesting +/// behaviour is not the socket anyway: it is that an all-zero reply is treated as a ban, and that a +/// banned connection stops talking. +/// +public class AniDBUdpConnectionTests +{ + /// + /// Reserved by RFC 2606 and guaranteed never to resolve. The socket is a stub and never dials + /// anything, but naming an unroutable host makes that impossible to get wrong by accident. + /// + private const string UnroutableHost = "anidb.invalid"; + + private const string Username = "tester"; + private const string Password = "secret"; + + private sealed class Harness + { + public AniDBUDPConnectionHandler Handler { get; } + + public AniDBTestDoubles.StubSocketHandler Socket { get; } = new(); + + public ServerSettings Settings { get; } = new(); + + public Harness(NetworkAvailability availability = NetworkAvailability.Internet, bool socketConnects = true) + { + Socket.IsConnected = socketConnects; + + var settingsProvider = new Mock(); + settingsProvider.Setup(p => p.GetSettings(It.IsAny())).Returns(Settings); + + var connectivity = new Mock(); + connectivity.SetupGet(c => c.NetworkAvailability).Returns(availability); + + Handler = new AniDBUDPConnectionHandler( + requestFactory: null!, + NullLoggerFactory.Instance, + settingsProvider.Object, + AniDBTestDoubles.UdpRateLimiter(), + connectivity.Object, + AniDBTestDoubles.SocketHandlerFactory(Socket)); + } + + public bool Init() => Handler.Init(Username, Password, UnroutableHost, 9000, 4556); + } + + /// + /// A plain reply. Without a byte order mark the handler decodes as ASCII, which is what AniDB + /// sends for ordinary status responses. + /// + private static byte[] Reply(string text) => Encoding.ASCII.GetBytes(text); + + /// A reply carrying a UTF-16 big-endian byte order mark, as used for text payloads. + private static byte[] UnicodeReply(string text) + => [0xFE, 0xFF, .. Encoding.BigEndianUnicode.GetBytes(text)]; + + #region Initialisation + + [Fact] + public void InitBuildsTheSocketAndRecordsWhetherItConnected() + { + var harness = new Harness(); + + Assert.True(harness.Init()); + Assert.True(harness.Socket.ConnectionAttempted); + Assert.True(harness.Handler.IsNetworkAvailable); + } + + [Fact] + public void InitRecordsAFailureToConnect() + { + var harness = new Harness(socketConnects: false); + + harness.Init(); + + Assert.False(harness.Handler.IsNetworkAvailable); + } + + [Theory] + [InlineData(null, Password)] + [InlineData("", Password)] + [InlineData(Username, null)] + [InlineData(Username, "")] + public void InitRefusesIncompleteCredentials(string? username, string? password) + { + var harness = new Harness(); + + Assert.False(harness.Handler.Init(username, password, UnroutableHost, 9000, 4556)); + Assert.False(harness.Socket.ConnectionAttempted); + } + + [Fact] + public void SendingBeforeInitIsRefused() + { + var harness = new Harness(); + + // No socket has been built, so there is nothing to send through. + Assert.Throws(() => harness.Handler.SendDirectly("PING")); + } + + #endregion + + #region Sending and receiving + + [Fact] + public void AReplyIsDecodedAndReturned() + { + var harness = new Harness(); + harness.Init(); + harness.Socket.Respond(Reply("300 PONG")); + + Assert.Equal("300 PONG", harness.Handler.SendDirectly("PING", isPing: true)); + } + + [Fact] + public void TheCommandIsSentAsUnicodeByDefault() + { + var harness = new Harness(); + harness.Init(); + harness.Socket.Respond(Reply("300 PONG")); + + harness.Handler.SendDirectly("PING"); + + var sent = Assert.Single(harness.Socket.Sent); + Assert.Equal(new UnicodeEncoding(true, false).GetBytes("PING"), sent); + } + + [Fact] + public void TheCommandCanBeSentAsAscii() + { + var harness = new Harness(); + harness.Init(); + harness.Socket.Respond(Reply("300 PONG")); + + harness.Handler.SendDirectly("PING", needsUnicode: false); + + Assert.Equal(Encoding.ASCII.GetBytes("PING"), Assert.Single(harness.Socket.Sent)); + } + + [Fact] + public void AByteOrderMarkIsStrippedFromTheReply() + { + var harness = new Harness(); + harness.Init(); + harness.Socket.Respond(UnicodeReply("300 PONG")); + + // The mark is a decoding artefact, not part of the response. + Assert.Equal("300 PONG", harness.Handler.SendDirectly("PING")); + } + + #endregion + + #region Ban handling + + [Fact] + public void AnAllZeroReplyIsTreatedAsABan() + { + var harness = new Harness(); + harness.Init(); + harness.Socket.Respond(new byte[16]); + + // A silent socket cannot be told apart from a ban, and assuming the worse is what stops the + // server digging the hole deeper. + var exception = Assert.Throws(() => harness.Handler.SendDirectly("PING")); + + Assert.Equal(UpdateType.UDPBan, exception.BanType); + Assert.True(harness.Handler.IsBanned); + } + + [Fact] + public void TheUdpBanExpiryIsAnHourAndAHalfAfterItStarted() + { + var harness = new Harness(); + harness.Init(); + harness.Socket.Respond(new byte[16]); + + var exception = Assert.Throws(() => harness.Handler.SendDirectly("PING")); + + Assert.Equal(1.5D, harness.Handler.BanTimerResetLength); + Assert.Equal(harness.Handler.BanTime!.Value.AddHours(1.5D), exception.BanExpires); + } + + [Fact] + public void SendRefusesToTalkWhileBanned() + { + var harness = new Harness(); + harness.Init(); + harness.Socket.Respond(new byte[16]); + Assert.Throws(() => harness.Handler.SendDirectly("PING")); + + Assert.Throws(() => harness.Handler.Send("PING")); + + // Only the first call reached the socket; the ban check short-circuits the rest. + Assert.Single(harness.Socket.Sent); + } + + [Fact] + public void ANonZeroReplyIsNotMistakenForABan() + { + var harness = new Harness(); + harness.Init(); + harness.Socket.Respond(Reply("500 LOGIN FAILED")); + + Assert.Equal("500 LOGIN FAILED", harness.Handler.SendDirectly("AUTH")); + Assert.False(harness.Handler.IsBanned); + } + + #endregion + + #region Connectivity + + [Fact] + public void NothingIsSentWithoutInternet() + { + var harness = new Harness(availability: NetworkAvailability.NoInterfaces); + harness.Init(); + + Assert.Throws(() => harness.Handler.SendDirectly("PING")); + + // The request is abandoned before it reaches the socket rather than timing out on it. + Assert.Empty(harness.Socket.Sent); + } + + [Fact] + public void APartialInternetConnectionIsGoodEnoughToTry() + { + var harness = new Harness(availability: NetworkAvailability.PartialInternet); + harness.Init(); + harness.Socket.Respond(Reply("300 PONG")); + + Assert.Equal("300 PONG", harness.Handler.SendDirectly("PING")); + } + + #endregion +} diff --git a/Shoko.Tests/Providers/AniDB/HttpAnimeParserTests.cs b/Shoko.Tests/Providers/AniDB/HttpAnimeParserTests.cs new file mode 100644 index 0000000000..67c7493ef7 --- /dev/null +++ b/Shoko.Tests/Providers/AniDB/HttpAnimeParserTests.cs @@ -0,0 +1,348 @@ +using System; +using System.Linq; +using Microsoft.Extensions.Logging.Abstractions; +using Shoko.Abstractions.Metadata; +using Shoko.Abstractions.Metadata.Enums; +using Shoko.Server.Providers.AniDB.HTTP; + +using AnimeType = Shoko.Server.Providers.AniDB.AnimeType; +using Xunit; + +namespace Shoko.Tests.Providers.AniDB; + +/// +/// Covers , which turns AniDB's anime XML into the records the rest of +/// the server is built on. It is pure — XML in, objects out — and had no tests, despite AniDB being +/// one of the most frequently fixed areas in the codebase. +/// +public class HttpAnimeParserTests +{ + private static HttpAnimeParser Parser() => new(NullLogger.Instance); + + /// Builds an anime document with sensible defaults, overriding the parts under test. + private static string Xml( + string id = "1", + string type = "TV Series", + string episodeCount = "12", + string startDate = "2020-01-05", + string endDate = "2020-03-22", + string? restricted = "false", + string titles = "Main Title", + string description = "A description.", + string extra = "") + => $""" + + {type} + {episodeCount} + {startDate} + {endDate} + https://example.invalid/show + 1234.jpg + {description} + {titles} + {extra} + + """; + + private static ResponseGetAnime ParseOrFail(string xml) + => Parser().Parse(1, xml) ?? throw new InvalidOperationException("Parse returned null."); + + #region Rejecting unusable documents + + [Fact] + public void ADocumentWithoutAnAnimeIdIsRejected() + => Assert.Null(Parser().Parse(1, "X")); + + [Fact] + public void ADocumentWithoutAMainTitleIsRejected() + { + // Everything downstream keys off the main title, so a document lacking one is unusable. + var xml = Xml(titles: "Official Only"); + + Assert.Null(Parser().Parse(1, xml)); + } + + [Fact] + public void AnEmptyMainTitleIsTreatedAsMissing() + => Assert.Null(Parser().Parse(1, Xml(titles: " "))); + + #endregion + + #region Anime details + + [Fact] + public void TheAnimeIdComesFromTheCallerNotTheDocument() + { + var response = Parser().Parse(99, Xml(id: "1"))!; + + Assert.Equal(99, response.Anime.AnimeID); + } + + [Theory] + [InlineData("Movie", AnimeType.Movie)] + [InlineData("OVA", AnimeType.OVA)] + [InlineData("TV Series", AnimeType.TVSeries)] + [InlineData("TV Special", AnimeType.TVSpecial)] + [InlineData("Web", AnimeType.Web)] + [InlineData("Music Video", AnimeType.MusicVideo)] + [InlineData("Other", AnimeType.Other)] + public void TheAnimeTypeIsMapped(string type, AnimeType expected) + => Assert.Equal(expected, ParseOrFail(Xml(type: type)).Anime.AnimeType); + + [Fact] + public void TheAnimeTypeIsMatchedWithoutRegardToCase() + => Assert.Equal(AnimeType.TVSeries, ParseOrFail(Xml(type: "tv series")).Anime.AnimeType); + + [Fact] + public void AnUnrecognisedAnimeTypeBecomesUnknown() + => Assert.Equal(AnimeType.Unknown, ParseOrFail(Xml(type: "Interpretive Dance")).Anime.AnimeType); + + [Fact] + public void TheEpisodeCountIsRead() + => Assert.Equal(12, ParseOrFail(Xml(episodeCount: "12")).Anime.EpisodeCount); + + [Fact] + public void AnUnreadableEpisodeCountBecomesZero() + => Assert.Equal(0, ParseOrFail(Xml(episodeCount: "lots")).Anime.EpisodeCount); + + [Theory] + [InlineData("true", true)] + [InlineData("false", false)] + [InlineData("nonsense", false)] + public void TheRestrictedFlagIsRead(string restricted, bool expected) + => Assert.Equal(expected, ParseOrFail(Xml(restricted: restricted)).Anime.IsRestricted); + + [Fact] + public void BackticksInTheDescriptionBecomeApostrophes() + { + // AniDB writes apostrophes as backticks throughout its API. + var response = ParseOrFail(Xml(description: "It`s a description.")); + + Assert.Equal("It's a description.", response.Anime.Description); + } + + #endregion + + #region Dates + + [Fact] + public void TheAirAndEndDatesAreRead() + { + var anime = ParseOrFail(Xml(startDate: "2020-01-05", endDate: "2020-03-22")).Anime; + + Assert.Equal(new PartialDateOnly(2020, 1, 5), anime.AirDate); + Assert.Equal(new PartialDateOnly(2020, 3, 22), anime.EndDate); + Assert.Equal(2020, anime.BeginYear); + Assert.Equal(2020, anime.EndYear); + } + + [Fact] + public void TheUnixEpochIsTreatedAsNoDate() + { + // AniDB uses 1970-01-01 as its "unknown" sentinel; taking it literally would place shows in + // 1970 and break every year filter and season grouping. + var anime = ParseOrFail(Xml(startDate: "1970-01-01", endDate: "1970-01-01")).Anime; + + Assert.Null(anime.AirDate); + Assert.Null(anime.EndDate); + Assert.Equal(0, anime.BeginYear); + Assert.Equal(0, anime.EndYear); + } + + [Fact] + public void AYearOnlyDateIsKeptPartial() + { + var anime = ParseOrFail(Xml(startDate: "2020", endDate: "")).Anime; + + Assert.Equal(2020, anime.AirDate!.Value.Year); + Assert.Null(anime.AirDate.Value.Month); + Assert.Null(anime.EndDate); + } + + [Fact] + public void AStillAiringShowHasNoEndDate() + { + var anime = ParseOrFail(Xml(endDate: "")).Anime; + + Assert.NotNull(anime.AirDate); + Assert.Null(anime.EndDate); + Assert.Equal(0, anime.EndYear); + } + + #endregion + + #region Titles + + [Fact] + public void TitlesAreReadWithTheirTypeAndLanguage() + { + var response = ParseOrFail(Xml(titles: """ + Romaji Title + English Title + Japanese Synonym + """)); + + Assert.Equal(3, response.Titles.Count); + var main = Assert.Single(response.Titles, t => t.TitleType == TitleType.Main); + Assert.Equal("Romaji Title", main.Title); + Assert.Equal(TitleLanguage.Romaji, main.Language); + Assert.Equal(TitleLanguage.English, Assert.Single(response.Titles, t => t.TitleType == TitleType.Official).Language); + Assert.Equal(TitleLanguage.Japanese, Assert.Single(response.Titles, t => t.TitleType == TitleType.Synonym).Language); + } + + [Fact] + public void TheMainTitleIsCopiedOntoTheAnime() + => Assert.Equal("Main Title", ParseOrFail(Xml()).Anime.MainTitle); + + [Fact] + public void BackticksInTitlesBecomeApostrophes() + { + var response = ParseOrFail(Xml(titles: "It`s Here")); + + Assert.Equal("It's Here", response.Anime.MainTitle); + } + + #endregion + + #region Episodes + + private static string EpisodeXml(string epno, string id = "1001", string extra = "") + => $""" + + + {epno} + 24 + 2020-01-05 + Episode Title + {extra} + + + """; + + [Theory] + [InlineData("1", 1)] + [InlineData("12", 12)] + [InlineData("S1", 1)] + [InlineData("C2", 2)] + [InlineData("T3", 3)] + [InlineData("P4", 4)] + [InlineData("O5", 5)] + public void TheEpisodeNumberIsReadWithoutItsTypePrefix(string epno, int expected) + { + var episode = Assert.Single(ParseOrFail(Xml(extra: EpisodeXml(epno))).Episodes); + + Assert.Equal(expected, episode.EpisodeNumber); + } + + [Theory] + [InlineData("1", "Episode")] + [InlineData("S1", "Special")] + [InlineData("C1", "Credits")] + [InlineData("T1", "Trailer")] + [InlineData("P1", "Parody")] + [InlineData("O1", "Other")] + public void TheEpisodeTypeComesFromThePrefix(string epno, string expected) + { + var episode = Assert.Single(ParseOrFail(Xml(extra: EpisodeXml(epno))).Episodes); + + Assert.Equal(expected, episode.EpisodeType.ToString()); + } + + [Fact] + public void ADoubleEpisodeTakesTheFirstNumber() + { + // AniDB writes a combined release as "1-2"; the first number is used as its number. + var episode = Assert.Single(ParseOrFail(Xml(extra: EpisodeXml("1-2"))).Episodes); + + Assert.Equal(1, episode.EpisodeNumber); + Assert.Equal(EpisodeType.Episode, episode.EpisodeType); + } + + [Fact] + public void TheEpisodeLengthIsConvertedFromMinutesToSeconds() + => Assert.Equal(24 * 60, Assert.Single(ParseOrFail(Xml(extra: EpisodeXml("1"))).Episodes).LengthSeconds); + + [Fact] + public void TheEpisodeIdAndAnimeIdAreRecorded() + { + var episode = Assert.Single(Parser().Parse(77, Xml(extra: EpisodeXml("1", id: "5150")))!.Episodes); + + Assert.Equal(5150, episode.EpisodeID); + Assert.Equal(77, episode.AnimeID); + } + + [Fact] + public void AMissingUpdateDateFallsBackToTheUnixEpoch() + { + var xml = Xml(extra: """ + + + 1 + 24 + + + """); + + Assert.Equal(DateTime.UnixEpoch, Assert.Single(ParseOrFail(xml).Episodes).LastUpdated); + } + + [Fact] + public void EpisodeTitlesInAnUnknownLanguageAreDropped() + { + var xml = Xml(extra: EpisodeXml("1", extra: "Unusable")); + + var episode = Assert.Single(ParseOrFail(xml).Episodes); + Assert.All(episode.Titles, title => Assert.NotEqual(TitleLanguage.Unknown, title.Language)); + Assert.Equal("Episode Title", Assert.Single(episode.Titles).Title); + } + + [Fact] + public void AnAnimeWithNoEpisodesParsesToAnEmptyList() + => Assert.Empty(ParseOrFail(Xml()).Episodes); + + #endregion + + #region Related collections + + [Fact] + public void RelationsAreRead() + { + var xml = Xml(extra: """ + + Next Season + + """); + + var relation = Assert.Single(ParseOrFail(xml).Relations); + Assert.Equal(200, relation.RelatedAnimeID); + } + + [Fact] + public void SimilarAnimeAreRead() + { + var xml = Xml(extra: """ + + Something Alike + + """); + + var similar = Assert.Single(ParseOrFail(xml).Similar); + Assert.Equal(300, similar.SimilarAnimeID); + } + + [Fact] + public void MissingCollectionsComeBackEmptyRatherThanNull() + { + var response = ParseOrFail(Xml()); + + Assert.Empty(response.Episodes); + Assert.Empty(response.Tags); + Assert.Empty(response.Characters); + Assert.Empty(response.Staff); + Assert.Empty(response.Relations); + Assert.Empty(response.Similar); + Assert.Empty(response.Resources); + } + + #endregion +} diff --git a/Shoko.Tests/Providers/TMDB/TmdbRateLimiterTests.cs b/Shoko.Tests/Providers/TMDB/TmdbRateLimiterTests.cs index a8f4cc5176..6b16b5d810 100644 --- a/Shoko.Tests/Providers/TMDB/TmdbRateLimiterTests.cs +++ b/Shoko.Tests/Providers/TMDB/TmdbRateLimiterTests.cs @@ -1,5 +1,4 @@ using System; -using System.Diagnostics; using System.Threading.Tasks; using Microsoft.Extensions.Logging.Abstractions; using Moq; @@ -14,16 +13,15 @@ namespace Shoko.Tests.Providers.TMDB; public class TmdbRateLimiterTests { [Fact] - public async Task CallsWithinWindow_ProceedImmediately() + public async Task CallsWithinWindow_ConsumeTheirSlots() { using var limiter = CreateRateLimiter(maxRequests: 3, windowMs: 500); - var sw = Stopwatch.StartNew(); for (var i = 0; i < 3; i++) await limiter.EnsureRateAsync(() => Task.FromResult(0)); - Assert.True(sw.Elapsed < TimeSpan.FromMilliseconds(200), - $"Expected < 200ms for 3 calls within a 3-request window, got {sw.Elapsed.TotalMilliseconds:F0}ms"); + Assert.Equal(0, limiter.RemainingInWindow); + Assert.Equal(3, limiter.CallsInWindow); } [Fact] @@ -34,29 +32,33 @@ public async Task ExtraCall_WaitsForWindowSlot() await limiter.EnsureRateAsync(() => Task.FromResult(0)); await limiter.EnsureRateAsync(() => Task.FromResult(0)); - var sw = Stopwatch.StartNew(); - await limiter.EnsureRateAsync(() => Task.FromResult(0)); + // That a caller with no permit left is made to wait is the framework's job; ours is to have + // handed it the right window, which is what an exhausted capacity shows. + Assert.Equal(0, limiter.RemainingInWindow); - Assert.True(sw.Elapsed >= TimeSpan.FromMilliseconds(200), - $"Expected >= 200ms wait for 3rd call with 2-request window, got {sw.Elapsed.TotalMilliseconds:F0}ms"); + await limiter.EnsureRateAsync(() => Task.FromResult(0)); } [Fact] public async Task SlotsExpireAfterWindow_AllowsNewCalls() { - using var limiter = CreateRateLimiter(maxRequests: 2, windowMs: 200); + using var limiter = CreateRateLimiter(maxRequests: 2, windowMs: 500); await limiter.EnsureRateAsync(() => Task.FromResult(0)); await limiter.EnsureRateAsync(() => Task.FromResult(0)); + Assert.Equal(0, limiter.RemainingInWindow); - await Task.Delay(300, TestContext.Current.CancellationToken); + // Waiting on the capacity rather than timing it. A loaded runner can starve the replenishment + // timer for several seconds, which measured 1933ms against a 500ms bound and says nothing + // about whether slots expire. + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(30); + while (limiter.RemainingInWindow < 2 && DateTime.UtcNow < deadline) + await Task.Delay(50, TestContext.Current.CancellationToken); + + Assert.Equal(2, limiter.RemainingInWindow); - var sw = Stopwatch.StartNew(); await limiter.EnsureRateAsync(() => Task.FromResult(0)); await limiter.EnsureRateAsync(() => Task.FromResult(0)); - - Assert.True(sw.Elapsed < TimeSpan.FromMilliseconds(200), - $"Expected < 200ms after window expiry, got {sw.Elapsed.TotalMilliseconds:F0}ms"); } [Fact] @@ -66,11 +68,10 @@ public async Task NotifyRateLimitExceeded_DelaysSubsequentCalls() limiter.NotifyRateLimitExceeded(TimeSpan.FromMilliseconds(300)); - var sw = Stopwatch.StartNew(); - await limiter.EnsureRateAsync(() => Task.FromResult(0)); + Assert.True(limiter.BackoffUntilTicks > DateTimeOffset.UtcNow.UtcTicks, + "Expected a backoff deadline in the future"); - Assert.True(sw.Elapsed >= TimeSpan.FromMilliseconds(250), - $"Expected >= 250ms backoff delay, got {sw.Elapsed.TotalMilliseconds:F0}ms"); + await limiter.EnsureRateAsync(() => Task.FromResult(0)); } [Fact] @@ -79,18 +80,17 @@ public async Task BackoffAppliesToAllConcurrentCallers_NotSequentially() using var limiter = CreateRateLimiter(maxRequests: 10, windowMs: 1000); limiter.NotifyRateLimitExceeded(TimeSpan.FromMilliseconds(300)); - var sw = Stopwatch.StartNew(); + var deadline = limiter.BackoffUntilTicks; + Assert.True(deadline > DateTimeOffset.UtcNow.UtcTicks, "Expected a backoff deadline in the future"); + await Task.WhenAll( limiter.EnsureRateAsync(() => Task.FromResult(0)), limiter.EnsureRateAsync(() => Task.FromResult(0)), limiter.EnsureRateAsync(() => Task.FromResult(0)) ); - // All three waited for the same backoff window (not 3 × 300ms). - Assert.True(sw.Elapsed >= TimeSpan.FromMilliseconds(250), - $"Expected >= 250ms, got {sw.Elapsed.TotalMilliseconds:F0}ms"); - Assert.True(sw.Elapsed < TimeSpan.FromMilliseconds(900), - $"Expected < 900ms (callers should share the wait, not queue it), got {sw.Elapsed.TotalMilliseconds:F0}ms"); + // Queueing the callers rather than sharing the pause would push the deadline out per caller. + Assert.Equal(deadline, limiter.BackoffUntilTicks); } [Fact] @@ -130,10 +130,8 @@ public async Task Notify5xxError_BelowThreshold_NoBackoff() limiter.Notify5xxError(); // Two errors — threshold not reached, no backoff applied. - var sw = Stopwatch.StartNew(); + Assert.Equal(0L, limiter.BackoffUntilTicks); await limiter.EnsureRateAsync(() => Task.FromResult(0)); - Assert.True(sw.Elapsed < TimeSpan.FromMilliseconds(200), - $"Expected no backoff delay, got {sw.Elapsed.TotalMilliseconds:F0}ms"); } [Fact] @@ -203,10 +201,11 @@ public async Task NotifySuccess_AfterPauseElapsed_ResetsRamp() // doesn't throw and that subsequent EnsureRateAsync proceeds immediately. limiter.NotifySuccess(); - var sw = Stopwatch.StartNew(); + // The 429 path leaves the elapsed deadline behind rather than zeroing it, so what matters is + // that it is in the past and nothing will wait on it. + Assert.True(limiter.BackoffUntilTicks <= DateTimeOffset.UtcNow.UtcTicks, + "Expected the backoff deadline to have elapsed"); await limiter.EnsureRateAsync(() => Task.FromResult(0)); - Assert.True(sw.Elapsed < TimeSpan.FromMilliseconds(200), - $"Expected immediate proceed after NotifySuccess reset, got {sw.Elapsed.TotalMilliseconds:F0}ms"); } [Fact] diff --git a/Shoko.Tests/Services/AnimeSeriesStatsTests.cs b/Shoko.Tests/Services/AnimeSeriesStatsTests.cs new file mode 100644 index 0000000000..e7240ce982 --- /dev/null +++ b/Shoko.Tests/Services/AnimeSeriesStatsTests.cs @@ -0,0 +1,408 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Shoko.Abstractions.Metadata; +using Shoko.Abstractions.Metadata.Enums; +using Shoko.Server.Models.AniDB; +using Shoko.Server.Models.CrossReference; +using Shoko.Server.Models.Release; + +using Shoko.Server.Models.Shoko; +using Shoko.Server.Repositories.Cached; +using Shoko.Server.Repositories.Cached.AniDB; +using Shoko.Server.Repositories.Direct; +using Shoko.Server.Databases; +using Shoko.QueueProcessor.Abstractions; +using Shoko.Server.Services; +using Shoko.Tests.Infrastructure; +using Xunit; + +namespace Shoko.Tests.Services; + +/// +/// Covers the missing-episode statistics writes onto a +/// series. These counts drive the missing-episode filters, the dashboard, and the calendar, and +/// nothing else recomputes them — a series carries whatever this last wrote. +/// +[Collection(nameof(RepoFactoryCollection))] +public class AnimeSeriesStatsTests +{ + private const int AnimeID = 100; + private const int SeriesID = 1; + + private static readonly DateTime s_aired = new(2020, 1, 1, 0, 0, 0, DateTimeKind.Unspecified); + + private static readonly DateTime s_unaired = DateTime.Now.AddYears(5); + + /// + /// One episode. null means AniDB gave no air date at all, which is a + /// different branch from an episode dated in the future. + /// + private sealed record EpisodeSpec( + int Number, + bool HasFile, + DateTime? AirsAt = null, + bool UnknownAirDate = false, + bool Hidden = false, + EpisodeType Type = EpisodeType.Episode, + int ReleaseGroupID = 0); + + private sealed class Harness : IDisposable + { + public AnimeSeriesService Service { get; } + + public AnimeSeries Series { get; } + + public Mock SeriesRepository { get; } + + private readonly RepoFactoryScope _scope; + + public Harness(IEnumerable specs, IEnumerable? groupStatuses = null, DateTime? animeEndDate = null) + { + Series = new AnimeSeries { AnimeSeriesID = SeriesID, AniDB_ID = AnimeID }; + + var anidbEpisodes = new List(); + var shokoEpisodes = new List(); + var videos = new List(); + var crossRefs = new List(); + var releaseInfos = new List(); + foreach (var spec in specs) + { + var episodeId = spec.Number + (int)spec.Type * 1000; + anidbEpisodes.Add(new AniDB_Episode + { + AniDB_EpisodeID = episodeId, + EpisodeID = episodeId, + AnimeID = AnimeID, + EpisodeNumber = spec.Number, + EpisodeType = spec.Type, + AirDate = spec.UnknownAirDate ? 0 : (int)((spec.AirsAt ?? s_aired) - new DateTime(1970, 1, 1)).TotalSeconds, + }); + shokoEpisodes.Add(new AnimeEpisode + { + AnimeEpisodeID = episodeId, + AnimeSeriesID = SeriesID, + AniDB_EpisodeID = episodeId, + IsHidden = spec.Hidden, + }); + + if (!spec.HasFile) + continue; + + var hash = $"hash-{episodeId}"; + videos.Add(new VideoLocal { VideoLocalID = episodeId, Hash = hash, FileSize = 1000 }); + if (spec.ReleaseGroupID > 0) + releaseInfos.Add(new StoredReleaseInfo + { + StoredReleaseInfoID = episodeId, + ED2K = hash, + FileSize = 1000, + GroupID = spec.ReleaseGroupID.ToString(), + GroupSource = "AniDB", + GroupName = $"Group {spec.ReleaseGroupID}", + // All four group fields are required before a release exposes a group. + GroupShortName = $"G{spec.ReleaseGroupID}", + }); + crossRefs.Add(new CrossRef_File_Episode + { + CrossRef_File_EpisodeID = episodeId, + Hash = hash, + AnimeID = AnimeID, + EpisodeID = episodeId, + Percentage = 100, + }); + } + + SeriesRepository = CachedRepo.BuildWritable(s => s.AnimeSeriesID, [Series]); + // UpdateStats persists through the three-argument overload. + SeriesRepository.Setup(r => r.Save(It.IsAny(), It.IsAny(), It.IsAny())); + + var episodeRepository = CachedRepo.Build(e => e.AnimeEpisodeID, shokoEpisodes); + + var releaseInfoRepository = CachedRepo.Build(r => r.StoredReleaseInfoID, releaseInfos); + + _scope = new RepoFactoryScope() + // VideoLocal.ReleaseGroup resolves through here while collecting the groups the + // user is currently following. + .Set(releaseInfoRepository) + .With(a => a.AniDB_AnimeID, + [new AniDB_Anime + { + AniDB_AnimeID = 1, AnimeID = AnimeID, MainTitle = "Test", AnimeType = AnimeType.TV, + AirDate = new PartialDateOnly(2020, 1, 1), + EndDate = animeEndDate is { } end ? new PartialDateOnly(end.Year, end.Month, end.Day) : null, + }]) + .With(e => e.AniDB_EpisodeID, anidbEpisodes) + .With(v => v.VideoLocalID, videos) + .With(x => x.CrossRef_File_EpisodeID, crossRefs) + .Set(episodeRepository); + + Service = new AnimeSeriesService( + NullLogger.Instance, + serviceProvider: null!, + schedulerFactory: null!, + groupService: null!, + vlUsers: null!, + videoReleaseService: null!, + userDataService: null!, + animeEpisodes: episodeRepository, + animeSeries: SeriesRepository.Object, + storedReleaseInfos: releaseInfoRepository, + anidbGroupStatuses: GroupStatuses(groupStatuses), + anidbAnimeStaff: null!, + xrefAnidbTmdbShows: null!, + xrefAnidbTmdbMovies: null!); + } + + /// + /// A direct repository, so it cannot be cache-backed; with no statuses the service treats + /// every aired episode as released, which is the common case. + /// + private static AniDB_GroupStatusRepository GroupStatuses(IEnumerable? statuses) + { + var mock = new Mock((DatabaseFactory)null!, (IQueueScheduler)null!); + mock.Setup(r => r.GetByAnimeID(It.IsAny())).Returns([.. statuses ?? []]); + return mock.Object; + } + + public AnimeSeries Update() + { + Service.UpdateStats(Series, watchedStats: false, missingEpsStats: true); + return Series; + } + + public void Dispose() => _scope.Dispose(); + } + + private static Harness Create(params EpisodeSpec[] specs) => new(specs); + + private static AniDB_GroupStatus GroupStatus( + int groupId, + Shoko.Server.Providers.AniDB.Group_CompletionStatus state = Shoko.Server.Providers.AniDB.Group_CompletionStatus.Complete, + string episodeRange = "") + => new() + { + AniDB_GroupStatusID = groupId, + AnimeID = AnimeID, + GroupID = groupId, + GroupName = $"Group {groupId}", + CompletionState = (int)state, + EpisodeRange = episodeRange, + }; + + #region Missing episode counts + + [Fact] + public void AnEpisodeWithoutAFileCountsAsMissing() + { + using var harness = Create(new EpisodeSpec(1, HasFile: false)); + + Assert.Equal(1, harness.Update().MissingEpisodeCount); + } + + [Fact] + public void AnEpisodeWithAFileDoesNotCountAsMissing() + { + using var harness = Create(new EpisodeSpec(1, HasFile: true)); + + Assert.Equal(0, harness.Update().MissingEpisodeCount); + } + + [Fact] + public void OnlyTheEpisodesWithoutFilesAreCounted() + { + using var harness = Create( + new EpisodeSpec(1, HasFile: true), + new EpisodeSpec(2, HasFile: false), + new EpisodeSpec(3, HasFile: false), + new EpisodeSpec(4, HasFile: true)); + + Assert.Equal(2, harness.Update().MissingEpisodeCount); + } + + [Fact] + public void AnEpisodeAiringInTheFutureIsNotCountedAsMissing() + { + using var harness = Create(new EpisodeSpec(1, HasFile: false, AirsAt: s_unaired)); + + // Nobody is missing an episode that has not been broadcast yet. + Assert.Equal(0, harness.Update().MissingEpisodeCount); + } + + [Fact] + public void AnEpisodeWithNoAirDateFallsBackToWhetherTheSeriesHasFinished() + { + // AniDB often has no date for an episode. The series having finished is then taken to mean + // the episode aired, so it counts as missing; a still-running series does not. + // Scoped one at a time: both install into the same RepoFactory statics. + int finished; + using (var harness = new Harness([new EpisodeSpec(1, HasFile: false, UnknownAirDate: true)], animeEndDate: s_aired)) + finished = harness.Update().MissingEpisodeCount; + + int running; + using (var harness = new Harness([new EpisodeSpec(1, HasFile: false, UnknownAirDate: true)])) + running = harness.Update().MissingEpisodeCount; + + Assert.Equal(1, finished); + Assert.Equal(0, running); + } + + [Fact] + public void AHiddenEpisodeIsCountedSeparately() + { + using var harness = Create( + new EpisodeSpec(1, HasFile: false, Hidden: true), + new EpisodeSpec(2, HasFile: false)); + + var series = harness.Update(); + + Assert.Equal(1, series.MissingEpisodeCount); + Assert.Equal(1, series.HiddenMissingEpisodeCount); + } + + [Fact] + public void OnlyRegularEpisodesAreCounted() + { + using var harness = Create( + new EpisodeSpec(1, HasFile: false), + new EpisodeSpec(1, HasFile: false, Type: EpisodeType.Special), + new EpisodeSpec(1, HasFile: false, Type: EpisodeType.Credits)); + + // Specials and credits are deliberately excluded; only regular episodes are counted. + Assert.Equal(1, harness.Update().MissingEpisodeCount); + } + + [Fact] + public void TheCountsAreRecomputedRatherThanAccumulated() + { + using var harness = Create(new EpisodeSpec(1, HasFile: false)); + + harness.Update(); + var second = harness.Update(); + + // Running twice must not double the counts. + Assert.Equal(1, second.MissingEpisodeCount); + } + + #endregion + + #region Release groups + + [Fact] + public void AnEpisodeNoGroupHasReleasedIsNotCountedAsMissing() + { + // A group list that covers only episode 1 means episode 2 does not exist to be collected + // yet, whatever AniDB says the episode count is. + using var harness = new Harness( + [new EpisodeSpec(1, HasFile: true), new EpisodeSpec(2, HasFile: false)], + [GroupStatus(7, Shoko.Server.Providers.AniDB.Group_CompletionStatus.Ongoing, episodeRange: "1")]); + + Assert.Equal(0, harness.Update().MissingEpisodeCount); + } + + [Fact] + public void AnEpisodeReleasedByAGroupTheUserCollectsCountsTowardsTheGroupTotal() + { + // The user holds episode 1 from group 7, so group 7 is one they collect; episode 2 is + // released by that same group and missing. + using var harness = new Harness( + [new EpisodeSpec(1, HasFile: true, ReleaseGroupID: 7), new EpisodeSpec(2, HasFile: false)], + [GroupStatus(7)]); + + var series = harness.Update(); + + Assert.Equal(1, series.MissingEpisodeCount); + Assert.Equal(1, series.MissingEpisodeCountGroups); + } + + [Fact] + public void AnEpisodeOnlyReleasedByAGroupTheUserDoesNotCollectIsExcludedFromTheGroupTotal() + { + // Still missing outright, but not from a group the user follows. + using var harness = new Harness( + [new EpisodeSpec(1, HasFile: true, ReleaseGroupID: 7), new EpisodeSpec(2, HasFile: false)], + [GroupStatus(9)]); + + var series = harness.Update(); + + Assert.Equal(1, series.MissingEpisodeCount); + Assert.Equal(0, series.MissingEpisodeCountGroups); + } + + [Fact] + public void WithNoGroupStatusesAtAllEveryAiredEpisodeCounts() + { + // The group list is only populated once the UDP command has run; until then nothing can be + // ruled out, so an aired episode counts as missing. + using var harness = Create(new EpisodeSpec(1, HasFile: false)); + + Assert.Equal(1, harness.Update().MissingEpisodeCount); + } + + #endregion + + #region Derived values + + [Fact(Skip = "Possible bug - Needs investigation")] + public void TheLatestLocalEpisodeNumberFollowsTheHighestHeldEpisode() + { + using var harness = Create( + new EpisodeSpec(1, HasFile: true), + new EpisodeSpec(2, HasFile: true), + new EpisodeSpec(3, HasFile: false)); + + Assert.Equal(2, harness.Update().LatestLocalEpisodeNumber); + } + + [Fact] + public void TheLatestLocalEpisodeNumberIsZeroWhenNothingIsHeld() + { + using var harness = Create(new EpisodeSpec(1, HasFile: false)); + + Assert.Equal(0, harness.Update().LatestLocalEpisodeNumber); + } + + [Fact] + public void TheLatestAirDateIsTakenFromTheAiredEpisodes() + { + using var harness = Create(new EpisodeSpec(1, HasFile: false)); + + Assert.Equal(s_aired, harness.Update().LatestEpisodeAirDate); + } + + [Fact] + public void NoAirDateIsRecordedWhenNothingHasAired() + { + using var harness = Create(new EpisodeSpec(1, HasFile: false, AirsAt: s_unaired)); + + Assert.Null(harness.Update().LatestEpisodeAirDate); + } + + #endregion + + #region Persistence + + [Fact] + public void TheUpdatedSeriesIsPersisted() + { + using var harness = Create(new EpisodeSpec(1, HasFile: false)); + + harness.Update(); + + harness.SeriesRepository.Verify(r => r.Save(harness.Series, false, It.IsAny()), Times.Once); + } + + [Fact] + public void ANullSeriesIsIgnored() + { + using var harness = Create(new EpisodeSpec(1, HasFile: false)); + + harness.Service.UpdateStats(null, watchedStats: false, missingEpsStats: true); + + harness.SeriesRepository.Verify(r => r.Save(It.IsAny(), It.IsAny(), It.IsAny()), Times.Never); + } + + #endregion +} diff --git a/Shoko.Tests/Services/EpisodeListTests.cs b/Shoko.Tests/Services/EpisodeListTests.cs new file mode 100644 index 0000000000..9705ff8552 --- /dev/null +++ b/Shoko.Tests/Services/EpisodeListTests.cs @@ -0,0 +1,244 @@ +using System.Linq; +using Shoko.Abstractions.Metadata.Enums; +using Shoko.Server.Models.Shoko; +using Shoko.Server.Services; +using Xunit; + +namespace Shoko.Tests.Services; + +/// +/// Covers , which decides whether a multi-part OVA or +/// movie counts as "available". Every part of a split release has to be present before the episode +/// is considered held, and the parts are matched together purely by their titles — so a change to +/// the title normalisation silently changes a user's missing-episode counts. +/// +public class EpisodeListTests +{ + private static AnimeEpisode Episode(string title, bool hidden = false) + => new() { EpisodeNameOverride = title, IsHidden = hidden }; + + private static AnimeSeriesService.EpisodeList List(AnimeType type) => new(type); + + #region Series types that are never part-matched + + [Theory] + [InlineData(AnimeType.TV)] + [InlineData(AnimeType.Web)] + [InlineData(AnimeType.TVSpecial)] + [InlineData(AnimeType.Other)] + public void NonOvaTypes_KeepEveryEpisodeSeparate(AnimeType type) + { + var list = List(type); + + list.Add(Episode("part 1 of 2"), available: true); + list.Add(Episode("part 2 of 2"), available: true); + + // Part matching is deliberately limited to OVA/Movie, so these stay two distinct entries. + Assert.Equal(2, list.Count); + Assert.All(list, group => Assert.Equal(string.Empty, group.Single().Match)); + } + + [Fact] + public void NonOvaTypes_AreAvailableWhenTheFileIsPresent() + { + var list = List(AnimeType.TV); + + list.Add(Episode("Episode 1"), available: true); + + Assert.True(list.Single().Available); + } + + [Fact] + public void NonOvaTypes_AreNotAvailableWhenTheFileIsMissing() + { + var list = List(AnimeType.TV); + + list.Add(Episode("Episode 1"), available: false); + + Assert.False(list.Single().Available); + } + + #endregion + + #region Part detection + + [Theory] + [InlineData(AnimeType.OVA)] + [InlineData(AnimeType.Movie)] + public void PartTitles_AreGroupedTogetherByTheirRemainingName(AnimeType type) + { + var list = List(type); + + list.Add(Episode("Some Movie part 1 of 2"), available: true); + list.Add(Episode("Some Movie part 2 of 2"), available: true); + + Assert.Single(list); + Assert.Equal(2, list.Single().Count); + Assert.All(list.Single(), part => Assert.Equal("Some Movie", part.Match)); + } + + [Fact] + public void PartTitles_RecordThePartCount() + { + var list = List(AnimeType.OVA); + + list.Add(Episode("Some Movie part 1 of 3"), available: true); + + var part = list.Single().Single(); + Assert.Equal(3, part.PartCount); + Assert.Equal(AnimeSeriesService.EpisodeList.StatEpisodes.StatEpisode.EpType.Part, part.EpisodeType); + } + + [Fact] + public void PartTitles_StripPunctuationWhenMatching() + { + var list = List(AnimeType.OVA); + + list.Add(Episode("Some: Movie! part 1 of 2"), available: true); + list.Add(Episode("Some Movie part 2 of 2"), available: true); + + // Symbols are removed and runs of whitespace collapsed, so both titles reduce to the same key. + Assert.Single(list); + } + + [Fact] + public void PartTitles_WithGenericNamesCollapseToTheEmptyKey() + { + var list = List(AnimeType.OVA); + + list.Add(Episode("complete movie part 1 of 2"), available: true); + + Assert.Equal(string.Empty, list.Single().Single().Match); + } + + [Fact] + public void DifferentTitles_AreKeptInSeparateGroups() + { + var list = List(AnimeType.OVA); + + list.Add(Episode("First Movie part 1 of 2"), available: true); + list.Add(Episode("Second Movie part 1 of 2"), available: true); + + Assert.Equal(2, list.Count); + } + + #endregion + + #region Whole-episode titles + + [Theory] + [InlineData("complete movie")] + [InlineData("movie")] + [InlineData("ova")] + public void GenericWholeEpisodeTitles_CollapseToTheEmptyKey(string title) + { + var list = List(AnimeType.OVA); + + list.Add(Episode(title), available: true); + + var episode = list.Single().Single(); + Assert.Equal(string.Empty, episode.Match); + Assert.Equal(AnimeSeriesService.EpisodeList.StatEpisodes.StatEpisode.EpType.Complete, episode.EpisodeType); + } + + [Fact] + public void GenericWholeEpisodeTitles_GroupWithEachOther() + { + var list = List(AnimeType.OVA); + + list.Add(Episode("complete movie"), available: true); + list.Add(Episode("movie"), available: false); + + Assert.Single(list); + } + + [Fact] + public void NamedWholeEpisodes_KeepTheirNormalisedTitleAsTheKey() + { + var list = List(AnimeType.OVA); + + list.Add(Episode("Some: Movie!"), available: true); + + Assert.Equal("Some Movie", list.Single().Single().Match); + } + + #endregion + + #region Availability + + [Fact] + public void AllPartsPresent_MakesTheEpisodeAvailable() + { + var list = List(AnimeType.OVA); + + list.Add(Episode("Some Movie part 1 of 2"), available: true); + list.Add(Episode("Some Movie part 2 of 2"), available: true); + + Assert.True(list.Single().Available); + } + + [Fact] + public void AMissingPart_LeavesTheEpisodeUnavailable() + { + var list = List(AnimeType.OVA); + + list.Add(Episode("Some Movie part 1 of 2"), available: true); + list.Add(Episode("Some Movie part 2 of 2"), available: false); + + Assert.False(list.Single().Available); + } + + [Fact] + public void AThreePartEpisodeNeedsEveryPart() + { + var list = List(AnimeType.OVA); + + list.Add(Episode("Some Movie part 1 of 3"), available: true); + list.Add(Episode("Some Movie part 2 of 3"), available: true); + + Assert.False(list.Single().Available); + + list.Add(Episode("Some Movie part 3 of 3"), available: true); + + Assert.True(list.Single().Available); + } + + [Fact] + public void ACompleteReleaseMakesTheEpisodeAvailableEvenWithMissingParts() + { + var list = List(AnimeType.OVA); + + list.Add(Episode("Some Movie part 1 of 2"), available: false); + list.Add(Episode("Some Movie"), available: true); + + // A single complete file covers the episode regardless of the part releases around it. + Assert.True(list.Single().Available); + } + + #endregion + + #region Hidden + + [Fact] + public void AGroupIsHiddenWhenAnyOfItsEpisodesIsHidden() + { + var list = List(AnimeType.OVA); + + list.Add(Episode("Some Movie part 1 of 2", hidden: false), available: true); + list.Add(Episode("Some Movie part 2 of 2", hidden: true), available: true); + + Assert.True(list.Single().Hidden); + } + + [Fact] + public void AGroupIsNotHiddenWhenNoEpisodeIsHidden() + { + var list = List(AnimeType.OVA); + + list.Add(Episode("Some Movie"), available: true); + + Assert.False(list.Single().Hidden); + } + + #endregion +} diff --git a/Shoko.Tests/Services/PlaylistParsingTests.cs b/Shoko.Tests/Services/PlaylistParsingTests.cs new file mode 100644 index 0000000000..c5c44d8358 --- /dev/null +++ b/Shoko.Tests/Services/PlaylistParsingTests.cs @@ -0,0 +1,152 @@ +using System.Linq; +using Microsoft.AspNetCore.Mvc.ModelBinding; +using Shoko.Server.Models.Shoko; +using Shoko.Server.Repositories.Cached; +using Shoko.Server.Services; +using Shoko.Tests.Infrastructure; +using Xunit; + +namespace Shoko.Tests.Services; + +/// +/// Covers the playlist DSL parsed by . It is +/// user-supplied text arriving from the v3 API, so how it rejects bad input matters as much as how +/// it accepts good input. +/// +/// +/// Only the paths that reject an entry are covered here. Once an entry parses, the service builds +/// the actual playlist, which needs the full service graph behind it — a separate exercise. +/// +[Collection(nameof(RepoFactoryCollection))] +public class PlaylistParsingTests +{ + private const int GroupID = 5; + + private sealed class Harness : System.IDisposable + { + public GeneratedPlaylistService Service { get; } + + private readonly RepoFactoryScope _scope; + + public Harness() + { + var groups = CachedRepo.Build( + g => g.AnimeGroupID, [new AnimeGroup { AnimeGroupID = GroupID, GroupName = "Group" }]); + var series = CachedRepo.Build(s => s.AnimeSeriesID, []); + var episodes = CachedRepo.Build(e => e.AnimeEpisodeID, []); + var videos = CachedRepo.Build(v => v.VideoLocalID, []); + + _scope = new RepoFactoryScope().Set(groups).Set(series).Set(episodes).Set(videos); + + Service = new GeneratedPlaylistService( + systemService: null!, imageManager: null!, contextAccessor: null!, + groupRepository: groups, animeSeriesService: null!, seriesRepository: series, + episodeRepository: episodes, videoRepository: videos, authTokensRepository: null!); + } + + public (bool Valid, string Errors, int Entries, string Keys) Parse(params string[] items) + { + var state = new ModelStateDictionary(); + var valid = Service.TryParsePlaylist(items, out var playlist, state); + var errors = string.Join(" | ", state.Values.SelectMany(v => v.Errors).Select(e => e.ErrorMessage)); + var keys = string.Join(",", state.Where(entry => entry.Value?.Errors.Count > 0).Select(entry => entry.Key)); + return (valid, errors, playlist.Count, keys); + } + + public void Dispose() => _scope.Dispose(); + } + + #region Nothing to play + + [Fact] + public void AnEmptyPlaylistProducesNothing() + { + using var harness = new Harness(); + + var (valid, _, entries, _) = harness.Parse(); + + Assert.True(valid); + Assert.Equal(0, entries); + } + + [Fact] + public void AnEmptyEntryIsSkippedWithoutDisturbingItsNeighbours() + { + using var harness = new Harness(); + + // The skip itself is not observable — with the guard removed an empty entry produces + // nothing and is discarded further down regardless. What is observable is that it still + // consumes a position, so the error is attributed to the right entry. + var (valid, errors, _, keys) = harness.Parse("", "g999", ""); + + Assert.False(valid); + Assert.Equal("Unknown group ID \"g999\".", errors); + Assert.Equal("playlist[1]", keys); + } + + #endregion + + #region Rejected entries + + [Fact] + public void AnUnknownGroupIsRejected() + { + using var harness = new Harness(); + + var (valid, errors, _, _) = harness.Parse("g999"); + + Assert.False(valid); + Assert.Contains("Unknown group ID", errors); + } + + [Theory] + [InlineData("gabc")] + [InlineData("g0")] + [InlineData("g-1")] + public void AGroupIdThatIsNotAPositiveNumberIsRejected(string item) + { + using var harness = new Harness(); + + var (valid, errors, _, _) = harness.Parse(item); + + Assert.False(valid); + Assert.Contains("Invalid group ID", errors); + } + + [Theory] + [InlineData("rabc")] + [InlineData("r0")] + public void AReleaseGroupIdThatIsNotAPositiveNumberIsRejected(string releaseItem) + { + using var harness = new Harness(); + + var (valid, errors, _, _) = harness.Parse($"g{GroupID} {releaseItem}"); + + Assert.False(valid); + Assert.Contains("Invalid release group ID", errors); + } + + [Fact] + public void AGroupEntryWithMoreThanAReleaseGroupIsRejected() + { + using var harness = new Harness(); + + var (valid, errors, _, _) = harness.Parse($"g{GroupID} r7 e9"); + + Assert.False(valid); + Assert.Contains("Invalid item", errors); + } + + [Fact] + public void AGroupEntryWithATrailingWordIsRejected() + { + using var harness = new Harness(); + + var (valid, errors, _, _) = harness.Parse($"g{GroupID} nonsense"); + + Assert.False(valid); + Assert.Contains("Invalid item", errors); + } + + #endregion +} diff --git a/Shoko.Tests/Services/ReleaseAutoManagementTests.cs b/Shoko.Tests/Services/ReleaseAutoManagementTests.cs new file mode 100644 index 0000000000..49d29c0471 --- /dev/null +++ b/Shoko.Tests/Services/ReleaseAutoManagementTests.cs @@ -0,0 +1,324 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Shoko.Abstractions.Metadata; +using Shoko.Abstractions.Metadata.Enums; +using Shoko.Server.Models.AniDB; +using Shoko.Server.Models.CrossReference; +using Shoko.Server.Models.Release; +using Shoko.Server.Models.Shoko; +using Shoko.Server.Repositories.Cached; +using Shoko.Server.Repositories.Cached.AniDB; +using Shoko.Server.Services; +using Shoko.Server.Settings; +using Shoko.Tests.Infrastructure; +using Xunit; + +namespace Shoko.Tests.Services; + +/// +/// Covers , which decides which of +/// a user's files get deleted when a better release of the same episodes is present. +/// +/// +/// This is the most destructive decision the server makes on its own, and it had no tests. The +/// method returns the list rather than acting on it, so every rule can be checked without deleting +/// anything. The real supplies the redundancy rules; +/// ranking is not involved, since the primary is whichever candidate is passed first. +/// +public class ReleaseAutoManagementTests +{ + private const int AnimeID = 100; + + private static readonly DateTime s_past = new(2000, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + private sealed class Harness + { + public ReleaseAutoManagementService Service { get; } + + public AnimeSeries Series { get; } = new() { AnimeSeriesID = 1, AniDB_ID = AnimeID }; + + public Dictionary VideoLookup { get; } = []; + + public Harness(ReleaseComparisonPreferences? preferences, bool seriesIsAiring, IEnumerable<(int placeId, int videoId, int episodeNumber)>? files) + { + var settings = new Mock(); + settings.Setup(s => s.GetSettings(It.IsAny())) + .Returns(new ServerSettings { ReleaseComparisonPreferences = preferences ?? new ReleaseComparisonPreferences() }); + + var episodes = new List(); + var crossRefs = new List(); + var videos = new List(); + foreach (var (placeId, videoId, episodeNumber) in files ?? []) + { + var hash = $"hash-{videoId}"; + if (!VideoLookup.ContainsKey(videoId)) + { + var video = new VideoLocal { VideoLocalID = videoId, Hash = hash, FileSize = 1000 + videoId }; + videos.Add(video); + VideoLookup[videoId] = video; + } + + if (episodes.All(e => e.EpisodeNumber != episodeNumber)) + episodes.Add(new AniDB_Episode + { + AniDB_EpisodeID = episodeNumber, + EpisodeID = episodeNumber, + AnimeID = AnimeID, + EpisodeNumber = episodeNumber, + EpisodeType = EpisodeType.Episode, + }); + + crossRefs.Add(new CrossRef_File_Episode + { + CrossRef_File_EpisodeID = placeId, + Hash = hash, + AnimeID = AnimeID, + EpisodeID = episodeNumber, + Percentage = 100, + }); + } + + var videoRepository = CachedRepo.Build(v => v.VideoLocalID, videos); + var episodeRepository = CachedRepo.Build(e => e.AniDB_EpisodeID, episodes); + var crossRefRepository = CachedRepo.Build(x => x.CrossRef_File_EpisodeID, crossRefs); + var releaseInfoRepository = CachedRepo.Build(r => r.StoredReleaseInfoID, []); + var animeRepository = CachedRepo.Build(a => a.AniDB_AnimeID, + [new AniDB_Anime { AniDB_AnimeID = 1, AnimeID = AnimeID, MainTitle = "Test", EndDate = seriesIsAiring ? null : new PartialDateOnly(s_past.Year, s_past.Month, s_past.Day) }]); + + Service = new ReleaseAutoManagementService( + settings.Object, + new VideoReleaseGroupingService(videoRepository, episodeRepository, releaseInfoRepository, crossRefRepository), + new ReleaseComparisonService(settings.Object, null!), + videoRepository, + videoLocalPlaces: null!, + crossRefRepository, + animeSeries: null!, + animeRepository, + videoService: null!, + NullLogger.Instance); + } + } + + private static VideoLocal_Place Place(int id, int videoId) + => new() { ID = id, VideoID = videoId, ManagedFolderID = 1, RelativePath = $"file-{id}.mkv" }; + + private static VideoReleaseCandidate Candidate( + string key, + IReadOnlyList places, + IEnumerable episodeNumbers, + bool hasReleaseInfo = true, + bool isMixed = false, + bool isCorrupted = false, + bool isChapteredMixed = false, + bool isCensoredMixed = false, + bool isCreditlessMixed = false) + => new() + { + Key = key, + Places = places, + EpisodeCoverage = episodeNumbers.Select(n => (EpisodeType.Episode, n)).ToHashSet(), + HasReleaseInfo = hasReleaseInfo, + IsMixed = isMixed, + IsCorrupted = isCorrupted, + IsChapteredMixed = isChapteredMixed, + IsCensoredMixed = isCensoredMixed, + IsCreditlessMixed = isCreditlessMixed, + }; + + /// Two candidates covering episode 1, ranked primary first. + private static Harness TwoCandidates(ReleaseComparisonPreferences? preferences = null, bool seriesIsAiring = false) + => new(preferences, seriesIsAiring, [(1, 1, 1), (2, 2, 1)]); + + #region Nothing to do + + [Fact] + public void NothingIsDeletedWhenThereIsOnlyOneCandidate() + { + var harness = TwoCandidates(); + var only = Candidate("primary", [Place(1, 1)], [1]); + + Assert.Empty(harness.Service.ComputeRedundantPlaces(harness.Series, [only], harness.VideoLookup)); + } + + [Fact] + public void NothingIsDeletedWhenThereAreNoCandidates() + { + var harness = TwoCandidates(); + + Assert.Empty(harness.Service.ComputeRedundantPlaces(harness.Series, [], harness.VideoLookup)); + } + + [Fact] + public void ASecondaryCoveringEpisodesThePrimaryDoesNotIsKept() + { + var harness = new Harness(null, seriesIsAiring: false, [(1, 1, 1), (2, 2, 2)]); + var primary = Candidate("primary", [Place(1, 1)], [1]); + var secondary = Candidate("secondary", [Place(2, 2)], [2]); + + // The primary does not provide episode 2, so nothing about the secondary is redundant. + Assert.Empty(harness.Service.ComputeRedundantPlaces(harness.Series, [primary, secondary], harness.VideoLookup)); + } + + #endregion + + #region The primary is never deleted + + [Fact] + public void AFileSharedByBothCandidatesSurvivesWhileTheRestOfTheSecondaryGoes() + { + var harness = new Harness(null, seriesIsAiring: false, [(1, 1, 1), (2, 2, 1)]); + var shared = Place(1, 1); + var primary = Candidate("primary", [shared], [1]); + var secondary = Candidate("secondary", [shared, Place(2, 2)], [1]); + + var redundant = harness.Service.ComputeRedundantPlaces(harness.Series, [primary, secondary], harness.VideoLookup); + + Assert.Equal([2], redundant.Select(p => p.ID).Order()); + } + + #endregion + + #region The eligibility gate + + [Theory] + [InlineData(false, false, false, false, false, true)] + [InlineData(true, false, false, false, false, false)] + [InlineData(false, true, false, false, false, false)] + [InlineData(false, false, true, false, false, false)] + [InlineData(false, false, false, true, false, false)] + [InlineData(false, false, false, false, true, false)] + public void AnIneligiblePrimaryDeletesNothing( + bool isMixed, bool isCorrupted, bool isChapteredMixed, bool isCensoredMixed, bool isCreditlessMixed, bool expectDeletion) + { + var harness = TwoCandidates(); + var primary = Candidate("primary", [Place(1, 1)], [1], + isMixed: isMixed, isCorrupted: isCorrupted, + isChapteredMixed: isChapteredMixed, isCensoredMixed: isCensoredMixed, isCreditlessMixed: isCreditlessMixed); + var secondary = Candidate("secondary", [Place(2, 2)], [1]); + + var redundant = harness.Service.ComputeRedundantPlaces(harness.Series, [primary, secondary], harness.VideoLookup); + + // A primary that cannot be trusted to stand in for the others must not cause deletions. + Assert.Equal(expectDeletion, redundant.Count > 0); + } + + [Fact] + public void APrimaryWithoutReleaseInfoDeletesNothing() + { + var harness = TwoCandidates(); + var primary = Candidate("primary", [Place(1, 1)], [1], hasReleaseInfo: false); + var secondary = Candidate("secondary", [Place(2, 2)], [1]); + + Assert.Empty(harness.Service.ComputeRedundantPlaces(harness.Series, [primary, secondary], harness.VideoLookup)); + } + + [Fact] + public void TheGateCanOnlyBeBypassedDeliberately() + { + var harness = TwoCandidates(); + var primary = Candidate("primary", [Place(1, 1)], [1], hasReleaseInfo: false); + var secondary = Candidate("secondary", [Place(2, 2)], [1]); + + // Reserved for a primary the user picked by hand; unattended paths must never pass this. + var redundant = harness.Service.ComputeRedundantPlaces( + harness.Series, [primary, secondary], harness.VideoLookup, bypassEligibilityGate: true); + + Assert.Equal([2], redundant.Select(p => p.ID)); + } + + #endregion + + #region Redundancy + + [Fact] + public void ASecondaryFullyCoveredByThePrimaryIsRedundant() + { + var harness = TwoCandidates(); + var primary = Candidate("primary", [Place(1, 1)], [1]); + var secondary = Candidate("secondary", [Place(2, 2)], [1]); + + var redundant = harness.Service.ComputeRedundantPlaces(harness.Series, [primary, secondary], harness.VideoLookup); + + Assert.Equal([2], redundant.Select(p => p.ID)); + } + + [Fact] + public void APlaceAppearingInTwoSecondariesIsOnlyListedOnce() + { + var harness = TwoCandidates(); + var duplicated = Place(2, 2); + var primary = Candidate("primary", [Place(1, 1)], [1]); + var first = Candidate("second", [duplicated], [1]); + var second = Candidate("third", [duplicated], [1]); + + var redundant = harness.Service.ComputeRedundantPlaces(harness.Series, [primary, first, second], harness.VideoLookup); + + // Deleting the same file twice would fail the second time round. + Assert.Single(redundant); + } + + #endregion + + #region Per-file mode + + [Fact] + public void AnAiringSeriesUsesPerFileDeletionWhenConfigured() + { + var preferences = new ReleaseComparisonPreferences { PerFileDeletionForAiringSeries = true }; + var harness = new Harness(preferences, seriesIsAiring: true, [(1, 1, 1), (2, 2, 1), (3, 3, 2)]); + var primary = Candidate("primary", [Place(1, 1)], [1]); + // One file duplicates episode 1, the other adds episode 2 the primary does not have. + var secondary = Candidate("secondary", [Place(2, 2), Place(3, 3)], [1, 2]); + + var redundant = harness.Service.ComputeRedundantPlaces(harness.Series, [primary, secondary], harness.VideoLookup); + + // Per file, only the duplicate goes; the file carrying episode 2 is kept. + Assert.Equal([2], redundant.Select(p => p.ID)); + } + + [Fact] + public void AFinishedSeriesComparesWholeCandidatesInstead() + { + var preferences = new ReleaseComparisonPreferences { PerFileDeletionForAiringSeries = true }; + var harness = new Harness(preferences, seriesIsAiring: false, [(1, 1, 1), (2, 2, 1), (3, 3, 2)]); + var primary = Candidate("primary", [Place(1, 1)], [1]); + var secondary = Candidate("secondary", [Place(2, 2), Place(3, 3)], [1, 2]); + + var redundant = harness.Service.ComputeRedundantPlaces(harness.Series, [primary, secondary], harness.VideoLookup); + + // The secondary as a whole is not covered by the primary, so none of it is removed. + Assert.Empty(redundant); + } + + [Fact] + public void AFileWithUnknownEpisodeCoverageIsKept() + { + var preferences = new ReleaseComparisonPreferences { PerFileDeletionForAiringSeries = true }; + // Video 9 is known but has no cross-reference, so its coverage cannot be resolved. + var harness = new Harness(preferences, seriesIsAiring: true, [(1, 1, 1)]); + harness.VideoLookup[9] = new VideoLocal { VideoLocalID = 9, Hash = "hash-9", FileSize = 9 }; + var primary = Candidate("primary", [Place(1, 1)], [1]); + var secondary = Candidate("secondary", [Place(9, 9)], [1]); + + var redundant = harness.Service.ComputeRedundantPlaces(harness.Series, [primary, secondary], harness.VideoLookup); + + // Never delete a file we cannot prove is covered elsewhere. + Assert.Empty(redundant); + } + + [Fact] + public void IsSeriesAiringFollowsTheEndDate() + { + Assert.True(new Harness(null, seriesIsAiring: true, null).Service.IsSeriesAiring(new AnimeSeries { AniDB_ID = AnimeID })); + Assert.False(new Harness(null, seriesIsAiring: false, null).Service.IsSeriesAiring(new AnimeSeries { AniDB_ID = AnimeID })); + } + + [Fact] + public void AnUnknownSeriesIsNotTreatedAsAiring() + => Assert.False(new Harness(null, seriesIsAiring: true, null).Service.IsSeriesAiring(new AnimeSeries { AniDB_ID = 999 })); + + #endregion +} diff --git a/Shoko.Tests/Services/UserDataServiceVideoTests.cs b/Shoko.Tests/Services/UserDataServiceVideoTests.cs new file mode 100644 index 0000000000..4c4434c3fc --- /dev/null +++ b/Shoko.Tests/Services/UserDataServiceVideoTests.cs @@ -0,0 +1,403 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Shoko.Abstractions.Metadata; +using Shoko.Abstractions.User; +using Shoko.Abstractions.User.Enums; +using Shoko.Abstractions.User.Events; +using Shoko.Abstractions.User.Update; +using Shoko.Abstractions.Video; +using Shoko.Abstractions.Video.Media; +using Shoko.Server.Models.Shoko; +using Shoko.Server.Repositories.Cached; +using Shoko.Server.Services; +using Shoko.Server.Settings; +using Shoko.Tests.Infrastructure; +using Xunit; + +namespace Shoko.Tests.Services; + +/// +/// Covers how folds a playback update into a video's stored user +/// data. This decides whether a file counts as watched, how far through it the user is, and how +/// many times they have seen it — the state the whole watched/unwatched view of a collection is +/// built from, and it had no tests. +/// +[Collection(nameof(RepoFactoryCollection))] +public class UserDataServiceVideoTests +{ + private const int UserID = 7; + private const int VideoID = 42; + + private static readonly TimeSpan s_duration = TimeSpan.FromMinutes(24); + + private sealed class Harness : IDisposable + { + public Mock Repository { get; } + + public UserDataService Service { get; } + + public IVideo Video { get; } + + public IUser User { get; } + + public VideoLocal_User? Stored => Repository.Object.GetByUserAndVideoLocalID(UserID, VideoID); + + private readonly RepoFactoryScope _scope; + + public void Dispose() => _scope.Dispose(); + + public Harness(VideoLocal_User? existing, TimeSpan? duration, bool isAnidbUser) + { + // VideoLocal_User.ToString() resolves the video through RepoFactory, and Moq calls it + // when rendering a failed verification. Without this a genuine assertion failure would + // surface as a NullReferenceException from the mocking library instead. + _scope = new RepoFactoryScope() + .With(v => v.VideoLocalID, + [new VideoLocal { VideoLocalID = VideoID, Hash = "abc", FileSize = 1 }]); + + Repository = CachedRepo.BuildWritable( + u => u.VideoLocal_UserID, existing is null ? [] : [existing]); + + var settings = new Mock(); + settings.Setup(s => s.GetSettings(It.IsAny())).Returns(new ServerSettings()); + + var video = new Mock(); + video.SetupGet(v => v.ID).Returns(VideoID); + video.SetupGet(v => v.CrossReferences).Returns([]); + if (duration.HasValue) + { + var mediaInfo = new Mock(); + mediaInfo.SetupGet(m => m.Duration).Returns(duration.Value); + video.SetupGet(v => v.MediaInfo).Returns(mediaInfo.Object); + } + + Video = video.Object; + + var user = new Mock(); + user.SetupGet(u => u.ID).Returns(UserID); + user.SetupGet(u => u.IsAnidbUser).Returns(isAnidbUser); + User = user.Object; + + Service = new UserDataService( + NullLogger.Instance, + settings.Object, + schedulerFactory: null!, + serviceProvider: null!, + videoUserDataRepository: Repository.Object, + episodeUserDataRepository: null!, + seriesUserDataRepository: null!, + groupUserDataRepository: null!, + userRepository: null!); + } + } + + private static Harness Create(VideoLocal_User? existing = null, TimeSpan? duration = null, bool isAnidbUser = false) + => new(existing, duration ?? s_duration, isAnidbUser); + + private static VideoLocal_User Existing(DateTime? watchedDate = null, int watchedCount = 0, TimeSpan? progress = null) + => new() + { + VideoLocal_UserID = 1, + JMMUserID = UserID, + VideoLocalID = VideoID, + WatchedDate = watchedDate, + WatchedCount = watchedCount, + ProgressPosition = progress, + LastUpdated = new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Local), + }; + + #region Argument validation + + [Fact] + public async Task SavingWithoutAUserIsRejected() + { + using var harness = Create(); + + await Assert.ThrowsAsync( + () => harness.Service.SetVideoWatchedStatus(harness.Video, null!)); + } + + [Fact] + public async Task SavingWithoutAVideoIsRejected() + { + using var harness = Create(); + + await Assert.ThrowsAsync( + () => harness.Service.SetVideoWatchedStatus(null!, harness.User)); + } + + #endregion + + #region Marking watched and unwatched + + [Fact] + public async Task MarkingAVideoWatchedRecordsTheWatchedDate() + { + using var harness = Create(); + var watchedAt = new DateTime(2024, 5, 1, 12, 0, 0, DateTimeKind.Local); + + var result = await harness.Service.SetVideoWatchedStatus(harness.Video, harness.User, watched: true, watchedAt: watchedAt); + + Assert.Equal(watchedAt, result.LastPlayedAt); + Assert.Equal(watchedAt, harness.Stored!.WatchedDate); + } + + [Fact] + public async Task MarkingAVideoWatchedIncrementsThePlaybackCount() + { + using var harness = Create(Existing(watchedCount: 3)); + + var result = await harness.Service.SetVideoWatchedStatus(harness.Video, harness.User); + + Assert.Equal(4, result.PlaybackCount); + } + + [Fact] + public async Task MarkingAnAlreadyWatchedVideoWatchedAtTheSameTimeDoesNotCountAgain() + { + var watchedAt = new DateTime(2024, 5, 1, 12, 0, 0, DateTimeKind.Local); + using var harness = Create(Existing(watchedDate: watchedAt, watchedCount: 1)); + + var result = await harness.Service.SetVideoWatchedStatus(harness.Video, harness.User, watched: true, watchedAt: watchedAt); + + // Nothing changed, so this is not a new viewing. + Assert.Equal(1, result.PlaybackCount); + } + + [Fact] + public async Task MarkingAVideoUnwatchedClearsTheWatchedDate() + { + using var harness = Create(Existing(watchedDate: new DateTime(2024, 5, 1, 12, 0, 0, DateTimeKind.Local), watchedCount: 1)); + + var result = await harness.Service.SetVideoWatchedStatus(harness.Video, harness.User, watched: false); + + Assert.Null(result.LastPlayedAt); + } + + [Fact] + public async Task MarkingAVideoUnwatchedKeepsThePlaybackCount() + { + using var harness = Create(Existing(watchedDate: new DateTime(2024, 5, 1, 12, 0, 0, DateTimeKind.Local), watchedCount: 2)); + + var result = await harness.Service.SetVideoWatchedStatus(harness.Video, harness.User, watched: false); + + // Un-watching says "I have not seen this now", not "I have never seen it". + Assert.Equal(2, result.PlaybackCount); + } + + [Fact] + public async Task AWatchedDateGivenInUtcIsStoredAsLocalTime() + { + using var harness = Create(); + var utc = new DateTime(2024, 5, 1, 12, 0, 0, DateTimeKind.Utc); + + var result = await harness.Service.SetVideoWatchedStatus(harness.Video, harness.User, watched: true, watchedAt: utc); + + // Stored as local, otherwise an unchanged date from a UTC source reads as a change every + // time it is compared against the stored local one. Asserted on the kind as well as the + // value, so this still means something when the tests run on a machine set to UTC. + Assert.Equal(utc.ToLocalTime(), result.LastPlayedAt); + Assert.Equal(DateTimeKind.Local, harness.Stored!.WatchedDate!.Value.Kind); + } + + #endregion + + #region Progress + + [Fact] + public async Task ProgressIsRecorded() + { + using var harness = Create(); + var progress = TimeSpan.FromMinutes(5); + + var result = await harness.Service.SaveVideoUserData(harness.Video, harness.User, new() { ProgressPosition = progress }); + + Assert.Equal(progress, result.ProgressPosition); + } + + [Fact] + public void NegativeProgressIsRejectedByTheUpdateItself() + { + // The guard sits on the update object, so a bad value never reaches the service. + Assert.Throws(() => new VideoUserDataUpdate { ProgressPosition = TimeSpan.FromMinutes(-5) }); + } + + [Fact] + public async Task ProgressBeyondTheEndCountsAsFinishedRatherThanBeingClamped() + { + using var harness = Create(); + + var result = await harness.Service.SaveVideoUserData(harness.Video, harness.User, new() { ProgressPosition = s_duration + TimeSpan.FromMinutes(10) }); + + // Anything past the end is already past the 97.5% threshold, so it is treated as watched + // and the position reset — the clamp to the duration never comes into play here. + Assert.NotNull(result.LastPlayedAt); + Assert.Equal(TimeSpan.Zero, result.ProgressPosition); + } + + [Fact] + public async Task ProgressPastTheNearlyFinishedThresholdMarksTheVideoWatched() + { + using var harness = Create(); + + // Anything past 97.5% counts as finished, so trailing credits do not leave it unwatched. + var result = await harness.Service.SaveVideoUserData(harness.Video, harness.User, new() { ProgressPosition = s_duration * 0.98 }); + + Assert.NotNull(result.LastPlayedAt); + Assert.Equal(TimeSpan.Zero, result.ProgressPosition); + } + + [Fact] + public async Task ProgressJustBelowTheThresholdLeavesTheVideoUnwatched() + { + using var harness = Create(); + + var result = await harness.Service.SaveVideoUserData(harness.Video, harness.User, new() { ProgressPosition = s_duration * 0.97 }); + + Assert.Null(result.LastPlayedAt); + Assert.Equal(s_duration * 0.97, result.ProgressPosition); + } + + [Fact] + public async Task MarkingAVideoWatchedClearsAnyStoredProgress() + { + using var harness = Create(Existing(progress: TimeSpan.FromMinutes(5))); + + var result = await harness.Service.SetVideoWatchedStatus(harness.Video, harness.User); + + Assert.Equal(TimeSpan.Zero, result.ProgressPosition); + } + + [Fact] + public async Task ProgressIsLeftAloneWhenTheDurationIsUnknown() + { + using var harness = new Harness(null, duration: null, isAnidbUser: false); + var progress = TimeSpan.FromHours(99); + + var result = await harness.Service.SaveVideoUserData(harness.Video, harness.User, new() { ProgressPosition = progress }); + + // With no media info there is nothing to clamp against. + Assert.Equal(progress, result.ProgressPosition); + Assert.Null(result.LastPlayedAt); + } + + #endregion + + #region Playback count + + [Fact] + public async Task AnExplicitPlaybackCountIsStored() + { + using var harness = Create(); + + var result = await harness.Service.SaveVideoUserData(harness.Video, harness.User, new() { PlaybackCount = 5 }); + + Assert.Equal(5, result.PlaybackCount); + } + + [Fact] + public async Task ANegativePlaybackCountIsInferredFromTheWatchedDate() + { + using var harness = Create(Existing(watchedDate: new DateTime(2024, 5, 1, 12, 0, 0, DateTimeKind.Local))); + + var result = await harness.Service.SaveVideoUserData(harness.Video, harness.User, new() { PlaybackCount = -1 }); + + Assert.Equal(1, result.PlaybackCount); + } + + [Fact] + public async Task ANegativePlaybackCountOnAnUnwatchedVideoInfersZero() + { + using var harness = Create(Existing()); + + var result = await harness.Service.SaveVideoUserData(harness.Video, harness.User, new() { PlaybackCount = -1 }); + + Assert.Equal(0, result.PlaybackCount); + } + + #endregion + + #region Persistence + + [Fact] + public async Task ANewRecordIsPersisted() + { + using var harness = Create(); + + await harness.Service.SetVideoWatchedStatus(harness.Video, harness.User); + + harness.Repository.Verify(r => r.Save(It.IsAny()), Times.Once); + } + + [Fact] + public async Task AnUpdateThatChangesNothingIsNotPersisted() + { + var watchedAt = new DateTime(2024, 5, 1, 12, 0, 0, DateTimeKind.Local); + using var harness = Create(Existing(watchedDate: watchedAt, watchedCount: 1)); + + await harness.Service.SaveVideoUserData(harness.Video, harness.User, new(), VideoUserDataSaveReason.PlaybackProgress); + + // Playback progress fires constantly; rewriting an unchanged row on every tick would be a + // needless write per second per client. + harness.Repository.Verify(r => r.Save(It.IsAny()), Times.Never); + } + + [Fact] + public async Task TheSavedRecordIsReturnedOnTheNextLookup() + { + using var harness = Create(); + + var result = await harness.Service.SetVideoWatchedStatus(harness.Video, harness.User); + + Assert.Equal(result.LastPlayedAt, harness.Stored!.WatchedDate); + } + + #endregion + + #region Events + + [Fact] + public async Task SavingRaisesTheVideoUserDataSavedEvent() + { + using var harness = Create(); + VideoUserDataSavedEventArgs? captured = null; + harness.Service.VideoUserDataSaved += (_, args) => captured = args; + + await harness.Service.SetVideoWatchedStatus(harness.Video, harness.User, reason: VideoUserDataSaveReason.UserInteraction); + + Assert.NotNull(captured); + Assert.Equal(VideoUserDataSaveReason.UserInteraction, captured!.Reason); + Assert.Same(harness.User, captured.User); + } + + [Fact] + public async Task NoEventIsRaisedWhenNothingChanged() + { + var watchedAt = new DateTime(2024, 5, 1, 12, 0, 0, DateTimeKind.Local); + using var harness = Create(Existing(watchedDate: watchedAt, watchedCount: 1)); + var raised = false; + harness.Service.VideoUserDataSaved += (_, _) => raised = true; + + await harness.Service.SaveVideoUserData(harness.Video, harness.User, new()); + + Assert.False(raised); + } + + [Fact] + public async Task AFailingEventHandlerDoesNotFailTheSave() + { + using var harness = Create(); + harness.Service.VideoUserDataSaved += (_, _) => throw new InvalidOperationException("boom"); + + var result = await harness.Service.SetVideoWatchedStatus(harness.Video, harness.User); + + // A misbehaving listener must not lose the user's watched state. + Assert.NotNull(result.LastPlayedAt); + Assert.NotNull(harness.Stored!.WatchedDate); + } + + #endregion +} diff --git a/Shoko.Tests/Services/VideoRelocationGuardTests.cs b/Shoko.Tests/Services/VideoRelocationGuardTests.cs new file mode 100644 index 0000000000..43eb397ff7 --- /dev/null +++ b/Shoko.Tests/Services/VideoRelocationGuardTests.cs @@ -0,0 +1,265 @@ +using System; +using System.IO; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging.Abstractions; +using Moq; +using Shoko.Abstractions.Video.Relocation; +using Shoko.Server.Models.Shoko; +using Shoko.Server.Repositories.Cached; +using Shoko.Server.Services; +using Shoko.Tests.Infrastructure; +using Xunit; + +namespace Shoko.Tests.Services; + +/// +/// Covers the guards applies before it +/// moves a user's file. Each one is the last thing standing between a bad request and a file being +/// written somewhere it should not be, and none of them had a test. +/// +/// +/// The file system is reached through the injected , so these run +/// against a mock and never touch a disk — a rejected request is proven by the mock never being +/// asked to move anything. +/// +[Collection(nameof(RepoFactoryCollection))] +public class VideoRelocationGuardTests +{ + private const int VideoID = 1; + private const int SourceFolderID = 1; + private const int DestinationFolderID = 2; + private const string SourcePath = "/media/anime"; + private const string DestinationPath = "/media/sorted"; + private const string RelativePath = "Show/episode.mkv"; + + private static ShokoManagedFolder Folder(int id, string path, bool isDropSource = true, bool isDropDestination = false) + => new() { ID = id, Name = $"folder-{id}", Path = path, IsDropSource = isDropSource, IsDropDestination = isDropDestination }; + + private sealed class Harness : IDisposable + { + public Mock FileSystem { get; } = new(MockBehavior.Loose); + + public VideoRelocationService Service { get; } + + public VideoLocal_Place Place { get; } + + private readonly RepoFactoryScope _scope; + + public Harness(ShokoManagedFolder sourceFolder, ShokoManagedFolder destinationFolder) + { + Place = new VideoLocal_Place { ID = 1, VideoID = VideoID, ManagedFolderID = sourceFolder.ID, RelativePath = RelativePath }; + + _scope = new RepoFactoryScope() + .With(v => v.VideoLocalID, + [new VideoLocal { VideoLocalID = VideoID, Hash = "abc", FileSize = 100 }]) + .With(f => f.ID, [sourceFolder, destinationFolder]) + .With(p => p.ID, [Place]); + + // The file is present unless a test says otherwise. + FileSystem.Setup(f => f.FileExists(It.IsAny())).Returns(true); + + Service = new VideoRelocationService( + NullLogger.Instance, + serviceProvider: null!, + pluginManager: null!, + settingsProvider: null!, + schedulerFactory: null!, + configurationService: null!, + fileWatcherService: null!, + videoLocalPlace: null!, + storedRelocationPresetRepository: null!, + fileNameHash: null!, + managedFolders: null!, + fileSystemHelpers: FileSystem.Object); + } + + /// Asserts that nothing was written to, moved on, or removed from disk. + public void AssertNothingWasMoved() + { + FileSystem.Verify(f => f.MoveFile(It.IsAny(), It.IsAny()), Times.Never); + FileSystem.Verify(f => f.DeleteFile(It.IsAny()), Times.Never); + FileSystem.Verify(f => f.CreateDirectory(It.IsAny()), Times.Never); + } + + public void Dispose() => _scope.Dispose(); + } + + private static Harness Create(bool sourceIsDropSource = true, bool sourceIsDropDestination = false) + => new(Folder(SourceFolderID, SourcePath, sourceIsDropSource, sourceIsDropDestination), Folder(DestinationFolderID, DestinationPath, false, true)); + + private static DirectlyRelocateRequest Request(ShokoManagedFolder? folder, string? relativePath, bool allowInsideDestination = true) + => new() + { + ManagedFolder = folder, + RelativePath = relativePath, + AllowRelocationInsideDestination = allowInsideDestination, + }; + + #region Request validation + + [Fact] + public async Task ARequestWithoutAManagedFolderIsRejected() + { + using var harness = Create(); + + var response = await harness.Service.DirectlyRelocateFile(harness.Place, Request(null, RelativePath)); + + Assert.False(response.Success); + harness.AssertNothingWasMoved(); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public async Task ARequestWithoutARelativePathIsRejected(string? relativePath) + { + using var harness = Create(); + + var response = await harness.Service.DirectlyRelocateFile( + harness.Place, Request(Folder(DestinationFolderID, DestinationPath), relativePath)); + + Assert.False(response.Success); + harness.AssertNothingWasMoved(); + } + + [Fact] + public async Task ACancelledRequestIsRejected() + { + using var harness = Create(); + using var cancellation = new CancellationTokenSource(); + cancellation.Cancel(); + + var request = Request(Folder(DestinationFolderID, DestinationPath), RelativePath) with { CancellationToken = cancellation.Token }; + var response = await harness.Service.DirectlyRelocateFile(harness.Place, request); + + Assert.False(response.Success); + harness.AssertNothingWasMoved(); + } + + #endregion + + #region Escaping the managed folder + + [Theory] + [InlineData("../outside/episode.mkv")] + [InlineData("Show/../../outside/episode.mkv")] + [InlineData("../../../etc/episode.mkv")] + public async Task ARelativePathThatClimbsOutOfTheManagedFolderIsRejected(string relativePath) + { + using var harness = Create(); + + var response = await harness.Service.DirectlyRelocateFile( + harness.Place, Request(Folder(DestinationFolderID, DestinationPath), relativePath)); + + Assert.False(response.Success); + harness.AssertNothingWasMoved(); + } + + [Fact] + public async Task ARelativePathLeadingIntoASiblingFolderWithASharedPrefixIsRejected() + { + using var harness = Create(); + + // "/media/anime" and "/media/animeX" share a prefix. The containment check only holds + // because ShokoManagedFolder.Path always ends in a directory separator; a plain prefix + // comparison would let this through and write the file outside the managed folder. + var response = await harness.Service.DirectlyRelocateFile( + harness.Place, Request(Folder(DestinationFolderID, "/media/anime"), "../animeX/episode.mkv")); + + Assert.False(response.Success); + Assert.Contains("outside the managed folder", response.Error?.Message ?? string.Empty); + harness.AssertNothingWasMoved(); + } + + [Fact] + public void AManagedFolderPathAlwaysEndsInASeparator() + { + // The containment check above depends on this, so it is pinned here rather than left to + // chance in ShokoManagedFolder. + Assert.EndsWith(Path.DirectorySeparatorChar.ToString(), Folder(SourceFolderID, SourcePath).Path); + } + + [Fact] + public async Task ARelativePathStayingInsideTheManagedFolderPassesTheContainmentCheck() + { + using var harness = Create(); + + var response = await harness.Service.DirectlyRelocateFile( + harness.Place, Request(Folder(DestinationFolderID, DestinationPath), "Show/Season 1/../episode.mkv")); + + // It may still fail later for other reasons, but not for leaving the folder. + Assert.DoesNotContain("outside the managed folder", response.Error?.Message ?? string.Empty); + } + + #endregion + + #region Drop folder rules + + [Fact] + public async Task AFileInAnExcludedFolderIsNotRelocated() + { + using var harness = Create(sourceIsDropSource: false, sourceIsDropDestination: false); + + var response = await harness.Service.DirectlyRelocateFile( + harness.Place, Request(Folder(DestinationFolderID, DestinationPath), RelativePath)); + + Assert.False(response.Success); + harness.AssertNothingWasMoved(); + } + + [Fact] + public async Task AFileInADropDestinationIsNotRelocatedWhenRelocatingInsideDestinationsIsDisabled() + { + using var harness = Create(sourceIsDropSource: false, sourceIsDropDestination: true); + + var response = await harness.Service.DirectlyRelocateFile( + harness.Place, Request(Folder(DestinationFolderID, DestinationPath), RelativePath, allowInsideDestination: false)); + + Assert.False(response.Success); + harness.AssertNothingWasMoved(); + } + + [Fact] + public async Task AFileInAFolderThatIsBothSourceAndDestinationIsAllowedThrough() + { + using var harness = Create(sourceIsDropSource: true, sourceIsDropDestination: true); + + var response = await harness.Service.DirectlyRelocateFile( + harness.Place, Request(Folder(DestinationFolderID, DestinationPath), RelativePath, allowInsideDestination: false)); + + Assert.DoesNotContain("drop destination", response.Error?.Message ?? string.Empty); + } + + #endregion + + #region File system state + + [Fact] + public async Task AMissingSourceFileIsReportedRatherThanMoved() + { + using var harness = Create(); + harness.FileSystem.Setup(f => f.FileExists(It.IsAny())).Returns(false); + + var response = await harness.Service.DirectlyRelocateFile( + harness.Place, Request(Folder(DestinationFolderID, DestinationPath), RelativePath)); + + Assert.False(response.Success); + harness.AssertNothingWasMoved(); + } + + [Fact] + public async Task TheSourceFileIsLookedForAtItsCurrentLocation() + { + using var harness = Create(); + harness.FileSystem.Setup(f => f.FileExists(It.IsAny())).Returns(false); + + await harness.Service.DirectlyRelocateFile( + harness.Place, Request(Folder(DestinationFolderID, DestinationPath), RelativePath)); + + harness.FileSystem.Verify(f => f.FileExists(Path.Combine(SourcePath, RelativePath)), Times.AtLeastOnce); + } + + #endregion +} diff --git a/Shoko.Tests/Shoko.Tests.csproj b/Shoko.Tests/Shoko.Tests.csproj index d99865c7ce..00b2bd0886 100644 --- a/Shoko.Tests/Shoko.Tests.csproj +++ b/Shoko.Tests/Shoko.Tests.csproj @@ -27,6 +27,7 @@ + diff --git a/Shoko.Tests/Startup.cs b/Shoko.Tests/Startup.cs deleted file mode 100644 index 197696d813..0000000000 --- a/Shoko.Tests/Startup.cs +++ /dev/null @@ -1,12 +0,0 @@ -using Microsoft.Extensions.DependencyInjection; - -namespace Shoko.Tests -{ - public class Startup - { - public void ConfigureServices(IServiceCollection services) - { - //Dunno what we can do with settings yet. - } - } -} diff --git a/Shoko.Tests/TagFilterTest.cs b/Shoko.Tests/TagFilterTest.cs index 07c140bb64..672b262ab4 100644 --- a/Shoko.Tests/TagFilterTest.cs +++ b/Shoko.Tests/TagFilterTest.cs @@ -1,336 +1,315 @@ -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using System.Threading.Tasks; -using Shoko.Server; -using Xunit; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Shoko.Server; +using Xunit; -// ReSharper disable StringLiteralTypo +// ReSharper disable StringLiteralTypo -namespace Shoko.Tests -{ - public class TagFilterTest - { - private readonly ITestOutputHelper _console; - - public TagFilterTest(ITestOutputHelper console) - { - _console = console; - } - - private static IEnumerable Input => - new[] - { - "comedy", - "Comedy", - "horror", - "18 restricted", - "large breasts", - "japan", - "violence", - "source material", - "manga", - "fantasy", - "shounen", - "Earth", - "Asia", - "noitamina", - "cgi", - "3DCG", - "long episodes", - "first girl wins", - "alternative past", - "past", - }; - - private static IEnumerable InputNoSource => - new[] - { - "horror", - "Horror", - "18 restricted", - "large breasts", - "japan", - "violence", - "fantasy", - "shounen", - "Earth", - "Asia", - "noitamina", - "cgi", - "3DCG", - "long episodes", - "first girl wins", - }; - - [Fact(DisplayName = "Full Test")] - public void TestFullList() - { - var actual = TagFilter.String.ProcessTags(TagFilter.Filter.Genre, Input); - var expected = new List - { - "large breasts", - "japan", - "source material", - "manga", - "Earth", - "Asia", - "noitamina", - "cgi", - "3DCG", - "long episodes", - "first girl wins", - "alternative past", - "past", - }; - - _console.WriteLine("Expected: [{0}]", string.Join(", ", expected)); - _console.WriteLine("Actual: [{0}]", string.Join(", ", actual)); - Assert.Equal(expected, actual); - } - - [Fact(DisplayName = "Full Test w/ 'original work'")] - public void TestFullListWithNew() - { - var actual = TagFilter.String.ProcessTags(TagFilter.Filter.Genre, Input.Concat(new[] { "original work" })); - var expected = new List - { - "large breasts", - "japan", - "source material", - "manga", - "Earth", - "Asia", - "noitamina", - "cgi", - "3DCG", - "long episodes", - "first girl wins", - "alternative past", - "past", - }; - - _console.WriteLine("Expected: [{0}]", string.Join(", ", expected)); - _console.WriteLine("Actual: [{0}]", string.Join(", ", actual)); - Assert.Equal(expected, actual); - } - - [Fact(DisplayName = "Full Test w/o Source")] - public void TestFullListNoSource() - { - var actual = TagFilter.String.ProcessTags(TagFilter.Filter.Genre, InputNoSource); - var expected = new List - { - "large breasts", - "japan", - "Earth", - "Asia", - "noitamina", - "cgi", - "3DCG", - "long episodes", - "first girl wins", - "original work", - }; - - _console.WriteLine("Expected: [{0}]", string.Join(", ", expected)); - _console.WriteLine("Actual: [{0}]", string.Join(", ", actual)); - Assert.Equal(expected, actual); - } - - [Fact(DisplayName = "Full Test Inverted")] - public void TestFullListInverted() - { - var filter = TagFilter.Filter.Invert | TagFilter.Filter.Source | TagFilter.Filter.Genre | TagFilter.Filter.Setting; - var actual = TagFilter.String.ProcessTags(filter, Input); - var expected = new List - { - "comedy", - "horror", - "18 restricted", - "japan", - "violence", - "manga", - "fantasy", - "shounen", - "alternative past", - "past", - }; - - _console.WriteLine( - "AniDB Internal: {0}, Art Style: {1}, Genre: {2}, Inverted: {3}, Misc: {4}, Plot: {5}, Programming: {6}, Setting: {7}, Source: {8}", filter.HasFlag(TagFilter.Filter.AnidbInternal), filter.HasFlag(TagFilter.Filter.ArtStyle), - filter.HasFlag(TagFilter.Filter.Genre), filter.HasFlag(TagFilter.Filter.Invert), filter.HasFlag(TagFilter.Filter.Misc), filter.HasFlag(TagFilter.Filter.Plot), filter.HasFlag(TagFilter.Filter.Programming), - filter.HasFlag(TagFilter.Filter.Setting), filter.HasFlag(TagFilter.Filter.Source) - ); - _console.WriteLine("Expected: [{0}]", string.Join(", ", expected)); - _console.WriteLine("Actual: [{0}]", string.Join(", ", actual)); - Assert.Equal(expected, actual); - } - - [Fact(DisplayName = "Source Exclusion with Full List")] - public void TestSourceFullList() - { - var actual = TagFilter.String.ProcessTags(TagFilter.Filter.Source, Input); - var expected = new List - { - "comedy", - "horror", - "18 restricted", - "large breasts", - "japan", - "violence", - "source material", - "fantasy", - "shounen", - "Earth", - "Asia", - "noitamina", - "cgi", - "3DCG", - "long episodes", - "first girl wins", - "alternative past", - "past", - }; - - _console.WriteLine("Expected: [{0}]", string.Join(", ", expected)); - _console.WriteLine("Actual: [{0}]", string.Join(", ", actual)); - Assert.Equal(expected, actual); - } - - [Fact(DisplayName = "Source Exclusion with Full List w/o Source")] - public void TestSourceFullListNoSource() - { - var actual = TagFilter.String.ProcessTags(TagFilter.Filter.Source, InputNoSource); - var expected = new List - { - "horror", - "18 restricted", - "large breasts", - "japan", - "violence", - "fantasy", - "shounen", - "Earth", - "Asia", - "noitamina", - "cgi", - "3DCG", - "long episodes", - "first girl wins", - }; - - _console.WriteLine("Expected: [{0}]", string.Join(", ", expected)); - _console.WriteLine("Actual: [{0}]", string.Join(", ", actual)); - Assert.Equal(expected, actual); - } - - [Fact(DisplayName = "Source Exclusion")] - public void TestSource() - { - Assert.Equal(new List(), TagFilter.String.ProcessTags(TagFilter.Filter.Source, new[] { "original work" })); - } - - [Fact(DisplayName = "Source Exclusion with No Source")] - public void TestSourceNoSource() - { - Assert.Equal(new List { "original work" }, TagFilter.String.ProcessTags(TagFilter.Filter.Genre, new List())); - } - - [Fact(DisplayName = "Source Inclusion with Source and Original Work")] - public void TestSourceInvertedDupeSource() - { - Assert.Equal(new List { "manga" }, TagFilter.String.ProcessTags(TagFilter.Filter.Source | TagFilter.Filter.Invert, new[] { "manga", "original work" })); - } - - [Fact(DisplayName = "Source Inclusion w/o Source")] - public void TestSourceInvertedNoSource() - { - Assert.Equal(new List { "original work" }, TagFilter.String.ProcessTags(TagFilter.Filter.Source | TagFilter.Filter.Invert, new[] { "action" })); - } - - [Fact(DisplayName = "Inverted w/o Source")] - public void TestInvertedNoSource() - { - Assert.Equal(new List { "action" }, TagFilter.String.ProcessTags(TagFilter.Filter.Genre | TagFilter.Filter.Invert, new[] { "action" })); - } - - [Fact(DisplayName = "AniDB Internal with Replacement Source")] - public void TestAniDBInternalWithSource() - { - Assert.Equal(new List { "original work" }, TagFilter.String.ProcessTags(TagFilter.Filter.AnidbInternal, new[] { "source material", "original work" })); - } - - [Fact(DisplayName = "AniDB Internal with Source")] - public void TestAniDBInternalWithTag() - { - Assert.Equal(new List(), TagFilter.String.ProcessTags(TagFilter.Filter.AnidbInternal | TagFilter.Filter.Source, new[] { "source material", "original work" })); - } - - [Fact(DisplayName = "AniDB Internal with Overlapping Source")] - public void TestAniDBInternalAndSourceWithOverlap() - { - Assert.Equal(new List { "action", "manga" }, TagFilter.String.ProcessTags(TagFilter.Filter.AnidbInternal, new[] { "action", "manga", "original work" })); - } - - [Fact(DisplayName = "Art Style Exclusion")] - public void TestArtStyle() - { - Assert.Equal(new List(), TagFilter.String.ProcessTags(TagFilter.Filter.ArtStyle | TagFilter.Filter.Source, new[] { "censored", "cgi" })); - } - - [Fact(DisplayName = "Plot Exclusion")] - public void TestPlot() - { - Assert.Equal(new List(), TagFilter.String.ProcessTags(TagFilter.Filter.Plot | TagFilter.Filter.Source, new[] { "everybody dies", "first girl wins" })); - } - - [Fact(DisplayName = "Settings Exclusion")] - public void TestSettings() - { - Assert.Equal(new List(), TagFilter.String.ProcessTags(TagFilter.Filter.Setting | TagFilter.Filter.Source, new[] { "meiji period", "meiji era", "japan", "high school" })); - } - - [Fact(DisplayName = "Misc Exclusion")] - public void TestMisc() - { - Assert.Equal(new List(), TagFilter.String.ProcessTags(TagFilter.Filter.Misc | TagFilter.Filter.Source, new[] { "previews suck" })); - } - - [Fact(DisplayName = "AniDB Internal Exclusion")] - public void TestAniDBInternal() - { - Assert.Equal( - new List(), - TagFilter.String.ProcessTags( - TagFilter.Filter.AnidbInternal | TagFilter.Filter.Source, - new[] - { - "old animetags", "missing frogs", "predominantly gay", "adapted into live action", "weekly monster", "needs removed", "to be merged", "to be improved", "to be split and deleted", - "to be moved", "to be split" - } - ) - ); - } - - [Fact(DisplayName = "Speed Test (Ideally <600ms on a good CPU)")] - public void TestSpeed() - { - const int Count = 4; - var times = new long[Count]; - for (var i = 0; i < Count; i++) - { - var stopwatch = Stopwatch.StartNew(); - Parallel.ForEach( - Enumerable.Range(0, 100000), new ParallelOptions() { MaxDegreeOfParallelism = 2 }, - _ => TagFilter.String.ProcessTags(TagFilter.Filter.Genre | TagFilter.Filter.AnidbInternal | TagFilter.Filter.Programming | TagFilter.Filter.Misc, Input) - ); - stopwatch.Stop(); - times[i] = stopwatch.ElapsedMilliseconds; - } - - _console.WriteLine("Average time is {0}ms", times.Average()); - Assert.True(times.Average() < 2000); - } - } -} +namespace Shoko.Tests +{ + public class TagFilterTest + { + private readonly ITestOutputHelper _console; + + public TagFilterTest(ITestOutputHelper console) + { + _console = console; + } + + private static IEnumerable Input => + new[] + { + "comedy", + "Comedy", + "horror", + "18 restricted", + "large breasts", + "japan", + "violence", + "source material", + "manga", + "fantasy", + "shounen", + "Earth", + "Asia", + "noitamina", + "cgi", + "3DCG", + "long episodes", + "first girl wins", + "alternative past", + "past", + }; + + private static IEnumerable InputNoSource => + new[] + { + "horror", + "Horror", + "18 restricted", + "large breasts", + "japan", + "violence", + "fantasy", + "shounen", + "Earth", + "Asia", + "noitamina", + "cgi", + "3DCG", + "long episodes", + "first girl wins", + }; + + [Fact(DisplayName = "Full Test")] + public void TestFullList() + { + var actual = TagFilter.String.ProcessTags(TagFilter.Filter.Genre, Input); + var expected = new List + { + "large breasts", + "japan", + "source material", + "manga", + "Earth", + "Asia", + "noitamina", + "cgi", + "3DCG", + "long episodes", + "first girl wins", + "alternative past", + "past", + }; + + _console.WriteLine("Expected: [{0}]", string.Join(", ", expected)); + _console.WriteLine("Actual: [{0}]", string.Join(", ", actual)); + Assert.Equal(expected, actual); + } + + [Fact(DisplayName = "Full Test w/ 'original work'")] + public void TestFullListWithNew() + { + var actual = TagFilter.String.ProcessTags(TagFilter.Filter.Genre, Input.Concat(new[] { "original work" })); + var expected = new List + { + "large breasts", + "japan", + "source material", + "manga", + "Earth", + "Asia", + "noitamina", + "cgi", + "3DCG", + "long episodes", + "first girl wins", + "alternative past", + "past", + }; + + _console.WriteLine("Expected: [{0}]", string.Join(", ", expected)); + _console.WriteLine("Actual: [{0}]", string.Join(", ", actual)); + Assert.Equal(expected, actual); + } + + [Fact(DisplayName = "Full Test w/o Source")] + public void TestFullListNoSource() + { + var actual = TagFilter.String.ProcessTags(TagFilter.Filter.Genre, InputNoSource); + var expected = new List + { + "large breasts", + "japan", + "Earth", + "Asia", + "noitamina", + "cgi", + "3DCG", + "long episodes", + "first girl wins", + "original work", + }; + + _console.WriteLine("Expected: [{0}]", string.Join(", ", expected)); + _console.WriteLine("Actual: [{0}]", string.Join(", ", actual)); + Assert.Equal(expected, actual); + } + + [Fact(DisplayName = "Full Test Inverted")] + public void TestFullListInverted() + { + var filter = TagFilter.Filter.Invert | TagFilter.Filter.Source | TagFilter.Filter.Genre | TagFilter.Filter.Setting; + var actual = TagFilter.String.ProcessTags(filter, Input); + var expected = new List + { + "comedy", + "horror", + "18 restricted", + "japan", + "violence", + "manga", + "fantasy", + "shounen", + "alternative past", + "past", + }; + + _console.WriteLine( + "AniDB Internal: {0}, Art Style: {1}, Genre: {2}, Inverted: {3}, Misc: {4}, Plot: {5}, Programming: {6}, Setting: {7}, Source: {8}", filter.HasFlag(TagFilter.Filter.AnidbInternal), filter.HasFlag(TagFilter.Filter.ArtStyle), + filter.HasFlag(TagFilter.Filter.Genre), filter.HasFlag(TagFilter.Filter.Invert), filter.HasFlag(TagFilter.Filter.Misc), filter.HasFlag(TagFilter.Filter.Plot), filter.HasFlag(TagFilter.Filter.Programming), + filter.HasFlag(TagFilter.Filter.Setting), filter.HasFlag(TagFilter.Filter.Source) + ); + _console.WriteLine("Expected: [{0}]", string.Join(", ", expected)); + _console.WriteLine("Actual: [{0}]", string.Join(", ", actual)); + Assert.Equal(expected, actual); + } + + [Fact(DisplayName = "Source Exclusion with Full List")] + public void TestSourceFullList() + { + var actual = TagFilter.String.ProcessTags(TagFilter.Filter.Source, Input); + var expected = new List + { + "comedy", + "horror", + "18 restricted", + "large breasts", + "japan", + "violence", + "source material", + "fantasy", + "shounen", + "Earth", + "Asia", + "noitamina", + "cgi", + "3DCG", + "long episodes", + "first girl wins", + "alternative past", + "past", + }; + + _console.WriteLine("Expected: [{0}]", string.Join(", ", expected)); + _console.WriteLine("Actual: [{0}]", string.Join(", ", actual)); + Assert.Equal(expected, actual); + } + + [Fact(DisplayName = "Source Exclusion with Full List w/o Source")] + public void TestSourceFullListNoSource() + { + var actual = TagFilter.String.ProcessTags(TagFilter.Filter.Source, InputNoSource); + var expected = new List + { + "horror", + "18 restricted", + "large breasts", + "japan", + "violence", + "fantasy", + "shounen", + "Earth", + "Asia", + "noitamina", + "cgi", + "3DCG", + "long episodes", + "first girl wins", + }; + + _console.WriteLine("Expected: [{0}]", string.Join(", ", expected)); + _console.WriteLine("Actual: [{0}]", string.Join(", ", actual)); + Assert.Equal(expected, actual); + } + + [Fact(DisplayName = "Source Exclusion")] + public void TestSource() + { + Assert.Equal(new List(), TagFilter.String.ProcessTags(TagFilter.Filter.Source, new[] { "original work" })); + } + + [Fact(DisplayName = "Source Exclusion with No Source")] + public void TestSourceNoSource() + { + Assert.Equal(new List { "original work" }, TagFilter.String.ProcessTags(TagFilter.Filter.Genre, new List())); + } + + [Fact(DisplayName = "Source Inclusion with Source and Original Work")] + public void TestSourceInvertedDupeSource() + { + Assert.Equal(new List { "manga" }, TagFilter.String.ProcessTags(TagFilter.Filter.Source | TagFilter.Filter.Invert, new[] { "manga", "original work" })); + } + + [Fact(DisplayName = "Source Inclusion w/o Source")] + public void TestSourceInvertedNoSource() + { + Assert.Equal(new List { "original work" }, TagFilter.String.ProcessTags(TagFilter.Filter.Source | TagFilter.Filter.Invert, new[] { "action" })); + } + + [Fact(DisplayName = "Inverted w/o Source")] + public void TestInvertedNoSource() + { + Assert.Equal(new List { "action" }, TagFilter.String.ProcessTags(TagFilter.Filter.Genre | TagFilter.Filter.Invert, new[] { "action" })); + } + + [Fact(DisplayName = "AniDB Internal with Replacement Source")] + public void TestAniDBInternalWithSource() + { + Assert.Equal(new List { "original work" }, TagFilter.String.ProcessTags(TagFilter.Filter.AnidbInternal, new[] { "source material", "original work" })); + } + + [Fact(DisplayName = "AniDB Internal with Source")] + public void TestAniDBInternalWithTag() + { + Assert.Equal(new List(), TagFilter.String.ProcessTags(TagFilter.Filter.AnidbInternal | TagFilter.Filter.Source, new[] { "source material", "original work" })); + } + + [Fact(DisplayName = "AniDB Internal with Overlapping Source")] + public void TestAniDBInternalAndSourceWithOverlap() + { + Assert.Equal(new List { "action", "manga" }, TagFilter.String.ProcessTags(TagFilter.Filter.AnidbInternal, new[] { "action", "manga", "original work" })); + } + + [Fact(DisplayName = "Art Style Exclusion")] + public void TestArtStyle() + { + Assert.Equal(new List(), TagFilter.String.ProcessTags(TagFilter.Filter.ArtStyle | TagFilter.Filter.Source, new[] { "censored", "cgi" })); + } + + [Fact(DisplayName = "Plot Exclusion")] + public void TestPlot() + { + Assert.Equal(new List(), TagFilter.String.ProcessTags(TagFilter.Filter.Plot | TagFilter.Filter.Source, new[] { "everybody dies", "first girl wins" })); + } + + [Fact(DisplayName = "Settings Exclusion")] + public void TestSettings() + { + Assert.Equal(new List(), TagFilter.String.ProcessTags(TagFilter.Filter.Setting | TagFilter.Filter.Source, new[] { "meiji period", "meiji era", "japan", "high school" })); + } + + [Fact(DisplayName = "Misc Exclusion")] + public void TestMisc() + { + Assert.Equal(new List(), TagFilter.String.ProcessTags(TagFilter.Filter.Misc | TagFilter.Filter.Source, new[] { "previews suck" })); + } + + [Fact(DisplayName = "AniDB Internal Exclusion")] + public void TestAniDBInternal() + { + Assert.Equal( + new List(), + TagFilter.String.ProcessTags( + TagFilter.Filter.AnidbInternal | TagFilter.Filter.Source, + new[] + { + "old animetags", "missing frogs", "predominantly gay", "adapted into live action", "weekly monster", "needs removed", "to be merged", "to be improved", "to be split and deleted", + "to be moved", "to be split" + } + ) + ); + } + } +} diff --git a/Shoko.Tests/Tasks/AutoAnimeGroupCalculatorTests.cs b/Shoko.Tests/Tasks/AutoAnimeGroupCalculatorTests.cs new file mode 100644 index 0000000000..3ca3a6b554 --- /dev/null +++ b/Shoko.Tests/Tasks/AutoAnimeGroupCalculatorTests.cs @@ -0,0 +1,381 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Shoko.Abstractions.Metadata; +using Shoko.Abstractions.Metadata.Enums; +using Shoko.Server.Tasks; +using Xunit; + +using AnimeRelation = Shoko.Server.Tasks.AutoAnimeGroupCalculator.AnimeRelation; +using RelationType = Shoko.Server.Tasks.AutoAnimeGroupCalculator.AnimeRelationType; + +namespace Shoko.Tests.Tasks; + +/// +/// Covers , which decides how a user's collection is carved +/// into groups. It is exercised entirely through the public constructor, which takes a prebuilt +/// relation lookup — the database-backed Create/CreateFromServerSettings factories are +/// thin adapters over that same constructor, so none of this needs a database or settings. +/// +public class AutoAnimeGroupCalculatorTests +{ + #region Helpers + + private sealed record Anime(int Id, string Title = "Alpha", AnimeType Type = AnimeType.TV, PartialDateOnly? AirDate = null); + + /// + /// Builds the two directed rows AniDB stores for a single relation, mirroring what the + /// AniDB_Anime_Relation query in produces. + /// + private static IEnumerable Link(Anime from, Anime to, RelationType forward, RelationType reverse) + { + yield return new AnimeRelation + { + FromId = from.Id, FromType = from.Type, FromMainTitle = from.Title, FromAirDate = from.AirDate, + ToId = to.Id, ToType = to.Type, ToMainTitle = to.Title, ToAirDate = to.AirDate, + RelationType = forward, + }; + yield return new AnimeRelation + { + FromId = to.Id, FromType = to.Type, FromMainTitle = to.Title, FromAirDate = to.AirDate, + ToId = from.Id, ToType = from.Type, ToMainTitle = from.Title, ToAirDate = from.AirDate, + RelationType = reverse, + }; + } + + private static IEnumerable Sequel(Anime earlier, Anime later) + => Link(earlier, later, RelationType.Sequel, RelationType.Prequel); + + private static IEnumerable SameSetting(Anime a, Anime b) + => Link(a, b, RelationType.SameSetting, RelationType.SameSetting); + + private static AutoAnimeGroupCalculator Calc( + IEnumerable relations, + AutoGroupExclude exclusions = AutoGroupExclude.None, + RelationType fuzzyTitleTest = RelationType.None, + MainAnimeSelectionStrategy strategy = MainAnimeSelectionStrategy.MinAirDate) + => new(relations.ToLookup(r => r.FromId), exclusions, fuzzyTitleTest, strategy); + + private static void AssertGrouped(AutoAnimeGroupCalculator calculator, int a, int b) + { + Assert.Equal(calculator.GetGroupAnimeId(a), calculator.GetGroupAnimeId(b)); + Assert.Contains(b, calculator.GetIdsOfAnimeInSameGroup(a)); + } + + private static void AssertNotGrouped(AutoAnimeGroupCalculator calculator, int a, int b) + { + Assert.NotEqual(calculator.GetGroupAnimeId(a), calculator.GetGroupAnimeId(b)); + Assert.DoesNotContain(b, calculator.GetIdsOfAnimeInSameGroup(a)); + } + + private static PartialDateOnly Year(int year) => new(year, 1, 1); + + #endregion + + #region Construction + + [Fact] + public void Constructor_Throws_WhenRelationMapIsNull() + => Assert.Throws(() => new AutoAnimeGroupCalculator( + null!, AutoGroupExclude.None, RelationType.None, MainAnimeSelectionStrategy.MinAirDate)); + + [Fact] + public void Exclusions_ExposesTheConfiguredValue() + { + var calculator = Calc([], AutoGroupExclude.SameSetting | AutoGroupExclude.Character); + + Assert.Equal(AutoGroupExclude.SameSetting | AutoGroupExclude.Character, calculator.Exclusions); + } + + #endregion + + #region Graph building + + [Fact] + public void GetGroupAnimeId_ReturnsTheAnimeItself_WhenItHasNoRelations() + { + var calculator = Calc([]); + + Assert.Equal(42, calculator.GetGroupAnimeId(42)); + } + + [Fact] + public void GetIdsOfAnimeInSameGroup_ReturnsOnlyTheAnime_WhenItHasNoRelations() + { + var calculator = Calc([]); + + Assert.Equal([42], calculator.GetIdsOfAnimeInSameGroup(42)); + } + + [Fact] + public void GetGroupAnimeId_GroupsDirectlyRelatedAnime() + { + var calculator = Calc(Sequel(new Anime(1, AirDate: Year(2000)), new Anime(2, AirDate: Year(2001)))); + + AssertGrouped(calculator, 1, 2); + } + + [Fact] + public void GetGroupAnimeId_GroupsTransitivelyRelatedAnime() + { + var first = new Anime(1, AirDate: Year(2000)); + var second = new Anime(2, AirDate: Year(2001)); + var third = new Anime(3, AirDate: Year(2002)); + var calculator = Calc([.. Sequel(first, second), .. Sequel(second, third)]); + + // 1 and 3 share no direct relation; they are only connected through 2. + AssertGrouped(calculator, 1, 3); + Assert.Equal([1, 2, 3], calculator.GetIdsOfAnimeInSameGroup(3).Order()); + } + + [Fact] + public void GetGroupAnimeId_TerminatesOnCyclicRelations() + { + var first = new Anime(1, AirDate: Year(2000)); + var second = new Anime(2, AirDate: Year(2001)); + var third = new Anime(3, AirDate: Year(2002)); + var calculator = Calc([.. Sequel(first, second), .. Sequel(second, third), .. Sequel(third, first)]); + + Assert.Equal([1, 2, 3], calculator.GetIdsOfAnimeInSameGroup(1).Order()); + } + + [Fact] + public void GetGroupAnimeId_HandlesSelfReferentialRelations() + { + var self = new Anime(1, AirDate: Year(2000)); + var calculator = Calc(Sequel(self, self)); + + Assert.Equal(1, calculator.GetGroupAnimeId(1)); + } + + [Fact] + public void GetGroupAnimeId_IsStableAcrossRepeatedCalls() + { + var calculator = Calc(Sequel(new Anime(7, AirDate: Year(2000)), new Anime(3, AirDate: Year(2001)))); + + var first = calculator.GetGroupAnimeId(3); + + // The second call is served from the memoised map rather than a rebuilt graph. + Assert.Equal(first, calculator.GetGroupAnimeId(3)); + Assert.Equal(first, calculator.GetGroupAnimeId(7)); + } + + #endregion + + #region MinAirDate selection strategy + + [Fact] + public void MinAirDate_SelectsTheEarliestAiringAnime() + { + var calculator = Calc( + Sequel(new Anime(9, AirDate: Year(1998)), new Anime(2, AirDate: Year(2005))), + strategy: MainAnimeSelectionStrategy.MinAirDate); + + // Chosen on air date, not on the lower ID. + Assert.Equal(9, calculator.GetGroupAnimeId(2)); + } + + [Fact] + public void MinAirDate_TreatsAMissingAirDateAsLastOfAll() + { + var calculator = Calc( + Sequel(new Anime(1, AirDate: null), new Anime(2, AirDate: Year(2005))), + strategy: MainAnimeSelectionStrategy.MinAirDate); + + Assert.Equal(2, calculator.GetGroupAnimeId(1)); + } + + #endregion + + #region Weighted selection strategy + + [Fact] + public void Weighted_PrefersTheAnimeWithASequel() + { + var calculator = Calc( + Sequel(new Anime(5), new Anime(1)), + strategy: MainAnimeSelectionStrategy.Weighted); + + // Both are TV (3 points); anime 5 additionally scores for having a sequel (+2), which beats + // the lowest-ID tiebreak that would otherwise pick anime 1. + Assert.Equal(5, calculator.GetGroupAnimeId(1)); + } + + [Fact] + public void Weighted_PrefersTvOverOva() + { + var calculator = Calc( + SameSetting(new Anime(9, Type: AnimeType.TV), new Anime(1, Type: AnimeType.OVA)), + strategy: MainAnimeSelectionStrategy.Weighted); + + // Both sides score one alternative version, so only the series type separates them. + Assert.Equal(9, calculator.GetGroupAnimeId(1)); + } + + [Fact] + public void Weighted_PrefersTvOverWeb() + { + var calculator = Calc( + SameSetting(new Anime(9, Type: AnimeType.TV), new Anime(1, Type: AnimeType.Web)), + strategy: MainAnimeSelectionStrategy.Weighted); + + Assert.Equal(9, calculator.GetGroupAnimeId(1)); + } + + [Fact] + public void Weighted_BreaksScoreTiesByLowestAnimeId() + { + var calculator = Calc( + SameSetting(new Anime(9, Type: AnimeType.TV), new Anime(4, Type: AnimeType.TV)), + strategy: MainAnimeSelectionStrategy.Weighted); + + Assert.Equal(4, calculator.GetGroupAnimeId(9)); + } + + #endregion + + #region Exclusions + + [Fact] + public void Exclusions_None_GroupsRelatedAnime() + { + var calculator = Calc(SameSetting(new Anime(1), new Anime(2)), AutoGroupExclude.None); + + AssertGrouped(calculator, 1, 2); + } + + [Fact] + public void Exclusions_SkipRelationsOfTheExcludedType() + { + var calculator = Calc(SameSetting(new Anime(1), new Anime(2)), AutoGroupExclude.SameSetting); + + AssertNotGrouped(calculator, 1, 2); + } + + [Fact] + public void Exclusions_OnlyApplyToTheExcludedRelationType() + { + // Excluding SameSetting must not disturb a prequel/sequel pair. + var calculator = Calc( + Sequel(new Anime(1, AirDate: Year(2000)), new Anime(2, AirDate: Year(2001))), + AutoGroupExclude.SameSetting); + + AssertGrouped(calculator, 1, 2); + } + + [Fact] + public void Exclusions_Movie_SkipsRelationsInvolvingAMovie() + { + var calculator = Calc( + Sequel(new Anime(1, Type: AnimeType.TV, AirDate: Year(2000)), new Anime(2, Type: AnimeType.Movie, AirDate: Year(2001))), + AutoGroupExclude.Movie); + + AssertNotGrouped(calculator, 1, 2); + } + + [Fact] + public void Exclusions_Ova_SkipsRelationsInvolvingAnOva() + { + var calculator = Calc( + Sequel(new Anime(1, Type: AnimeType.TV, AirDate: Year(2000)), new Anime(2, Type: AnimeType.OVA, AirDate: Year(2001))), + AutoGroupExclude.Ova); + + AssertNotGrouped(calculator, 1, 2); + } + + [Fact] + public void Exclusions_Movie_LeavesNonMovieRelationsAlone() + { + var calculator = Calc( + Sequel(new Anime(1, Type: AnimeType.TV, AirDate: Year(2000)), new Anime(2, Type: AnimeType.TV, AirDate: Year(2001))), + AutoGroupExclude.Movie); + + AssertGrouped(calculator, 1, 2); + } + + #endregion + + #region Fuzzy title matching + + /// + /// Builds a calculator whose only relation is a pair with + /// the given titles, with fuzzy title testing switched on for the secondary relation types. + /// + private static AutoAnimeGroupCalculator FuzzyCalc(string firstTitle, string secondTitle) + => Calc( + SameSetting(new Anime(1, firstTitle), new Anime(2, secondTitle)), + AutoGroupExclude.None, + RelationType.SecondaryRelations); + + [Fact] + public void FuzzyTitle_GroupsAnimeWithOverlappingTitles() + => AssertGrouped(FuzzyCalc("Fullmetal Alchemist", "Fullmetal Alchemist Brotherhood"), 1, 2); + + [Fact] + public void FuzzyTitle_DoesNotGroupAnimeWithUnrelatedTitles() + => AssertNotGrouped(FuzzyCalc("Naruto", "Bleach"), 1, 2); + + [Fact] + public void FuzzyTitle_IsNotAppliedToPrimaryRelationTypes() + { + // Prequel/Sequel is outside SecondaryRelations, so wildly different titles still group. + var calculator = Calc( + Sequel(new Anime(1, "Naruto", AirDate: Year(2000)), new Anime(2, "Bleach", AirDate: Year(2001))), + AutoGroupExclude.None, + RelationType.SecondaryRelations); + + AssertGrouped(calculator, 1, 2); + } + + [Fact] + public void FuzzyTitle_StripsTheMovieSuffixSoItCannotCreateAMatch() + { + // Were "The Movie" not stripped, the shared words would group two unrelated franchises. + AssertNotGrouped(FuzzyCalc("Bleach The Movie", "Naruto The Movie"), 1, 2); + } + + [Fact] + public void FuzzyTitle_StripsTheAnimationSuffixSoItCannotCreateAMatch() + => AssertNotGrouped(FuzzyCalc("Bleach The Animation", "Naruto The Animation"), 1, 2); + + [Fact] + public void FuzzyTitle_IgnoresTheGekijoubanPrefix() + => AssertGrouped(FuzzyCalc("Gekijouban Naruto", "Naruto"), 1, 2); + + [Fact] + public void FuzzyTitle_IgnoresDigits() + => AssertGrouped(FuzzyCalc("Gundam 00", "Gundam 2"), 1, 2); + + [Fact] + public void FuzzyTitle_TreatsHyphensAsWordSeparators() + => AssertGrouped(FuzzyCalc("Cowboy-Bebop", "Cowboy Bebop"), 1, 2); + + [Fact] + public void FuzzyTitle_DropsOtherPunctuationWithoutSplittingTheWord() + { + // "Cowboy/Bebop" collapses to the single token "CowboyBebop", which matches neither word. + AssertNotGrouped(FuzzyCalc("Cowboy/Bebop", "Cowboy Bebop"), 1, 2); + } + + [Fact] + public void FuzzyTitle_GroupsWhenMatchedCharactersReachFortyPercentOfTheShorterTitle() + { + // Two of six tokens match, which fails the "half the words" rule (2 < 3), but the matched + // characters (19) clear 40% of the shorter title (36 chars -> 14). + AssertGrouped(FuzzyCalc( + "Alphabetical Betamax Gamma Delta Epsilon Zeta", + "Alphabetical Betamax Eta Theta Iota Kappa"), 1, 2); + } + + [Fact] + public void FuzzyTitle_DoesNotGroupWhenNeitherTheWordNorCharacterThresholdIsMet() + { + // Same two-of-six token overlap, but the matched characters (9) fall short of 40% of the + // shorter title (26 chars -> 10). + AssertNotGrouped(FuzzyCalc( + "Alpha Beta Gamma Delta Epsilon Zeta", + "Alpha Beta Eta Theta Iota Kappa"), 1, 2); + } + + #endregion +} diff --git a/Shoko.Tests/TestDataTests.cs b/Shoko.Tests/TestDataTests.cs new file mode 100644 index 0000000000..3d7b6fdb6f --- /dev/null +++ b/Shoko.Tests/TestDataTests.cs @@ -0,0 +1,35 @@ +using System.Linq; +using Xunit; + +using Fixtures = Shoko.TestData.TestData; + +namespace Shoko.Tests; + +/// +/// Guards the shared fixtures in . Each accessor pulls a different embedded +/// resource, and because the two accessors are otherwise identical it is easy for one to end up +/// reading the other's file — which yields a silently empty or nonsensical collection rather than +/// an error. +/// +public class TestDataTests +{ + [Fact] + public void AniDBAnime_LoadsPopulatedRecords() + { + var anime = Fixtures.AniDB_Anime.Value.ToList(); + + Assert.NotEmpty(anime); + Assert.All(anime, a => Assert.NotEqual(0, a.AnimeID)); + } + + [Fact] + public void CrossRefFileEpisode_LoadsPopulatedRecords() + { + var crossRefs = Fixtures.CrossRef_File_Episode.Value.ToList(); + + Assert.NotEmpty(crossRefs); + // Reading the wrong resource still deserialises, but every field comes back at its default. + Assert.All(crossRefs, x => Assert.NotEqual(0, x.EpisodeID)); + Assert.All(crossRefs, x => Assert.False(string.IsNullOrEmpty(x.Hash))); + } +} diff --git a/Shoko.Tests/TestFilterable.cs b/Shoko.Tests/TestFilterable.cs index 7e39f26630..cb98d8893e 100644 --- a/Shoko.Tests/TestFilterable.cs +++ b/Shoko.Tests/TestFilterable.cs @@ -33,8 +33,8 @@ public class TestFilterable : IFilterableInfo public IReadOnlySet CustomTags { get; init; } = null!; public IReadOnlySet Years { get; init; } = null!; public IReadOnlySet<(int year, YearlySeason season)> Seasons { get; init; } = null!; - public IReadOnlySet AvailableImageTypes { get; } = null!; - public IReadOnlySet PreferredImageTypes { get; } = null!; + public IReadOnlySet AvailableImageTypes { get; init; } = null!; + public IReadOnlySet PreferredImageTypes { get; init; } = null!; public bool HasTmdbLink { get; init; } public bool HasTmdbAutoLinkingDisabled { get; init; } public bool HasMissingTmdbLink { get; init; } diff --git a/Shoko.Tests/TestServerSettings.cs b/Shoko.Tests/TestServerSettings.cs deleted file mode 100644 index 8a6a66a5b6..0000000000 --- a/Shoko.Tests/TestServerSettings.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System.Collections.Generic; -using Shoko.Abstractions.Metadata.Enums; -using Shoko.Server.Settings; - -namespace Shoko.Tests; - -public class TestServerSettings -{ - public ushort ServerPort { get; set; } = 8111; - public double PluginAutoWatchThreshold { get; set; } = 0.89; - public int CachingDatabaseTimeout { get; set; } = 180; - public string Culture { get; set; } = "en"; - public string WebUI_Settings { get; set; } = ""; - public bool FirstRun { get; set; } = true; - public int LegacyRenamerMaxEpisodeLength { get; set; } = 33; - public LoggingSettings Logging { get; set; } = new(); - public DatabaseSettings Database { get; set; } = new(); - public AniDbSettings AniDb { get; set; } = new(); - public TMDBSettings TMDB { get; set; } = new(); - public ImportSettings Import { get; set; } = new(); - public PlexSettings Plex { get; set; } = new(); - public PluginSettings Plugins { get; set; } = new(); - public bool AutoGroupSeries { get; set; } - public string AutoGroupSeriesRelationExclusions { get; set; } = "same setting|character"; - public bool AutoGroupSeriesUseScoreAlgorithm { get; set; } - public List LanguagePreference { get; set; } = new() { "x-jat", "en" }; - public string EpisodeLanguagePreference { get; set; } = string.Empty; - public bool LanguageUseSynonyms { get; set; } = true; - public int CloudWatcherTime { get; set; } = 3; - public DataSource EpisodeTitleSource { get; set; } = DataSource.AniDB; - public DataSource SeriesDescriptionSource { get; set; } = DataSource.AniDB; - public DataSource SeriesNameSource { get; set; } = DataSource.AniDB; - public string ImagesPath { get; set; } = null!; - public string UpdateChannel { get; set; } = "Stable"; - public LinuxSettings Linux { get; set; } = new(); -} diff --git a/Shoko.Tests/Utilities/PocoCacheTests.cs b/Shoko.Tests/Utilities/PocoCacheTests.cs new file mode 100644 index 0000000000..0b430571ef --- /dev/null +++ b/Shoko.Tests/Utilities/PocoCacheTests.cs @@ -0,0 +1,265 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using NutzCode.InMemoryIndex; +using Xunit; + +namespace Shoko.Tests.Utilities; + +/// +/// Covers and . +/// Every cached repository in the server keeps its rows in one of these and answers reads from the +/// secondary indexes, so a defect here silently corrupts lookups across the whole application. +/// +public class PocoCacheTests +{ + private sealed class Item(int id, string category, params string[] tags) + { + public int Id { get; set; } = id; + + public string Category { get; set; } = category; + + public IReadOnlyList Tags { get; set; } = tags; + } + + private static PocoCache Cache(params Item[] items) + => new(items, i => i.Id); + + #region Cache basics + + [Fact] + public void Get_ReturnsTheEntityForAKnownKey() + { + var item = new Item(1, "a"); + + Assert.Same(item, Cache(item).Get(1)); + } + + [Fact] + public void Get_ReturnsNullForAnUnknownKey() + => Assert.Null(Cache(new Item(1, "a")).Get(99)); + + [Fact] + public void GetAll_ReturnsEveryEntity() + { + var cache = Cache(new Item(1, "a"), new Item(2, "b")); + + Assert.Equal([1, 2], cache.GetAll().Select(i => i.Id).Order()); + } + + [Fact] + public void GetAllKeys_ReturnsEveryKey() + { + var cache = Cache(new Item(1, "a"), new Item(2, "b")); + + Assert.Equal([1, 2], cache.GetAllKeys().Order()); + } + + [Fact] + public void GetAll_ReturnsASnapshotThatDoesNotTrackLaterWrites() + { + var cache = Cache(new Item(1, "a")); + + var snapshot = cache.GetAll(); + cache.Update(new Item(2, "b")); + + Assert.Single(snapshot); + } + + [Fact] + public void Constructor_ThrowsWhenTheKeySelectorProducesDuplicates() + => Assert.Throws(() => Cache(new Item(1, "a"), new Item(1, "b"))); + + [Fact] + public void Update_AddsAnEntityThatWasNotPresent() + { + var cache = Cache(); + var item = new Item(1, "a"); + + cache.Update(item); + + Assert.Same(item, cache.Get(1)); + } + + [Fact] + public void Update_ReplacesTheEntityStoredUnderTheSameKey() + { + var cache = Cache(new Item(1, "a")); + var replacement = new Item(1, "b"); + + cache.Update(replacement); + + Assert.Same(replacement, cache.Get(1)); + Assert.Single(cache.GetAll()); + } + + [Fact] + public void Remove_DropsTheEntity() + { + var item = new Item(1, "a"); + var cache = Cache(item); + + cache.Remove(item); + + Assert.Null(cache.Get(1)); + Assert.Empty(cache.GetAll()); + } + + [Fact] + public void Clear_DropsEveryEntity() + { + var cache = Cache(new Item(1, "a"), new Item(2, "b")); + + cache.Clear(); + + Assert.Empty(cache.GetAll()); + } + + #endregion + + #region Single-valued index + + [Fact] + public void Index_IsPopulatedFromTheEntitiesPresentWhenItIsCreated() + { + var cache = Cache(new Item(1, "a"), new Item(2, "b")); + + var index = cache.CreateIndex(i => i.Category); + + Assert.Equal(1, index.GetOne("a")!.Id); + Assert.Equal(2, index.GetOne("b")!.Id); + } + + [Fact] + public void Index_GetOne_ReturnsNullForAnUnknownKey() + => Assert.Null(Cache(new Item(1, "a")).CreateIndex(i => i.Category).GetOne("zzz")); + + [Fact] + public void Index_GetMultiple_ReturnsEveryMatch() + { + var cache = Cache(new Item(1, "a"), new Item(2, "a"), new Item(3, "b")); + var index = cache.CreateIndex(i => i.Category); + + Assert.Equal([1, 2], index.GetMultiple("a").Select(i => i.Id).Order()); + } + + [Fact] + public void Index_GetMultiple_ReturnsEmptyForAnUnknownKey() + => Assert.Empty(Cache(new Item(1, "a")).CreateIndex(i => i.Category).GetMultiple("zzz")); + + #endregion + + #region Index maintenance + + [Fact] + public void Index_PicksUpAnEntityAddedAfterTheIndexWasCreated() + { + var cache = Cache(); + var index = cache.CreateIndex(i => i.Category); + + cache.Update(new Item(1, "a")); + + Assert.Equal(1, index.GetOne("a")!.Id); + } + + [Fact] + public void Index_MovesAnEntityWhenItsIndexedValueChanges() + { + var cache = Cache(new Item(1, "a")); + var index = cache.CreateIndex(i => i.Category); + + cache.Update(new Item(1, "b")); + + // The stale mapping must not survive, or lookups return entities that no longer match. + Assert.Null(index.GetOne("a")); + Assert.Equal(1, index.GetOne("b")!.Id); + } + + [Fact] + public void Index_DropsAnEntityThatIsRemovedFromTheCache() + { + var item = new Item(1, "a"); + var cache = Cache(item); + var index = cache.CreateIndex(i => i.Category); + + cache.Remove(item); + + Assert.Null(index.GetOne("a")); + Assert.Empty(index.GetMultiple("a")); + } + + [Fact] + public void Index_IsEmptiedWhenTheCacheIsCleared() + { + var cache = Cache(new Item(1, "a"), new Item(2, "b")); + var index = cache.CreateIndex(i => i.Category); + + cache.Clear(); + + Assert.Null(index.GetOne("a")); + Assert.Null(index.GetOne("b")); + } + + [Fact] + public void MultipleIndexesOverTheSameCacheAreAllMaintained() + { + var cache = Cache(new Item(1, "a")); + var byCategory = cache.CreateIndex(i => i.Category); + var byId = cache.CreateIndex(i => i.Id); + + cache.Update(new Item(1, "b")); + + Assert.Null(byCategory.GetOne("a")); + Assert.Equal(1, byCategory.GetOne("b")!.Id); + Assert.Equal("b", byId.GetOne(1)!.Category); + } + + #endregion + + #region Many-valued index + + [Fact] + public void MultiValuedIndex_IndexesAnEntityUnderEveryValue() + { + var cache = Cache(new Item(1, "a", "x", "y")); + var index = cache.CreateIndex(i => i.Tags); + + Assert.Equal(1, index.GetOne("x")!.Id); + Assert.Equal(1, index.GetOne("y")!.Id); + } + + [Fact] + public void MultiValuedIndex_ReturnsEveryEntitySharingAValue() + { + var cache = Cache(new Item(1, "a", "x"), new Item(2, "b", "x", "y")); + var index = cache.CreateIndex(i => i.Tags); + + Assert.Equal([1, 2], index.GetMultiple("x").Select(i => i.Id).Order()); + Assert.Equal([2], index.GetMultiple("y").Select(i => i.Id)); + } + + [Fact] + public void MultiValuedIndex_ReplacesTheWholeValueSetOnUpdate() + { + var cache = Cache(new Item(1, "a", "x", "y")); + var index = cache.CreateIndex(i => i.Tags); + + cache.Update(new Item(1, "a", "y", "z")); + + Assert.Null(index.GetOne("x")); + Assert.Equal(1, index.GetOne("y")!.Id); + Assert.Equal(1, index.GetOne("z")!.Id); + } + + [Fact] + public void MultiValuedIndex_HandlesAnEntityWithNoValues() + { + var cache = Cache(new Item(1, "a")); + var index = cache.CreateIndex(i => i.Tags); + + Assert.Empty(index.GetMultiple("x")); + Assert.Equal([1], cache.GetAllKeys()); + } + + #endregion +} diff --git a/Shoko.Tests/Utilities/ReflectionUtilsTests.cs b/Shoko.Tests/Utilities/ReflectionUtilsTests.cs new file mode 100644 index 0000000000..8e2955c79e --- /dev/null +++ b/Shoko.Tests/Utilities/ReflectionUtilsTests.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Concurrent; +using System.Linq; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Moq; +using Shoko.Abstractions.Filtering.Expressions; +using Shoko.Server.Utilities; +using Xunit; + +namespace Shoko.Tests.Utilities; + +/// +/// , which every type scan in the server runs over. +/// +/// +/// Thirteen of them look up job types, subtitle providers, filter expressions and mapped entities this +/// way. throws on an +/// assembly still being written to, which surfaced as two unrelated CI failures before the scan +/// learned to skip them. +/// +public class ReflectionUtilsTests +{ + [Fact] + public void TheServerAssemblyIsScanned() + => Assert.Contains(typeof(ReflectionUtils).Assembly, ReflectionUtils.ScannableAssemblies()); + + [Fact] + public void RuntimeEmittedAssembliesAreNotScanned() + { + GC.KeepAlive(new Mock().Object); + + Assert.DoesNotContain(ReflectionUtils.ScannableAssemblies(), assembly => assembly.IsDynamic); + } + + [Fact] + public async Task TheScanSurvivesAssembliesBeingEmittedAlongsideIt() + { + // Mocking distinct interfaces makes Castle emit a new proxy type for each rather than serve a + // cache, which is what a full test run does across its classes. + var interfaces = typeof(FilterExpression).Assembly.GetTypes() + .Where(type => type.IsInterface && type.IsPublic && !type.ContainsGenericParameters) + .Take(200) + .ToArray(); + Assert.NotEmpty(interfaces); + + // Ends the loops; deliberately not passed to Task.Run, where a task not yet scheduled when + // it fires would come back cancelled rather than having run at all. + using var stop = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var failures = new ConcurrentBag(); + + var emitting = Enumerable.Range(0, 4).Select(worker => Task.Run(() => + { + foreach (var type in interfaces.Skip(worker)) + { + if (stop.IsCancellationRequested) + return; + + try + { + GC.KeepAlive(((Mock)Activator.CreateInstance(typeof(Mock<>).MakeGenericType(type))!).Object); + } + catch + { + // Not every interface can be proxied, and only the emitting matters here. + } + } + })).ToArray(); + + var scanning = Enumerable.Range(0, 4).Select(_ => Task.Run(() => + { + while (!stop.IsCancellationRequested) + { + try + { + _ = ReflectionUtils.ScannableAssemblies().SelectMany(assembly => assembly.GetTypes()).Count(); + } + catch (Exception exception) + { + failures.Add(exception.GetType().Name); + } + } + })).ToArray(); + + await Task.WhenAll(emitting); + await stop.CancelAsync(); + await Task.WhenAll(scanning); + + Assert.Equal(string.Empty, string.Join(", ", failures.GroupBy(f => f).Select(g => $"{g.Count()}x {g.Key}"))); + } +} diff --git a/scripts/compare_schemas.sh b/scripts/compare_schemas.sh new file mode 100755 index 0000000000..a0e7197d52 --- /dev/null +++ b/scripts/compare_schemas.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# Migrates a real database of each supported backend and compares the three schemas against each +# other. This is what CI does across its backend matrix; here it runs on one machine, with MariaDB +# and SQL Server in Docker. +# +# Needs docker, the .NET SDK, mediainfo and librhash-dev. + +set -euo pipefail + +cd "$(dirname "$0")/.." + +export SHOKO_SCHEMA_DIR="${SHOKO_SCHEMA_DIR:-$(mktemp -d)}" +MYSQL_PASS=root +MSSQL_PASS='ShokoTest1!' +KEEP="${KEEP_CONTAINERS:-0}" + +cleanup() { + [ "$KEEP" = "1" ] || docker rm -f shoko-schema-maria shoko-schema-mssql >/dev/null 2>&1 || true +} +trap cleanup EXIT + +echo "==> starting databases" +docker rm -f shoko-schema-maria shoko-schema-mssql >/dev/null 2>&1 || true +docker run -d --name shoko-schema-maria -e MARIADB_ROOT_PASSWORD="$MYSQL_PASS" -e MARIADB_DATABASE=shoko \ + -p 3306:3306 mariadb:lts >/dev/null +docker run -d --name shoko-schema-mssql -e SA_PASSWORD="$MSSQL_PASS" -e ACCEPT_EULA=Y -e MSSQL_PID=Express \ + -p 1433:1433 mcr.microsoft.com/mssql/server:2022-latest >/dev/null + +echo -n "==> waiting for them to accept connections" +for _ in $(seq 1 60); do + if docker exec shoko-schema-maria mariadb -uroot -p"$MYSQL_PASS" -e "SELECT 1" >/dev/null 2>&1 && + docker exec shoko-schema-mssql /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P "$MSSQL_PASS" -Q "SELECT 1" -No >/dev/null 2>&1; then + echo " ok" + break + fi + echo -n "." + sleep 2 +done + +# Every backend has to start empty, or the dump describes a schema nobody will ever migrate into. +# The database is created without naming a collation, the way CI's MARIADB_DATABASE creates it and +# the way anyone who creates one outside Shoko will: it then keeps the server's default, which is +# what makes MySQLFixUTF8MB4 convert columns it leaves alone in a database Shoko created itself. +docker exec shoko-schema-maria mariadb -uroot -p"$MYSQL_PASS" \ + -e "DROP DATABASE IF EXISTS shoko; CREATE DATABASE shoko;" >/dev/null +docker exec shoko-schema-mssql /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P "$MSSQL_PASS" -No \ + -Q "IF DB_ID('shoko') IS NOT NULL BEGIN ALTER DATABASE shoko SET SINGLE_USER WITH ROLLBACK IMMEDIATE; DROP DATABASE shoko; END; CREATE DATABASE shoko" >/dev/null + +dump() { + echo "==> migrating $1" + DB_TYPE="$1" DB_HOST=127.0.0.1 DB_USER="${2:-}" DB_PASS="${3:-}" DB_NAME=shoko \ + dotnet test Shoko.IntegrationTests/Shoko.IntegrationTests.csproj -c Release \ + --filter "FullyQualifiedName~SchemaSnapshotTests" --nologo +} + +dump SQLite +dump MySQL root "$MYSQL_PASS" +dump SQLServer sa "$MSSQL_PASS" + +echo "==> comparing" +dotnet test Shoko.Tests/Shoko.Tests.csproj -c Release --filter "FullyQualifiedName~SchemaTypeParityTests" --nologo