From 2f71d6e2286a8a04a01024ef1fe1e7cee25ccc56 Mon Sep 17 00:00:00 2001 From: krbrs <57227244+krbrs@users.noreply.github.com> Date: Thu, 21 May 2026 08:40:04 +0200 Subject: [PATCH 01/13] feat(cli): add mssql to sqlite database migrator --- Shoko.CLI/DatabaseConverterCommand.cs | 940 ++++++++++++++++++++++++++ Shoko.CLI/Program.cs | 7 +- 2 files changed, 946 insertions(+), 1 deletion(-) create mode 100644 Shoko.CLI/DatabaseConverterCommand.cs diff --git a/Shoko.CLI/DatabaseConverterCommand.cs b/Shoko.CLI/DatabaseConverterCommand.cs new file mode 100644 index 0000000000..6797b7c9ce --- /dev/null +++ b/Shoko.CLI/DatabaseConverterCommand.cs @@ -0,0 +1,940 @@ +using System; +using System.Collections.Generic; +using System.Data; +using System.Data.Common; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Data.SqlClient; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Shoko.Abstractions.Plugin.Models; +using Shoko.Abstractions.Extensions; +using Shoko.Server.Databases; +using Shoko.Server.Plugin; +using Shoko.Server.Repositories; +using Shoko.Server.Services; +using Shoko.Server.Utilities; + +namespace Shoko.CLI; + +internal static class DatabaseConverterCommand +{ + private static readonly HashSet ExcludedTables = new(StringComparer.OrdinalIgnoreCase) + { + "Versions", + }; + + private static readonly HashSet LegacyMigratedTables = new(StringComparer.OrdinalIgnoreCase) + { + "AniDB_Vote", + "CrossRef_AniDB_TvDBV2", + }; + + public static async Task RunAsync(string[] args) + { + var options = ParseArgs(args); + if (options.ShowHelp) + { + PrintUsage(); + return 0; + } + + if (string.IsNullOrWhiteSpace(options.SourceConnectionString) || string.IsNullOrWhiteSpace(options.TargetFile)) + { + PrintUsage(); + return 1; + } + + try + { + await ConvertAsync(options.SourceConnectionString, options.TargetFile, options.Overwrite); + return 0; + } + catch (Exception ex) + { + Console.Error.WriteLine($"Database conversion failed: {ex.Message}"); + Console.Error.WriteLine(ex); + return 1; + } + } + + private static async Task ConvertAsync(string sourceConnectionString, string targetFile, bool overwrite) + { + var fullTargetPath = Path.GetFullPath(targetFile); + if (File.Exists(fullTargetPath)) + { + if (!overwrite) + { + throw new InvalidOperationException($"Target file already exists: {fullTargetPath}. Use --overwrite to replace it."); + } + + File.Delete(fullTargetPath); + } + + var targetDirectory = Path.GetDirectoryName(fullTargetPath); + if (!string.IsNullOrWhiteSpace(targetDirectory)) + { + Directory.CreateDirectory(targetDirectory); + } + + var targetConnectionString = new SqliteConnectionStringBuilder + { + DataSource = fullTargetPath, + Mode = SqliteOpenMode.ReadWriteCreate, + Pooling = false, + }.ToString(); + + await InitializeSqliteDatabaseAsync(targetConnectionString); + + await using var source = new SqlConnection(sourceConnectionString); + await source.OpenAsync(); + await using var target = new SqliteConnection(targetConnectionString); + await target.OpenAsync(); + + await ConfigureSqliteAsync(target); + + var sourceTables = await GetSqlServerTablesAsync(source); + await PruneLegacyTargetTablesAsync(source, target, sourceTables); + var targetTables = await GetSqliteTablesAsync(target); + var tablesToCopy = targetTables + .Where(sourceTables.Contains) + .Where(tableName => !ExcludedTables.Contains(tableName)) + .ToList(); + var sourceOnlyTables = sourceTables + .Except(targetTables) + .OrderBy(a => a, StringComparer.OrdinalIgnoreCase) + .ToList(); + var targetOnlyTables = targetTables + .Except(sourceTables) + .Where(tableName => !ExcludedTables.Contains(tableName)) + .OrderBy(a => a, StringComparer.OrdinalIgnoreCase) + .ToList(); + + Console.WriteLine($"Source tables found: {sourceTables.Count}"); + Console.WriteLine($"Target tables found: {targetTables.Count}"); + Console.WriteLine($"Tables to copy: {tablesToCopy.Count}"); + + if (sourceOnlyTables.Count > 0) + { + Console.WriteLine("Source-only tables not present in the generated SQLite schema:"); + foreach (var skippedTable in sourceOnlyTables) + { + Console.WriteLine($" {skippedTable}"); + } + } + + if (targetOnlyTables.Count > 0) + { + Console.WriteLine("Target-only tables missing from the source database:"); + foreach (var targetOnlyTable in targetOnlyTables) + { + Console.WriteLine($" {targetOnlyTable}"); + } + + await ReportSqlServerObjectsAsync(source, targetOnlyTables); + } + + if (ExcludedTables.Count > 0) + { + Console.WriteLine("Excluded control tables:"); + foreach (var excludedTable in ExcludedTables.OrderBy(a => a, StringComparer.OrdinalIgnoreCase)) + { + Console.WriteLine($" {excludedTable}"); + } + } + + foreach (var tableName in tablesToCopy) + { + await CopyTableAsync(source, target, tableName); + } + + await VerifyCopyAsync(source, target, tablesToCopy); + Console.WriteLine($"Conversion completed successfully: {fullTargetPath}"); + } + + private static async Task InitializeSqliteDatabaseAsync(string connectionString) + { + await using var bootstrapHome = new TemporaryShokoHomeScope(); + + var systemService = new SystemService(); + var settings = Utils.SettingsProvider.GetSettings(); + settings.Database.Type = Shoko.Server.Server.Constants.DatabaseType.SQLite; + settings.Database.OverrideConnectionString = connectionString; + var pluginManager = GetRequiredPrivateField(systemService, "_pluginManager"); + InitializeCorePluginOnly(pluginManager, systemService); + + using var host = CreateBootstrapHost(systemService, settings); + Utils.ServiceContainer = host.Services; + pluginManager.InitPlugins(); + + var databaseFactory = host.Services.GetRequiredService(); + var repositoryFactory = host.Services.GetRequiredService(); + if (!RunInitializeDatabase(systemService, databaseFactory, repositoryFactory)) + { + throw new InvalidOperationException(systemService.StartupMessage ?? "Shoko database bootstrap failed."); + } + + databaseFactory.CloseSessionFactory(); + } + + private static async Task ConfigureSqliteAsync(SqliteConnection connection) + { + var commands = new[] + { + "PRAGMA foreign_keys = OFF;", + "PRAGMA journal_mode = WAL;", + "PRAGMA synchronous = OFF;", + }; + + foreach (var commandText in commands) + { + await using var command = connection.CreateCommand(); + command.CommandText = commandText; + await command.ExecuteNonQueryAsync(); + } + } + + private static async Task CopyTableAsync(SqlConnection source, SqliteConnection target, string tableName) + { + var sourceColumns = await GetSqlServerColumnsAsync(source, tableName); + var targetColumns = await GetSqliteColumnsAsync(target, tableName); + var sourceColumnSet = sourceColumns.ToHashSet(StringComparer.OrdinalIgnoreCase); + var sourceBackedColumns = targetColumns.Where(column => sourceColumnSet.Contains(column.Name)).ToList(); + var fallbackColumns = targetColumns + .Where(column => !sourceColumnSet.Contains(column.Name) && column.NotNull && string.IsNullOrWhiteSpace(column.DefaultValue)) + .ToList(); + var insertColumns = sourceBackedColumns + .Concat(fallbackColumns) + .ToList(); + + if (insertColumns.Count == 0) + { + Console.WriteLine($"Skipping {tableName}: no shared columns."); + return; + } + + if (fallbackColumns.Count > 0) + { + Console.WriteLine($"Applying fallback values for {tableName}: {string.Join(", ", fallbackColumns.Select(column => column.Name))}"); + } + + await using var transaction = target.BeginTransaction(); + await using (var deleteCommand = target.CreateCommand()) + { + deleteCommand.Transaction = transaction; + deleteCommand.CommandText = $"DELETE FROM {QuoteSqliteIdentifier(tableName)};"; + await deleteCommand.ExecuteNonQueryAsync(); + } + + var selectSql = $"SELECT {string.Join(", ", sourceBackedColumns.Select(column => QuoteSqlServerIdentifier(column.Name)))} FROM {QuoteSqlServerIdentifier(tableName)};"; + await using var selectCommand = new SqlCommand(selectSql, source); + await using var reader = await selectCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess); + + await using var insertCommand = target.CreateCommand(); + insertCommand.Transaction = transaction; + insertCommand.CommandText = + $"INSERT INTO {QuoteSqliteIdentifier(tableName)} ({string.Join(", ", insertColumns.Select(column => QuoteSqliteIdentifier(column.Name)))}) VALUES ({string.Join(", ", insertColumns.Select((_, index) => $"@p{index}"))});"; + + for (var index = 0; index < insertColumns.Count; index++) + { + insertCommand.Parameters.Add(new SqliteParameter($"@p{index}", DbType.Object)); + } + + var rowCount = 0; + while (await reader.ReadAsync()) + { + for (var index = 0; index < sourceBackedColumns.Count; index++) + { + var value = await reader.IsDBNullAsync(index) ? DBNull.Value : reader.GetValue(index); + insertCommand.Parameters[index].Value = NormalizeValue(value); + } + + for (var index = 0; index < fallbackColumns.Count; index++) + { + var column = fallbackColumns[index]; + insertCommand.Parameters[sourceBackedColumns.Count + index].Value = GetFallbackValue(tableName, column); + } + + await insertCommand.ExecuteNonQueryAsync(); + rowCount++; + } + + await transaction.CommitAsync(); + Console.WriteLine($"Copied {tableName}: {rowCount} rows, {insertColumns.Count} columns."); + } + + private static async Task VerifyCopyAsync(SqlConnection source, SqliteConnection target, IReadOnlyList tablesToCopy) + { + Console.WriteLine("Verifying migrated data..."); + + foreach (var tableName in tablesToCopy) + { + var sourceColumns = await GetSqlServerColumnsAsync(source, tableName); + var targetColumns = await GetSqliteColumnsAsync(target, tableName); + var sharedColumns = targetColumns + .Where(column => sourceColumns.Contains(column.Name, StringComparer.OrdinalIgnoreCase)) + .Select(column => column.Name) + .ToList(); + + if (sharedColumns.Count == 0) + { + Console.WriteLine($"Verified {tableName}: skipped, no shared columns."); + continue; + } + + var sourceCount = await GetRowCountAsync(source, tableName); + var targetCount = await GetRowCountAsync(target, tableName); + if (sourceCount != targetCount) + { + throw new InvalidOperationException($"Verification failed for {tableName}: row count mismatch. Source={sourceCount}, Target={targetCount}."); + } + + await VerifyTableContentAsync(source, target, tableName, sharedColumns); + + Console.WriteLine($"Verified {tableName}: {sourceCount} rows, {sharedColumns.Count} shared columns."); + } + } + + private static async Task GetRowCountAsync(SqlConnection connection, string tableName) + { + await using var command = connection.CreateCommand(); + command.CommandText = $"SELECT COUNT(*) FROM {QuoteSqlServerIdentifier(tableName)};"; + return Convert.ToInt64(await command.ExecuteScalarAsync(), CultureInfo.InvariantCulture); + } + + private static async Task GetRowCountAsync(SqliteConnection connection, string tableName) + { + await using var command = connection.CreateCommand(); + command.CommandText = $"SELECT COUNT(*) FROM {QuoteSqliteIdentifier(tableName)};"; + return Convert.ToInt64(await command.ExecuteScalarAsync(), CultureInfo.InvariantCulture); + } + + private static async Task VerifyTableContentAsync(SqlConnection source, SqliteConnection target, string tableName, IReadOnlyList sharedColumns) + { + var sourcePrimaryKeys = await GetSqlServerPrimaryKeyColumnsAsync(source, tableName); + var targetPrimaryKeys = await GetSqlitePrimaryKeyColumnsAsync(target, tableName); + var sourceColumnTypes = await GetSqlServerColumnTypesAsync(source, tableName); + var orderColumns = sourcePrimaryKeys + .Where(column => targetPrimaryKeys.Contains(column, StringComparer.OrdinalIgnoreCase)) + .Where(column => sharedColumns.Contains(column, StringComparer.OrdinalIgnoreCase)) + .ToList(); + + if (orderColumns.Count == 0) + { + orderColumns = sharedColumns + .Where(column => sourceColumnTypes.TryGetValue(column, out var type) && IsSqlServerOrderableType(type)) + .ToList(); + } + + if (orderColumns.Count == 0) + { + throw new InvalidOperationException($"Verification failed for {tableName}: no primary key or SQL-orderable shared columns are available for deterministic comparison."); + } + + var sourceSql = BuildSqlServerOrderedSelectSql(tableName, sharedColumns, orderColumns, sourceColumnTypes); + var targetSql = BuildSqliteOrderedSelectSql(tableName, sharedColumns, orderColumns); + + await using var sourceCommand = new SqlCommand(sourceSql, source); + await using var sourceReader = await sourceCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess); + await using var targetCommand = target.CreateCommand(); + targetCommand.CommandText = targetSql; + await using var targetReader = await targetCommand.ExecuteReaderAsync(); + + var rowNumber = 0L; + while (true) + { + var sourceHasRow = await sourceReader.ReadAsync(); + var targetHasRow = await targetReader.ReadAsync(); + if (sourceHasRow != targetHasRow) + { + throw new InvalidOperationException($"Verification failed for {tableName}: row stream length mismatch near row {rowNumber + 1}."); + } + + if (!sourceHasRow) + { + return; + } + + rowNumber++; + var sourceRow = new string[sharedColumns.Count]; + var targetRow = new string[sharedColumns.Count]; + for (var index = 0; index < sharedColumns.Count; index++) + { + sourceRow[index] = NormalizeComparisonValue(await sourceReader.IsDBNullAsync(index) ? null : sourceReader.GetValue(index)); + targetRow[index] = NormalizeComparisonValue(targetReader.IsDBNull(index) ? null : targetReader.GetValue(index)); + } + + for (var index = 0; index < sharedColumns.Count; index++) + { + var sourceValue = sourceRow[index]; + var targetValue = targetRow[index]; + if (string.Equals(sourceValue, targetValue, StringComparison.Ordinal)) + { + continue; + } + + var keyDescription = string.Join(", ", orderColumns.Select(column => + { + var keyIndex = sharedColumns.IndexOf(column); + var keyValue = keyIndex >= 0 ? sourceRow[keyIndex] : ""; + return $"{column}={keyValue}"; + })); + throw new InvalidOperationException($"Verification failed for {tableName}: column {sharedColumns[index]} mismatch at row {rowNumber} ({keyDescription}). Source={sourceValue}, Target={targetValue}."); + } + } + } + + private static void AppendValue(List buffer, object? value) + { + switch (value) + { + case null: + case DBNull: + buffer.AddRange("NULL"u8.ToArray()); + break; + case byte[] bytes: + buffer.AddRange(Convert.ToHexString(bytes).Select(c => (byte)c)); + break; + case string text when TryNormalizeTemporalString(text, out var normalizedText): + buffer.AddRange(Encoding.UTF8.GetBytes(normalizedText)); + break; + case DateTime dateTime: + buffer.AddRange(Encoding.UTF8.GetBytes(NormalizeDateTime(dateTime))); + break; + case DateTimeOffset offset: + buffer.AddRange(Encoding.UTF8.GetBytes(offset.ToString("O", CultureInfo.InvariantCulture))); + break; + case bool boolean: + buffer.AddRange(boolean ? "1"u8.ToArray() : "0"u8.ToArray()); + break; + case IFormattable formattable: + buffer.AddRange(Encoding.UTF8.GetBytes(formattable.ToString(null, CultureInfo.InvariantCulture) ?? string.Empty)); + break; + default: + buffer.AddRange(Encoding.UTF8.GetBytes(value.ToString() ?? string.Empty)); + break; + } + } + + private static object NormalizeValue(object value) + { + return value switch + { + DBNull => DBNull.Value, + Guid guid => guid.ToString(), + DateTimeOffset offset => offset.UtcDateTime, + _ => value, + }; + } + + private static string NormalizeDateTime(DateTime value) + { + if (value.TimeOfDay == TimeSpan.Zero) + { + return value.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); + } + + return value.ToString("O", CultureInfo.InvariantCulture); + } + + private static bool TryNormalizeTemporalString(string value, out string normalized) + { + if (DateTime.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var dateTime)) + { + normalized = NormalizeDateTime(dateTime); + return true; + } + + if (DateOnly.TryParseExact(value, ["yyyy-MM-dd"], CultureInfo.InvariantCulture, DateTimeStyles.None, out var dateOnly)) + { + normalized = dateOnly.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture); + return true; + } + + normalized = string.Empty; + return false; + } + + private static async Task> GetSqlServerTablesAsync(SqlConnection connection) + { + const string sql = """ + SELECT TABLE_NAME + FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_TYPE = 'BASE TABLE' + """; + + var tables = new HashSet(StringComparer.OrdinalIgnoreCase); + await using var command = new SqlCommand(sql, connection); + await using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + tables.Add(reader.GetString(0)); + } + + return tables; + } + + private static async Task> GetSqliteTablesAsync(SqliteConnection connection) + { + const string sql = """ + SELECT name + FROM sqlite_master + WHERE type = 'table' + AND name NOT LIKE 'sqlite_%' + ORDER BY name + """; + + var tables = new List(); + await using var command = connection.CreateCommand(); + command.CommandText = sql; + await using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + tables.Add(reader.GetString(0)); + } + + return tables; + } + + private static async Task> GetSqlServerColumnsAsync(SqlConnection connection, string tableName) + { + const string sql = """ + SELECT COLUMN_NAME + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_NAME = @tableName + ORDER BY ORDINAL_POSITION + """; + + var columns = new List(); + await using var command = new SqlCommand(sql, connection); + command.Parameters.AddWithValue("@tableName", tableName); + await using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + columns.Add(reader.GetString(0)); + } + + return columns; + } + + private static async Task> GetSqlServerPrimaryKeyColumnsAsync(SqlConnection connection, string tableName) + { + const string sql = """ + SELECT COLUMN_NAME + FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE + WHERE OBJECTPROPERTY(OBJECT_ID(CONSTRAINT_SCHEMA + '.' + QUOTENAME(CONSTRAINT_NAME)), 'IsPrimaryKey') = 1 + AND TABLE_NAME = @tableName + ORDER BY ORDINAL_POSITION + """; + + var columns = new List(); + await using var command = new SqlCommand(sql, connection); + command.Parameters.AddWithValue("@tableName", tableName); + await using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + columns.Add(reader.GetString(0)); + } + + return columns; + } + + private static async Task> GetSqlServerColumnTypesAsync(SqlConnection connection, string tableName) + { + const string sql = """ + SELECT COLUMN_NAME, DATA_TYPE + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_NAME = @tableName + """; + + var columns = new Dictionary(StringComparer.OrdinalIgnoreCase); + await using var command = new SqlCommand(sql, connection); + command.Parameters.AddWithValue("@tableName", tableName); + await using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + columns[reader.GetString(0)] = reader.GetString(1); + } + + return columns; + } + + private static async Task> GetSqlitePrimaryKeyColumnsAsync(SqliteConnection connection, string tableName) + { + var columns = new List<(int Position, string Name)>(); + await using var command = connection.CreateCommand(); + command.CommandText = $"PRAGMA table_info({QuoteSqliteLiteral(tableName)});"; + await using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + var position = reader.GetInt32(5); + if (position > 0) + { + columns.Add((position, reader.GetString(1))); + } + } + + return columns.OrderBy(column => column.Position).Select(column => column.Name).ToList(); + } + + private static async Task ReportSqlServerObjectsAsync(SqlConnection connection, IReadOnlyList objectNames) + { + const string sql = """ + SELECT s.name AS SchemaName, o.name AS ObjectName, o.type AS ObjectType, o.type_desc AS ObjectTypeDescription + FROM sys.objects o + INNER JOIN sys.schemas s ON s.schema_id = o.schema_id + WHERE o.name = @objectName + ORDER BY s.name, o.type_desc + """; + + Console.WriteLine("Source object lookup for target-only names:"); + foreach (var objectName in objectNames) + { + await using var command = new SqlCommand(sql, connection); + command.Parameters.AddWithValue("@objectName", objectName); + await using var reader = await command.ExecuteReaderAsync(); + if (!reader.HasRows) + { + Console.WriteLine($" {objectName}: not found in sys.objects"); + continue; + } + + while (await reader.ReadAsync()) + { + Console.WriteLine($" {objectName}: schema={reader.GetString(0)}, type={reader.GetString(2)}, type_desc={reader.GetString(3)}"); + } + } + } + + private static object GetFallbackValue(string tableName, SqliteColumnInfo column) + { + if (string.Equals(tableName, "CrossRef_File_Episode", StringComparison.OrdinalIgnoreCase) && + string.Equals(column.Name, "CrossRefSource", StringComparison.OrdinalIgnoreCase)) + { + return 1; + } + + var normalizedType = (column.Type ?? string.Empty).ToUpperInvariant(); + if (normalizedType.Contains("INT", StringComparison.Ordinal) || + normalizedType.Contains("REAL", StringComparison.Ordinal) || + normalizedType.Contains("NUM", StringComparison.Ordinal) || + normalizedType.Contains("DEC", StringComparison.Ordinal) || + normalizedType.Contains("BOOL", StringComparison.Ordinal)) + { + return 0; + } + + if (normalizedType.Contains("DATE", StringComparison.Ordinal) || normalizedType.Contains("TIME", StringComparison.Ordinal)) + { + return new DateTime(2000, 1, 1, 0, 0, 0, DateTimeKind.Utc); + } + + if (normalizedType.Contains("BLOB", StringComparison.Ordinal)) + { + return Array.Empty(); + } + + return string.Empty; + } + + private static async Task> GetSqliteColumnsAsync(SqliteConnection connection, string tableName) + { + var columns = new List(); + await using var command = connection.CreateCommand(); + command.CommandText = $"PRAGMA table_info({QuoteSqliteLiteral(tableName)});"; + await using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + columns.Add(new() + { + Name = reader.GetString(1), + Type = reader.IsDBNull(2) ? string.Empty : reader.GetString(2), + NotNull = reader.GetInt32(3) != 0, + DefaultValue = reader.IsDBNull(4) ? null : reader.GetString(4), + }); + } + + return columns; + } + + private static async Task PruneLegacyTargetTablesAsync(SqlConnection source, SqliteConnection target, IReadOnlySet sourceTables) + { + var targetTables = await GetSqliteTablesAsync(target); + foreach (var tableName in LegacyMigratedTables) + { + if (sourceTables.Contains(tableName) || !targetTables.Contains(tableName, StringComparer.OrdinalIgnoreCase)) + { + continue; + } + + await using var command = target.CreateCommand(); + command.CommandText = $"DROP TABLE {QuoteSqliteIdentifier(tableName)};"; + await command.ExecuteNonQueryAsync(); + Console.WriteLine($"Pruned legacy target-only table: {tableName}"); + } + } + + private static string BuildSqlServerOrderedSelectSql(string tableName, IReadOnlyList selectColumns, IReadOnlyList orderColumns, IReadOnlyDictionary columnTypes) + { + var selectList = string.Join(", ", selectColumns.Select(column => BuildSqlServerSelectExpression(column, columnTypes))); + var orderList = string.Join(", ", orderColumns.Select(column => BuildSqlServerOrderExpression(column, columnTypes))); + return $"SELECT {selectList} FROM {QuoteSqlServerIdentifier(tableName)} ORDER BY {orderList};"; + } + + private static string BuildSqliteOrderedSelectSql(string tableName, IReadOnlyList selectColumns, IReadOnlyList orderColumns) + { + return $"SELECT {string.Join(", ", selectColumns.Select(QuoteSqliteIdentifier))} FROM {QuoteSqliteIdentifier(tableName)} ORDER BY {string.Join(", ", orderColumns.Select(QuoteSqliteIdentifier))};"; + } + + private static string BuildSqlServerSelectExpression(string columnName, IReadOnlyDictionary columnTypes) + { + var expression = BuildSqlServerOrderExpression(columnName, columnTypes); + return $"{expression} AS {QuoteSqlServerIdentifier(columnName)}"; + } + + private static string BuildSqlServerOrderExpression(string columnName, IReadOnlyDictionary columnTypes) + { + var identifier = QuoteSqlServerIdentifier(columnName); + if (columnTypes.TryGetValue(columnName, out var type) && + type.Equals("uniqueidentifier", StringComparison.OrdinalIgnoreCase)) + { + return $"LOWER(CONVERT(varchar(36), {identifier}))"; + } + + return identifier; + } + + private static bool IsSqlServerOrderableType(string dataType) + => !dataType.Equals("text", StringComparison.OrdinalIgnoreCase) && + !dataType.Equals("ntext", StringComparison.OrdinalIgnoreCase) && + !dataType.Equals("image", StringComparison.OrdinalIgnoreCase); + + private static string NormalizeComparisonValue(object? value) + { + return value switch + { + null or DBNull => "", + byte[] bytes => Convert.ToHexString(bytes), + string text when TryNormalizeTemporalString(text, out var normalizedText) => normalizedText, + DateTime dateTime => NormalizeDateTime(dateTime), + DateTimeOffset offset => offset.ToString("O", CultureInfo.InvariantCulture), + bool boolean => boolean ? "1" : "0", + sbyte number => NormalizeNumericValue(number), + byte number => NormalizeNumericValue(number), + short number => NormalizeNumericValue(number), + ushort number => NormalizeNumericValue(number), + int number => NormalizeNumericValue(number), + uint number => NormalizeNumericValue(number), + long number => NormalizeNumericValue(number), + ulong number => NormalizeNumericValue(number), + decimal number => NormalizeNumericValue(number), + double number => NormalizeNumericValue(number), + float number => NormalizeNumericValue(number), + IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture) ?? string.Empty, + _ => value.ToString() ?? string.Empty, + }; + } + + private static string NormalizeNumericValue(T value) + where T : struct, ISpanFormattable + { + return value switch + { + decimal decimalValue => decimalValue.ToString("G29", CultureInfo.InvariantCulture), + double doubleValue => doubleValue.ToString("R", CultureInfo.InvariantCulture), + float floatValue => floatValue.ToString("R", CultureInfo.InvariantCulture), + _ => value.ToString(null, CultureInfo.InvariantCulture), + }; + } + + private static string QuoteSqlServerIdentifier(string identifier) + => $"[{identifier.Replace("]", "]]", StringComparison.Ordinal)}]"; + + private static string QuoteSqliteIdentifier(string identifier) + => $"\"{identifier.Replace("\"", "\"\"", StringComparison.Ordinal)}\""; + + private static string QuoteSqliteLiteral(string value) + => $"'{value.Replace("'", "''", StringComparison.Ordinal)}'"; + + private static Options ParseArgs(string[] args) + { + var options = new Options(); + for (var index = 0; index < args.Length; index++) + { + var argument = args[index]; + switch (argument) + { + case "--source-connection-string": + options.SourceConnectionString = GetRequiredValue(args, ref index, argument); + break; + case "--target-file": + options.TargetFile = GetRequiredValue(args, ref index, argument); + break; + case "--overwrite": + options.Overwrite = true; + break; + case "--help": + case "-h": + case "/?": + options.ShowHelp = true; + break; + default: + throw new ArgumentException($"Unknown argument: {argument}"); + } + } + + return options; + } + + private static string GetRequiredValue(string[] args, ref int index, string argumentName) + { + if (index + 1 >= args.Length) + { + throw new ArgumentException($"Missing value for {argumentName}"); + } + + index++; + return args[index]; + } + + private static void PrintUsage() + { + Console.WriteLine("Usage:"); + Console.WriteLine(" Shoko.CLI convert-db --source-connection-string \"\" --target-file \"/path/to/Shoko.sqlite\" [--overwrite]"); + Console.WriteLine(); + Console.WriteLine("Notes:"); + Console.WriteLine(" - This creates a fresh SQLite database using Shoko's built-in SQLite schema commands."); + Console.WriteLine(" - It copies tables and columns shared by the source SQL Server schema and target SQLite schema."); + Console.WriteLine(" - Quartz tables are not included because Quartz uses a separate database configuration."); + } + + private sealed class Options + { + public string SourceConnectionString { get; set; } = string.Empty; + public string TargetFile { get; set; } = string.Empty; + public bool Overwrite { get; set; } + public bool ShowHelp { get; set; } + } + + private sealed class TemporaryShokoHomeScope : IAsyncDisposable + { + private readonly string? _previousShokoHome; + private readonly string _temporaryHomePath; + + public TemporaryShokoHomeScope() + { + _previousShokoHome = Environment.GetEnvironmentVariable("SHOKO_HOME"); + _temporaryHomePath = Path.Combine(Path.GetTempPath(), $"shoko-convert-bootstrap-{Guid.NewGuid():N}"); + Directory.CreateDirectory(_temporaryHomePath); + Environment.SetEnvironmentVariable("SHOKO_HOME", _temporaryHomePath); + ResetApplicationPaths(); + } + + public ValueTask DisposeAsync() + { + Utils.ServiceContainer = null; + Environment.SetEnvironmentVariable("SHOKO_HOME", _previousShokoHome); + ResetApplicationPaths(); + try + { + if (Directory.Exists(_temporaryHomePath)) + { + Directory.Delete(_temporaryHomePath, true); + } + } + catch + { + // Best-effort cleanup only. Bootstrap isolation matters more than temp dir removal. + } + + return ValueTask.CompletedTask; + } + + private static void ResetApplicationPaths() + { + SetPrivateStaticField("_dataPath", null); + SetPrivateStaticField("_instance", null); + } + } + + private sealed class SqliteColumnInfo + { + public string Name { get; set; } = string.Empty; + public string Type { get; set; } = string.Empty; + public bool NotNull { get; set; } + public string? DefaultValue { get; set; } + } + + private static IHost CreateBootstrapHost(SystemService systemService, object settings) + { + var initWebHostMethod = typeof(SystemService).GetMethod("InitWebHost", BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("Unable to locate SystemService.InitWebHost."); + return (IHost)(initWebHostMethod.Invoke(systemService, [settings]) + ?? throw new InvalidOperationException("SystemService.InitWebHost returned null.")); + } + + private static bool RunInitializeDatabase(SystemService systemService, DatabaseFactory databaseFactory, RepoFactory repositoryFactory) + { + var initializeDatabaseMethod = typeof(SystemService).GetMethod("InitializeDatabase", BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("Unable to locate SystemService.InitializeDatabase."); + return (bool)(initializeDatabaseMethod.Invoke(systemService, [databaseFactory, repositoryFactory, default(CancellationToken)]) + ?? throw new InvalidOperationException("SystemService.InitializeDatabase returned null.")); + } + + private static T GetRequiredPrivateField(object instance, string fieldName) where T : class + { + var field = instance.GetType().GetField(fieldName, BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException($"Unable to locate field {instance.GetType().Name}.{fieldName}."); + return (T)(field.GetValue(instance) ?? throw new InvalidOperationException($"Field {instance.GetType().Name}.{fieldName} was null.")); + } + + private static void SetPrivateStaticField(string fieldName, object? value) + { + var field = typeof(TDeclaring).GetField(fieldName, BindingFlags.Static | BindingFlags.NonPublic) + ?? throw new InvalidOperationException($"Unable to locate static field {typeof(TDeclaring).Name}.{fieldName}."); + field.SetValue(null, value); + } + + private static void InitializeCorePluginOnly(PluginManager pluginManager, SystemService systemService) + { + var pluginTypes = GetRequiredPrivateField>(pluginManager, "_pluginTypes"); + if (pluginTypes.Count > 0) + { + return; + } + + var coreAssembly = typeof(CorePlugin).Assembly; + pluginTypes.Add(new() + { + ID = typeof(CorePlugin).FullName!.ToUuidV5(), + Name = "Shoko Core", + Description = string.Empty, + Version = systemService.Version, + Authors = null, + RepositoryUrl = null, + HomepageUrl = null, + Tags = [], + LoadOrder = 0, + Thumbnail = null, + InstalledAt = DateTime.MinValue, + IsEnabled = true, + IsActive = false, + CanLoad = true, + CanUninstall = false, + Plugin = null, + PluginType = typeof(CorePlugin), + ServiceRegistrationType = null, + ApplicationRegistrationType = null, + ContainingDirectory = null, + DLLs = [coreAssembly.Location], + Types = coreAssembly.GetExportedTypes(), + }); + } +} diff --git a/Shoko.CLI/Program.cs b/Shoko.CLI/Program.cs index 33c51ee38b..c6acd06bd3 100644 --- a/Shoko.CLI/Program.cs +++ b/Shoko.CLI/Program.cs @@ -15,8 +15,13 @@ public static class Program { private static ILogger _logger = null!; - public static async Task Main() + public static async Task Main(string[] args) { + if (args.Length > 0 && string.Equals(args[0], "convert-db", StringComparison.OrdinalIgnoreCase)) + { + return await DatabaseConverterCommand.RunAsync(args[1..]); + } + try { UnhandledExceptionManager.AddHandler(); From 52e38cd65f927bc55d3bb39eae16d3c4a7b15b65 Mon Sep 17 00:00:00 2001 From: krbrs <57227244+krbrs@users.noreply.github.com> Date: Thu, 21 May 2026 10:01:51 +0200 Subject: [PATCH 02/13] feat(cli): add mariadb to sqlite database migrator --- Shoko.CLI/DatabaseConverterCommand.cs | 325 +++++++++++++++++++++++--- 1 file changed, 297 insertions(+), 28 deletions(-) diff --git a/Shoko.CLI/DatabaseConverterCommand.cs b/Shoko.CLI/DatabaseConverterCommand.cs index 6797b7c9ce..76d433198f 100644 --- a/Shoko.CLI/DatabaseConverterCommand.cs +++ b/Shoko.CLI/DatabaseConverterCommand.cs @@ -14,6 +14,7 @@ using Microsoft.Data.Sqlite; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using MySqlConnector; using Shoko.Abstractions.Plugin.Models; using Shoko.Abstractions.Extensions; using Shoko.Server.Databases; @@ -26,6 +27,12 @@ namespace Shoko.CLI; internal static class DatabaseConverterCommand { + private enum SourceDatabaseType + { + SqlServer, + MySql, + } + private static readonly HashSet ExcludedTables = new(StringComparer.OrdinalIgnoreCase) { "Versions", @@ -37,6 +44,14 @@ internal static class DatabaseConverterCommand "CrossRef_AniDB_TvDBV2", }; + private static readonly string[] LegacyUnifiedImageTables = + [ + "TMDB_Image", + "TMDB_Image_Entity", + "AniDB_Anime_PreferredImage", + "AniDB_Episode_PreferredImage", + ]; + public static async Task RunAsync(string[] args) { var options = ParseArgs(args); @@ -54,7 +69,7 @@ public static async Task RunAsync(string[] args) try { - await ConvertAsync(options.SourceConnectionString, options.TargetFile, options.Overwrite); + await ConvertAsync(options.SourceType, options.SourceConnectionString, options.TargetFile, options.Overwrite); return 0; } catch (Exception ex) @@ -65,7 +80,7 @@ public static async Task RunAsync(string[] args) } } - private static async Task ConvertAsync(string sourceConnectionString, string targetFile, bool overwrite) + private static async Task ConvertAsync(SourceDatabaseType sourceType, string sourceConnectionString, string targetFile, bool overwrite) { var fullTargetPath = Path.GetFullPath(targetFile); if (File.Exists(fullTargetPath)) @@ -93,15 +108,14 @@ private static async Task ConvertAsync(string sourceConnectionString, string tar await InitializeSqliteDatabaseAsync(targetConnectionString); - await using var source = new SqlConnection(sourceConnectionString); - await source.OpenAsync(); + await using var source = await OpenSourceConnectionAsync(sourceType, sourceConnectionString); await using var target = new SqliteConnection(targetConnectionString); await target.OpenAsync(); await ConfigureSqliteAsync(target); - var sourceTables = await GetSqlServerTablesAsync(source); - await PruneLegacyTargetTablesAsync(source, target, sourceTables); + var sourceTables = await GetSourceTablesAsync(source, sourceType); + await PruneLegacyTargetTablesAsync(target, sourceTables); var targetTables = await GetSqliteTablesAsync(target); var tablesToCopy = targetTables .Where(sourceTables.Contains) @@ -138,7 +152,7 @@ private static async Task ConvertAsync(string sourceConnectionString, string tar Console.WriteLine($" {targetOnlyTable}"); } - await ReportSqlServerObjectsAsync(source, targetOnlyTables); + await ReportSourceObjectsAsync(source, sourceType, targetOnlyTables); } if (ExcludedTables.Count > 0) @@ -152,10 +166,10 @@ private static async Task ConvertAsync(string sourceConnectionString, string tar foreach (var tableName in tablesToCopy) { - await CopyTableAsync(source, target, tableName); + await CopyTableAsync(source, sourceType, target, tableName); } - await VerifyCopyAsync(source, target, tablesToCopy); + await VerifyCopyAsync(source, sourceType, target, tablesToCopy); Console.WriteLine($"Conversion completed successfully: {fullTargetPath}"); } @@ -201,9 +215,22 @@ private static async Task ConfigureSqliteAsync(SqliteConnection connection) } } - private static async Task CopyTableAsync(SqlConnection source, SqliteConnection target, string tableName) + private static async Task OpenSourceConnectionAsync(SourceDatabaseType sourceType, string connectionString) { - var sourceColumns = await GetSqlServerColumnsAsync(source, tableName); + DbConnection connection = sourceType switch + { + SourceDatabaseType.SqlServer => new SqlConnection(connectionString), + SourceDatabaseType.MySql => new MySqlConnection(connectionString), + _ => throw new InvalidOperationException($"Unsupported source database type: {sourceType}"), + }; + + await connection.OpenAsync(); + return connection; + } + + private static async Task CopyTableAsync(DbConnection source, SourceDatabaseType sourceType, SqliteConnection target, string tableName) + { + var sourceColumns = await GetSourceColumnsAsync(source, sourceType, tableName); var targetColumns = await GetSqliteColumnsAsync(target, tableName); var sourceColumnSet = sourceColumns.ToHashSet(StringComparer.OrdinalIgnoreCase); var sourceBackedColumns = targetColumns.Where(column => sourceColumnSet.Contains(column.Name)).ToList(); @@ -233,8 +260,9 @@ private static async Task CopyTableAsync(SqlConnection source, SqliteConnection await deleteCommand.ExecuteNonQueryAsync(); } - var selectSql = $"SELECT {string.Join(", ", sourceBackedColumns.Select(column => QuoteSqlServerIdentifier(column.Name)))} FROM {QuoteSqlServerIdentifier(tableName)};"; - await using var selectCommand = new SqlCommand(selectSql, source); + var selectSql = $"SELECT {string.Join(", ", sourceBackedColumns.Select(column => QuoteSourceIdentifier(sourceType, column.Name)))} FROM {QuoteSourceIdentifier(sourceType, tableName)};"; + await using var selectCommand = source.CreateCommand(); + selectCommand.CommandText = selectSql; await using var reader = await selectCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess); await using var insertCommand = target.CreateCommand(); @@ -270,13 +298,13 @@ private static async Task CopyTableAsync(SqlConnection source, SqliteConnection Console.WriteLine($"Copied {tableName}: {rowCount} rows, {insertColumns.Count} columns."); } - private static async Task VerifyCopyAsync(SqlConnection source, SqliteConnection target, IReadOnlyList tablesToCopy) + private static async Task VerifyCopyAsync(DbConnection source, SourceDatabaseType sourceType, SqliteConnection target, IReadOnlyList tablesToCopy) { Console.WriteLine("Verifying migrated data..."); foreach (var tableName in tablesToCopy) { - var sourceColumns = await GetSqlServerColumnsAsync(source, tableName); + var sourceColumns = await GetSourceColumnsAsync(source, sourceType, tableName); var targetColumns = await GetSqliteColumnsAsync(target, tableName); var sharedColumns = targetColumns .Where(column => sourceColumns.Contains(column.Name, StringComparer.OrdinalIgnoreCase)) @@ -289,23 +317,23 @@ private static async Task VerifyCopyAsync(SqlConnection source, SqliteConnection continue; } - var sourceCount = await GetRowCountAsync(source, tableName); + var sourceCount = await GetRowCountAsync(source, sourceType, tableName); var targetCount = await GetRowCountAsync(target, tableName); if (sourceCount != targetCount) { throw new InvalidOperationException($"Verification failed for {tableName}: row count mismatch. Source={sourceCount}, Target={targetCount}."); } - await VerifyTableContentAsync(source, target, tableName, sharedColumns); + await VerifyTableContentAsync(source, sourceType, target, tableName, sharedColumns); Console.WriteLine($"Verified {tableName}: {sourceCount} rows, {sharedColumns.Count} shared columns."); } } - private static async Task GetRowCountAsync(SqlConnection connection, string tableName) + private static async Task GetRowCountAsync(DbConnection connection, SourceDatabaseType sourceType, string tableName) { await using var command = connection.CreateCommand(); - command.CommandText = $"SELECT COUNT(*) FROM {QuoteSqlServerIdentifier(tableName)};"; + command.CommandText = $"SELECT COUNT(*) FROM {QuoteSourceIdentifier(sourceType, tableName)};"; return Convert.ToInt64(await command.ExecuteScalarAsync(), CultureInfo.InvariantCulture); } @@ -316,11 +344,11 @@ private static async Task GetRowCountAsync(SqliteConnection connection, st return Convert.ToInt64(await command.ExecuteScalarAsync(), CultureInfo.InvariantCulture); } - private static async Task VerifyTableContentAsync(SqlConnection source, SqliteConnection target, string tableName, IReadOnlyList sharedColumns) + private static async Task VerifyTableContentAsync(DbConnection source, SourceDatabaseType sourceType, SqliteConnection target, string tableName, IReadOnlyList sharedColumns) { - var sourcePrimaryKeys = await GetSqlServerPrimaryKeyColumnsAsync(source, tableName); + var sourcePrimaryKeys = await GetSourcePrimaryKeyColumnsAsync(source, sourceType, tableName); var targetPrimaryKeys = await GetSqlitePrimaryKeyColumnsAsync(target, tableName); - var sourceColumnTypes = await GetSqlServerColumnTypesAsync(source, tableName); + var sourceColumnTypes = await GetSourceColumnTypesAsync(source, sourceType, tableName); var orderColumns = sourcePrimaryKeys .Where(column => targetPrimaryKeys.Contains(column, StringComparer.OrdinalIgnoreCase)) .Where(column => sharedColumns.Contains(column, StringComparer.OrdinalIgnoreCase)) @@ -329,7 +357,7 @@ private static async Task VerifyTableContentAsync(SqlConnection source, SqliteCo if (orderColumns.Count == 0) { orderColumns = sharedColumns - .Where(column => sourceColumnTypes.TryGetValue(column, out var type) && IsSqlServerOrderableType(type)) + .Where(column => sourceColumnTypes.TryGetValue(column, out var type) && IsSourceOrderableType(sourceType, type)) .ToList(); } @@ -338,10 +366,11 @@ private static async Task VerifyTableContentAsync(SqlConnection source, SqliteCo throw new InvalidOperationException($"Verification failed for {tableName}: no primary key or SQL-orderable shared columns are available for deterministic comparison."); } - var sourceSql = BuildSqlServerOrderedSelectSql(tableName, sharedColumns, orderColumns, sourceColumnTypes); + var sourceSql = BuildSourceOrderedSelectSql(sourceType, tableName, sharedColumns, orderColumns, sourceColumnTypes); var targetSql = BuildSqliteOrderedSelectSql(tableName, sharedColumns, orderColumns); - await using var sourceCommand = new SqlCommand(sourceSql, source); + await using var sourceCommand = source.CreateCommand(); + sourceCommand.CommandText = sourceSql; await using var sourceReader = await sourceCommand.ExecuteReaderAsync(CommandBehavior.SequentialAccess); await using var targetCommand = target.CreateCommand(); targetCommand.CommandText = targetSql; @@ -462,6 +491,16 @@ private static bool TryNormalizeTemporalString(string value, out string normaliz return false; } + private static async Task> GetSourceTablesAsync(DbConnection connection, SourceDatabaseType sourceType) + { + return sourceType switch + { + SourceDatabaseType.SqlServer => await GetSqlServerTablesAsync((SqlConnection)connection), + SourceDatabaseType.MySql => await GetMySqlTablesAsync((MySqlConnection)connection), + _ => throw new InvalidOperationException($"Unsupported source database type: {sourceType}"), + }; + } + private static async Task> GetSqlServerTablesAsync(SqlConnection connection) { const string sql = """ @@ -481,6 +520,26 @@ FROM INFORMATION_SCHEMA.TABLES return tables; } + private static async Task> GetMySqlTablesAsync(MySqlConnection connection) + { + const string sql = """ + SELECT TABLE_NAME + FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_TYPE = 'BASE TABLE' + """; + + var tables = new HashSet(StringComparer.OrdinalIgnoreCase); + await using var command = new MySqlCommand(sql, connection); + await using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + tables.Add(reader.GetString(0)); + } + + return tables; + } + private static async Task> GetSqliteTablesAsync(SqliteConnection connection) { const string sql = """ @@ -503,6 +562,16 @@ ORDER BY name return tables; } + private static async Task> GetSourceColumnsAsync(DbConnection connection, SourceDatabaseType sourceType, string tableName) + { + return sourceType switch + { + SourceDatabaseType.SqlServer => await GetSqlServerColumnsAsync((SqlConnection)connection, tableName), + SourceDatabaseType.MySql => await GetMySqlColumnsAsync((MySqlConnection)connection, tableName), + _ => throw new InvalidOperationException($"Unsupported source database type: {sourceType}"), + }; + } + private static async Task> GetSqlServerColumnsAsync(SqlConnection connection, string tableName) { const string sql = """ @@ -524,6 +593,38 @@ ORDER BY ORDINAL_POSITION return columns; } + private static async Task> GetMySqlColumnsAsync(MySqlConnection connection, string tableName) + { + const string sql = """ + SELECT COLUMN_NAME + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = @tableName + ORDER BY ORDINAL_POSITION + """; + + var columns = new List(); + await using var command = new MySqlCommand(sql, connection); + command.Parameters.AddWithValue("@tableName", tableName); + await using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + columns.Add(reader.GetString(0)); + } + + return columns; + } + + private static async Task> GetSourcePrimaryKeyColumnsAsync(DbConnection connection, SourceDatabaseType sourceType, string tableName) + { + return sourceType switch + { + SourceDatabaseType.SqlServer => await GetSqlServerPrimaryKeyColumnsAsync((SqlConnection)connection, tableName), + SourceDatabaseType.MySql => await GetMySqlPrimaryKeyColumnsAsync((MySqlConnection)connection, tableName), + _ => throw new InvalidOperationException($"Unsupported source database type: {sourceType}"), + }; + } + private static async Task> GetSqlServerPrimaryKeyColumnsAsync(SqlConnection connection, string tableName) { const string sql = """ @@ -546,6 +647,39 @@ ORDER BY ORDINAL_POSITION return columns; } + private static async Task> GetMySqlPrimaryKeyColumnsAsync(MySqlConnection connection, string tableName) + { + const string sql = """ + SELECT COLUMN_NAME + FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = @tableName + AND CONSTRAINT_NAME = 'PRIMARY' + ORDER BY ORDINAL_POSITION + """; + + var columns = new List(); + await using var command = new MySqlCommand(sql, connection); + command.Parameters.AddWithValue("@tableName", tableName); + await using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + columns.Add(reader.GetString(0)); + } + + return columns; + } + + private static async Task> GetSourceColumnTypesAsync(DbConnection connection, SourceDatabaseType sourceType, string tableName) + { + return sourceType switch + { + SourceDatabaseType.SqlServer => await GetSqlServerColumnTypesAsync((SqlConnection)connection, tableName), + SourceDatabaseType.MySql => await GetMySqlColumnTypesAsync((MySqlConnection)connection, tableName), + _ => throw new InvalidOperationException($"Unsupported source database type: {sourceType}"), + }; + } + private static async Task> GetSqlServerColumnTypesAsync(SqlConnection connection, string tableName) { const string sql = """ @@ -566,6 +700,27 @@ FROM INFORMATION_SCHEMA.COLUMNS return columns; } + private static async Task> GetMySqlColumnTypesAsync(MySqlConnection connection, string tableName) + { + const string sql = """ + SELECT COLUMN_NAME, DATA_TYPE + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = @tableName + """; + + var columns = new Dictionary(StringComparer.OrdinalIgnoreCase); + await using var command = new MySqlCommand(sql, connection); + command.Parameters.AddWithValue("@tableName", tableName); + await using var reader = await command.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + columns[reader.GetString(0)] = reader.GetString(1); + } + + return columns; + } + private static async Task> GetSqlitePrimaryKeyColumnsAsync(SqliteConnection connection, string tableName) { var columns = new List<(int Position, string Name)>(); @@ -584,6 +739,21 @@ private static async Task> GetSqlitePrimaryKeyColumnsAsync(SqliteCo return columns.OrderBy(column => column.Position).Select(column => column.Name).ToList(); } + private static async Task ReportSourceObjectsAsync(DbConnection connection, SourceDatabaseType sourceType, IReadOnlyList objectNames) + { + switch (sourceType) + { + case SourceDatabaseType.SqlServer: + await ReportSqlServerObjectsAsync((SqlConnection)connection, objectNames); + return; + case SourceDatabaseType.MySql: + await ReportMySqlObjectsAsync((MySqlConnection)connection, objectNames); + return; + default: + throw new InvalidOperationException($"Unsupported source database type: {sourceType}"); + } + } + private static async Task ReportSqlServerObjectsAsync(SqlConnection connection, IReadOnlyList objectNames) { const string sql = """ @@ -613,6 +783,35 @@ FROM sys.objects o } } + private static async Task ReportMySqlObjectsAsync(MySqlConnection connection, IReadOnlyList objectNames) + { + const string sql = """ + SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE + FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = @objectName + ORDER BY TABLE_TYPE, TABLE_NAME + """; + + Console.WriteLine("Source object lookup for target-only names:"); + foreach (var objectName in objectNames) + { + await using var command = new MySqlCommand(sql, connection); + command.Parameters.AddWithValue("@objectName", objectName); + await using var reader = await command.ExecuteReaderAsync(); + if (!reader.HasRows) + { + Console.WriteLine($" {objectName}: not found in information_schema.tables"); + continue; + } + + while (await reader.ReadAsync()) + { + Console.WriteLine($" {objectName}: schema={reader.GetString(0)}, type={reader.GetString(2)}"); + } + } + } + private static object GetFallbackValue(string tableName, SqliteColumnInfo column) { if (string.Equals(tableName, "CrossRef_File_Episode", StringComparison.OrdinalIgnoreCase) && @@ -664,7 +863,7 @@ private static async Task> GetSqliteColumnsAsync(SqliteCo return columns; } - private static async Task PruneLegacyTargetTablesAsync(SqlConnection source, SqliteConnection target, IReadOnlySet sourceTables) + private static async Task PruneLegacyTargetTablesAsync(SqliteConnection target, IReadOnlySet sourceTables) { var targetTables = await GetSqliteTablesAsync(target); foreach (var tableName in LegacyMigratedTables) @@ -681,6 +880,16 @@ private static async Task PruneLegacyTargetTablesAsync(SqlConnection source, Sql } } + private static string BuildSourceOrderedSelectSql(SourceDatabaseType sourceType, string tableName, IReadOnlyList selectColumns, IReadOnlyList orderColumns, IReadOnlyDictionary columnTypes) + { + return sourceType switch + { + SourceDatabaseType.SqlServer => BuildSqlServerOrderedSelectSql(tableName, selectColumns, orderColumns, columnTypes), + SourceDatabaseType.MySql => BuildMySqlOrderedSelectSql(tableName, selectColumns, orderColumns, columnTypes), + _ => throw new InvalidOperationException($"Unsupported source database type: {sourceType}"), + }; + } + private static string BuildSqlServerOrderedSelectSql(string tableName, IReadOnlyList selectColumns, IReadOnlyList orderColumns, IReadOnlyDictionary columnTypes) { var selectList = string.Join(", ", selectColumns.Select(column => BuildSqlServerSelectExpression(column, columnTypes))); @@ -688,6 +897,13 @@ private static string BuildSqlServerOrderedSelectSql(string tableName, IReadOnly return $"SELECT {selectList} FROM {QuoteSqlServerIdentifier(tableName)} ORDER BY {orderList};"; } + private static string BuildMySqlOrderedSelectSql(string tableName, IReadOnlyList selectColumns, IReadOnlyList orderColumns, IReadOnlyDictionary columnTypes) + { + var selectList = string.Join(", ", selectColumns.Select(column => BuildMySqlSelectExpression(column, columnTypes))); + var orderList = string.Join(", ", orderColumns.Select(column => BuildMySqlOrderExpression(column, columnTypes))); + return $"SELECT {selectList} FROM {QuoteMySqlIdentifier(tableName)} ORDER BY {orderList};"; + } + private static string BuildSqliteOrderedSelectSql(string tableName, IReadOnlyList selectColumns, IReadOnlyList orderColumns) { return $"SELECT {string.Join(", ", selectColumns.Select(QuoteSqliteIdentifier))} FROM {QuoteSqliteIdentifier(tableName)} ORDER BY {string.Join(", ", orderColumns.Select(QuoteSqliteIdentifier))};"; @@ -711,11 +927,36 @@ private static string BuildSqlServerOrderExpression(string columnName, IReadOnly return identifier; } + private static string BuildMySqlSelectExpression(string columnName, IReadOnlyDictionary columnTypes) + { + return QuoteMySqlIdentifier(columnName); + } + + private static string BuildMySqlOrderExpression(string columnName, IReadOnlyDictionary columnTypes) + { + return QuoteMySqlIdentifier(columnName); + } + + private static bool IsSourceOrderableType(SourceDatabaseType sourceType, string dataType) + { + return sourceType switch + { + SourceDatabaseType.SqlServer => IsSqlServerOrderableType(dataType), + SourceDatabaseType.MySql => IsMySqlOrderableType(dataType), + _ => false, + }; + } + private static bool IsSqlServerOrderableType(string dataType) => !dataType.Equals("text", StringComparison.OrdinalIgnoreCase) && !dataType.Equals("ntext", StringComparison.OrdinalIgnoreCase) && !dataType.Equals("image", StringComparison.OrdinalIgnoreCase); + private static bool IsMySqlOrderableType(string dataType) + => !dataType.Contains("blob", StringComparison.OrdinalIgnoreCase) && + !dataType.Equals("json", StringComparison.OrdinalIgnoreCase) && + !dataType.Equals("geometry", StringComparison.OrdinalIgnoreCase); + private static string NormalizeComparisonValue(object? value) { return value switch @@ -757,6 +998,19 @@ private static string NormalizeNumericValue(T value) private static string QuoteSqlServerIdentifier(string identifier) => $"[{identifier.Replace("]", "]]", StringComparison.Ordinal)}]"; + private static string QuoteMySqlIdentifier(string identifier) + => $"`{identifier.Replace("`", "``", StringComparison.Ordinal)}`"; + + private static string QuoteSourceIdentifier(SourceDatabaseType sourceType, string identifier) + { + return sourceType switch + { + SourceDatabaseType.SqlServer => QuoteSqlServerIdentifier(identifier), + SourceDatabaseType.MySql => QuoteMySqlIdentifier(identifier), + _ => throw new InvalidOperationException($"Unsupported source database type: {sourceType}"), + }; + } + private static string QuoteSqliteIdentifier(string identifier) => $"\"{identifier.Replace("\"", "\"\"", StringComparison.Ordinal)}\""; @@ -771,6 +1025,9 @@ private static Options ParseArgs(string[] args) var argument = args[index]; switch (argument) { + case "--source-type": + options.SourceType = ParseSourceType(GetRequiredValue(args, ref index, argument)); + break; case "--source-connection-string": options.SourceConnectionString = GetRequiredValue(args, ref index, argument); break; @@ -804,19 +1061,31 @@ private static string GetRequiredValue(string[] args, ref int index, string argu return args[index]; } + private static SourceDatabaseType ParseSourceType(string value) + { + return value.Trim().ToLowerInvariant() switch + { + "mssql" or "sqlserver" or "sql-server" => SourceDatabaseType.SqlServer, + "mysql" or "mariadb" or "maria" => SourceDatabaseType.MySql, + _ => throw new ArgumentException($"Unsupported source type: {value}. Expected mssql or mariadb."), + }; + } + private static void PrintUsage() { Console.WriteLine("Usage:"); - Console.WriteLine(" Shoko.CLI convert-db --source-connection-string \"\" --target-file \"/path/to/Shoko.sqlite\" [--overwrite]"); + Console.WriteLine(" Shoko.CLI convert-db [--source-type mssql|mariadb] --source-connection-string \"\" --target-file \"/path/to/Shoko.sqlite\" [--overwrite]"); Console.WriteLine(); Console.WriteLine("Notes:"); Console.WriteLine(" - This creates a fresh SQLite database using Shoko's built-in SQLite schema commands."); - Console.WriteLine(" - It copies tables and columns shared by the source SQL Server schema and target SQLite schema."); + Console.WriteLine(" - Supported source backends: SQL Server and MySQL/MariaDB."); + Console.WriteLine(" - It copies tables and columns shared by the source schema and target SQLite schema."); Console.WriteLine(" - Quartz tables are not included because Quartz uses a separate database configuration."); } private sealed class Options { + public SourceDatabaseType SourceType { get; set; } = SourceDatabaseType.SqlServer; public string SourceConnectionString { get; set; } = string.Empty; public string TargetFile { get; set; } = string.Empty; public bool Overwrite { get; set; } From b9d1e251dd525a8a8ca4fbdd45657947bcf5cc68 Mon Sep 17 00:00:00 2001 From: krbrs <57227244+krbrs@users.noreply.github.com> Date: Thu, 21 May 2026 10:38:08 +0200 Subject: [PATCH 03/13] feat(cli): guard database conversion by source version --- Shoko.CLI/DatabaseConverterCommand.cs | 148 +++++++++++++++++++++++++- 1 file changed, 147 insertions(+), 1 deletion(-) diff --git a/Shoko.CLI/DatabaseConverterCommand.cs b/Shoko.CLI/DatabaseConverterCommand.cs index 76d433198f..a0ae269972 100644 --- a/Shoko.CLI/DatabaseConverterCommand.cs +++ b/Shoko.CLI/DatabaseConverterCommand.cs @@ -4,6 +4,7 @@ using System.Data.Common; using System.Globalization; using System.IO; +using System.Collections; using System.Linq; using System.Reflection; using System.Security.Cryptography; @@ -99,6 +100,9 @@ private static async Task ConvertAsync(SourceDatabaseType sourceType, string sou Directory.CreateDirectory(targetDirectory); } + await using var source = await OpenSourceConnectionAsync(sourceType, sourceConnectionString); + await EnsureSourceVersionSupportedAsync(source, sourceType); + var targetConnectionString = new SqliteConnectionStringBuilder { DataSource = fullTargetPath, @@ -108,7 +112,6 @@ private static async Task ConvertAsync(SourceDatabaseType sourceType, string sou await InitializeSqliteDatabaseAsync(targetConnectionString); - await using var source = await OpenSourceConnectionAsync(sourceType, sourceConnectionString); await using var target = new SqliteConnection(targetConnectionString); await target.OpenAsync(); @@ -173,6 +176,35 @@ private static async Task ConvertAsync(SourceDatabaseType sourceType, string sou Console.WriteLine($"Conversion completed successfully: {fullTargetPath}"); } + private static async Task EnsureSourceVersionSupportedAsync(DbConnection source, SourceDatabaseType sourceType) + { + var sourceVersion = await GetSourceDatabaseVersionAsync(source, sourceType); + if (sourceVersion is null) + { + throw new InvalidOperationException("The source database does not contain a current Database version entry in Versions. Upgrade it with the matching Shoko Server build before conversion."); + } + + var expectedVersion = await GetExpectedSourceDatabaseVersionAsync(sourceType); + if (sourceVersion.Value.Version != expectedVersion.Version || sourceVersion.Value.Revision != expectedVersion.Revision) + { + throw new InvalidOperationException( + $"Unsupported source database version for {GetSourceTypeDisplayName(sourceType)}. " + + $"Found {sourceVersion.Value.Version}.{sourceVersion.Value.Revision} " + + $"(program: {sourceVersion.Value.Program ?? "unknown"}), expected {expectedVersion.Version}.{expectedVersion.Revision} " + + $"for this Shoko build. Upgrade the source database with the matching Shoko Server build before conversion."); + } + } + + private static string GetSourceTypeDisplayName(SourceDatabaseType sourceType) + { + return sourceType switch + { + SourceDatabaseType.SqlServer => "SQL Server", + SourceDatabaseType.MySql => "MySQL/MariaDB", + _ => sourceType.ToString(), + }; + } + private static async Task InitializeSqliteDatabaseAsync(string connectionString) { await using var bootstrapHome = new TemporaryShokoHomeScope(); @@ -228,6 +260,118 @@ private static async Task OpenSourceConnectionAsync(SourceDatabase return connection; } + private static async Task GetSourceDatabaseVersionAsync(DbConnection connection, SourceDatabaseType sourceType) + { + return sourceType switch + { + SourceDatabaseType.SqlServer => await GetSqlServerDatabaseVersionAsync((SqlConnection)connection), + SourceDatabaseType.MySql => await GetMySqlDatabaseVersionAsync((MySqlConnection)connection), + _ => throw new InvalidOperationException($"Unsupported source database type: {sourceType}"), + }; + } + + private static async Task GetSqlServerDatabaseVersionAsync(SqlConnection connection) + { + const string sql = """ + SELECT TOP 1 VersionValue, VersionRevision, VersionProgram + FROM Versions + WHERE VersionType = 'Database' + ORDER BY TRY_CONVERT(int, VersionValue) DESC, TRY_CONVERT(int, VersionRevision) DESC + """; + + await using var command = new SqlCommand(sql, connection); + await using var reader = await command.ExecuteReaderAsync(); + if (!await reader.ReadAsync()) + { + return null; + } + + return new DatabaseVersionInfo( + ParseVersionPart(reader.GetString(0), "VersionValue"), + ParseVersionPart(reader.GetString(1), "VersionRevision"), + reader.IsDBNull(2) ? null : reader.GetString(2)); + } + + private static async Task GetMySqlDatabaseVersionAsync(MySqlConnection connection) + { + const string sql = """ + SELECT VersionValue, VersionRevision, VersionProgram + FROM Versions + WHERE VersionType = 'Database' + ORDER BY CAST(VersionValue AS SIGNED) DESC, CAST(VersionRevision AS SIGNED) DESC + LIMIT 1 + """; + + await using var command = new MySqlCommand(sql, connection); + await using var reader = await command.ExecuteReaderAsync(); + if (!await reader.ReadAsync()) + { + return null; + } + + return new DatabaseVersionInfo( + ParseVersionPart(reader.GetString(0), "VersionValue"), + ParseVersionPart(reader.GetString(1), "VersionRevision"), + reader.IsDBNull(2) ? null : reader.GetString(2)); + } + + private static async Task GetExpectedSourceDatabaseVersionAsync(SourceDatabaseType sourceType) + { + await using var bootstrapHome = new TemporaryShokoHomeScope(); + var systemService = new SystemService(); + object database = sourceType switch + { + SourceDatabaseType.SqlServer => new SQLServer(systemService), + SourceDatabaseType.MySql => new MySQL(systemService), + _ => throw new InvalidOperationException($"Unsupported source database type: {sourceType}"), + }; + + var latest = GetDatabaseCommands(database) + .Where(command => command.Version > 0) + .Select(command => new DatabaseVersionInfo(command.Version, command.Revision, null)) + .OrderByDescending(command => command.Version) + .ThenByDescending(command => command.Revision) + .First(); + + return latest; + } + + private static IEnumerable GetDatabaseCommands(object database) + { + foreach (var field in database.GetType().GetFields(BindingFlags.Instance | BindingFlags.NonPublic)) + { + var value = field.GetValue(database); + if (value is DatabaseCommand singleCommand) + { + yield return singleCommand; + continue; + } + + if (value is not IEnumerable enumerable) + { + continue; + } + + foreach (var item in enumerable) + { + if (item is DatabaseCommand command) + { + yield return command; + } + } + } + } + + private static int ParseVersionPart(string value, string columnName) + { + if (!int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsed)) + { + throw new InvalidOperationException($"Invalid {columnName} value in Versions: {value}"); + } + + return parsed; + } + private static async Task CopyTableAsync(DbConnection source, SourceDatabaseType sourceType, SqliteConnection target, string tableName) { var sourceColumns = await GetSourceColumnsAsync(source, sourceType, tableName); @@ -1092,6 +1236,8 @@ private sealed class Options public bool ShowHelp { get; set; } } + private readonly record struct DatabaseVersionInfo(int Version, int Revision, string? Program); + private sealed class TemporaryShokoHomeScope : IAsyncDisposable { private readonly string? _previousShokoHome; From 528c362e708ed1f46c7573f853cd300d1bfb1cbd Mon Sep 17 00:00:00 2001 From: krbrs <57227244+krbrs@users.noreply.github.com> Date: Thu, 21 May 2026 15:04:45 +0200 Subject: [PATCH 04/13] refactor(cli): remove temporary legacy table handling from db converter --- Shoko.CLI/DatabaseConverterCommand.cs | 32 --------------------------- 1 file changed, 32 deletions(-) diff --git a/Shoko.CLI/DatabaseConverterCommand.cs b/Shoko.CLI/DatabaseConverterCommand.cs index a0ae269972..83ad153e25 100644 --- a/Shoko.CLI/DatabaseConverterCommand.cs +++ b/Shoko.CLI/DatabaseConverterCommand.cs @@ -39,20 +39,6 @@ private enum SourceDatabaseType "Versions", }; - private static readonly HashSet LegacyMigratedTables = new(StringComparer.OrdinalIgnoreCase) - { - "AniDB_Vote", - "CrossRef_AniDB_TvDBV2", - }; - - private static readonly string[] LegacyUnifiedImageTables = - [ - "TMDB_Image", - "TMDB_Image_Entity", - "AniDB_Anime_PreferredImage", - "AniDB_Episode_PreferredImage", - ]; - public static async Task RunAsync(string[] args) { var options = ParseArgs(args); @@ -118,7 +104,6 @@ private static async Task ConvertAsync(SourceDatabaseType sourceType, string sou await ConfigureSqliteAsync(target); var sourceTables = await GetSourceTablesAsync(source, sourceType); - await PruneLegacyTargetTablesAsync(target, sourceTables); var targetTables = await GetSqliteTablesAsync(target); var tablesToCopy = targetTables .Where(sourceTables.Contains) @@ -1007,23 +992,6 @@ private static async Task> GetSqliteColumnsAsync(SqliteCo return columns; } - private static async Task PruneLegacyTargetTablesAsync(SqliteConnection target, IReadOnlySet sourceTables) - { - var targetTables = await GetSqliteTablesAsync(target); - foreach (var tableName in LegacyMigratedTables) - { - if (sourceTables.Contains(tableName) || !targetTables.Contains(tableName, StringComparer.OrdinalIgnoreCase)) - { - continue; - } - - await using var command = target.CreateCommand(); - command.CommandText = $"DROP TABLE {QuoteSqliteIdentifier(tableName)};"; - await command.ExecuteNonQueryAsync(); - Console.WriteLine($"Pruned legacy target-only table: {tableName}"); - } - } - private static string BuildSourceOrderedSelectSql(SourceDatabaseType sourceType, string tableName, IReadOnlyList selectColumns, IReadOnlyList orderColumns, IReadOnlyDictionary columnTypes) { return sourceType switch From e364ac3d99c59768ad3bb31a46721307634b0660 Mon Sep 17 00:00:00 2001 From: krbrs <57227244+krbrs@users.noreply.github.com> Date: Thu, 21 May 2026 15:30:10 +0200 Subject: [PATCH 05/13] chore(cli): polish database converter review follow-ups --- Shoko.CLI/DatabaseConverterCommand.cs | 145 ++++++++++++++++---------- 1 file changed, 92 insertions(+), 53 deletions(-) diff --git a/Shoko.CLI/DatabaseConverterCommand.cs b/Shoko.CLI/DatabaseConverterCommand.cs index 83ad153e25..e0c8cd6e89 100644 --- a/Shoko.CLI/DatabaseConverterCommand.cs +++ b/Shoko.CLI/DatabaseConverterCommand.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Data; using System.Data.Common; +using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.IO; using System.Collections; @@ -28,6 +29,8 @@ namespace Shoko.CLI; internal static class DatabaseConverterCommand { + private const string TableNameParameter = "@tableName"; + private enum SourceDatabaseType { SqlServer, @@ -61,8 +64,8 @@ public static async Task RunAsync(string[] args) } catch (Exception ex) { - Console.Error.WriteLine($"Database conversion failed: {ex.Message}"); - Console.Error.WriteLine(ex); + await Console.Error.WriteLineAsync($"Database conversion failed: {ex.Message}"); + await Console.Error.WriteLineAsync(ex.ToString()); return 1; } } @@ -274,7 +277,7 @@ ORDER BY TRY_CONVERT(int, VersionValue) DESC, TRY_CONVERT(int, VersionRevision) return new DatabaseVersionInfo( ParseVersionPart(reader.GetString(0), "VersionValue"), ParseVersionPart(reader.GetString(1), "VersionRevision"), - reader.IsDBNull(2) ? null : reader.GetString(2)); + await reader.IsDBNullAsync(2) ? null : reader.GetString(2)); } private static async Task GetMySqlDatabaseVersionAsync(MySqlConnection connection) @@ -297,7 +300,7 @@ LIMIT 1 return new DatabaseVersionInfo( ParseVersionPart(reader.GetString(0), "VersionValue"), ParseVersionPart(reader.GetString(1), "VersionRevision"), - reader.IsDBNull(2) ? null : reader.GetString(2)); + await reader.IsDBNullAsync(2) ? null : reader.GetString(2)); } private static async Task GetExpectedSourceDatabaseVersionAsync(SourceDatabaseType sourceType) @@ -321,6 +324,7 @@ private static async Task GetExpectedSourceDatabaseVersionA return latest; } + [SuppressMessage("Major Code Smell", "S3011", Justification = "The converter inspects internal database command containers to determine the expected backend migration version without duplicating migration metadata.")] private static IEnumerable GetDatabaseCommands(object database) { foreach (var field in database.GetType().GetFields(BindingFlags.Instance | BindingFlags.NonPublic)) @@ -381,7 +385,7 @@ private static async Task CopyTableAsync(DbConnection source, SourceDatabaseType Console.WriteLine($"Applying fallback values for {tableName}: {string.Join(", ", fallbackColumns.Select(column => column.Name))}"); } - await using var transaction = target.BeginTransaction(); + await using var transaction = (SqliteTransaction)await target.BeginTransactionAsync(); await using (var deleteCommand = target.CreateCommand()) { deleteCommand.Transaction = transaction; @@ -475,25 +479,8 @@ private static async Task GetRowCountAsync(SqliteConnection connection, st private static async Task VerifyTableContentAsync(DbConnection source, SourceDatabaseType sourceType, SqliteConnection target, string tableName, IReadOnlyList sharedColumns) { - var sourcePrimaryKeys = await GetSourcePrimaryKeyColumnsAsync(source, sourceType, tableName); - var targetPrimaryKeys = await GetSqlitePrimaryKeyColumnsAsync(target, tableName); var sourceColumnTypes = await GetSourceColumnTypesAsync(source, sourceType, tableName); - var orderColumns = sourcePrimaryKeys - .Where(column => targetPrimaryKeys.Contains(column, StringComparer.OrdinalIgnoreCase)) - .Where(column => sharedColumns.Contains(column, StringComparer.OrdinalIgnoreCase)) - .ToList(); - - if (orderColumns.Count == 0) - { - orderColumns = sharedColumns - .Where(column => sourceColumnTypes.TryGetValue(column, out var type) && IsSourceOrderableType(sourceType, type)) - .ToList(); - } - - if (orderColumns.Count == 0) - { - throw new InvalidOperationException($"Verification failed for {tableName}: no primary key or SQL-orderable shared columns are available for deterministic comparison."); - } + var orderColumns = await ResolveOrderColumnsAsync(source, sourceType, target, tableName, sharedColumns, sourceColumnTypes); var sourceSql = BuildSourceOrderedSelectSql(sourceType, tableName, sharedColumns, orderColumns, sourceColumnTypes); var targetSql = BuildSqliteOrderedSelectSql(tableName, sharedColumns, orderColumns); @@ -521,31 +508,79 @@ private static async Task VerifyTableContentAsync(DbConnection source, SourceDat } rowNumber++; - var sourceRow = new string[sharedColumns.Count]; - var targetRow = new string[sharedColumns.Count]; - for (var index = 0; index < sharedColumns.Count; index++) + var sourceRow = await ReadNormalizedRowAsync(sourceReader, sharedColumns.Count); + var targetRow = await ReadNormalizedRowAsync(targetReader, sharedColumns.Count); + ThrowIfRowMismatch(tableName, sharedColumns, orderColumns, rowNumber, sourceRow, targetRow); + } + } + + private static async Task> ResolveOrderColumnsAsync( + DbConnection source, + SourceDatabaseType sourceType, + SqliteConnection target, + string tableName, + IReadOnlyList sharedColumns, + IReadOnlyDictionary sourceColumnTypes) + { + var sourcePrimaryKeys = await GetSourcePrimaryKeyColumnsAsync(source, sourceType, tableName); + var targetPrimaryKeys = await GetSqlitePrimaryKeyColumnsAsync(target, tableName); + var orderColumns = sourcePrimaryKeys + .Where(column => targetPrimaryKeys.Contains(column, StringComparer.OrdinalIgnoreCase)) + .Where(column => sharedColumns.Contains(column, StringComparer.OrdinalIgnoreCase)) + .ToList(); + + if (orderColumns.Count > 0) + { + return orderColumns; + } + + orderColumns = sharedColumns + .Where(column => sourceColumnTypes.TryGetValue(column, out var type) && IsSourceOrderableType(sourceType, type)) + .ToList(); + + if (orderColumns.Count > 0) + { + return orderColumns; + } + + throw new InvalidOperationException($"Verification failed for {tableName}: no primary key or SQL-orderable shared columns are available for deterministic comparison."); + } + + private static async Task ReadNormalizedRowAsync(DbDataReader reader, int columnCount) + { + var row = new string[columnCount]; + for (var index = 0; index < columnCount; index++) + { + row[index] = NormalizeComparisonValue(await reader.IsDBNullAsync(index) ? null : reader.GetValue(index)); + } + + return row; + } + + private static void ThrowIfRowMismatch( + string tableName, + IReadOnlyList sharedColumns, + IReadOnlyList orderColumns, + long rowNumber, + IReadOnlyList sourceRow, + IReadOnlyList targetRow) + { + for (var index = 0; index < sharedColumns.Count; index++) + { + var sourceValue = sourceRow[index]; + var targetValue = targetRow[index]; + if (string.Equals(sourceValue, targetValue, StringComparison.Ordinal)) { - sourceRow[index] = NormalizeComparisonValue(await sourceReader.IsDBNullAsync(index) ? null : sourceReader.GetValue(index)); - targetRow[index] = NormalizeComparisonValue(targetReader.IsDBNull(index) ? null : targetReader.GetValue(index)); + continue; } - for (var index = 0; index < sharedColumns.Count; index++) + var keyDescription = string.Join(", ", orderColumns.Select(column => { - var sourceValue = sourceRow[index]; - var targetValue = targetRow[index]; - if (string.Equals(sourceValue, targetValue, StringComparison.Ordinal)) - { - continue; - } - - var keyDescription = string.Join(", ", orderColumns.Select(column => - { - var keyIndex = sharedColumns.IndexOf(column); - var keyValue = keyIndex >= 0 ? sourceRow[keyIndex] : ""; - return $"{column}={keyValue}"; - })); - throw new InvalidOperationException($"Verification failed for {tableName}: column {sharedColumns[index]} mismatch at row {rowNumber} ({keyDescription}). Source={sourceValue}, Target={targetValue}."); - } + var keyIndex = sharedColumns.IndexOf(column); + var keyValue = keyIndex >= 0 ? sourceRow[keyIndex] : ""; + return $"{column}={keyValue}"; + })); + throw new InvalidOperationException($"Verification failed for {tableName}: column {sharedColumns[index]} mismatch at row {rowNumber} ({keyDescription}). Source={sourceValue}, Target={targetValue}."); } } @@ -712,7 +747,7 @@ ORDER BY ORDINAL_POSITION var columns = new List(); await using var command = new SqlCommand(sql, connection); - command.Parameters.AddWithValue("@tableName", tableName); + command.Parameters.AddWithValue(TableNameParameter, tableName); await using var reader = await command.ExecuteReaderAsync(); while (await reader.ReadAsync()) { @@ -983,9 +1018,9 @@ private static async Task> GetSqliteColumnsAsync(SqliteCo columns.Add(new() { Name = reader.GetString(1), - Type = reader.IsDBNull(2) ? string.Empty : reader.GetString(2), + Type = await reader.IsDBNullAsync(2) ? string.Empty : reader.GetString(2), NotNull = reader.GetInt32(3) != 0, - DefaultValue = reader.IsDBNull(4) ? null : reader.GetString(4), + DefaultValue = await reader.IsDBNullAsync(4) ? null : reader.GetString(4), }); } @@ -997,7 +1032,7 @@ private static string BuildSourceOrderedSelectSql(SourceDatabaseType sourceType, return sourceType switch { SourceDatabaseType.SqlServer => BuildSqlServerOrderedSelectSql(tableName, selectColumns, orderColumns, columnTypes), - SourceDatabaseType.MySql => BuildMySqlOrderedSelectSql(tableName, selectColumns, orderColumns, columnTypes), + SourceDatabaseType.MySql => BuildMySqlOrderedSelectSql(tableName, selectColumns, orderColumns), _ => throw new InvalidOperationException($"Unsupported source database type: {sourceType}"), }; } @@ -1009,10 +1044,10 @@ private static string BuildSqlServerOrderedSelectSql(string tableName, IReadOnly return $"SELECT {selectList} FROM {QuoteSqlServerIdentifier(tableName)} ORDER BY {orderList};"; } - private static string BuildMySqlOrderedSelectSql(string tableName, IReadOnlyList selectColumns, IReadOnlyList orderColumns, IReadOnlyDictionary columnTypes) + private static string BuildMySqlOrderedSelectSql(string tableName, IReadOnlyList selectColumns, IReadOnlyList orderColumns) { - var selectList = string.Join(", ", selectColumns.Select(column => BuildMySqlSelectExpression(column, columnTypes))); - var orderList = string.Join(", ", orderColumns.Select(column => BuildMySqlOrderExpression(column, columnTypes))); + var selectList = string.Join(", ", selectColumns.Select(BuildMySqlSelectExpression)); + var orderList = string.Join(", ", orderColumns.Select(BuildMySqlOrderExpression)); return $"SELECT {selectList} FROM {QuoteMySqlIdentifier(tableName)} ORDER BY {orderList};"; } @@ -1039,12 +1074,12 @@ private static string BuildSqlServerOrderExpression(string columnName, IReadOnly return identifier; } - private static string BuildMySqlSelectExpression(string columnName, IReadOnlyDictionary columnTypes) + private static string BuildMySqlSelectExpression(string columnName) { return QuoteMySqlIdentifier(columnName); } - private static string BuildMySqlOrderExpression(string columnName, IReadOnlyDictionary columnTypes) + private static string BuildMySqlOrderExpression(string columnName) { return QuoteMySqlIdentifier(columnName); } @@ -1255,6 +1290,7 @@ private sealed class SqliteColumnInfo public string? DefaultValue { get; set; } } + [SuppressMessage("Major Code Smell", "S3011", Justification = "The converter must call Shoko's internal bootstrap path to create a real target database without changing server startup surface area.")] private static IHost CreateBootstrapHost(SystemService systemService, object settings) { var initWebHostMethod = typeof(SystemService).GetMethod("InitWebHost", BindingFlags.Instance | BindingFlags.NonPublic) @@ -1263,6 +1299,7 @@ private static IHost CreateBootstrapHost(SystemService systemService, object set ?? throw new InvalidOperationException("SystemService.InitWebHost returned null.")); } + [SuppressMessage("Major Code Smell", "S3011", Justification = "The converter must run Shoko's internal database initialization path so bootstrap and migrations behave exactly like server startup.")] private static bool RunInitializeDatabase(SystemService systemService, DatabaseFactory databaseFactory, RepoFactory repositoryFactory) { var initializeDatabaseMethod = typeof(SystemService).GetMethod("InitializeDatabase", BindingFlags.Instance | BindingFlags.NonPublic) @@ -1271,6 +1308,7 @@ private static bool RunInitializeDatabase(SystemService systemService, DatabaseF ?? throw new InvalidOperationException("SystemService.InitializeDatabase returned null.")); } + [SuppressMessage("Major Code Smell", "S3011", Justification = "The converter uses limited reflective access to initialize only Shoko's built-in core plugin state during isolated database bootstrap, without loading external plugins.")] private static T GetRequiredPrivateField(object instance, string fieldName) where T : class { var field = instance.GetType().GetField(fieldName, BindingFlags.Instance | BindingFlags.NonPublic) @@ -1278,6 +1316,7 @@ private static T GetRequiredPrivateField(object instance, string fieldName) w return (T)(field.GetValue(instance) ?? throw new InvalidOperationException($"Field {instance.GetType().Name}.{fieldName} was null.")); } + [SuppressMessage("Major Code Smell", "S3011", Justification = "The converter resets cached application paths between temporary SHOKO_HOME scopes to keep bootstrap isolated from the user's normal data directory.")] private static void SetPrivateStaticField(string fieldName, object? value) { var field = typeof(TDeclaring).GetField(fieldName, BindingFlags.Static | BindingFlags.NonPublic) From 87c70bee858d225793d292f59ca55f1ba84d9f46 Mon Sep 17 00:00:00 2001 From: krbrs <57227244+krbrs@users.noreply.github.com> Date: Thu, 21 May 2026 15:41:06 +0200 Subject: [PATCH 06/13] chore(cli): validate dynamic sql identifiers in db converter --- Shoko.CLI/DatabaseConverterCommand.cs | 68 ++++++++++++++++++--------- 1 file changed, 47 insertions(+), 21 deletions(-) diff --git a/Shoko.CLI/DatabaseConverterCommand.cs b/Shoko.CLI/DatabaseConverterCommand.cs index e0c8cd6e89..1128181a03 100644 --- a/Shoko.CLI/DatabaseConverterCommand.cs +++ b/Shoko.CLI/DatabaseConverterCommand.cs @@ -738,10 +738,10 @@ private static async Task> GetSourceColumnsAsync(DbConnection conne private static async Task> GetSqlServerColumnsAsync(SqlConnection connection, string tableName) { - const string sql = """ + var sql = $""" SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS - WHERE TABLE_NAME = @tableName + WHERE TABLE_NAME = {TableNameParameter} ORDER BY ORDINAL_POSITION """; @@ -759,17 +759,17 @@ ORDER BY ORDINAL_POSITION private static async Task> GetMySqlColumnsAsync(MySqlConnection connection, string tableName) { - const string sql = """ + var sql = $""" SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() - AND TABLE_NAME = @tableName + AND TABLE_NAME = {TableNameParameter} ORDER BY ORDINAL_POSITION """; var columns = new List(); await using var command = new MySqlCommand(sql, connection); - command.Parameters.AddWithValue("@tableName", tableName); + command.Parameters.AddWithValue(TableNameParameter, tableName); await using var reader = await command.ExecuteReaderAsync(); while (await reader.ReadAsync()) { @@ -791,17 +791,17 @@ private static async Task> GetSourcePrimaryKeyColumnsAsync(DbConnec private static async Task> GetSqlServerPrimaryKeyColumnsAsync(SqlConnection connection, string tableName) { - const string sql = """ + var sql = $""" SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE WHERE OBJECTPROPERTY(OBJECT_ID(CONSTRAINT_SCHEMA + '.' + QUOTENAME(CONSTRAINT_NAME)), 'IsPrimaryKey') = 1 - AND TABLE_NAME = @tableName + AND TABLE_NAME = {TableNameParameter} ORDER BY ORDINAL_POSITION """; var columns = new List(); await using var command = new SqlCommand(sql, connection); - command.Parameters.AddWithValue("@tableName", tableName); + command.Parameters.AddWithValue(TableNameParameter, tableName); await using var reader = await command.ExecuteReaderAsync(); while (await reader.ReadAsync()) { @@ -813,18 +813,18 @@ ORDER BY ORDINAL_POSITION private static async Task> GetMySqlPrimaryKeyColumnsAsync(MySqlConnection connection, string tableName) { - const string sql = """ + var sql = $""" SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA = DATABASE() - AND TABLE_NAME = @tableName + AND TABLE_NAME = {TableNameParameter} AND CONSTRAINT_NAME = 'PRIMARY' ORDER BY ORDINAL_POSITION """; var columns = new List(); await using var command = new MySqlCommand(sql, connection); - command.Parameters.AddWithValue("@tableName", tableName); + command.Parameters.AddWithValue(TableNameParameter, tableName); await using var reader = await command.ExecuteReaderAsync(); while (await reader.ReadAsync()) { @@ -846,15 +846,15 @@ private static async Task> GetSourceColumnTypesAsync( private static async Task> GetSqlServerColumnTypesAsync(SqlConnection connection, string tableName) { - const string sql = """ + var sql = $""" SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS - WHERE TABLE_NAME = @tableName + WHERE TABLE_NAME = {TableNameParameter} """; var columns = new Dictionary(StringComparer.OrdinalIgnoreCase); await using var command = new SqlCommand(sql, connection); - command.Parameters.AddWithValue("@tableName", tableName); + command.Parameters.AddWithValue(TableNameParameter, tableName); await using var reader = await command.ExecuteReaderAsync(); while (await reader.ReadAsync()) { @@ -866,16 +866,16 @@ FROM INFORMATION_SCHEMA.COLUMNS private static async Task> GetMySqlColumnTypesAsync(MySqlConnection connection, string tableName) { - const string sql = """ + var sql = $""" SELECT COLUMN_NAME, DATA_TYPE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = DATABASE() - AND TABLE_NAME = @tableName + AND TABLE_NAME = {TableNameParameter} """; var columns = new Dictionary(StringComparer.OrdinalIgnoreCase); await using var command = new MySqlCommand(sql, connection); - command.Parameters.AddWithValue("@tableName", tableName); + command.Parameters.AddWithValue(TableNameParameter, tableName); await using var reader = await command.ExecuteReaderAsync(); while (await reader.ReadAsync()) { @@ -1142,11 +1142,37 @@ private static string NormalizeNumericValue(T value) }; } + private static string ValidateIdentifier(string identifier) + { + if (string.IsNullOrWhiteSpace(identifier)) + { + throw new InvalidOperationException("SQL identifier cannot be null or empty."); + } + + if (!(char.IsLetter(identifier[0]) || identifier[0] == '_')) + { + throw new InvalidOperationException($"Unsupported SQL identifier: {identifier}"); + } + + for (var index = 1; index < identifier.Length; index++) + { + var character = identifier[index]; + if (char.IsLetterOrDigit(character) || character == '_') + { + continue; + } + + throw new InvalidOperationException($"Unsupported SQL identifier: {identifier}"); + } + + return identifier; + } + private static string QuoteSqlServerIdentifier(string identifier) - => $"[{identifier.Replace("]", "]]", StringComparison.Ordinal)}]"; + => $"[{ValidateIdentifier(identifier).Replace("]", "]]", StringComparison.Ordinal)}]"; private static string QuoteMySqlIdentifier(string identifier) - => $"`{identifier.Replace("`", "``", StringComparison.Ordinal)}`"; + => $"`{ValidateIdentifier(identifier).Replace("`", "``", StringComparison.Ordinal)}`"; private static string QuoteSourceIdentifier(SourceDatabaseType sourceType, string identifier) { @@ -1159,10 +1185,10 @@ private static string QuoteSourceIdentifier(SourceDatabaseType sourceType, strin } private static string QuoteSqliteIdentifier(string identifier) - => $"\"{identifier.Replace("\"", "\"\"", StringComparison.Ordinal)}\""; + => $"\"{ValidateIdentifier(identifier).Replace("\"", "\"\"", StringComparison.Ordinal)}\""; private static string QuoteSqliteLiteral(string value) - => $"'{value.Replace("'", "''", StringComparison.Ordinal)}'"; + => $"'{ValidateIdentifier(value).Replace("'", "''", StringComparison.Ordinal)}'"; private static Options ParseArgs(string[] args) { From dba5b73c605fd724166d10a82d073d8bada01b76 Mon Sep 17 00:00:00 2001 From: krbrs <57227244+krbrs@users.noreply.github.com> Date: Thu, 21 May 2026 18:47:04 +0200 Subject: [PATCH 07/13] fix(cli): normalize guid values during sqlite conversion --- Shoko.CLI/DatabaseConverterCommand.cs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/Shoko.CLI/DatabaseConverterCommand.cs b/Shoko.CLI/DatabaseConverterCommand.cs index 1128181a03..3b16bf11ce 100644 --- a/Shoko.CLI/DatabaseConverterCommand.cs +++ b/Shoko.CLI/DatabaseConverterCommand.cs @@ -595,6 +595,9 @@ private static void AppendValue(List buffer, object? value) case byte[] bytes: buffer.AddRange(Convert.ToHexString(bytes).Select(c => (byte)c)); break; + case string text when TryNormalizeGuidString(text, out var normalizedGuidText): + buffer.AddRange(Encoding.UTF8.GetBytes(normalizedGuidText)); + break; case string text when TryNormalizeTemporalString(text, out var normalizedText): buffer.AddRange(Encoding.UTF8.GetBytes(normalizedText)); break; @@ -621,7 +624,8 @@ private static object NormalizeValue(object value) return value switch { DBNull => DBNull.Value, - Guid guid => guid.ToString(), + Guid guid => NormalizeGuid(guid), + string text when TryNormalizeGuidString(text, out var normalizedGuidText) => normalizedGuidText, DateTimeOffset offset => offset.UtcDateTime, _ => value, }; @@ -655,6 +659,23 @@ private static bool TryNormalizeTemporalString(string value, out string normaliz return false; } + private static bool TryNormalizeGuidString(string value, out string normalized) + { + if (Guid.TryParse(value, out var guid)) + { + normalized = NormalizeGuid(guid); + return true; + } + + normalized = string.Empty; + return false; + } + + private static string NormalizeGuid(Guid guid) + { + return guid.ToString("D").ToUpperInvariant(); + } + private static async Task> GetSourceTablesAsync(DbConnection connection, SourceDatabaseType sourceType) { return sourceType switch @@ -1110,6 +1131,8 @@ private static string NormalizeComparisonValue(object? value) { null or DBNull => "", byte[] bytes => Convert.ToHexString(bytes), + Guid guid => NormalizeGuid(guid), + string text when TryNormalizeGuidString(text, out var normalizedGuidText) => normalizedGuidText, string text when TryNormalizeTemporalString(text, out var normalizedText) => normalizedText, DateTime dateTime => NormalizeDateTime(dateTime), DateTimeOffset offset => offset.ToString("O", CultureInfo.InvariantCulture), From 9c511f7f0bba69e317f3d3c61491300938551475 Mon Sep 17 00:00:00 2001 From: krbrs <57227244+krbrs@users.noreply.github.com> Date: Thu, 21 May 2026 19:29:00 +0200 Subject: [PATCH 08/13] fix(cli): limit guid normalization to guid columns --- Shoko.CLI/DatabaseConverterCommand.cs | 92 +++++++++++++-------------- 1 file changed, 44 insertions(+), 48 deletions(-) diff --git a/Shoko.CLI/DatabaseConverterCommand.cs b/Shoko.CLI/DatabaseConverterCommand.cs index 3b16bf11ce..02c73f89a9 100644 --- a/Shoko.CLI/DatabaseConverterCommand.cs +++ b/Shoko.CLI/DatabaseConverterCommand.cs @@ -8,8 +8,6 @@ using System.Collections; using System.Linq; using System.Reflection; -using System.Security.Cryptography; -using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Data.SqlClient; @@ -364,9 +362,14 @@ private static int ParseVersionPart(string value, string columnName) private static async Task CopyTableAsync(DbConnection source, SourceDatabaseType sourceType, SqliteConnection target, string tableName) { var sourceColumns = await GetSourceColumnsAsync(source, sourceType, tableName); + var sourceColumnTypes = await GetSourceColumnTypesAsync(source, sourceType, tableName); var targetColumns = await GetSqliteColumnsAsync(target, tableName); var sourceColumnSet = sourceColumns.ToHashSet(StringComparer.OrdinalIgnoreCase); var sourceBackedColumns = targetColumns.Where(column => sourceColumnSet.Contains(column.Name)).ToList(); + var guidColumns = sourceBackedColumns + .Where(column => IsGuidColumn(sourceType, sourceColumnTypes, column)) + .Select(column => column.Name) + .ToHashSet(StringComparer.OrdinalIgnoreCase); var fallbackColumns = targetColumns .Where(column => !sourceColumnSet.Contains(column.Name) && column.NotNull && string.IsNullOrWhiteSpace(column.DefaultValue)) .ToList(); @@ -414,7 +417,7 @@ private static async Task CopyTableAsync(DbConnection source, SourceDatabaseType for (var index = 0; index < sourceBackedColumns.Count; index++) { var value = await reader.IsDBNullAsync(index) ? DBNull.Value : reader.GetValue(index); - insertCommand.Parameters[index].Value = NormalizeValue(value); + insertCommand.Parameters[index].Value = NormalizeValue(value, guidColumns.Contains(sourceBackedColumns[index].Name)); } for (var index = 0; index < fallbackColumns.Count; index++) @@ -480,6 +483,11 @@ private static async Task GetRowCountAsync(SqliteConnection connection, st private static async Task VerifyTableContentAsync(DbConnection source, SourceDatabaseType sourceType, SqliteConnection target, string tableName, IReadOnlyList sharedColumns) { var sourceColumnTypes = await GetSourceColumnTypesAsync(source, sourceType, tableName); + var targetColumns = await GetSqliteColumnsAsync(target, tableName); + var targetColumnLookup = targetColumns.ToDictionary(column => column.Name, StringComparer.OrdinalIgnoreCase); + var guidColumns = sharedColumns + .Where(column => targetColumnLookup.TryGetValue(column, out var targetColumn) && IsGuidColumn(sourceType, sourceColumnTypes, targetColumn)) + .ToHashSet(StringComparer.OrdinalIgnoreCase); var orderColumns = await ResolveOrderColumnsAsync(source, sourceType, target, tableName, sharedColumns, sourceColumnTypes); var sourceSql = BuildSourceOrderedSelectSql(sourceType, tableName, sharedColumns, orderColumns, sourceColumnTypes); @@ -508,8 +516,8 @@ private static async Task VerifyTableContentAsync(DbConnection source, SourceDat } rowNumber++; - var sourceRow = await ReadNormalizedRowAsync(sourceReader, sharedColumns.Count); - var targetRow = await ReadNormalizedRowAsync(targetReader, sharedColumns.Count); + var sourceRow = await ReadNormalizedRowAsync(sourceReader, sharedColumns, guidColumns); + var targetRow = await ReadNormalizedRowAsync(targetReader, sharedColumns, guidColumns); ThrowIfRowMismatch(tableName, sharedColumns, orderColumns, rowNumber, sourceRow, targetRow); } } @@ -546,12 +554,12 @@ private static async Task> ResolveOrderColumnsAsync( throw new InvalidOperationException($"Verification failed for {tableName}: no primary key or SQL-orderable shared columns are available for deterministic comparison."); } - private static async Task ReadNormalizedRowAsync(DbDataReader reader, int columnCount) + private static async Task ReadNormalizedRowAsync(DbDataReader reader, IReadOnlyList columnNames, IReadOnlySet guidColumns) { - var row = new string[columnCount]; - for (var index = 0; index < columnCount; index++) + var row = new string[columnNames.Count]; + for (var index = 0; index < columnNames.Count; index++) { - row[index] = NormalizeComparisonValue(await reader.IsDBNullAsync(index) ? null : reader.GetValue(index)); + row[index] = NormalizeComparisonValue(await reader.IsDBNullAsync(index) ? null : reader.GetValue(index), guidColumns.Contains(columnNames[index])); } return row; @@ -584,48 +592,13 @@ private static void ThrowIfRowMismatch( } } - private static void AppendValue(List buffer, object? value) - { - switch (value) - { - case null: - case DBNull: - buffer.AddRange("NULL"u8.ToArray()); - break; - case byte[] bytes: - buffer.AddRange(Convert.ToHexString(bytes).Select(c => (byte)c)); - break; - case string text when TryNormalizeGuidString(text, out var normalizedGuidText): - buffer.AddRange(Encoding.UTF8.GetBytes(normalizedGuidText)); - break; - case string text when TryNormalizeTemporalString(text, out var normalizedText): - buffer.AddRange(Encoding.UTF8.GetBytes(normalizedText)); - break; - case DateTime dateTime: - buffer.AddRange(Encoding.UTF8.GetBytes(NormalizeDateTime(dateTime))); - break; - case DateTimeOffset offset: - buffer.AddRange(Encoding.UTF8.GetBytes(offset.ToString("O", CultureInfo.InvariantCulture))); - break; - case bool boolean: - buffer.AddRange(boolean ? "1"u8.ToArray() : "0"u8.ToArray()); - break; - case IFormattable formattable: - buffer.AddRange(Encoding.UTF8.GetBytes(formattable.ToString(null, CultureInfo.InvariantCulture) ?? string.Empty)); - break; - default: - buffer.AddRange(Encoding.UTF8.GetBytes(value.ToString() ?? string.Empty)); - break; - } - } - - private static object NormalizeValue(object value) + private static object NormalizeValue(object value, bool isGuidColumn) { return value switch { DBNull => DBNull.Value, Guid guid => NormalizeGuid(guid), - string text when TryNormalizeGuidString(text, out var normalizedGuidText) => normalizedGuidText, + string text when isGuidColumn && TryNormalizeGuidString(text, out var normalizedGuidText) => normalizedGuidText, DateTimeOffset offset => offset.UtcDateTime, _ => value, }; @@ -676,6 +649,29 @@ private static string NormalizeGuid(Guid guid) return guid.ToString("D").ToUpperInvariant(); } + private static bool IsGuidColumn(SourceDatabaseType sourceType, IReadOnlyDictionary sourceColumnTypes, SqliteColumnInfo targetColumn) + { + if (IsGuidType(targetColumn.Type)) + { + return true; + } + + return sourceColumnTypes.TryGetValue(targetColumn.Name, out var sourceTypeName) && IsGuidType(sourceType, sourceTypeName); + } + + private static bool IsGuidType(SourceDatabaseType sourceType, string typeName) + { + return sourceType switch + { + SourceDatabaseType.SqlServer => typeName.Equals("uniqueidentifier", StringComparison.OrdinalIgnoreCase), + SourceDatabaseType.MySql => false, + _ => false, + }; + } + + private static bool IsGuidType(string typeName) + => typeName.Contains("UNIQUEIDENTIFIER", StringComparison.OrdinalIgnoreCase); + private static async Task> GetSourceTablesAsync(DbConnection connection, SourceDatabaseType sourceType) { return sourceType switch @@ -1125,14 +1121,14 @@ private static bool IsMySqlOrderableType(string dataType) !dataType.Equals("json", StringComparison.OrdinalIgnoreCase) && !dataType.Equals("geometry", StringComparison.OrdinalIgnoreCase); - private static string NormalizeComparisonValue(object? value) + private static string NormalizeComparisonValue(object? value, bool isGuidColumn) { return value switch { null or DBNull => "", byte[] bytes => Convert.ToHexString(bytes), Guid guid => NormalizeGuid(guid), - string text when TryNormalizeGuidString(text, out var normalizedGuidText) => normalizedGuidText, + string text when isGuidColumn && TryNormalizeGuidString(text, out var normalizedGuidText) => normalizedGuidText, string text when TryNormalizeTemporalString(text, out var normalizedText) => normalizedText, DateTime dateTime => NormalizeDateTime(dateTime), DateTimeOffset offset => offset.ToString("O", CultureInfo.InvariantCulture), From e4064b1e833eb874f35ba392a75cb32ee19d36f7 Mon Sep 17 00:00:00 2001 From: krbrs <57227244+krbrs@users.noreply.github.com> Date: Thu, 21 May 2026 23:30:49 +0200 Subject: [PATCH 09/13] fix: isolate conversion startup side effects Run conversion mode before hosted services and web endpoints start, and wrap early startup in a temporary conversion Shoko home. This keeps Quartz, bootstrap, and other startup-time filesystem/database initialization from touching the user's real Shoko home during conversion. Source settings are copied before isolation, while the target SQLite file still uses the resolved conversion output path. --- Shoko.CLI/Program.cs | 7 +- Shoko.Server/Properties/AssemblyInfo.cs | 2 + Shoko.Server/Services/ApplicationPaths.cs | 43 +- .../Services/DatabaseConversionService.cs | 595 +++++++++++------- Shoko.Server/Services/SystemService.cs | 111 +++- Shoko.Server/Utilities/Utils.cs | 46 +- Shoko.Tests/DatabaseConversionOptionsTests.cs | 268 ++++++++ Shoko.TrayService/App.axaml.cs | 2 +- Shoko.TrayService/Program.cs | 3 + 9 files changed, 814 insertions(+), 263 deletions(-) rename Shoko.CLI/DatabaseConverterCommand.cs => Shoko.Server/Services/DatabaseConversionService.cs (77%) create mode 100644 Shoko.Tests/DatabaseConversionOptionsTests.cs diff --git a/Shoko.CLI/Program.cs b/Shoko.CLI/Program.cs index c6acd06bd3..edd159a9e9 100644 --- a/Shoko.CLI/Program.cs +++ b/Shoko.CLI/Program.cs @@ -17,11 +17,6 @@ public static class Program public static async Task Main(string[] args) { - if (args.Length > 0 && string.Equals(args[0], "convert-db", StringComparison.OrdinalIgnoreCase)) - { - return await DatabaseConverterCommand.RunAsync(args[1..]); - } - try { UnhandledExceptionManager.AddHandler(); @@ -31,7 +26,7 @@ public static async Task Main(string[] args) Console.WriteLine(ex.ToString()); } - var systemService = new SystemService(); + var systemService = new SystemService(args); systemService.StartupFailed += OnStartupFailed; systemService.AboutToStart += (_, args) => AddEventHandlers(args.ServiceProvider); var host = await systemService.StartAsync(); diff --git a/Shoko.Server/Properties/AssemblyInfo.cs b/Shoko.Server/Properties/AssemblyInfo.cs index 6ce951098b..eaad376852 100755 --- a/Shoko.Server/Properties/AssemblyInfo.cs +++ b/Shoko.Server/Properties/AssemblyInfo.cs @@ -1,4 +1,5 @@ using System.Reflection; +using System.Runtime.CompilerServices; using System.Runtime.InteropServices; // General Information about an assembly is controlled through the following @@ -11,6 +12,7 @@ // COM, set the ComVisible attribute to true on that type. [assembly: ComVisible(false)] +[assembly: InternalsVisibleTo("Shoko.Tests")] //In order to begin building localizable applications, set //CultureYouAreCodingWith in your .csproj file diff --git a/Shoko.Server/Services/ApplicationPaths.cs b/Shoko.Server/Services/ApplicationPaths.cs index 106b63c053..067c7aec8a 100644 --- a/Shoko.Server/Services/ApplicationPaths.cs +++ b/Shoko.Server/Services/ApplicationPaths.cs @@ -2,6 +2,7 @@ using System; using System.IO; using System.Reflection; +using System.Threading; using Shoko.Abstractions.Plugin; using Shoko.Abstractions.Utilities; using Shoko.Server.Utilities; @@ -23,10 +24,13 @@ public string ApplicationPath => _applicationPath ??= Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location)!; private string? _webPath = null; + private static readonly AsyncLocal s_dataPathOverride = new(); /// public string WebPath - => _webPath ??= Path.Combine(DataPath, Utils.SettingsProvider.GetSettings().Web.WebUIPath); + => s_dataPathOverride.Value is { Length: > 0 } + ? Path.Combine(DataPath, Utils.SettingsProvider.GetSettings().Web.WebUIPath) + : _webPath ??= Path.Combine(DataPath, Utils.SettingsProvider.GetSettings().Web.WebUIPath); private static string? _dataPath = null; @@ -37,6 +41,9 @@ public static string StaticDataPath { get { + if (s_dataPathOverride.Value is { Length: > 0 } overriddenPath) + return overriddenPath; + if (_dataPath != null) return _dataPath; @@ -94,9 +101,13 @@ public static void SetHome(string[] args) /// public string ImagesPath - => _imagesPath ??= Utils.SettingsProvider.GetSettings().ImagesPath is { Length: > 0 } imagePath - ? Path.Combine(DataPath, imagePath) - : DefaultImagePath; + => s_dataPathOverride.Value is { Length: > 0 } + ? Utils.SettingsProvider.GetSettings().ImagesPath is { Length: > 0 } configuredImagePath + ? Path.Combine(DataPath, configuredImagePath) + : DefaultImagePath + : _imagesPath ??= Utils.SettingsProvider.GetSettings().ImagesPath is { Length: > 0 } cachedConfiguredImagePath + ? Path.Combine(DataPath, cachedConfiguredImagePath) + : DefaultImagePath; public static string DefaultImagePath => Path.Combine(StaticDataPath, "images"); @@ -114,4 +125,28 @@ public string ConfigurationsPath /// public string LogsPath => Path.Combine(DataPath, "logs"); + + internal static IDisposable PushDataPathOverride(string absolutePath) + => new ScopedDataPathOverride(absolutePath); + + private sealed class ScopedDataPathOverride : IDisposable + { + private readonly string? _previousValue; + private bool _disposed; + + public ScopedDataPathOverride(string value) + { + _previousValue = s_dataPathOverride.Value; + s_dataPathOverride.Value = value; + } + + public void Dispose() + { + if (_disposed) + return; + + s_dataPathOverride.Value = _previousValue; + _disposed = true; + } + } } diff --git a/Shoko.CLI/DatabaseConverterCommand.cs b/Shoko.Server/Services/DatabaseConversionService.cs similarity index 77% rename from Shoko.CLI/DatabaseConverterCommand.cs rename to Shoko.Server/Services/DatabaseConversionService.cs index 02c73f89a9..91dc5664b4 100644 --- a/Shoko.CLI/DatabaseConverterCommand.cs +++ b/Shoko.Server/Services/DatabaseConversionService.cs @@ -10,96 +10,264 @@ using System.Reflection; using System.Threading; using System.Threading.Tasks; +using Force.DeepCloner; using Microsoft.Data.SqlClient; using Microsoft.Data.Sqlite; using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; using MySqlConnector; -using Shoko.Abstractions.Plugin.Models; -using Shoko.Abstractions.Extensions; +using Shoko.Abstractions.Plugin; using Shoko.Server.Databases; -using Shoko.Server.Plugin; using Shoko.Server.Repositories; -using Shoko.Server.Services; using Shoko.Server.Utilities; -namespace Shoko.CLI; +using ISettingsProvider = Shoko.Server.Settings.ISettingsProvider; -internal static class DatabaseConverterCommand +#nullable enable +namespace Shoko.Server.Services; + +internal sealed class DatabaseConversionOptions +{ + public DatabaseConversionService.SourceDatabaseType SourceType { get; set; } = DatabaseConversionService.SourceDatabaseType.SqlServer; + public bool SourceTypeProvided { get; set; } + public string SourceConnectionString { get; set; } = string.Empty; + public bool SourceConnectionStringProvided { get; set; } + public string TargetFile { get; set; } = string.Empty; + public bool TargetFileProvided { get; set; } + public bool Overwrite { get; set; } + public bool ShowHelp { get; set; } + + public static bool TryParse(string[] args, [NotNullWhen(true)] out DatabaseConversionOptions? options) + { + options = null; + var conversionModeDetected = DetectConversionMode(args); + if (!conversionModeDetected) + { + return false; + } + + var parsed = new DatabaseConversionOptions(); + for (var index = 0; index < args.Length; index++) + { + var argument = args[index]; + switch (argument) + { + case "convert-db": + case "--convert-db": + break; + case "--source-type": + parsed.SourceType = DatabaseConversionService.ParseSourceType(GetRequiredValue(args, ref index, argument)); + parsed.SourceTypeProvided = true; + break; + case "--source-connection-string": + parsed.SourceConnectionString = GetRequiredValue(args, ref index, argument); + parsed.SourceConnectionStringProvided = true; + break; + case "--target-file": + parsed.TargetFile = GetRequiredValue(args, ref index, argument); + parsed.TargetFileProvided = true; + break; + case "--overwrite": + parsed.Overwrite = true; + break; + case "--help": + case "-h": + case "/?": + parsed.ShowHelp = true; + break; + } + } + + options = parsed; + return true; + } + + private static bool DetectConversionMode(string[] args) + { + if (args.Any(argument => string.Equals(argument, "--convert-db", StringComparison.OrdinalIgnoreCase))) + { + return true; + } + + for (var index = 0; index < args.Length; index++) + { + var argument = args[index]; + if (argument.StartsWith("-", StringComparison.Ordinal)) + { + continue; + } + + return string.Equals(argument, "convert-db", StringComparison.OrdinalIgnoreCase) && + (index is 0 || !args[index - 1].StartsWith("-", StringComparison.Ordinal)); + } + + return false; + } + + private static string GetRequiredValue(string[] args, ref int index, string argumentName) + { + if (index + 1 >= args.Length) + { + throw new ArgumentException($"Missing value for {argumentName}"); + } + + index++; + return args[index]; + } +} + +internal static class DatabaseConversionService { private const string TableNameParameter = "@tableName"; - private enum SourceDatabaseType + internal enum SourceDatabaseType { SqlServer, MySql, } + internal readonly record struct ResolvedSource(SourceDatabaseType SourceType, string SourceConnectionString); + internal readonly record struct ResolvedTarget(string TargetFile); + internal readonly record struct ConversionRuntimeContext( + DatabaseConversionOptions Options, + ResolvedSource Source, + string PreparedTargetFile, + string TargetConnectionString, + string TemporaryHomePath, + ISettingsProvider RuntimeSettingsProvider); + private static readonly HashSet ExcludedTables = new(StringComparer.OrdinalIgnoreCase) { "Versions", }; - public static async Task RunAsync(string[] args) + internal static ConversionRuntimeContext PrepareRuntime(DatabaseConversionOptions options, Shoko.Server.Settings.ServerSettings realSettings) { - var options = ParseArgs(args); if (options.ShowHelp) { - PrintUsage(); - return 0; + throw new InvalidOperationException("Help output does not require conversion runtime preparation."); } - if (string.IsNullOrWhiteSpace(options.SourceConnectionString) || string.IsNullOrWhiteSpace(options.TargetFile)) + var resolvedSource = ResolveSource(options, realSettings); + var resolvedTarget = ResolveTarget(options, realSettings); + var preparedTargetFile = PrepareTargetPath(resolvedTarget.TargetFile, options.Overwrite); + var targetConnectionString = BuildSqliteConnectionString(preparedTargetFile); + var temporaryHomePath = Path.Combine(Path.GetTempPath(), $"shoko-convert-home-{Guid.NewGuid():N}"); + var runtimeSettings = realSettings.DeepClone(); + runtimeSettings.Database.Type = Shoko.Server.Server.Constants.DatabaseType.SQLite; + runtimeSettings.Database.OverrideConnectionString = targetConnectionString; + runtimeSettings.Quartz.DatabaseType = Shoko.Server.Server.Constants.DatabaseType.SQLite; + runtimeSettings.Quartz.ConnectionString = BuildTemporaryQuartzConnectionString(temporaryHomePath); + return new(options, resolvedSource, preparedTargetFile, targetConnectionString, temporaryHomePath, new InMemorySettingsProvider(runtimeSettings)); + } + + internal static IDisposable BeginIsolatedRuntime(ConversionRuntimeContext context) + => new ConversionIsolationScope(context); + + internal static async Task RunAsync(SystemService systemService, IServiceProvider services, ConversionRuntimeContext context, CancellationToken cancellationToken) + { + await ConvertAsync(systemService, services, context.Source.SourceType, context.Source.SourceConnectionString, context.PreparedTargetFile, cancellationToken); + } + + internal static ResolvedSource ResolveSource(DatabaseConversionOptions options, Shoko.Server.Settings.IServerSettings settings) + { + var configuredSourceType = TryResolveConfiguredSourceType(settings.Database.Type); + var resolvedSourceType = options.SourceTypeProvided + ? options.SourceType + : configuredSourceType ?? throw new InvalidOperationException( + "The current configured source database is SQLite. Conversion only supports SQL Server/MySQL/MariaDB -> SQLite. " + + "Provide both --source-type and --source-connection-string to convert from an external supported source."); + + var resolvedConnectionString = options.SourceConnectionStringProvided + ? options.SourceConnectionString + : ResolveConfiguredSourceConnectionString(settings.Database, configuredSourceType, resolvedSourceType); + + return new(resolvedSourceType, resolvedConnectionString); + } + + internal static ResolvedTarget ResolveTarget(DatabaseConversionOptions options, Shoko.Server.Settings.IServerSettings settings) + { + if (options.TargetFileProvided) { - PrintUsage(); - return 1; + if (string.IsNullOrWhiteSpace(options.TargetFile)) + { + throw new InvalidOperationException(GetUsage()); + } + + return new(Path.GetFullPath(options.TargetFile)); } - try + return new(GetDefaultSqliteTargetPath(settings.Database)); + } + + private static string ResolveConfiguredSourceConnectionString( + Shoko.Server.Settings.DatabaseSettings settings, + SourceDatabaseType? configuredSourceType, + SourceDatabaseType requestedSourceType) + { + if (!configuredSourceType.HasValue) { - await ConvertAsync(options.SourceType, options.SourceConnectionString, options.TargetFile, options.Overwrite); - return 0; + throw new InvalidOperationException( + "The current configured source database is SQLite. Conversion only supports SQL Server/MySQL/MariaDB -> SQLite. " + + "Provide --source-connection-string to override the source details."); } - catch (Exception ex) + + if (configuredSourceType.Value != requestedSourceType) { - await Console.Error.WriteLineAsync($"Database conversion failed: {ex.Message}"); - await Console.Error.WriteLineAsync(ex.ToString()); - return 1; + throw new InvalidOperationException( + $"The current configured source database is {GetSourceTypeDisplayName(configuredSourceType.Value)}, but the requested source type is {GetSourceTypeDisplayName(requestedSourceType)}. " + + "Provide --source-connection-string when overriding the source type."); } + + return BuildConfiguredSourceConnectionString(settings, configuredSourceType.Value); } - private static async Task ConvertAsync(SourceDatabaseType sourceType, string sourceConnectionString, string targetFile, bool overwrite) + private static SourceDatabaseType? TryResolveConfiguredSourceType(Shoko.Server.Server.Constants.DatabaseType configuredType) { - var fullTargetPath = Path.GetFullPath(targetFile); - if (File.Exists(fullTargetPath)) + return configuredType switch { - if (!overwrite) - { - throw new InvalidOperationException($"Target file already exists: {fullTargetPath}. Use --overwrite to replace it."); - } + Shoko.Server.Server.Constants.DatabaseType.SQLServer => SourceDatabaseType.SqlServer, + Shoko.Server.Server.Constants.DatabaseType.MySQL => SourceDatabaseType.MySql, + Shoko.Server.Server.Constants.DatabaseType.SQLite => null, + _ => null, + }; + } - File.Delete(fullTargetPath); + private static string BuildConfiguredSourceConnectionString(Shoko.Server.Settings.DatabaseSettings settings, SourceDatabaseType sourceType) + { + if (!string.IsNullOrWhiteSpace(settings.OverrideConnectionString)) + { + return settings.OverrideConnectionString; } - var targetDirectory = Path.GetDirectoryName(fullTargetPath); - if (!string.IsNullOrWhiteSpace(targetDirectory)) + return sourceType switch { - Directory.CreateDirectory(targetDirectory); - } + SourceDatabaseType.SqlServer => + $"data source={settings.Hostname},{settings.Port};Initial Catalog={settings.Schema};user id={settings.Username};password={settings.Password};persist security info=True;MultipleActiveResultSets=True;TrustServerCertificate=True", + SourceDatabaseType.MySql => + $"Server={settings.Hostname};Port={settings.Port};Database={settings.Schema};User ID={settings.Username};Password={settings.Password};Default Command Timeout=3600;Allow User Variables=true", + _ => throw new InvalidOperationException($"Unsupported source database type: {sourceType}"), + }; + } - await using var source = await OpenSourceConnectionAsync(sourceType, sourceConnectionString); - await EnsureSourceVersionSupportedAsync(source, sourceType); + private static string GetDefaultSqliteTargetPath(Shoko.Server.Settings.DatabaseSettings settings) + { + var databaseDirectory = string.IsNullOrWhiteSpace(settings.MySqliteDirectory) + ? ApplicationPaths.StaticDataPath + : Path.Combine(ApplicationPaths.StaticDataPath, settings.MySqliteDirectory); + var databaseFile = string.IsNullOrWhiteSpace(settings.SQLite_DatabaseFile) ? "Shoko.db3" : settings.SQLite_DatabaseFile; + return Path.GetFullPath(Path.Combine(databaseDirectory, databaseFile)); + } - var targetConnectionString = new SqliteConnectionStringBuilder - { - DataSource = fullTargetPath, - Mode = SqliteOpenMode.ReadWriteCreate, - Pooling = false, - }.ToString(); + private static async Task ConvertAsync(SystemService systemService, IServiceProvider services, SourceDatabaseType sourceType, string sourceConnectionString, string targetFile, CancellationToken cancellationToken) + { + Console.WriteLine($"Resolved target SQLite path: {targetFile}"); - await InitializeSqliteDatabaseAsync(targetConnectionString); + await using var source = await OpenSourceConnectionAsync(sourceType, sourceConnectionString); + await EnsureSourceVersionSupportedAsync(systemService, source, sourceType); - await using var target = new SqliteConnection(targetConnectionString); + await InitializeSqliteDatabaseAsync(systemService, services, cancellationToken); + + await using var target = new SqliteConnection(BuildSqliteConnectionString(targetFile)); await target.OpenAsync(); await ConfigureSqliteAsync(target); @@ -159,10 +327,32 @@ private static async Task ConvertAsync(SourceDatabaseType sourceType, string sou } await VerifyCopyAsync(source, sourceType, target, tablesToCopy); - Console.WriteLine($"Conversion completed successfully: {fullTargetPath}"); + Console.WriteLine($"Conversion completed successfully: {targetFile}"); } - private static async Task EnsureSourceVersionSupportedAsync(DbConnection source, SourceDatabaseType sourceType) + internal static string PrepareTargetPath(string targetFile, bool overwrite) + { + var fullTargetPath = Path.GetFullPath(targetFile); + if (File.Exists(fullTargetPath)) + { + if (!overwrite) + { + throw new InvalidOperationException($"Target file already exists: {fullTargetPath}. Use --overwrite to replace it."); + } + + File.Delete(fullTargetPath); + } + + var targetDirectory = Path.GetDirectoryName(fullTargetPath); + if (!string.IsNullOrWhiteSpace(targetDirectory)) + { + Directory.CreateDirectory(targetDirectory); + } + + return fullTargetPath; + } + + private static async Task EnsureSourceVersionSupportedAsync(SystemService systemService, DbConnection source, SourceDatabaseType sourceType) { var sourceVersion = await GetSourceDatabaseVersionAsync(source, sourceType); if (sourceVersion is null) @@ -170,7 +360,7 @@ private static async Task EnsureSourceVersionSupportedAsync(DbConnection source, throw new InvalidOperationException("The source database does not contain a current Database version entry in Versions. Upgrade it with the matching Shoko Server build before conversion."); } - var expectedVersion = await GetExpectedSourceDatabaseVersionAsync(sourceType); + var expectedVersion = GetExpectedSourceDatabaseVersion(systemService, sourceType); if (sourceVersion.Value.Version != expectedVersion.Version || sourceVersion.Value.Revision != expectedVersion.Revision) { throw new InvalidOperationException( @@ -191,31 +381,102 @@ private static string GetSourceTypeDisplayName(SourceDatabaseType sourceType) }; } - private static async Task InitializeSqliteDatabaseAsync(string connectionString) + private static async Task InitializeSqliteDatabaseAsync(SystemService systemService, IServiceProvider services, CancellationToken cancellationToken) { - await using var bootstrapHome = new TemporaryShokoHomeScope(); - - var systemService = new SystemService(); - var settings = Utils.SettingsProvider.GetSettings(); - settings.Database.Type = Shoko.Server.Server.Constants.DatabaseType.SQLite; - settings.Database.OverrideConnectionString = connectionString; - var pluginManager = GetRequiredPrivateField(systemService, "_pluginManager"); - InitializeCorePluginOnly(pluginManager, systemService); - - using var host = CreateBootstrapHost(systemService, settings); - Utils.ServiceContainer = host.Services; - pluginManager.InitPlugins(); + var databaseFactory = services.GetRequiredService(); + var repositoryFactory = services.GetRequiredService(); + // Conversion mode enters its isolated temp-home/settings scope before host build, so + // Quartz and any other early services already point at the conversion runtime. Database + // bootstrap can therefore reuse the existing isolated runtime without touching the real + // Shoko home or source database settings. + databaseFactory.CloseSessionFactory(); + databaseFactory.Instance = null; - var databaseFactory = host.Services.GetRequiredService(); - var repositoryFactory = host.Services.GetRequiredService(); - if (!RunInitializeDatabase(systemService, databaseFactory, repositoryFactory)) + if (!systemService.InitializeDatabaseForConversion(databaseFactory, repositoryFactory, cancellationToken)) { throw new InvalidOperationException(systemService.StartupMessage ?? "Shoko database bootstrap failed."); } databaseFactory.CloseSessionFactory(); + databaseFactory.Instance = null; } + private sealed class InMemorySettingsProvider(Shoko.Server.Settings.ServerSettings settings) : ISettingsProvider + { + private Shoko.Server.Settings.ServerSettings _settings = settings; + + public Shoko.Server.Settings.IServerSettings GetSettings(bool copy = false) + => copy ? _settings.DeepClone() : _settings; + + public void SaveSettings(Shoko.Server.Settings.IServerSettings settings) + { + if (settings is Shoko.Server.Settings.ServerSettings serverSettings) + { + _settings = serverSettings; + } + } + + public void SaveSettings() + { + } + + public void DebugSettingsToLog() + { + } + } + + private sealed class ConversionIsolationScope : IDisposable + { + private readonly IDisposable _settingsOverride; + private readonly IDisposable _dataPathOverride; + private readonly string _temporaryHomePath; + private bool _disposed; + + public ConversionIsolationScope(ConversionRuntimeContext context) + { + _temporaryHomePath = context.TemporaryHomePath; + Directory.CreateDirectory(_temporaryHomePath); + _settingsOverride = Utils.PushSettingsProviderOverride(context.RuntimeSettingsProvider); + _dataPathOverride = ApplicationPaths.PushDataPathOverride(_temporaryHomePath); + } + + public void Dispose() + { + if (_disposed) + { + return; + } + + _dataPathOverride.Dispose(); + _settingsOverride.Dispose(); + + try + { + if (Directory.Exists(_temporaryHomePath)) + { + Directory.Delete(_temporaryHomePath, true); + } + } + catch + { + // Best-effort cleanup only. + } + + _disposed = true; + } + } + + private static string BuildTemporaryQuartzConnectionString(string temporaryHomePath) + => $"Data Source={Path.Combine(temporaryHomePath, "SQLite", "Quartz.db3")};Mode=ReadWriteCreate;Pooling=True"; + + private static string BuildSqliteConnectionString(string targetFile) + => new SqliteConnectionStringBuilder + { + DataSource = targetFile, + Mode = SqliteOpenMode.ReadWriteCreate, + Pooling = false, + }.ToString(); + private static async Task ConfigureSqliteAsync(SqliteConnection connection) { var commands = new[] @@ -301,10 +562,8 @@ LIMIT 1 await reader.IsDBNullAsync(2) ? null : reader.GetString(2)); } - private static async Task GetExpectedSourceDatabaseVersionAsync(SourceDatabaseType sourceType) + private static DatabaseVersionInfo GetExpectedSourceDatabaseVersion(SystemService systemService, SourceDatabaseType sourceType) { - await using var bootstrapHome = new TemporaryShokoHomeScope(); - var systemService = new SystemService(); object database = sourceType switch { SourceDatabaseType.SqlServer => new SQLServer(systemService), @@ -584,7 +843,18 @@ private static void ThrowIfRowMismatch( var keyDescription = string.Join(", ", orderColumns.Select(column => { - var keyIndex = sharedColumns.IndexOf(column); + var keyIndex = -1; + for (var sharedColumnIndex = 0; sharedColumnIndex < sharedColumns.Count; sharedColumnIndex++) + { + if (!string.Equals(sharedColumns[sharedColumnIndex], column, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + keyIndex = sharedColumnIndex; + break; + } + var keyValue = keyIndex >= 0 ? sourceRow[keyIndex] : ""; return $"{column}={keyValue}"; })); @@ -1209,51 +1479,7 @@ private static string QuoteSqliteIdentifier(string identifier) private static string QuoteSqliteLiteral(string value) => $"'{ValidateIdentifier(value).Replace("'", "''", StringComparison.Ordinal)}'"; - private static Options ParseArgs(string[] args) - { - var options = new Options(); - for (var index = 0; index < args.Length; index++) - { - var argument = args[index]; - switch (argument) - { - case "--source-type": - options.SourceType = ParseSourceType(GetRequiredValue(args, ref index, argument)); - break; - case "--source-connection-string": - options.SourceConnectionString = GetRequiredValue(args, ref index, argument); - break; - case "--target-file": - options.TargetFile = GetRequiredValue(args, ref index, argument); - break; - case "--overwrite": - options.Overwrite = true; - break; - case "--help": - case "-h": - case "/?": - options.ShowHelp = true; - break; - default: - throw new ArgumentException($"Unknown argument: {argument}"); - } - } - - return options; - } - - private static string GetRequiredValue(string[] args, ref int index, string argumentName) - { - if (index + 1 >= args.Length) - { - throw new ArgumentException($"Missing value for {argumentName}"); - } - - index++; - return args[index]; - } - - private static SourceDatabaseType ParseSourceType(string value) + internal static SourceDatabaseType ParseSourceType(string value) { return value.Trim().ToLowerInvariant() switch { @@ -1263,70 +1489,25 @@ private static SourceDatabaseType ParseSourceType(string value) }; } - private static void PrintUsage() - { - Console.WriteLine("Usage:"); - Console.WriteLine(" Shoko.CLI convert-db [--source-type mssql|mariadb] --source-connection-string \"\" --target-file \"/path/to/Shoko.sqlite\" [--overwrite]"); - Console.WriteLine(); - Console.WriteLine("Notes:"); - Console.WriteLine(" - This creates a fresh SQLite database using Shoko's built-in SQLite schema commands."); - Console.WriteLine(" - Supported source backends: SQL Server and MySQL/MariaDB."); - Console.WriteLine(" - It copies tables and columns shared by the source schema and target SQLite schema."); - Console.WriteLine(" - Quartz tables are not included because Quartz uses a separate database configuration."); - } + internal static string GetUsage() + => """ + Usage: + ShokoServer --convert-db [--source-type mssql|mariadb] [--source-connection-string ""] [--target-file "/path/to/Shoko.sqlite"] [--overwrite] - private sealed class Options - { - public SourceDatabaseType SourceType { get; set; } = SourceDatabaseType.SqlServer; - public string SourceConnectionString { get; set; } = string.Empty; - public string TargetFile { get; set; } = string.Empty; - public bool Overwrite { get; set; } - public bool ShowHelp { get; set; } - } + Notes: + - If omitted, source type and connection details default to the current ServerSettings.Database configuration. + - If omitted, the target SQLite file defaults to Shoko's normal SQLite database path under the current Shoko home/data directory. + - Explicit --source-type and/or --source-connection-string override the configured source details. + - Explicit --target-file overrides the default SQLite target path. + - The resolved source database must be SQL Server or MySQL/MariaDB. SQLite cannot be used as a source. + - This creates a fresh SQLite database using Shoko's built-in SQLite schema commands. + - Supported source backends: SQL Server and MySQL/MariaDB. + - It copies tables and columns shared by the source schema and target SQLite schema. + - Quartz tables are not included because Quartz uses a separate database configuration. + """; private readonly record struct DatabaseVersionInfo(int Version, int Revision, string? Program); - private sealed class TemporaryShokoHomeScope : IAsyncDisposable - { - private readonly string? _previousShokoHome; - private readonly string _temporaryHomePath; - - public TemporaryShokoHomeScope() - { - _previousShokoHome = Environment.GetEnvironmentVariable("SHOKO_HOME"); - _temporaryHomePath = Path.Combine(Path.GetTempPath(), $"shoko-convert-bootstrap-{Guid.NewGuid():N}"); - Directory.CreateDirectory(_temporaryHomePath); - Environment.SetEnvironmentVariable("SHOKO_HOME", _temporaryHomePath); - ResetApplicationPaths(); - } - - public ValueTask DisposeAsync() - { - Utils.ServiceContainer = null; - Environment.SetEnvironmentVariable("SHOKO_HOME", _previousShokoHome); - ResetApplicationPaths(); - try - { - if (Directory.Exists(_temporaryHomePath)) - { - Directory.Delete(_temporaryHomePath, true); - } - } - catch - { - // Best-effort cleanup only. Bootstrap isolation matters more than temp dir removal. - } - - return ValueTask.CompletedTask; - } - - private static void ResetApplicationPaths() - { - SetPrivateStaticField("_dataPath", null); - SetPrivateStaticField("_instance", null); - } - } - private sealed class SqliteColumnInfo { public string Name { get; set; } = string.Empty; @@ -1334,74 +1515,4 @@ private sealed class SqliteColumnInfo public bool NotNull { get; set; } public string? DefaultValue { get; set; } } - - [SuppressMessage("Major Code Smell", "S3011", Justification = "The converter must call Shoko's internal bootstrap path to create a real target database without changing server startup surface area.")] - private static IHost CreateBootstrapHost(SystemService systemService, object settings) - { - var initWebHostMethod = typeof(SystemService).GetMethod("InitWebHost", BindingFlags.Instance | BindingFlags.NonPublic) - ?? throw new InvalidOperationException("Unable to locate SystemService.InitWebHost."); - return (IHost)(initWebHostMethod.Invoke(systemService, [settings]) - ?? throw new InvalidOperationException("SystemService.InitWebHost returned null.")); - } - - [SuppressMessage("Major Code Smell", "S3011", Justification = "The converter must run Shoko's internal database initialization path so bootstrap and migrations behave exactly like server startup.")] - private static bool RunInitializeDatabase(SystemService systemService, DatabaseFactory databaseFactory, RepoFactory repositoryFactory) - { - var initializeDatabaseMethod = typeof(SystemService).GetMethod("InitializeDatabase", BindingFlags.Instance | BindingFlags.NonPublic) - ?? throw new InvalidOperationException("Unable to locate SystemService.InitializeDatabase."); - return (bool)(initializeDatabaseMethod.Invoke(systemService, [databaseFactory, repositoryFactory, default(CancellationToken)]) - ?? throw new InvalidOperationException("SystemService.InitializeDatabase returned null.")); - } - - [SuppressMessage("Major Code Smell", "S3011", Justification = "The converter uses limited reflective access to initialize only Shoko's built-in core plugin state during isolated database bootstrap, without loading external plugins.")] - private static T GetRequiredPrivateField(object instance, string fieldName) where T : class - { - var field = instance.GetType().GetField(fieldName, BindingFlags.Instance | BindingFlags.NonPublic) - ?? throw new InvalidOperationException($"Unable to locate field {instance.GetType().Name}.{fieldName}."); - return (T)(field.GetValue(instance) ?? throw new InvalidOperationException($"Field {instance.GetType().Name}.{fieldName} was null.")); - } - - [SuppressMessage("Major Code Smell", "S3011", Justification = "The converter resets cached application paths between temporary SHOKO_HOME scopes to keep bootstrap isolated from the user's normal data directory.")] - private static void SetPrivateStaticField(string fieldName, object? value) - { - var field = typeof(TDeclaring).GetField(fieldName, BindingFlags.Static | BindingFlags.NonPublic) - ?? throw new InvalidOperationException($"Unable to locate static field {typeof(TDeclaring).Name}.{fieldName}."); - field.SetValue(null, value); - } - - private static void InitializeCorePluginOnly(PluginManager pluginManager, SystemService systemService) - { - var pluginTypes = GetRequiredPrivateField>(pluginManager, "_pluginTypes"); - if (pluginTypes.Count > 0) - { - return; - } - - var coreAssembly = typeof(CorePlugin).Assembly; - pluginTypes.Add(new() - { - ID = typeof(CorePlugin).FullName!.ToUuidV5(), - Name = "Shoko Core", - Description = string.Empty, - Version = systemService.Version, - Authors = null, - RepositoryUrl = null, - HomepageUrl = null, - Tags = [], - LoadOrder = 0, - Thumbnail = null, - InstalledAt = DateTime.MinValue, - IsEnabled = true, - IsActive = false, - CanLoad = true, - CanUninstall = false, - Plugin = null, - PluginType = typeof(CorePlugin), - ServiceRegistrationType = null, - ApplicationRegistrationType = null, - ContainingDirectory = null, - DLLs = [coreAssembly.Location], - Types = coreAssembly.GetExportedTypes(), - }); - } } diff --git a/Shoko.Server/Services/SystemService.cs b/Shoko.Server/Services/SystemService.cs index ebe713f1dd..ba1e20dd89 100644 --- a/Shoko.Server/Services/SystemService.cs +++ b/Shoko.Server/Services/SystemService.cs @@ -5,6 +5,7 @@ using System.IO; using System.Net; using System.Runtime.InteropServices; +using System.Linq; using System.Threading; using System.Threading.Tasks; using System.Timers; @@ -68,6 +69,7 @@ namespace Shoko.Server.Services; public class SystemService : ISystemService { private readonly ILogger _logger; + private readonly DatabaseConversionOptions? _databaseConversionOptions; private readonly PluginManager _pluginManager; @@ -80,11 +82,14 @@ public class SystemService : ISystemService private Timer? _autoUpdateTimer; private IHost? _webHost; + private bool _hostStarted; + private bool _oneShotModeCompleted; - public SystemService() + public SystemService(string[]? startupArgs = null) { var now = DateTime.UtcNow; - var args = Environment.GetCommandLineArgs(); + var args = startupArgs ?? Environment.GetCommandLineArgs().Skip(1).ToArray(); + _databaseConversionOptions = DatabaseConversionOptions.TryParse(args, out var conversionOptions) ? conversionOptions : null; ApplicationPaths.SetHome(args); @@ -199,6 +204,7 @@ private set /// public async Task StartAsync() { + IDisposable? conversionIsolation = null; try { // Check if any of the DLL are blocked, common issue with daily builds. @@ -210,6 +216,27 @@ private set } var settings = _settingsProvider.GetSettings(); + ISettingsProvider effectiveSettingsProvider = _settingsProvider; + + if (_databaseConversionOptions?.ShowHelp == true) + { + Console.WriteLine(DatabaseConversionService.GetUsage()); + CompleteOneShotShutdown(); + return null; + } + + DatabaseConversionService.ConversionRuntimeContext? conversionRuntime = null; + if (_databaseConversionOptions is not null) + { + var realSettings = (ServerSettings)_settingsProvider.GetSettings(copy: true); + conversionRuntime = DatabaseConversionService.PrepareRuntime(_databaseConversionOptions, realSettings); + // Enter conversion isolation before starting log maintenance, plugin scanning, + // host building, or Quartz registration so any startup side effects are scoped + // to the temporary conversion home instead of the user's real Shoko home. + conversionIsolation = DatabaseConversionService.BeginIsolatedRuntime(conversionRuntime.Value); + effectiveSettingsProvider = conversionRuntime.Value.RuntimeSettingsProvider; + settings = effectiveSettingsProvider.GetSettings(); + } LogService.ApplyLoggingSettings(settings.Logging); @@ -250,7 +277,7 @@ private set StartupMessage = "Initializing Web Host & Services."; - _webHost = InitWebHost(settings); + _webHost = InitWebHost(settings, effectiveSettingsProvider); #pragma warning disable CS0618 // Type or member is obsolete ISystemService.StaticServices = _webHost.Services; @@ -266,10 +293,17 @@ private set StartupMessage = "Plugins initialized."; + if (conversionRuntime is not null) + { + await RunDatabaseConversionModeAsync(conversionRuntime.Value, hostStarted: false); + return _webHost; + } + StartupMessage = "Starting Web Hosts."; // Start the web server and all IHostedService services. await _webHost.StartAsync(); + _hostStarted = true; StartupMessage = "Web Host started."; @@ -304,6 +338,43 @@ private set StartupFailedException = new(innerException: ex); return null; } + finally + { + conversionIsolation?.Dispose(); + } + } + + private async Task RunDatabaseConversionModeAsync(DatabaseConversionService.ConversionRuntimeContext context, bool hostStarted) + { + try + { + var cancellationToken = hostStarted + ? CancellationTokenSource.CreateLinkedTokenSource(_webHost!.Services.GetRequiredService().ApplicationStopping, _shutdownTokenSource.Token).Token + : _shutdownTokenSource.Token; + if (cancellationToken.IsCancellationRequested) + return; + + await DatabaseConversionService.RunAsync(this, _webHost!.Services, context, cancellationToken); + + _startupTaskSource?.SetResult(); + _startupTaskSource = null; + } + catch (Exception ex) + { + StartupMessage = "Database conversion failed."; + StartupFailedException = new($"Database conversion failed. Error Message: {ex.Message}", innerException: ex); + } + finally + { + if (hostStarted) + { + _webHost?.Services.GetRequiredService().StopApplication(); + } + else + { + CompleteOneShotShutdown(); + } + } } public Task WaitForStartupAsync() @@ -342,14 +413,14 @@ private bool CheckBlockedFiles() #region Startup | Services - private IHost InitWebHost(IServerSettings settings) + private IHost InitWebHost(IServerSettings settings, ISettingsProvider settingsProvider) => new HostBuilder() .ConfigureWebHost(webHostBuilder => webHostBuilder .UseKestrel(options => options.ListenAnyIP(settings.Web.Port)) .ConfigureApp() .ConfigureServiceProvider() - .UseStartup(_ => new Startup(this, _logService, _configurationService, _settingsProvider, _pluginManager)) + .UseStartup(_ => new Startup(this, _logService, _configurationService, settingsProvider, _pluginManager)) .ConfigureLogging(logging => { logging.ClearProviders(); @@ -361,7 +432,7 @@ private IHost InitWebHost(IServerSettings settings) #endif }) .UseNLog() - .UseSentryConfig(_settingsProvider) + .UseSentryConfig(settingsProvider) ) .Build(); @@ -702,6 +773,9 @@ private bool InitializeDatabase(DatabaseFactory databaseFactory, RepoFactory rep } } + internal bool InitializeDatabaseForConversion(DatabaseFactory databaseFactory, RepoFactory repositoryFactory, CancellationToken cancellationToken) + => InitializeDatabase(databaseFactory, repositoryFactory, cancellationToken); + #endregion #endregion @@ -760,7 +834,28 @@ public bool RequestShutdown() /// public Task WaitForShutdownAsync() - => _webHost?.WaitForShutdownAsync() ?? Task.CompletedTask; + => _oneShotModeCompleted ? Task.CompletedTask : _webHost?.WaitForShutdownAsync() ?? Task.CompletedTask; + + private void CompleteOneShotShutdown() + { + lock (_logger) + { + if (!RestartPending && !ShutdownPending) + ShutdownPending = true; + } + + _shutdownTokenSource.Cancel(); + _oneShotModeCompleted = true; + + try + { + Shutdown?.Invoke(this, EventArgs.Empty); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error while invoking Shutdown"); + } + } /// internal void OnShutdown() @@ -773,7 +868,7 @@ internal void OnShutdown() } _shutdownTokenSource.Cancel(); - if (_webHost is not null) + if (_hostStarted && _webHost is not null) { _autoUpdateTimer?.Stop(); diff --git a/Shoko.Server/Utilities/Utils.cs b/Shoko.Server/Utilities/Utils.cs index 7927d070a9..0d9ba249f8 100644 --- a/Shoko.Server/Utilities/Utils.cs +++ b/Shoko.Server/Utilities/Utils.cs @@ -1,18 +1,60 @@ using System; using System.IO; +using System.Threading; using Shoko.Server.Settings; +#nullable enable namespace Shoko.Server.Utilities; public static partial class Utils { - public static IServiceProvider ServiceContainer { get; set; } + private static readonly AsyncLocal s_settingsProviderOverride = new(); - public static ISettingsProvider SettingsProvider { get; set; } + private static IServiceProvider? _serviceContainer; + + public static IServiceProvider ServiceContainer + { + get => _serviceContainer!; + set => _serviceContainer = value; + } + + private static ISettingsProvider? _settingsProvider; + + public static ISettingsProvider SettingsProvider + { + get => s_settingsProviderOverride.Value ?? _settingsProvider!; + set => _settingsProvider = value; + } + + internal static IDisposable PushSettingsProviderOverride(ISettingsProvider settingsProvider) + => new ScopedOverride(s_settingsProviderOverride, settingsProvider); public static string GetDistinctPath(string fullPath) { var parent = Path.GetDirectoryName(fullPath); return string.IsNullOrEmpty(parent) ? fullPath : Path.Combine(Path.GetFileName(parent), Path.GetFileName(fullPath)); } + + private sealed class ScopedOverride : IDisposable + { + private readonly AsyncLocal _slot; + private readonly T? _previousValue; + private bool _disposed; + + public ScopedOverride(AsyncLocal slot, T value) + { + _slot = slot; + _previousValue = slot.Value; + _slot.Value = value; + } + + public void Dispose() + { + if (_disposed) + return; + + _slot.Value = _previousValue; + _disposed = true; + } + } } diff --git a/Shoko.Tests/DatabaseConversionOptionsTests.cs b/Shoko.Tests/DatabaseConversionOptionsTests.cs new file mode 100644 index 0000000000..bdb075d7b8 --- /dev/null +++ b/Shoko.Tests/DatabaseConversionOptionsTests.cs @@ -0,0 +1,268 @@ +using System; +using System.IO; +using Shoko.Server.Services; +using Shoko.Server.Settings; +using Shoko.Server.Server; +using Xunit; + +namespace Shoko.Tests; + +public class DatabaseConversionOptionsTests +{ + public static TheoryData ConversionModeArguments => + [ + ["--convert-db"], + ["--config", "/config", "--convert-db"], + ["convert-db", "--config", "/config"], + ]; + + [Theory] + [MemberData(nameof(ConversionModeArguments))] + public void ShouldDetectConversionModeIndependentOfPosition(string[] args) + { + var detected = DatabaseConversionOptions.TryParse(args, out var options); + + Assert.True(detected); + Assert.NotNull(options); + } + + [Fact] + public void ShouldParseKnownConversionArgumentsWhileIgnoringHostArguments() + { + var args = new[] + { + "convert-db", + "--config", "/config", + "--source-type", "mariadb", + "--source-connection-string", "Server=127.0.0.1;Database=shoko;", + "--target-file", "/tmp/Shoko.db3", + "--overwrite", + }; + + var detected = DatabaseConversionOptions.TryParse(args, out var options); + + Assert.True(detected); + Assert.NotNull(options); + Assert.Equal(DatabaseConversionService.SourceDatabaseType.MySql, options.SourceType); + Assert.Equal("Server=127.0.0.1;Database=shoko;", options.SourceConnectionString); + Assert.Equal("/tmp/Shoko.db3", options.TargetFile); + Assert.True(options.Overwrite); + } + + [Fact] + public void ShouldNotDetectConversionModeWhenModeArgumentIsMissing() + { + var detected = DatabaseConversionOptions.TryParse(["--config", "/config"], out var options); + + Assert.False(detected); + Assert.Null(options); + } + + [Fact] + public void ShouldNotDetectConversionModeWhenConfigValueMatchesModeToken() + { + var detected = DatabaseConversionOptions.TryParse(["--config", "convert-db"], out var options); + + Assert.False(detected); + Assert.Null(options); + } + + [Fact] + public void ShouldNotTreatTargetFileValueAsModeTokenWithoutActualModeArgument() + { + var detected = DatabaseConversionOptions.TryParse(["--target-file", "convert-db"], out var options); + + Assert.False(detected); + Assert.Null(options); + } + + [Fact] + public void ShouldNotDetectConversionModeWhenFutureOptionValueMatchesModeToken() + { + var detected = DatabaseConversionOptions.TryParse(["--some-future-option", "convert-db"], out var options); + + Assert.False(detected); + Assert.Null(options); + } + + [Fact] + public void ShouldUseConfiguredSettingsAsSourceWhenSourceArgsAreOmitted() + { + var settings = new ServerSettings + { + Database = + { + Type = Constants.DatabaseType.MySQL, + Host = "db.example:3307", + Username = "user", + Password = "pass", + Schema = "shoko", + } + }; + var options = new DatabaseConversionOptions + { + TargetFile = "/tmp/Shoko.db3", + }; + + var resolved = DatabaseConversionService.ResolveSource(options, settings); + + Assert.Equal(DatabaseConversionService.SourceDatabaseType.MySql, resolved.SourceType); + Assert.Equal("Server=db.example;Port=3307;Database=shoko;User ID=user;Password=pass;Default Command Timeout=3600;Allow User Variables=true", resolved.SourceConnectionString); + } + + [Fact] + public void ExplicitSourceArgsShouldOverrideConfiguredSettings() + { + var settings = new ServerSettings + { + Database = + { + Type = Constants.DatabaseType.MySQL, + Host = "db.example:3307", + Username = "user", + Password = "pass", + Schema = "shoko", + } + }; + var options = new DatabaseConversionOptions + { + SourceType = DatabaseConversionService.SourceDatabaseType.SqlServer, + SourceTypeProvided = true, + SourceConnectionString = "Server=override;Database=override;", + SourceConnectionStringProvided = true, + TargetFile = "/tmp/Shoko.db3", + }; + + var resolved = DatabaseConversionService.ResolveSource(options, settings); + + Assert.Equal(DatabaseConversionService.SourceDatabaseType.SqlServer, resolved.SourceType); + Assert.Equal("Server=override;Database=override;", resolved.SourceConnectionString); + } + + [Fact] + public void ConfiguredSqliteSourceShouldFailClearly() + { + var settings = new ServerSettings + { + Database = + { + Type = Constants.DatabaseType.SQLite, + } + }; + var options = new DatabaseConversionOptions + { + TargetFile = "/tmp/Shoko.db3", + }; + + var ex = Assert.Throws(() => DatabaseConversionService.ResolveSource(options, settings)); + + Assert.Contains("SQLite", ex.Message); + } + + [Fact] + public void OmittedTargetFileShouldResolveToDefaultSqlitePath() + { + var settings = new ServerSettings + { + Database = + { + MySqliteDirectory = "SQLite", + SQLite_DatabaseFile = "Shoko.db3", + } + }; + var options = new DatabaseConversionOptions(); + + var resolved = DatabaseConversionService.ResolveTarget(options, settings); + + Assert.Equal(Path.GetFullPath(Path.Combine(ApplicationPaths.StaticDataPath, "SQLite", "Shoko.db3")), resolved.TargetFile); + } + + [Fact] + public void ExplicitTargetFileShouldOverrideDefaultSqlitePath() + { + var settings = new ServerSettings + { + Database = + { + MySqliteDirectory = "SQLite", + SQLite_DatabaseFile = "Shoko.db3", + } + }; + var options = new DatabaseConversionOptions + { + TargetFile = "/tmp/custom.db3", + TargetFileProvided = true, + }; + + var resolved = DatabaseConversionService.ResolveTarget(options, settings); + + Assert.Equal(Path.GetFullPath("/tmp/custom.db3"), resolved.TargetFile); + } + + [Fact] + public void ExistingTargetFileShouldFailWithoutOverwrite() + { + var targetFile = Path.Combine(Path.GetTempPath(), $"shoko-converter-target-{Guid.NewGuid():N}.db3"); + try + { + File.WriteAllText(targetFile, "existing"); + + var ex = Assert.Throws(() => DatabaseConversionService.PrepareTargetPath(targetFile, overwrite: false)); + + Assert.Contains("already exists", ex.Message); + } + finally + { + if (File.Exists(targetFile)) + { + File.Delete(targetFile); + } + } + } + + [Fact] + public void PreparedConversionRuntimeShouldIsolateQuartzAndTargetBootstrapPaths() + { + var realQuartzConnectionString = $"Data Source={Path.Combine(ApplicationPaths.StaticDataPath, "SQLite", "Quartz.db3")};Mode=ReadWriteCreate;Pooling=True"; + var settings = new ServerSettings + { + Database = + { + Type = Constants.DatabaseType.MySQL, + Host = "db.example:3306", + Username = "user", + Password = "pass", + Schema = "shoko", + }, + Quartz = + { + DatabaseType = Constants.DatabaseType.SQLServer, + ConnectionString = realQuartzConnectionString, + }, + }; + var options = new DatabaseConversionOptions + { + TargetFile = Path.Combine(Path.GetTempPath(), $"shoko-converter-target-{Guid.NewGuid():N}.db3"), + TargetFileProvided = true, + }; + + try + { + var runtime = DatabaseConversionService.PrepareRuntime(options, settings); + var runtimeSettings = (ServerSettings)runtime.RuntimeSettingsProvider.GetSettings(copy: true); + + Assert.Equal(Constants.DatabaseType.SQLite, runtimeSettings.Database.Type); + Assert.Contains(runtime.PreparedTargetFile, runtimeSettings.Database.OverrideConnectionString, StringComparison.Ordinal); + Assert.Equal(Constants.DatabaseType.SQLite, runtimeSettings.Quartz.DatabaseType); + Assert.Contains(runtime.TemporaryHomePath, runtimeSettings.Quartz.ConnectionString, StringComparison.Ordinal); + Assert.DoesNotContain(realQuartzConnectionString, runtimeSettings.Quartz.ConnectionString, StringComparison.Ordinal); + } + finally + { + if (File.Exists(options.TargetFile)) + { + File.Delete(options.TargetFile); + } + } + } +} diff --git a/Shoko.TrayService/App.axaml.cs b/Shoko.TrayService/App.axaml.cs index bc42255325..28bcc204be 100644 --- a/Shoko.TrayService/App.axaml.cs +++ b/Shoko.TrayService/App.axaml.cs @@ -40,7 +40,7 @@ public override void OnFrameworkInitializationCompleted() Console.CancelKeyPress += OnConsoleOnCancelKeyPress; InitialiseTrayIcon(); - _systemService = new SystemService(); + _systemService = new SystemService(Program.StartupArgs); _systemService.Shutdown += (_, _) => DispatchShutdown(); var host = _systemService.StartAsync() .ConfigureAwait(true) diff --git a/Shoko.TrayService/Program.cs b/Shoko.TrayService/Program.cs index 0c9a388b1a..50923b43dc 100644 --- a/Shoko.TrayService/Program.cs +++ b/Shoko.TrayService/Program.cs @@ -6,9 +6,12 @@ namespace Shoko.TrayService; public static class Program { + internal static string[] StartupArgs { get; private set; } = Array.Empty(); + [STAThread] public static int Main(string[] args) { + StartupArgs = args; try { UnhandledExceptionManager.AddHandler(); From 51cdb56cab485c1f650bea9c8da18c5abbd88a41 Mon Sep 17 00:00:00 2001 From: revam Date: Fri, 22 May 2026 12:36:24 +0200 Subject: [PATCH 10/13] chore: don't pass in args from cli/tray service But keep the override for the db fixture. --- Shoko.CLI/Program.cs | 4 ++-- Shoko.IntegrationTests/DatabaseMigrationFixture.cs | 6 +----- Shoko.Server/Services/SystemService.cs | 3 ++- Shoko.TrayService/App.axaml.cs | 2 +- Shoko.TrayService/Program.cs | 3 --- 5 files changed, 6 insertions(+), 12 deletions(-) diff --git a/Shoko.CLI/Program.cs b/Shoko.CLI/Program.cs index edd159a9e9..33c51ee38b 100644 --- a/Shoko.CLI/Program.cs +++ b/Shoko.CLI/Program.cs @@ -15,7 +15,7 @@ public static class Program { private static ILogger _logger = null!; - public static async Task Main(string[] args) + public static async Task Main() { try { @@ -26,7 +26,7 @@ public static async Task Main(string[] args) Console.WriteLine(ex.ToString()); } - var systemService = new SystemService(args); + var systemService = new SystemService(); systemService.StartupFailed += OnStartupFailed; systemService.AboutToStart += (_, args) => AddEventHandlers(args.ServiceProvider); var host = await systemService.StartAsync(); diff --git a/Shoko.IntegrationTests/DatabaseMigrationFixture.cs b/Shoko.IntegrationTests/DatabaseMigrationFixture.cs index 0bf404fa44..d39c7dffa4 100644 --- a/Shoko.IntegrationTests/DatabaseMigrationFixture.cs +++ b/Shoko.IntegrationTests/DatabaseMigrationFixture.cs @@ -35,13 +35,9 @@ public DatabaseMigrationFixture() _tempDir = Path.Combine(Path.GetTempPath(), $"shoko-integration-{Guid.NewGuid():N}"); Directory.CreateDirectory(_tempDir); - // SHOKO_HOME controls Utils.ApplicationPath. Must be set before SystemService() reads it. - // Forward slashes avoid bad JSON escape sequences when the config service parses env vars. - Environment.SetEnvironmentVariable("SHOKO_HOME", _tempDir.Replace('\\', '/')); - // SystemService() bootstraps Utils.SettingsProvider with default settings (FirstRun=true). // No settings file yet — defaults are valid and pass schema validation. - var systemService = new SystemService(); + var systemService = new SystemService(["--config", _tempDir.Replace('\\', '/')]); // Mutate the live settings: disable first-run, inject fake AniDB credentials so the // settings custom-validator is satisfied, and move the web port away from 8111 so this diff --git a/Shoko.Server/Services/SystemService.cs b/Shoko.Server/Services/SystemService.cs index ba1e20dd89..21ae918141 100644 --- a/Shoko.Server/Services/SystemService.cs +++ b/Shoko.Server/Services/SystemService.cs @@ -89,10 +89,11 @@ public SystemService(string[]? startupArgs = null) { var now = DateTime.UtcNow; var args = startupArgs ?? Environment.GetCommandLineArgs().Skip(1).ToArray(); - _databaseConversionOptions = DatabaseConversionOptions.TryParse(args, out var conversionOptions) ? conversionOptions : null; ApplicationPaths.SetHome(args); + _databaseConversionOptions = DatabaseConversionOptions.TryParse(args, out var conversionOptions) ? conversionOptions : null; + LogService.InitLogger(ApplicationPaths.Instance); var loggerFactory = LoggerFactory.Create(o => o.AddNLog()); diff --git a/Shoko.TrayService/App.axaml.cs b/Shoko.TrayService/App.axaml.cs index 28bcc204be..bc42255325 100644 --- a/Shoko.TrayService/App.axaml.cs +++ b/Shoko.TrayService/App.axaml.cs @@ -40,7 +40,7 @@ public override void OnFrameworkInitializationCompleted() Console.CancelKeyPress += OnConsoleOnCancelKeyPress; InitialiseTrayIcon(); - _systemService = new SystemService(Program.StartupArgs); + _systemService = new SystemService(); _systemService.Shutdown += (_, _) => DispatchShutdown(); var host = _systemService.StartAsync() .ConfigureAwait(true) diff --git a/Shoko.TrayService/Program.cs b/Shoko.TrayService/Program.cs index 50923b43dc..0c9a388b1a 100644 --- a/Shoko.TrayService/Program.cs +++ b/Shoko.TrayService/Program.cs @@ -6,12 +6,9 @@ namespace Shoko.TrayService; public static class Program { - internal static string[] StartupArgs { get; private set; } = Array.Empty(); - [STAThread] public static int Main(string[] args) { - StartupArgs = args; try { UnhandledExceptionManager.AddHandler(); From c23ced68c00ab8d9ef74cb49d9bf04e3d0fb6a38 Mon Sep 17 00:00:00 2001 From: krbrs <57227244+krbrs@users.noreply.github.com> Date: Fri, 22 May 2026 13:33:28 +0200 Subject: [PATCH 11/13] test: use supported home override and flag-only convert-db mode - switch the database migration fixture from --config to the supported --home override - remove bare convert-db detection and require --convert-db explicitly - update conversion parser tests to use --home instead of --config - keep parser coverage for false positives on option values --- .../DatabaseMigrationFixture.cs | 2 +- .../Services/DatabaseConversionService.cs | 22 +------------------ Shoko.Tests/DatabaseConversionOptionsTests.cs | 22 +++++++++++++------ 3 files changed, 17 insertions(+), 29 deletions(-) diff --git a/Shoko.IntegrationTests/DatabaseMigrationFixture.cs b/Shoko.IntegrationTests/DatabaseMigrationFixture.cs index d39c7dffa4..56682ad3fb 100644 --- a/Shoko.IntegrationTests/DatabaseMigrationFixture.cs +++ b/Shoko.IntegrationTests/DatabaseMigrationFixture.cs @@ -37,7 +37,7 @@ public DatabaseMigrationFixture() // SystemService() bootstraps Utils.SettingsProvider with default settings (FirstRun=true). // No settings file yet — defaults are valid and pass schema validation. - var systemService = new SystemService(["--config", _tempDir.Replace('\\', '/')]); + var systemService = new SystemService(["--home", _tempDir.Replace('\\', '/')]); // Mutate the live settings: disable first-run, inject fake AniDB credentials so the // settings custom-validator is satisfied, and move the web port away from 8111 so this diff --git a/Shoko.Server/Services/DatabaseConversionService.cs b/Shoko.Server/Services/DatabaseConversionService.cs index 91dc5664b4..e177ef7cfc 100644 --- a/Shoko.Server/Services/DatabaseConversionService.cs +++ b/Shoko.Server/Services/DatabaseConversionService.cs @@ -51,7 +51,6 @@ public static bool TryParse(string[] args, [NotNullWhen(true)] out DatabaseConve var argument = args[index]; switch (argument) { - case "convert-db": case "--convert-db": break; case "--source-type": @@ -82,26 +81,7 @@ public static bool TryParse(string[] args, [NotNullWhen(true)] out DatabaseConve } private static bool DetectConversionMode(string[] args) - { - if (args.Any(argument => string.Equals(argument, "--convert-db", StringComparison.OrdinalIgnoreCase))) - { - return true; - } - - for (var index = 0; index < args.Length; index++) - { - var argument = args[index]; - if (argument.StartsWith("-", StringComparison.Ordinal)) - { - continue; - } - - return string.Equals(argument, "convert-db", StringComparison.OrdinalIgnoreCase) && - (index is 0 || !args[index - 1].StartsWith("-", StringComparison.Ordinal)); - } - - return false; - } + => args.Any(argument => string.Equals(argument, "--convert-db", StringComparison.OrdinalIgnoreCase)); private static string GetRequiredValue(string[] args, ref int index, string argumentName) { diff --git a/Shoko.Tests/DatabaseConversionOptionsTests.cs b/Shoko.Tests/DatabaseConversionOptionsTests.cs index bdb075d7b8..b3c14c5961 100644 --- a/Shoko.Tests/DatabaseConversionOptionsTests.cs +++ b/Shoko.Tests/DatabaseConversionOptionsTests.cs @@ -12,8 +12,7 @@ public class DatabaseConversionOptionsTests public static TheoryData ConversionModeArguments => [ ["--convert-db"], - ["--config", "/config", "--convert-db"], - ["convert-db", "--config", "/config"], + ["--home", "/tmp/shoko", "--convert-db"], ]; [Theory] @@ -31,8 +30,8 @@ public void ShouldParseKnownConversionArgumentsWhileIgnoringHostArguments() { var args = new[] { - "convert-db", - "--config", "/config", + "--convert-db", + "--home", "/tmp/shoko", "--source-type", "mariadb", "--source-connection-string", "Server=127.0.0.1;Database=shoko;", "--target-file", "/tmp/Shoko.db3", @@ -52,16 +51,16 @@ public void ShouldParseKnownConversionArgumentsWhileIgnoringHostArguments() [Fact] public void ShouldNotDetectConversionModeWhenModeArgumentIsMissing() { - var detected = DatabaseConversionOptions.TryParse(["--config", "/config"], out var options); + var detected = DatabaseConversionOptions.TryParse(["--home", "/tmp/shoko"], out var options); Assert.False(detected); Assert.Null(options); } [Fact] - public void ShouldNotDetectConversionModeWhenConfigValueMatchesModeToken() + public void ShouldNotDetectConversionModeWhenHomeValueMatchesModeToken() { - var detected = DatabaseConversionOptions.TryParse(["--config", "convert-db"], out var options); + var detected = DatabaseConversionOptions.TryParse(["--home", "convert-db"], out var options); Assert.False(detected); Assert.Null(options); @@ -85,6 +84,15 @@ public void ShouldNotDetectConversionModeWhenFutureOptionValueMatchesModeToken() Assert.Null(options); } + [Fact] + public void ShouldNotDetectConversionModeForBareConvertDbToken() + { + var detected = DatabaseConversionOptions.TryParse(["convert-db"], out var options); + + Assert.False(detected); + Assert.Null(options); + } + [Fact] public void ShouldUseConfiguredSettingsAsSourceWhenSourceArgsAreOmitted() { From f26555a5f8ca5a60464c21bb8efa9275cc5129d1 Mon Sep 17 00:00:00 2001 From: krbrs <57227244+krbrs@users.noreply.github.com> Date: Fri, 22 May 2026 22:06:34 +0200 Subject: [PATCH 12/13] fix: resolve ApplicationPaths merge artifact --- Shoko.Server/Services/ApplicationPaths.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Shoko.Server/Services/ApplicationPaths.cs b/Shoko.Server/Services/ApplicationPaths.cs index ee7aa74832..8a1da894d4 100644 --- a/Shoko.Server/Services/ApplicationPaths.cs +++ b/Shoko.Server/Services/ApplicationPaths.cs @@ -29,7 +29,7 @@ public string ApplicationPath /// public string WebPath => s_dataPathOverride.Value is { Length: > 0 } - ? Path.Combine(DataPath, Utils.SettingsProvider.GetSettings().Web.WebUIPath) + ? Path.Combine(DataPath, ISettingsProvider.Instance.GetSettings().Web.WebUIPath) : _webPath ??= Path.Combine(DataPath, ISettingsProvider.Instance.GetSettings().Web.WebUIPath); private static string? _dataPath = null; @@ -101,9 +101,6 @@ public static void SetHome(string[] args) /// public string ImagesPath - => _imagesPath ??= ISettingsProvider.Instance.GetSettings().ImagesPath is { Length: > 0 } imagePath - ? Path.Combine(DataPath, imagePath) - : DefaultImagePath; => s_dataPathOverride.Value is { Length: > 0 } ? ISettingsProvider.Instance.GetSettings().ImagesPath is { Length: > 0 } configuredImagePath ? Path.Combine(DataPath, configuredImagePath) From f44c6ac8a46a7ad6bb23f50cb0d89095d83abcf2 Mon Sep 17 00:00:00 2001 From: krbrs <57227244+krbrs@users.noreply.github.com> Date: Fri, 22 May 2026 22:17:14 +0200 Subject: [PATCH 13/13] fix: resolve settings provider merge conflicts --- Shoko.Server/Services/DatabaseConversionService.cs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Shoko.Server/Services/DatabaseConversionService.cs b/Shoko.Server/Services/DatabaseConversionService.cs index e177ef7cfc..635d494b0f 100644 --- a/Shoko.Server/Services/DatabaseConversionService.cs +++ b/Shoko.Server/Services/DatabaseConversionService.cs @@ -18,7 +18,6 @@ using Shoko.Abstractions.Plugin; using Shoko.Server.Databases; using Shoko.Server.Repositories; -using Shoko.Server.Utilities; using ISettingsProvider = Shoko.Server.Settings.ISettingsProvider; @@ -407,7 +406,7 @@ public void DebugSettingsToLog() private sealed class ConversionIsolationScope : IDisposable { - private readonly IDisposable _settingsOverride; + private readonly ISettingsProvider _previousSettingsProvider; private readonly IDisposable _dataPathOverride; private readonly string _temporaryHomePath; private bool _disposed; @@ -416,7 +415,10 @@ public ConversionIsolationScope(ConversionRuntimeContext context) { _temporaryHomePath = context.TemporaryHomePath; Directory.CreateDirectory(_temporaryHomePath); - _settingsOverride = Utils.PushSettingsProviderOverride(context.RuntimeSettingsProvider); + + _previousSettingsProvider = ISettingsProvider.Instance; + ISettingsProvider.Instance = context.RuntimeSettingsProvider; + _dataPathOverride = ApplicationPaths.PushDataPathOverride(_temporaryHomePath); } @@ -428,7 +430,7 @@ public void Dispose() } _dataPathOverride.Dispose(); - _settingsOverride.Dispose(); + ISettingsProvider.Instance = _previousSettingsProvider; try {