From dc6d3410db1dbdf385baaa4aba9678953987205b Mon Sep 17 00:00:00 2001 From: realzombee <209545148+realzombee@users.noreply.github.com> Date: Fri, 4 Sep 2026 22:23:23 +0100 Subject: [PATCH 1/7] wip - Add SQLite-backed cache for downloads and queries --- .../Extensions/PathExtensions.cs | 6 + .../Cache/BrotliCompressionHelper.cs | 40 ++++ src/NzbDrone.Core/Cache/CacheKeyHasher.cs | 19 ++ src/NzbDrone.Core/Cache/DiskCacheService.cs | 207 ------------------ .../Cache/DownloadCacheMigrator.cs | 130 +++++++++++ .../Cache/DownloadCacheService.cs | 200 +++++++++++++++++ .../Cache/IDownloadCacheMigrator.cs | 7 + .../Cache/IDownloadCacheService.cs | 15 ++ .../Cache/ISqliteCacheDatabase.cs | 10 + .../Cache/SqliteCacheDatabase.cs | 124 +++++++++++ .../Housekeepers/CleanupOldCacheEntries.cs | 36 +++ .../CleanupOldDiskCacheEntries.cs | 17 -- .../Caching/SqliteOutputCacheStore.cs | 112 ++++++++++ src/NzbDrone.Host/Startup.cs | 10 + .../Indexers/NewznabController.cs | 12 +- 15 files changed, 715 insertions(+), 230 deletions(-) create mode 100644 src/NzbDrone.Core/Cache/BrotliCompressionHelper.cs create mode 100644 src/NzbDrone.Core/Cache/CacheKeyHasher.cs delete mode 100644 src/NzbDrone.Core/Cache/DiskCacheService.cs create mode 100644 src/NzbDrone.Core/Cache/DownloadCacheMigrator.cs create mode 100644 src/NzbDrone.Core/Cache/DownloadCacheService.cs create mode 100644 src/NzbDrone.Core/Cache/IDownloadCacheMigrator.cs create mode 100644 src/NzbDrone.Core/Cache/IDownloadCacheService.cs create mode 100644 src/NzbDrone.Core/Cache/ISqliteCacheDatabase.cs create mode 100644 src/NzbDrone.Core/Cache/SqliteCacheDatabase.cs create mode 100644 src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOldCacheEntries.cs delete mode 100644 src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOldDiskCacheEntries.cs create mode 100644 src/NzbDrone.Host/Caching/SqliteOutputCacheStore.cs diff --git a/src/NzbDrone.Common/Extensions/PathExtensions.cs b/src/NzbDrone.Common/Extensions/PathExtensions.cs index 30a467f2108..cb5178c072b 100644 --- a/src/NzbDrone.Common/Extensions/PathExtensions.cs +++ b/src/NzbDrone.Common/Extensions/PathExtensions.cs @@ -15,6 +15,7 @@ public static class PathExtensions private const string DB = "prowlarr.db"; private const string DB_RESTORE = "prowlarr.restore"; private const string LOG_DB = "logs.db"; + private const string CACHE_DB = "cache.db"; private const string NLOG_CONFIG_FILE = "nlog.config"; private const string UPDATE_CLIENT_EXE_NAME = "Prowlarr.Update"; @@ -364,6 +365,11 @@ public static string GetLogDatabase(this IAppFolderInfo appFolderInfo) return Path.Combine(GetAppDataPath(appFolderInfo), LOG_DB); } + public static string GetCacheDatabase(this IAppFolderInfo appFolderInfo) + { + return Path.Combine(GetAppDataPath(appFolderInfo), CACHE_DB); + } + public static string GetNlogConfigPath(this IAppFolderInfo appFolderInfo) { return Path.Combine(appFolderInfo.StartUpFolder, NLOG_CONFIG_FILE); diff --git a/src/NzbDrone.Core/Cache/BrotliCompressionHelper.cs b/src/NzbDrone.Core/Cache/BrotliCompressionHelper.cs new file mode 100644 index 00000000000..cfc39ba684a --- /dev/null +++ b/src/NzbDrone.Core/Cache/BrotliCompressionHelper.cs @@ -0,0 +1,40 @@ +using System; +using System.IO; +using System.IO.Compression; + +namespace NzbDrone.Core.Cache +{ + public static class BrotliCompressionHelper + { + public static byte[] Compress(byte[] data, CompressionLevel level = CompressionLevel.Fastest) + { + if (data == null || data.Length == 0) + { + return Array.Empty(); + } + + using var outputStream = new MemoryStream(); + using (var brotliStream = new BrotliStream(outputStream, level)) + { + brotliStream.Write(data, 0, data.Length); + } + + return outputStream.ToArray(); + } + + public static byte[] Decompress(byte[] compressedData) + { + if (compressedData == null || compressedData.Length == 0) + { + return Array.Empty(); + } + + using var inputStream = new MemoryStream(compressedData); + using var brotliStream = new BrotliStream(inputStream, CompressionMode.Decompress); + using var outputStream = new MemoryStream(); + + brotliStream.CopyTo(outputStream); + return outputStream.ToArray(); + } + } +} diff --git a/src/NzbDrone.Core/Cache/CacheKeyHasher.cs b/src/NzbDrone.Core/Cache/CacheKeyHasher.cs new file mode 100644 index 00000000000..238b32822c2 --- /dev/null +++ b/src/NzbDrone.Core/Cache/CacheKeyHasher.cs @@ -0,0 +1,19 @@ +using System; +using System.Security.Cryptography; +using System.Text; + +namespace NzbDrone.Core.Cache +{ + public static class CacheKeyHasher + { + public static string Hash(string key) + { + if (string.IsNullOrEmpty(key)) + { + return string.Empty; + } + + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(key))).ToLowerInvariant(); + } + } +} diff --git a/src/NzbDrone.Core/Cache/DiskCacheService.cs b/src/NzbDrone.Core/Cache/DiskCacheService.cs deleted file mode 100644 index c410435effc..00000000000 --- a/src/NzbDrone.Core/Cache/DiskCacheService.cs +++ /dev/null @@ -1,207 +0,0 @@ -using System; -using System.IO; -using System.Linq; -using System.Security.Cryptography; -using System.Text; -using System.Threading.Tasks; -using NLog; -using NzbDrone.Common.EnvironmentInfo; -using NzbDrone.Common.Extensions; - -namespace NzbDrone.Core.Cache -{ - public interface IDiskCacheService - { - bool IsEnabled { get; } - - Task Get(string key); - - Task Store(string key, byte[] value, string fileName); - - void Cleanup(); - } - - public class DiskCacheService : IDiskCacheService - { - private readonly IAppFolderInfo _appFolderInfo; - private readonly Logger _logger; - - public bool IsEnabled => - bool.TryParse(Environment.GetEnvironmentVariable("ENABLE_DOWNLOAD_CACHE"), out var enabled) && enabled; - - public DiskCacheService(IAppFolderInfo appFolderInfo, Logger logger) - { - _appFolderInfo = appFolderInfo; - _logger = logger; - } - - public async Task Get(string key) - { - var hash = GenerateHash(key); - - var directory = Path.Combine(GetDiskCacheDir(), hash[..2], hash); - - if (!Directory.Exists(directory)) - { - return null; - } - - var files = Directory.GetFiles(directory); - - if (files.Length == 0) - { - return null; - } - - var cachedFile = files[0]; - _logger.Debug("Download cache hit for {0}: {1}", key, cachedFile.CleanFileName()); - - var value = await File.ReadAllBytesAsync(cachedFile); - - try - { - File.SetLastWriteTimeUtc(cachedFile, DateTime.UtcNow); - } - catch - { - // Cache metadata update failure shouldn't invalidate a successful read. - } - - return value; - } - - public async Task Store(string key, byte[] value, string fileName) - { - var hash = GenerateHash(key); - - var directory = Path.Combine(GetDiskCacheDir(), hash[..2], hash); - - Directory.CreateDirectory(directory); - - var safeFileName = GetSafeFileName(fileName); - var path = Path.Combine(directory, safeFileName); - - var tempPath = path + "." + Guid.NewGuid().ToString("N") + ".tmp"; - - try - { - await File.WriteAllBytesAsync(tempPath, value); - File.Move(tempPath, path, overwrite: true); - _logger.Debug("Stored file in disk cache for key {0}: {1}", key, path); - } - catch (Exception e) - { - _logger.Error(e, "Failed to store file in disk cache for key {0}: {1}", key, path); - } - finally - { - // Clean up if the move/write failed. - File.Delete(tempPath); - } - } - - public void Cleanup() - { - var cacheDir = GetDiskCacheDir(); - - if (!Directory.Exists(cacheDir)) - { - return; - } - - var cacheMaxSize = long.TryParse( - Environment.GetEnvironmentVariable("DOWNLOAD_CACHE_MAX_SIZE_MB"), - out var mega) - ? mega - : 1000; - - var maxBytes = cacheMaxSize * 1024 * 1024; - - var files = Directory - .EnumerateFiles(cacheDir, "*", SearchOption.AllDirectories) - .Select(path => - { - var info = new FileInfo(path); - - return new - { - Path = path, - Size = info.Length, - LastAccessed = info.LastWriteTimeUtc - }; - }) - .OrderBy(x => x.LastAccessed) - .ToList(); - - var totalSize = files.Sum(x => x.Size); - var deletedCount = 0; - - _logger.Debug("Total size of disk cache: {0} MB, Limit: {1} MB", totalSize / 1024 / 1024, maxBytes / 1024 / 1024); - - foreach (var file in files) - { - if (totalSize <= maxBytes) - { - break; - } - - try - { - var entryDirectory = Path.GetDirectoryName(file.Path); - - if (entryDirectory is null) - { - _logger.Warn("Unable to determine parent directory for cache file {0}", file.Path); - continue; - } - - Directory.Delete(entryDirectory, recursive: true); - - totalSize -= file.Size; - deletedCount += 1; - } - catch (DirectoryNotFoundException) - { - // Already deleted, possibly by another operation. - } - } - - _logger.Info("Cleaned up {0} files from disk cache", deletedCount); - } - - private string GetDiskCacheDir() - { - return Path.Combine(_appFolderInfo.AppDataFolder, "download-cache"); - } - - private static string GenerateHash(string key) - { - return Convert.ToHexString( - SHA256.HashData(Encoding.UTF8.GetBytes(key))) - .ToLowerInvariant(); - } - - private static string GetSafeFileName(string fileName) - { - if (string.IsNullOrWhiteSpace(fileName)) - { - return "data"; - } - - var safeFileName = Path.GetFileName(fileName); - - if (string.IsNullOrWhiteSpace(safeFileName) || - safeFileName is "." or "..") - { - return "data"; - } - - if (safeFileName.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0) - { - return "data"; - } - - return safeFileName; - } - } -} diff --git a/src/NzbDrone.Core/Cache/DownloadCacheMigrator.cs b/src/NzbDrone.Core/Cache/DownloadCacheMigrator.cs new file mode 100644 index 00000000000..96ffb813b15 --- /dev/null +++ b/src/NzbDrone.Core/Cache/DownloadCacheMigrator.cs @@ -0,0 +1,130 @@ +using System; +using System.IO; +using Dapper; +using NLog; +using NzbDrone.Common.EnvironmentInfo; + +namespace NzbDrone.Core.Cache +{ + public class DownloadCacheMigrator : IDownloadCacheMigrator + { + private readonly ISqliteCacheDatabase _cacheDatabase; + private readonly IAppFolderInfo _appFolderInfo; + private readonly Logger _logger; + private static readonly object MigrationLock = new object(); + private static bool _migrated; + + public DownloadCacheMigrator(ISqliteCacheDatabase cacheDatabase, IAppFolderInfo appFolderInfo, Logger logger) + { + _cacheDatabase = cacheDatabase; + _appFolderInfo = appFolderInfo; + _logger = logger; + } + + public void Migrate() + { + if (_migrated) + { + return; + } + + lock (MigrationLock) + { + if (_migrated) + { + return; + } + + var legacyCacheDir = Path.Combine(_appFolderInfo.AppDataFolder, "download-cache"); + + if (!Directory.Exists(legacyCacheDir)) + { + _migrated = true; + return; + } + + try + { + _logger.Info("Starting migration of legacy download cache directory from {0}...", legacyCacheDir); + + var files = Directory.GetFiles(legacyCacheDir, "*", SearchOption.AllDirectories); + var migratedCount = 0; + + using (var connection = _cacheDatabase.OpenConnection()) + using (var transaction = connection.BeginTransaction()) + { + const string insertSql = @" + INSERT OR IGNORE INTO DownloadCache (KeyHash, OriginalKey, Payload, CompressedSize, UncompressedSize, FileName, CreatedAt, LastAccessedAt) + VALUES (@hash, NULL, @compressed, @compressedSize, @uncompressedSize, @fileName, @createdAt, @lastAccessedAt); + "; + + foreach (var filePath in files) + { + try + { + var fileInfo = new FileInfo(filePath); + var hash = fileInfo.Directory?.Name; + + if (string.IsNullOrWhiteSpace(hash) || hash.Length != 64) + { + continue; + } + + var uncompressedBytes = File.ReadAllBytes(filePath); + if (uncompressedBytes.Length == 0) + { + continue; + } + + var compressedBytes = BrotliCompressionHelper.Compress(uncompressedBytes); + var createdAt = new DateTimeOffset(fileInfo.CreationTimeUtc).ToUnixTimeSeconds(); + var lastAccessedAt = new DateTimeOffset(fileInfo.LastWriteTimeUtc).ToUnixTimeSeconds(); + + connection.Execute(insertSql, + new + { + hash, + compressed = compressedBytes, + compressedSize = compressedBytes.Length, + uncompressedSize = uncompressedBytes.Length, + fileName = fileInfo.Name, + createdAt, + lastAccessedAt + }, + transaction); + + migratedCount++; + } + catch (Exception ex) + { + _logger.Warn(ex, "Failed to migrate legacy download cache file {0}", filePath); + } + } + + transaction.Commit(); + } + + _logger.Info("Successfully migrated {0} files from legacy download cache into SQLite.", migratedCount); + + try + { + Directory.Delete(legacyCacheDir, recursive: true); + _logger.Debug("Removed legacy download cache directory: {0}", legacyCacheDir); + } + catch (Exception ex) + { + _logger.Warn(ex, "Failed to remove legacy download cache directory after migration: {0}", legacyCacheDir); + } + } + catch (Exception ex) + { + _logger.Error(ex, "An error occurred during legacy download cache migration."); + } + finally + { + _migrated = true; + } + } + } + } +} diff --git a/src/NzbDrone.Core/Cache/DownloadCacheService.cs b/src/NzbDrone.Core/Cache/DownloadCacheService.cs new file mode 100644 index 00000000000..570ed24b223 --- /dev/null +++ b/src/NzbDrone.Core/Cache/DownloadCacheService.cs @@ -0,0 +1,200 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Threading.Tasks; +using Dapper; +using NLog; +using NzbDrone.Common.Extensions; + +namespace NzbDrone.Core.Cache +{ + public class DownloadCacheService : IDownloadCacheService + { + private readonly ISqliteCacheDatabase _cacheDatabase; + private readonly Logger _logger; + + public bool IsEnabled => + bool.TryParse(Environment.GetEnvironmentVariable("ENABLE_DOWNLOAD_CACHE"), out var enabled) && enabled; + + public DownloadCacheService(ISqliteCacheDatabase cacheDatabase, + Logger logger) + { + _cacheDatabase = cacheDatabase; + _logger = logger; + } + + public async Task Get(string key) + { + if (!IsEnabled) + { + return null; + } + + var hash = CacheKeyHasher.Hash(key); + + try + { + using var connection = _cacheDatabase.OpenConnection(); + + const string selectSql = "SELECT Payload, FileName FROM DownloadCache WHERE KeyHash = @hash;"; + var result = await connection.QueryFirstOrDefaultAsync(selectSql, new { hash }); + + if (result == null || result.Payload == null || result.Payload.Length == 0) + { + return null; + } + + _logger.Debug("Download cache hit for {0}: {1}", key, result.FileName.CleanFileName()); + + var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); + const string updateSql = "UPDATE DownloadCache SET LastAccessedAt = @now WHERE KeyHash = @hash;"; + + try + { + await connection.ExecuteAsync(updateSql, new { now, hash }); + } + catch (Exception ex) + { + // Cache metadata update failure shouldn't invalidate a successful read. + _logger.Warn(ex, "Failed to update last access metadata for cache key {0}", key); + } + + return BrotliCompressionHelper.Decompress(result.Payload); + } + catch (Exception ex) + { + _logger.Error(ex, "Failed to read download cache for key {0}", key); + return null; + } + } + + public async Task Store(string key, byte[] value, string fileName) + { + if (!IsEnabled || value == null || value.Length == 0) + { + return; + } + + var hash = CacheKeyHasher.Hash(key); + var safeFileName = GetSafeFileName(fileName); + + try + { + var compressed = BrotliCompressionHelper.Compress(value, CompressionLevel.Fastest); + var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); + + using var connection = _cacheDatabase.OpenConnection(); + + const string upsertSql = @" + INSERT INTO DownloadCache (KeyHash, OriginalKey, Payload, CompressedSize, UncompressedSize, FileName, CreatedAt, LastAccessedAt) + VALUES (@hash, @key, @compressed, @compressedSize, @uncompressedSize, @safeFileName, @now, @now) + ON CONFLICT(KeyHash) DO UPDATE SET + OriginalKey = @key, + Payload = @compressed, + CompressedSize = @compressedSize, + UncompressedSize = @uncompressedSize, + FileName = @safeFileName, + LastAccessedAt = @now; + "; + + await connection.ExecuteAsync(upsertSql, new + { + hash, + key, + compressed, + compressedSize = compressed.Length, + uncompressedSize = value.Length, + safeFileName, + now + }); + + _logger.Debug("Stored download in SQLite cache for key {0}: {1}", key, safeFileName); + } + catch (Exception ex) + { + _logger.Error(ex, "Failed to store download in cache for key {0}: {1}", key, fileName); + } + } + + public void Cleanup() + { + if (!IsEnabled) + { + return; + } + + var cacheMaxSize = long.TryParse( + Environment.GetEnvironmentVariable("DOWNLOAD_CACHE_MAX_SIZE_MB"), + out var mega) + ? mega + : 1000; + + var maxBytes = cacheMaxSize * 1024 * 1024; + + try + { + using var connection = _cacheDatabase.OpenConnection(); + + var currentTotalSize = connection.ExecuteScalar("SELECT COALESCE(SUM(CompressedSize), 0) FROM DownloadCache;"); + + _logger.Debug("Total compressed size of download cache: {0} MB, Limit: {1} MB", + currentTotalSize / 1024 / 1024, + maxBytes / 1024 / 1024); + + if (currentTotalSize <= maxBytes) + { + return; + } + + const string lruEvictionSql = @" + WITH Excess AS ( + SELECT KeyHash, + SUM(CompressedSize) OVER (ORDER BY LastAccessedAt DESC) AS CumulativeSize + FROM DownloadCache + ) + DELETE FROM DownloadCache + WHERE KeyHash IN ( + SELECT KeyHash FROM Excess WHERE CumulativeSize > @maxBytes + ); + "; + + var deletedCount = connection.Execute(lruEvictionSql, new { maxBytes }); + + _logger.Info("Cleaned up {0} entries from download cache", deletedCount); + } + catch (Exception ex) + { + _logger.Error(ex, "Failed to clean up download cache"); + } + } + + private static string GetSafeFileName(string fileName) + { + if (string.IsNullOrWhiteSpace(fileName)) + { + return "data"; + } + + var safeFileName = Path.GetFileName(fileName); + + if (string.IsNullOrWhiteSpace(safeFileName) || + safeFileName is "." or "..") + { + return "data"; + } + + if (safeFileName.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0) + { + return "data"; + } + + return safeFileName; + } + + private class DownloadCacheRecord + { + public byte[] Payload { get; set; } + public string FileName { get; set; } + } + } +} diff --git a/src/NzbDrone.Core/Cache/IDownloadCacheMigrator.cs b/src/NzbDrone.Core/Cache/IDownloadCacheMigrator.cs new file mode 100644 index 00000000000..2bf90ae7346 --- /dev/null +++ b/src/NzbDrone.Core/Cache/IDownloadCacheMigrator.cs @@ -0,0 +1,7 @@ +namespace NzbDrone.Core.Cache +{ + public interface IDownloadCacheMigrator + { + void Migrate(); + } +} diff --git a/src/NzbDrone.Core/Cache/IDownloadCacheService.cs b/src/NzbDrone.Core/Cache/IDownloadCacheService.cs new file mode 100644 index 00000000000..bd91c153b76 --- /dev/null +++ b/src/NzbDrone.Core/Cache/IDownloadCacheService.cs @@ -0,0 +1,15 @@ +using System.Threading.Tasks; + +namespace NzbDrone.Core.Cache +{ + public interface IDownloadCacheService + { + bool IsEnabled { get; } + + Task Get(string key); + + Task Store(string key, byte[] value, string fileName); + + void Cleanup(); + } +} diff --git a/src/NzbDrone.Core/Cache/ISqliteCacheDatabase.cs b/src/NzbDrone.Core/Cache/ISqliteCacheDatabase.cs new file mode 100644 index 00000000000..42612ed8f9d --- /dev/null +++ b/src/NzbDrone.Core/Cache/ISqliteCacheDatabase.cs @@ -0,0 +1,10 @@ +using System.Data; + +namespace NzbDrone.Core.Cache +{ + public interface ISqliteCacheDatabase + { + IDbConnection OpenConnection(); + void Initialize(); + } +} diff --git a/src/NzbDrone.Core/Cache/SqliteCacheDatabase.cs b/src/NzbDrone.Core/Cache/SqliteCacheDatabase.cs new file mode 100644 index 00000000000..6d66793d4d2 --- /dev/null +++ b/src/NzbDrone.Core/Cache/SqliteCacheDatabase.cs @@ -0,0 +1,124 @@ +using System; +using System.Data; +using System.Data.SQLite; +using System.IO; +using NLog; +using NzbDrone.Common.EnvironmentInfo; +using NzbDrone.Common.Extensions; + +namespace NzbDrone.Core.Cache +{ + public class SqliteCacheDatabase : ISqliteCacheDatabase + { + private readonly IAppFolderInfo _appFolderInfo; + private readonly Logger _logger; + private readonly string _connectionString; + private readonly object _initLock = new object(); + private bool _isInitialized; + + public SqliteCacheDatabase(IAppFolderInfo appFolderInfo, Logger logger) + { + _appFolderInfo = appFolderInfo; + _logger = logger; + + var dbPath = _appFolderInfo.GetCacheDatabase(); + var connectionBuilder = new SQLiteConnectionStringBuilder + { + DataSource = dbPath, + CacheSize = (int)-20000, + DateTimeKind = DateTimeKind.Utc, + JournalMode = OsInfo.IsOsx ? SQLiteJournalModeEnum.Truncate : SQLiteJournalModeEnum.Wal, + Pooling = true, + Version = 3, + BusyTimeout = 5000 + }; + + if (OsInfo.IsOsx) + { + connectionBuilder.Add("Full FSync", true); + } + + _connectionString = connectionBuilder.ConnectionString; + } + + public IDbConnection OpenConnection() + { + Initialize(); + + var connection = new SQLiteConnection(_connectionString); + connection.Open(); + return connection; + } + + public void Initialize() + { + if (_isInitialized) + { + return; + } + + lock (_initLock) + { + if (_isInitialized) + { + return; + } + + try + { + var dbPath = _appFolderInfo.GetCacheDatabase(); + var directory = Path.GetDirectoryName(dbPath); + + if (directory != null && !Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + + using (var connection = new SQLiteConnection(_connectionString)) + { + connection.Open(); + + using (var cmd = connection.CreateCommand()) + { + cmd.CommandText = @" + PRAGMA synchronous = NORMAL; + + CREATE TABLE IF NOT EXISTS DownloadCache ( + KeyHash TEXT PRIMARY KEY NOT NULL, + OriginalKey TEXT, + Payload BLOB NOT NULL, + CompressedSize INTEGER NOT NULL, + UncompressedSize INTEGER NOT NULL, + FileName TEXT, + CreatedAt INTEGER NOT NULL, + LastAccessedAt INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS IX_DownloadCache_LastAccessedAt ON DownloadCache (LastAccessedAt); + + CREATE TABLE IF NOT EXISTS OutputCache ( + KeyHash TEXT PRIMARY KEY NOT NULL, + OriginalKey TEXT, + Payload BLOB NOT NULL, + CompressedSize INTEGER NOT NULL, + UncompressedSize INTEGER NOT NULL, + CreatedAt INTEGER NOT NULL, + ExpiresAt INTEGER NOT NULL + ); + CREATE INDEX IF NOT EXISTS IX_OutputCache_ExpiresAt ON OutputCache (ExpiresAt); + "; + cmd.ExecuteNonQuery(); + } + } + + _isInitialized = true; + _logger.Debug("Initialized cache database at {0}", dbPath); + } + catch (Exception ex) + { + _logger.Error(ex, "Failed to initialize SQLite cache database"); + throw; + } + } + } + } +} diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOldCacheEntries.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOldCacheEntries.cs new file mode 100644 index 00000000000..fdf7320888f --- /dev/null +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOldCacheEntries.cs @@ -0,0 +1,36 @@ +using System; +using Dapper; +using NLog; +using NzbDrone.Core.Cache; + +namespace NzbDrone.Core.Housekeeping.Housekeepers +{ + public class CleanupOldDownloadCacheEntries(IDownloadCacheService downloadCacheService, + ISqliteCacheDatabase cacheDatabase, + Logger logger) : IHousekeepingTask + { + public void Clean() + { + if (downloadCacheService.IsEnabled) + { + downloadCacheService.Cleanup(); + } + + try + { + var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); + using var connection = cacheDatabase.OpenConnection(); + var expiredOutputCount = connection.Execute("DELETE FROM OutputCache WHERE ExpiresAt <= @now;", new { now }); + + if (expiredOutputCount > 0) + { + logger.Debug("Evicted {0} expired records from OutputCache", expiredOutputCount); + } + } + catch (Exception ex) + { + logger.Warn(ex, "Failed to evict expired records from OutputCache"); + } + } + } +} diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOldDiskCacheEntries.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOldDiskCacheEntries.cs deleted file mode 100644 index af4d764c8bf..00000000000 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOldDiskCacheEntries.cs +++ /dev/null @@ -1,17 +0,0 @@ -using NzbDrone.Core.Cache; - -namespace NzbDrone.Core.Housekeeping.Housekeepers -{ - public class CleanupOldDiskCacheEntries(IDiskCacheService diskCacheService) : IHousekeepingTask - { - public void Clean() - { - if (!diskCacheService.IsEnabled) - { - return; - } - - diskCacheService.Cleanup(); - } - } -} diff --git a/src/NzbDrone.Host/Caching/SqliteOutputCacheStore.cs b/src/NzbDrone.Host/Caching/SqliteOutputCacheStore.cs new file mode 100644 index 00000000000..fdea55c0e7b --- /dev/null +++ b/src/NzbDrone.Host/Caching/SqliteOutputCacheStore.cs @@ -0,0 +1,112 @@ +using System; +using System.IO.Compression; +using System.Threading; +using System.Threading.Tasks; +using Dapper; +using Microsoft.AspNetCore.OutputCaching; +using NLog; +using NzbDrone.Common.Instrumentation; +using NzbDrone.Core.Cache; + +namespace NzbDrone.Host.Caching +{ + public class SqliteOutputCacheStore : IOutputCacheStore + { + private readonly ISqliteCacheDatabase _cacheDatabase; + private readonly Logger _logger = NzbDroneLogger.GetLogger(typeof(SqliteOutputCacheStore)); + + public SqliteOutputCacheStore(ISqliteCacheDatabase cacheDatabase) + { + _cacheDatabase = cacheDatabase; + } + + public async ValueTask GetAsync(string key, CancellationToken cancellationToken) + { + var hash = CacheKeyHasher.Hash(key); + var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); + + try + { + using var connection = _cacheDatabase.OpenConnection(); + + const string selectSql = "SELECT Payload, ExpiresAt FROM OutputCache WHERE KeyHash = @hash;"; + var entry = await connection.QueryFirstOrDefaultAsync(selectSql, new { hash }); + + if (entry == null || entry.Payload == null || entry.Payload.Length == 0) + { + return null; + } + + if (entry.ExpiresAt <= now) + { + // Expired entry + return null; + } + + return BrotliCompressionHelper.Decompress(entry.Payload); + } + catch (Exception ex) + { + _logger.Warn(ex, "Failed to read OutputCache for key: {0}", key); + return null; + } + } + + public async ValueTask SetAsync(string key, byte[] value, string[] tags, TimeSpan validFor, CancellationToken cancellationToken) + { + if (value == null || value.Length == 0) + { + return; + } + + var hash = CacheKeyHasher.Hash(key); + + try + { + var compressed = BrotliCompressionHelper.Compress(value, CompressionLevel.Fastest); + var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); + var expiresAt = now + (long)validFor.TotalSeconds; + + using var connection = _cacheDatabase.OpenConnection(); + + const string upsertSql = @" + INSERT INTO OutputCache (KeyHash, OriginalKey, Payload, CompressedSize, UncompressedSize, CreatedAt, ExpiresAt) + VALUES (@hash, @key, @compressed, @compressedSize, @uncompressedSize, @now, @expiresAt) + ON CONFLICT(KeyHash) DO UPDATE SET + OriginalKey = @key, + Payload = @compressed, + CompressedSize = @compressedSize, + UncompressedSize = @uncompressedSize, + ExpiresAt = @expiresAt; + "; + + await connection.ExecuteAsync(upsertSql, new + { + hash, + key, + compressed, + compressedSize = compressed.Length, + uncompressedSize = value.Length, + now, + expiresAt + }); + } + catch (Exception ex) + { + _logger.Warn(ex, "Failed to store into OutputCache for key: {0}", key); + } + } + + public ValueTask EvictByTagAsync(string tag, CancellationToken cancellationToken) + { + // Tag-based eviction is not used + return ValueTask.CompletedTask; + } + + private class OutputCacheRecord + { + public byte[] Payload { get; set; } + public long ExpiresAt { get; set; } + } + } +} diff --git a/src/NzbDrone.Host/Startup.cs b/src/NzbDrone.Host/Startup.cs index e72628a6ec3..c1404e3c315 100644 --- a/src/NzbDrone.Host/Startup.cs +++ b/src/NzbDrone.Host/Startup.cs @@ -10,6 +10,7 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.HttpOverrides; using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.OutputCaching; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -19,12 +20,14 @@ using NzbDrone.Common.Instrumentation; using NzbDrone.Common.Processes; using NzbDrone.Common.Serializer; +using NzbDrone.Core.Cache; using NzbDrone.Core.Configuration; using NzbDrone.Core.Datastore; using NzbDrone.Core.Instrumentation; using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Messaging.Events; using NzbDrone.Host.AccessControl; +using NzbDrone.Host.Caching; using NzbDrone.SignalR; using Prowlarr.Api.V1.System; using Prowlarr.Http; @@ -108,6 +111,8 @@ public void ConfigureServices(IServiceCollection services) builder.With(context => !IsRssRequest(context.HttpContext.Request))); }); + services.AddSingleton(); + services .AddControllers(options => { @@ -240,6 +245,8 @@ public void Configure(IApplicationBuilder app, IStartupContext startupContext, Lazy mainDatabaseFactory, Lazy logDatabaseFactory, + ISqliteCacheDatabase sqliteCacheDatabase, + IDownloadCacheMigrator downloadCacheMigrator, DatabaseTarget dbTarget, ISingleInstancePolicy singleInstancePolicy, InitializeLogger initializeLogger, @@ -271,6 +278,9 @@ public void Configure(IApplicationBuilder app, dbTarget.Register(); } + sqliteCacheDatabase.Initialize(); + downloadCacheMigrator.Migrate(); + SchemaBuilder.Initialize(container); if (OsInfo.IsNotWindows) diff --git a/src/Prowlarr.Api.V1/Indexers/NewznabController.cs b/src/Prowlarr.Api.V1/Indexers/NewznabController.cs index 40040f74ab4..ab9b65ef6ac 100644 --- a/src/Prowlarr.Api.V1/Indexers/NewznabController.cs +++ b/src/Prowlarr.Api.V1/Indexers/NewznabController.cs @@ -37,7 +37,7 @@ public class NewznabController : Controller private IIndexerStatusService _indexerStatusService; private IDownloadMappingService _downloadMappingService { get; set; } private IDownloadService _downloadService { get; set; } - private IDiskCacheService _diskCacheService { get; set; } + private IDownloadCacheService _downloadCacheService { get; set; } private readonly Logger _logger; public NewznabController(IndexerFactory indexerFactory, @@ -46,7 +46,7 @@ public NewznabController(IndexerFactory indexerFactory, IIndexerStatusService indexerStatusService, IDownloadMappingService downloadMappingService, IDownloadService downloadService, - IDiskCacheService diskCacheService, + IDownloadCacheService downloadCacheService, Logger logger) { _indexerFactory = indexerFactory; @@ -55,7 +55,7 @@ public NewznabController(IndexerFactory indexerFactory, _indexerStatusService = indexerStatusService; _downloadMappingService = downloadMappingService; _downloadService = downloadService; - _diskCacheService = diskCacheService; + _downloadCacheService = downloadCacheService; _logger = logger; } @@ -268,7 +268,7 @@ public async Task GetDownload(int id, string link, string file) throw new BadRequestException("Failed to normalize provided link"); } - var enableDownloadCache = _diskCacheService.IsEnabled; + var enableDownloadCache = _downloadCacheService.IsEnabled; // If Indexer is set to download via Redirect then just redirect to the link unless it's a Usenet indexer at which point it forces Redirect. if (!enableDownloadCache) @@ -289,7 +289,7 @@ public async Task GetDownload(int id, string link, string file) if (enableDownloadCache) { - downloadBytes = await _diskCacheService.Get(unprotectedLink); + downloadBytes = await _downloadCacheService.Get(unprotectedLink); } if (downloadBytes == null) @@ -300,7 +300,7 @@ public async Task GetDownload(int id, string link, string file) if (enableDownloadCache) { - await _diskCacheService.Store(unprotectedLink, downloadBytes, filename); + await _downloadCacheService.Store(unprotectedLink, downloadBytes, filename); } } catch (ReleaseUnavailableException ex) From d82ac5d034bdef9c43f849738bd843868670bd1e Mon Sep 17 00:00:00 2001 From: realzombee <209545148+realzombee@users.noreply.github.com> Date: Fri, 4 Sep 2026 23:53:56 +0100 Subject: [PATCH 2/7] Re-order wiring in SqliteOutputCacheStore --- src/NzbDrone.Host/Startup.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/NzbDrone.Host/Startup.cs b/src/NzbDrone.Host/Startup.cs index c1404e3c315..0f334043c0a 100644 --- a/src/NzbDrone.Host/Startup.cs +++ b/src/NzbDrone.Host/Startup.cs @@ -103,6 +103,8 @@ public void ConfigureServices(IServiceCollection services) ? mega : 100; + services.AddSingleton(); + services.AddOutputCache(options => { options.DefaultExpirationTimeSpan = TimeSpan.FromMinutes(cacheTtl); @@ -111,8 +113,6 @@ public void ConfigureServices(IServiceCollection services) builder.With(context => !IsRssRequest(context.HttpContext.Request))); }); - services.AddSingleton(); - services .AddControllers(options => { From 7a9668167908a5bbd21ce2a6edbb9cae073cf5e7 Mon Sep 17 00:00:00 2001 From: realzombee <209545148+realzombee@users.noreply.github.com> Date: Sat, 5 Sep 2026 19:08:21 +0100 Subject: [PATCH 3/7] refactor: Move away from OutputCache - The OutputCache behavior is opaque and does not seem to work as expected. Manually cache responses using the SQLite cache database --- .../Cache/QueryCacheService.cs} | 70 ++++++++++--------- .../Cache/SqliteCacheDatabase.cs | 4 +- .../Housekeepers/CleanupOldCacheEntries.cs | 22 +----- src/NzbDrone.Host/Startup.cs | 46 ------------ .../Indexers/NewznabController.cs | 48 ++++++++++++- 5 files changed, 85 insertions(+), 105 deletions(-) rename src/{NzbDrone.Host/Caching/SqliteOutputCacheStore.cs => NzbDrone.Core/Cache/QueryCacheService.cs} (55%) diff --git a/src/NzbDrone.Host/Caching/SqliteOutputCacheStore.cs b/src/NzbDrone.Core/Cache/QueryCacheService.cs similarity index 55% rename from src/NzbDrone.Host/Caching/SqliteOutputCacheStore.cs rename to src/NzbDrone.Core/Cache/QueryCacheService.cs index fdea55c0e7b..4d25d701acd 100644 --- a/src/NzbDrone.Host/Caching/SqliteOutputCacheStore.cs +++ b/src/NzbDrone.Core/Cache/QueryCacheService.cs @@ -1,58 +1,45 @@ using System; using System.IO.Compression; -using System.Threading; using System.Threading.Tasks; using Dapper; -using Microsoft.AspNetCore.OutputCaching; using NLog; -using NzbDrone.Common.Instrumentation; -using NzbDrone.Core.Cache; -namespace NzbDrone.Host.Caching +namespace NzbDrone.Core.Cache { - public class SqliteOutputCacheStore : IOutputCacheStore + public class QueryCacheService(ISqliteCacheDatabase cacheDatabase, Logger logger) { - private readonly ISqliteCacheDatabase _cacheDatabase; - private readonly Logger _logger = NzbDroneLogger.GetLogger(typeof(SqliteOutputCacheStore)); + private static readonly TimeSpan CacheTtl = TimeSpan.FromMinutes( + int.TryParse(Environment.GetEnvironmentVariable("CACHE_TTL_MINS"), out var mins) && mins > 0 + ? mins + : 10); - public SqliteOutputCacheStore(ISqliteCacheDatabase cacheDatabase) - { - _cacheDatabase = cacheDatabase; - } - - public async ValueTask GetAsync(string key, CancellationToken cancellationToken) + public async ValueTask GetAsync(string key) { var hash = CacheKeyHasher.Hash(key); var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); try { - using var connection = _cacheDatabase.OpenConnection(); + using var connection = cacheDatabase.OpenConnection(); - const string selectSql = "SELECT Payload, ExpiresAt FROM OutputCache WHERE KeyHash = @hash;"; - var entry = await connection.QueryFirstOrDefaultAsync(selectSql, new { hash }); + const string selectSql = "SELECT Payload, ExpiresAt FROM QueryCache WHERE KeyHash = @hash;"; + var entry = await connection.QueryFirstOrDefaultAsync(selectSql, new { hash }); - if (entry == null || entry.Payload == null || entry.Payload.Length == 0) + if (entry?.Payload == null || entry.Payload.Length == 0 || entry.ExpiresAt <= now) { return null; } - if (entry.ExpiresAt <= now) - { - // Expired entry - return null; - } - return BrotliCompressionHelper.Decompress(entry.Payload); } catch (Exception ex) { - _logger.Warn(ex, "Failed to read OutputCache for key: {0}", key); + logger.Warn(ex, "Failed to read QueryCache for key: {0}", key); return null; } } - public async ValueTask SetAsync(string key, byte[] value, string[] tags, TimeSpan validFor, CancellationToken cancellationToken) + public async ValueTask SetAsync(string key, byte[] value) { if (value == null || value.Length == 0) { @@ -65,12 +52,12 @@ public async ValueTask SetAsync(string key, byte[] value, string[] tags, TimeSpa { var compressed = BrotliCompressionHelper.Compress(value, CompressionLevel.Fastest); var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); - var expiresAt = now + (long)validFor.TotalSeconds; + var expiresAt = now + (long)CacheTtl.TotalSeconds; - using var connection = _cacheDatabase.OpenConnection(); + using var connection = cacheDatabase.OpenConnection(); const string upsertSql = @" - INSERT INTO OutputCache (KeyHash, OriginalKey, Payload, CompressedSize, UncompressedSize, CreatedAt, ExpiresAt) + INSERT INTO QueryCache (KeyHash, OriginalKey, Payload, CompressedSize, UncompressedSize, CreatedAt, ExpiresAt) VALUES (@hash, @key, @compressed, @compressedSize, @uncompressedSize, @now, @expiresAt) ON CONFLICT(KeyHash) DO UPDATE SET OriginalKey = @key, @@ -93,17 +80,32 @@ ON CONFLICT(KeyHash) DO UPDATE SET } catch (Exception ex) { - _logger.Warn(ex, "Failed to store into OutputCache for key: {0}", key); + logger.Warn(ex, "Failed to store into QueryCache for key: {0}", key); } } - public ValueTask EvictByTagAsync(string tag, CancellationToken cancellationToken) + public void Cleanup() { - // Tag-based eviction is not used - return ValueTask.CompletedTask; + try + { + var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); + + using var connection = cacheDatabase.OpenConnection(); + var expiredOutputCount = + connection.Execute("DELETE FROM QueryCache WHERE ExpiresAt <= @now;", new { now }); + + if (expiredOutputCount > 0) + { + logger.Debug("Evicted {0} expired records from QueryCache", expiredOutputCount); + } + } + catch (Exception ex) + { + logger.Warn(ex, "Failed to evict expired records from QueryCache"); + } } - private class OutputCacheRecord + private class QueryCacheRecord { public byte[] Payload { get; set; } public long ExpiresAt { get; set; } diff --git a/src/NzbDrone.Core/Cache/SqliteCacheDatabase.cs b/src/NzbDrone.Core/Cache/SqliteCacheDatabase.cs index 6d66793d4d2..bb59802bdb9 100644 --- a/src/NzbDrone.Core/Cache/SqliteCacheDatabase.cs +++ b/src/NzbDrone.Core/Cache/SqliteCacheDatabase.cs @@ -95,7 +95,7 @@ LastAccessedAt INTEGER NOT NULL ); CREATE INDEX IF NOT EXISTS IX_DownloadCache_LastAccessedAt ON DownloadCache (LastAccessedAt); - CREATE TABLE IF NOT EXISTS OutputCache ( + CREATE TABLE IF NOT EXISTS QueryCache ( KeyHash TEXT PRIMARY KEY NOT NULL, OriginalKey TEXT, Payload BLOB NOT NULL, @@ -104,7 +104,7 @@ CREATE TABLE IF NOT EXISTS OutputCache ( CreatedAt INTEGER NOT NULL, ExpiresAt INTEGER NOT NULL ); - CREATE INDEX IF NOT EXISTS IX_OutputCache_ExpiresAt ON OutputCache (ExpiresAt); + CREATE INDEX IF NOT EXISTS IX_QueryCache_ExpiresAt ON QueryCache (ExpiresAt); "; cmd.ExecuteNonQuery(); } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOldCacheEntries.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOldCacheEntries.cs index fdf7320888f..e6b002792aa 100644 --- a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOldCacheEntries.cs +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOldCacheEntries.cs @@ -1,13 +1,9 @@ -using System; -using Dapper; -using NLog; using NzbDrone.Core.Cache; namespace NzbDrone.Core.Housekeeping.Housekeepers { public class CleanupOldDownloadCacheEntries(IDownloadCacheService downloadCacheService, - ISqliteCacheDatabase cacheDatabase, - Logger logger) : IHousekeepingTask + QueryCacheService queryCacheService) : IHousekeepingTask { public void Clean() { @@ -16,21 +12,7 @@ public void Clean() downloadCacheService.Cleanup(); } - try - { - var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); - using var connection = cacheDatabase.OpenConnection(); - var expiredOutputCount = connection.Execute("DELETE FROM OutputCache WHERE ExpiresAt <= @now;", new { now }); - - if (expiredOutputCount > 0) - { - logger.Debug("Evicted {0} expired records from OutputCache", expiredOutputCount); - } - } - catch (Exception ex) - { - logger.Warn(ex, "Failed to evict expired records from OutputCache"); - } + queryCacheService.Cleanup(); } } } diff --git a/src/NzbDrone.Host/Startup.cs b/src/NzbDrone.Host/Startup.cs index 0f334043c0a..207ebd0240f 100644 --- a/src/NzbDrone.Host/Startup.cs +++ b/src/NzbDrone.Host/Startup.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Net; using DryIoc; using Microsoft.AspNetCore.Authorization; @@ -10,7 +9,6 @@ using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.HttpOverrides; using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.OutputCaching; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -27,7 +25,6 @@ using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Messaging.Events; using NzbDrone.Host.AccessControl; -using NzbDrone.Host.Caching; using NzbDrone.SignalR; using Prowlarr.Api.V1.System; using Prowlarr.Http; @@ -91,28 +88,6 @@ public void ConfigureServices(IServiceCollection services) .AllowAnyHeader()); }); - var cacheTtl = int.TryParse( - Environment.GetEnvironmentVariable("CACHE_TTL_MINS"), - out var minutes) - ? minutes - : 10; - - var cacheSize = int.TryParse( - Environment.GetEnvironmentVariable("CACHE_MAX_SIZE_MB"), - out var mega) - ? mega - : 100; - - services.AddSingleton(); - - services.AddOutputCache(options => - { - options.DefaultExpirationTimeSpan = TimeSpan.FromMinutes(cacheTtl); - options.SizeLimit = cacheSize * 1024 * 1024; - options.AddPolicy("NewznabQuery", builder => - builder.With(context => !IsRssRequest(context.HttpContext.Request))); - }); - services .AddControllers(options => { @@ -308,7 +283,6 @@ public void Configure(IApplicationBuilder app, app.UseCors(); app.UseAuthentication(); app.UseAuthorization(); - app.UseOutputCache(); app.UseResponseCompression(); app.Properties["host.AppName"] = BuildInfo.AppName; @@ -361,25 +335,5 @@ private void EnsureSingleInstance(bool isService, IStartupContext startupContext instancePolicy.PreventStartIfAlreadyRunning(); } } - - private static bool IsRssRequest(HttpRequest request) - { - var query = request.Query; - var requestType = query["t"].ToString(); - - if (requestType is not ("search" or "tvsearch" or "movie" or "music" or "book")) - { - return false; - } - - string[] searchParams = - { - "q", "imdbid", "tmdbid", "tvdbid", "rid", "tvmazeid", "traktid", "doubanid", - "season", "ep", "album", "artist", "label", "track", "year", "genre", - "author", "title", "publisher" - }; - - return searchParams.All(param => string.IsNullOrWhiteSpace(query[param].ToString())); - } } } diff --git a/src/Prowlarr.Api.V1/Indexers/NewznabController.cs b/src/Prowlarr.Api.V1/Indexers/NewznabController.cs index ab9b65ef6ac..f501fefaaae 100644 --- a/src/Prowlarr.Api.V1/Indexers/NewznabController.cs +++ b/src/Prowlarr.Api.V1/Indexers/NewznabController.cs @@ -8,7 +8,6 @@ using Microsoft.AspNetCore.Cors; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; -using Microsoft.AspNetCore.OutputCaching; using Microsoft.Net.Http.Headers; using NLog; using NzbDrone.Common.Extensions; @@ -23,6 +22,7 @@ using Prowlarr.Http.Extensions; using Prowlarr.Http.REST; using BadRequestException = NzbDrone.Core.Exceptions.BadRequestException; +using HttpRequest = Microsoft.AspNetCore.Http.HttpRequest; namespace NzbDrone.Api.V1.Indexers { @@ -37,6 +37,7 @@ public class NewznabController : Controller private IIndexerStatusService _indexerStatusService; private IDownloadMappingService _downloadMappingService { get; set; } private IDownloadService _downloadService { get; set; } + private QueryCacheService _queryCacheService { get; set; } private IDownloadCacheService _downloadCacheService { get; set; } private readonly Logger _logger; @@ -47,6 +48,7 @@ public NewznabController(IndexerFactory indexerFactory, IDownloadMappingService downloadMappingService, IDownloadService downloadService, IDownloadCacheService downloadCacheService, + QueryCacheService queryCacheService, Logger logger) { _indexerFactory = indexerFactory; @@ -56,12 +58,37 @@ public NewznabController(IndexerFactory indexerFactory, _downloadMappingService = downloadMappingService; _downloadService = downloadService; _downloadCacheService = downloadCacheService; + _queryCacheService = queryCacheService; _logger = logger; } + private string BuildCacheKey() + { + return $"{Request.Path}{Request.QueryString}"; + } + + private static bool IsRssRequest(HttpRequest request) + { + var query = request.Query; + var requestType = query["t"].ToString(); + + if (requestType is not ("search" or "tvsearch" or "movie" or "music" or "book")) + { + return false; + } + + string[] searchParams = + { + "q", "imdbid", "tmdbid", "tvdbid", "rid", "tvmazeid", "traktid", "doubanid", + "season", "ep", "album", "artist", "label", "track", "year", "genre", + "author", "title", "publisher" + }; + + return searchParams.All(param => string.IsNullOrWhiteSpace(query[param].ToString())); + } + [HttpGet("/api/v1/indexer/{id:int}/newznab")] [HttpGet("{id:int}/api")] - [OutputCache(PolicyName = "NewznabQuery")] public async Task GetNewznabResponse(int id, [FromQuery] NewznabRequest request) { var requestType = request.t; @@ -69,6 +96,14 @@ public async Task GetNewznabResponse(int id, [FromQuery] NewznabR request.server = Request.GetServerUrl(); request.host = Request.GetHostName(); + var cacheKey = BuildCacheKey(); + var cachedBytes = await _queryCacheService.GetAsync(cacheKey); + if (cachedBytes != null) + { + var cachedXml = Encoding.UTF8.GetString(cachedBytes); + return CreateResponse(cachedXml); + } + if (requestType.IsNullOrWhiteSpace()) { return CreateResponse(CreateErrorXML(200, "Missing parameter (t)"), statusCode: StatusCodes.Status400BadRequest); @@ -206,7 +241,14 @@ public async Task GetNewznabResponse(int id, [FromQuery] NewznabR var preferMagnetUrl = indexer.Protocol == DownloadProtocol.Torrent && indexerDef.Settings is ITorrentIndexerSettings torrentIndexerSettings && (torrentIndexerSettings.TorrentBaseSettings?.PreferMagnetUrl ?? false); - return CreateResponse(results.ToXml(indexer.Protocol, preferMagnetUrl)); + var resultsXml = results.ToXml(indexer.Protocol, preferMagnetUrl); + + if (!IsRssRequest(Request)) + { + await _queryCacheService.SetAsync(cacheKey, Encoding.UTF8.GetBytes(resultsXml)); + } + + return CreateResponse(resultsXml); default: return CreateResponse(CreateErrorXML(202, $"No such function ({requestType})"), statusCode: StatusCodes.Status400BadRequest); } From 99625986c66bf79733f5f4995e9200cc883bb4b8 Mon Sep 17 00:00:00 2001 From: realzombee <209545148+realzombee@users.noreply.github.com> Date: Sat, 5 Sep 2026 19:28:06 +0100 Subject: [PATCH 4/7] refactor: Use CreatedAt directly for checking cache expiration --- src/NzbDrone.Core/Cache/QueryCacheService.cs | 19 +++++++++---------- .../Cache/SqliteCacheDatabase.cs | 5 ++--- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/NzbDrone.Core/Cache/QueryCacheService.cs b/src/NzbDrone.Core/Cache/QueryCacheService.cs index 4d25d701acd..833499fba53 100644 --- a/src/NzbDrone.Core/Cache/QueryCacheService.cs +++ b/src/NzbDrone.Core/Cache/QueryCacheService.cs @@ -22,10 +22,10 @@ public async ValueTask GetAsync(string key) { using var connection = cacheDatabase.OpenConnection(); - const string selectSql = "SELECT Payload, ExpiresAt FROM QueryCache WHERE KeyHash = @hash;"; + const string selectSql = "SELECT Payload, CreatedAt FROM QueryCache WHERE KeyHash = @hash;"; var entry = await connection.QueryFirstOrDefaultAsync(selectSql, new { hash }); - if (entry?.Payload == null || entry.Payload.Length == 0 || entry.ExpiresAt <= now) + if (entry?.Payload == null || entry.Payload.Length == 0 || (entry.CreatedAt + (long)CacheTtl.TotalSeconds) <= now) { return null; } @@ -52,19 +52,18 @@ public async ValueTask SetAsync(string key, byte[] value) { var compressed = BrotliCompressionHelper.Compress(value, CompressionLevel.Fastest); var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); - var expiresAt = now + (long)CacheTtl.TotalSeconds; using var connection = cacheDatabase.OpenConnection(); const string upsertSql = @" - INSERT INTO QueryCache (KeyHash, OriginalKey, Payload, CompressedSize, UncompressedSize, CreatedAt, ExpiresAt) - VALUES (@hash, @key, @compressed, @compressedSize, @uncompressedSize, @now, @expiresAt) + INSERT INTO QueryCache (KeyHash, OriginalKey, Payload, CompressedSize, UncompressedSize, CreatedAt) + VALUES (@hash, @key, @compressed, @compressedSize, @uncompressedSize, @now) ON CONFLICT(KeyHash) DO UPDATE SET OriginalKey = @key, Payload = @compressed, CompressedSize = @compressedSize, UncompressedSize = @uncompressedSize, - ExpiresAt = @expiresAt; + CreatedAt = @now; "; await connection.ExecuteAsync(upsertSql, new @@ -74,8 +73,7 @@ ON CONFLICT(KeyHash) DO UPDATE SET compressed, compressedSize = compressed.Length, uncompressedSize = value.Length, - now, - expiresAt + now }); } catch (Exception ex) @@ -89,10 +87,11 @@ public void Cleanup() try { var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); + var threshold = now - (long)CacheTtl.TotalSeconds; using var connection = cacheDatabase.OpenConnection(); var expiredOutputCount = - connection.Execute("DELETE FROM QueryCache WHERE ExpiresAt <= @now;", new { now }); + connection.Execute("DELETE FROM QueryCache WHERE CreatedAt <= @threshold;", new { threshold }); if (expiredOutputCount > 0) { @@ -108,7 +107,7 @@ public void Cleanup() private class QueryCacheRecord { public byte[] Payload { get; set; } - public long ExpiresAt { get; set; } + public long CreatedAt { get; set; } } } } diff --git a/src/NzbDrone.Core/Cache/SqliteCacheDatabase.cs b/src/NzbDrone.Core/Cache/SqliteCacheDatabase.cs index bb59802bdb9..f7e3e151dd3 100644 --- a/src/NzbDrone.Core/Cache/SqliteCacheDatabase.cs +++ b/src/NzbDrone.Core/Cache/SqliteCacheDatabase.cs @@ -101,10 +101,9 @@ CREATE TABLE IF NOT EXISTS QueryCache ( Payload BLOB NOT NULL, CompressedSize INTEGER NOT NULL, UncompressedSize INTEGER NOT NULL, - CreatedAt INTEGER NOT NULL, - ExpiresAt INTEGER NOT NULL + CreatedAt INTEGER NOT NULL ); - CREATE INDEX IF NOT EXISTS IX_QueryCache_ExpiresAt ON QueryCache (ExpiresAt); + CREATE INDEX IF NOT EXISTS IX_QueryCache_CreatedAt ON QueryCache (CreatedAt); "; cmd.ExecuteNonQuery(); } From 40905fe8faec1b98705bf7c2a525e24ae59d2f89 Mon Sep 17 00:00:00 2001 From: realzombee <209545148+realzombee@users.noreply.github.com> Date: Sat, 5 Sep 2026 22:16:32 +0100 Subject: [PATCH 5/7] refactor: Use DATETIME strings for SQLite timestamps --- .../Cache/DownloadCacheMigrator.cs | 4 +-- .../Cache/DownloadCacheService.cs | 17 +++++------ src/NzbDrone.Core/Cache/QueryCacheService.cs | 28 +++++++++---------- .../Cache/SqliteCacheDatabase.cs | 6 ++-- 4 files changed, 26 insertions(+), 29 deletions(-) diff --git a/src/NzbDrone.Core/Cache/DownloadCacheMigrator.cs b/src/NzbDrone.Core/Cache/DownloadCacheMigrator.cs index 96ffb813b15..793fe37132f 100644 --- a/src/NzbDrone.Core/Cache/DownloadCacheMigrator.cs +++ b/src/NzbDrone.Core/Cache/DownloadCacheMigrator.cs @@ -77,8 +77,8 @@ INSERT OR IGNORE INTO DownloadCache (KeyHash, OriginalKey, Payload, CompressedSi } var compressedBytes = BrotliCompressionHelper.Compress(uncompressedBytes); - var createdAt = new DateTimeOffset(fileInfo.CreationTimeUtc).ToUnixTimeSeconds(); - var lastAccessedAt = new DateTimeOffset(fileInfo.LastWriteTimeUtc).ToUnixTimeSeconds(); + var createdAt = fileInfo.CreationTimeUtc.ToString("yyyy-MM-dd HH:mm:ss"); + var lastAccessedAt = fileInfo.LastWriteTimeUtc.ToString("yyyy-MM-dd HH:mm:ss"); connection.Execute(insertSql, new diff --git a/src/NzbDrone.Core/Cache/DownloadCacheService.cs b/src/NzbDrone.Core/Cache/DownloadCacheService.cs index 570ed24b223..eb57cf04b6b 100644 --- a/src/NzbDrone.Core/Cache/DownloadCacheService.cs +++ b/src/NzbDrone.Core/Cache/DownloadCacheService.cs @@ -39,19 +39,18 @@ public async Task Get(string key) const string selectSql = "SELECT Payload, FileName FROM DownloadCache WHERE KeyHash = @hash;"; var result = await connection.QueryFirstOrDefaultAsync(selectSql, new { hash }); - if (result == null || result.Payload == null || result.Payload.Length == 0) + if (result?.Payload == null || result.Payload.Length == 0) { return null; } _logger.Debug("Download cache hit for {0}: {1}", key, result.FileName.CleanFileName()); - var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); - const string updateSql = "UPDATE DownloadCache SET LastAccessedAt = @now WHERE KeyHash = @hash;"; + const string updateSql = "UPDATE DownloadCache SET LastAccessedAt = datetime('now') WHERE KeyHash = @hash;"; try { - await connection.ExecuteAsync(updateSql, new { now, hash }); + await connection.ExecuteAsync(updateSql, new { hash }); } catch (Exception ex) { @@ -81,20 +80,19 @@ public async Task Store(string key, byte[] value, string fileName) try { var compressed = BrotliCompressionHelper.Compress(value, CompressionLevel.Fastest); - var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); using var connection = _cacheDatabase.OpenConnection(); const string upsertSql = @" INSERT INTO DownloadCache (KeyHash, OriginalKey, Payload, CompressedSize, UncompressedSize, FileName, CreatedAt, LastAccessedAt) - VALUES (@hash, @key, @compressed, @compressedSize, @uncompressedSize, @safeFileName, @now, @now) + VALUES (@hash, @key, @compressed, @compressedSize, @uncompressedSize, @safeFileName, datetime('now'), datetime('now')) ON CONFLICT(KeyHash) DO UPDATE SET OriginalKey = @key, Payload = @compressed, CompressedSize = @compressedSize, UncompressedSize = @uncompressedSize, FileName = @safeFileName, - LastAccessedAt = @now; + LastAccessedAt = datetime('now'); "; await connection.ExecuteAsync(upsertSql, new @@ -104,8 +102,7 @@ ON CONFLICT(KeyHash) DO UPDATE SET compressed, compressedSize = compressed.Length, uncompressedSize = value.Length, - safeFileName, - now + safeFileName }); _logger.Debug("Stored download in SQLite cache for key {0}: {1}", key, safeFileName); @@ -137,7 +134,7 @@ public void Cleanup() var currentTotalSize = connection.ExecuteScalar("SELECT COALESCE(SUM(CompressedSize), 0) FROM DownloadCache;"); - _logger.Debug("Total compressed size of download cache: {0} MB, Limit: {1} MB", + _logger.Debug("Total size of download cache: {0} MB, Limit: {1} MB", currentTotalSize / 1024 / 1024, maxBytes / 1024 / 1024); diff --git a/src/NzbDrone.Core/Cache/QueryCacheService.cs b/src/NzbDrone.Core/Cache/QueryCacheService.cs index 833499fba53..4fed33c8ce2 100644 --- a/src/NzbDrone.Core/Cache/QueryCacheService.cs +++ b/src/NzbDrone.Core/Cache/QueryCacheService.cs @@ -16,16 +16,21 @@ public class QueryCacheService(ISqliteCacheDatabase cacheDatabase, Logger logger public async ValueTask GetAsync(string key) { var hash = CacheKeyHasher.Hash(key); - var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); try { using var connection = cacheDatabase.OpenConnection(); - const string selectSql = "SELECT Payload, CreatedAt FROM QueryCache WHERE KeyHash = @hash;"; - var entry = await connection.QueryFirstOrDefaultAsync(selectSql, new { hash }); + const string selectSql = @" + SELECT Payload + FROM QueryCache + WHERE KeyHash = @hash + AND CreatedAt >= (datetime('now', '-' || @ttlMinutes || ' minutes')); + "; - if (entry?.Payload == null || entry.Payload.Length == 0 || (entry.CreatedAt + (long)CacheTtl.TotalSeconds) <= now) + var entry = await connection.QueryFirstOrDefaultAsync(selectSql, new { hash, ttlMinutes = CacheTtl.TotalMinutes }); + + if (entry?.Payload == null || entry.Payload.Length == 0) { return null; } @@ -51,19 +56,18 @@ public async ValueTask SetAsync(string key, byte[] value) try { var compressed = BrotliCompressionHelper.Compress(value, CompressionLevel.Fastest); - var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); using var connection = cacheDatabase.OpenConnection(); const string upsertSql = @" INSERT INTO QueryCache (KeyHash, OriginalKey, Payload, CompressedSize, UncompressedSize, CreatedAt) - VALUES (@hash, @key, @compressed, @compressedSize, @uncompressedSize, @now) + VALUES (@hash, @key, @compressed, @compressedSize, @uncompressedSize, datetime('now')) ON CONFLICT(KeyHash) DO UPDATE SET OriginalKey = @key, Payload = @compressed, CompressedSize = @compressedSize, UncompressedSize = @uncompressedSize, - CreatedAt = @now; + CreatedAt = datetime('now'); "; await connection.ExecuteAsync(upsertSql, new @@ -72,8 +76,7 @@ ON CONFLICT(KeyHash) DO UPDATE SET key, compressed, compressedSize = compressed.Length, - uncompressedSize = value.Length, - now + uncompressedSize = value.Length }); } catch (Exception ex) @@ -86,12 +89,10 @@ public void Cleanup() { try { - var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); - var threshold = now - (long)CacheTtl.TotalSeconds; - using var connection = cacheDatabase.OpenConnection(); var expiredOutputCount = - connection.Execute("DELETE FROM QueryCache WHERE CreatedAt <= @threshold;", new { threshold }); + connection.Execute("DELETE FROM QueryCache WHERE CreatedAt <= (datetime('now', '-' || @ttlMinutes || ' minutes'))", + new { ttlMinutes = CacheTtl.TotalMinutes }); if (expiredOutputCount > 0) { @@ -107,7 +108,6 @@ public void Cleanup() private class QueryCacheRecord { public byte[] Payload { get; set; } - public long CreatedAt { get; set; } } } } diff --git a/src/NzbDrone.Core/Cache/SqliteCacheDatabase.cs b/src/NzbDrone.Core/Cache/SqliteCacheDatabase.cs index f7e3e151dd3..dd5d9f57145 100644 --- a/src/NzbDrone.Core/Cache/SqliteCacheDatabase.cs +++ b/src/NzbDrone.Core/Cache/SqliteCacheDatabase.cs @@ -90,8 +90,8 @@ CREATE TABLE IF NOT EXISTS DownloadCache ( CompressedSize INTEGER NOT NULL, UncompressedSize INTEGER NOT NULL, FileName TEXT, - CreatedAt INTEGER NOT NULL, - LastAccessedAt INTEGER NOT NULL + CreatedAt DATETIME NOT NULL, + LastAccessedAt DATETIME NOT NULL ); CREATE INDEX IF NOT EXISTS IX_DownloadCache_LastAccessedAt ON DownloadCache (LastAccessedAt); @@ -101,7 +101,7 @@ CREATE TABLE IF NOT EXISTS QueryCache ( Payload BLOB NOT NULL, CompressedSize INTEGER NOT NULL, UncompressedSize INTEGER NOT NULL, - CreatedAt INTEGER NOT NULL + CreatedAt DATETIME NOT NULL ); CREATE INDEX IF NOT EXISTS IX_QueryCache_CreatedAt ON QueryCache (CreatedAt); "; From 9817ec5b6ab31271e8b7857e7f706a52d75f1574 Mon Sep 17 00:00:00 2001 From: realzombee <209545148+realzombee@users.noreply.github.com> Date: Sat, 5 Sep 2026 22:25:11 +0100 Subject: [PATCH 6/7] Update README.md --- README.md | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 7cf22737ccc..e67a662fe96 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,6 @@ prowlarr: - TZ=Etc/UTC # Configure caching behavior - CACHE_TTL_MINS=10 - - CACHE_MAX_SIZE_MB=100 - ENABLE_DOWNLOAD_CACHE=true - DOWNLOAD_CACHE_MAX_SIZE_MB=1000 volumes: @@ -35,8 +34,6 @@ This fork aims to improve certain aspects of Prowlarr to make it work better wit | Env Var | Default | Description | |-------------------|---------|--------------------------------------------------------------------------------------------------------| | CACHE_TTL_MINS | 10 | How long a particular query response should be cached for. RSS queries are not cached. | -| CACHE_MAX_SIZE_MB | 100 | Maximum size of cache in memory before old records are cleaned up. Higher values will use more memory. | - Debrid/Usenet mounting tools cause a lot of repeated queries to the indexer that waste time and API queries. In particular, the workflow for most Usenet streaming setups is: - Arrs search for an item @@ -53,16 +50,10 @@ Generally, if you're using any of the streaming clients, this fork will give you WITH enriched AS ( SELECT IndexerId, - json_extract(Data, '$.season') AS season, - json_extract(Data, '$.query') AS query, - json_extract(Data, '$.categories') AS categories, - json_extract(Data, '$.queryType') AS queryType, - json_extract(Data, '$.tvdbId') AS tvdbId, - json_extract(Data, '$.tmdbId') AS tmdbId, - json_extract(Data, '$.imdbId') AS imdbId, + json_extract(Data, '$.url') AS url, CAST(strftime('%s', date) / 600 AS INTEGER) AS window_id FROM History - WHERE date >= datetime('now', '-90 days') AND (EventType = 2 OR EventType = 3) + WHERE date >= datetime('now', '-90 days') AND EventType = 2 ), grouped AS ( SELECT @@ -70,7 +61,7 @@ SELECT COUNT(*) - 1 AS duplicate_calls FROM enriched GROUP BY - IndexerId, season, query, categories, queryType, tvdbId, tmdbId, imdbId, window_id + IndexerId, url, window_id ) SELECT SUM(total_calls) AS total_requests, @@ -81,10 +72,10 @@ FROM grouped; ## Cache nzb/torrent files -| Env Var | Default | Description | -|----------------------------|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| ENABLE_DOWNLOAD_CACHE | false | Whether Prowlarr should download and cache nzb/torrent files | -| DOWNLOAD_CACHE_MAX_SIZE_MB | 1000 | Maximum size of download cache on disk. The cleanup job runs with the housekeeping tasks every 24 hours so this is not a strict limit. In testing, 1GB of disk cache stored ~6k nzbs. | +| Env Var | Default | Description | +|----------------------------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ENABLE_DOWNLOAD_CACHE | false | Whether Prowlarr should download and cache nzb/torrent files | +| DOWNLOAD_CACHE_MAX_SIZE_MB | 1000 | Maximum size of download cache on disk. The cleanup job runs with the housekeeping tasks every 24 hours so this is not a strict limit. In testing, 1GB of disk cache stored ~7k nzbs. | There is potential for download loops in arrs where the same release is re-downloaded repeatedly due to mismatches in the parsed release custom format score and custom format score after import. This problem gets exacerbated when you use tools like Newtarr/Houndarr/Huntarr/etc to automate searching. From 8cefdaac2d5d59d16187028c30b0ccca6da47eeb Mon Sep 17 00:00:00 2001 From: actuallyevan <56329378+actuallyevan@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:29:22 -0700 Subject: [PATCH 7/7] ci: add sequential fork release versioning to release workflows --- .github/actions/package/package.sh | 12 +++++--- .github/release.yml | 9 ++++++ .github/workflows/build.yml | 2 +- .github/workflows/full-release.yml | 41 +++++++++++++++++++++++++ .github/workflows/publish-binaries.yml | 32 +++++++++++++------ .github/workflows/push-docker-image.yml | 21 +++++++++---- 6 files changed, 96 insertions(+), 21 deletions(-) create mode 100644 .github/release.yml diff --git a/.github/actions/package/package.sh b/.github/actions/package/package.sh index 1c80dd2c766..a1cf82d49aa 100644 --- a/.github/actions/package/package.sh +++ b/.github/actions/package/package.sh @@ -63,7 +63,11 @@ do fi done -# Copy Inno Setup Windows installers if present -if compgen -G "distribution/windows/setup/output/Prowlarr.*.exe" > /dev/null; then - cp distribution/windows/setup/output/Prowlarr.*.exe _artifacts/ -fi +# Copy Inno Setup Windows installers if present and apply fork version +upstream_ver="${PROWLARRVERSION%%-*}" +for exe in distribution/windows/setup/output/Prowlarr.*.exe; do + if [ -f "$exe" ]; then + name="$(basename "$exe")" + cp "$exe" "_artifacts/${name/$upstream_ver/$PROWLARRVERSION}" + fi +done diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 00000000000..10ca39069fc --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,9 @@ +changelog: + exclude: + authors: + - Weblate + - ProwlarrBot + categories: + - title: Changes + labels: + - '*' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fae81dfaa27..97c7cef52fe 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,7 +49,7 @@ jobs: id: variables shell: bash run: | - TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0.0") + TAG=$(git describe --tags --abbrev=0 --exclude "*-*" 2>/dev/null || echo "v0.0.0.0") PROWLARR_VERSION="${TAG#v}" FRAMEWORK=$(git grep -h -m 1 "" src/NzbDrone.Core/ | sed -e 's/.*\(.*\)<\/TargetFrameworks>.*/\1/') echo "PROWLARR_VERSION=$PROWLARR_VERSION" >> "$GITHUB_ENV" diff --git a/.github/workflows/full-release.yml b/.github/workflows/full-release.yml index 67d784d41aa..236b88ed954 100644 --- a/.github/workflows/full-release.yml +++ b/.github/workflows/full-release.yml @@ -14,16 +14,57 @@ permissions: statuses: write jobs: + prepare-release: + name: Prepare Release + runs-on: ubuntu-latest + outputs: + fork_version: ${{ steps.version.outputs.fork_version }} + fork_tag: ${{ steps.version.outputs.fork_tag }} + steps: + - name: Check out + uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + fetch-depth: 0 + + - name: Compute Versions & Tag + id: version + shell: bash + run: | + git fetch --tags origin + + UPSTREAM_TAG=$(git describe --tags --abbrev=0 --exclude "*-*" 2>/dev/null || echo "v0.0.0.0") + UPSTREAM_VERSION="${UPSTREAM_TAG#v}" + + LATEST_REV=$(git tag -l "v${UPSTREAM_VERSION}-*" | sed "s/^v${UPSTREAM_VERSION}-//" | sort -n | tail -n 1) + NEXT_REV=$(( ${LATEST_REV:-0} + 1 )) + + FORK_VERSION="${UPSTREAM_VERSION}-${NEXT_REV}" + FORK_TAG="v${FORK_VERSION}" + + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + git tag "$FORK_TAG" + git push origin "$FORK_TAG" + + echo "upstream_version=$UPSTREAM_VERSION" >> "$GITHUB_OUTPUT" + echo "fork_version=$FORK_VERSION" >> "$GITHUB_OUTPUT" + echo "fork_tag=$FORK_TAG" >> "$GITHUB_OUTPUT" + publish-binaries: name: Publish Binaries + needs: prepare-release uses: ./.github/workflows/publish-binaries.yml with: branch: ${{ inputs.branch }} + fork_tag: ${{ needs.prepare-release.outputs.fork_tag }} secrets: inherit push-docker-image: name: Push Docker Image + needs: prepare-release uses: ./.github/workflows/push-docker-image.yml with: branch: ${{ inputs.branch }} + fork_version: ${{ needs.prepare-release.outputs.fork_version }} secrets: inherit diff --git a/.github/workflows/publish-binaries.yml b/.github/workflows/publish-binaries.yml index 4fc05941f47..ad486a95d3f 100644 --- a/.github/workflows/publish-binaries.yml +++ b/.github/workflows/publish-binaries.yml @@ -14,6 +14,9 @@ on: required: true type: string default: master + fork_tag: + type: string + required: false env: INNOVERSION: 6.7.2 @@ -34,12 +37,18 @@ jobs: id: get_version shell: bash run: | - TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0.0") - VERSION="${TAG#v}" - echo "TAG_NAME=$TAG" >> "$GITHUB_OUTPUT" - echo "VERSION=$VERSION" >> "$GITHUB_OUTPUT" - echo "MAJORVERSION=${VERSION%.*}" >> "$GITHUB_ENV" - echo "MINORVERSION=${VERSION##*.}" >> "$GITHUB_ENV" + TAG="${{ inputs.fork_tag }}" + if [ -z "$TAG" ]; then + TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0.0") + fi + FORK_VERSION="${TAG#v}" + UPSTREAM_VERSION="${FORK_VERSION%%-*}" + + echo "FORK_TAG=$TAG" >> "$GITHUB_OUTPUT" + echo "FORK_VERSION=$FORK_VERSION" >> "$GITHUB_OUTPUT" + echo "UPSTREAM_VERSION=$UPSTREAM_VERSION" >> "$GITHUB_OUTPUT" + echo "MAJORVERSION=${UPSTREAM_VERSION%.*}" >> "$GITHUB_ENV" + echo "MINORVERSION=${UPSTREAM_VERSION##*.}" >> "$GITHUB_ENV" - name: Extract Target Framework shell: bash @@ -59,7 +68,7 @@ jobs: shell: bash run: ./build.sh --backend --frontend --packages --installer -f "${{ env.FRAMEWORK }}" env: - PROWLARRVERSION: ${{ steps.get_version.outputs.VERSION }} + PROWLARRVERSION: ${{ steps.get_version.outputs.UPSTREAM_VERSION }} BRANCH: ${{ inputs.branch }} BUILD_SOURCEBRANCHNAME: ${{ inputs.branch }} @@ -73,13 +82,16 @@ jobs: env: FRAMEWORK: ${{ env.FRAMEWORK }} BRANCH: ${{ inputs.branch }} - PROWLARRVERSION: ${{ steps.get_version.outputs.VERSION }} + PROWLARRVERSION: ${{ steps.get_version.outputs.FORK_VERSION }} - name: Publish GitHub Release uses: softprops/action-gh-release@v2 with: - tag_name: ${{ steps.get_version.outputs.TAG_NAME }} - name: Prowlarr v${{ steps.get_version.outputs.VERSION }} + tag_name: ${{ steps.get_version.outputs.FORK_TAG }} + name: Prowlarr v${{ steps.get_version.outputs.FORK_VERSION }} + body: | + Based on upstream Prowlarr [v${{ steps.get_version.outputs.UPSTREAM_VERSION }}](https://github.com/Prowlarr/Prowlarr/releases/tag/v${{ steps.get_version.outputs.UPSTREAM_VERSION }}). + generate_release_notes: true draft: false prerelease: false files: | diff --git a/.github/workflows/push-docker-image.yml b/.github/workflows/push-docker-image.yml index 3f4c3788fa3..265ef5f26e4 100644 --- a/.github/workflows/push-docker-image.yml +++ b/.github/workflows/push-docker-image.yml @@ -14,6 +14,9 @@ on: required: true type: string default: master + fork_version: + type: string + required: false workflow_run: workflows: ["Build"] types: [completed] @@ -92,11 +95,17 @@ jobs: echo "BRANCH=$BRANCH" >> "$GITHUB_ENV" fi - TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0.0") - VERSION="${TAG#v}" + FORK_VERSION="${{ inputs.fork_version }}" + if [ -z "$FORK_VERSION" ]; then + TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0.0") + FORK_VERSION="${TAG#v}" + fi + UPSTREAM_VERSION="${FORK_VERSION%%-*}" + FRAMEWORK=$(git grep -h -m 1 "" src/NzbDrone.Core/ | sed -e 's/.*\(.*\)<\/TargetFrameworks>.*/\1/') - echo "VERSION=$VERSION" >> "$GITHUB_OUTPUT" + echo "FORK_VERSION=$FORK_VERSION" >> "$GITHUB_OUTPUT" + echo "UPSTREAM_VERSION=$UPSTREAM_VERSION" >> "$GITHUB_OUTPUT" echo "FRAMEWORK=$FRAMEWORK" >> "$GITHUB_ENV" - name: Set up Node.js (Volta) @@ -117,7 +126,7 @@ jobs: mkdir -p _artifacts/linux-musl-arm64/${{ env.FRAMEWORK }}/Prowlarr fi env: - PROWLARRVERSION: ${{ steps.meta_env.outputs.VERSION }} + PROWLARRVERSION: ${{ steps.meta_env.outputs.UPSTREAM_VERSION }} BRANCH: ${{ env.BRANCH }} BUILD_SOURCEBRANCHNAME: ${{ env.BRANCH }} @@ -142,7 +151,7 @@ jobs: images: ${{ env.IMAGE_NAME }} tags: | type=raw,value=pr-${{ steps.meta_env.outputs.PR_NUM }},enable=${{ env.IS_PR == 'true' }} - type=raw,value=${{ steps.meta_env.outputs.VERSION }},enable=${{ env.IS_PR != 'true' }} + type=raw,value=${{ steps.meta_env.outputs.FORK_VERSION }},enable=${{ env.IS_PR != 'true' }} type=raw,value=latest,enable=${{ env.IS_PR != 'true' && env.BRANCH == 'master' }} type=raw,value=${{ env.BRANCH }},enable=${{ env.IS_PR != 'true' }} @@ -157,7 +166,7 @@ jobs: labels: ${{ steps.meta.outputs.labels }} build-args: | BUILD_DATE=${{ env.BUILD_DATE }} - VERSION=${{ steps.meta_env.outputs.VERSION }} + VERSION=${{ steps.meta_env.outputs.UPSTREAM_VERSION }} PROWLARR_BRANCH=${{ env.BRANCH }} PACKAGE_AUTHOR=github.com/${{ github.repository }}