Add SQLite-backed cache for downloads and queries

.github/actions/package/package.sh

@@ -63,7 +63,11 @@ do
   fi
 done
 
-# Copy Inno Setup Windows installers if present
-if compgen -G "distribution/windows/setup/output/Prowlarr.*.exe" > /dev/null; then
-  cp distribution/windows/setup/output/Prowlarr.*.exe _artifacts/
-fi
+# Copy Inno Setup Windows installers if present and apply fork version
+upstream_ver="${PROWLARRVERSION%%-*}"
+for exe in distribution/windows/setup/output/Prowlarr.*.exe; do
+  if [ -f "$exe" ]; then
+    name="$(basename "$exe")"
+    cp "$exe" "_artifacts/${name/$upstream_ver/$PROWLARRVERSION}"
+  fi
+done

.github/release.yml

@@ -0,0 +1,9 @@
+changelog:
+  exclude:
+    authors:
+      - Weblate
+      - ProwlarrBot
+  categories:
+    - title: Changes
+      labels:
+        - '*'

.github/workflows/build.yml

@@ -49,7 +49,7 @@ jobs:
         id: variables
         shell: bash
         run: |
-          TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0.0")
+          TAG=$(git describe --tags --abbrev=0 --exclude "*-*" 2>/dev/null || echo "v0.0.0.0")
           PROWLARR_VERSION="${TAG#v}"
           FRAMEWORK=$(git grep -h -m 1 "<TargetFrameworks>" src/NzbDrone.Core/ | sed -e 's/.*<TargetFrameworks>\(.*\)<\/TargetFrameworks>.*/\1/')
           echo "PROWLARR_VERSION=$PROWLARR_VERSION" >> "$GITHUB_ENV"

.github/workflows/full-release.yml

@@ -14,16 +14,57 @@ permissions:
   statuses: write
 
 jobs:
+  prepare-release:
+    name: Prepare Release
+    runs-on: ubuntu-latest
+    outputs:
+      fork_version: ${{ steps.version.outputs.fork_version }}
+      fork_tag: ${{ steps.version.outputs.fork_tag }}
+    steps:
+      - name: Check out
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch }}
+          fetch-depth: 0
+
+      - name: Compute Versions & Tag
+        id: version
+        shell: bash
+        run: |
+          git fetch --tags origin
+
+          UPSTREAM_TAG=$(git describe --tags --abbrev=0 --exclude "*-*" 2>/dev/null || echo "v0.0.0.0")
+          UPSTREAM_VERSION="${UPSTREAM_TAG#v}"
+
+          LATEST_REV=$(git tag -l "v${UPSTREAM_VERSION}-*" | sed "s/^v${UPSTREAM_VERSION}-//" | sort -n | tail -n 1)
+          NEXT_REV=$(( ${LATEST_REV:-0} + 1 ))
+
+          FORK_VERSION="${UPSTREAM_VERSION}-${NEXT_REV}"
+          FORK_TAG="v${FORK_VERSION}"
+
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git tag "$FORK_TAG"
+          git push origin "$FORK_TAG"
+
+          echo "upstream_version=$UPSTREAM_VERSION" >> "$GITHUB_OUTPUT"
+          echo "fork_version=$FORK_VERSION" >> "$GITHUB_OUTPUT"
+          echo "fork_tag=$FORK_TAG" >> "$GITHUB_OUTPUT"
+
   publish-binaries:
     name: Publish Binaries
+    needs: prepare-release
     uses: ./.github/workflows/publish-binaries.yml
     with:
       branch: ${{ inputs.branch }}
+      fork_tag: ${{ needs.prepare-release.outputs.fork_tag }}
     secrets: inherit
 
   push-docker-image:
     name: Push Docker Image
+    needs: prepare-release
     uses: ./.github/workflows/push-docker-image.yml
     with:
       branch: ${{ inputs.branch }}
+      fork_version: ${{ needs.prepare-release.outputs.fork_version }}
     secrets: inherit

.github/workflows/publish-binaries.yml

@@ -14,6 +14,9 @@ on:
         required: true
         type: string
         default: master
+      fork_tag:
+        type: string
+        required: false
 
 env:
   INNOVERSION: 6.7.2
@@ -34,12 +37,18 @@ jobs:
         id: get_version
         shell: bash
         run: |
-          TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0.0")
-          VERSION="${TAG#v}"
-          echo "TAG_NAME=$TAG" >> "$GITHUB_OUTPUT"
-          echo "VERSION=$VERSION" >> "$GITHUB_OUTPUT"
-          echo "MAJORVERSION=${VERSION%.*}" >> "$GITHUB_ENV"
-          echo "MINORVERSION=${VERSION##*.}" >> "$GITHUB_ENV"
+          TAG="${{ inputs.fork_tag }}"
+          if [ -z "$TAG" ]; then
+            TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0.0")
+          fi
+          FORK_VERSION="${TAG#v}"
+          UPSTREAM_VERSION="${FORK_VERSION%%-*}"
+
+          echo "FORK_TAG=$TAG" >> "$GITHUB_OUTPUT"
+          echo "FORK_VERSION=$FORK_VERSION" >> "$GITHUB_OUTPUT"
+          echo "UPSTREAM_VERSION=$UPSTREAM_VERSION" >> "$GITHUB_OUTPUT"
+          echo "MAJORVERSION=${UPSTREAM_VERSION%.*}" >> "$GITHUB_ENV"
+          echo "MINORVERSION=${UPSTREAM_VERSION##*.}" >> "$GITHUB_ENV"
 
       - name: Extract Target Framework
         shell: bash
@@ -59,7 +68,7 @@ jobs:
         shell: bash
         run: ./build.sh --backend --frontend --packages --installer -f "${{ env.FRAMEWORK }}"
         env:
-          PROWLARRVERSION: ${{ steps.get_version.outputs.VERSION }}
+          PROWLARRVERSION: ${{ steps.get_version.outputs.UPSTREAM_VERSION }}
           BRANCH: ${{ inputs.branch }}
           BUILD_SOURCEBRANCHNAME: ${{ inputs.branch }}
 
@@ -73,13 +82,16 @@ jobs:
         env:
           FRAMEWORK: ${{ env.FRAMEWORK }}
           BRANCH: ${{ inputs.branch }}
-          PROWLARRVERSION: ${{ steps.get_version.outputs.VERSION }}
+          PROWLARRVERSION: ${{ steps.get_version.outputs.FORK_VERSION }}
 
       - name: Publish GitHub Release
         uses: softprops/action-gh-release@v2
         with:
-          tag_name: ${{ steps.get_version.outputs.TAG_NAME }}
-          name: Prowlarr v${{ steps.get_version.outputs.VERSION }}
+          tag_name: ${{ steps.get_version.outputs.FORK_TAG }}
+          name: Prowlarr v${{ steps.get_version.outputs.FORK_VERSION }}
+          body: |
+            Based on upstream Prowlarr [v${{ steps.get_version.outputs.UPSTREAM_VERSION }}](https://github.com/Prowlarr/Prowlarr/releases/tag/v${{ steps.get_version.outputs.UPSTREAM_VERSION }}).
+          generate_release_notes: true
           draft: false
           prerelease: false
           files: |

.github/workflows/push-docker-image.yml

@@ -14,6 +14,9 @@ on:
         required: true
         type: string
         default: master
+      fork_version:
+        type: string
+        required: false
   workflow_run:
     workflows: ["Build"]
     types: [completed]
@@ -92,11 +95,17 @@ jobs:
             echo "BRANCH=$BRANCH" >> "$GITHUB_ENV"
           fi
 
-          TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0.0")
-          VERSION="${TAG#v}"
+          FORK_VERSION="${{ inputs.fork_version }}"
+          if [ -z "$FORK_VERSION" ]; then
+            TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0.0")
+            FORK_VERSION="${TAG#v}"
+          fi
+          UPSTREAM_VERSION="${FORK_VERSION%%-*}"
+
           FRAMEWORK=$(git grep -h -m 1 "<TargetFrameworks>" src/NzbDrone.Core/ | sed -e 's/.*<TargetFrameworks>\(.*\)<\/TargetFrameworks>.*/\1/')
 
-          echo "VERSION=$VERSION" >> "$GITHUB_OUTPUT"
+          echo "FORK_VERSION=$FORK_VERSION" >> "$GITHUB_OUTPUT"
+          echo "UPSTREAM_VERSION=$UPSTREAM_VERSION" >> "$GITHUB_OUTPUT"
           echo "FRAMEWORK=$FRAMEWORK" >> "$GITHUB_ENV"
 
       - name: Set up Node.js (Volta)
@@ -112,7 +121,7 @@ jobs:
           ./build.sh --backend --frontend --packages -f "${{ env.FRAMEWORK }}" -r linux-musl-x64
           ./build.sh --backend --frontend --packages -f "${{ env.FRAMEWORK }}" -r linux-musl-arm64
         env:
-          PROWLARRVERSION: ${{ steps.meta_env.outputs.VERSION }}
+          PROWLARRVERSION: ${{ steps.meta_env.outputs.UPSTREAM_VERSION }}
           BRANCH: ${{ env.BRANCH }}
           BUILD_SOURCEBRANCHNAME: ${{ env.BRANCH }}
 
@@ -137,7 +146,7 @@ jobs:
           images: ${{ env.IMAGE_NAME }}
           tags: |
             type=raw,value=pr-${{ steps.meta_env.outputs.PR_NUM }},enable=${{ env.IS_PR == 'true' }}
-            type=raw,value=${{ steps.meta_env.outputs.VERSION }},enable=${{ env.IS_PR != 'true' }}
+            type=raw,value=${{ steps.meta_env.outputs.FORK_VERSION }},enable=${{ env.IS_PR != 'true' }}
             type=raw,value=latest,enable=${{ env.IS_PR != 'true' && env.BRANCH == 'master' }}
             type=raw,value=${{ env.BRANCH }},enable=${{ env.IS_PR != 'true' }}
 
@@ -152,7 +161,7 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           build-args: |
             BUILD_DATE=${{ env.BUILD_DATE }}
-            VERSION=${{ steps.meta_env.outputs.VERSION }}
+            VERSION=${{ steps.meta_env.outputs.UPSTREAM_VERSION }}
             PROWLARR_BRANCH=${{ env.BRANCH }}
             PACKAGE_AUTHOR=github.com/${{ github.repository }}
 

README.md

@@ -15,7 +15,6 @@ prowlarr:
       - TZ=Etc/UTC
       # Configure caching behavior
       - CACHE_TTL_MINS=10
-      - CACHE_MAX_SIZE_MB=100
       - ENABLE_DOWNLOAD_CACHE=true
       - DOWNLOAD_CACHE_MAX_SIZE_MB=1000
     volumes:
@@ -35,8 +34,6 @@ This fork aims to improve certain aspects of Prowlarr to make it work better wit
 | Env Var           | Default | Description                                                                                            |
 |-------------------|---------|--------------------------------------------------------------------------------------------------------|
 | CACHE_TTL_MINS    | 10      | How long a particular query response should be cached for. RSS queries are not cached.                 |
-| CACHE_MAX_SIZE_MB | 100     | Maximum size of cache in memory before old records are cleaned up. Higher values will use more memory. |
-
 
 Debrid/Usenet mounting tools cause a lot of repeated queries to the indexer that waste time and API queries. In particular, the workflow for most Usenet streaming setups is:
 - Arrs search for an item
@@ -53,24 +50,18 @@ Generally, if you're using any of the streaming clients, this fork will give you
 WITH enriched AS (
     SELECT
         IndexerId,
-        json_extract(Data, '$.season') AS season,
-        json_extract(Data, '$.query') AS query,
-        json_extract(Data, '$.categories') AS categories,
-        json_extract(Data, '$.queryType') AS queryType,
-        json_extract(Data, '$.tvdbId') AS tvdbId,
-        json_extract(Data, '$.tmdbId') AS tmdbId,
-        json_extract(Data, '$.imdbId') AS imdbId,
+        json_extract(Data, '$.url') AS url,
         CAST(strftime('%s', date) / 600 AS INTEGER) AS window_id
     FROM History
-    WHERE date >= datetime('now', '-90 days') AND (EventType = 2 OR EventType = 3)
+    WHERE date >= datetime('now', '-90 days') AND EventType = 2
     ),
     grouped AS (
 SELECT
     COUNT(*) AS total_calls,
     COUNT(*) - 1 AS duplicate_calls
 FROM enriched
 GROUP BY
-    IndexerId, season, query, categories, queryType, tvdbId, tmdbId, imdbId, window_id
+    IndexerId, url, window_id
     )
 SELECT
     SUM(total_calls) AS total_requests,
@@ -81,10 +72,10 @@ FROM grouped;
 
 ## Cache nzb/torrent files
 
-| Env Var                    | Default | Description                                                                                                                                                                                        |
-|----------------------------|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| ENABLE_DOWNLOAD_CACHE      | false   | Whether Prowlarr should download and cache nzb/torrent files                                                                                                                                       |
-| DOWNLOAD_CACHE_MAX_SIZE_MB | 1000    | Maximum size of download cache on disk. The cleanup job runs with the housekeeping tasks every 24 hours so this is not a strict limit. In testing, 1GB of disk cache stored ~6k nzbs. |
+| Env Var                    | Default | Description                                                                                                                                                                           |
+|----------------------------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| ENABLE_DOWNLOAD_CACHE      | false   | Whether Prowlarr should download and cache nzb/torrent files                                                                                                                          |
+| DOWNLOAD_CACHE_MAX_SIZE_MB | 1000    | Maximum size of download cache on disk. The cleanup job runs with the housekeeping tasks every 24 hours so this is not a strict limit. In testing, 1GB of disk cache stored ~7k nzbs. |
 
 There is potential for download loops in arrs where the same release is re-downloaded repeatedly due to mismatches in the parsed release custom format score and custom format score after import. This problem gets exacerbated when you use tools like Newtarr/Houndarr/Huntarr/etc to automate searching.
 

src/NzbDrone.Common/Extensions/PathExtensions.cs

@@ -15,6 +15,7 @@ public static class PathExtensions
         private const string DB = "prowlarr.db";
         private const string DB_RESTORE = "prowlarr.restore";
         private const string LOG_DB = "logs.db";
+        private const string CACHE_DB = "cache.db";
         private const string NLOG_CONFIG_FILE = "nlog.config";
         private const string UPDATE_CLIENT_EXE_NAME = "Prowlarr.Update";
 
@@ -364,6 +365,11 @@ public static string GetLogDatabase(this IAppFolderInfo appFolderInfo)
             return Path.Combine(GetAppDataPath(appFolderInfo), LOG_DB);
         }
 
+        public static string GetCacheDatabase(this IAppFolderInfo appFolderInfo)
+        {
+            return Path.Combine(GetAppDataPath(appFolderInfo), CACHE_DB);
+        }
+
         public static string GetNlogConfigPath(this IAppFolderInfo appFolderInfo)
         {
             return Path.Combine(appFolderInfo.StartUpFolder, NLOG_CONFIG_FILE);

src/NzbDrone.Core/Cache/BrotliCompressionHelper.cs

@@ -0,0 +1,40 @@
+using System;
+using System.IO;
+using System.IO.Compression;
+
+namespace NzbDrone.Core.Cache
+{
+    public static class BrotliCompressionHelper
+    {
+        public static byte[] Compress(byte[] data, CompressionLevel level = CompressionLevel.Fastest)
+        {
+            if (data == null || data.Length == 0)
+            {
+                return Array.Empty<byte>();
+            }
+
+            using var outputStream = new MemoryStream();
+            using (var brotliStream = new BrotliStream(outputStream, level))
+            {
+                brotliStream.Write(data, 0, data.Length);
+            }
+
+            return outputStream.ToArray();
+        }
+
+        public static byte[] Decompress(byte[] compressedData)
+        {
+            if (compressedData == null || compressedData.Length == 0)
+            {
+                return Array.Empty<byte>();
+            }
+
+            using var inputStream = new MemoryStream(compressedData);
+            using var brotliStream = new BrotliStream(inputStream, CompressionMode.Decompress);
+            using var outputStream = new MemoryStream();
+
+            brotliStream.CopyTo(outputStream);
+            return outputStream.ToArray();
+        }
+    }
+}

src/NzbDrone.Core/Cache/CacheKeyHasher.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Security.Cryptography;
+using System.Text;
+
+namespace NzbDrone.Core.Cache
+{
+    public static class CacheKeyHasher
+    {
+        public static string Hash(string key)
+        {
+            if (string.IsNullOrEmpty(key))
+            {
+                return string.Empty;
+            }
+
+            return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(key))).ToLowerInvariant();
+        }
+    }
+}