Add regular Upload cleanup (#1219)

Author: Swatinem

services/cleanup/regular.py

@@ -1,5 +1,9 @@
 import logging
 import random
+from datetime import datetime, timedelta, timezone
+
+from django.db.models.query import QuerySet
+from shared.django_apps.reports.models import ReportSession as Upload
 
 from services.cleanup.cleanup import run_cleanup
 from services.cleanup.utils import CleanupResult, CleanupSummary, cleanup_context
@@ -11,8 +15,7 @@ def run_regular_cleanup() -> CleanupSummary:
     log.info("Starting regular cleanup job")
     complete_summary = CleanupSummary(CleanupResult(0), summary={})
 
-    # Usage of these model was removed, and we should clean up all its data before dropping the table for good.
-    cleanups_to_run = []
+    cleanups_to_run = create_upload_cleanup_jobs()
 
     # as we expect this job to have frequent retries, and cleanup to take a long time,
     # lets shuffle the various cleanups so that each one of those makes a little progress.
@@ -27,9 +30,51 @@ def run_regular_cleanup() -> CleanupSummary:
             complete_summary.add(summary)
 
     # TODO:
-    # - cleanup old `ReportSession`s (aka `Upload`s)
     # - cleanup `Commit`s that are `deleted`
     # - figure out a way how we can first mark, and then fully delete `Branch`es
 
     log.info("Regular cleanup finished")
     return complete_summary
+
+
+UPLOAD_RETENTION_PERIOD = 150
+MONTH_SLOTS = 120
+
+
+def create_upload_cleanup_jobs() -> list[QuerySet]:
+    """
+    This returns a list of `Upload` querysets, each targetting a subset of to-delete data.
+
+    As the `Upload` table is one of our biggest tables, running an (almost)
+    unbounded `DELETE` query would certainly cause problems.
+
+    Fortunately though, the (production) table has an index on `created_at`,
+    so queries targetting a range on that field should be fairly quick, and we
+    can use that to devide up the deletion workload onto more manageable chunks.
+
+    We are targetting 30-day chunks, going back ~10 years.
+    As the main cleanup task above is using a `random.shuffle`, and the cleanup
+    task itself is being restarted/retried on timeouts, this will end up with
+    an even distribution of cleanup tasks running concurrently, deleting different
+    chunks of this table.
+    """
+    latest_timestamp = datetime.now(timezone.utc) - timedelta(
+        days=UPLOAD_RETENTION_PERIOD
+    )
+    timestamps = [latest_timestamp]
+    for _ in range(MONTH_SLOTS):
+        timestamps.append(timestamps[-1] - timedelta(days=30))
+    timestamps.reverse()
+
+    begin_timestamp = None
+    queries: list[QuerySet] = []
+    for timestamp in timestamps:
+        query = Upload.objects
+        if begin_timestamp:
+            query = query.filter(created_at__gte=begin_timestamp)
+        query = query.filter(created_at__lt=timestamp)
+        queries.append(query)
+
+        begin_timestamp = timestamp
+
+    return queries

services/cleanup/tests/snapshots/regular_cleanup__generates_sliced_upload_cleanups__upload.txt

@@ -0,0 +1,50 @@
+-- UploadError
+DELETE
+FROM "reports_uploaderror"
+WHERE "reports_uploaderror"."upload_id" IN
+    (SELECT U0."id"
+     FROM "reports_upload" U0
+     WHERE (U0."created_at" >= %s
+            AND U0."created_at" < %s));
+-- [2024-10-25 00:00:00+00:00, 2024-11-24 00:00:00+00:00]
+
+
+-- UploadFlagMembership
+DELETE
+FROM "reports_uploadflagmembership"
+WHERE "reports_uploadflagmembership"."upload_id" IN
+    (SELECT U0."id"
+     FROM "reports_upload" U0
+     WHERE (U0."created_at" >= %s
+            AND U0."created_at" < %s));
+-- [2024-10-25 00:00:00+00:00, 2024-11-24 00:00:00+00:00]
+
+
+-- UploadLevelTotals
+DELETE
+FROM "reports_uploadleveltotals"
+WHERE "reports_uploadleveltotals"."upload_id" IN
+    (SELECT U0."id"
+     FROM "reports_upload" U0
+     WHERE (U0."created_at" >= %s
+            AND U0."created_at" < %s));
+-- [2024-10-25 00:00:00+00:00, 2024-11-24 00:00:00+00:00]
+
+
+-- TestInstance
+DELETE
+FROM "reports_testinstance"
+WHERE "reports_testinstance"."upload_id" IN
+    (SELECT U0."id"
+     FROM "reports_upload" U0
+     WHERE (U0."created_at" >= %s
+            AND U0."created_at" < %s));
+-- [2024-10-25 00:00:00+00:00, 2024-11-24 00:00:00+00:00]
+
+
+-- ReportSession
+DELETE
+FROM "reports_upload"
+WHERE ("reports_upload"."created_at" >= %s
+       AND "reports_upload"."created_at" < %s);
+-- [2024-10-25 00:00:00+00:00, 2024-11-24 00:00:00+00:00]