From 58b7f760413efedafd805280d6c6cfaeb8877330 Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh Date: Thu, 30 Jan 2025 18:42:12 -0500 Subject: [PATCH 1/2] Add script for retrying all missed job webhooks --- .../commands/retry_failed_job_webhooks.py | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 analytics/analytics/core/management/commands/retry_failed_job_webhooks.py diff --git a/analytics/analytics/core/management/commands/retry_failed_job_webhooks.py b/analytics/analytics/core/management/commands/retry_failed_job_webhooks.py new file mode 100644 index 000000000..e6edd238e --- /dev/null +++ b/analytics/analytics/core/management/commands/retry_failed_job_webhooks.py @@ -0,0 +1,146 @@ +from concurrent.futures import Future, ThreadPoolExecutor, wait +from dataclasses import dataclass +from datetime import timedelta +import itertools +import re + +from django.db import connections +import djclick as click + +from analytics.core.models import JobFact +from analytics.job_processor.utils import get_gitlab_handle + +# The URL of the webhook handler service specified in the GitLab project settings. +# This is the URL in the web_hook_logs table in the GitLab DB. +WEBHOOK_URL = "http://webhook-handler.custom.svc.cluster.local" + + +@dataclass +class WebhookEvent: + created_at: str + build_id: int + project_id: int + webhook_id: int + webhook_event_id: int + + def __str__(self) -> str: + return f"[{self.created_at}] build_id: {self.build_id}, project_id: {self.project_id}, webhook_id: {self.webhook_id}, webhook_event_id: {self.webhook_event_id}" + + +def retry_webhook(webhook_event: WebhookEvent, dry_run: bool) -> None: + if dry_run: + click.echo(f"Would retry webhook {webhook_event}") + return + + click.echo(f"Retrying webhook {webhook_event}") + gl = get_gitlab_handle() + + # https://docs.gitlab.com/ee/api/project_webhooks.html#resend-a-project-webhook-event + retry_url = f"/projects/{webhook_event.project_id}/hooks/{webhook_event.webhook_id}/events/{webhook_event.webhook_event_id}/resend" + gl.http_post(retry_url) + + +@click.command() +@click.option( + "--seconds", + type=int, + default=timedelta(days=1).total_seconds(), + help="Retry webhooks that failed in the last N seconds", +) +@click.option( + "--dry-run", + is_flag=True, + default=False, + help="Print the webhooks that would be retried without actually retrying them", +) +def retry_failed_job_webhooks(seconds: int, dry_run: bool) -> None: + with connections["gitlab"].cursor() as cursor: + cursor.execute("BEGIN;") + + cursor.execute( + """ + DECLARE + webhook_cursor + CURSOR FOR + SELECT + created_at, + request_data, + web_hook_id, + id + FROM + public.web_hook_logs + WHERE + url = %s AND + created_at > NOW() - INTERVAL %s; + """, + [WEBHOOK_URL, f"{seconds} seconds"], + ) + + futures: list[Future] = [] + + with ThreadPoolExecutor() as executor: + while True: + # Fetch a batch of rows from the cursor + cursor.execute("FETCH FORWARD %s FROM webhook_cursor", [5000]) + rows = cursor.fetchall() + if not rows: + break + + webhook_events = [ + WebhookEvent( + created_at=row[0], + build_id=int(re.search(r"build_id: (\d+)", row[1]).group(1)), + project_id=int(re.search(r"project_id: (\d+)", row[1]).group(1)), + webhook_id=row[2], + webhook_event_id=row[3], + ) + for row in rows + ] + + # We only want to retry webhooks for builds that have finished (i.e. + # status is 'success' or 'failed'). Skipped or cancelled builds are + # not stored in the analytics DB. + cursor.execute( + """ + SELECT + id + FROM + ci_builds + WHERE + id IN %s AND + status IN ('success', 'failed'); + """, + [tuple(event.build_id for event in webhook_events)], + ) + finished_jobs: set[int] = set(itertools.chain.from_iterable(cursor.fetchall())) + + # Build a mapping of build ID to webhook event object for fast lookup by build ID + build_id_to_webhook_mapping: dict[int, WebhookEvent] = { + event.build_id: event + for event in webhook_events + if event.build_id in finished_jobs + } + + # Collect all build IDs + build_ids: set[int] = set(build_id_to_webhook_mapping.keys()) + + # Filter out build IDs that already have a corresponding analytics DB record + existing_build_ids: set[int] = set( + JobFact.objects.filter(job_id__in=build_ids).values_list("job_id", flat=True) + ) + + # Calculate the missing build IDs + missing_build_ids: set[int] = build_ids - existing_build_ids + + # Retry the webhooks for the missing build IDs + for build_id in missing_build_ids: + futures.append( + executor.submit( + retry_webhook, build_id_to_webhook_mapping[build_id], dry_run + ) + ) + + cursor.execute("CLOSE webhook_cursor;") + cursor.execute("COMMIT;") + + wait(futures) From a9f4488e24f512b00373c0ff74e29b2cb108ef59 Mon Sep 17 00:00:00 2001 From: Mike VanDenburgh Date: Thu, 29 May 2025 12:14:21 -0400 Subject: [PATCH 2/2] Bump image version --- .github/workflows/custom_docker_builds.yml | 2 +- k8s/production/custom/webhook-handler/deployments.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/custom_docker_builds.yml b/.github/workflows/custom_docker_builds.yml index d587320ad..4e7be287e 100644 --- a/.github/workflows/custom_docker_builds.yml +++ b/.github/workflows/custom_docker_builds.yml @@ -45,7 +45,7 @@ jobs: - docker-image: ./images/cache-indexer image-tags: ghcr.io/spack/cache-indexer:0.0.6 - docker-image: ./analytics - image-tags: ghcr.io/spack/django:0.5.1 + image-tags: ghcr.io/spack/django:0.5.2 - docker-image: ./images/ci-prune-buildcache image-tags: ghcr.io/spack/ci-prune-buildcache:0.0.4 - docker-image: ./images/protected-publish diff --git a/k8s/production/custom/webhook-handler/deployments.yaml b/k8s/production/custom/webhook-handler/deployments.yaml index 21924093b..7a0f7d2bd 100644 --- a/k8s/production/custom/webhook-handler/deployments.yaml +++ b/k8s/production/custom/webhook-handler/deployments.yaml @@ -23,7 +23,7 @@ spec: serviceAccountName: webhook-handler containers: - name: webhook-handler - image: ghcr.io/spack/django:0.5.1 + image: ghcr.io/spack/django:0.5.2 imagePullPolicy: Always resources: requests: @@ -146,7 +146,7 @@ spec: serviceAccountName: webhook-handler containers: - name: webhook-handler-worker - image: ghcr.io/spack/django:0.5.1 + image: ghcr.io/spack/django:0.5.2 command: [ "celery",