Harden the vulnerability refresh pipeline against long-running scans

DOCKER.md

@@ -81,6 +81,8 @@ Common JSON paths:
 | `DockerUpdateGuard:Vulnerabilities:Provider` | `None`, `DockerScout`, or `Trivy` |
 | `DockerUpdateGuard:Vulnerabilities:TrivyBaseUrl` | Required when `Provider=Trivy`; base URL of the Trivy server used by the bundled Trivy CLI in client mode |
 | `DockerUpdateGuard:Vulnerabilities:TrivyExecutablePath` | Optional path to the Trivy executable; defaults to `trivy` (bundled in the image) |
+| `DockerUpdateGuard:Vulnerabilities:RequestTimeoutSeconds` | Timeout for vulnerability provider requests; defaults to `120` because first-time scans of large images are slow |
+| `DockerUpdateGuard:Vulnerabilities:MaxParallelScans` | Number of vulnerability scans running at the same time (`1` to `8`); defaults to `1` |
 | `Telemetry:OtlpEndpoint` | Optional OTLP collector endpoint |
 | `DockerUpdateGuard:DisplayVersion` | Version label shown in the UI; set by the image build argument |
 

README.md

@@ -132,7 +132,8 @@ versions. Override them to route the lookups through an internal mirror or proxy
 | `TrivyExecutablePath` | `trivy` | No | Path to the Trivy executable (resolved through `PATH` by default; bundled in the published container image) |
 | `DockerScoutLoginUrl` | `https://hub.docker.com/v2/users/login` | No | Docker Hub login endpoint used by the `DockerScout` provider |
 | `DockerScoutBaseUrl` | `https://api.scout.docker.com` | No | Base address of the Docker Scout API |
-| `RequestTimeoutSeconds` | `30` | No | Timeout for vulnerability provider requests |
+| `RequestTimeoutSeconds` | `120` | No | Timeout for vulnerability provider requests; first-time Trivy scans of large images regularly exceed shorter timeouts |
+| `MaxParallelScans` | `1` | No | Maximum number of vulnerability provider scans running at the same time (`1` to `8`) |
 
 ### `DockerUpdateGuard:Scanning`
 
@@ -232,7 +233,8 @@ If all three telemetry switches are `false`, telemetry is effectively disabled.
       "Enabled": true,
       "Provider": "Trivy",
       "TrivyBaseUrl": "http://trivy:4954",
-      "RequestTimeoutSeconds": 30
+      "RequestTimeoutSeconds": 120,
+      "MaxParallelScans": 1
     },
     "Scanning": {
       "DiscoveryIntervalMinutes": 5,

src/DockerUpdateGuard/Configuration/DockerUpdateGuardOptionsValidator.cs

@@ -148,6 +148,11 @@ private static void ValidateVulnerabilityOptions(VulnerabilityOptions options, L
             failures.Add($"'{DockerUpdateGuardOptions.SectionName}:Vulnerabilities:RequestTimeoutSeconds' must be between 1 and 300");
         }
 
+        ValidateRange(options.MaxParallelScans,
+                      1,
+                      8,
+                      $"{DockerUpdateGuardOptions.SectionName}:Vulnerabilities:MaxParallelScans",
+                      failures);
         ValidateAbsoluteHttpUri(options.DockerScoutLoginUrl,
                                 $"{DockerUpdateGuardOptions.SectionName}:Vulnerabilities:DockerScoutLoginUrl",
                                 failures);

src/DockerUpdateGuard/Configuration/VulnerabilityOptions.cs

@@ -40,7 +40,12 @@ public class VulnerabilityOptions
     /// <summary>
     /// Request timeout in seconds
     /// </summary>
-    public int RequestTimeoutSeconds { get; set; } = 30;
+    public int RequestTimeoutSeconds { get; set; } = 120;
+
+    /// <summary>
+    /// Maximum number of vulnerability provider scans that may run at the same time
+    /// </summary>
+    public int MaxParallelScans { get; set; } = 1;
 
     #endregion // Properties
 }

src/DockerUpdateGuard/Images/ImageHostLoggingExtensions.cs

@@ -112,6 +112,16 @@ public static partial void ScanCleanupCompleted(this ILogger logger,
                                                     int removedSnapshots,
                                                     int removedScanRuns);
 
+    /// <summary>
+    /// Log the number of abandoned running scan runs that were marked as failed
+    /// </summary>
+    /// <param name="logger">Logger</param>
+    /// <param name="repairedScanRuns">Repaired scan run count</param>
+    [LoggerMessage(EventId = 2021,
+                   Level = LogLevel.Warning,
+                   Message = "Marked {RepairedScanRuns} abandoned running scan runs as failed because they never completed")]
+    public static partial void StaleScanRunsRepaired(this ILogger logger, int repairedScanRuns);
+
     /// <summary>
     /// Log an observed image scan failure
     /// </summary>

src/DockerUpdateGuard/Images/ScanCleanupBackgroundService.cs

@@ -1,5 +1,6 @@
 using DockerUpdateGuard.Configuration;
 using DockerUpdateGuard.Data;
+using DockerUpdateGuard.Data.Entities;
 
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Options;
@@ -18,6 +19,16 @@ public class ScanCleanupBackgroundService : ScheduledBackgroundService
     /// </summary>
     private const int MaxRetainedScanRuns = 20;
 
+    /// <summary>
+    /// Shortest age a running scan run must reach before it is treated as abandoned
+    /// </summary>
+    private const int MinimumStaleRunThresholdHours = 24;
+
+    /// <summary>
+    /// Message stored on running scan runs that were repaired by the cleanup
+    /// </summary>
+    private const string StaleRunErrorMessage = "Marked as failed by cleanup: the run never completed (process restart or crash)";
+
     #endregion // Constants
 
     #region Fields
@@ -59,6 +70,73 @@ public ScanCleanupBackgroundService(ILogger<ScanCleanupBackgroundService> logger
 
     #endregion // Constructors
 
+    #region Static methods
+
+    /// <summary>
+    /// Resolve the age a running scan run of the supplied type must reach before it is treated as abandoned
+    /// </summary>
+    /// <param name="scanRunType">Scan run type</param>
+    /// <param name="scanningOptions">Scan scheduling options</param>
+    /// <returns>Stale threshold</returns>
+    private static TimeSpan GetStaleRunThreshold(ScanRunType scanRunType, ScanningOptions scanningOptions)
+    {
+        var intervalMinutes = scanRunType switch
+                              {
+                                  ScanRunType.ObservedImage => scanningOptions.OwnImageBaseScanIntervalMinutes,
+                                  ScanRunType.RuntimeContainer => scanningOptions.RuntimeImageUpdateScanIntervalMinutes,
+                                  ScanRunType.Vulnerability => scanningOptions.VulnerabilityRefreshIntervalMinutes,
+                                  _ => 0,
+                              };
+        var intervalThreshold = TimeSpan.FromMinutes(2d * intervalMinutes);
+        var minimumThreshold = TimeSpan.FromHours(MinimumStaleRunThresholdHours);
+
+        return intervalThreshold < minimumThreshold ? minimumThreshold : intervalThreshold;
+    }
+
+    #endregion // Static methods
+
+    #region Methods
+
+    /// <summary>
+    /// Mark running scan runs that never completed because of a process restart or crash as failed
+    /// </summary>
+    /// <param name="dbContext">Database context</param>
+    /// <param name="utcNow">Current point in time</param>
+    /// <param name="cancellationToken">Cancellation token</param>
+    /// <returns>Task</returns>
+    private async Task RepairStaleRunningScanRunsAsync(DockerUpdateGuardDbContext dbContext,
+                                                       DateTimeOffset utcNow,
+                                                       CancellationToken cancellationToken)
+    {
+        var runningScanRuns = await dbContext.ScanRuns.Where(entity => entity.Status == ScanRunStatus.Running)
+                                                      .ToListAsync(cancellationToken)
+                                                      .ConfigureAwait(false);
+
+        var scanningOptions = _optionsMonitor.CurrentValue.Scanning;
+
+        var staleScanRuns = runningScanRuns.Where(entity => entity.StartedAtUtc < utcNow - GetStaleRunThreshold(entity.Type, scanningOptions))
+                                           .ToList();
+
+        if (staleScanRuns.Count == 0)
+        {
+            return;
+        }
+
+        foreach (var staleScanRun in staleScanRuns)
+        {
+            staleScanRun.Status = ScanRunStatus.Failed;
+            staleScanRun.CompletedAtUtc = utcNow;
+            staleScanRun.ErrorMessage = StaleRunErrorMessage;
+        }
+
+        await dbContext.SaveChangesAsync(cancellationToken)
+                       .ConfigureAwait(false);
+
+        _logger.StaleScanRunsRepaired(staleScanRuns.Count);
+    }
+
+    #endregion // Methods
+
     #region ScheduledBackgroundService
 
     /// <inheritdoc/>
@@ -73,6 +151,21 @@ protected override bool ShouldExecuteImmediately()
         return false;
     }
 
+    /// <inheritdoc/>
+    protected override async Task ExecuteStartupAsync(CancellationToken stoppingToken)
+    {
+        var scope = _serviceScopeFactory.CreateAsyncScope();
+
+        await using (scope.ConfigureAwait(false))
+        {
+            var dbContext = scope.ServiceProvider.GetRequiredService<DockerUpdateGuardDbContext>();
+
+            await RepairStaleRunningScanRunsAsync(dbContext,
+                                                  DateTimeOffset.UtcNow,
+                                                  stoppingToken).ConfigureAwait(false);
+        }
+    }
+
     /// <inheritdoc/>
     protected override async Task ExecuteCoreAsync(CancellationToken stoppingToken)
     {
@@ -84,6 +177,10 @@ protected override async Task ExecuteCoreAsync(CancellationToken stoppingToken)
             var applicationTelemetry = scope.ServiceProvider.GetRequiredService<ApplicationTelemetry>();
             var cleanupStartedAtUtc = DateTimeOffset.UtcNow;
 
+            await RepairStaleRunningScanRunsAsync(dbContext,
+                                                  cleanupStartedAtUtc,
+                                                  stoppingToken).ConfigureAwait(false);
+
             var cutoff = cleanupStartedAtUtc.AddDays(-_optionsMonitor.CurrentValue.Scanning.RetainScanRunsDays);
 
             var completedUnreferencedScanRuns = dbContext.ScanRuns

src/DockerUpdateGuard/Images/ScheduledBackgroundService.cs

@@ -53,6 +53,42 @@ protected virtual bool ShouldExecuteImmediately()
     /// <returns>Task</returns>
     protected abstract Task ExecuteCoreAsync(CancellationToken stoppingToken);
 
+    /// <summary>
+    /// Execute a short startup step before the scheduled loop begins, independent of <see cref="ShouldExecuteImmediately"/>
+    /// </summary>
+    /// <param name="stoppingToken">Cancellation token</param>
+    /// <returns>Task</returns>
+    protected virtual Task ExecuteStartupAsync(CancellationToken stoppingToken)
+    {
+        return Task.CompletedTask;
+    }
+
+    /// <summary>
+    /// Execute the startup step with exception logging
+    /// </summary>
+    /// <param name="stoppingToken">Cancellation token</param>
+    /// <returns>Task</returns>
+    private async Task ExecuteStartupSafelyAsync(CancellationToken stoppingToken)
+    {
+        var backgroundServiceName = GetType().Name;
+        var stopwatch = Stopwatch.StartNew();
+
+        try
+        {
+            await ExecuteStartupAsync(stoppingToken).ConfigureAwait(false);
+        }
+        catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested)
+        {
+            // Expected during host shutdown, the startup step is neither completed nor failed
+        }
+        catch (Exception exception)
+        {
+            _logger.BackgroundServiceExecutionFailed(exception,
+                                                     backgroundServiceName,
+                                                     stopwatch.ElapsedMilliseconds);
+        }
+    }
+
     /// <summary>
     /// Execute the background operation with exception logging
     /// </summary>
@@ -97,6 +133,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken)
             _logger.BackgroundServiceStarted(backgroundServiceName, GetInterval().TotalMinutes);
         }
 
+        await ExecuteStartupSafelyAsync(stoppingToken).ConfigureAwait(false);
+
         if (ShouldExecuteImmediately())
         {
             await ExecuteSafelyAsync(stoppingToken).ConfigureAwait(false);

src/DockerUpdateGuard/Images/VulnerabilityEnrichmentService.cs

@@ -19,6 +19,15 @@ namespace DockerUpdateGuard.Images;
 /// </summary>
 public class VulnerabilityEnrichmentService : IVulnerabilityEnrichmentService
 {
+    #region Constants
+
+    /// <summary>
+    /// Highest supported number of provider scans running at the same time
+    /// </summary>
+    private const int MaxSupportedParallelScans = 8;
+
+    #endregion // Constants
+
     #region Fields
 
     /// <summary>
@@ -265,27 +274,39 @@ private async Task<int> UpsertVulnerabilityFindingsAsync(ScanRun scanRun,
     }
 
     /// <summary>
-    /// Enrich a single image version with the advisories reported by the vulnerability provider
+    /// Start the provider scan of a single image version without touching the database context
+    /// </summary>
+    /// <param name="image">Image version to scan</param>
+    /// <param name="vulnerabilityProvider">Vulnerability provider resolved for this refresh run</param>
+    /// <param name="cancellationToken">Cancellation token</param>
+    /// <returns>Reported advisories</returns>
+    private Task<ExternalOperationResult<IReadOnlyList<VulnerabilityAdvisoryData>>> ScanImageVulnerabilitiesAsync(ImageVersion image,
+                                                                                                                  IVulnerabilityProvider vulnerabilityProvider,
+                                                                                                                  CancellationToken cancellationToken)
+    {
+        var imageReference = _imageReferenceParser.Parse(_imageReferenceParser.Format(image));
+
+        return vulnerabilityProvider.GetVulnerabilitiesAsync(imageReference, cancellationToken);
+    }
+
+    /// <summary>
+    /// Apply the reported advisories of a single image version to the tracked entities
     /// </summary>
     /// <param name="scanRun">Owning scan run</param>
     /// <param name="image">Image version to enrich</param>
     /// <param name="currentStatus">Scan status before the image was processed</param>
-    /// <param name="vulnerabilityProvider">Vulnerability provider resolved for this refresh run</param>
+    /// <param name="advisoryResult">Result reported by the vulnerability provider for this image version</param>
     /// <param name="cancellationToken">Cancellation token</param>
     /// <returns>Resulting scan status and failure message when the image could not be enriched</returns>
-    private async Task<(ScanRunStatus Status, string? Message)> ProcessImageVulnerabilitiesAsync(ScanRun scanRun,
-                                                                                                 ImageVersion image,
-                                                                                                 ScanRunStatus currentStatus,
-                                                                                                 IVulnerabilityProvider vulnerabilityProvider,
-                                                                                                 CancellationToken cancellationToken)
+    private async Task<(ScanRunStatus Status, string? Message)> ApplyImageVulnerabilitiesAsync(ScanRun scanRun,
+                                                                                               ImageVersion image,
+                                                                                               ScanRunStatus currentStatus,
+                                                                                               ExternalOperationResult<IReadOnlyList<VulnerabilityAdvisoryData>> advisoryResult,
+                                                                                               CancellationToken cancellationToken)
     {
-        var formattedImageReference = _imageReferenceParser.Format(image);
-        var imageReference = _imageReferenceParser.Parse(formattedImageReference);
-        var advisoryResult = await vulnerabilityProvider.GetVulnerabilitiesAsync(imageReference, cancellationToken)
-                                                        .ConfigureAwait(false);
-
         if (advisoryResult.Status != ExternalOperationStatus.Succeeded || advisoryResult.Data is null)
         {
+            var formattedImageReference = _imageReferenceParser.Format(image);
             var failureMessage = advisoryResult.Message ?? $"Unable to enrich vulnerabilities for '{image.RegistryRepository.Repository}:{image.Tag}'";
             var failureStatus = advisoryResult.Status == ExternalOperationStatus.Failed
                                 || currentStatus == ScanRunStatus.Failed
@@ -415,23 +436,41 @@ await _dbContext.SaveChangesAsync(cancellationToken)
                 _logger.VulnerabilityRefreshNoImages(triggerSource);
             }
 
-            foreach (var image in images)
+            var maxParallelScans = Math.Clamp(_optionsMonitor.CurrentValue.Vulnerabilities.MaxParallelScans,
+                                              1,
+                                              MaxSupportedParallelScans);
+
+            // Only the provider calls of a batch run in parallel, every database interaction stays on this flow
+            for (var batchStart = 0; batchStart < images.Count; batchStart += maxParallelScans)
             {
-                var imageOutcome = await ProcessImageVulnerabilitiesAsync(scanRun,
-                                                                          image,
-                                                                          finalStatus,
-                                                                          vulnerabilityProvider,
-                                                                          cancellationToken).ConfigureAwait(false);
+                var batch = images.Skip(batchStart)
+                                  .Take(maxParallelScans)
+                                  .ToList();
+
+                var scanTasks = batch.Select(image => ScanImageVulnerabilitiesAsync(image, vulnerabilityProvider, cancellationToken))
+                                     .ToList();
 
-                finalStatus = imageOutcome.Status;
+                var advisoryResults = await Task.WhenAll(scanTasks)
+                                                .ConfigureAwait(false);
 
-                if (imageOutcome.Message is not null)
+                for (var batchIndex = 0; batchIndex < batch.Count; batchIndex++)
                 {
-                    statusMessages.Add(imageOutcome.Message);
-                }
+                    var imageOutcome = await ApplyImageVulnerabilitiesAsync(scanRun,
+                                                                            batch[batchIndex],
+                                                                            finalStatus,
+                                                                            advisoryResults[batchIndex],
+                                                                            cancellationToken).ConfigureAwait(false);
 
-                await _dbContext.SaveChangesAsync(cancellationToken)
-                                .ConfigureAwait(false);
+                    finalStatus = imageOutcome.Status;
+
+                    if (imageOutcome.Message is not null)
+                    {
+                        statusMessages.Add(imageOutcome.Message);
+                    }
+
+                    await _dbContext.SaveChangesAsync(cancellationToken)
+                                    .ConfigureAwait(false);
+                }
             }
 
             var deactivatedFindingCount = await DeactivateStaleFindingsAsync(scanRun,

src/DockerUpdateGuard/appsettings.json

@@ -29,7 +29,8 @@
       "TrivyExecutablePath": "trivy",
       "DockerScoutLoginUrl": "https://hub.docker.com/v2/users/login",
       "DockerScoutBaseUrl": "https://api.scout.docker.com",
-      "RequestTimeoutSeconds": 30
+      "RequestTimeoutSeconds": 120,
+      "MaxParallelScans": 1
     },
     "Scanning": {
       "DiscoveryIntervalMinutes": 15,