diff --git a/DOCKER.md b/DOCKER.md index ee2d120..b9a9fc6 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -81,6 +81,8 @@ Common JSON paths: | `DockerUpdateGuard:Vulnerabilities:Provider` | `None`, `DockerScout`, or `Trivy` | | `DockerUpdateGuard:Vulnerabilities:TrivyBaseUrl` | Required when `Provider=Trivy`; base URL of the Trivy server used by the bundled Trivy CLI in client mode | | `DockerUpdateGuard:Vulnerabilities:TrivyExecutablePath` | Optional path to the Trivy executable; defaults to `trivy` (bundled in the image) | +| `DockerUpdateGuard:Vulnerabilities:RequestTimeoutSeconds` | Timeout for vulnerability provider requests; defaults to `120` because first-time scans of large images are slow | +| `DockerUpdateGuard:Vulnerabilities:MaxParallelScans` | Number of vulnerability scans running at the same time (`1` to `8`); defaults to `1` | | `Telemetry:OtlpEndpoint` | Optional OTLP collector endpoint | | `DockerUpdateGuard:DisplayVersion` | Version label shown in the UI; set by the image build argument | diff --git a/README.md b/README.md index 3d4da83..80bd837 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,8 @@ versions. Override them to route the lookups through an internal mirror or proxy | `TrivyExecutablePath` | `trivy` | No | Path to the Trivy executable (resolved through `PATH` by default; bundled in the published container image) | | `DockerScoutLoginUrl` | `https://hub.docker.com/v2/users/login` | No | Docker Hub login endpoint used by the `DockerScout` provider | | `DockerScoutBaseUrl` | `https://api.scout.docker.com` | No | Base address of the Docker Scout API | -| `RequestTimeoutSeconds` | `30` | No | Timeout for vulnerability provider requests | +| `RequestTimeoutSeconds` | `120` | No | Timeout for vulnerability provider requests; first-time Trivy scans of large images regularly exceed shorter timeouts | +| `MaxParallelScans` | `1` | No | Maximum number of vulnerability provider scans running at the same time (`1` to `8`) | ### `DockerUpdateGuard:Scanning` @@ -232,7 +233,8 @@ If all three telemetry switches are `false`, telemetry is effectively disabled. "Enabled": true, "Provider": "Trivy", "TrivyBaseUrl": "http://trivy:4954", - "RequestTimeoutSeconds": 30 + "RequestTimeoutSeconds": 120, + "MaxParallelScans": 1 }, "Scanning": { "DiscoveryIntervalMinutes": 5, diff --git a/src/DockerUpdateGuard/Configuration/DockerUpdateGuardOptionsValidator.cs b/src/DockerUpdateGuard/Configuration/DockerUpdateGuardOptionsValidator.cs index 04eca07..c3c8e00 100644 --- a/src/DockerUpdateGuard/Configuration/DockerUpdateGuardOptionsValidator.cs +++ b/src/DockerUpdateGuard/Configuration/DockerUpdateGuardOptionsValidator.cs @@ -148,6 +148,11 @@ private static void ValidateVulnerabilityOptions(VulnerabilityOptions options, L failures.Add($"'{DockerUpdateGuardOptions.SectionName}:Vulnerabilities:RequestTimeoutSeconds' must be between 1 and 300"); } + ValidateRange(options.MaxParallelScans, + 1, + 8, + $"{DockerUpdateGuardOptions.SectionName}:Vulnerabilities:MaxParallelScans", + failures); ValidateAbsoluteHttpUri(options.DockerScoutLoginUrl, $"{DockerUpdateGuardOptions.SectionName}:Vulnerabilities:DockerScoutLoginUrl", failures); diff --git a/src/DockerUpdateGuard/Configuration/VulnerabilityOptions.cs b/src/DockerUpdateGuard/Configuration/VulnerabilityOptions.cs index f08e279..d35bc8d 100644 --- a/src/DockerUpdateGuard/Configuration/VulnerabilityOptions.cs +++ b/src/DockerUpdateGuard/Configuration/VulnerabilityOptions.cs @@ -40,7 +40,12 @@ public class VulnerabilityOptions /// /// Request timeout in seconds /// - public int RequestTimeoutSeconds { get; set; } = 30; + public int RequestTimeoutSeconds { get; set; } = 120; + + /// + /// Maximum number of vulnerability provider scans that may run at the same time + /// + public int MaxParallelScans { get; set; } = 1; #endregion // Properties } \ No newline at end of file diff --git a/src/DockerUpdateGuard/Images/ImageHostLoggingExtensions.cs b/src/DockerUpdateGuard/Images/ImageHostLoggingExtensions.cs index d2d30f1..a2caf18 100644 --- a/src/DockerUpdateGuard/Images/ImageHostLoggingExtensions.cs +++ b/src/DockerUpdateGuard/Images/ImageHostLoggingExtensions.cs @@ -112,6 +112,16 @@ public static partial void ScanCleanupCompleted(this ILogger logger, int removedSnapshots, int removedScanRuns); + /// + /// Log the number of abandoned running scan runs that were marked as failed + /// + /// Logger + /// Repaired scan run count + [LoggerMessage(EventId = 2021, + Level = LogLevel.Warning, + Message = "Marked {RepairedScanRuns} abandoned running scan runs as failed because they never completed")] + public static partial void StaleScanRunsRepaired(this ILogger logger, int repairedScanRuns); + /// /// Log an observed image scan failure /// diff --git a/src/DockerUpdateGuard/Images/ScanCleanupBackgroundService.cs b/src/DockerUpdateGuard/Images/ScanCleanupBackgroundService.cs index 7cf867e..077062f 100644 --- a/src/DockerUpdateGuard/Images/ScanCleanupBackgroundService.cs +++ b/src/DockerUpdateGuard/Images/ScanCleanupBackgroundService.cs @@ -1,5 +1,6 @@ using DockerUpdateGuard.Configuration; using DockerUpdateGuard.Data; +using DockerUpdateGuard.Data.Entities; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; @@ -18,6 +19,16 @@ public class ScanCleanupBackgroundService : ScheduledBackgroundService /// private const int MaxRetainedScanRuns = 20; + /// + /// Shortest age a running scan run must reach before it is treated as abandoned + /// + private const int MinimumStaleRunThresholdHours = 24; + + /// + /// Message stored on running scan runs that were repaired by the cleanup + /// + private const string StaleRunErrorMessage = "Marked as failed by cleanup: the run never completed (process restart or crash)"; + #endregion // Constants #region Fields @@ -59,6 +70,73 @@ public ScanCleanupBackgroundService(ILogger logger #endregion // Constructors + #region Static methods + + /// + /// Resolve the age a running scan run of the supplied type must reach before it is treated as abandoned + /// + /// Scan run type + /// Scan scheduling options + /// Stale threshold + private static TimeSpan GetStaleRunThreshold(ScanRunType scanRunType, ScanningOptions scanningOptions) + { + var intervalMinutes = scanRunType switch + { + ScanRunType.ObservedImage => scanningOptions.OwnImageBaseScanIntervalMinutes, + ScanRunType.RuntimeContainer => scanningOptions.RuntimeImageUpdateScanIntervalMinutes, + ScanRunType.Vulnerability => scanningOptions.VulnerabilityRefreshIntervalMinutes, + _ => 0, + }; + var intervalThreshold = TimeSpan.FromMinutes(2d * intervalMinutes); + var minimumThreshold = TimeSpan.FromHours(MinimumStaleRunThresholdHours); + + return intervalThreshold < minimumThreshold ? minimumThreshold : intervalThreshold; + } + + #endregion // Static methods + + #region Methods + + /// + /// Mark running scan runs that never completed because of a process restart or crash as failed + /// + /// Database context + /// Current point in time + /// Cancellation token + /// Task + private async Task RepairStaleRunningScanRunsAsync(DockerUpdateGuardDbContext dbContext, + DateTimeOffset utcNow, + CancellationToken cancellationToken) + { + var runningScanRuns = await dbContext.ScanRuns.Where(entity => entity.Status == ScanRunStatus.Running) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + var scanningOptions = _optionsMonitor.CurrentValue.Scanning; + + var staleScanRuns = runningScanRuns.Where(entity => entity.StartedAtUtc < utcNow - GetStaleRunThreshold(entity.Type, scanningOptions)) + .ToList(); + + if (staleScanRuns.Count == 0) + { + return; + } + + foreach (var staleScanRun in staleScanRuns) + { + staleScanRun.Status = ScanRunStatus.Failed; + staleScanRun.CompletedAtUtc = utcNow; + staleScanRun.ErrorMessage = StaleRunErrorMessage; + } + + await dbContext.SaveChangesAsync(cancellationToken) + .ConfigureAwait(false); + + _logger.StaleScanRunsRepaired(staleScanRuns.Count); + } + + #endregion // Methods + #region ScheduledBackgroundService /// @@ -73,6 +151,21 @@ protected override bool ShouldExecuteImmediately() return false; } + /// + protected override async Task ExecuteStartupAsync(CancellationToken stoppingToken) + { + var scope = _serviceScopeFactory.CreateAsyncScope(); + + await using (scope.ConfigureAwait(false)) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await RepairStaleRunningScanRunsAsync(dbContext, + DateTimeOffset.UtcNow, + stoppingToken).ConfigureAwait(false); + } + } + /// protected override async Task ExecuteCoreAsync(CancellationToken stoppingToken) { @@ -84,6 +177,10 @@ protected override async Task ExecuteCoreAsync(CancellationToken stoppingToken) var applicationTelemetry = scope.ServiceProvider.GetRequiredService(); var cleanupStartedAtUtc = DateTimeOffset.UtcNow; + await RepairStaleRunningScanRunsAsync(dbContext, + cleanupStartedAtUtc, + stoppingToken).ConfigureAwait(false); + var cutoff = cleanupStartedAtUtc.AddDays(-_optionsMonitor.CurrentValue.Scanning.RetainScanRunsDays); var completedUnreferencedScanRuns = dbContext.ScanRuns diff --git a/src/DockerUpdateGuard/Images/ScheduledBackgroundService.cs b/src/DockerUpdateGuard/Images/ScheduledBackgroundService.cs index 69147d0..f8138fa 100644 --- a/src/DockerUpdateGuard/Images/ScheduledBackgroundService.cs +++ b/src/DockerUpdateGuard/Images/ScheduledBackgroundService.cs @@ -53,6 +53,42 @@ protected virtual bool ShouldExecuteImmediately() /// Task protected abstract Task ExecuteCoreAsync(CancellationToken stoppingToken); + /// + /// Execute a short startup step before the scheduled loop begins, independent of + /// + /// Cancellation token + /// Task + protected virtual Task ExecuteStartupAsync(CancellationToken stoppingToken) + { + return Task.CompletedTask; + } + + /// + /// Execute the startup step with exception logging + /// + /// Cancellation token + /// Task + private async Task ExecuteStartupSafelyAsync(CancellationToken stoppingToken) + { + var backgroundServiceName = GetType().Name; + var stopwatch = Stopwatch.StartNew(); + + try + { + await ExecuteStartupAsync(stoppingToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + // Expected during host shutdown, the startup step is neither completed nor failed + } + catch (Exception exception) + { + _logger.BackgroundServiceExecutionFailed(exception, + backgroundServiceName, + stopwatch.ElapsedMilliseconds); + } + } + /// /// Execute the background operation with exception logging /// @@ -97,6 +133,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) _logger.BackgroundServiceStarted(backgroundServiceName, GetInterval().TotalMinutes); } + await ExecuteStartupSafelyAsync(stoppingToken).ConfigureAwait(false); + if (ShouldExecuteImmediately()) { await ExecuteSafelyAsync(stoppingToken).ConfigureAwait(false); diff --git a/src/DockerUpdateGuard/Images/VulnerabilityEnrichmentService.cs b/src/DockerUpdateGuard/Images/VulnerabilityEnrichmentService.cs index 56ae52b..1f52643 100644 --- a/src/DockerUpdateGuard/Images/VulnerabilityEnrichmentService.cs +++ b/src/DockerUpdateGuard/Images/VulnerabilityEnrichmentService.cs @@ -19,6 +19,15 @@ namespace DockerUpdateGuard.Images; /// public class VulnerabilityEnrichmentService : IVulnerabilityEnrichmentService { + #region Constants + + /// + /// Highest supported number of provider scans running at the same time + /// + private const int MaxSupportedParallelScans = 8; + + #endregion // Constants + #region Fields /// @@ -265,27 +274,39 @@ private async Task UpsertVulnerabilityFindingsAsync(ScanRun scanRun, } /// - /// Enrich a single image version with the advisories reported by the vulnerability provider + /// Start the provider scan of a single image version without touching the database context + /// + /// Image version to scan + /// Vulnerability provider resolved for this refresh run + /// Cancellation token + /// Reported advisories + private Task>> ScanImageVulnerabilitiesAsync(ImageVersion image, + IVulnerabilityProvider vulnerabilityProvider, + CancellationToken cancellationToken) + { + var imageReference = _imageReferenceParser.Parse(_imageReferenceParser.Format(image)); + + return vulnerabilityProvider.GetVulnerabilitiesAsync(imageReference, cancellationToken); + } + + /// + /// Apply the reported advisories of a single image version to the tracked entities /// /// Owning scan run /// Image version to enrich /// Scan status before the image was processed - /// Vulnerability provider resolved for this refresh run + /// Result reported by the vulnerability provider for this image version /// Cancellation token /// Resulting scan status and failure message when the image could not be enriched - private async Task<(ScanRunStatus Status, string? Message)> ProcessImageVulnerabilitiesAsync(ScanRun scanRun, - ImageVersion image, - ScanRunStatus currentStatus, - IVulnerabilityProvider vulnerabilityProvider, - CancellationToken cancellationToken) + private async Task<(ScanRunStatus Status, string? Message)> ApplyImageVulnerabilitiesAsync(ScanRun scanRun, + ImageVersion image, + ScanRunStatus currentStatus, + ExternalOperationResult> advisoryResult, + CancellationToken cancellationToken) { - var formattedImageReference = _imageReferenceParser.Format(image); - var imageReference = _imageReferenceParser.Parse(formattedImageReference); - var advisoryResult = await vulnerabilityProvider.GetVulnerabilitiesAsync(imageReference, cancellationToken) - .ConfigureAwait(false); - if (advisoryResult.Status != ExternalOperationStatus.Succeeded || advisoryResult.Data is null) { + var formattedImageReference = _imageReferenceParser.Format(image); var failureMessage = advisoryResult.Message ?? $"Unable to enrich vulnerabilities for '{image.RegistryRepository.Repository}:{image.Tag}'"; var failureStatus = advisoryResult.Status == ExternalOperationStatus.Failed || currentStatus == ScanRunStatus.Failed @@ -415,23 +436,41 @@ await _dbContext.SaveChangesAsync(cancellationToken) _logger.VulnerabilityRefreshNoImages(triggerSource); } - foreach (var image in images) + var maxParallelScans = Math.Clamp(_optionsMonitor.CurrentValue.Vulnerabilities.MaxParallelScans, + 1, + MaxSupportedParallelScans); + + // Only the provider calls of a batch run in parallel, every database interaction stays on this flow + for (var batchStart = 0; batchStart < images.Count; batchStart += maxParallelScans) { - var imageOutcome = await ProcessImageVulnerabilitiesAsync(scanRun, - image, - finalStatus, - vulnerabilityProvider, - cancellationToken).ConfigureAwait(false); + var batch = images.Skip(batchStart) + .Take(maxParallelScans) + .ToList(); + + var scanTasks = batch.Select(image => ScanImageVulnerabilitiesAsync(image, vulnerabilityProvider, cancellationToken)) + .ToList(); - finalStatus = imageOutcome.Status; + var advisoryResults = await Task.WhenAll(scanTasks) + .ConfigureAwait(false); - if (imageOutcome.Message is not null) + for (var batchIndex = 0; batchIndex < batch.Count; batchIndex++) { - statusMessages.Add(imageOutcome.Message); - } + var imageOutcome = await ApplyImageVulnerabilitiesAsync(scanRun, + batch[batchIndex], + finalStatus, + advisoryResults[batchIndex], + cancellationToken).ConfigureAwait(false); - await _dbContext.SaveChangesAsync(cancellationToken) - .ConfigureAwait(false); + finalStatus = imageOutcome.Status; + + if (imageOutcome.Message is not null) + { + statusMessages.Add(imageOutcome.Message); + } + + await _dbContext.SaveChangesAsync(cancellationToken) + .ConfigureAwait(false); + } } var deactivatedFindingCount = await DeactivateStaleFindingsAsync(scanRun, diff --git a/src/DockerUpdateGuard/appsettings.json b/src/DockerUpdateGuard/appsettings.json index 9bc3b05..16836ee 100644 --- a/src/DockerUpdateGuard/appsettings.json +++ b/src/DockerUpdateGuard/appsettings.json @@ -29,7 +29,8 @@ "TrivyExecutablePath": "trivy", "DockerScoutLoginUrl": "https://hub.docker.com/v2/users/login", "DockerScoutBaseUrl": "https://api.scout.docker.com", - "RequestTimeoutSeconds": 30 + "RequestTimeoutSeconds": 120, + "MaxParallelScans": 1 }, "Scanning": { "DiscoveryIntervalMinutes": 15, diff --git a/src/Tests/DockerUpdateGuard.Tests/DockerUpdateGuardOptionsValidatorTests.cs b/src/Tests/DockerUpdateGuard.Tests/DockerUpdateGuardOptionsValidatorTests.cs index 0fd1e1d..f958928 100644 --- a/src/Tests/DockerUpdateGuard.Tests/DockerUpdateGuardOptionsValidatorTests.cs +++ b/src/Tests/DockerUpdateGuard.Tests/DockerUpdateGuardOptionsValidatorTests.cs @@ -256,6 +256,7 @@ public void DockerUpdateGuardOptionsValidatorOutOfRangeUpperBoundsReturnsFailure options.DockerHub.RequestTimeoutSeconds = 301; options.DockerHub.MaxParallelRequests = 33; options.Vulnerabilities.RequestTimeoutSeconds = 301; + options.Vulnerabilities.MaxParallelScans = 9; options.Scanning.DiscoveryIntervalMinutes = 1441; options.Scanning.OwnImageBaseScanIntervalMinutes = 10081; options.Scanning.DockerHubRequestLimitWindowHours = 169; @@ -297,6 +298,9 @@ public void DockerUpdateGuardOptionsValidatorOutOfRangeUpperBoundsReturnsFailure Assert.Contains(message => message.Contains("Vulnerabilities:RequestTimeoutSeconds", StringComparison.Ordinal), failures, "A vulnerability timeout above the upper bound must be reported"); + Assert.Contains(message => message.Contains("Vulnerabilities:MaxParallelScans", StringComparison.Ordinal), + failures, + "A vulnerability scan parallelism above the upper bound must be reported"); Assert.Contains(message => message.Contains("Scanning:DiscoveryIntervalMinutes", StringComparison.Ordinal), failures, "A discovery interval above the upper bound must be reported"); diff --git a/src/Tests/DockerUpdateGuard.Tests/Helper/ConcurrencyTrackingVulnerabilityProvider.cs b/src/Tests/DockerUpdateGuard.Tests/Helper/ConcurrencyTrackingVulnerabilityProvider.cs new file mode 100644 index 0000000..7621434 --- /dev/null +++ b/src/Tests/DockerUpdateGuard.Tests/Helper/ConcurrencyTrackingVulnerabilityProvider.cs @@ -0,0 +1,137 @@ +using DockerUpdateGuard.Data.Entities; +using DockerUpdateGuard.Images.Data; +using DockerUpdateGuard.Infrastructure; +using DockerUpdateGuard.Vulnerabilities; +using DockerUpdateGuard.Vulnerabilities.Interfaces; + +namespace DockerUpdateGuard.Tests.Helper; + +/// +/// Vulnerability provider that records the peak number of concurrent scans +/// +internal sealed class ConcurrencyTrackingVulnerabilityProvider : IVulnerabilityProvider +{ + #region Fields + + /// + /// Artificial delay applied while a scan is in flight + /// + private readonly TimeSpan _delay; + + /// + /// Image references passed to the provider + /// + private readonly List _scannedReferences; + + /// + /// Number of scans currently in flight + /// + private int _currentConcurrency; + + /// + /// Peak number of scans observed in flight at the same time + /// + private int _maxConcurrency; + + #endregion // Fields + + #region Constructors + + /// + /// Constructor + /// + /// Artificial delay applied while a scan is in flight + public ConcurrencyTrackingVulnerabilityProvider(TimeSpan delay) + { + _delay = delay; + _scannedReferences = []; + } + + #endregion // Constructors + + #region Properties + + /// + /// Peak number of scans observed in flight at the same time + /// + public int MaxObservedConcurrency => Volatile.Read(ref _maxConcurrency); + + /// + /// Image references passed to the provider + /// + public IReadOnlyList ScannedReferences + { + get + { + lock (_scannedReferences) + { + return _scannedReferences.ToList(); + } + } + } + + #endregion // Properties + + #region Methods + + /// + /// Raise the observed peak concurrency to the supplied value when it is higher + /// + /// Candidate concurrency value + private void UpdateMaxConcurrency(int candidate) + { + var observedMax = Volatile.Read(ref _maxConcurrency); + + while (candidate > observedMax) + { + var previousMax = Interlocked.CompareExchange(ref _maxConcurrency, candidate, observedMax); + + if (previousMax == observedMax) + { + break; + } + + observedMax = previousMax; + } + } + + #endregion // Methods + + #region IVulnerabilityProvider + + /// + public async Task>> GetVulnerabilitiesAsync(ImageReference imageReference, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(imageReference); + + var current = Interlocked.Increment(ref _currentConcurrency); + + UpdateMaxConcurrency(current); + + lock (_scannedReferences) + { + _scannedReferences.Add(imageReference.Repository); + } + + try + { + await Task.Delay(_delay, cancellationToken).ConfigureAwait(false); + + return ExternalOperationResult>.Succeeded([ + new VulnerabilityAdvisoryData + { + AdvisoryId = $"CVE-{imageReference.Repository}", + Title = "Concurrency advisory", + Severity = VulnerabilitySeverity.High, + AffectedPackage = "openssl", + }, + ]); + } + finally + { + Interlocked.Decrement(ref _currentConcurrency); + } + } + + #endregion // IVulnerabilityProvider +} \ No newline at end of file diff --git a/src/Tests/DockerUpdateGuard.Tests/Helper/TestScanCleanupBackgroundService.cs b/src/Tests/DockerUpdateGuard.Tests/Helper/TestScanCleanupBackgroundService.cs index 2db7bc2..fa512f4 100644 --- a/src/Tests/DockerUpdateGuard.Tests/Helper/TestScanCleanupBackgroundService.cs +++ b/src/Tests/DockerUpdateGuard.Tests/Helper/TestScanCleanupBackgroundService.cs @@ -42,5 +42,15 @@ public Task ExecuteOnceAsync(CancellationToken cancellationToken) return ExecuteCoreAsync(cancellationToken); } + /// + /// Execute the startup step of the cleanup service + /// + /// Cancellation token + /// Task + public Task ExecuteStartupOnceAsync(CancellationToken cancellationToken) + { + return ExecuteStartupAsync(cancellationToken); + } + #endregion // Methods } \ No newline at end of file diff --git a/src/Tests/DockerUpdateGuard.Tests/Helper/TestScheduledBackgroundService.cs b/src/Tests/DockerUpdateGuard.Tests/Helper/TestScheduledBackgroundService.cs new file mode 100644 index 0000000..1b1c46c --- /dev/null +++ b/src/Tests/DockerUpdateGuard.Tests/Helper/TestScheduledBackgroundService.cs @@ -0,0 +1,138 @@ +using DockerUpdateGuard.Images; + +using Microsoft.Extensions.Logging; + +namespace DockerUpdateGuard.Tests.Helper; + +/// +/// Testable scheduled background service that records how the base class drives its steps +/// +internal sealed class TestScheduledBackgroundService : ScheduledBackgroundService +{ + #region Fields + + /// + /// Signal completed as soon as the scheduled execution ran once + /// + private readonly TaskCompletionSource _executionSignal; + + /// + /// Configured execution interval + /// + private readonly TimeSpan _interval; + + /// + /// Exception thrown by the startup step, or when the startup step succeeds + /// + private readonly Exception? _startupException; + + /// + /// Number of scheduled executions + /// + private int _executionCount; + + /// + /// Number of startup executions + /// + private int _startupExecutionCount; + + /// + /// Indicates whether the startup step ran before the first scheduled execution + /// + private bool _startupRanBeforeExecution; + + #endregion // Fields + + #region Constructors + + /// + /// Constructor + /// + /// Logger + /// Configured execution interval + /// Exception thrown by the startup step, or when the startup step succeeds + public TestScheduledBackgroundService(ILogger logger, TimeSpan interval, Exception? startupException) + : base(logger) + { + _executionSignal = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _interval = interval; + _startupException = startupException; + } + + #endregion // Constructors + + #region Properties + + /// + /// Signal completed as soon as the scheduled execution ran once + /// + public Task ExecutionSignal => _executionSignal.Task; + + /// + /// Number of scheduled executions + /// + public int ExecutionCount => Volatile.Read(ref _executionCount); + + /// + /// Number of startup executions + /// + public int StartupExecutionCount => Volatile.Read(ref _startupExecutionCount); + + /// + /// Indicates whether the startup step ran before the first scheduled execution + /// + public bool StartupRanBeforeExecution => Volatile.Read(ref _startupRanBeforeExecution); + + #endregion // Properties + + #region Methods + + /// + /// Run the scheduled loop of the base class until the supplied token is cancelled + /// + /// Cancellation token + /// Task + public Task RunAsync(CancellationToken stoppingToken) + { + return ExecuteAsync(stoppingToken); + } + + #endregion // Methods + + #region ScheduledBackgroundService + + /// + protected override TimeSpan GetInterval() + { + return _interval; + } + + /// + protected override async Task ExecuteStartupAsync(CancellationToken stoppingToken) + { + await base.ExecuteStartupAsync(stoppingToken).ConfigureAwait(false); + + stoppingToken.ThrowIfCancellationRequested(); + + Volatile.Write(ref _startupRanBeforeExecution, Volatile.Read(ref _executionCount) == 0); + Interlocked.Increment(ref _startupExecutionCount); + + if (_startupException is not null) + { + throw _startupException; + } + } + + /// + protected override Task ExecuteCoreAsync(CancellationToken stoppingToken) + { + stoppingToken.ThrowIfCancellationRequested(); + + Interlocked.Increment(ref _executionCount); + _executionSignal.TrySetResult(); + + return Task.CompletedTask; + } + + #endregion // ScheduledBackgroundService +} \ No newline at end of file diff --git a/src/Tests/DockerUpdateGuard.Tests/ScanCleanupBackgroundServiceTests.cs b/src/Tests/DockerUpdateGuard.Tests/ScanCleanupBackgroundServiceTests.cs index d58dc20..414ab16 100644 --- a/src/Tests/DockerUpdateGuard.Tests/ScanCleanupBackgroundServiceTests.cs +++ b/src/Tests/DockerUpdateGuard.Tests/ScanCleanupBackgroundServiceTests.cs @@ -197,5 +197,289 @@ await service.ExecuteOnceAsync(CancellationToken.None) } } + /// + /// Verify cleanup marks abandoned running scan runs as failed while fresh runs stay untouched + /// + /// Task + [TestMethod] + public async Task ScanCleanupBackgroundServiceExecuteCoreAsyncMarksStaleRunningScanRunsAsFailedAsync() + { + var databaseName = Guid.NewGuid().ToString(); + var serviceCollection = new ServiceCollection(); + + serviceCollection.AddScoped(_ => + { + var options = new DbContextOptionsBuilder().UseInMemoryDatabase(databaseName) + .Options; + + return new DockerUpdateGuardDbContext(options); + }); + serviceCollection.AddScoped(_ => new ApplicationTelemetry()); + + var serviceProvider = serviceCollection.BuildServiceProvider(); + + await using (serviceProvider.ConfigureAwait(false)) + { + var scope = serviceProvider.CreateAsyncScope(); + + await using (scope.ConfigureAwait(false)) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + var now = DateTimeOffset.UtcNow; + + dbContext.ScanRuns.Add(new ScanRun + { + Type = ScanRunType.Vulnerability, + Status = ScanRunStatus.Running, + TriggerSource = ScanTriggerSource.Scheduled, + StartedAtUtc = now.AddDays(-3), + CorrelationId = "stale-run", + }); + dbContext.ScanRuns.Add(new ScanRun + { + Type = ScanRunType.ObservedImage, + Status = ScanRunStatus.Running, + TriggerSource = ScanTriggerSource.Scheduled, + StartedAtUtc = now.AddDays(-3), + CorrelationId = "stale-observed-run", + }); + dbContext.ScanRuns.Add(new ScanRun + { + Type = ScanRunType.NotSet, + Status = ScanRunStatus.Running, + TriggerSource = ScanTriggerSource.Scheduled, + StartedAtUtc = now.AddDays(-3), + CorrelationId = "stale-untyped-run", + }); + dbContext.ScanRuns.Add(new ScanRun + { + Type = ScanRunType.Vulnerability, + Status = ScanRunStatus.Running, + TriggerSource = ScanTriggerSource.Scheduled, + StartedAtUtc = now.AddMinutes(-5), + CorrelationId = "fresh-run", + }); + + await dbContext.SaveChangesAsync(CancellationToken.None) + .ConfigureAwait(false); + } + + var options = new DockerUpdateGuardOptions + { + Scanning = new ScanningOptions + { + CleanupIntervalMinutes = 60, + RetainScanRunsDays = 30, + VulnerabilityRefreshIntervalMinutes = 180, + }, + }; + var logger = new TestLogger(); + + var service = new TestScanCleanupBackgroundService(logger, + new TestOptionsMonitor(options), + serviceProvider.GetRequiredService()); + + await service.ExecuteOnceAsync(CancellationToken.None) + .ConfigureAwait(false); + + var verificationScope = serviceProvider.CreateAsyncScope(); + + await using (verificationScope.ConfigureAwait(false)) + { + var dbContext = verificationScope.ServiceProvider.GetRequiredService(); + var staleScanRun = await dbContext.ScanRuns.SingleAsync(entity => entity.CorrelationId == "stale-run", CancellationToken.None) + .ConfigureAwait(false); + var staleObservedScanRun = await dbContext.ScanRuns.SingleAsync(entity => entity.CorrelationId == "stale-observed-run", CancellationToken.None) + .ConfigureAwait(false); + var staleUntypedScanRun = await dbContext.ScanRuns.SingleAsync(entity => entity.CorrelationId == "stale-untyped-run", CancellationToken.None) + .ConfigureAwait(false); + var freshScanRun = await dbContext.ScanRuns.SingleAsync(entity => entity.CorrelationId == "fresh-run", CancellationToken.None) + .ConfigureAwait(false); + + Assert.AreEqual(ScanRunStatus.Failed, + staleScanRun.Status, + "Cleanup must mark running scan runs that never completed as failed"); + Assert.AreEqual(ScanRunStatus.Failed, + staleObservedScanRun.Status, + "Cleanup must repair abandoned observed image scan runs as well"); + Assert.AreEqual(ScanRunStatus.Failed, + staleUntypedScanRun.Status, + "Cleanup must repair abandoned scan runs that carry no run type"); + Assert.IsNotNull(staleScanRun.CompletedAtUtc, "A repaired scan run must record a completion timestamp"); + Assert.AreEqual("Marked as failed by cleanup: the run never completed (process restart or crash)", + staleScanRun.ErrorMessage, + "A repaired scan run must explain why it was marked as failed"); + Assert.AreEqual(ScanRunStatus.Running, + freshScanRun.Status, + "Cleanup must not touch running scan runs that are still within their expected runtime"); + Assert.IsNull(freshScanRun.CompletedAtUtc, "Cleanup must not complete running scan runs that are still within their expected runtime"); + Assert.Contains(entry => entry.EventId.Id == 2021, logger.Entries, "Repairing abandoned scan runs must be logged"); + } + } + } + + /// + /// Verify a long refresh interval stretches the stale threshold beyond the minimum of 24 hours + /// + /// Task + [TestMethod] + public async Task ScanCleanupBackgroundServiceExecuteCoreAsyncLongIntervalStretchesStaleThresholdAsync() + { + var databaseName = Guid.NewGuid().ToString(); + var serviceCollection = new ServiceCollection(); + + serviceCollection.AddScoped(_ => + { + var options = new DbContextOptionsBuilder().UseInMemoryDatabase(databaseName) + .Options; + + return new DockerUpdateGuardDbContext(options); + }); + serviceCollection.AddScoped(_ => new ApplicationTelemetry()); + + var serviceProvider = serviceCollection.BuildServiceProvider(); + + await using (serviceProvider.ConfigureAwait(false)) + { + var scope = serviceProvider.CreateAsyncScope(); + + await using (scope.ConfigureAwait(false)) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + var now = DateTimeOffset.UtcNow; + + dbContext.ScanRuns.Add(new ScanRun + { + Type = ScanRunType.Vulnerability, + Status = ScanRunStatus.Running, + TriggerSource = ScanTriggerSource.Scheduled, + StartedAtUtc = now.AddHours(-40), + CorrelationId = "beyond-threshold-run", + }); + dbContext.ScanRuns.Add(new ScanRun + { + Type = ScanRunType.Vulnerability, + Status = ScanRunStatus.Running, + TriggerSource = ScanTriggerSource.Scheduled, + StartedAtUtc = now.AddHours(-26), + CorrelationId = "within-threshold-run", + }); + + await dbContext.SaveChangesAsync(CancellationToken.None) + .ConfigureAwait(false); + } + + // Twice the refresh interval of 15 hours is 30 hours and therefore longer than the minimum threshold + var options = new DockerUpdateGuardOptions + { + Scanning = new ScanningOptions + { + CleanupIntervalMinutes = 60, + RetainScanRunsDays = 30, + VulnerabilityRefreshIntervalMinutes = 900, + }, + }; + + var service = new TestScanCleanupBackgroundService(new TestLogger(), + new TestOptionsMonitor(options), + serviceProvider.GetRequiredService()); + + await service.ExecuteOnceAsync(CancellationToken.None) + .ConfigureAwait(false); + + var verificationScope = serviceProvider.CreateAsyncScope(); + + await using (verificationScope.ConfigureAwait(false)) + { + var dbContext = verificationScope.ServiceProvider.GetRequiredService(); + var beyondThresholdScanRun = await dbContext.ScanRuns.SingleAsync(entity => entity.CorrelationId == "beyond-threshold-run", CancellationToken.None) + .ConfigureAwait(false); + var withinThresholdScanRun = await dbContext.ScanRuns.SingleAsync(entity => entity.CorrelationId == "within-threshold-run", CancellationToken.None) + .ConfigureAwait(false); + + Assert.AreEqual(ScanRunStatus.Failed, + beyondThresholdScanRun.Status, + "A running scan run older than twice the configured refresh interval must be repaired"); + Assert.AreEqual(ScanRunStatus.Running, + withinThresholdScanRun.Status, + "A long refresh interval must keep running scan runs untouched beyond the minimum threshold of 24 hours"); + } + } + } + + /// + /// Verify the cleanup startup step repairs abandoned running scan runs before the first scheduled cycle + /// + /// Task + [TestMethod] + public async Task ScanCleanupBackgroundServiceExecuteStartupAsyncMarksStaleRunningScanRunsAsFailedAsync() + { + var databaseName = Guid.NewGuid().ToString(); + var serviceCollection = new ServiceCollection(); + + serviceCollection.AddScoped(_ => + { + var options = new DbContextOptionsBuilder().UseInMemoryDatabase(databaseName) + .Options; + + return new DockerUpdateGuardDbContext(options); + }); + serviceCollection.AddScoped(_ => new ApplicationTelemetry()); + + var serviceProvider = serviceCollection.BuildServiceProvider(); + + await using (serviceProvider.ConfigureAwait(false)) + { + var scope = serviceProvider.CreateAsyncScope(); + + await using (scope.ConfigureAwait(false)) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + + dbContext.ScanRuns.Add(new ScanRun + { + Type = ScanRunType.RuntimeContainer, + Status = ScanRunStatus.Running, + TriggerSource = ScanTriggerSource.Scheduled, + StartedAtUtc = DateTimeOffset.UtcNow.AddDays(-2), + CorrelationId = "crashed-run", + }); + + await dbContext.SaveChangesAsync(CancellationToken.None) + .ConfigureAwait(false); + } + + var options = new DockerUpdateGuardOptions + { + Scanning = new ScanningOptions + { + CleanupIntervalMinutes = 60, + RetainScanRunsDays = 30, + }, + }; + + var service = new TestScanCleanupBackgroundService(new TestLogger(), + new TestOptionsMonitor(options), + serviceProvider.GetRequiredService()); + + await service.ExecuteStartupOnceAsync(CancellationToken.None) + .ConfigureAwait(false); + + var verificationScope = serviceProvider.CreateAsyncScope(); + + await using (verificationScope.ConfigureAwait(false)) + { + var dbContext = verificationScope.ServiceProvider.GetRequiredService(); + var crashedScanRun = await dbContext.ScanRuns.SingleAsync(entity => entity.CorrelationId == "crashed-run", CancellationToken.None) + .ConfigureAwait(false); + + Assert.AreEqual(ScanRunStatus.Failed, + crashedScanRun.Status, + "The cleanup startup step must repair running scan runs left behind by a crashed process"); + Assert.IsNotNull(crashedScanRun.CompletedAtUtc, "A scan run repaired at startup must record a completion timestamp"); + } + } + } + #endregion // Methods } \ No newline at end of file diff --git a/src/Tests/DockerUpdateGuard.Tests/ScheduledBackgroundServiceTests.cs b/src/Tests/DockerUpdateGuard.Tests/ScheduledBackgroundServiceTests.cs new file mode 100644 index 0000000..a6ee8af --- /dev/null +++ b/src/Tests/DockerUpdateGuard.Tests/ScheduledBackgroundServiceTests.cs @@ -0,0 +1,104 @@ +using DockerUpdateGuard.Images; +using DockerUpdateGuard.Tests.Data; +using DockerUpdateGuard.Tests.Helper; + +namespace DockerUpdateGuard.Tests; + +/// +/// Tests for +/// +[TestClass] +public class ScheduledBackgroundServiceTests +{ + #region Methods + + /// + /// Verify the startup step runs once before the first scheduled execution + /// + /// Task + [TestMethod] + public async Task ScheduledBackgroundServiceExecuteAsyncRunsStartupStepBeforeFirstExecutionAsync() + { + var logger = new TestLogger(); + + using (var cancellationTokenSource = new CancellationTokenSource()) + { + var service = new TestScheduledBackgroundService(logger, + TimeSpan.FromMinutes(5), + null); + var runTask = service.RunAsync(cancellationTokenSource.Token); + + await service.ExecutionSignal.ConfigureAwait(false); + await cancellationTokenSource.CancelAsync().ConfigureAwait(false); + await runTask.ConfigureAwait(false); + + Assert.AreEqual(1, + service.StartupExecutionCount, + "The scheduled loop must run the startup step exactly once"); + Assert.IsTrue(service.StartupRanBeforeExecution, "The startup step must run before the first scheduled execution"); + Assert.IsGreaterThanOrEqualTo(1, + service.ExecutionCount, + "The scheduled loop must still execute the background operation after the startup step"); + Assert.DoesNotContain(entry => entry.EventId.Id == 2004, logger.Entries, "A successful startup step must not be logged as a failure"); + } + } + + /// + /// Verify a failing startup step is logged and does not stop the scheduled loop + /// + /// Task + [TestMethod] + public async Task ScheduledBackgroundServiceExecuteAsyncStartupFailureIsLoggedAndLoopContinuesAsync() + { + var logger = new TestLogger(); + + using (var cancellationTokenSource = new CancellationTokenSource()) + { + var service = new TestScheduledBackgroundService(logger, + TimeSpan.FromMinutes(5), + new InvalidOperationException("startup boom")); + var runTask = service.RunAsync(cancellationTokenSource.Token); + + await service.ExecutionSignal.ConfigureAwait(false); + await cancellationTokenSource.CancelAsync().ConfigureAwait(false); + await runTask.ConfigureAwait(false); + + Assert.Contains(entry => entry.EventId.Id == 2004 + && entry.Exception is InvalidOperationException, + logger.Entries, + "A failing startup step must be logged as a background service failure"); + Assert.IsGreaterThanOrEqualTo(1, + service.ExecutionCount, + "A failing startup step must not stop the scheduled loop"); + } + } + + /// + /// Verify a startup step cancelled during shutdown is not logged as a failure + /// + /// Task + [TestMethod] + public async Task ScheduledBackgroundServiceExecuteAsyncStartupCancellationDuringShutdownIsNotLoggedAsync() + { + var logger = new TestLogger(); + + using (var cancellationTokenSource = new CancellationTokenSource()) + { + await cancellationTokenSource.CancelAsync().ConfigureAwait(false); + + var service = new TestScheduledBackgroundService(logger, + TimeSpan.FromMinutes(5), + null); + + await service.RunAsync(cancellationTokenSource.Token) + .ConfigureAwait(false); + + Assert.AreEqual(0, + service.StartupExecutionCount, + "A startup step cancelled during shutdown must not complete"); + Assert.DoesNotContain(entry => entry.EventId.Id == 2004, logger.Entries, "Cancelling the startup step during shutdown must not be logged as a failure"); + } + } + + #endregion // Methods +} \ No newline at end of file diff --git a/src/Tests/DockerUpdateGuard.Tests/VulnerabilityEnrichmentServiceTests.cs b/src/Tests/DockerUpdateGuard.Tests/VulnerabilityEnrichmentServiceTests.cs index 75ef1b7..877ee47 100644 --- a/src/Tests/DockerUpdateGuard.Tests/VulnerabilityEnrichmentServiceTests.cs +++ b/src/Tests/DockerUpdateGuard.Tests/VulnerabilityEnrichmentServiceTests.cs @@ -6,6 +6,7 @@ using DockerUpdateGuard.Images.Data; using DockerUpdateGuard.Infrastructure; using DockerUpdateGuard.Tests.Data; +using DockerUpdateGuard.Tests.Helper; using DockerUpdateGuard.Vulnerabilities; using DockerUpdateGuard.Vulnerabilities.Interfaces; @@ -1740,5 +1741,91 @@ await service.RefreshAsync(ScanTriggerSource.Manual, CancellationToken.None) } } + /// + /// Verify the configured scan parallelism bounds the concurrent provider calls while every finding is persisted + /// + /// Task + [TestMethod] + public async Task VulnerabilityEnrichmentServiceRefreshAsyncBoundsProviderCallsToConfiguredParallelismAsync() + { + using (var database = new SqliteTestDatabase()) + { + var dbContext = database.CreateDbContext(); + + await using (dbContext.ConfigureAwait(false)) + { + var imageCatalogRepository = new ImageCatalogRepository(dbContext); + var repositories = new[] { "company/api", "company/web", "company/worker" }; + + foreach (var repository in repositories) + { + var imageVersion = await imageCatalogRepository.GetOrCreateImageVersionAsync("docker.io", + repository, + "1.0.0", + $"sha256:{repository.Replace('/', '-')}", + cancellationToken: CancellationToken.None) + .ConfigureAwait(false); + + dbContext.ObservedImages.Add(new ObservedImage + { + Name = repository, + CurrentImageVersionId = imageVersion.Id, + }); + } + + await dbContext.SaveChangesAsync(TestContext.CancellationToken) + .ConfigureAwait(false); + + var vulnerabilityProvider = new ConcurrencyTrackingVulnerabilityProvider(TimeSpan.FromMilliseconds(150)); + var vulnerabilityProviderResolver = Substitute.For(); + + vulnerabilityProviderResolver.Resolve().Returns(vulnerabilityProvider); + + var optionsMonitor = new TestOptionsMonitor(new DockerUpdateGuardOptions + { + Vulnerabilities = new VulnerabilityOptions + { + Enabled = true, + Provider = VulnerabilityProviderKind.Trivy, + MaxParallelScans = 2, + }, + }); + + var service = new VulnerabilityEnrichmentService(new ApplicationTelemetry(), + dbContext, + new ImageReferenceParser(), + new LiveImageInventoryQueryService(dbContext), + new TestLogger(), + optionsMonitor, + vulnerabilityProviderResolver); + + await service.RefreshAsync(ScanTriggerSource.Manual, CancellationToken.None) + .ConfigureAwait(false); + + var findings = await dbContext.VulnerabilityFindings.ToListAsync(TestContext.CancellationToken) + .ConfigureAwait(false); + var scanRun = await dbContext.ScanRuns.SingleAsync(entity => entity.Type == ScanRunType.Vulnerability, TestContext.CancellationToken) + .ConfigureAwait(false); + + Assert.IsLessThanOrEqualTo(2, + vulnerabilityProvider.MaxObservedConcurrency, + "The configured scan parallelism must bound the number of concurrent provider calls"); + Assert.IsGreaterThan(1, + vulnerabilityProvider.MaxObservedConcurrency, + "A parallelism above one must actually run provider calls next to each other"); + Assert.HasCount(3, + vulnerabilityProvider.ScannedReferences, + "Every relevant image version must be handed to the vulnerability provider exactly once"); + Assert.HasCount(3, + findings, + "Parallel provider calls must persist a finding for every scanned image version"); + Assert.IsTrue(findings.TrueForAll(entity => entity.IsActive), "Findings reported by parallel provider calls must be stored as active"); + Assert.AreEqual(ScanRunStatus.Succeeded, + scanRun.Status, + "A refresh with bounded parallelism must complete successfully"); + } + } + } + #endregion // Methods } \ No newline at end of file