From 5e78624dd4783559624eee7af9594a63fa0fb27d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 05:28:44 +0000 Subject: [PATCH 1/2] Harden the vulnerability refresh against long-running provider scans Add the Vulnerabilities:MaxParallelScans option (validated 1..8, default 1) and process image versions in batches of that size, running only the provider calls in parallel while every database interaction stays on a single flow. Raise the vulnerability request timeout default from 30 to 120 seconds because first-time Trivy scans of large images regularly exceed the shorter window. Let the scan cleanup mark abandoned Running scan runs as Failed on startup and on every cleanup cycle, using twice the run type interval with a floor of 24 hours as the stale threshold. Closes #170 --- DOCKER.md | 2 + README.md | 6 +- .../DockerUpdateGuardOptionsValidator.cs | 5 + .../Configuration/VulnerabilityOptions.cs | 7 +- .../Images/ImageHostLoggingExtensions.cs | 10 ++ .../Images/ScanCleanupBackgroundService.cs | 97 ++++++++++ .../Images/ScheduledBackgroundService.cs | 38 ++++ .../Images/VulnerabilityEnrichmentService.cs | 87 ++++++--- src/DockerUpdateGuard/appsettings.json | 3 +- .../DockerUpdateGuardOptionsValidatorTests.cs | 4 + ...oncurrencyTrackingVulnerabilityProvider.cs | 137 ++++++++++++++ .../TestScanCleanupBackgroundService.cs | 10 ++ .../ScanCleanupBackgroundServiceTests.cs | 169 ++++++++++++++++++ .../VulnerabilityEnrichmentServiceTests.cs | 87 +++++++++ 14 files changed, 634 insertions(+), 28 deletions(-) create mode 100644 src/Tests/DockerUpdateGuard.Tests/Helper/ConcurrencyTrackingVulnerabilityProvider.cs diff --git a/DOCKER.md b/DOCKER.md index ee2d120..b9a9fc6 100644 --- a/DOCKER.md +++ b/DOCKER.md @@ -81,6 +81,8 @@ Common JSON paths: | `DockerUpdateGuard:Vulnerabilities:Provider` | `None`, `DockerScout`, or `Trivy` | | `DockerUpdateGuard:Vulnerabilities:TrivyBaseUrl` | Required when `Provider=Trivy`; base URL of the Trivy server used by the bundled Trivy CLI in client mode | | `DockerUpdateGuard:Vulnerabilities:TrivyExecutablePath` | Optional path to the Trivy executable; defaults to `trivy` (bundled in the image) | +| `DockerUpdateGuard:Vulnerabilities:RequestTimeoutSeconds` | Timeout for vulnerability provider requests; defaults to `120` because first-time scans of large images are slow | +| `DockerUpdateGuard:Vulnerabilities:MaxParallelScans` | Number of vulnerability scans running at the same time (`1` to `8`); defaults to `1` | | `Telemetry:OtlpEndpoint` | Optional OTLP collector endpoint | | `DockerUpdateGuard:DisplayVersion` | Version label shown in the UI; set by the image build argument | diff --git a/README.md b/README.md index 3d4da83..80bd837 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,8 @@ versions. Override them to route the lookups through an internal mirror or proxy | `TrivyExecutablePath` | `trivy` | No | Path to the Trivy executable (resolved through `PATH` by default; bundled in the published container image) | | `DockerScoutLoginUrl` | `https://hub.docker.com/v2/users/login` | No | Docker Hub login endpoint used by the `DockerScout` provider | | `DockerScoutBaseUrl` | `https://api.scout.docker.com` | No | Base address of the Docker Scout API | -| `RequestTimeoutSeconds` | `30` | No | Timeout for vulnerability provider requests | +| `RequestTimeoutSeconds` | `120` | No | Timeout for vulnerability provider requests; first-time Trivy scans of large images regularly exceed shorter timeouts | +| `MaxParallelScans` | `1` | No | Maximum number of vulnerability provider scans running at the same time (`1` to `8`) | ### `DockerUpdateGuard:Scanning` @@ -232,7 +233,8 @@ If all three telemetry switches are `false`, telemetry is effectively disabled. "Enabled": true, "Provider": "Trivy", "TrivyBaseUrl": "http://trivy:4954", - "RequestTimeoutSeconds": 30 + "RequestTimeoutSeconds": 120, + "MaxParallelScans": 1 }, "Scanning": { "DiscoveryIntervalMinutes": 5, diff --git a/src/DockerUpdateGuard/Configuration/DockerUpdateGuardOptionsValidator.cs b/src/DockerUpdateGuard/Configuration/DockerUpdateGuardOptionsValidator.cs index 04eca07..c3c8e00 100644 --- a/src/DockerUpdateGuard/Configuration/DockerUpdateGuardOptionsValidator.cs +++ b/src/DockerUpdateGuard/Configuration/DockerUpdateGuardOptionsValidator.cs @@ -148,6 +148,11 @@ private static void ValidateVulnerabilityOptions(VulnerabilityOptions options, L failures.Add($"'{DockerUpdateGuardOptions.SectionName}:Vulnerabilities:RequestTimeoutSeconds' must be between 1 and 300"); } + ValidateRange(options.MaxParallelScans, + 1, + 8, + $"{DockerUpdateGuardOptions.SectionName}:Vulnerabilities:MaxParallelScans", + failures); ValidateAbsoluteHttpUri(options.DockerScoutLoginUrl, $"{DockerUpdateGuardOptions.SectionName}:Vulnerabilities:DockerScoutLoginUrl", failures); diff --git a/src/DockerUpdateGuard/Configuration/VulnerabilityOptions.cs b/src/DockerUpdateGuard/Configuration/VulnerabilityOptions.cs index f08e279..d35bc8d 100644 --- a/src/DockerUpdateGuard/Configuration/VulnerabilityOptions.cs +++ b/src/DockerUpdateGuard/Configuration/VulnerabilityOptions.cs @@ -40,7 +40,12 @@ public class VulnerabilityOptions /// /// Request timeout in seconds /// - public int RequestTimeoutSeconds { get; set; } = 30; + public int RequestTimeoutSeconds { get; set; } = 120; + + /// + /// Maximum number of vulnerability provider scans that may run at the same time + /// + public int MaxParallelScans { get; set; } = 1; #endregion // Properties } \ No newline at end of file diff --git a/src/DockerUpdateGuard/Images/ImageHostLoggingExtensions.cs b/src/DockerUpdateGuard/Images/ImageHostLoggingExtensions.cs index d2d30f1..a2caf18 100644 --- a/src/DockerUpdateGuard/Images/ImageHostLoggingExtensions.cs +++ b/src/DockerUpdateGuard/Images/ImageHostLoggingExtensions.cs @@ -112,6 +112,16 @@ public static partial void ScanCleanupCompleted(this ILogger logger, int removedSnapshots, int removedScanRuns); + /// + /// Log the number of abandoned running scan runs that were marked as failed + /// + /// Logger + /// Repaired scan run count + [LoggerMessage(EventId = 2021, + Level = LogLevel.Warning, + Message = "Marked {RepairedScanRuns} abandoned running scan runs as failed because they never completed")] + public static partial void StaleScanRunsRepaired(this ILogger logger, int repairedScanRuns); + /// /// Log an observed image scan failure /// diff --git a/src/DockerUpdateGuard/Images/ScanCleanupBackgroundService.cs b/src/DockerUpdateGuard/Images/ScanCleanupBackgroundService.cs index 7cf867e..077062f 100644 --- a/src/DockerUpdateGuard/Images/ScanCleanupBackgroundService.cs +++ b/src/DockerUpdateGuard/Images/ScanCleanupBackgroundService.cs @@ -1,5 +1,6 @@ using DockerUpdateGuard.Configuration; using DockerUpdateGuard.Data; +using DockerUpdateGuard.Data.Entities; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; @@ -18,6 +19,16 @@ public class ScanCleanupBackgroundService : ScheduledBackgroundService /// private const int MaxRetainedScanRuns = 20; + /// + /// Shortest age a running scan run must reach before it is treated as abandoned + /// + private const int MinimumStaleRunThresholdHours = 24; + + /// + /// Message stored on running scan runs that were repaired by the cleanup + /// + private const string StaleRunErrorMessage = "Marked as failed by cleanup: the run never completed (process restart or crash)"; + #endregion // Constants #region Fields @@ -59,6 +70,73 @@ public ScanCleanupBackgroundService(ILogger logger #endregion // Constructors + #region Static methods + + /// + /// Resolve the age a running scan run of the supplied type must reach before it is treated as abandoned + /// + /// Scan run type + /// Scan scheduling options + /// Stale threshold + private static TimeSpan GetStaleRunThreshold(ScanRunType scanRunType, ScanningOptions scanningOptions) + { + var intervalMinutes = scanRunType switch + { + ScanRunType.ObservedImage => scanningOptions.OwnImageBaseScanIntervalMinutes, + ScanRunType.RuntimeContainer => scanningOptions.RuntimeImageUpdateScanIntervalMinutes, + ScanRunType.Vulnerability => scanningOptions.VulnerabilityRefreshIntervalMinutes, + _ => 0, + }; + var intervalThreshold = TimeSpan.FromMinutes(2d * intervalMinutes); + var minimumThreshold = TimeSpan.FromHours(MinimumStaleRunThresholdHours); + + return intervalThreshold < minimumThreshold ? minimumThreshold : intervalThreshold; + } + + #endregion // Static methods + + #region Methods + + /// + /// Mark running scan runs that never completed because of a process restart or crash as failed + /// + /// Database context + /// Current point in time + /// Cancellation token + /// Task + private async Task RepairStaleRunningScanRunsAsync(DockerUpdateGuardDbContext dbContext, + DateTimeOffset utcNow, + CancellationToken cancellationToken) + { + var runningScanRuns = await dbContext.ScanRuns.Where(entity => entity.Status == ScanRunStatus.Running) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + var scanningOptions = _optionsMonitor.CurrentValue.Scanning; + + var staleScanRuns = runningScanRuns.Where(entity => entity.StartedAtUtc < utcNow - GetStaleRunThreshold(entity.Type, scanningOptions)) + .ToList(); + + if (staleScanRuns.Count == 0) + { + return; + } + + foreach (var staleScanRun in staleScanRuns) + { + staleScanRun.Status = ScanRunStatus.Failed; + staleScanRun.CompletedAtUtc = utcNow; + staleScanRun.ErrorMessage = StaleRunErrorMessage; + } + + await dbContext.SaveChangesAsync(cancellationToken) + .ConfigureAwait(false); + + _logger.StaleScanRunsRepaired(staleScanRuns.Count); + } + + #endregion // Methods + #region ScheduledBackgroundService /// @@ -73,6 +151,21 @@ protected override bool ShouldExecuteImmediately() return false; } + /// + protected override async Task ExecuteStartupAsync(CancellationToken stoppingToken) + { + var scope = _serviceScopeFactory.CreateAsyncScope(); + + await using (scope.ConfigureAwait(false)) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + + await RepairStaleRunningScanRunsAsync(dbContext, + DateTimeOffset.UtcNow, + stoppingToken).ConfigureAwait(false); + } + } + /// protected override async Task ExecuteCoreAsync(CancellationToken stoppingToken) { @@ -84,6 +177,10 @@ protected override async Task ExecuteCoreAsync(CancellationToken stoppingToken) var applicationTelemetry = scope.ServiceProvider.GetRequiredService(); var cleanupStartedAtUtc = DateTimeOffset.UtcNow; + await RepairStaleRunningScanRunsAsync(dbContext, + cleanupStartedAtUtc, + stoppingToken).ConfigureAwait(false); + var cutoff = cleanupStartedAtUtc.AddDays(-_optionsMonitor.CurrentValue.Scanning.RetainScanRunsDays); var completedUnreferencedScanRuns = dbContext.ScanRuns diff --git a/src/DockerUpdateGuard/Images/ScheduledBackgroundService.cs b/src/DockerUpdateGuard/Images/ScheduledBackgroundService.cs index 69147d0..f8138fa 100644 --- a/src/DockerUpdateGuard/Images/ScheduledBackgroundService.cs +++ b/src/DockerUpdateGuard/Images/ScheduledBackgroundService.cs @@ -53,6 +53,42 @@ protected virtual bool ShouldExecuteImmediately() /// Task protected abstract Task ExecuteCoreAsync(CancellationToken stoppingToken); + /// + /// Execute a short startup step before the scheduled loop begins, independent of + /// + /// Cancellation token + /// Task + protected virtual Task ExecuteStartupAsync(CancellationToken stoppingToken) + { + return Task.CompletedTask; + } + + /// + /// Execute the startup step with exception logging + /// + /// Cancellation token + /// Task + private async Task ExecuteStartupSafelyAsync(CancellationToken stoppingToken) + { + var backgroundServiceName = GetType().Name; + var stopwatch = Stopwatch.StartNew(); + + try + { + await ExecuteStartupAsync(stoppingToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + // Expected during host shutdown, the startup step is neither completed nor failed + } + catch (Exception exception) + { + _logger.BackgroundServiceExecutionFailed(exception, + backgroundServiceName, + stopwatch.ElapsedMilliseconds); + } + } + /// /// Execute the background operation with exception logging /// @@ -97,6 +133,8 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) _logger.BackgroundServiceStarted(backgroundServiceName, GetInterval().TotalMinutes); } + await ExecuteStartupSafelyAsync(stoppingToken).ConfigureAwait(false); + if (ShouldExecuteImmediately()) { await ExecuteSafelyAsync(stoppingToken).ConfigureAwait(false); diff --git a/src/DockerUpdateGuard/Images/VulnerabilityEnrichmentService.cs b/src/DockerUpdateGuard/Images/VulnerabilityEnrichmentService.cs index 56ae52b..1f52643 100644 --- a/src/DockerUpdateGuard/Images/VulnerabilityEnrichmentService.cs +++ b/src/DockerUpdateGuard/Images/VulnerabilityEnrichmentService.cs @@ -19,6 +19,15 @@ namespace DockerUpdateGuard.Images; /// public class VulnerabilityEnrichmentService : IVulnerabilityEnrichmentService { + #region Constants + + /// + /// Highest supported number of provider scans running at the same time + /// + private const int MaxSupportedParallelScans = 8; + + #endregion // Constants + #region Fields /// @@ -265,27 +274,39 @@ private async Task UpsertVulnerabilityFindingsAsync(ScanRun scanRun, } /// - /// Enrich a single image version with the advisories reported by the vulnerability provider + /// Start the provider scan of a single image version without touching the database context + /// + /// Image version to scan + /// Vulnerability provider resolved for this refresh run + /// Cancellation token + /// Reported advisories + private Task>> ScanImageVulnerabilitiesAsync(ImageVersion image, + IVulnerabilityProvider vulnerabilityProvider, + CancellationToken cancellationToken) + { + var imageReference = _imageReferenceParser.Parse(_imageReferenceParser.Format(image)); + + return vulnerabilityProvider.GetVulnerabilitiesAsync(imageReference, cancellationToken); + } + + /// + /// Apply the reported advisories of a single image version to the tracked entities /// /// Owning scan run /// Image version to enrich /// Scan status before the image was processed - /// Vulnerability provider resolved for this refresh run + /// Result reported by the vulnerability provider for this image version /// Cancellation token /// Resulting scan status and failure message when the image could not be enriched - private async Task<(ScanRunStatus Status, string? Message)> ProcessImageVulnerabilitiesAsync(ScanRun scanRun, - ImageVersion image, - ScanRunStatus currentStatus, - IVulnerabilityProvider vulnerabilityProvider, - CancellationToken cancellationToken) + private async Task<(ScanRunStatus Status, string? Message)> ApplyImageVulnerabilitiesAsync(ScanRun scanRun, + ImageVersion image, + ScanRunStatus currentStatus, + ExternalOperationResult> advisoryResult, + CancellationToken cancellationToken) { - var formattedImageReference = _imageReferenceParser.Format(image); - var imageReference = _imageReferenceParser.Parse(formattedImageReference); - var advisoryResult = await vulnerabilityProvider.GetVulnerabilitiesAsync(imageReference, cancellationToken) - .ConfigureAwait(false); - if (advisoryResult.Status != ExternalOperationStatus.Succeeded || advisoryResult.Data is null) { + var formattedImageReference = _imageReferenceParser.Format(image); var failureMessage = advisoryResult.Message ?? $"Unable to enrich vulnerabilities for '{image.RegistryRepository.Repository}:{image.Tag}'"; var failureStatus = advisoryResult.Status == ExternalOperationStatus.Failed || currentStatus == ScanRunStatus.Failed @@ -415,23 +436,41 @@ await _dbContext.SaveChangesAsync(cancellationToken) _logger.VulnerabilityRefreshNoImages(triggerSource); } - foreach (var image in images) + var maxParallelScans = Math.Clamp(_optionsMonitor.CurrentValue.Vulnerabilities.MaxParallelScans, + 1, + MaxSupportedParallelScans); + + // Only the provider calls of a batch run in parallel, every database interaction stays on this flow + for (var batchStart = 0; batchStart < images.Count; batchStart += maxParallelScans) { - var imageOutcome = await ProcessImageVulnerabilitiesAsync(scanRun, - image, - finalStatus, - vulnerabilityProvider, - cancellationToken).ConfigureAwait(false); + var batch = images.Skip(batchStart) + .Take(maxParallelScans) + .ToList(); + + var scanTasks = batch.Select(image => ScanImageVulnerabilitiesAsync(image, vulnerabilityProvider, cancellationToken)) + .ToList(); - finalStatus = imageOutcome.Status; + var advisoryResults = await Task.WhenAll(scanTasks) + .ConfigureAwait(false); - if (imageOutcome.Message is not null) + for (var batchIndex = 0; batchIndex < batch.Count; batchIndex++) { - statusMessages.Add(imageOutcome.Message); - } + var imageOutcome = await ApplyImageVulnerabilitiesAsync(scanRun, + batch[batchIndex], + finalStatus, + advisoryResults[batchIndex], + cancellationToken).ConfigureAwait(false); - await _dbContext.SaveChangesAsync(cancellationToken) - .ConfigureAwait(false); + finalStatus = imageOutcome.Status; + + if (imageOutcome.Message is not null) + { + statusMessages.Add(imageOutcome.Message); + } + + await _dbContext.SaveChangesAsync(cancellationToken) + .ConfigureAwait(false); + } } var deactivatedFindingCount = await DeactivateStaleFindingsAsync(scanRun, diff --git a/src/DockerUpdateGuard/appsettings.json b/src/DockerUpdateGuard/appsettings.json index 9bc3b05..16836ee 100644 --- a/src/DockerUpdateGuard/appsettings.json +++ b/src/DockerUpdateGuard/appsettings.json @@ -29,7 +29,8 @@ "TrivyExecutablePath": "trivy", "DockerScoutLoginUrl": "https://hub.docker.com/v2/users/login", "DockerScoutBaseUrl": "https://api.scout.docker.com", - "RequestTimeoutSeconds": 30 + "RequestTimeoutSeconds": 120, + "MaxParallelScans": 1 }, "Scanning": { "DiscoveryIntervalMinutes": 15, diff --git a/src/Tests/DockerUpdateGuard.Tests/DockerUpdateGuardOptionsValidatorTests.cs b/src/Tests/DockerUpdateGuard.Tests/DockerUpdateGuardOptionsValidatorTests.cs index 0fd1e1d..f958928 100644 --- a/src/Tests/DockerUpdateGuard.Tests/DockerUpdateGuardOptionsValidatorTests.cs +++ b/src/Tests/DockerUpdateGuard.Tests/DockerUpdateGuardOptionsValidatorTests.cs @@ -256,6 +256,7 @@ public void DockerUpdateGuardOptionsValidatorOutOfRangeUpperBoundsReturnsFailure options.DockerHub.RequestTimeoutSeconds = 301; options.DockerHub.MaxParallelRequests = 33; options.Vulnerabilities.RequestTimeoutSeconds = 301; + options.Vulnerabilities.MaxParallelScans = 9; options.Scanning.DiscoveryIntervalMinutes = 1441; options.Scanning.OwnImageBaseScanIntervalMinutes = 10081; options.Scanning.DockerHubRequestLimitWindowHours = 169; @@ -297,6 +298,9 @@ public void DockerUpdateGuardOptionsValidatorOutOfRangeUpperBoundsReturnsFailure Assert.Contains(message => message.Contains("Vulnerabilities:RequestTimeoutSeconds", StringComparison.Ordinal), failures, "A vulnerability timeout above the upper bound must be reported"); + Assert.Contains(message => message.Contains("Vulnerabilities:MaxParallelScans", StringComparison.Ordinal), + failures, + "A vulnerability scan parallelism above the upper bound must be reported"); Assert.Contains(message => message.Contains("Scanning:DiscoveryIntervalMinutes", StringComparison.Ordinal), failures, "A discovery interval above the upper bound must be reported"); diff --git a/src/Tests/DockerUpdateGuard.Tests/Helper/ConcurrencyTrackingVulnerabilityProvider.cs b/src/Tests/DockerUpdateGuard.Tests/Helper/ConcurrencyTrackingVulnerabilityProvider.cs new file mode 100644 index 0000000..7621434 --- /dev/null +++ b/src/Tests/DockerUpdateGuard.Tests/Helper/ConcurrencyTrackingVulnerabilityProvider.cs @@ -0,0 +1,137 @@ +using DockerUpdateGuard.Data.Entities; +using DockerUpdateGuard.Images.Data; +using DockerUpdateGuard.Infrastructure; +using DockerUpdateGuard.Vulnerabilities; +using DockerUpdateGuard.Vulnerabilities.Interfaces; + +namespace DockerUpdateGuard.Tests.Helper; + +/// +/// Vulnerability provider that records the peak number of concurrent scans +/// +internal sealed class ConcurrencyTrackingVulnerabilityProvider : IVulnerabilityProvider +{ + #region Fields + + /// + /// Artificial delay applied while a scan is in flight + /// + private readonly TimeSpan _delay; + + /// + /// Image references passed to the provider + /// + private readonly List _scannedReferences; + + /// + /// Number of scans currently in flight + /// + private int _currentConcurrency; + + /// + /// Peak number of scans observed in flight at the same time + /// + private int _maxConcurrency; + + #endregion // Fields + + #region Constructors + + /// + /// Constructor + /// + /// Artificial delay applied while a scan is in flight + public ConcurrencyTrackingVulnerabilityProvider(TimeSpan delay) + { + _delay = delay; + _scannedReferences = []; + } + + #endregion // Constructors + + #region Properties + + /// + /// Peak number of scans observed in flight at the same time + /// + public int MaxObservedConcurrency => Volatile.Read(ref _maxConcurrency); + + /// + /// Image references passed to the provider + /// + public IReadOnlyList ScannedReferences + { + get + { + lock (_scannedReferences) + { + return _scannedReferences.ToList(); + } + } + } + + #endregion // Properties + + #region Methods + + /// + /// Raise the observed peak concurrency to the supplied value when it is higher + /// + /// Candidate concurrency value + private void UpdateMaxConcurrency(int candidate) + { + var observedMax = Volatile.Read(ref _maxConcurrency); + + while (candidate > observedMax) + { + var previousMax = Interlocked.CompareExchange(ref _maxConcurrency, candidate, observedMax); + + if (previousMax == observedMax) + { + break; + } + + observedMax = previousMax; + } + } + + #endregion // Methods + + #region IVulnerabilityProvider + + /// + public async Task>> GetVulnerabilitiesAsync(ImageReference imageReference, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(imageReference); + + var current = Interlocked.Increment(ref _currentConcurrency); + + UpdateMaxConcurrency(current); + + lock (_scannedReferences) + { + _scannedReferences.Add(imageReference.Repository); + } + + try + { + await Task.Delay(_delay, cancellationToken).ConfigureAwait(false); + + return ExternalOperationResult>.Succeeded([ + new VulnerabilityAdvisoryData + { + AdvisoryId = $"CVE-{imageReference.Repository}", + Title = "Concurrency advisory", + Severity = VulnerabilitySeverity.High, + AffectedPackage = "openssl", + }, + ]); + } + finally + { + Interlocked.Decrement(ref _currentConcurrency); + } + } + + #endregion // IVulnerabilityProvider +} \ No newline at end of file diff --git a/src/Tests/DockerUpdateGuard.Tests/Helper/TestScanCleanupBackgroundService.cs b/src/Tests/DockerUpdateGuard.Tests/Helper/TestScanCleanupBackgroundService.cs index 2db7bc2..fa512f4 100644 --- a/src/Tests/DockerUpdateGuard.Tests/Helper/TestScanCleanupBackgroundService.cs +++ b/src/Tests/DockerUpdateGuard.Tests/Helper/TestScanCleanupBackgroundService.cs @@ -42,5 +42,15 @@ public Task ExecuteOnceAsync(CancellationToken cancellationToken) return ExecuteCoreAsync(cancellationToken); } + /// + /// Execute the startup step of the cleanup service + /// + /// Cancellation token + /// Task + public Task ExecuteStartupOnceAsync(CancellationToken cancellationToken) + { + return ExecuteStartupAsync(cancellationToken); + } + #endregion // Methods } \ No newline at end of file diff --git a/src/Tests/DockerUpdateGuard.Tests/ScanCleanupBackgroundServiceTests.cs b/src/Tests/DockerUpdateGuard.Tests/ScanCleanupBackgroundServiceTests.cs index d58dc20..6311f1a 100644 --- a/src/Tests/DockerUpdateGuard.Tests/ScanCleanupBackgroundServiceTests.cs +++ b/src/Tests/DockerUpdateGuard.Tests/ScanCleanupBackgroundServiceTests.cs @@ -197,5 +197,174 @@ await service.ExecuteOnceAsync(CancellationToken.None) } } + /// + /// Verify cleanup marks abandoned running scan runs as failed while fresh runs stay untouched + /// + /// Task + [TestMethod] + public async Task ScanCleanupBackgroundServiceExecuteCoreAsyncMarksStaleRunningScanRunsAsFailedAsync() + { + var databaseName = Guid.NewGuid().ToString(); + var serviceCollection = new ServiceCollection(); + + serviceCollection.AddScoped(_ => + { + var options = new DbContextOptionsBuilder().UseInMemoryDatabase(databaseName) + .Options; + + return new DockerUpdateGuardDbContext(options); + }); + serviceCollection.AddScoped(_ => new ApplicationTelemetry()); + + var serviceProvider = serviceCollection.BuildServiceProvider(); + + await using (serviceProvider.ConfigureAwait(false)) + { + var scope = serviceProvider.CreateAsyncScope(); + + await using (scope.ConfigureAwait(false)) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + var now = DateTimeOffset.UtcNow; + + dbContext.ScanRuns.Add(new ScanRun + { + Type = ScanRunType.Vulnerability, + Status = ScanRunStatus.Running, + TriggerSource = ScanTriggerSource.Scheduled, + StartedAtUtc = now.AddDays(-3), + CorrelationId = "stale-run", + }); + dbContext.ScanRuns.Add(new ScanRun + { + Type = ScanRunType.Vulnerability, + Status = ScanRunStatus.Running, + TriggerSource = ScanTriggerSource.Scheduled, + StartedAtUtc = now.AddMinutes(-5), + CorrelationId = "fresh-run", + }); + + await dbContext.SaveChangesAsync(CancellationToken.None) + .ConfigureAwait(false); + } + + var options = new DockerUpdateGuardOptions + { + Scanning = new ScanningOptions + { + CleanupIntervalMinutes = 60, + RetainScanRunsDays = 30, + VulnerabilityRefreshIntervalMinutes = 180, + }, + }; + var logger = new TestLogger(); + + var service = new TestScanCleanupBackgroundService(logger, + new TestOptionsMonitor(options), + serviceProvider.GetRequiredService()); + + await service.ExecuteOnceAsync(CancellationToken.None) + .ConfigureAwait(false); + + var verificationScope = serviceProvider.CreateAsyncScope(); + + await using (verificationScope.ConfigureAwait(false)) + { + var dbContext = verificationScope.ServiceProvider.GetRequiredService(); + var staleScanRun = await dbContext.ScanRuns.SingleAsync(entity => entity.CorrelationId == "stale-run", CancellationToken.None) + .ConfigureAwait(false); + var freshScanRun = await dbContext.ScanRuns.SingleAsync(entity => entity.CorrelationId == "fresh-run", CancellationToken.None) + .ConfigureAwait(false); + + Assert.AreEqual(ScanRunStatus.Failed, + staleScanRun.Status, + "Cleanup must mark running scan runs that never completed as failed"); + Assert.IsNotNull(staleScanRun.CompletedAtUtc, "A repaired scan run must record a completion timestamp"); + Assert.AreEqual("Marked as failed by cleanup: the run never completed (process restart or crash)", + staleScanRun.ErrorMessage, + "A repaired scan run must explain why it was marked as failed"); + Assert.AreEqual(ScanRunStatus.Running, + freshScanRun.Status, + "Cleanup must not touch running scan runs that are still within their expected runtime"); + Assert.IsNull(freshScanRun.CompletedAtUtc, "Cleanup must not complete running scan runs that are still within their expected runtime"); + Assert.Contains(entry => entry.EventId.Id == 2021, logger.Entries, "Repairing abandoned scan runs must be logged"); + } + } + } + + /// + /// Verify the cleanup startup step repairs abandoned running scan runs before the first scheduled cycle + /// + /// Task + [TestMethod] + public async Task ScanCleanupBackgroundServiceExecuteStartupAsyncMarksStaleRunningScanRunsAsFailedAsync() + { + var databaseName = Guid.NewGuid().ToString(); + var serviceCollection = new ServiceCollection(); + + serviceCollection.AddScoped(_ => + { + var options = new DbContextOptionsBuilder().UseInMemoryDatabase(databaseName) + .Options; + + return new DockerUpdateGuardDbContext(options); + }); + serviceCollection.AddScoped(_ => new ApplicationTelemetry()); + + var serviceProvider = serviceCollection.BuildServiceProvider(); + + await using (serviceProvider.ConfigureAwait(false)) + { + var scope = serviceProvider.CreateAsyncScope(); + + await using (scope.ConfigureAwait(false)) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + + dbContext.ScanRuns.Add(new ScanRun + { + Type = ScanRunType.RuntimeContainer, + Status = ScanRunStatus.Running, + TriggerSource = ScanTriggerSource.Scheduled, + StartedAtUtc = DateTimeOffset.UtcNow.AddDays(-2), + CorrelationId = "crashed-run", + }); + + await dbContext.SaveChangesAsync(CancellationToken.None) + .ConfigureAwait(false); + } + + var options = new DockerUpdateGuardOptions + { + Scanning = new ScanningOptions + { + CleanupIntervalMinutes = 60, + RetainScanRunsDays = 30, + }, + }; + + var service = new TestScanCleanupBackgroundService(new TestLogger(), + new TestOptionsMonitor(options), + serviceProvider.GetRequiredService()); + + await service.ExecuteStartupOnceAsync(CancellationToken.None) + .ConfigureAwait(false); + + var verificationScope = serviceProvider.CreateAsyncScope(); + + await using (verificationScope.ConfigureAwait(false)) + { + var dbContext = verificationScope.ServiceProvider.GetRequiredService(); + var crashedScanRun = await dbContext.ScanRuns.SingleAsync(entity => entity.CorrelationId == "crashed-run", CancellationToken.None) + .ConfigureAwait(false); + + Assert.AreEqual(ScanRunStatus.Failed, + crashedScanRun.Status, + "The cleanup startup step must repair running scan runs left behind by a crashed process"); + Assert.IsNotNull(crashedScanRun.CompletedAtUtc, "A scan run repaired at startup must record a completion timestamp"); + } + } + } + #endregion // Methods } \ No newline at end of file diff --git a/src/Tests/DockerUpdateGuard.Tests/VulnerabilityEnrichmentServiceTests.cs b/src/Tests/DockerUpdateGuard.Tests/VulnerabilityEnrichmentServiceTests.cs index 75ef1b7..877ee47 100644 --- a/src/Tests/DockerUpdateGuard.Tests/VulnerabilityEnrichmentServiceTests.cs +++ b/src/Tests/DockerUpdateGuard.Tests/VulnerabilityEnrichmentServiceTests.cs @@ -6,6 +6,7 @@ using DockerUpdateGuard.Images.Data; using DockerUpdateGuard.Infrastructure; using DockerUpdateGuard.Tests.Data; +using DockerUpdateGuard.Tests.Helper; using DockerUpdateGuard.Vulnerabilities; using DockerUpdateGuard.Vulnerabilities.Interfaces; @@ -1740,5 +1741,91 @@ await service.RefreshAsync(ScanTriggerSource.Manual, CancellationToken.None) } } + /// + /// Verify the configured scan parallelism bounds the concurrent provider calls while every finding is persisted + /// + /// Task + [TestMethod] + public async Task VulnerabilityEnrichmentServiceRefreshAsyncBoundsProviderCallsToConfiguredParallelismAsync() + { + using (var database = new SqliteTestDatabase()) + { + var dbContext = database.CreateDbContext(); + + await using (dbContext.ConfigureAwait(false)) + { + var imageCatalogRepository = new ImageCatalogRepository(dbContext); + var repositories = new[] { "company/api", "company/web", "company/worker" }; + + foreach (var repository in repositories) + { + var imageVersion = await imageCatalogRepository.GetOrCreateImageVersionAsync("docker.io", + repository, + "1.0.0", + $"sha256:{repository.Replace('/', '-')}", + cancellationToken: CancellationToken.None) + .ConfigureAwait(false); + + dbContext.ObservedImages.Add(new ObservedImage + { + Name = repository, + CurrentImageVersionId = imageVersion.Id, + }); + } + + await dbContext.SaveChangesAsync(TestContext.CancellationToken) + .ConfigureAwait(false); + + var vulnerabilityProvider = new ConcurrencyTrackingVulnerabilityProvider(TimeSpan.FromMilliseconds(150)); + var vulnerabilityProviderResolver = Substitute.For(); + + vulnerabilityProviderResolver.Resolve().Returns(vulnerabilityProvider); + + var optionsMonitor = new TestOptionsMonitor(new DockerUpdateGuardOptions + { + Vulnerabilities = new VulnerabilityOptions + { + Enabled = true, + Provider = VulnerabilityProviderKind.Trivy, + MaxParallelScans = 2, + }, + }); + + var service = new VulnerabilityEnrichmentService(new ApplicationTelemetry(), + dbContext, + new ImageReferenceParser(), + new LiveImageInventoryQueryService(dbContext), + new TestLogger(), + optionsMonitor, + vulnerabilityProviderResolver); + + await service.RefreshAsync(ScanTriggerSource.Manual, CancellationToken.None) + .ConfigureAwait(false); + + var findings = await dbContext.VulnerabilityFindings.ToListAsync(TestContext.CancellationToken) + .ConfigureAwait(false); + var scanRun = await dbContext.ScanRuns.SingleAsync(entity => entity.Type == ScanRunType.Vulnerability, TestContext.CancellationToken) + .ConfigureAwait(false); + + Assert.IsLessThanOrEqualTo(2, + vulnerabilityProvider.MaxObservedConcurrency, + "The configured scan parallelism must bound the number of concurrent provider calls"); + Assert.IsGreaterThan(1, + vulnerabilityProvider.MaxObservedConcurrency, + "A parallelism above one must actually run provider calls next to each other"); + Assert.HasCount(3, + vulnerabilityProvider.ScannedReferences, + "Every relevant image version must be handed to the vulnerability provider exactly once"); + Assert.HasCount(3, + findings, + "Parallel provider calls must persist a finding for every scanned image version"); + Assert.IsTrue(findings.TrueForAll(entity => entity.IsActive), "Findings reported by parallel provider calls must be stored as active"); + Assert.AreEqual(ScanRunStatus.Succeeded, + scanRun.Status, + "A refresh with bounded parallelism must complete successfully"); + } + } + } + #endregion // Methods } \ No newline at end of file From 68018ae541e376ddffd932e9ceb622311a25a793 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 29 Jul 2026 06:20:05 +0000 Subject: [PATCH 2/2] Cover the scheduled startup hook and the stale run threshold with tests Add ScheduledBackgroundServiceTests together with a test double that drives the scheduled loop, covering the startup hook ordering, the logged startup failure, and the cancellation path during shutdown. Extend the scan cleanup tests with abandoned observed image and untyped runs and with a refresh interval whose doubled value exceeds the minimum threshold of 24 hours. All lines and branches added by this branch are covered now. --- .../Helper/TestScheduledBackgroundService.cs | 138 ++++++++++++++++++ .../ScanCleanupBackgroundServiceTests.cs | 115 +++++++++++++++ .../ScheduledBackgroundServiceTests.cs | 104 +++++++++++++ 3 files changed, 357 insertions(+) create mode 100644 src/Tests/DockerUpdateGuard.Tests/Helper/TestScheduledBackgroundService.cs create mode 100644 src/Tests/DockerUpdateGuard.Tests/ScheduledBackgroundServiceTests.cs diff --git a/src/Tests/DockerUpdateGuard.Tests/Helper/TestScheduledBackgroundService.cs b/src/Tests/DockerUpdateGuard.Tests/Helper/TestScheduledBackgroundService.cs new file mode 100644 index 0000000..1b1c46c --- /dev/null +++ b/src/Tests/DockerUpdateGuard.Tests/Helper/TestScheduledBackgroundService.cs @@ -0,0 +1,138 @@ +using DockerUpdateGuard.Images; + +using Microsoft.Extensions.Logging; + +namespace DockerUpdateGuard.Tests.Helper; + +/// +/// Testable scheduled background service that records how the base class drives its steps +/// +internal sealed class TestScheduledBackgroundService : ScheduledBackgroundService +{ + #region Fields + + /// + /// Signal completed as soon as the scheduled execution ran once + /// + private readonly TaskCompletionSource _executionSignal; + + /// + /// Configured execution interval + /// + private readonly TimeSpan _interval; + + /// + /// Exception thrown by the startup step, or when the startup step succeeds + /// + private readonly Exception? _startupException; + + /// + /// Number of scheduled executions + /// + private int _executionCount; + + /// + /// Number of startup executions + /// + private int _startupExecutionCount; + + /// + /// Indicates whether the startup step ran before the first scheduled execution + /// + private bool _startupRanBeforeExecution; + + #endregion // Fields + + #region Constructors + + /// + /// Constructor + /// + /// Logger + /// Configured execution interval + /// Exception thrown by the startup step, or when the startup step succeeds + public TestScheduledBackgroundService(ILogger logger, TimeSpan interval, Exception? startupException) + : base(logger) + { + _executionSignal = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _interval = interval; + _startupException = startupException; + } + + #endregion // Constructors + + #region Properties + + /// + /// Signal completed as soon as the scheduled execution ran once + /// + public Task ExecutionSignal => _executionSignal.Task; + + /// + /// Number of scheduled executions + /// + public int ExecutionCount => Volatile.Read(ref _executionCount); + + /// + /// Number of startup executions + /// + public int StartupExecutionCount => Volatile.Read(ref _startupExecutionCount); + + /// + /// Indicates whether the startup step ran before the first scheduled execution + /// + public bool StartupRanBeforeExecution => Volatile.Read(ref _startupRanBeforeExecution); + + #endregion // Properties + + #region Methods + + /// + /// Run the scheduled loop of the base class until the supplied token is cancelled + /// + /// Cancellation token + /// Task + public Task RunAsync(CancellationToken stoppingToken) + { + return ExecuteAsync(stoppingToken); + } + + #endregion // Methods + + #region ScheduledBackgroundService + + /// + protected override TimeSpan GetInterval() + { + return _interval; + } + + /// + protected override async Task ExecuteStartupAsync(CancellationToken stoppingToken) + { + await base.ExecuteStartupAsync(stoppingToken).ConfigureAwait(false); + + stoppingToken.ThrowIfCancellationRequested(); + + Volatile.Write(ref _startupRanBeforeExecution, Volatile.Read(ref _executionCount) == 0); + Interlocked.Increment(ref _startupExecutionCount); + + if (_startupException is not null) + { + throw _startupException; + } + } + + /// + protected override Task ExecuteCoreAsync(CancellationToken stoppingToken) + { + stoppingToken.ThrowIfCancellationRequested(); + + Interlocked.Increment(ref _executionCount); + _executionSignal.TrySetResult(); + + return Task.CompletedTask; + } + + #endregion // ScheduledBackgroundService +} \ No newline at end of file diff --git a/src/Tests/DockerUpdateGuard.Tests/ScanCleanupBackgroundServiceTests.cs b/src/Tests/DockerUpdateGuard.Tests/ScanCleanupBackgroundServiceTests.cs index 6311f1a..414ab16 100644 --- a/src/Tests/DockerUpdateGuard.Tests/ScanCleanupBackgroundServiceTests.cs +++ b/src/Tests/DockerUpdateGuard.Tests/ScanCleanupBackgroundServiceTests.cs @@ -235,6 +235,22 @@ public async Task ScanCleanupBackgroundServiceExecuteCoreAsyncMarksStaleRunningS StartedAtUtc = now.AddDays(-3), CorrelationId = "stale-run", }); + dbContext.ScanRuns.Add(new ScanRun + { + Type = ScanRunType.ObservedImage, + Status = ScanRunStatus.Running, + TriggerSource = ScanTriggerSource.Scheduled, + StartedAtUtc = now.AddDays(-3), + CorrelationId = "stale-observed-run", + }); + dbContext.ScanRuns.Add(new ScanRun + { + Type = ScanRunType.NotSet, + Status = ScanRunStatus.Running, + TriggerSource = ScanTriggerSource.Scheduled, + StartedAtUtc = now.AddDays(-3), + CorrelationId = "stale-untyped-run", + }); dbContext.ScanRuns.Add(new ScanRun { Type = ScanRunType.Vulnerability, @@ -273,12 +289,22 @@ await service.ExecuteOnceAsync(CancellationToken.None) var dbContext = verificationScope.ServiceProvider.GetRequiredService(); var staleScanRun = await dbContext.ScanRuns.SingleAsync(entity => entity.CorrelationId == "stale-run", CancellationToken.None) .ConfigureAwait(false); + var staleObservedScanRun = await dbContext.ScanRuns.SingleAsync(entity => entity.CorrelationId == "stale-observed-run", CancellationToken.None) + .ConfigureAwait(false); + var staleUntypedScanRun = await dbContext.ScanRuns.SingleAsync(entity => entity.CorrelationId == "stale-untyped-run", CancellationToken.None) + .ConfigureAwait(false); var freshScanRun = await dbContext.ScanRuns.SingleAsync(entity => entity.CorrelationId == "fresh-run", CancellationToken.None) .ConfigureAwait(false); Assert.AreEqual(ScanRunStatus.Failed, staleScanRun.Status, "Cleanup must mark running scan runs that never completed as failed"); + Assert.AreEqual(ScanRunStatus.Failed, + staleObservedScanRun.Status, + "Cleanup must repair abandoned observed image scan runs as well"); + Assert.AreEqual(ScanRunStatus.Failed, + staleUntypedScanRun.Status, + "Cleanup must repair abandoned scan runs that carry no run type"); Assert.IsNotNull(staleScanRun.CompletedAtUtc, "A repaired scan run must record a completion timestamp"); Assert.AreEqual("Marked as failed by cleanup: the run never completed (process restart or crash)", staleScanRun.ErrorMessage, @@ -292,6 +318,95 @@ await service.ExecuteOnceAsync(CancellationToken.None) } } + /// + /// Verify a long refresh interval stretches the stale threshold beyond the minimum of 24 hours + /// + /// Task + [TestMethod] + public async Task ScanCleanupBackgroundServiceExecuteCoreAsyncLongIntervalStretchesStaleThresholdAsync() + { + var databaseName = Guid.NewGuid().ToString(); + var serviceCollection = new ServiceCollection(); + + serviceCollection.AddScoped(_ => + { + var options = new DbContextOptionsBuilder().UseInMemoryDatabase(databaseName) + .Options; + + return new DockerUpdateGuardDbContext(options); + }); + serviceCollection.AddScoped(_ => new ApplicationTelemetry()); + + var serviceProvider = serviceCollection.BuildServiceProvider(); + + await using (serviceProvider.ConfigureAwait(false)) + { + var scope = serviceProvider.CreateAsyncScope(); + + await using (scope.ConfigureAwait(false)) + { + var dbContext = scope.ServiceProvider.GetRequiredService(); + var now = DateTimeOffset.UtcNow; + + dbContext.ScanRuns.Add(new ScanRun + { + Type = ScanRunType.Vulnerability, + Status = ScanRunStatus.Running, + TriggerSource = ScanTriggerSource.Scheduled, + StartedAtUtc = now.AddHours(-40), + CorrelationId = "beyond-threshold-run", + }); + dbContext.ScanRuns.Add(new ScanRun + { + Type = ScanRunType.Vulnerability, + Status = ScanRunStatus.Running, + TriggerSource = ScanTriggerSource.Scheduled, + StartedAtUtc = now.AddHours(-26), + CorrelationId = "within-threshold-run", + }); + + await dbContext.SaveChangesAsync(CancellationToken.None) + .ConfigureAwait(false); + } + + // Twice the refresh interval of 15 hours is 30 hours and therefore longer than the minimum threshold + var options = new DockerUpdateGuardOptions + { + Scanning = new ScanningOptions + { + CleanupIntervalMinutes = 60, + RetainScanRunsDays = 30, + VulnerabilityRefreshIntervalMinutes = 900, + }, + }; + + var service = new TestScanCleanupBackgroundService(new TestLogger(), + new TestOptionsMonitor(options), + serviceProvider.GetRequiredService()); + + await service.ExecuteOnceAsync(CancellationToken.None) + .ConfigureAwait(false); + + var verificationScope = serviceProvider.CreateAsyncScope(); + + await using (verificationScope.ConfigureAwait(false)) + { + var dbContext = verificationScope.ServiceProvider.GetRequiredService(); + var beyondThresholdScanRun = await dbContext.ScanRuns.SingleAsync(entity => entity.CorrelationId == "beyond-threshold-run", CancellationToken.None) + .ConfigureAwait(false); + var withinThresholdScanRun = await dbContext.ScanRuns.SingleAsync(entity => entity.CorrelationId == "within-threshold-run", CancellationToken.None) + .ConfigureAwait(false); + + Assert.AreEqual(ScanRunStatus.Failed, + beyondThresholdScanRun.Status, + "A running scan run older than twice the configured refresh interval must be repaired"); + Assert.AreEqual(ScanRunStatus.Running, + withinThresholdScanRun.Status, + "A long refresh interval must keep running scan runs untouched beyond the minimum threshold of 24 hours"); + } + } + } + /// /// Verify the cleanup startup step repairs abandoned running scan runs before the first scheduled cycle /// diff --git a/src/Tests/DockerUpdateGuard.Tests/ScheduledBackgroundServiceTests.cs b/src/Tests/DockerUpdateGuard.Tests/ScheduledBackgroundServiceTests.cs new file mode 100644 index 0000000..a6ee8af --- /dev/null +++ b/src/Tests/DockerUpdateGuard.Tests/ScheduledBackgroundServiceTests.cs @@ -0,0 +1,104 @@ +using DockerUpdateGuard.Images; +using DockerUpdateGuard.Tests.Data; +using DockerUpdateGuard.Tests.Helper; + +namespace DockerUpdateGuard.Tests; + +/// +/// Tests for +/// +[TestClass] +public class ScheduledBackgroundServiceTests +{ + #region Methods + + /// + /// Verify the startup step runs once before the first scheduled execution + /// + /// Task + [TestMethod] + public async Task ScheduledBackgroundServiceExecuteAsyncRunsStartupStepBeforeFirstExecutionAsync() + { + var logger = new TestLogger(); + + using (var cancellationTokenSource = new CancellationTokenSource()) + { + var service = new TestScheduledBackgroundService(logger, + TimeSpan.FromMinutes(5), + null); + var runTask = service.RunAsync(cancellationTokenSource.Token); + + await service.ExecutionSignal.ConfigureAwait(false); + await cancellationTokenSource.CancelAsync().ConfigureAwait(false); + await runTask.ConfigureAwait(false); + + Assert.AreEqual(1, + service.StartupExecutionCount, + "The scheduled loop must run the startup step exactly once"); + Assert.IsTrue(service.StartupRanBeforeExecution, "The startup step must run before the first scheduled execution"); + Assert.IsGreaterThanOrEqualTo(1, + service.ExecutionCount, + "The scheduled loop must still execute the background operation after the startup step"); + Assert.DoesNotContain(entry => entry.EventId.Id == 2004, logger.Entries, "A successful startup step must not be logged as a failure"); + } + } + + /// + /// Verify a failing startup step is logged and does not stop the scheduled loop + /// + /// Task + [TestMethod] + public async Task ScheduledBackgroundServiceExecuteAsyncStartupFailureIsLoggedAndLoopContinuesAsync() + { + var logger = new TestLogger(); + + using (var cancellationTokenSource = new CancellationTokenSource()) + { + var service = new TestScheduledBackgroundService(logger, + TimeSpan.FromMinutes(5), + new InvalidOperationException("startup boom")); + var runTask = service.RunAsync(cancellationTokenSource.Token); + + await service.ExecutionSignal.ConfigureAwait(false); + await cancellationTokenSource.CancelAsync().ConfigureAwait(false); + await runTask.ConfigureAwait(false); + + Assert.Contains(entry => entry.EventId.Id == 2004 + && entry.Exception is InvalidOperationException, + logger.Entries, + "A failing startup step must be logged as a background service failure"); + Assert.IsGreaterThanOrEqualTo(1, + service.ExecutionCount, + "A failing startup step must not stop the scheduled loop"); + } + } + + /// + /// Verify a startup step cancelled during shutdown is not logged as a failure + /// + /// Task + [TestMethod] + public async Task ScheduledBackgroundServiceExecuteAsyncStartupCancellationDuringShutdownIsNotLoggedAsync() + { + var logger = new TestLogger(); + + using (var cancellationTokenSource = new CancellationTokenSource()) + { + await cancellationTokenSource.CancelAsync().ConfigureAwait(false); + + var service = new TestScheduledBackgroundService(logger, + TimeSpan.FromMinutes(5), + null); + + await service.RunAsync(cancellationTokenSource.Token) + .ConfigureAwait(false); + + Assert.AreEqual(0, + service.StartupExecutionCount, + "A startup step cancelled during shutdown must not complete"); + Assert.DoesNotContain(entry => entry.EventId.Id == 2004, logger.Entries, "Cancelling the startup step during shutdown must not be logged as a failure"); + } + } + + #endregion // Methods +} \ No newline at end of file