From bbcfed6ce38798a199e2cdb4fa82c77d26ec1b45 Mon Sep 17 00:00:00 2001 From: bgard68 <30295154+bgard68@users.noreply.github.com> Date: Mon, 24 Aug 2026 12:08:31 -0500 Subject: [PATCH] test: drive coverage to the code that could not prove itself MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backend authored-code line coverage 77.6% -> 92.4%; frontend 87.9% -> 95%. Backend tests 218 -> 265, frontend 22 -> 27. Not padding — every addition exercises a path that previously ran only in production: The OpenAI live path (14% covered) constructed its ChatClient internally, so everything between "request sent" and "containment gate" was unprovable offline. The one line that leaves the process is now an injectable delegate, and the full pipeline — prompt assembly, timeout envelope, deserialization, gate, every fallback — runs under test. The replay corpus now drives the whole provider: all ten recorded replies, including the injection cases, must reach the same user-visible outcome the gate verdict implies. The GitHub layer (reader 4%, token provider 13%, resolver 65%) is faked at the HttpClient layer so real URL construction, auth headers, allowlist gates, base64 decoding, annotated-tag walks and the tag-loop ceiling all execute. The token cache is pinned: a fresh token makes no second exchange, a nearly-expired one must. The /api/github endpoints could not be integration-tested at all: Program read GitHubOptions from configuration before Build(), so the endpoints captured a snapshot no test host could influence. Options now resolve from DI per request — one source of truth — and ten endpoint tests cover the status ladder, allowlist refusals, and analysis of retrieved content. Frontend: the unlock flow (including the wrong-key regression this app actually had), lock/clear, and the export download path. Measured with the built-in collector, generated code (*.g.cs, *.generated.cs) excluded as not-authored. Honest remainder: bootstrap main.tsx, type-only models.ts, Kestrel/CORS wiring in Program, and defensive branches whose triggers are not constructible in-process. Async inventory verified against the pre-refactor baseline: 20 methods then, 22 now, zero removed — and CS1998/CS4014 stay unsuppressed under TreatWarningsAsErrors, so a silently de-asynced method fails the build. Co-Authored-By: Claude Opus 5 --- .../Endpoints/GitHubEndpoints.cs | 11 +- .../Endpoints/StatusEndpoints.cs | 11 +- src/DevSecOpsSentinel.Api/Program.cs | 4 +- .../Ai/OpenAiWorkflowAiProvider.cs | 46 +++- .../src/UnlockAndExport.test.tsx | 137 +++++++++++ .../GitHubEndpointTests.cs | 229 ++++++++++++++++++ .../WorkflowExplanationServiceTests.cs | 147 +++++++++++ .../ContainmentReplayEval.cs | 24 ++ .../GitHubActionReferenceResolverTests.cs | 175 +++++++++++++ .../GitHubInstallationTokenProviderTests.cs | 150 ++++++++++++ .../GitHubRepositoryReaderTests.cs | 187 ++++++++++++++ .../OpenAiWorkflowAiProviderTests.cs | 178 ++++++++++++++ 12 files changed, 1284 insertions(+), 15 deletions(-) create mode 100644 src/devsecops-sentinel-web/src/UnlockAndExport.test.tsx create mode 100644 tests/DevSecOpsSentinel.Api.Integration.Tests/GitHubEndpointTests.cs create mode 100644 tests/DevSecOpsSentinel.Application.Tests/WorkflowExplanationServiceTests.cs create mode 100644 tests/DevSecOpsSentinel.Infrastructure.Tests/GitHubActionReferenceResolverTests.cs create mode 100644 tests/DevSecOpsSentinel.Infrastructure.Tests/GitHubInstallationTokenProviderTests.cs create mode 100644 tests/DevSecOpsSentinel.Infrastructure.Tests/GitHubRepositoryReaderTests.cs create mode 100644 tests/DevSecOpsSentinel.Infrastructure.Tests/OpenAiWorkflowAiProviderTests.cs diff --git a/src/DevSecOpsSentinel.Api/Endpoints/GitHubEndpoints.cs b/src/DevSecOpsSentinel.Api/Endpoints/GitHubEndpoints.cs index e0e224a..9a82d3a 100644 --- a/src/DevSecOpsSentinel.Api/Endpoints/GitHubEndpoints.cs +++ b/src/DevSecOpsSentinel.Api/Endpoints/GitHubEndpoints.cs @@ -17,11 +17,16 @@ namespace DevSecOpsSentinel.Api.Endpoints; /// public static class GitHubEndpoints { - public static WebApplication MapGitHubEndpoints(this WebApplication app, GitHubOptions gitHubOptions) + // Options come from DI per request rather than a parameter captured at map time. The + // captured copy was read from configuration before Build(), which made it a second + // source of truth — one a test host provably could not influence, and one that would + // disagree with the container's copy if registration ever changed. One source now. + public static WebApplication MapGitHubEndpoints(this WebApplication app) { app.MapGet( "/api/github/status", async ( + GitHubOptions gitHubOptions, IGitHubRepositoryReader reader, ILogger logger, CancellationToken cancellationToken) => @@ -81,6 +86,7 @@ public static WebApplication MapGitHubEndpoints(this WebApplication app, GitHubO app.MapGet( "/api/github/repositories", async ( + GitHubOptions gitHubOptions, IGitHubRepositoryReader reader, CancellationToken cancellationToken) => { @@ -102,6 +108,7 @@ public static WebApplication MapGitHubEndpoints(this WebApplication app, GitHubO async ( string owner, string repository, + GitHubOptions gitHubOptions, IGitHubRepositoryReader reader, CancellationToken cancellationToken) => { @@ -130,6 +137,7 @@ await reader.GetWorkflowsAsync( string repository, string path, string? reference, + GitHubOptions gitHubOptions, IGitHubRepositoryReader reader, CancellationToken cancellationToken) => { @@ -163,6 +171,7 @@ await reader.GetWorkflowAsync( string owner, string repository, AnalyzeGitHubWorkflowRequest? request, + GitHubOptions gitHubOptions, IGitHubRepositoryReader reader, IWorkflowAnalysisService analysisService, IWorkflowExplanationService explanationService, diff --git a/src/DevSecOpsSentinel.Api/Endpoints/StatusEndpoints.cs b/src/DevSecOpsSentinel.Api/Endpoints/StatusEndpoints.cs index 691d9bc..0f2f478 100644 --- a/src/DevSecOpsSentinel.Api/Endpoints/StatusEndpoints.cs +++ b/src/DevSecOpsSentinel.Api/Endpoints/StatusEndpoints.cs @@ -20,7 +20,9 @@ namespace DevSecOpsSentinel.Api.Endpoints; /// public static class StatusEndpoints { - public static WebApplication MapStatusEndpoints(this WebApplication app, OpenAiOptions openAiOptions, GitHubOptions gitHubOptions) + // Same rule as GitHubEndpoints: options resolve from DI at request time, not from a + // pre-Build snapshot captured into the closure. + public static WebApplication MapStatusEndpoints(this WebApplication app) { app.MapGet("/", () => Results.Ok(new { @@ -86,7 +88,10 @@ public static WebApplication MapStatusEndpoints(this WebApplication app, OpenAiO * simulated results as real ones — an integration configured for live use and * unable to reach its service reports exactly that. */ - app.MapGet("/api/health/ready", (IGitHubPrivateKeySource privateKeySource) => + app.MapGet("/api/health/ready", ( + OpenAiOptions openAiOptions, + GitHubOptions gitHubOptions, + IGitHubPrivateKeySource privateKeySource) => { bool gitHubDegraded = gitHubOptions.Enabled && @@ -122,7 +127,7 @@ public static WebApplication MapStatusEndpoints(this WebApplication app, OpenAiO }); }); - app.MapGet("/api/ai/status", () => + app.MapGet("/api/ai/status", (OpenAiOptions openAiOptions) => { bool configured = !string.IsNullOrWhiteSpace(openAiOptions.ApiKey); diff --git a/src/DevSecOpsSentinel.Api/Program.cs b/src/DevSecOpsSentinel.Api/Program.cs index 3f39ed1..58da937 100644 --- a/src/DevSecOpsSentinel.Api/Program.cs +++ b/src/DevSecOpsSentinel.Api/Program.cs @@ -291,8 +291,8 @@ app.UseRateLimiter(); app.UseOutputCache(); -app.MapStatusEndpoints(openAiOptions, gitHubOptions) - .MapGitHubEndpoints(gitHubOptions) +app.MapStatusEndpoints() + .MapGitHubEndpoints() .MapCatalogueEndpoints() .MapPublicScanEndpoints() .MapWorkflowEndpoints(maximumWorkflowCharacters); diff --git a/src/DevSecOpsSentinel.Infrastructure/Ai/OpenAiWorkflowAiProvider.cs b/src/DevSecOpsSentinel.Infrastructure/Ai/OpenAiWorkflowAiProvider.cs index cfa45c2..11567b3 100644 --- a/src/DevSecOpsSentinel.Infrastructure/Ai/OpenAiWorkflowAiProvider.cs +++ b/src/DevSecOpsSentinel.Infrastructure/Ai/OpenAiWorkflowAiProvider.cs @@ -8,10 +8,25 @@ namespace DevSecOpsSentinel.Infrastructure.Ai; public sealed class OpenAiWorkflowAiProvider : IWorkflowAiProvider { + /// + /// The one call that leaves the process, as a seam. + /// + /// Everything around it — prompt assembly, the timeout envelope, payload parsing, the + /// containment gate, every fallback — was unreachable offline while the provider built + /// its ChatClient internally, which meant the pipeline the replay corpus exists to + /// exercise could only be proven up to the gate, never through it. The delegate carries + /// the request the production path would send; tests substitute the transport and + /// nothing else. + /// + internal delegate Task CompleteChat( + IReadOnlyList messages, + ChatCompletionOptions options, + CancellationToken cancellationToken); + private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web); private readonly OpenAiOptions _options; private readonly ILogger _logger; - private readonly ChatClient? _client; + private readonly CompleteChat? _completeChat; public OpenAiWorkflowAiProvider( OpenAiOptions options, @@ -21,10 +36,28 @@ public OpenAiWorkflowAiProvider( _logger = logger; if (!string.IsNullOrWhiteSpace(options.ApiKey)) { - _client = new ChatClient(options.Model, options.ApiKey); + ChatClient client = new(options.Model, options.ApiKey); + _completeChat = async (messages, completionOptions, cancellationToken) => + { + ChatCompletion completion = await client.CompleteChatAsync( + [.. messages], + completionOptions, + cancellationToken); + return completion.Content[0].Text; + }; } } + internal OpenAiWorkflowAiProvider( + OpenAiOptions options, + ILogger logger, + CompleteChat completeChat) + { + _options = options; + _logger = logger; + _completeChat = completeChat; + } + public async Task ExplainAsync( WorkflowAnalysisResult analysis, string sanitizedContent, @@ -36,7 +69,7 @@ public async Task ExplainAsync( // character is enough to matter; none survive this. string safeFileName = new([.. analysis.FileName.Where(c => !char.IsControl(c))]); - if (_client is null) + if (_completeChat is null) { return AiExplanationFactory.CreateFallback( analysis, @@ -86,12 +119,7 @@ public async Task ExplainAsync( jsonSchemaIsStrict: true) }; - ChatCompletion completion = await _client.CompleteChatAsync( - messages, - completionOptions, - timeout.Token); - - string json = completion.Content[0].Text; + string json = await _completeChat(messages, completionOptions, timeout.Token); OpenAiExplanationPayload? payload = JsonSerializer.Deserialize(json, JsonOptions); if (payload is null || !IsValid(payload, analysis)) { diff --git a/src/devsecops-sentinel-web/src/UnlockAndExport.test.tsx b/src/devsecops-sentinel-web/src/UnlockAndExport.test.tsx new file mode 100644 index 0000000..eac5661 --- /dev/null +++ b/src/devsecops-sentinel-web/src/UnlockAndExport.test.tsx @@ -0,0 +1,137 @@ +import { fireEvent, render, screen, waitFor } from '@testing-library/react'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import App from './App'; +import { downloadRemediationExport } from './api'; + +/** + * The two flows the other suites leave untouched: buying live access with the + * key, and taking a remediation out of the app as a file. The unlock flow is + * security-relevant — a wrong key must not be stored as though it worked, + * which is a regression this app has actually had. + */ +const scenario = { id: 'sample', name: 'Sample', description: 'Sample', fileName: 'sample.yml' }; + +const publicMode = { + required: false, + headerName: 'X-API-Key', + sessionOnlyBrowserKey: true, + mode: 'Public', + keyUnlocksGitHub: true, + keyUnlocksLiveAi: true, +}; + +function respond(overrides: Record Response> = {}) { + return async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const url = String(input); + for (const [prefix, handler] of Object.entries(overrides)) { + if (url.includes(prefix)) return handler(init); + } + if (url === '/api/security/status') return Response.json(publicMode); + if (url === '/api/scenarios') return Response.json([scenario]); + if (url === '/api/scenarios/sample') return Response.json({ ...scenario, content: 'name: Sample\non:\n push:\n' }); + if (url === '/api/ai/status') return Response.json({ enabled: true, configured: false, provider: 'OpenAI', mode: 'Mock', model: 'gpt-5-mini', costProtection: { explicitRequestOnly: true, mockModeConsumesCredits: false } }); + if (url === '/api/github/status') return Response.json({ enabled: false, configured: false, connected: false, mode: 'ReadOnly', allowedRepositoryCount: 0, message: 'Not configured.' }); + return new Response('{}', { status: 404 }); + }; +} + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + sessionStorage.clear(); +}); + +describe('Unlocking live access', () => { + beforeEach(() => { + vi.stubGlobal('fetch', vi.fn(respond())); + }); + + it('accepts a key the API verifies and switches the header to locked mode', async () => { + render(); + await waitFor(() => expect(screen.getByRole('button', { name: 'Unlock live AI and GitHub' })).toBeInTheDocument()); + + fireEvent.click(screen.getByRole('button', { name: 'Unlock live AI and GitHub' })); + fireEvent.change(screen.getByLabelText('X-API-Key'), { target: { value: 'the-key' } }); + fireEvent.click(screen.getByRole('button', { name: 'Unlock' })); + + await waitFor(() => expect(screen.getByRole('button', { name: 'Lock API' })).toBeInTheDocument()); + expect(sessionStorage.getItem('devsecops-sentinel-api-key')).toBe('the-key'); + }); + + it('rejects a key the API refuses and does not keep it', async () => { + // The regression this pins: an unverified key was stored, the header said + // "Lock API" as though it had worked, and the only symptom was GitHub + // quietly staying unavailable. + // The app verifies a candidate key by probing /api/github/status with it. + vi.stubGlobal('fetch', vi.fn(respond({ + '/api/github/status': (init) => { + const headers = new Headers(init?.headers); + return headers.get('X-API-Key') + ? new Response('{"title":"Invalid key"}', { status: 401 }) + : Response.json({ enabled: false, configured: false, connected: false, mode: 'ReadOnly', allowedRepositoryCount: 0, message: 'Not configured.' }); + }, + }))); + + render(); + await waitFor(() => expect(screen.getByRole('button', { name: 'Unlock live AI and GitHub' })).toBeInTheDocument()); + + fireEvent.click(screen.getByRole('button', { name: 'Unlock live AI and GitHub' })); + fireEvent.change(screen.getByLabelText('X-API-Key'), { target: { value: 'wrong' } }); + fireEvent.click(screen.getByRole('button', { name: 'Unlock' })); + + await waitFor(() => expect(screen.getAllByText('That key was not accepted. Check it and try again.').length).toBeGreaterThan(0)); + expect(sessionStorage.getItem('devsecops-sentinel-api-key')).toBeNull(); + expect(screen.queryByRole('button', { name: 'Lock API' })).not.toBeInTheDocument(); + }); + + it('locking again clears the stored key', async () => { + render(); + await waitFor(() => expect(screen.getByRole('button', { name: 'Unlock live AI and GitHub' })).toBeInTheDocument()); + + fireEvent.click(screen.getByRole('button', { name: 'Unlock live AI and GitHub' })); + fireEvent.change(screen.getByLabelText('X-API-Key'), { target: { value: 'the-key' } }); + fireEvent.click(screen.getByRole('button', { name: 'Unlock' })); + await waitFor(() => expect(screen.getByRole('button', { name: 'Lock API' })).toBeInTheDocument()); + + fireEvent.click(screen.getByRole('button', { name: 'Lock API' })); + + await waitFor(() => expect(screen.getByRole('button', { name: 'Unlock live AI and GitHub' })).toBeInTheDocument()); + expect(sessionStorage.getItem('devsecops-sentinel-api-key')).toBeNull(); + }); +}); + +describe('Remediation export', () => { + it('downloads the export under the server-supplied file name', async () => { + vi.stubGlobal('fetch', vi.fn(async (input: RequestInfo | URL) => { + const url = String(input); + if (url.includes('/api/workflows/remediation/export/markdown')) { + return new Response(new Blob(['# report']), { + status: 200, + headers: { 'content-disposition': 'attachment; filename="sample-remediation.md"' }, + }); + } + return new Response('{}', { status: 404 }); + })); + vi.stubGlobal('URL', { + ...URL, + createObjectURL: vi.fn(() => 'blob:url'), + revokeObjectURL: vi.fn(), + }); + const click = vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(() => {}); + + await downloadRemediationExport('sample.yml', 'name: x', 'markdown'); + + expect(click).toHaveBeenCalledTimes(1); + expect(URL.createObjectURL).toHaveBeenCalledTimes(1); + expect(URL.revokeObjectURL).toHaveBeenCalledWith('blob:url'); + }); + + it('a failed export throws with the status instead of downloading', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response('{}', { status: 500 }))); + const click = vi.spyOn(HTMLAnchorElement.prototype, 'click').mockImplementation(() => {}); + + await expect(downloadRemediationExport('sample.yml', 'name: x', 'sarif')) + .rejects.toThrow('Export failed (500)'); + expect(click).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/DevSecOpsSentinel.Api.Integration.Tests/GitHubEndpointTests.cs b/tests/DevSecOpsSentinel.Api.Integration.Tests/GitHubEndpointTests.cs new file mode 100644 index 0000000..f7cafc2 --- /dev/null +++ b/tests/DevSecOpsSentinel.Api.Integration.Tests/GitHubEndpointTests.cs @@ -0,0 +1,229 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using DevSecOpsSentinel.Application; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace DevSecOpsSentinel.Api.Integration.Tests; + +/// +/// The /api/github surface with the reader stubbed: allowlist refusals, the +/// status ladder, analysis of retrieved content — every branch that previously +/// only ran against the real GitHub App. +/// +public sealed class GitHubEndpointTests : IClassFixture +{ + private const string VulnerableYaml = "name: CI\non:\n push:\njobs:\n build:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@v4\n"; + + private readonly HttpClient _client; + + public GitHubEndpointTests(ConfiguredFactory factory) => _client = factory.CreateClient(); + + [Fact] + public async Task Status_reports_connected_when_the_reader_answers() + { + JsonElement status = await Get("/api/github/status"); + + Assert.True(status.GetProperty("connected").GetBoolean(), status.GetRawText()); + Assert.Equal("ReadOnly", status.GetProperty("mode").GetString()); + } + + [Fact] + public async Task Repositories_come_back_allowlisted_only_because_the_reader_already_filtered() + { + HttpResponseMessage response = await _client.GetAsync("/api/github/repositories"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + JsonElement items = JsonDocument.Parse(await response.Content.ReadAsStringAsync()).RootElement; + Assert.Equal(1, items.GetArrayLength()); + Assert.Equal("octo/Sandbox", items[0].GetProperty("fullName").GetString()); + } + + [Fact] + public async Task A_repository_outside_the_allowlist_is_refused_with_403() + { + HttpResponseMessage response = + await _client.GetAsync("/api/github/repositories/octo/Other/workflows"); + + Assert.Equal(HttpStatusCode.Forbidden, response.StatusCode); + } + + [Fact] + public async Task Workflows_list_for_an_allowlisted_repository() + { + JsonElement items = await Get("/api/github/repositories/octo/Sandbox/workflows"); + + Assert.Equal(1, items.GetArrayLength()); + Assert.Equal(".github/workflows/ci.yml", items[0].GetProperty("path").GetString()); + } + + [Fact] + public async Task Workflow_content_returns_the_file_and_a_missing_path_is_404() + { + JsonElement file = await Get( + "/api/github/repositories/octo/Sandbox/workflows/content?path=.github/workflows/ci.yml"); + Assert.Equal(VulnerableYaml, file.GetProperty("content").GetString()); + + HttpResponseMessage missing = await _client.GetAsync( + "/api/github/repositories/octo/Sandbox/workflows/content?path=.github/workflows/nope.yml"); + Assert.Equal(HttpStatusCode.NotFound, missing.StatusCode); + } + + [Fact] + public async Task Analyze_runs_the_deterministic_rules_over_the_retrieved_workflow() + { + HttpResponseMessage response = await _client.PostAsJsonAsync( + "/api/github/repositories/octo/Sandbox/analyze", + new { path = ".github/workflows/ci.yml", useAi = false }); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + string body = await response.Content.ReadAsStringAsync(); + Assert.Contains("GHA001", body); + } + + [Fact] + public async Task Analyze_refuses_a_repository_outside_the_allowlist() + { + HttpResponseMessage response = await _client.PostAsJsonAsync( + "/api/github/repositories/octo/Other/analyze", + new { path = ".github/workflows/ci.yml", useAi = false }); + + Assert.Equal(HttpStatusCode.Forbidden, response.StatusCode); + } + + [Fact] + public async Task Analyze_of_a_missing_workflow_is_404() + { + HttpResponseMessage response = await _client.PostAsJsonAsync( + "/api/github/repositories/octo/Sandbox/analyze", + new { path = ".github/workflows/nope.yml", useAi = false }); + + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + } + + [Fact] + public async Task Analyze_with_ai_returns_the_mock_explanation() + { + HttpResponseMessage response = await _client.PostAsJsonAsync( + "/api/github/repositories/octo/Sandbox/analyze", + new { path = ".github/workflows/ci.yml", useAi = true }); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + string body = await response.Content.ReadAsStringAsync(); + Assert.Contains("explanation", body); + } + + [Fact] + public async Task Status_degrades_to_not_connected_when_the_reader_throws() + { + using BrokenFactory broken = new(); + using HttpClient client = broken.CreateClient(); + + HttpResponseMessage response = await client.GetAsync("/api/github/status"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + JsonElement status = JsonDocument.Parse(await response.Content.ReadAsStringAsync()).RootElement; + Assert.False(status.GetProperty("connected").GetBoolean()); + } + + private async Task Get(string path) + { + HttpResponseMessage response = await _client.GetAsync(path); + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + return JsonDocument.Parse(await response.Content.ReadAsStringAsync()).RootElement; + } + + private static DevSecOpsSentinel.Infrastructure.GitHub.GitHubOptions ConfiguredOptions() => new() + { + Enabled = true, + AppId = 1, + InstallationId = 2, + PrivateKey = "-----BEGIN key", + AllowedRepositories = ["octo/Sandbox"] + }; + + private static Dictionary ConfiguredGitHub() => new() + { + ["OpenAI:Mode"] = "Mock", + ["OpenAI:ApiKey"] = string.Empty, + ["GitHub:Enabled"] = "true", + ["GitHub:AppId"] = "1", + ["GitHub:InstallationId"] = "2", + ["GitHub:PrivateKey"] = "-----BEGIN key", + ["GitHub:AllowedRepositories:0"] = "octo/Sandbox", + ["GitHub:ResolveActionReferences"] = "false", + ["Security:Mode"] = "Disabled", + ["Security:ApiKey"] = string.Empty + }; + + public sealed class ConfiguredFactory : WebApplicationFactory + { + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.UseEnvironment("Testing"); + builder.ConfigureAppConfiguration((_, configuration) => + configuration.AddInMemoryCollection(ConfiguredGitHub())); + builder.ConfigureServices(services => + { + // Program reads GitHubOptions from configuration before Build(), so factory + // configuration lands too late for it. The endpoints resolve the options + // from DI, so replacing the singleton is the supported seam. + services.RemoveAll(); + services.AddSingleton(ConfiguredOptions()); + services.RemoveAll(); + services.AddSingleton(new StubReader(throwOnList: false)); + }); + } + } + + private sealed class BrokenFactory : WebApplicationFactory + { + protected override void ConfigureWebHost(IWebHostBuilder builder) + { + builder.UseEnvironment("Testing"); + builder.ConfigureAppConfiguration((_, configuration) => + configuration.AddInMemoryCollection(ConfiguredGitHub())); + builder.ConfigureServices(services => + { + services.RemoveAll(); + services.AddSingleton(ConfiguredOptions()); + services.RemoveAll(); + services.AddSingleton(new StubReader(throwOnList: true)); + }); + } + } + + private sealed class StubReader(bool throwOnList) : IGitHubRepositoryReader + { + public Task> GetRepositoriesAsync( + CancellationToken cancellationToken) => + throwOnList + ? Task.FromException>( + new HttpRequestException("GitHub unreachable")) + : Task.FromResult>( + [ + new GitHubRepositorySummary( + "octo", "Sandbox", "octo/Sandbox", "main", true, "https://github.test/octo/Sandbox") + ]); + + public Task> GetWorkflowsAsync( + string owner, string repository, CancellationToken cancellationToken) => + Task.FromResult>( + [ + new GitHubWorkflowSummary("ci.yml", ".github/workflows/ci.yml", "abc", "https://github.test/ci") + ]); + + public Task GetWorkflowAsync( + string owner, string repository, string path, string? reference, + CancellationToken cancellationToken) => + Task.FromResult(path.EndsWith("ci.yml", StringComparison.Ordinal) + ? new GitHubWorkflowFile( + owner, repository, reference ?? "main", path, "abc", + VulnerableYaml, "https://github.test/ci", DateTimeOffset.UnixEpoch) + : null); + } +} diff --git a/tests/DevSecOpsSentinel.Application.Tests/WorkflowExplanationServiceTests.cs b/tests/DevSecOpsSentinel.Application.Tests/WorkflowExplanationServiceTests.cs new file mode 100644 index 0000000..691c7a4 --- /dev/null +++ b/tests/DevSecOpsSentinel.Application.Tests/WorkflowExplanationServiceTests.cs @@ -0,0 +1,147 @@ +using DevSecOpsSentinel.Domain; + +namespace DevSecOpsSentinel.Application.Tests; + +/// +/// The routing above the providers: which caller gets which provider, and which +/// situations never reach a provider at all. The second half is the security-relevant +/// one — invalid YAML and un-requested AI must short-circuit before anything that +/// could cost money runs. +/// +public sealed class WorkflowExplanationServiceTests +{ + private sealed class FakeAnalysis(WorkflowAnalysisResult result) : IWorkflowAnalysisService + { + public Task AnalyzeAsync( + WorkflowDocument document, + CancellationToken cancellationToken) => Task.FromResult(result); + } + + private sealed class RecordingProvider(string label) : IWorkflowAiProvider + { + public int Calls { get; private set; } + + public Task ExplainAsync( + WorkflowAnalysisResult analysis, + string sanitizedContent, + CancellationToken cancellationToken) + { + Calls++; + return Task.FromResult(new WorkflowAiExplanation( + $"from {label}", [], "next", [], GeneratedByAi: true, Mode: label)); + } + } + + private sealed class RecordingSelector(IWorkflowAiProvider provider) : IWorkflowAiProviderSelector + { + public AiCallerAccess? SelectedWith { get; private set; } + + public IWorkflowAiProvider Select(AiCallerAccess access) + { + SelectedWith = access; + return provider; + } + } + + private sealed class PassthroughSanitizer : ISensitiveDataSanitizer + { + public SanitizedWorkflow Sanitize(string content) => new(content, WasRedacted: false); + } + + private static WorkflowAnalysisResult Valid() => + new("wf.yml", IsValid: true, [], [], Patch: null); + + private static WorkflowAnalysisResult Invalid() => + new("wf.yml", IsValid: false, ["bad yaml"], [], Patch: null); + + private static WorkflowDocument Document() => new("wf.yml", "name: x\non:\n push:\n"); + + [Fact] + public async Task Invalid_yaml_never_reaches_a_provider() + { + RecordingProvider provider = new("Live"); + RecordingSelector selector = new(provider); + var service = new WorkflowExplanationService( + new FakeAnalysis(Invalid()), selector, new PassthroughSanitizer()); + + WorkflowExplanationResult result = await service.ExplainAsync( + Document(), useAi: true, AiCallerAccess.Configured, CancellationToken.None); + + Assert.Equal(0, provider.Calls); + Assert.Equal("Deterministic", result.Explanation.Mode); + Assert.False(result.Explanation.GeneratedByAi); + } + + [Fact] + public async Task Unrequested_ai_never_reaches_a_provider() + { + RecordingProvider provider = new("Live"); + RecordingSelector selector = new(provider); + var service = new WorkflowExplanationService( + new FakeAnalysis(Valid()), selector, new PassthroughSanitizer()); + + WorkflowExplanationResult result = await service.ExplainAsync( + Document(), useAi: false, AiCallerAccess.Configured, CancellationToken.None); + + Assert.Equal(0, provider.Calls); + Assert.Equal("Disabled", result.Explanation.Mode); + } + + [Theory] + [InlineData(AiCallerAccess.MockOnly)] + [InlineData(AiCallerAccess.Configured)] + public async Task The_callers_access_level_is_what_reaches_the_selector(AiCallerAccess access) + { + // The selector is where "anonymous callers cannot spend" is decided, so the + // access value must arrive exactly as the endpoint stated it. + RecordingProvider provider = new("Selected"); + RecordingSelector selector = new(provider); + var service = new WorkflowExplanationService( + new FakeAnalysis(Valid()), selector, new PassthroughSanitizer()); + + WorkflowExplanationResult result = await service.ExplainAsync( + Document(), useAi: true, access, CancellationToken.None); + + Assert.Equal(access, selector.SelectedWith); + Assert.Equal(1, provider.Calls); + Assert.True(result.Explanation.GeneratedByAi); + } + + [Fact] + public async Task Redaction_flag_travels_from_the_sanitizer_to_the_result() + { + var service = new WorkflowExplanationService( + new FakeAnalysis(Valid()), + new RecordingSelector(new RecordingProvider("Live")), + new RedactingSanitizer()); + + WorkflowExplanationResult result = await service.ExplainAsync( + Document(), useAi: true, AiCallerAccess.Configured, CancellationToken.None); + + Assert.True(result.SensitiveContentRedacted); + } + + private sealed class RedactingSanitizer : ISensitiveDataSanitizer + { + public SanitizedWorkflow Sanitize(string content) => new("[redacted]", WasRedacted: true); + } + + [Fact] + public void The_fallback_carries_every_deterministic_finding() + { + WorkflowAnalysisResult analysis = new( + "wf.yml", IsValid: true, [], + [ + new WorkflowFinding("GHA001", WorkflowSeverity.High, "t", "d", 1, "r", false), + new WorkflowFinding("GHA002", WorkflowSeverity.High, "t", "d", 2, "r", false) + ], + Patch: null); + + WorkflowAiExplanation fallback = AiExplanationFactory.CreateFallback(analysis, "Mode", "why"); + + Assert.Equal(2, fallback.Findings.Count); + Assert.Contains("2 finding(s)", fallback.Summary); + Assert.Equal("why", fallback.FallbackReason); + Assert.False(fallback.GeneratedByAi); + } +} diff --git a/tests/DevSecOpsSentinel.Evals/ContainmentReplayEval.cs b/tests/DevSecOpsSentinel.Evals/ContainmentReplayEval.cs index b13e43e..ef990e7 100644 --- a/tests/DevSecOpsSentinel.Evals/ContainmentReplayEval.cs +++ b/tests/DevSecOpsSentinel.Evals/ContainmentReplayEval.cs @@ -52,6 +52,30 @@ public void Gate_reaches_the_right_verdict(string responseFile) """); } + [Theory] + [MemberData(nameof(Replies))] + public async Task The_full_provider_reaches_the_same_verdict_as_the_gate(string responseFile) + { + // The gate tests prove the comparison; this proves the pipeline around it. Each + // recorded reply is served through the provider's transport seam, so prompt + // assembly, deserialization, the gate and the fallback all run exactly as they do + // against the live API — and the user-visible outcome (a live explanation versus + // the deterministic fallback) must agree with the per-reply verdict. + ReplayEntry entry = ReplayCorpus.Entries.Single(candidate => candidate.ResponseFile == responseFile); + WorkflowAnalysisResult analysis = CorpusEval.AnalyzeForReplay(entry.WorkflowFile); + string reply = File.ReadAllText(Path.Join(ResponsesDirectory, entry.ResponseFile)); + + var provider = new OpenAiWorkflowAiProvider( + new OpenAiOptions { ApiKey = string.Empty, Model = "test", TimeoutSeconds = 5, MaximumContextCharacters = 10_000 }, + Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance, + (_, _, _) => Task.FromResult(reply)); + + WorkflowAiExplanation explanation = + await provider.ExplainAsync(analysis, "sanitized", CancellationToken.None); + + Assert.Equal(entry.ShouldBeAccepted, explanation.GeneratedByAi); + } + [Fact] public void No_invented_rule_id_is_ever_accepted() { diff --git a/tests/DevSecOpsSentinel.Infrastructure.Tests/GitHubActionReferenceResolverTests.cs b/tests/DevSecOpsSentinel.Infrastructure.Tests/GitHubActionReferenceResolverTests.cs new file mode 100644 index 0000000..0d65589 --- /dev/null +++ b/tests/DevSecOpsSentinel.Infrastructure.Tests/GitHubActionReferenceResolverTests.cs @@ -0,0 +1,175 @@ +using System.Net; +using System.Text; +using DevSecOpsSentinel.Application; +using DevSecOpsSentinel.Infrastructure.GitHub; + +namespace DevSecOpsSentinel.Infrastructure.Tests; + +/// +/// Tag-to-SHA resolution against a faked GitHub git-data API. The patch generator +/// pins actions with what this returns, so a wrong answer here becomes a wrong pin +/// in a proposed remediation — the annotated-tag walk and the lightweight-tag path +/// have to agree on the commit they land on. +/// +public sealed class GitHubActionReferenceResolverTests +{ + private const string CommitSha = "1111111111111111111111111111111111111111"; + private const string AnnotatedTagSha = "2222222222222222222222222222222222222222"; + + private static GitHubOptions Options() => new() + { + Enabled = true, + AppId = 1, + InstallationId = 2, + PrivateKey = "-----BEGIN key", + AllowedRepositories = ["octo/Sandbox"], + ApiBaseUrl = "https://api.github.test" + }; + + private sealed class FakeTokens : IGitHubInstallationTokenProvider + { + public Task GetTokenAsync(CancellationToken cancellationToken) => + Task.FromResult("token"); + } + + private sealed class FakeHttp(Func respond) + : HttpMessageHandler, IHttpClientFactory + { + public int Requests { get; private set; } + public HttpClient CreateClient(string name) => new(this); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Requests++; + return Task.FromResult(respond(request)); + } + } + + private static HttpResponseMessage Json(string body) => + new(HttpStatusCode.OK) { Content = new StringContent(body, Encoding.UTF8, "application/json") }; + + private static GitHubActionReferenceResolver Resolver(FakeHttp http) => + new(http, Options(), new FakeTokens()); + + [Theory] + [InlineData("./local/action")] + [InlineData("docker://alpine:3")] + [InlineData("not-a-reference")] + public async Task Local_docker_and_malformed_references_are_unsupported_without_any_request(string reference) + { + FakeHttp http = new(_ => throw new InvalidOperationException("must not be called")); + + ActionReferenceResolutionResult result = + await Resolver(http).ResolveAsync(reference, CancellationToken.None); + + Assert.Equal(ActionReferenceResolutionStatus.Unsupported, result.Status); + Assert.Equal(0, http.Requests); + } + + [Fact] + public async Task An_already_pinned_reference_resolves_to_itself_without_any_request() + { + FakeHttp http = new(_ => throw new InvalidOperationException("must not be called")); + + ActionReferenceResolutionResult result = await Resolver(http) + .ResolveAsync($"actions/checkout@{CommitSha.ToUpperInvariant()}", CancellationToken.None); + + Assert.Equal(ActionReferenceResolutionStatus.Resolved, result.Status); + Assert.Equal(CommitSha, result.CommitSha); + Assert.Equal(0, http.Requests); + } + + [Fact] + public async Task A_lightweight_tag_resolves_straight_to_its_commit() + { + FakeHttp http = new(request => request.RequestUri!.AbsolutePath.Contains("/git/ref/tags/v4") + ? Json($$"""{ "object": { "sha": "{{CommitSha}}", "type": "commit" } }""") + : new HttpResponseMessage(HttpStatusCode.NotFound)); + + ActionReferenceResolutionResult result = + await Resolver(http).ResolveAsync("actions/checkout@v4", CancellationToken.None); + + Assert.Equal(ActionReferenceResolutionStatus.Resolved, result.Status); + Assert.Equal(CommitSha, result.CommitSha); + } + + [Fact] + public async Task An_annotated_tag_is_dereferenced_to_the_commit_it_wraps() + { + // Annotated tags point at a tag object, not the commit. Pinning to the tag + // object's SHA would produce a reference Actions cannot check out. + FakeHttp http = new(request => + { + string path = request.RequestUri!.AbsolutePath; + if (path.Contains("/git/ref/tags/v4")) + return Json($$"""{ "object": { "sha": "{{AnnotatedTagSha}}", "type": "tag" } }"""); + if (path.Contains($"/git/tags/{AnnotatedTagSha}")) + return Json($$"""{ "object": { "sha": "{{CommitSha}}", "type": "commit" } }"""); + return new HttpResponseMessage(HttpStatusCode.NotFound); + }); + + ActionReferenceResolutionResult result = + await Resolver(http).ResolveAsync("actions/checkout@v4", CancellationToken.None); + + Assert.Equal(ActionReferenceResolutionStatus.Resolved, result.Status); + Assert.Equal(CommitSha, result.CommitSha); + } + + [Fact] + public async Task A_branch_reference_falls_back_to_the_heads_lookup() + { + FakeHttp http = new(request => + { + string path = request.RequestUri!.AbsolutePath; + if (path.Contains("/git/ref/tags/main")) + return new HttpResponseMessage(HttpStatusCode.NotFound); + if (path.Contains("/git/ref/heads/main")) + return Json($$"""{ "object": { "sha": "{{CommitSha}}", "type": "commit" } }"""); + return new HttpResponseMessage(HttpStatusCode.NotFound); + }); + + ActionReferenceResolutionResult result = + await Resolver(http).ResolveAsync("actions/checkout@main", CancellationToken.None); + + Assert.Equal(ActionReferenceResolutionStatus.Resolved, result.Status); + Assert.Equal(CommitSha, result.CommitSha); + } + + [Fact] + public async Task A_reference_that_is_neither_tag_nor_branch_reports_not_found() + { + // Not Failed: the lookup worked and the answer is "no such reference". The patch + // generator treats the two differently — NotFound is a wrong tag in the workflow, + // Failed is GitHub being unreachable. + FakeHttp http = new(_ => new HttpResponseMessage(HttpStatusCode.NotFound)); + + ActionReferenceResolutionResult result = + await Resolver(http).ResolveAsync("actions/checkout@nope", CancellationToken.None); + + Assert.Equal(ActionReferenceResolutionStatus.NotFound, result.Status); + Assert.Null(result.CommitSha); + } + + [Fact] + public async Task A_tag_loop_stops_at_the_dereference_ceiling_instead_of_spinning() + { + // A hostile or broken repository can make tag objects point at tag objects + // forever. The resolver must give up, not follow. + FakeHttp http = new(request => + { + string path = request.RequestUri!.AbsolutePath; + if (path.Contains("/git/ref/tags/v4")) + return Json($$"""{ "object": { "sha": "{{AnnotatedTagSha}}", "type": "tag" } }"""); + if (path.Contains("/git/tags/")) + return Json($$"""{ "object": { "sha": "{{AnnotatedTagSha}}", "type": "tag" } }"""); + return new HttpResponseMessage(HttpStatusCode.NotFound); + }); + + ActionReferenceResolutionResult result = + await Resolver(http).ResolveAsync("actions/checkout@v4", CancellationToken.None); + + Assert.NotEqual(ActionReferenceResolutionStatus.Resolved, result.Status); + Assert.True(http.Requests <= 7, $"made {http.Requests} requests"); + } +} diff --git a/tests/DevSecOpsSentinel.Infrastructure.Tests/GitHubInstallationTokenProviderTests.cs b/tests/DevSecOpsSentinel.Infrastructure.Tests/GitHubInstallationTokenProviderTests.cs new file mode 100644 index 0000000..c835576 --- /dev/null +++ b/tests/DevSecOpsSentinel.Infrastructure.Tests/GitHubInstallationTokenProviderTests.cs @@ -0,0 +1,150 @@ +using System.Net; +using System.Security.Cryptography; +using System.Text; +using DevSecOpsSentinel.Infrastructure.GitHub; + +namespace DevSecOpsSentinel.Infrastructure.Tests; + +/// +/// The installation-token exchange: a signed App JWT goes out, a short-lived +/// installation token comes back and is cached until near expiry. The cache is the +/// security-relevant part — every avoidable exchange is an avoidable place for the +/// App credential to travel. +/// +public sealed class GitHubInstallationTokenProviderTests : IDisposable +{ + private readonly string _keyDirectory = + Directory.CreateDirectory(Path.Join(Path.GetTempPath(), Guid.NewGuid().ToString("N"))).FullName; + + private GitHubOptions Options() + { + string keyPath = Path.Join(_keyDirectory, "app.pem"); + using RSA rsa = RSA.Create(2048); + File.WriteAllText(keyPath, rsa.ExportRSAPrivateKeyPem()); + return new GitHubOptions + { + Enabled = true, + AppId = 1, + InstallationId = 42, + PrivateKeyPath = keyPath, + AllowedRepositories = ["octo/Sandbox"], + ApiBaseUrl = "https://api.github.test" + }; + } + + private sealed class FakeHttp(Func respond) + : HttpMessageHandler, IHttpClientFactory + { + public List Requests { get; } = []; + public HttpClient CreateClient(string name) => new(this); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + Requests.Add(request); + return Task.FromResult(respond(request)); + } + } + + private static HttpResponseMessage TokenResponse(string token, DateTimeOffset expires) => + new(HttpStatusCode.Created) + { + Content = new StringContent( + $$"""{ "token": "{{token}}", "expires_at": "{{expires:O}}" }""", + Encoding.UTF8, + "application/json") + }; + + private GitHubInstallationTokenProvider Provider(GitHubOptions options, FakeHttp http) => + new(http, options, new GitHubAppJwtFactory(options, new GitHubPrivateKeySource(options)), + new GitHubPrivateKeySource(options)); + + [Fact] + public async Task Exchanges_a_signed_app_jwt_for_the_installation_token() + { + GitHubOptions options = Options(); + FakeHttp http = new(_ => TokenResponse("inst-token", DateTimeOffset.UtcNow.AddMinutes(50))); + + string token = await Provider(options, http).GetTokenAsync(CancellationToken.None); + + Assert.Equal("inst-token", token); + HttpRequestMessage request = Assert.Single(http.Requests); + Assert.EndsWith("/app/installations/42/access_tokens", request.RequestUri!.AbsolutePath); + // The outgoing credential is the App JWT, not anything cached or stored. + Assert.Equal("Bearer", request.Headers.Authorization?.Scheme); + Assert.Equal(2, request.Headers.Authorization!.Parameter!.Count(c => c == '.')); + } + + [Fact] + public async Task A_fresh_token_is_served_from_cache_without_a_second_exchange() + { + GitHubOptions options = Options(); + FakeHttp http = new(_ => TokenResponse("inst-token", DateTimeOffset.UtcNow.AddMinutes(50))); + GitHubInstallationTokenProvider provider = Provider(options, http); + + await provider.GetTokenAsync(CancellationToken.None); + string second = await provider.GetTokenAsync(CancellationToken.None); + + Assert.Equal("inst-token", second); + Assert.Single(http.Requests); + } + + [Fact] + public async Task A_nearly_expired_token_is_exchanged_again() + { + GitHubOptions options = Options(); + int calls = 0; + FakeHttp http = new(_ => TokenResponse($"token-{++calls}", DateTimeOffset.UtcNow.AddSeconds(5))); + GitHubInstallationTokenProvider provider = Provider(options, http); + + await provider.GetTokenAsync(CancellationToken.None); + string second = await provider.GetTokenAsync(CancellationToken.None); + + // Five seconds of validity is inside any sane refresh margin, so the second + // request must not trust the cache. + Assert.Equal("token-2", second); + Assert.Equal(2, http.Requests.Count); + } + + [Fact] + public async Task Unconfigured_options_are_refused_before_any_request() + { + FakeHttp http = new(_ => throw new InvalidOperationException("must not be called")); + var options = new GitHubOptions(); + var provider = new GitHubInstallationTokenProvider( + http, options, new GitHubAppJwtFactory(options, new GitHubPrivateKeySource(options)), + new GitHubPrivateKeySource(options)); + + await Assert.ThrowsAsync( + () => provider.GetTokenAsync(CancellationToken.None)); + Assert.Empty(http.Requests); + } + + [Fact] + public async Task A_github_error_carries_no_token_and_says_so() + { + GitHubOptions options = Options(); + FakeHttp http = new(_ => new HttpResponseMessage(HttpStatusCode.Unauthorized)); + + await Assert.ThrowsAsync( + () => Provider(options, http).GetTokenAsync(CancellationToken.None)); + } + + [Fact] + public async Task A_success_without_a_token_in_the_body_is_rejected() + { + GitHubOptions options = Options(); + FakeHttp http = new(_ => new HttpResponseMessage(HttpStatusCode.Created) + { + Content = new StringContent("""{ "token": "" }""", Encoding.UTF8, "application/json") + }); + + await Assert.ThrowsAsync( + () => Provider(options, http).GetTokenAsync(CancellationToken.None)); + } + + public void Dispose() + { + try { Directory.Delete(_keyDirectory, recursive: true); } catch { /* best effort */ } + } +} diff --git a/tests/DevSecOpsSentinel.Infrastructure.Tests/GitHubRepositoryReaderTests.cs b/tests/DevSecOpsSentinel.Infrastructure.Tests/GitHubRepositoryReaderTests.cs new file mode 100644 index 0000000..23ff686 --- /dev/null +++ b/tests/DevSecOpsSentinel.Infrastructure.Tests/GitHubRepositoryReaderTests.cs @@ -0,0 +1,187 @@ +using System.Net; +using System.Text; +using DevSecOpsSentinel.Application; +using DevSecOpsSentinel.Infrastructure.GitHub; + +namespace DevSecOpsSentinel.Infrastructure.Tests; + +/// +/// The App-authenticated reader, faked at the HttpClient layer so the real request +/// pipeline runs: URL construction, the bearer token, the allowlist gates, the base64 +/// decode. ADR-004 says this integration is read-only and allowlisted; the allowlist +/// half of that claim lives in this class and is pinned here. +/// +public sealed class GitHubRepositoryReaderTests +{ + private static GitHubOptions Options(params string[] allowed) => new() + { + Enabled = true, + AppId = 1, + InstallationId = 2, + PrivateKey = "-----BEGIN key", + AllowedRepositories = allowed, + ApiBaseUrl = "https://api.github.test" + }; + + private sealed class FakeTokenProvider : IGitHubInstallationTokenProvider + { + public Task GetTokenAsync(CancellationToken cancellationToken) => + Task.FromResult("installation-token"); + } + + private sealed class FakeHttp(Func respond) + : HttpMessageHandler, IHttpClientFactory + { + public List Requests { get; } = []; + + public HttpClient CreateClient(string name) => new(this); + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + Requests.Add(request); + return Task.FromResult(respond(request)); + } + } + + private static HttpResponseMessage Json(string body, HttpStatusCode status = HttpStatusCode.OK) => + new(status) { Content = new StringContent(body, Encoding.UTF8, "application/json") }; + + [Fact] + public async Task Repositories_outside_the_allowlist_are_dropped_even_when_the_installation_grants_them() + { + // The installation and the allowlist are separate gates on purpose: someone + // widening the App's installation must not silently widen this application. + FakeHttp http = new(_ => Json(""" + { "repositories": [ + { "name": "Sandbox", "full_name": "octo/Sandbox", "default_branch": "main", + "private": true, "html_url": "https://github.test/octo/Sandbox", + "owner": { "login": "octo" } }, + { "name": "Other", "full_name": "octo/Other", "default_branch": "main", + "private": true, "html_url": "https://github.test/octo/Other", + "owner": { "login": "octo" } } + ]} + """)); + var reader = new GitHubRepositoryReader(http, Options("octo/Sandbox"), new FakeTokenProvider()); + + IReadOnlyList repositories = + await reader.GetRepositoriesAsync(CancellationToken.None); + + Assert.Equal("octo/Sandbox", Assert.Single(repositories).FullName); + } + + [Fact] + public async Task Requests_carry_the_installation_token_and_the_api_version() + { + FakeHttp http = new(_ => Json("""{ "repositories": [] }""")); + var reader = new GitHubRepositoryReader(http, Options("octo/Sandbox"), new FakeTokenProvider()); + + await reader.GetRepositoriesAsync(CancellationToken.None); + + HttpRequestMessage request = Assert.Single(http.Requests); + Assert.Equal("Bearer", request.Headers.Authorization?.Scheme); + Assert.Equal("installation-token", request.Headers.Authorization?.Parameter); + Assert.Contains("2022-11-28", request.Headers.GetValues("X-GitHub-Api-Version")); + } + + [Fact] + public async Task Workflow_listing_for_a_repository_outside_the_allowlist_is_refused_before_any_request() + { + FakeHttp http = new(_ => throw new InvalidOperationException("must not be called")); + var reader = new GitHubRepositoryReader(http, Options("octo/Sandbox"), new FakeTokenProvider()); + + await Assert.ThrowsAsync( + () => reader.GetWorkflowsAsync("octo", "Other", CancellationToken.None)); + Assert.Empty(http.Requests); + } + + [Fact] + public async Task Unconfigured_integration_is_refused_before_any_request() + { + FakeHttp http = new(_ => throw new InvalidOperationException("must not be called")); + var reader = new GitHubRepositoryReader(http, new GitHubOptions(), new FakeTokenProvider()); + + await Assert.ThrowsAsync( + () => reader.GetRepositoriesAsync(CancellationToken.None)); + Assert.Empty(http.Requests); + } + + [Fact] + public async Task Workflow_listing_returns_only_yaml_files_sorted_by_name() + { + FakeHttp http = new(_ => Json(""" + [ + { "name": "b.yml", "path": ".github/workflows/b.yml", "sha": "b1", "type": "file", + "html_url": "https://github.test/b", "encoding": null, "content": null }, + { "name": "a.yaml", "path": ".github/workflows/a.yaml", "sha": "a1", "type": "file", + "html_url": "https://github.test/a", "encoding": null, "content": null }, + { "name": "README.md", "path": ".github/workflows/README.md", "sha": "r1", "type": "file", + "html_url": "https://github.test/r", "encoding": null, "content": null }, + { "name": "dir.yml", "path": ".github/workflows/dir.yml", "sha": "d1", "type": "dir", + "html_url": "https://github.test/d", "encoding": null, "content": null } + ] + """)); + var reader = new GitHubRepositoryReader(http, Options("octo/Sandbox"), new FakeTokenProvider()); + + IReadOnlyList workflows = + await reader.GetWorkflowsAsync("octo", "Sandbox", CancellationToken.None); + + Assert.Equal(["a.yaml", "b.yml"], workflows.Select(workflow => workflow.Name).ToArray()); + } + + [Fact] + public async Task A_missing_workflows_directory_is_an_empty_list_not_an_error() + { + FakeHttp http = new(_ => new HttpResponseMessage(HttpStatusCode.NotFound)); + var reader = new GitHubRepositoryReader(http, Options("octo/Sandbox"), new FakeTokenProvider()); + + Assert.Empty(await reader.GetWorkflowsAsync("octo", "Sandbox", CancellationToken.None)); + } + + [Fact] + public async Task Workflow_content_is_base64_decoded_with_the_reference_applied() + { + string yaml = "name: CI\non:\n push:\n"; + string encoded = Convert.ToBase64String(Encoding.UTF8.GetBytes(yaml)); + FakeHttp http = new(request => request.RequestUri!.Query.Contains("ref=feature") + ? Json($$""" + { "name": "ci.yml", "path": ".github/workflows/ci.yml", "sha": "abc", "type": "file", + "html_url": "https://github.test/ci", "encoding": "base64", "content": "{{encoded}}" } + """) + : new HttpResponseMessage(HttpStatusCode.NotFound)); + var reader = new GitHubRepositoryReader(http, Options("octo/Sandbox"), new FakeTokenProvider()); + + GitHubWorkflowFile? file = await reader.GetWorkflowAsync( + "octo", "Sandbox", ".github/workflows/ci.yml", "feature", CancellationToken.None); + + Assert.NotNull(file); + Assert.Equal(yaml, file.Content); + Assert.Equal("feature", file.DefaultBranch); + } + + [Fact] + public async Task Missing_workflow_content_is_null_not_an_error() + { + FakeHttp http = new(_ => new HttpResponseMessage(HttpStatusCode.NotFound)); + var reader = new GitHubRepositoryReader(http, Options("octo/Sandbox"), new FakeTokenProvider()); + + Assert.Null(await reader.GetWorkflowAsync( + "octo", "Sandbox", ".github/workflows/ci.yml", null, CancellationToken.None)); + } + + [Fact] + public async Task Content_that_is_not_base64_is_rejected_loudly() + { + // Silent acceptance here would analyze garbage and report it as the repository's + // workflow. Loud is correct. + FakeHttp http = new(_ => Json(""" + { "name": "ci.yml", "path": "p", "sha": "s", "type": "file", + "html_url": "u", "encoding": "utf-8", "content": "plain" } + """)); + var reader = new GitHubRepositoryReader(http, Options("octo/Sandbox"), new FakeTokenProvider()); + + await Assert.ThrowsAsync(() => reader.GetWorkflowAsync( + "octo", "Sandbox", ".github/workflows/ci.yml", null, CancellationToken.None)); + } +} diff --git a/tests/DevSecOpsSentinel.Infrastructure.Tests/OpenAiWorkflowAiProviderTests.cs b/tests/DevSecOpsSentinel.Infrastructure.Tests/OpenAiWorkflowAiProviderTests.cs new file mode 100644 index 0000000..6aeda6b --- /dev/null +++ b/tests/DevSecOpsSentinel.Infrastructure.Tests/OpenAiWorkflowAiProviderTests.cs @@ -0,0 +1,178 @@ +using DevSecOpsSentinel.Domain; +using DevSecOpsSentinel.Infrastructure.Ai; +using Microsoft.Extensions.Logging.Abstractions; + +namespace DevSecOpsSentinel.Infrastructure.Tests; + +/// +/// The live path, end to end, with only the transport substituted. +/// +/// Until the seam existed, everything between "request assembled" and "containment gate" +/// ran only against the real OpenAI API — which is to say it ran in production and nowhere +/// else. These tests drive the exact pipeline production uses: prompt assembly, the +/// timeout envelope, deserialization, the gate, and every fallback branch. +/// +public sealed class OpenAiWorkflowAiProviderTests +{ + private static readonly OpenAiOptions Options = new() + { + ApiKey = string.Empty, + Model = "gpt-5-mini", + TimeoutSeconds = 5, + MaximumContextCharacters = 200 + }; + + private static WorkflowAnalysisResult Analysis(params string[] ruleIds) => + new( + "workflow.yml", + IsValid: true, + ValidationErrors: [], + Findings: [.. ruleIds.Select(id => new WorkflowFinding( + id, WorkflowSeverity.High, $"{id} title", $"{id} description", + LineNumber: 1, Recommendation: $"{id} fix", IsAutomaticallyFixable: false))], + Patch: null); + + private static OpenAiWorkflowAiProvider Provider(OpenAiWorkflowAiProvider.CompleteChat completeChat) => + new(Options, NullLogger.Instance, completeChat); + + private static string ValidReply(params string[] ruleIds) => + System.Text.Json.JsonSerializer.Serialize(new + { + summary = "What the findings mean together.", + findings = ruleIds.Select(id => new + { + ruleId = id, + whyItMatters = "why", + recommendedAction = "action", + confidence = "high" + }), + recommendedNextStep = "Fix the pin first.", + limitations = Array.Empty() + }); + + [Fact] + public async Task A_valid_reply_becomes_a_live_explanation() + { + var provider = Provider((_, _, _) => Task.FromResult(ValidReply("GHA001"))); + + WorkflowAiExplanation explanation = + await provider.ExplainAsync(Analysis("GHA001"), "on: push", CancellationToken.None); + + Assert.True(explanation.GeneratedByAi); + Assert.Equal("Live", explanation.Mode); + Assert.Equal("GHA001", Assert.Single(explanation.Findings).RuleId); + Assert.Null(explanation.FallbackReason); + } + + [Fact] + public async Task The_prompt_carries_the_findings_and_the_sanitized_content() + { + string? prompt = null; + var provider = Provider((messages, _, _) => + { + prompt = messages[^1].Content[0].Text; + return Task.FromResult(ValidReply("GHA002")); + }); + + await provider.ExplainAsync(Analysis("GHA002"), "permissions: write-all", CancellationToken.None); + + Assert.NotNull(prompt); + Assert.Contains("GHA002", prompt); + Assert.Contains("permissions: write-all", prompt); + } + + [Fact] + public async Task Context_beyond_the_configured_maximum_is_truncated_before_it_is_sent() + { + string? prompt = null; + var provider = Provider((messages, _, _) => + { + prompt = messages[^1].Content[0].Text; + return Task.FromResult(ValidReply("GHA001")); + }); + + string oversized = new('x', Options.MaximumContextCharacters + 50); + await provider.ExplainAsync(Analysis("GHA001"), oversized, CancellationToken.None); + + Assert.NotNull(prompt); + Assert.DoesNotContain(oversized, prompt); + Assert.Contains(new string('x', Options.MaximumContextCharacters), prompt); + } + + [Fact] + public async Task A_reply_that_fails_the_gate_degrades_to_the_deterministic_fallback() + { + var provider = Provider((_, _, _) => Task.FromResult(ValidReply("GHA999"))); + + WorkflowAiExplanation explanation = + await provider.ExplainAsync(Analysis("GHA001"), "on: push", CancellationToken.None); + + Assert.False(explanation.GeneratedByAi); + Assert.Equal("OpenAI returned an invalid structured explanation.", explanation.FallbackReason); + // The fallback still explains the real finding; the user loses polish, not facts. + Assert.Equal("GHA001", Assert.Single(explanation.Findings).RuleId); + } + + [Fact] + public async Task A_reply_that_is_not_json_degrades_rather_than_throws() + { + var provider = Provider((_, _, _) => Task.FromResult("I am not JSON.")); + + WorkflowAiExplanation explanation = + await provider.ExplainAsync(Analysis("GHA001"), "on: push", CancellationToken.None); + + Assert.False(explanation.GeneratedByAi); + Assert.NotNull(explanation.FallbackReason); + } + + [Fact] + public async Task A_transport_failure_degrades_to_the_unavailable_fallback() + { + var provider = Provider((_, _, _) => + Task.FromException(new HttpRequestException("boom"))); + + WorkflowAiExplanation explanation = + await provider.ExplainAsync(Analysis("GHA001"), "on: push", CancellationToken.None); + + Assert.False(explanation.GeneratedByAi); + Assert.Equal("The OpenAI provider was unavailable.", explanation.FallbackReason); + } + + [Fact] + public async Task A_request_that_outlives_the_timeout_reports_the_timeout() + { + // The delegate honours the token it is handed — the token the provider's own + // timeout envelope controls. Nothing here waits five seconds; the envelope is + // driven by the option, and the option is one second above the clamp floor. + var provider = new OpenAiWorkflowAiProvider( + new OpenAiOptions { ApiKey = string.Empty, Model = "m", TimeoutSeconds = 5, MaximumContextCharacters = 200 }, + NullLogger.Instance, + async (_, _, cancellationToken) => + { + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + return "unreachable"; + }); + + WorkflowAiExplanation explanation = + await provider.ExplainAsync(Analysis("GHA001"), "on: push", CancellationToken.None); + + Assert.False(explanation.GeneratedByAi); + Assert.Equal("The OpenAI request timed out.", explanation.FallbackReason); + } + + [Fact] + public async Task Without_an_api_key_the_public_constructor_degrades_before_any_request() + { + var provider = new OpenAiWorkflowAiProvider( + new OpenAiOptions { ApiKey = " ", Model = "m" }, + NullLogger.Instance); + + WorkflowAiExplanation explanation = + await provider.ExplainAsync(Analysis("GHA001"), "on: push", CancellationToken.None); + + Assert.False(explanation.GeneratedByAi); + Assert.Equal( + "OpenAI is configured for live mode, but no API key is available.", + explanation.FallbackReason); + } +}