diff --git a/docs/analysis-rules.md b/docs/analysis-rules.md index aee6d11..207fdaa 100644 --- a/docs/analysis-rules.md +++ b/docs/analysis-rules.md @@ -201,6 +201,10 @@ tmforge analyze model.tm7 --rules ./rules.tmrules.json # one spec file tmforge analyze model.tm7 --rules ./rules/ # a directory of specs (searched recursively) ``` +The [starter rule-pack library](../examples/README.md#starter-rule-packs) provides opt-in PCI-inspired, +HIPAA-inspired, and internal-service examples, with a runnable synthetic model, source/control +references, and tested command snippets. These examples do not certify compliance. + To compile an existing MTMT template instead of hand-authoring JSON, use [`rules import`](cli-reference.md#rules): ```bash diff --git a/examples/README.md b/examples/README.md index 6757b35..33d3d34 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,7 +1,8 @@ -# Sample threat models +# Sample threat models and rule packs Small, synthetic threat models used to demo Threat Model Forge, exercise the first-party -[GitHub Action](../action.yml), and dogfood the CLI in [CI](../.github/workflows/ci.yml). +[GitHub Action](../action.yml), and dogfood the CLI in [CI](../.github/workflows/ci.yml), plus +opt-in starter packs for authoring model-specific policies. ## `webshop` @@ -58,3 +59,97 @@ tmforge analyze examples/webshop.tm7 \ CI runs this pair through both the CLI and the first-party Action and requires the same outcome from each, which is what keeps the Action's rule and suppression wiring honest. A suppressed finding is still produced and recorded — it simply stops gating the build. + +## Starter rule packs + +Three small, inspectable version 2 packs demonstrate numeric, regex, Boolean, and connectivity +policies. **These examples do not certify compliance.** They evaluate recorded model properties and +directed connections, not deployed configurations, legal applicability, or control effectiveness. +They are opt-in examples, not new built-in rules. Use a build with the additional matchers described +in [the rule guide](../docs/analysis-rules.md#numeric-and-regex-predicates); older builds reject +unsupported version 2 fields. + +| Pack | Checks | Source and Intent | +| --- | --- | --- | +| [PCI-inspired](rule-packs/pci-inspired.tmrules.json) | Stored PAN encryption; at least 12 months of CDE audit retention. | Inspired by [PCI DSS v4.0.1](https://www.pcisecuritystandards.org/document_library/), 3.5.1 and 10.5.1. The first rule covers encryption-based designs only; other permitted ways to protect PAN require adapting the policy. The second checks duration only, not log availability or completeness. | +| [HIPAA-inspired](rule-packs/hipaa-inspired.tmrules.json) | ePHI flows use an allowed TLS transport and certificate validation; receiving processes have a direct audit-store connection. | Inspired by [45 CFR 164.312(b) and (e)](https://www.hhs.gov/hipaa/for-professionals/security/laws-regulations/index.html). TLS allowlists and direct-store topology are example choices, not the legal text. Addressable encryption specifications still require assessment; this pack does not assess alternatives. | +| [Internal service](rule-packs/internal-service.tmrules.json) | Internal service names follow `svc-name`; externally reachable internal processes connect directly to an audit store. | Illustrative `EXAMPLE-SVC-1`/`EXAMPLE-SVC-2` policies, with [OWASP logging guidance](https://cheatsheetseries.owasp.org/cheatsheets/Logging_Cheat_Sheet.html#event-collection). Naming is hygiene, not a threat or an OWASP mandate. | + +Each file contains two rules, versioned pack identity, control references, remediation guidance, and +scope limitations. PCI uses flat guard/requirement predicates and numeric retention; HIPAA shows +`allOf`/`not` Boolean composition; internal service shows whole-string regex and directed reachability. +All findings are warnings by default. Five rules declare STRIDE metadata and produce triageable +threats; `SERVICE-NAME` is a finding only. Use your existing ruleset/severity policy to choose gates. + +### Record scope and evidence + +Use these **property names** consistently; property lookup and enum comparisons are case-insensitive. +The service-name regex is case-sensitive. Record custom properties on the appropriate objects; +there is no automatic classification or data discovery. + +| Property | Where | Meaning in These Examples | +| --- | --- | --- | +| `StoresPAN` | Datastore | `Yes` opts into the PAN rule; `No` and `Unknown` do not. | +| `Encrypted` | Datastore | The PAN rule accepts `At-rest`, `TDE`, `Client-side`, or `Platform`; `No`, `Unknown`, or missing evidence fails. These labels do not prove cryptographic strength or key management. | +| `AuditScope` | Datastore | `CDE` opts into audit retention; `Other` and `Unknown` do not. This marker denotes an audit store, not every datastore in a CDE. | +| `RetentionMonths` | Datastore | An invariant numeric string of at least `12`. No approximation of a month/year using days is made. | +| `DataClassification` | Flow | `ePHI` opts into HIPAA-inspired rules; `Other` and `Unknown` do not. This custom field is separate from the built-in `DataType` vocabulary. | +| `Protocol`, `CertificateValidation` | Flow | `HTTPS`, `TLS`, or `mTLS`, together with `CertificateValidation=Yes`. Unqualified `gRPC`, for example, does not establish TLS use. | +| `ServiceScope`, `ServiceName` | Process | `ServiceScope=Internal` opts into internal-service rules. Names use the case-sensitive `svc-` prefix and lowercase alphanumeric segments separated by single hyphens, such as `svc-billing-v2`. | +| `StoresLogData` | Datastore | `Yes` qualifies a direct audit destination. A process/collector with the same property does not qualify as a datastore. | + +**Missing or Unknown scope means unassessed, not compliant or proven out of scope.** Within an +explicit scope, missing/Unknown control evidence fails the relevant requirement. Do not clear +findings by removing scope markers or asserting unevidenced controls. + +Connectivity is directed and page-local. Trust boundaries do not block traversal. The audit checks +require a direct outgoing store connection; brokered/collector-based logging needs a policy adapted +to that architecture. HIPAA audit findings are per incoming ePHI flow, not per process, and isolated +processes are not assessed by an interaction rule. See the rule guide for +[graph and input limitations](../docs/analysis-rules.md#directed-connectivity). + +### Try the demonstration model + +[starter-model.tmforge.json](rule-packs/starter-model.tmforge.json) is a deliberately incomplete, +synthetic model that combines independent examples so every rule fires once. It contains no real +payment or health data and is not a recommended architecture. From the repository root: + +```bash +tmforge analyze examples/rule-packs/starter-model.tmforge.json --rules examples/rule-packs/pci-inspired.tmrules.json --max-severity warning +tmforge analyze examples/rule-packs/starter-model.tmforge.json --rules examples/rule-packs/hipaa-inspired.tmrules.json --max-severity warning +tmforge analyze examples/rule-packs/starter-model.tmforge.json --rules examples/rule-packs/internal-service.tmrules.json --max-severity warning +tmforge analyze examples/rule-packs/starter-model.tmforge.json --rules examples/rule-packs --max-severity warning +``` + +Each command exits `2` for findings at the warning threshold, not a tool failure. Each individual +pack adds two findings; the directory loads all three and adds six. Built-in findings are also +reported, so total output is not limited to those counts. Add `--reportFolder ` to retain +the analysis JSON, HTML, and SARIF using the usual CLI workflow. Tests run the command lines above +and check the resulting rule IDs and report identities. + +To satisfy the example policies while retaining their scope declarations, change the demonstration +model as follows (only record such values on a real model when evidenced): + +| Object | Example Change | +| --- | --- | +| Example PAN store | Set `Encrypted=At-rest`. | +| Example audit store | Set `RetentionMonths=12`. | +| Example gateway | Set `ServiceName=svc-gateway`. | +| Example ePHI request | Set `Protocol=TLS` and `CertificateValidation=Yes`. | +| Example gateway to Example audit store | Add a direct outgoing audit flow. The existing route through Example worker is indirect. | + +Tests verify these changes clear the six starter findings without erasing scope markers. This does +not clear every built-in finding or establish compliance. + +### Use Your Own Model + +In Studio, open a model, load a chosen `.tmrules.json` under **Analysis Rules**, and run **Analyze**. +The browser/WASM engine accepts custom pack content; an API-backed Studio uses the server's trusted +startup configuration instead. API operators can point `TmForge__Rules` at the chosen pack or pack +directory, and MCP clients can use the existing sandboxed `rulesPath`. The first-party Action's +`rules` input accepts the same files. No model or pack is uploaded automatically by these examples. + +Fork a pack with your own pack ID and version before adapting it; preserve useful source references +and record the changes from the example policy. Use the existing expected-pack fingerprints to +detect missing or changed policy content. Do not claim these selected checks implement the whole +standard, and do not treat a regex, protocol label, or drawn connection as proof of runtime behavior. diff --git a/examples/rule-packs/hipaa-inspired.tmrules.json b/examples/rule-packs/hipaa-inspired.tmrules.json new file mode 100644 index 0000000..da5964a --- /dev/null +++ b/examples/rule-packs/hipaa-inspired.tmrules.json @@ -0,0 +1,64 @@ +{ + "schema": "tmforge-rules", + "version": 2, + "dialect": "urn:tmforge:rules:interaction-v1", + "pack": { + "id": "example-hipaa", + "name": "HIPAA-inspired starter policy", + "version": "1.0.0", + "description": "Two illustrative model checks inspired by HIPAA technical safeguards, not a complete implementation of the Security Rule. These examples do not certify compliance or verify deployed controls.", + "source": { + "type": "urn:tmforge:source:example-policy", + "name": "HHS Summary of the HIPAA Security Rule; 45 CFR 164.312", + "uri": "https://www.hhs.gov/hipaa/for-professionals/security/laws-regulations/index.html" + } + }, + "properties": [ + { "name": "DataClassification", "allowedValues": ["ePHI", "Other", "Unknown"] }, + { "name": "Protocol", "allowedValues": ["Unknown", "HTTPS", "HTTP", "TLS", "mTLS", "gRPC", "AMQP", "SQL", "SSH", "FTP", "Other"] }, + { "name": "CertificateValidation", "allowedValues": ["Yes", "No", "Unknown"] }, + { "name": "StoresLogData", "allowedValues": ["Yes", "No", "Unknown"] } + ], + "rules": [ + { + "id": "EPHI-TRANSPORT", + "severity": "warning", + "stride": "InformationDisclosure", + "message": "{flow.Name} is marked ePHI but does not evidence both an allowed TLS transport and certificate validation.", + "fullDescription": "Example TLS policy inspired by 45 CFR 164.312(e), transmission security. The allowlist and CertificateValidation=Yes are example design choices, not the legal text. Encryption under 164.312(e)(2)(ii) is addressable: a documented assessment and appropriate implementation or alternative is still required. A protocol label alone cannot prove deployed protection.", + "helpText": "For an explicitly ePHI flow, record Protocol as HTTPS, TLS, or mTLS and CertificateValidation=Yes only when evidenced. Adapt this policy for other justified protections; do not relabel an unencrypted flow or treat Unknown classification as out-of-scope evidence.", + "helpUri": "https://www.hhs.gov/hipaa/for-professionals/security/laws-regulations/index.html", + "provenance": { "sourceId": "45-CFR-164.312(e)", "location": "45 CFR 164.312(e)(1) and (e)(2)(ii); illustrative TLS policy" }, + "expression": { + "allOf": [ + { "subject": "flow", "property": "DataClassification", "valueIn": ["ePHI"] }, + { + "not": { + "allOf": [ + { "subject": "flow", "property": "Protocol", "valueIn": ["HTTPS", "TLS", "mTLS"] }, + { "subject": "flow", "property": "CertificateValidation", "valueIn": ["Yes"] } + ] + } + } + ] + } + }, + { + "id": "EPHI-AUDIT", + "severity": "warning", + "stride": "Repudiation", + "message": "{target.Name} receives ePHI on {flow.Name} but has no direct outgoing connection to a store marked StoresLogData=Yes.", + "fullDescription": "Audit-destination example inspired by 45 CFR 164.312(b), audit controls. A modeled process receiving an explicitly ePHI flow should have an audit destination. A direct store connection is an example topology, not a HIPAA requirement, and does not prove that the required activities are recorded or examined. Findings are per incoming ePHI flow.", + "helpText": "Model the evidenced audit destination and mark the destination StoresLogData=Yes. Adapt the direct-connection policy for collectors or message brokers. This interaction rule does not assess isolated processes or infer ePHI classification from a name or boundary.", + "helpUri": "https://www.hhs.gov/hipaa/for-professionals/security/laws-regulations/index.html", + "provenance": { "sourceId": "45-CFR-164.312(b)", "location": "45 CFR 164.312(b); illustrative audit-destination policy" }, + "expression": { + "allOf": [ + { "subject": "flow", "property": "DataClassification", "valueIn": ["ePHI"] }, + { "subject": "target", "kind": "process" }, + { "not": { "subject": "target", "connectsTo": { "kind": "datastore", "property": "StoresLogData", "equals": "Yes" } } } + ] + } + } + ] +} diff --git a/examples/rule-packs/internal-service.tmrules.json b/examples/rule-packs/internal-service.tmrules.json new file mode 100644 index 0000000..e0d53b6 --- /dev/null +++ b/examples/rule-packs/internal-service.tmrules.json @@ -0,0 +1,51 @@ +{ + "schema": "tmforge-rules", + "version": 2, + "dialect": "urn:tmforge:rules:flat-v1", + "pack": { + "id": "example-internal-service", + "name": "Internal-service starter policy", + "version": "1.0.0", + "description": "Illustrative policies over recorded model properties and connections, not an organization standard. These examples do not certify compliance or verify deployed controls.", + "source": { + "type": "urn:tmforge:source:example-policy", + "name": "Illustrative internal-service policy; logging guidance from OWASP", + "uri": "https://cheatsheetseries.owasp.org/cheatsheets/Logging_Cheat_Sheet.html" + } + }, + "properties": [ + { "name": "ServiceScope", "allowedValues": ["Internal", "External", "Unknown"] }, + { "name": "ServiceName" }, + { "name": "StoresLogData", "allowedValues": ["Yes", "No", "Unknown"] } + ], + "rules": [ + { + "id": "SERVICE-NAME", + "severity": "warning", + "appliesTo": "process", + "message": "{name} is marked Internal but does not record a ServiceName matching the example svc-name convention.", + "fullDescription": "EXAMPLE-SVC-1: Internal services use a svc- prefix and lowercase alphanumeric segments separated by single hyphens. This is an illustrative naming policy, not an OWASP requirement or evidence of a security control.", + "helpText": "Record an evidenced ServiceName such as svc-billing, or adapt the example pattern to your own naming policy. Missing names also fail this requirement.", + "provenance": { "sourceId": "EXAMPLE-SVC-1", "location": "examples/README.md#starter-rule-packs" }, + "when": { "property": "ServiceScope", "equals": "Internal" }, + "assert": { "property": "ServiceName", "matches": "\\Asvc-[a-z0-9]+(?:-[a-z0-9]+)*\\z" } + }, + { + "id": "AUDIT-CONNECTION", + "severity": "warning", + "appliesTo": "process", + "stride": "Repudiation", + "message": "{name} is an externally reachable Internal service with no direct outgoing connection to a store marked StoresLogData=Yes.", + "fullDescription": "EXAMPLE-SVC-2: A service reachable along modeled directed paths from an external entity should have an explicit audit destination. The direct-store topology is an illustrative policy, not a requirement imposed by OWASP. It does not prove that events are actually logged.", + "helpText": "Model the evidenced audit-store connection and mark that store StoresLogData=Yes. Adapt this example for brokered or collector-based logging; an indirect path does not satisfy connectsTo.", + "helpUri": "https://cheatsheetseries.owasp.org/cheatsheets/Logging_Cheat_Sheet.html", + "provenance": { "sourceId": "EXAMPLE-SVC-2", "location": "OWASP Logging Cheat Sheet: Event collection" }, + "when": { + "property": "ServiceScope", + "equals": "Internal", + "reachableFrom": { "kind": "external" } + }, + "assert": { "connectsTo": { "kind": "datastore", "property": "StoresLogData", "equals": "Yes" } } + } + ] +} diff --git a/examples/rule-packs/pci-inspired.tmrules.json b/examples/rule-packs/pci-inspired.tmrules.json new file mode 100644 index 0000000..d7a9a66 --- /dev/null +++ b/examples/rule-packs/pci-inspired.tmrules.json @@ -0,0 +1,51 @@ +{ + "schema": "tmforge-rules", + "version": 2, + "dialect": "urn:tmforge:rules:flat-v1", + "pack": { + "id": "example-pci", + "name": "PCI-inspired starter policy", + "version": "1.0.0", + "description": "Two illustrative model checks inspired by PCI DSS v4.0.1, not a complete implementation of any requirement. These examples do not certify compliance or verify deployed controls.", + "source": { + "type": "urn:tmforge:source:example-policy", + "name": "PCI DSS", + "version": "4.0.1", + "uri": "https://www.pcisecuritystandards.org/document_library/" + } + }, + "properties": [ + { "name": "StoresPAN", "allowedValues": ["Yes", "No", "Unknown"] }, + { "name": "Encrypted", "allowedValues": ["Unknown", "No", "At-rest", "TDE", "Client-side", "Platform"] }, + { "name": "AuditScope", "allowedValues": ["CDE", "Other", "Unknown"] }, + { "name": "RetentionMonths" } + ], + "rules": [ + { + "id": "PAN-ENCRYPTION", + "severity": "warning", + "appliesTo": "datastore", + "stride": "InformationDisclosure", + "message": "{name} is marked StoresPAN=Yes but does not evidence an allowed at-rest encryption mode.", + "fullDescription": "Encryption-focused example inspired by PCI DSS v4.0.1 requirement 3.5.1 (protect stored PAN). The standard also permits non-encryption approaches; this rule intentionally covers only encryption-based designs. An encryption label does not establish strong cryptography, key management, or full requirement satisfaction.", + "helpText": "Record the implemented Encrypted mode as At-rest, TDE, Client-side, or Platform. Adapt the policy for evidenced tokenization, truncation, or hashing designs rather than claiming encryption that is not implemented. Missing or Unknown scope is not assessed.", + "helpUri": "https://www.pcisecuritystandards.org/document_library/", + "provenance": { "sourceId": "PCI-DSS-4.0.1-3.5.1", "location": "PCI DSS v4.0.1, requirement 3.5.1; encryption-only example" }, + "when": { "property": "StoresPAN", "equals": "Yes" }, + "assert": { "property": "Encrypted", "anyOf": ["At-rest", "TDE", "Client-side", "Platform"] } + }, + { + "id": "AUDIT-RETENTION", + "severity": "warning", + "appliesTo": "datastore", + "stride": "Repudiation", + "message": "{name} is marked AuditScope=CDE but does not evidence at least 12 months of audit-log retention.", + "fullDescription": "Retention-duration example inspired by PCI DSS v4.0.1 requirement 10.5.1. It checks only a declared minimum of 12 months, not actual log history, the recent three months' availability, or the other logging requirements. Months are used deliberately rather than approximating a year with 365 days.", + "helpText": "For an audit store in the cardholder data environment, set AuditScope=CDE and record the evidenced RetentionMonths value. A missing, non-numeric, or Unknown duration fails; unclassified stores are not assessed.", + "helpUri": "https://www.pcisecuritystandards.org/document_library/", + "provenance": { "sourceId": "PCI-DSS-4.0.1-10.5.1", "location": "PCI DSS v4.0.1, requirement 10.5.1; minimum duration only" }, + "when": { "property": "AuditScope", "equals": "CDE" }, + "assert": { "property": "RetentionMonths", "greaterThanOrEqual": 12 } + } + ] +} diff --git a/examples/rule-packs/starter-model.tmforge.json b/examples/rule-packs/starter-model.tmforge.json new file mode 100644 index 0000000..859c540 --- /dev/null +++ b/examples/rule-packs/starter-model.tmforge.json @@ -0,0 +1,82 @@ +{ + "schema": "tmforge-json", + "version": "0.1", + "elements": [ + { + "id": "10000000-0000-4000-8000-000000000001", + "kind": "external", + "name": "Example client", + "x": 40, + "y": 80, + "width": 160, + "height": 80 + }, + { + "id": "10000000-0000-4000-8000-000000000002", + "kind": "process", + "name": "Example gateway", + "x": 280, + "y": 80, + "width": 160, + "height": 100, + "properties": { "ServiceScope": "Internal", "ServiceName": "gateway" } + }, + { + "id": "10000000-0000-4000-8000-000000000003", + "kind": "process", + "name": "Example worker", + "x": 520, + "y": 80, + "width": 160, + "height": 100, + "properties": { "ServiceScope": "Internal", "ServiceName": "svc-worker" } + }, + { + "id": "10000000-0000-4000-8000-000000000004", + "kind": "datastore", + "name": "Example audit store", + "x": 760, + "y": 80, + "width": 180, + "height": 100, + "properties": { "StoresLogData": "Yes", "AuditScope": "CDE", "RetentionMonths": "6" } + }, + { + "id": "10000000-0000-4000-8000-000000000005", + "kind": "datastore", + "name": "Example PAN store", + "x": 280, + "y": 340, + "width": 180, + "height": 100, + "properties": { "StoresPAN": "Yes", "Encrypted": "No" } + } + ], + "flows": [ + { + "id": "20000000-0000-4000-8000-000000000001", + "name": "Example ePHI request", + "source": "10000000-0000-4000-8000-000000000001", + "target": "10000000-0000-4000-8000-000000000002", + "properties": { "DataClassification": "ePHI", "Protocol": "HTTP", "CertificateValidation": "No" } + }, + { + "id": "20000000-0000-4000-8000-000000000002", + "name": "Example work request", + "source": "10000000-0000-4000-8000-000000000002", + "target": "10000000-0000-4000-8000-000000000003" + }, + { + "id": "20000000-0000-4000-8000-000000000003", + "name": "Example audit event", + "source": "10000000-0000-4000-8000-000000000003", + "target": "10000000-0000-4000-8000-000000000004" + }, + { + "id": "20000000-0000-4000-8000-000000000004", + "name": "Example payment record", + "source": "10000000-0000-4000-8000-000000000002", + "target": "10000000-0000-4000-8000-000000000005" + } + ] +} diff --git a/test/ThreatModelForge.Api.Tests/ApiCustomRulesTest.cs b/test/ThreatModelForge.Api.Tests/ApiCustomRulesTest.cs index e172350..7a7d262 100644 --- a/test/ThreatModelForge.Api.Tests/ApiCustomRulesTest.cs +++ b/test/ThreatModelForge.Api.Tests/ApiCustomRulesTest.cs @@ -2,6 +2,7 @@ namespace ThreatModelForge.Api.Tests { using System; using System.IO; + using System.Linq; using System.Net; using System.Net.Http; using System.Net.Http.Json; @@ -168,6 +169,41 @@ public async Task AdditionalMatchersMatchTheSharedEngine() Assert.AreEqual(JsonSerializer.Serialize(expected), JsonSerializer.Serialize(actual)); } + /// A trusted starter-pack directory contributes every example to HTTP analysis. + /// A task. + [TestMethod] + public async Task StarterPackDirectoryIsListedAndEnforced() + { + string directory = Path.Join(AppContext.BaseDirectory, "Fixtures", "RulePacks"); + using WebApplicationFactory factory = HostWithRules(directory); + using HttpClient client = factory.CreateClient(); + using HttpResponseMessage bundle = await client.GetAsync("/v1/rule-bundle"); + Assert.AreEqual(HttpStatusCode.OK, bundle.StatusCode); + RuleBundleDto? loaded = await bundle.Content.ReadFromJsonAsync(); + Assert.IsNotNull(loaded); + Assert.AreEqual(0, loaded.Diagnostics.Count); + CollectionAssert.AreEquivalent( + new[] { "example-pci", "example-hipaa", "example-internal-service" }, loaded.RulePacks.Select(pack => pack.Id).ToArray()); + using StringContent content = new StringContent( + File.ReadAllText(Path.Join(directory, "starter-model.tmforge.json")), Encoding.UTF8, "application/json"); + using HttpResponseMessage response = await client.PostAsync("/v1/model/analysis", content); + Assert.AreEqual(HttpStatusCode.OK, response.StatusCode); + AnalysisResultDto? result = await response.Content.ReadFromJsonAsync(); + Assert.IsNotNull(result); + Assert.AreEqual(0, result.Diagnostics.Count); + Assert.IsFalse(result.Findings.Any(finding => finding.Id == "engine-error")); + CollectionAssert.AreEquivalent( + new[] + { + "example-pci/PAN-ENCRYPTION", "example-pci/AUDIT-RETENTION", + "example-hipaa/EPHI-TRANSPORT", "example-hipaa/EPHI-AUDIT", + "example-internal-service/SERVICE-NAME", "example-internal-service/AUDIT-CONNECTION", + }, + result.Findings.Where(finding => finding.RuleId?.StartsWith("example-", StringComparison.Ordinal) == true).Select(finding => finding.RuleId).ToArray()); + Assert.AreEqual(5, result.Threats.Count(threat => threat.RuleId?.StartsWith("example-", StringComparison.Ordinal) == true)); + CollectionAssert.AreEquivalent(loaded.RulePacks.Select(pack => pack.Fingerprint).ToArray(), result.RulePacks.Select(pack => pack.Fingerprint).ToArray()); + } + /// Builds a host that loads rule packs from the given paths. /// The configured rule pack paths. /// The factory. diff --git a/test/ThreatModelForge.Api.Tests/EngineCustomRulesTest.cs b/test/ThreatModelForge.Api.Tests/EngineCustomRulesTest.cs index 372ef11..a7c97ca 100644 --- a/test/ThreatModelForge.Api.Tests/EngineCustomRulesTest.cs +++ b/test/ThreatModelForge.Api.Tests/EngineCustomRulesTest.cs @@ -6,6 +6,7 @@ namespace ThreatModelForge.Api.Tests using System.Linq; using System.Text; using System.Text.Json; + using System.Text.Json.Nodes; using Microsoft.VisualStudio.TestTools.UnitTesting; using ThreatModelForge.Engine; @@ -320,6 +321,285 @@ public void RegexTimeoutIsVisibleOnTheEngineFacade() Assert.IsFalse(result.Findings.Any(finding => finding.RuleId == "TIMEOUT")); } + /// The internal-service starter pack is opt-in and enforces whole-string service names. + /// The recorded service scope, or null when unclassified. + /// The recorded service name, or null when absent. + /// Whether the naming policy should report a finding. + [TestMethod] + [DataRow("Internal", "svc-billing", false)] + [DataRow("Internal", "svc-billing-v2", false)] + [DataRow("Internal", "billing", true)] + [DataRow("Internal", "svc-Billing", true)] + [DataRow("Internal", "svc-billing\n", true)] + [DataRow("Internal", "svc--billing", true)] + [DataRow("Internal", "svc-", true)] + [DataRow("Internal", "Unknown", true)] + [DataRow("Internal", null, true)] + [DataRow("External", "billing", false)] + [DataRow("Unknown", "billing", false)] + [DataRow(null, "billing", false)] + public void StarterServiceNamesAreScoped(string? scope, string? name, bool expected) + { + Dictionary properties = new Dictionary(); + if (scope != null) + { + properties["ServiceScope"] = scope; + } + + if (name != null) + { + properties["ServiceName"] = name; + } + + TmForgeModelDto model = new TmForgeModelDto + { + Elements = new[] { new TmForgeElementDto { Id = "service", Name = "Billing", Kind = "process", Properties = properties } }, + }; + AnalysisResultDto result = EngineService.RunAnalysis(model, StarterRules("internal-service")); + + Assert.AreEqual(0, result.Diagnostics.Count, string.Join("; ", result.Diagnostics)); + Assert.AreEqual(2, result.RulePacks.Single().RuleCount); + Assert.IsFalse(result.Findings.Any(finding => finding.Id == "engine-error")); + Assert.AreEqual(expected ? 1 : 0, result.Findings.Count(finding => finding.RuleId == "example-internal-service/SERVICE-NAME")); + Assert.IsFalse(result.Threats.Any(threat => threat.RuleId == "example-internal-service/SERVICE-NAME")); + } + + /// Each starter pack loads as two opt-in rules with source references and non-certification wording. + /// The pack file name. + /// The published pack id. + [TestMethod] + [DataRow("internal-service", "example-internal-service")] + [DataRow("pci-inspired", "example-pci")] + [DataRow("hipaa-inspired", "example-hipaa")] + public void StarterPacksLoadWithScopeAndProvenance(string name, string id) + { + EngineRuleOptions options = StarterRules(name); + RuleBundleDto result = EngineService.DescribeRules(options); + Assert.AreEqual(0, result.Diagnostics.Count, string.Join("; ", result.Diagnostics)); + RulePackInfoDto pack = result.RulePacks.Single(); + Assert.AreEqual(id, pack.Id); + Assert.AreEqual("1.0.0", pack.Version); + Assert.AreEqual(2, pack.RuleCount); + StringAssert.StartsWith(pack.Fingerprint, "sha256:"); + string? json = options.Sources![0].Json; + Assert.IsNotNull(json); + using JsonDocument document = JsonDocument.Parse(json); + JsonElement root = document.RootElement; + StringAssert.Contains(root.GetProperty("pack").GetProperty("description").GetString() ?? string.Empty, "do not certify compliance"); + Assert.IsTrue(Uri.TryCreate(root.GetProperty("pack").GetProperty("source").GetProperty("uri").GetString(), UriKind.Absolute, out Uri? source)); + Assert.AreEqual("https", source!.Scheme); + foreach (JsonElement rule in root.GetProperty("rules").EnumerateArray()) + { + Assert.IsFalse(string.IsNullOrWhiteSpace(rule.GetProperty("provenance").GetProperty("sourceId").GetString())); + Assert.IsFalse(string.IsNullOrWhiteSpace(rule.GetProperty("provenance").GetProperty("location").GetString())); + Assert.IsFalse(string.IsNullOrWhiteSpace(rule.GetProperty("fullDescription").GetString())); + Assert.IsFalse(string.IsNullOrWhiteSpace(rule.GetProperty("helpText").GetString())); + } + + AnalysisResultDto unclassified = EngineService.RunAnalysis(new TmForgeModelDto(), options); + Assert.AreEqual(0, unclassified.Diagnostics.Count); + Assert.IsFalse(unclassified.Findings.Any(finding => finding.RuleId?.StartsWith(id + "/", StringComparison.Ordinal) == true)); + } + + /// The published demonstration model exercises both rules in every starter pack. + /// The starter pack file. + /// The first expected rule id. + /// The second expected rule id. + [TestMethod] + [DataRow("internal-service", "SERVICE-NAME", "AUDIT-CONNECTION")] + [DataRow("pci-inspired", "PAN-ENCRYPTION", "AUDIT-RETENTION")] + [DataRow("hipaa-inspired", "EPHI-TRANSPORT", "EPHI-AUDIT")] + public void StarterDemonstrationTriggersEveryRule(string name, string first, string second) + { + TmForgeModelDto model = StarterModel(); + string original = JsonSerializer.Serialize(model); + AnalysisResultDto result = AssertStarterFindings(model, name, first, second); + string packId = result.RulePacks.Single().Id; + Assert.AreEqual(name == "internal-service" ? 1 : 2, result.Threats.Count(threat => threat.RuleId?.StartsWith(packId + "/", StringComparison.Ordinal) == true)); + Assert.AreEqual(original, JsonSerializer.Serialize(model)); + } + + /// PCI-inspired PAN protection accepts only evidenced encryption on explicitly scoped stores. + /// Whether PAN storage is declared. + /// The recorded encryption mode. + /// Whether the rule should fire. + [TestMethod] + [DataRow("Yes", "At-rest", false)] + [DataRow("Yes", "TDE", false)] + [DataRow("Yes", "Client-side", false)] + [DataRow("Yes", "Platform", false)] + [DataRow("Yes", "No", true)] + [DataRow("Yes", "Unknown", true)] + [DataRow("Yes", "Tokenized", true)] + [DataRow("Yes", "", true)] + [DataRow("Yes", null, true)] + [DataRow("No", "No", false)] + [DataRow("Unknown", "No", false)] + [DataRow(null, "No", false)] + public void StarterPanEncryptionRequiresEvidence(string? scope, string? encryption, bool expected) + { + TmForgeModelDto model = StarterStore("StoresPAN", scope, "Encrypted", encryption); + AssertStarterFindings(model, "pci-inspired", expected ? new[] { "PAN-ENCRYPTION" } : Array.Empty()); + } + + /// The retention example checks a minimum in months without coercing invalid evidence. + /// The audit scope. + /// The declared retention in months. + /// Whether the rule should fire. + [TestMethod] + [DataRow("CDE", "11.99", true)] + [DataRow("CDE", "12", false)] + [DataRow("CDE", "12.5", false)] + [DataRow("CDE", "24", false)] + [DataRow("CDE", "1.2e1", false)] + [DataRow("CDE", "0", true)] + [DataRow("CDE", "-1", true)] + [DataRow("CDE", "12,0", true)] + [DataRow("CDE", "Unknown", true)] + [DataRow("CDE", "twelve", true)] + [DataRow("CDE", null, true)] + [DataRow("Other", "0", false)] + [DataRow("Unknown", "0", false)] + [DataRow(null, "0", false)] + public void StarterAuditRetentionRequiresTwelveMonths(string? scope, string? months, bool expected) + { + TmForgeModelDto model = StarterStore("AuditScope", scope, "RetentionMonths", months); + AssertStarterFindings(model, "pci-inspired", expected ? new[] { "AUDIT-RETENTION" } : Array.Empty()); + } + + /// Both transport and certificate validation are required on explicitly ePHI flows. + /// The flow classification. + /// The recorded transport. + /// The recorded certificate validation. + /// Whether the rule should fire. + [TestMethod] + [DataRow("ePHI", "HTTPS", "Yes", false)] + [DataRow("ePHI", "TLS", "Yes", false)] + [DataRow("ePHI", "mTLS", "Yes", false)] + [DataRow("ePHI", "HTTP", "Yes", true)] + [DataRow("ePHI", "gRPC", "Yes", true)] + [DataRow("ePHI", "Unknown", "Yes", true)] + [DataRow("ePHI", null, "Yes", true)] + [DataRow("ePHI", "TLS", "No", true)] + [DataRow("ePHI", "TLS", "Unknown", true)] + [DataRow("ePHI", "TLS", null, true)] + [DataRow("ePHI", null, null, true)] + [DataRow("Other", "HTTP", "No", false)] + [DataRow("Unknown", "HTTP", "No", false)] + [DataRow(null, "HTTP", "No", false)] + public void StarterEphiTransportRequiresBothControls(string? classification, string? protocol, string? validation, bool expected) + { + Dictionary properties = new Dictionary(); + foreach ((string key, string? value) in new[] { ("DataClassification", classification), ("Protocol", protocol), ("CertificateValidation", validation) }) + { + if (value != null) + { + properties[key] = value; + } + } + + TmForgeModelDto model = new TmForgeModelDto + { + Elements = new[] + { + new TmForgeElementDto { Id = "source", Kind = "external", Name = "Client" }, + new TmForgeElementDto { Id = "target", Kind = "datastore", Name = "Records" }, + }, + Flows = new[] { new TmForgeFlowDto { Id = "request", Source = "source", Target = "target", Name = "Request", Properties = properties } }, + }; + AssertStarterFindings(model, "hipaa-inspired", expected ? new[] { "EPHI-TRANSPORT" } : Array.Empty()); + } + + /// Audit policies require a direct outgoing connection to an evidenced audit datastore. + /// The modeled audit connection topology. + /// The destination's logging property. + /// The destination's primitive kind. + /// Whether the audit rule should fire. + [TestMethod] + [DataRow("direct", "Yes", "datastore", false)] + [DataRow("direct", "No", "datastore", true)] + [DataRow("direct", "Unknown", "datastore", true)] + [DataRow("direct", null, "datastore", true)] + [DataRow("indirect", "Yes", "datastore", true)] + [DataRow("reverse", "Yes", "datastore", true)] + [DataRow("none", "Yes", "datastore", true)] + [DataRow("direct", "Yes", "process", true)] + public void StarterAuditConnectionsRequireDirectEvidencedStores(string connection, string? marker, string kind, bool expected) + { + TmForgeModelDto model = StarterAuditModel(connection, marker, kind); + AssertStarterFindings(model, "internal-service", expected ? new[] { "AUDIT-CONNECTION" } : Array.Empty()); + AssertStarterFindings(model, "hipaa-inspired", expected ? new[] { "EPHI-AUDIT" } : Array.Empty()); + } + + /// Scope must be recorded; neither example infers scope from names or topology. + /// The internal-service scope. + /// The incoming flow's classification. + [TestMethod] + [DataRow("External", "Other")] + [DataRow("Unknown", "Unknown")] + [DataRow(null, null)] + public void StarterAuditRulesDoNotInferScope(string? scope, string? classification) + { + TmForgeModelDto model = StarterAuditModel("none", "Yes", "datastore", scope, classification); + AssertStarterFindings(model, "internal-service"); + AssertStarterFindings(model, "hipaa-inspired"); + } + + /// Only the internal-service example uses external reachability as its scope guard. + [TestMethod] + public void StarterInternalAuditRequiresAnExternalPath() + { + TmForgeModelDto source = StarterAuditModel("none", "Yes", "datastore"); + TmForgeModelDto isolated = new TmForgeModelDto { Elements = source.Elements, Flows = Array.Empty() }; + AssertStarterFindings(isolated, "internal-service"); + } + + /// The documented remediation clears only starter-policy findings and keeps every scope declaration. + /// The pack file name. + [TestMethod] + [DataRow("internal-service")] + [DataRow("pci-inspired")] + [DataRow("hipaa-inspired")] + public void StarterRemediationSatisfiesTheExamplePolicies(string name) + { + TmForgeModelDto model = StarterModel(satisfyPolicies: true); + AssertStarterFindings(model, name); + Assert.AreEqual("Internal", model.Elements!.Single(element => element.Name == "Example gateway").Properties["ServiceScope"]); + Assert.AreEqual("Yes", model.Elements!.Single(element => element.Name == "Example PAN store").Properties["StoresPAN"]); + Assert.AreEqual("CDE", model.Elements!.Single(element => element.Name == "Example audit store").Properties["AuditScope"]); + Assert.AreEqual("ePHI", model.Flows!.Single(flow => flow.Name == "Example ePHI request").Properties["DataClassification"]); + } + + /// All packs compose without collisions and their threat identities survive model export. + /// The exported format. + [TestMethod] + [DataRow("tmforge-json")] + [DataRow("tm7")] + public void StarterLibraryPreservesFindingsAndThreatsAcrossFormats(string format) + { + EngineRuleOptions rules = new EngineRuleOptions + { + Sources = new[] { "pci-inspired", "hipaa-inspired", "internal-service" }.SelectMany(name => StarterRules(name).Sources!).ToArray(), + }; + TmForgeModelDto model = StarterModel(); + AnalysisResultDto before = EngineService.RunAnalysis(model, rules); + Assert.AreEqual(0, before.Diagnostics.Count); + Assert.AreEqual(3, before.RulePacks.Count); + FindingDto[] findings = before.Findings.Where(finding => finding.RuleId?.StartsWith("example-", StringComparison.Ordinal) == true).ToArray(); + ThreatDto[] threats = before.Threats.Where(threat => threat.RuleId?.StartsWith("example-", StringComparison.Ordinal) == true).ToArray(); + Assert.AreEqual(6, findings.Length); + Assert.AreEqual(5, threats.Length); + + TmForgeModelDto restored = EngineService.ReadModel(EngineService.Convert(model, format, rules), format); + AnalysisResultDto after = EngineService.RunAnalysis(restored, rules); + Assert.AreEqual(0, after.Diagnostics.Count); + Assert.IsFalse(after.Findings.Any(finding => finding.Id == "engine-error")); + CollectionAssert.AreEquivalent(findings.Select(finding => finding.Id).ToArray(), after.Findings + .Where(finding => finding.RuleId?.StartsWith("example-", StringComparison.Ordinal) == true).Select(finding => finding.Id).ToArray()); + CollectionAssert.AreEquivalent(threats.Select(threat => threat.Id).ToArray(), after.Threats + .Where(threat => threat.RuleId?.StartsWith("example-", StringComparison.Ordinal) == true).Select(threat => threat.Id).ToArray()); + } + /// Reads identical rule content and model input for direct-engine and HTTP parity checks. /// The fixture model and rule sources. internal static (TmForgeModelDto Model, EngineRuleOptions Rules) AdditionalMatchers() @@ -334,6 +614,131 @@ internal static (TmForgeModelDto Model, EngineRuleOptions Rules) AdditionalMatch return (model, rules); } + private static TmForgeModelDto StarterModel(bool satisfyPolicies = false) + { + static JsonNode Required(JsonNode node, string key) => node[key] ?? throw new InvalidDataException("The starter example requires " + key + "."); + string file = Path.Join(AppContext.BaseDirectory, "Fixtures", "RulePacks", "starter-model.tmforge.json"); + JsonNode document = JsonNode.Parse(File.ReadAllText(file)) ?? throw new InvalidDataException("The starter example requires a model."); + if (satisfyPolicies) + { + JsonArray elements = Required(document, "elements").AsArray(); + JsonObject gateway = elements.OfType().Single(element => Required(element, "name").GetValue() == "Example gateway"); + JsonObject audit = elements.OfType().Single(element => Required(element, "name").GetValue() == "Example audit store"); + JsonObject pan = elements.OfType().Single(element => Required(element, "name").GetValue() == "Example PAN store"); + Required(gateway, "properties")["ServiceName"] = "svc-gateway"; + Required(audit, "properties")["RetentionMonths"] = "12"; + Required(pan, "properties")["Encrypted"] = "At-rest"; + JsonArray flows = Required(document, "flows").AsArray(); + JsonObject request = flows.OfType().Single(flow => Required(flow, "name").GetValue() == "Example ePHI request"); + Required(request, "properties")["Protocol"] = "TLS"; + Required(request, "properties")["CertificateValidation"] = "Yes"; + flows.Add(new JsonObject + { + ["id"] = "20000000-0000-4000-8000-000000000005", + ["name"] = "Example direct audit event", + ["source"] = Required(gateway, "id").GetValue(), + ["target"] = Required(audit, "id").GetValue(), + }); + } + + return document.Deserialize(new JsonSerializerOptions(JsonSerializerDefaults.Web)) + ?? throw new InvalidDataException("The starter example requires a model."); + } + + private static TmForgeModelDto StarterAuditModel(string connection, string? marker, string kind, string? scope = "Internal", string? classification = "ePHI") + { + Dictionary serviceProperties = new Dictionary { ["ServiceName"] = "svc-service" }; + if (scope != null) + { + serviceProperties["ServiceScope"] = scope; + } + + Dictionary flowProperties = new Dictionary { ["Protocol"] = "TLS", ["CertificateValidation"] = "Yes" }; + if (classification != null) + { + flowProperties["DataClassification"] = classification; + } + + Dictionary logProperties = new Dictionary(); + if (marker != null) + { + logProperties["StoresLogData"] = marker; + } + + List flows = new List + { + new TmForgeFlowDto { Id = "request", Source = "entry", Target = "service", Name = "Request", Properties = flowProperties }, + }; + if (connection != "none") + { + flows.Add(new TmForgeFlowDto + { + Id = "audit", + Source = connection == "reverse" ? "logs" : "service", + Target = connection == "reverse" ? "service" : connection == "indirect" ? "relay" : "logs", + Name = "Audit event", + }); + } + + if (connection == "indirect") + { + flows.Add(new TmForgeFlowDto { Id = "forward", Source = "relay", Target = "logs", Name = "Forward event" }); + } + + return new TmForgeModelDto + { + Elements = new[] + { + new TmForgeElementDto { Id = "entry", Kind = "external", Name = "Entry" }, + new TmForgeElementDto { Id = "service", Kind = "process", Name = "Service", Properties = serviceProperties }, + new TmForgeElementDto { Id = "relay", Kind = "process", Name = "Collector" }, + new TmForgeElementDto { Id = "logs", Kind = kind, Name = "Logs", Properties = logProperties }, + }, + Flows = flows, + }; + } + + private static TmForgeModelDto StarterStore(string scopeName, string? scope, string valueName, string? value) + { + Dictionary properties = new Dictionary(); + if (scope != null) + { + properties[scopeName] = scope; + } + + if (value != null) + { + properties[valueName] = value; + } + + return new TmForgeModelDto + { + Elements = new[] { new TmForgeElementDto { Id = "store", Name = "Store", Kind = "datastore", Properties = properties } }, + }; + } + + private static AnalysisResultDto AssertStarterFindings(TmForgeModelDto model, string name, params string[] expectedRules) + { + AnalysisResultDto result = EngineService.RunAnalysis(model, StarterRules(name)); + Assert.AreEqual(0, result.Diagnostics.Count, string.Join("; ", result.Diagnostics)); + Assert.IsFalse(result.Findings.Any(finding => finding.Id == "engine-error")); + Assert.IsFalse(result.Threats.Any(threat => threat.Id == "engine-error")); + string prefix = result.RulePacks.Single().Id + "/"; + FindingDto[] findings = result.Findings.Where(finding => finding.RuleId?.StartsWith(prefix, StringComparison.Ordinal) == true).ToArray(); + CollectionAssert.AreEquivalent(expectedRules.Select(rule => prefix + rule).ToArray(), findings.Select(finding => finding.RuleId).ToArray()); + Assert.IsTrue(findings.All(finding => finding.Severity == "warning")); + return result; + } + + private static EngineRuleOptions StarterRules(string name) + { + string file = Path.Join(AppContext.BaseDirectory, "Fixtures", "RulePacks", name + ".tmrules.json"); + return new EngineRuleOptions + { + Sources = new[] { new RuleSourceDto { Name = Path.GetFileName(file), Json = File.ReadAllText(file) } }, + }; + } + private static EngineRuleOptions Rules() { return new EngineRuleOptions diff --git a/test/ThreatModelForge.Api.Tests/ThreatModelForge.Api.Tests.csproj b/test/ThreatModelForge.Api.Tests/ThreatModelForge.Api.Tests.csproj index 5bc0dba..db9d82d 100644 --- a/test/ThreatModelForge.Api.Tests/ThreatModelForge.Api.Tests.csproj +++ b/test/ThreatModelForge.Api.Tests/ThreatModelForge.Api.Tests.csproj @@ -23,6 +23,8 @@ + + diff --git a/test/ThreatModelForge.Cli.Tests/AnalyzeAnalysisDocumentTest.cs b/test/ThreatModelForge.Cli.Tests/AnalyzeAnalysisDocumentTest.cs index 0783841..7ee5d3b 100644 --- a/test/ThreatModelForge.Cli.Tests/AnalyzeAnalysisDocumentTest.cs +++ b/test/ThreatModelForge.Cli.Tests/AnalyzeAnalysisDocumentTest.cs @@ -182,6 +182,58 @@ public void AdditionalMatchersAgreeWithTheEngine() Assert.AreEqual(modelJson, File.ReadAllText(modelPath)); } + /// The actual documented starter commands execute every pack and preserve report identities. + [TestMethod] + public void StarterDocumentationCommandsAreExecutable() + { + string examples = Path.Join(AppContext.BaseDirectory, "Fixtures", "Examples"); + string[] commands = File.ReadAllLines(Path.Join(examples, "README.md")) + .Where(line => line.StartsWith("tmforge analyze examples/rule-packs/", StringComparison.Ordinal)).ToArray(); + Assert.AreEqual(4, commands.Length); + Dictionary expected = new Dictionary(StringComparer.Ordinal) + { + ["pci-inspired.tmrules.json"] = new[] { "example-pci/PAN-ENCRYPTION", "example-pci/AUDIT-RETENTION" }, + ["hipaa-inspired.tmrules.json"] = new[] { "example-hipaa/EPHI-TRANSPORT", "example-hipaa/EPHI-AUDIT" }, + ["internal-service.tmrules.json"] = new[] { "example-internal-service/SERVICE-NAME", "example-internal-service/AUDIT-CONNECTION" }, + }; + CollectionAssert.AreEquivalent(expected.Keys.ToArray(), Directory.GetFiles(Path.Join(examples, "rule-packs"), "*.tmrules.json").Select(Path.GetFileName).ToArray()); + HashSet covered = new HashSet(StringComparer.Ordinal); + foreach (string command in commands) + { + string[] words = command.Split(' ', StringSplitOptions.RemoveEmptyEntries); + Assert.AreEqual(7, words.Length, command); + Assert.AreEqual("--rules", words[3], command); + Assert.AreEqual("--max-severity", words[5], command); + Assert.AreEqual("warning", words[6], command); + string[] args = words.Skip(2).Select(word => word.StartsWith("examples/", StringComparison.Ordinal) + ? Path.Join(examples, word.Substring("examples/".Length)) : word).ToArray(); + Assert.IsTrue(covered.Add(words[4]), "A duplicate command does not cover another pack."); + byte[] original = File.ReadAllBytes(args[0]); + Assert.AreEqual(2, Run(args), command); + string reports = Path.Join(this.WorkingDirectory, "snippet-" + covered.Count); + Assert.AreEqual(2, Run(args.Concat(new[] { "--reportFolder", reports }).ToArray()), command); + string reportName = Path.GetFileNameWithoutExtension(args[0]); + using JsonDocument evidence = JsonDocument.Parse(File.ReadAllText(Path.Join(reports, reportName + ".analysis.json"))); + JsonElement[] findings = evidence.RootElement.GetProperty("findings").EnumerateArray() + .Where(finding => finding.GetProperty("ruleId").GetString()?.StartsWith("example-", StringComparison.Ordinal) == true).ToArray(); + string[] ruleIds = Directory.Exists(args[2]) ? expected.Values.SelectMany(ids => ids).ToArray() : expected[Path.GetFileName(args[2])]; + CollectionAssert.AreEquivalent(ruleIds, findings.Select(finding => finding.GetProperty("ruleId").GetString()).ToArray(), command); + foreach (JsonElement finding in findings) + { + string disposition = finding.GetProperty("ruleId").GetString() == "example-internal-service/SERVICE-NAME" ? "hygiene" : "generated-threat"; + Assert.AreEqual(disposition, finding.GetProperty("disposition").GetString()); + } + + using JsonDocument sarif = JsonDocument.Parse(File.ReadAllText(Path.Join(reports, reportName + ".sarif"))); + string[] sarifIds = sarif.RootElement.GetProperty("runs")[0].GetProperty("results").EnumerateArray() + .Where(result => result.GetProperty("ruleId").GetString()?.StartsWith("example-", StringComparison.Ordinal) == true) + .Select(result => result.GetProperty("partialFingerprints").GetProperty("tmforgeFindingId/v1").GetString() ?? string.Empty).ToArray(); + CollectionAssert.AreEquivalent(findings.Select(finding => finding.GetProperty("id").GetString()).ToArray(), sarifIds); + Assert.IsTrue(File.Exists(Path.Join(reports, reportName + ".html"))); + CollectionAssert.AreEqual(original, File.ReadAllBytes(args[0])); + } + } + private static int Run(string[] args) { TextWriter originalOut = Console.Out; diff --git a/test/ThreatModelForge.Cli.Tests/ThreatModelForge.Cli.Tests.csproj b/test/ThreatModelForge.Cli.Tests/ThreatModelForge.Cli.Tests.csproj index 5b2ea12..8a42d66 100644 --- a/test/ThreatModelForge.Cli.Tests/ThreatModelForge.Cli.Tests.csproj +++ b/test/ThreatModelForge.Cli.Tests/ThreatModelForge.Cli.Tests.csproj @@ -21,6 +21,8 @@ + +