From 088abdb5b1ae5b6e0d3b8394b4ad921037970c93 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 19:09:53 +0000 Subject: [PATCH 1/5] Add NIST CMVP module validation status fetcher New nist_cmvp category with one fetcher, nist_cmvp_module_validation_status. It reads the public CMVP certificate page for each certificate number in the `certificates` config and reports whether the module is currently FIPS 140-2 or FIPS 140-3 validated: the page is for that number, Status is Active, and the sunset date has not passed. Anything else is not validated, with a reason code. Tests run against a local HTTP server with fixtures trimmed from the live pages for certificates 4816 (Active), 4282 (Historical) and 1381 (Revoked). docs/ksi_mapping.md is regenerated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WA75NTunK2CFNDjwSpwg8m --- CHANGELOG.md | 9 + README.md | 3 + docs/ksi_mapping.md | 17 +- fetchers/_categories/nist_cmvp.yaml | 23 ++ fetchers/nist_cmvp/README.md | 75 ++++ .../module_validation_status/fetcher.py | 391 ++++++++++++++++++ .../module_validation_status/fetcher.yaml | 60 +++ tests/fixtures/nist_cmvp/cert_1381.html | 134 ++++++ tests/fixtures/nist_cmvp/cert_4282.html | 145 +++++++ tests/fixtures/nist_cmvp/cert_4816.html | 129 ++++++ tests/test_contracts.py | 1 + tests/test_nist_cmvp.py | 249 +++++++++++ 12 files changed, 1231 insertions(+), 5 deletions(-) create mode 100644 fetchers/_categories/nist_cmvp.yaml create mode 100644 fetchers/nist_cmvp/README.md create mode 100644 fetchers/nist_cmvp/module_validation_status/fetcher.py create mode 100644 fetchers/nist_cmvp/module_validation_status/fetcher.yaml create mode 100644 tests/fixtures/nist_cmvp/cert_1381.html create mode 100644 tests/fixtures/nist_cmvp/cert_4282.html create mode 100644 tests/fixtures/nist_cmvp/cert_4816.html create mode 100644 tests/test_nist_cmvp.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 51d2f4979..771280e7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -47,6 +47,15 @@ schemas and the `paramify` CLI — not the internal code. ### Added +- **NIST CMVP category: cryptographic module validation status.** + `nist_cmvp_module_validation_status` reads the public CMVP certificate page + for each certificate number in its `certificates` config and reports whether + the module is currently validated: the page is for that number, Status is + Active, the standard is FIPS 140-2 or FIPS 140-3, and the sunset date has not + passed. Anything else is not validated, with a reason (`historical`, + `revoked`, `sunset_passed`, `status_unknown`, `not_found`, `parse_error` or + `fetch_error`). Maps to KSI-SVC-SIN and KSI-SVC-VRI. No credential. + - **`wiz_vulnerability_findings` 0.4.0: SLA columns.** After the 43 legacy columns (unchanged, so existing intake presets keep mapping), the CSV adds the Wiz fields Paramify's remediation SLAs depend on: `Is Internet Reachable` and diff --git a/README.md b/README.md index 97946d2b2..addf124ca 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,9 @@ If you have a question about this repo, ask DeepWiki. It answers in plain Englis FedRAMP Marketplace
FedRAMP Marketplace Qualys
Qualys + + NIST
NIST CMVP + diff --git a/docs/ksi_mapping.md b/docs/ksi_mapping.md index 042915df7..2379e6132 100644 --- a/docs/ksi_mapping.md +++ b/docs/ksi_mapping.md @@ -144,7 +144,7 @@ python tools/gen_ksi_mapping.py *Controls:* `ac-1`, `ac-17.2`, `cp-9.8`, `sc-8`, `sc-8.1`, `sc-13`, `sc-20`, `sc-21`, `sc-22`, `sc-23`, `sc-28`, `sc-28.1` -*74 fetchers:* [`aws_apigateway_tls_enforcement`](../fetchers/aws/apigateway_tls_enforcement), [`aws_athena_encryption_status`](../fetchers/aws/athena_encryption_status), [`aws_block_storage_encryption_status`](../fetchers/aws/block_storage_encryption_status), [`aws_cloudfront_distribution_security`](../fetchers/aws/cloudfront_distribution_security), [`aws_codeartifact_encryption_status`](../fetchers/aws/codeartifact_encryption_status), [`aws_component_ssl_enforcement_status`](../fetchers/aws/component_ssl_enforcement_status), [`aws_dms_encryption_status`](../fetchers/aws/dms_encryption_status), [`aws_documentdb_encryption_status`](../fetchers/aws/documentdb_encryption_status), [`aws_dynamodb_encryption_status`](../fetchers/aws/dynamodb_encryption_status), [`aws_ebs_snapshot_status`](../fetchers/aws/ebs_snapshot_status), [`aws_efs_encryption_status`](../fetchers/aws/efs_encryption_status), [`aws_elasticache_encryption_status`](../fetchers/aws/elasticache_encryption_status), [`aws_emr_encryption_status`](../fetchers/aws/emr_encryption_status), [`aws_firehose_encryption_status`](../fetchers/aws/firehose_encryption_status), [`aws_fsx_encryption_status`](../fetchers/aws/fsx_encryption_status), [`aws_glacier_encryption_status`](../fetchers/aws/glacier_encryption_status), [`aws_glue_encryption_status`](../fetchers/aws/glue_encryption_status), [`aws_kafka_encryption_status`](../fetchers/aws/kafka_encryption_status), [`aws_kinesis_encryption_status`](../fetchers/aws/kinesis_encryption_status), [`aws_load_balancer_encryption_status`](../fetchers/aws/load_balancer_encryption_status), [`aws_macie_data_discovery`](../fetchers/aws/macie_data_discovery), [`aws_memorydb_encryption_status`](../fetchers/aws/memorydb_encryption_status), [`aws_neptune_encryption_status`](../fetchers/aws/neptune_encryption_status), [`aws_opensearch_encryption_status`](../fetchers/aws/opensearch_encryption_status), [`aws_rds_encryption_status`](../fetchers/aws/rds_encryption_status), [`aws_rds_tls_configuration`](../fetchers/aws/rds_tls_configuration), [`aws_redshift_encryption_status`](../fetchers/aws/redshift_encryption_status), [`aws_s3_encryption_status`](../fetchers/aws/s3_encryption_status), [`aws_sagemaker_encryption_status`](../fetchers/aws/sagemaker_encryption_status), [`aws_sns_encryption_status`](../fetchers/aws/sns_encryption_status), [`aws_sqs_encryption_status`](../fetchers/aws/sqs_encryption_status), [`aws_transfer_tls_enforcement`](../fetchers/aws/transfer_tls_enforcement), [`azure_ai_hub_configuration`](../fetchers/azure/ai_hub_configuration), [`azure_ai_services_configuration`](../fetchers/azure/ai_services_configuration), [`azure_api_management_configuration`](../fetchers/azure/api_management_configuration), [`azure_app_gateway_backend_tls`](../fetchers/azure/app_gateway_backend_tls), [`azure_app_gateway_tls_listeners`](../fetchers/azure/app_gateway_tls_listeners), [`azure_app_service_configuration`](../fetchers/azure/app_service_configuration), [`azure_container_registry_configuration`](../fetchers/azure/container_registry_configuration), [`azure_container_workloads_configuration`](../fetchers/azure/container_workloads_configuration), [`azure_cosmosdb_configuration`](../fetchers/azure/cosmosdb_configuration), [`azure_data_services_configuration`](../fetchers/azure/data_services_configuration), [`azure_databricks_workspace_configuration`](../fetchers/azure/databricks_workspace_configuration), [`azure_disk_encryption_status`](../fetchers/azure/disk_encryption_status), [`azure_dns_configuration`](../fetchers/azure/dns_configuration), [`azure_entra_sign_in_logs`](../fetchers/azure/entra_sign_in_logs), [`azure_front_door_tls`](../fetchers/azure/front_door_tls), [`azure_function_app_configuration`](../fetchers/azure/function_app_configuration), [`azure_key_vault_configuration`](../fetchers/azure/key_vault_configuration), [`azure_key_vault_key_rotation`](../fetchers/azure/key_vault_key_rotation), [`azure_managed_redis_configuration`](../fetchers/azure/managed_redis_configuration), [`azure_mysql_configuration`](../fetchers/azure/mysql_configuration), [`azure_postgresql_configuration`](../fetchers/azure/postgresql_configuration), [`azure_redis_cache_configuration`](../fetchers/azure/redis_cache_configuration), [`azure_sql_encryption_status`](../fetchers/azure/sql_encryption_status), [`azure_sql_managed_instance_configuration`](../fetchers/azure/sql_managed_instance_configuration), [`azure_sql_managed_instance_encryption`](../fetchers/azure/sql_managed_instance_encryption), [`azure_sql_server_configuration`](../fetchers/azure/sql_server_configuration), [`azure_storage_encryption_status`](../fetchers/azure/storage_encryption_status), [`azure_virtual_wan_configuration`](../fetchers/azure/virtual_wan_configuration), [`azure_vm_hardening_status`](../fetchers/azure/vm_hardening_status), [`azure_vpn_gateway_configuration`](../fetchers/azure/vpn_gateway_configuration), [`gcp_bigquery_dataset_configuration`](../fetchers/gcp/bigquery_dataset_configuration), [`gcp_cloud_sql_encryption_status`](../fetchers/gcp/cloud_sql_encryption_status), [`gcp_cloud_storage_encryption_status`](../fetchers/gcp/cloud_storage_encryption_status), [`gcp_dns_configuration`](../fetchers/gcp/dns_configuration), [`gcp_load_balancer_tls_configuration`](../fetchers/gcp/load_balancer_tls_configuration), [`gcp_persistent_disk_encryption_status`](../fetchers/gcp/persistent_disk_encryption_status), [`oci_block_volume_encryption`](../fetchers/oci/block_volume_encryption), [`oci_certificates`](../fetchers/oci/certificates), [`oci_compute_instances`](../fetchers/oci/compute_instances), [`oci_data_service_exposure`](../fetchers/oci/data_service_exposure), [`oci_object_storage_buckets`](../fetchers/oci/object_storage_buckets), [`oci_vault_keys`](../fetchers/oci/vault_keys) +*75 fetchers:* [`aws_apigateway_tls_enforcement`](../fetchers/aws/apigateway_tls_enforcement), [`aws_athena_encryption_status`](../fetchers/aws/athena_encryption_status), [`aws_block_storage_encryption_status`](../fetchers/aws/block_storage_encryption_status), [`aws_cloudfront_distribution_security`](../fetchers/aws/cloudfront_distribution_security), [`aws_codeartifact_encryption_status`](../fetchers/aws/codeartifact_encryption_status), [`aws_component_ssl_enforcement_status`](../fetchers/aws/component_ssl_enforcement_status), [`aws_dms_encryption_status`](../fetchers/aws/dms_encryption_status), [`aws_documentdb_encryption_status`](../fetchers/aws/documentdb_encryption_status), [`aws_dynamodb_encryption_status`](../fetchers/aws/dynamodb_encryption_status), [`aws_ebs_snapshot_status`](../fetchers/aws/ebs_snapshot_status), [`aws_efs_encryption_status`](../fetchers/aws/efs_encryption_status), [`aws_elasticache_encryption_status`](../fetchers/aws/elasticache_encryption_status), [`aws_emr_encryption_status`](../fetchers/aws/emr_encryption_status), [`aws_firehose_encryption_status`](../fetchers/aws/firehose_encryption_status), [`aws_fsx_encryption_status`](../fetchers/aws/fsx_encryption_status), [`aws_glacier_encryption_status`](../fetchers/aws/glacier_encryption_status), [`aws_glue_encryption_status`](../fetchers/aws/glue_encryption_status), [`aws_kafka_encryption_status`](../fetchers/aws/kafka_encryption_status), [`aws_kinesis_encryption_status`](../fetchers/aws/kinesis_encryption_status), [`aws_load_balancer_encryption_status`](../fetchers/aws/load_balancer_encryption_status), [`aws_macie_data_discovery`](../fetchers/aws/macie_data_discovery), [`aws_memorydb_encryption_status`](../fetchers/aws/memorydb_encryption_status), [`aws_neptune_encryption_status`](../fetchers/aws/neptune_encryption_status), [`aws_opensearch_encryption_status`](../fetchers/aws/opensearch_encryption_status), [`aws_rds_encryption_status`](../fetchers/aws/rds_encryption_status), [`aws_rds_tls_configuration`](../fetchers/aws/rds_tls_configuration), [`aws_redshift_encryption_status`](../fetchers/aws/redshift_encryption_status), [`aws_s3_encryption_status`](../fetchers/aws/s3_encryption_status), [`aws_sagemaker_encryption_status`](../fetchers/aws/sagemaker_encryption_status), [`aws_sns_encryption_status`](../fetchers/aws/sns_encryption_status), [`aws_sqs_encryption_status`](../fetchers/aws/sqs_encryption_status), [`aws_transfer_tls_enforcement`](../fetchers/aws/transfer_tls_enforcement), [`azure_ai_hub_configuration`](../fetchers/azure/ai_hub_configuration), [`azure_ai_services_configuration`](../fetchers/azure/ai_services_configuration), [`azure_api_management_configuration`](../fetchers/azure/api_management_configuration), [`azure_app_gateway_backend_tls`](../fetchers/azure/app_gateway_backend_tls), [`azure_app_gateway_tls_listeners`](../fetchers/azure/app_gateway_tls_listeners), [`azure_app_service_configuration`](../fetchers/azure/app_service_configuration), [`azure_container_registry_configuration`](../fetchers/azure/container_registry_configuration), [`azure_container_workloads_configuration`](../fetchers/azure/container_workloads_configuration), [`azure_cosmosdb_configuration`](../fetchers/azure/cosmosdb_configuration), [`azure_data_services_configuration`](../fetchers/azure/data_services_configuration), [`azure_databricks_workspace_configuration`](../fetchers/azure/databricks_workspace_configuration), [`azure_disk_encryption_status`](../fetchers/azure/disk_encryption_status), [`azure_dns_configuration`](../fetchers/azure/dns_configuration), [`azure_entra_sign_in_logs`](../fetchers/azure/entra_sign_in_logs), [`azure_front_door_tls`](../fetchers/azure/front_door_tls), [`azure_function_app_configuration`](../fetchers/azure/function_app_configuration), [`azure_key_vault_configuration`](../fetchers/azure/key_vault_configuration), [`azure_key_vault_key_rotation`](../fetchers/azure/key_vault_key_rotation), [`azure_managed_redis_configuration`](../fetchers/azure/managed_redis_configuration), [`azure_mysql_configuration`](../fetchers/azure/mysql_configuration), [`azure_postgresql_configuration`](../fetchers/azure/postgresql_configuration), [`azure_redis_cache_configuration`](../fetchers/azure/redis_cache_configuration), [`azure_sql_encryption_status`](../fetchers/azure/sql_encryption_status), [`azure_sql_managed_instance_configuration`](../fetchers/azure/sql_managed_instance_configuration), [`azure_sql_managed_instance_encryption`](../fetchers/azure/sql_managed_instance_encryption), [`azure_sql_server_configuration`](../fetchers/azure/sql_server_configuration), [`azure_storage_encryption_status`](../fetchers/azure/storage_encryption_status), [`azure_virtual_wan_configuration`](../fetchers/azure/virtual_wan_configuration), [`azure_vm_hardening_status`](../fetchers/azure/vm_hardening_status), [`azure_vpn_gateway_configuration`](../fetchers/azure/vpn_gateway_configuration), [`gcp_bigquery_dataset_configuration`](../fetchers/gcp/bigquery_dataset_configuration), [`gcp_cloud_sql_encryption_status`](../fetchers/gcp/cloud_sql_encryption_status), [`gcp_cloud_storage_encryption_status`](../fetchers/gcp/cloud_storage_encryption_status), [`gcp_dns_configuration`](../fetchers/gcp/dns_configuration), [`gcp_load_balancer_tls_configuration`](../fetchers/gcp/load_balancer_tls_configuration), [`gcp_persistent_disk_encryption_status`](../fetchers/gcp/persistent_disk_encryption_status), `nist_cmvp_module_validation_status`, [`oci_block_volume_encryption`](../fetchers/oci/block_volume_encryption), [`oci_certificates`](../fetchers/oci/certificates), [`oci_compute_instances`](../fetchers/oci/compute_instances), [`oci_data_service_exposure`](../fetchers/oci/data_service_exposure), [`oci_object_storage_buckets`](../fetchers/oci/object_storage_buckets), [`oci_vault_keys`](../fetchers/oci/vault_keys) #### `KSI-SVC-VCM` — Validating Communications *(optional at Low)* @@ -160,7 +160,7 @@ python tools/gen_ksi_mapping.py *Controls:* `cm-2.2`, `cm-8.3`, `sc-13`, `sc-23`, `si-7`, `si-7.1`, `sr-10` -*9 fetchers:* [`azure_container_registry_configuration`](../fetchers/azure/container_registry_configuration), [`azure_devops_pipelines`](../fetchers/azure/devops_pipelines), [`azure_vm_hardening_status`](../fetchers/azure/vm_hardening_status), [`gcp_compute_instance_configuration`](../fetchers/gcp/compute_instance_configuration), [`gcp_gke_cluster_configuration`](../fetchers/gcp/gke_cluster_configuration), [`github_branch_protection`](../fetchers/github/branch_protection), [`github_release_integrity`](../fetchers/github/release_integrity), [`oci_compute_instances`](../fetchers/oci/compute_instances), [`wiz_file_integrity_monitoring`](../fetchers/wiz/file_integrity_monitoring) +*10 fetchers:* [`azure_container_registry_configuration`](../fetchers/azure/container_registry_configuration), [`azure_devops_pipelines`](../fetchers/azure/devops_pipelines), [`azure_vm_hardening_status`](../fetchers/azure/vm_hardening_status), [`gcp_compute_instance_configuration`](../fetchers/gcp/compute_instance_configuration), [`gcp_gke_cluster_configuration`](../fetchers/gcp/gke_cluster_configuration), [`github_branch_protection`](../fetchers/github/branch_protection), [`github_release_integrity`](../fetchers/github/release_integrity), `nist_cmvp_module_validation_status`, [`oci_compute_instances`](../fetchers/oci/compute_instances), [`wiz_file_integrity_monitoring`](../fetchers/wiz/file_integrity_monitoring) ### MLA — Monitoring, Logging, and Auditing (5/5) @@ -330,7 +330,7 @@ python tools/gen_ksi_mapping.py *Controls:* `cm-2.2`, `cm-7.5`, `cm-8`, `cm-8.1`, `cm-12`, `cm-12.1`, `cp-2.8` -*18 fetchers:* [`aws_detect_new_aws_resource`](../fetchers/aws/detect_new_aws_resource), [`aws_resource_inventory`](../fetchers/aws/resource_inventory), [`azure_entra_service_principals`](../fetchers/azure/entra_service_principals), [`azure_resource_inventory`](../fetchers/azure/resource_inventory), [`azure_sql_managed_instance_configuration`](../fetchers/azure/sql_managed_instance_configuration), [`azure_storage_encryption_status`](../fetchers/azure/storage_encryption_status), [`crowdstrike_hosts`](../fetchers/crowdstrike/hosts), [`datadog_agent_hosts`](../fetchers/datadog/agent_hosts), [`datadog_apm_services`](../fetchers/datadog/apm_services), [`datadog_containers`](../fetchers/datadog/containers), [`github_repository_inventory`](../fetchers/github/repository_inventory), [`gitlab_project_summary`](../fetchers/gitlab/project_summary), [`k8s_eks_pod_inventory`](../fetchers/k8s/eks_pod_inventory), [`rippling_devices`](../fetchers/rippling/devices), [`sentinelone_agents`](../fetchers/sentinelone/agents), [`sentinelone_xdr_assets`](../fetchers/sentinelone/xdr_assets), [`sophos_endpoint_inventory`](../fetchers/sophos/endpoint_inventory), [`wiz_scan_coverage`](../fetchers/wiz/scan_coverage) +*19 fetchers:* [`aws_config_inventory`](../fetchers/aws/config_inventory), [`aws_detect_new_aws_resource`](../fetchers/aws/detect_new_aws_resource), [`aws_resource_inventory`](../fetchers/aws/resource_inventory), [`azure_entra_service_principals`](../fetchers/azure/entra_service_principals), [`azure_resource_inventory`](../fetchers/azure/resource_inventory), [`azure_sql_managed_instance_configuration`](../fetchers/azure/sql_managed_instance_configuration), [`azure_storage_encryption_status`](../fetchers/azure/storage_encryption_status), [`crowdstrike_hosts`](../fetchers/crowdstrike/hosts), [`datadog_agent_hosts`](../fetchers/datadog/agent_hosts), [`datadog_apm_services`](../fetchers/datadog/apm_services), [`datadog_containers`](../fetchers/datadog/containers), [`github_repository_inventory`](../fetchers/github/repository_inventory), [`gitlab_project_summary`](../fetchers/gitlab/project_summary), [`k8s_eks_pod_inventory`](../fetchers/k8s/eks_pod_inventory), [`rippling_devices`](../fetchers/rippling/devices), [`sentinelone_agents`](../fetchers/sentinelone/agents), [`sentinelone_xdr_assets`](../fetchers/sentinelone/xdr_assets), [`sophos_endpoint_inventory`](../fetchers/sophos/endpoint_inventory), [`wiz_scan_coverage`](../fetchers/wiz/scan_coverage) #### `KSI-PIY-RES` — Reviewing Executive Support @@ -418,9 +418,9 @@ python tools/gen_ksi_mapping.py ## By fetcher -329 of 345 fetchers carry a mapping. +331 of 347 fetchers carry a mapping. -### aws (82) +### aws (83) | Fetcher | Indicators | |---|---| @@ -441,6 +441,7 @@ python tools/gen_ksi_mapping.py | [`aws_codepipeline_config`](../fetchers/aws/codepipeline_config) | `KSI-CMT-RMV`, `KSI-CMT-VTD` | | [`aws_component_ssl_enforcement_status`](../fetchers/aws/component_ssl_enforcement_status) | `KSI-SVC-SIN` | | [`aws_config_conformance_packs`](../fetchers/aws/config_conformance_packs) | `KSI-CNA-IBP`, `KSI-MLA-EVC` | +| [`aws_config_inventory`](../fetchers/aws/config_inventory) | `KSI-PIY-GIV` | | [`aws_config_monitoring`](../fetchers/aws/config_monitoring) | `KSI-CMT-LMC`, `KSI-MLA-EVC`, `KSI-SVC-ACM` | | [`aws_database_high_availability`](../fetchers/aws/database_high_availability) | `KSI-CNA-OFA` | | [`aws_detect_new_aws_resource`](../fetchers/aws/detect_new_aws_resource) | `KSI-CMT-LMC`, `KSI-PIY-GIV` | @@ -758,6 +759,12 @@ python tools/gen_ksi_mapping.py | [`knowbe4_module_based_summary`](../fetchers/knowbe4/module_based_summary) | `KSI-CED-RAT` | | [`knowbe4_security_awareness_training`](../fetchers/knowbe4/security_awareness_training) | `KSI-CED-RAT` | +### nist (1) + +| Fetcher | Indicators | +|---|---| +| `nist_cmvp_module_validation_status` | `KSI-SVC-SIN`, `KSI-SVC-VRI` | + ### oci (15) | Fetcher | Indicators | diff --git a/fetchers/_categories/nist_cmvp.yaml b/fetchers/_categories/nist_cmvp.yaml new file mode 100644 index 000000000..5c6602c4e --- /dev/null +++ b/fetchers/_categories/nist_cmvp.yaml @@ -0,0 +1,23 @@ +# NIST Cryptographic Module Validation Program (CMVP) certificates, read from +# the public certificate pages on csrc.nist.gov. +# +# No secrets and no `auth:` block: the certificate pages are public. No +# `requires:` block either: `requests` is a core dependency (pyproject.toml), +# and HTML is parsed with the standard library. + +name: nist_cmvp +display_name: NIST CMVP +description: >- + NIST Cryptographic Module Validation Program certificates: whether each + cryptographic module you list by certificate number is currently FIPS 140-2 + or FIPS 140-3 validated. + +config_schema: + base_url: + type: string + default: https://csrc.nist.gov + env: NIST_CMVP_BASE_URL + description: >- + Base URL the certificate pages are read from, as + /projects/cryptographic-module-validation-program/certificate/. + Change it only to point at a mirror. diff --git a/fetchers/nist_cmvp/README.md b/fetchers/nist_cmvp/README.md new file mode 100644 index 000000000..c1f0cc8bc --- /dev/null +++ b/fetchers/nist_cmvp/README.md @@ -0,0 +1,75 @@ +# NIST CMVP + +This category checks whether the cryptographic modules you depend on are +currently FIPS 140-2 or FIPS 140-3 validated. It reads the public certificate +page that the NIST Cryptographic Module Validation Program (CMVP) publishes for +each module: + +`https://csrc.nist.gov/projects/cryptographic-module-validation-program/certificate/` + +No credential is needed. + +| Fetcher | Certificates come from | Credential | +|---|---|---| +| `nist_cmvp_module_validation_status` | `certificates` config: `4816, 4282` or `AWS-LC=4816, ...` | none | + +## Set it up + +Add the fetcher to your manifest and list your certificate numbers: + +```yaml +run: + fetchers: + - use: nist_cmvp_module_validation_status + config: + certificates: "AWS-LC=4816, 4282" +``` + +Or use the manifest builder: + +```bash +paramify manifest add nist_cmvp_module_validation_status +paramify manifest set-config nist_cmvp_module_validation_status certificates="AWS-LC=4816, 4282" +``` + +The label before `=` is optional. It is copied into the evidence so that a +reader knows what each module is. + +## What a record says + +A certificate is validated only when all of these are true: + +- The page is for that certificate number. +- Status is Active. +- Standard is FIPS 140-2 or FIPS 140-3. +- The sunset date is today or later (UTC). + +Every other certificate is not validated, and `reason` says why: + +| Reason | Meaning | +|---|---| +| `historical` | Status is Historical. | +| `revoked` | Status is Revoked. | +| `sunset_passed` | Status is Active, but the sunset date is before today. All FIPS 140-2 certificates sunset on 9/21/2026. | +| `status_unknown` | Status is missing or has another value, or the standard is not FIPS 140-2 or FIPS 140-3. | +| `not_found` | NIST has no page for the number (HTTP 404), or the page is for a different number. | +| `parse_error` | The page could not be read, for example a missing sunset date. | +| `fetch_error` | The page could not be fetched after retries. | + +Each record also carries the module name, vendor, standard, status, historical +reason, sunset date, overall level, module type, embodiment, caveat, security +level exceptions and validation history. + +A `parse_error` or `fetch_error` on any certificate makes the run exit 1, but +the other certificates are still checked and written. The other reasons are +verdicts, so the run still exits 0. + +## Settings + +| Setting | Env | Default | +|---|---|---| +| `certificates` | `NIST_CMVP_CERTIFICATES` | required | +| `base_url` (category) | `NIST_CMVP_BASE_URL` | `https://csrc.nist.gov` | + +Requests go one at a time, with a 30-second timeout. A timeout, connection +error or HTTP 5xx is retried twice. diff --git a/fetchers/nist_cmvp/module_validation_status/fetcher.py b/fetchers/nist_cmvp/module_validation_status/fetcher.py new file mode 100644 index 000000000..828d03125 --- /dev/null +++ b/fetchers/nist_cmvp/module_validation_status/fetcher.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +"""NIST CMVP validation status for a configured list of cryptographic module certificate numbers.""" + +from __future__ import annotations + +import json +import logging +import os +import re +import sys +import time +from datetime import date, datetime, timezone +from html.parser import HTMLParser +from pathlib import Path +from typing import Any, Dict, Iterator, List, Optional, Tuple + +import requests +from dotenv import load_dotenv + +SCRIPT_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPT_DIR.parents[1] / "_lib")) + +from fetcher_status import report_failure # noqa: E402 + +NAME = "nist_cmvp_module_validation_status" +VERSION = "0.1.0" +logger = logging.getLogger(NAME) + +DEFAULT_BASE_URL = "https://csrc.nist.gov" +CERT_PATH = "/projects/cryptographic-module-validation-program/certificate/{number}" +USER_AGENT = f"paramify-fetchers {NAME}/{VERSION}" +REQUEST_TIMEOUT = 30 +RETRIES = 2 +RETRY_BACKOFF_SECONDS = 2.0 + +REASONS = ( + "historical", "revoked", "sunset_passed", "status_unknown", "not_found", "parse_error", "fetch_error", +) +# Reasons that mean the run could not check a certificate, rather than a verdict on it. +COLLECTION_FAILURES = ("parse_error", "fetch_error") +ACCEPTED_STANDARD = re.compile(r"^FIPS\s*140-[23]$", re.IGNORECASE) +US_DATE = re.compile(r"^(\d{1,2})/(\d{1,2})/(\d{4})$") +CERT_HEADING = re.compile(r"^Certificate\s*#\s*(\d+)$", re.IGNORECASE) +HEADINGS = {"h1", "h2", "h3", "h4", "h5", "h6"} + +# Labels in the certificate page's Details panel. Rows are found by label text, +# not by id or class, so a change to the page's styling does not break them. +DETAIL_LABELS = { + "module name": "module_name", + "standard": "standard", + "status": "status", + "historical reason": "historical_reason", + "sunset date": "sunset_date", + "overall level": "overall_level", + "caveat": "caveat", + "security level exceptions": "security_level_exceptions", + "module type": "module_type", + "embodiment": "embodiment", +} + + +class ConfigError(Exception): + pass + + +# -- a minimal DOM over html.parser ------------------------------------------ + +VOID_TAGS = {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "source", "track", "wbr"} + + +class Node: + def __init__(self, tag: str, attrs: Dict[str, str], parent: Optional["Node"]) -> None: + self.tag, self.attrs, self.parent = tag, attrs, parent + self.children: List[Any] = [] # Node or str + self._text: Optional[str] = None + + @property + def elements(self) -> List["Node"]: + return [c for c in self.children if isinstance(c, Node)] + + def text(self) -> str: + """Whitespace-collapsed text, cached: a certificate page nests thousands of algorithm rows.""" + if self._text is None: + parts = [c if isinstance(c, str) else c.text() for c in self.children if not ( + isinstance(c, Node) and c.tag in ("script", "style"))] + self._text = " ".join(" ".join(parts).split()) + return self._text + + def walk(self) -> Iterator["Node"]: + for child in self.elements: + yield child + yield from child.walk() + + def ancestors(self) -> Iterator["Node"]: + node = self.parent + while node is not None: + yield node + node = node.parent + + +class _TreeBuilder(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.root = Node("#document", {}, None) + self.current = self.root + + def handle_starttag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]) -> None: + node = Node(tag, {k: v or "" for k, v in attrs}, self.current) + self.current.children.append(node) + if tag not in VOID_TAGS: + self.current = node + + def handle_startendtag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]) -> None: + self.current.children.append(Node(tag, {k: v or "" for k, v in attrs}, self.current)) + + def handle_endtag(self, tag: str) -> None: + # Close back to the nearest open element of this tag; ignore a stray end tag. + for node in [self.current, *self.current.ancestors()]: + if node.tag == tag: + self.current = node.parent or self.root + return + + def handle_data(self, data: str) -> None: + self.current.children.append(data) + + +def parse_html(html: str) -> Node: + builder = _TreeBuilder() + builder.feed(html) + builder.close() + return builder.root + + +# -- certificate page ---------------------------------------------------------- + +def _label(text: str) -> str: + return text.strip().rstrip(":").strip().lower() + + +def _section(root: Node, title: str, has_content) -> Optional[Node]: + """The container of the section headed `title`: the nearest ancestor of the heading that holds its content.""" + for heading in root.walk(): + if heading.tag in HEADINGS and _label(heading.text()) == title.lower(): + for ancestor in heading.ancestors(): + if has_content(ancestor): + return ancestor + return None + + +def _label_rows(container: Node) -> Dict[str, Node]: + """Known label -> its value element: any element whose text is a label, followed by a sibling element.""" + rows: Dict[str, Node] = {} + for node in [container, *container.walk()]: + siblings = node.elements + for i, cell in enumerate(siblings[:-1]): + key = DETAIL_LABELS.get(_label(cell.text())) + if key and key not in rows: + rows[key] = siblings[i + 1] + return rows + + +def us_date_to_iso(value: Optional[str]) -> Optional[str]: + match = US_DATE.match((value or "").strip()) + if not match: + return None + month, day, year = (int(g) for g in match.groups()) + try: + return date(year, month, day).isoformat() + except ValueError: + return None + + +def parse_certificate_page(html: str) -> Dict[str, Any]: + """The page's certificate number, Details rows, vendor and validation history. Empty fields are None.""" + root = parse_html(html) + heading = next( + (m for n in root.walk() if n.tag in HEADINGS and (m := CERT_HEADING.match(n.text()))), None + ) + details = _section(root, "Details", lambda n: bool({"status", "module_name"} & set(_label_rows(n)))) + rows = _label_rows(details) if details else {} + page: Dict[str, Any] = { + "page_certificate_number": int(heading.group(1)) if heading else None, + "has_details": details is not None, + } + for key in DETAIL_LABELS.values(): + cell = rows.get(key) + if key == "security_level_exceptions": + items = [li.text() for li in cell.walk() if li.tag == "li"] if cell else [] + page[key] = [i for i in items if i] or ([cell.text()] if cell and cell.text() else []) + else: + page[key] = (cell.text() or None) if cell else None + + vendor = _section(root, "Vendor", lambda n: any(a.tag == "a" for a in n.walk())) + link = next((a for a in vendor.walk() if a.tag == "a"), None) if vendor else None + page["vendor"] = (link.text() or None) if link else None + + history_box = _section(root, "Validation History", lambda n: any(t.tag == "table" for t in n.walk())) + history = [] + if history_box: + for tr in (n for n in history_box.walk() if n.tag == "tr"): + cells = [c.text() for c in tr.elements if c.tag == "td"] + if len(cells) >= 3 and any(cells): + history.append({"date": us_date_to_iso(cells[0]) or cells[0] or None, + "type": cells[1] or None, "lab": cells[2] or None}) + page["validation_history"] = history + return page + + +def verdict(number: int, page: Dict[str, Any], today: date) -> Tuple[Optional[str], str]: + """(reason, detail); reason is None when the certificate is currently validated.""" + if not page["has_details"]: + if page["page_certificate_number"] == number: + return "parse_error", "the page names the certificate but has no Details rows" + return "not_found", "the page has no Details section for a certificate" + if page["page_certificate_number"] is None: + return "parse_error", "the page does not name its certificate number" + if page["page_certificate_number"] != number: + return "not_found", f"the page is for certificate {page['page_certificate_number']}" + status = (page["status"] or "").strip().lower() + if status == "revoked": + return "revoked", "Status is Revoked" + if status == "historical": + return "historical", "Status is Historical" + if status != "active": + return "status_unknown", f"Status is {page['status']!r}, not Active" + if not page["standard"]: + return "parse_error", "the page shows no Standard" + if not ACCEPTED_STANDARD.match(page["standard"]): + return "status_unknown", f"Standard is {page['standard']!r}, not FIPS 140-2 or FIPS 140-3" + sunset = us_date_to_iso(page["sunset_date"]) + if sunset is None: + return "parse_error", f"Sunset Date {page['sunset_date']!r} is not a M/D/YYYY date" + if date.fromisoformat(sunset) < today: + return "sunset_passed", f"Status is Active but the sunset date {sunset} is before {today.isoformat()}" + return None, f"Active {page['standard']} certificate, sunset {sunset}" + + +# -- collection ---------------------------------------------------------------- + +def parse_certificates(raw: str) -> List[Tuple[Optional[str], int]]: + """'AWS-LC=4816, #4282' -> [('AWS-LC', 4816), (None, 4282)].""" + entries: List[Tuple[Optional[str], int]] = [] + for chunk in re.split(r"[,;\n]", raw or ""): + chunk = chunk.strip() + if not chunk: + continue + label, _, number = chunk.rpartition("=") + number = number.strip().lstrip("#").strip() + if not number.isdigit(): + raise ConfigError(f"NIST_CMVP_CERTIFICATES: {chunk!r} is not a certificate number or 'Label=NUMBER'") + entries.append((label.strip() or None, int(number))) + if not entries: + raise ConfigError("NIST_CMVP_CERTIFICATES is empty: list certificate numbers as '4816, 4282'") + return entries + + +def fetch_page(session: requests.Session, url: str) -> Tuple[Optional[int], Optional[str], Optional[str]]: + """(http_status, body, error). Retries timeouts, connection errors and 5xx; one request at a time.""" + error = None + for attempt in range(RETRIES + 1): + if attempt: + time.sleep(RETRY_BACKOFF_SECONDS * attempt) + try: + response = session.get(url, timeout=REQUEST_TIMEOUT) + except requests.exceptions.RequestException as exc: + error = f"GET {url} failed: {type(exc).__name__}: {exc}" + continue + if response.status_code >= 500: + error = f"GET {url} returned HTTP {response.status_code}" + continue + if response.status_code in (200, 404, 410): + return response.status_code, response.text, None + return response.status_code, None, f"GET {url} returned HTTP {response.status_code}" + return None, None, error + + +def check_certificate( + session: requests.Session, base_url: str, label: Optional[str], number: int, today: date +) -> Dict[str, Any]: + url = base_url + CERT_PATH.format(number=number) + record: Dict[str, Any] = {"certificate_number": number, "label": label, "url": url} + status, body, error = fetch_page(session, url) + record["http_status"] = status + page: Dict[str, Any] = {} + reason: Optional[str] + if error: + reason, detail = "fetch_error", error + elif status != 200: + reason, detail = "not_found", f"HTTP {status}" + else: + try: + page = parse_certificate_page(body or "") + reason, detail = verdict(number, page, today) + except Exception as exc: # noqa: BLE001 - one unreadable page must not stop the others + reason, detail = "parse_error", f"{type(exc).__name__}: {exc}" + history = page.get("validation_history") or [] + record.update({ + "validated": reason is None, + "reason": reason, + "reason_detail": detail, + "module_name": page.get("module_name"), + "vendor": page.get("vendor"), + "standard": page.get("standard"), + "status": page.get("status"), + "historical_reason": page.get("historical_reason"), + "sunset_date": us_date_to_iso(page.get("sunset_date")), + "overall_level": page.get("overall_level"), + "module_type": page.get("module_type"), + "embodiment": page.get("embodiment"), + "caveat": page.get("caveat"), + "security_level_exceptions": page.get("security_level_exceptions") or [], + "validation_dates": [h["date"] for h in history if h["date"]], + "validation_history": history, + }) + return record + + +def summarize(records: List[Dict[str, Any]]) -> Dict[str, Any]: + validated = sum(1 for r in records if r["validated"]) + return { + "certificates_total": len(records), + "certificates_validated": validated, + "certificates_not_validated": len(records) - validated, + "not_validated_by_reason": {reason: sum(1 for r in records if r["reason"] == reason) for reason in REASONS}, + } + + +def collect(entries: List[Tuple[Optional[str], int]], base_url: str, now: datetime) -> Tuple[Dict[str, Any], List[str]]: + session = requests.Session() + session.headers.update({"User-Agent": USER_AGENT, "Accept": "text/html"}) + records = [check_certificate(session, base_url, label, number, now.date()) for label, number in entries] + failures = [f"certificate {r['certificate_number']}: {r['reason_detail']}" + for r in records if r["reason"] in COLLECTION_FAILURES] + # summary comes before the records: a validator reads the first match of each count. + evidence: Dict[str, Any] = { + "collected_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + "base_url": base_url, + "run_date": now.date().isoformat(), + "summary": summarize(records), + "certificates": records, + } + if failures: + evidence["metadata"] = { + "partial_failure": True, + "api_failures": [ + {"operation": f"read CMVP certificate {r['certificate_number']}", "type": r["reason"], + "message": r["reason_detail"]} + for r in records if r["reason"] in COLLECTION_FAILURES + ], + } + return evidence, failures + + +def main() -> int: + logging.basicConfig( + level=os.environ.get("LOG_LEVEL", "INFO"), + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + load_dotenv() + output_dir = Path(os.environ.get("EVIDENCE_DIR", "./evidence")) + output_dir.mkdir(parents=True, exist_ok=True) + output_path = output_dir / f"{NAME}.json" + now = datetime.now(timezone.utc) + base_url = (os.environ.get("NIST_CMVP_BASE_URL") or DEFAULT_BASE_URL).strip().rstrip("/") + + try: + entries = parse_certificates(os.environ.get("NIST_CMVP_CERTIFICATES", "")) + except ConfigError as exc: + evidence = { + "collected_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), + "base_url": base_url, + "certificates": [], + "metadata": {"partial_failure": True, + "api_failures": [{"operation": "read config", "type": "bad_config", "message": str(exc)}]}, + } + output_path.write_text(json.dumps(evidence, indent=2)) + report_failure(str(exc), "bad_config") + return 1 + + evidence, failures = collect(entries, base_url, now) + output_path.write_text(json.dumps(evidence, indent=2)) + logger.info("Evidence saved to %s", output_path) + if failures: + code = "target_unreachable" if len(failures) == len(entries) else "partial_failure" + report_failure(f"{len(failures)} of {len(entries)} certificates could not be checked: " + "; ".join(failures), code) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/fetchers/nist_cmvp/module_validation_status/fetcher.yaml b/fetchers/nist_cmvp/module_validation_status/fetcher.yaml new file mode 100644 index 000000000..f17c5ab46 --- /dev/null +++ b/fetchers/nist_cmvp/module_validation_status/fetcher.yaml @@ -0,0 +1,60 @@ +name: nist_cmvp_module_validation_status +version: 0.1.0 +description: >- + Shows whether each listed cryptographic module certificate is currently FIPS + 140-2 or FIPS 140-3 validated, with its status, sunset date and module details. + +category: nist_cmvp + +runtime: + type: python + entry: fetcher.py + +output: + type: json + path: nist_cmvp_module_validation_status.json + +secrets: [] # NIST CMVP certificate pages are public + +config_schema: + certificates: + type: string + required: true + env: NIST_CMVP_CERTIFICATES + description: >- + Comma-separated CMVP certificate numbers, for example `4816, 4282`. A + number can carry your own label as `Label=NUMBER`, for example + `AWS-LC=4816`; the label is copied into the evidence so a reader knows + what each module is. + +ksis: + - KSI-SVC-SIN + - KSI-SVC-VRI + +evidence_set: + reference_id: EVD-NIST-CMVP-MODULE-VALIDATION + name: NIST CMVP Module Validation Status + instructions: | + Script: nist_cmvp/module_validation_status/fetcher.py + + Access: None. The certificate pages on csrc.nist.gov are public. + + Commands Executed: + - GET /projects/cryptographic-module-validation-program/certificate/ — reads the certificate page (per certificate) + + Output: nist_cmvp_module_validation_status.json — one record per certificate with its module name, vendor, standard, status, sunset date and verdict. + + Summary fields: + + Is every listed module currently validated? + - certificates_total — Number of certificates checked + - certificates_validated — Number of certificates that are validated: the page is for that number, Status is Active, the standard is FIPS 140-2 or FIPS 140-3, and the sunset date is today or later + - certificates_not_validated — Number of certificates that are not validated + - not_validated_by_reason — Number of not-validated certificates for each reason: + - historical (Status is Historical) + - revoked (Status is Revoked) + - sunset_passed (Status is Active but the sunset date is before today) + - status_unknown (Status is missing or another value, or the standard is not FIPS 140-2 or 140-3) + - not_found (NIST has no page for the number, or the page is for another number) + - parse_error (the page could not be read, for example a missing sunset date) + - fetch_error (the page could not be fetched) diff --git a/tests/fixtures/nist_cmvp/cert_1381.html b/tests/fixtures/nist_cmvp/cert_1381.html new file mode 100644 index 000000000..a81124f42 --- /dev/null +++ b/tests/fixtures/nist_cmvp/cert_1381.html @@ -0,0 +1,134 @@ + + +Cryptographic Module Validation Program | CSRC + +
+
+
+
+

Certificate #1381

+ + +
+
+
+
+
+
+
+

Details

+
+
+
+
+ Module Name +
+
Accellion Secure File Transfer Cryptographic Module
+
+
+
Standard
+
FIPS 140-2
+
+
+
Status
+
Revoked
+
+
+
 Historical Reason
+
Revoked
+
+
+
+ Overall Level +
+
1
+
+
+
+ Caveat +
+
+ None +
+
+
+
+ Security Level Exceptions +
+
+
    +
  • Design Assurance: Level 2
  • +
+
+
+
+
+ Module Type +
+
Software
+
+
+
+ Embodiment +
+
Multi-chip standalone
+
+
+
+
+
+
+
+
+
+

Vendor

+
+
+ Accellion, Inc. +
+ 1900 Embarcadero Road, Suite 207 +
+ Palo Alto, CA 94303 +
+ USA +
+
+
+
+
+
+
+
+

Validation History

+
+
+ + + + + + + + + + + + + + + +
DateTypeLab
8/12/2010InitialUL Verification Services, Inc.
+
+
+
+
+
+ + diff --git a/tests/fixtures/nist_cmvp/cert_4282.html b/tests/fixtures/nist_cmvp/cert_4282.html new file mode 100644 index 000000000..e1abe4609 --- /dev/null +++ b/tests/fixtures/nist_cmvp/cert_4282.html @@ -0,0 +1,145 @@ + + +Cryptographic Module Validation Program | CSRC + +
+
+
+
+

Certificate #4282

+ +
+
+
+
+
+
+
+

Details

+
+
+
+
+ Module Name +
+
OpenSSL FIPS Provider
+
+
+
Standard
+
FIPS 140-2
+
+
+
Status
+
Historical
+
+
+
 Historical Reason
+
Moved to historical list due to sunsetting
+
+
+
+ Overall Level +
+
1
+
+
+
+ Caveat +
+
+ When operated in FIPS mode. No assurance of the minimum strength of generated keys. +
+
+
+
+ Security Level Exceptions +
+
+
    +
  • Physical Security: N/A
  • +
  • Design Assurance: Level 3
  • +
+
+
+
+
+ Module Type +
+
Software
+
+
+
+ Embodiment +
+
Multi-Chip Stand Alone
+
+
+
+
+
+
+
+
+
+

Vendor

+
+ +
+
+
+
+
+

Validation History

+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
DateTypeLab
8/23/2022InitialAcumen Security
3/10/2023UpdateAcumen Security
5/23/2023UpdateAcumen Security
1/22/2024UpdateAcumen Security
7/10/2024UpdateAcumen Security
+
+
+
+
+
+ + diff --git a/tests/fixtures/nist_cmvp/cert_4816.html b/tests/fixtures/nist_cmvp/cert_4816.html new file mode 100644 index 000000000..57c62d6e9 --- /dev/null +++ b/tests/fixtures/nist_cmvp/cert_4816.html @@ -0,0 +1,129 @@ + + +Cryptographic Module Validation Program | CSRC + +
+
+
+
+

Certificate #4816

+
+
+
+
+
+
+
+

Details

+
+
+
+
+ Module Name +
+
AWS-LC Cryptographic Module (static)
+
+
+
Standard
+
FIPS 140-3
+
+
+
Status
+
Active
+
+
+
+ Sunset Date +
+
9/30/2029
+
+
+
+ Overall Level +
+
1
+
+
+
+ Caveat +
+
+ When operated in approved mode. No assurance of the minimum strength of generated SSPs (e.g., keys) +
+
+
+
+ Security Level Exceptions +
+
+
    +
  • Physical security: N/A
  • +
  • Non-invasive security: N/A
  • +
+
+
+
+
+ Module Type +
+
Software
+
+
+
+ Embodiment +
+
Multi-Chip Stand Alone
+
+
+
+
+
+
+
+
+
+

Vendor

+
+
+ Amazon Web Services Inc. +
+ 1918 8th Ave. +
+ Seattle, WA 98121 +
+ USA +
+
+
+
+
+
+
+
+

Validation History

+
+
+ + + + + + + + + + + + + + + +
DateTypeLab
10/1/2024Initialatsec information security corporation
+
+
+
+
+
+ + diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 181753c90..07c6a5f12 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -319,6 +319,7 @@ def test_instructions_open_with_the_entry_path() -> None: "azure/vpn_gateway_configuration", "azure/waf_ddos_activity", "azure/zone_redundancy", + "nist_cmvp", "oci", "qualys", "sophos", diff --git a/tests/test_nist_cmvp.py b/tests/test_nist_cmvp.py new file mode 100644 index 000000000..f21180da6 --- /dev/null +++ b/tests/test_nist_cmvp.py @@ -0,0 +1,249 @@ +"""nist_cmvp_module_validation_status against a local HTTP server. + +The pages under tests/fixtures/nist_cmvp/ are trimmed from the live certificate +pages on csrc.nist.gov (4816 Active, 4282 Historical, 1381 Revoked), as served +on 2026-11-13. The other cases are edits of the 4816 page. +""" + +from __future__ import annotations + +import importlib.util +import json +import socket +import threading +from datetime import datetime, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Dict, Tuple + +import pytest + +REPO_ROOT = Path(__file__).resolve().parent.parent +FIXTURES = REPO_ROOT / "tests" / "fixtures" / "nist_cmvp" +_spec = importlib.util.spec_from_file_location( + "nist_cmvp_module_validation_status", + REPO_ROOT / "fetchers" / "nist_cmvp" / "module_validation_status" / "fetcher.py", +) +cmvp = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(cmvp) + +NOW = datetime(2026, 11, 13, 18, 0, tzinfo=timezone.utc) +ACTIVE = (FIXTURES / "cert_4816.html").read_text() +HISTORICAL = (FIXTURES / "cert_4282.html").read_text() +REVOKED = (FIXTURES / "cert_1381.html").read_text() + + +def edit(page: str, number: int, **swap: str) -> str: + """The 4816 page renumbered, with each old=new text swap applied.""" + page = page.replace("Certificate #4816", f"Certificate #{number}") + for old, new in swap.values(): + assert old in page, old + page = page.replace(old, new) + return page + + +STATUS_ROW = '
Status
\n
Active
' +PAGES: Dict[int, str] = { + 4816: ACTIVE, + 4282: HISTORICAL, + 1381: REVOKED, + # FIPS 140-2 certificates sunset on 9/21/2026; one still shown Active must not pass. + 1001: edit(ACTIVE, 1001, std=("FIPS 140-3", "FIPS 140-2"), sunset=("9/30/2029", "9/21/2026")), + 1002: edit(ACTIVE, 1002, status=(STATUS_ROW, "")), + 1003: edit(ACTIVE, 1003, status=(">Active<", ">In Review<")), + 1004: edit(ACTIVE, 1004, sunset=("9/30/2029", "TBD")), + 1005: edit(ACTIVE, 1005, std=("FIPS 140-3", "FIPS 140-1")), + 1006: edit(ACTIVE, 1006, heading=("

Certificate #1006

", "")), + 1007: "

Certificate #1007

Something went wrong", + 1009: "

Search results

No modules match.

", + 1008: edit(ACTIVE, 1008, sunset=("9/30/2029", "11/13/2026")), # sunset today still counts + 5000: ACTIVE, # serves another certificate's page +} + +Routes = Dict[str, Tuple[int, str]] + + +class _Handler(BaseHTTPRequestHandler): + routes: Routes = {} + hits: Dict[str, int] = {} + user_agents: list = [] + + def do_GET(self) -> None: # noqa: N802 + _Handler.hits[self.path] = _Handler.hits.get(self.path, 0) + 1 + _Handler.user_agents.append(self.headers.get("User-Agent")) + status, body = self.routes.get(self.path, (404, "Page Not Found | CSRC")) + if isinstance(status, list): # a sequence of statuses, one per request + status = status.pop(0) if len(status) > 1 else status[0] + data = body.encode() + self.send_response(status) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, *args) -> None: + pass + + +def path(number: int) -> str: + return cmvp.CERT_PATH.format(number=number) + + +@pytest.fixture(scope="module") +def server(): + httpd = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + threading.Thread(target=httpd.serve_forever, daemon=True).start() + yield f"http://127.0.0.1:{httpd.server_address[1]}" + httpd.shutdown() + + +@pytest.fixture +def site(server, monkeypatch): + _Handler.routes = {path(n): (200, page) for n, page in PAGES.items()} + _Handler.hits, _Handler.user_agents = {}, [] + monkeypatch.setattr(cmvp, "RETRY_BACKOFF_SECONDS", 0) + return server + + +def check(site: str, number: int) -> dict: + evidence, _ = cmvp.collect([(None, number)], site, NOW) + return evidence["certificates"][0] + + +def test_live_active_page_parses(): + page = cmvp.parse_certificate_page(ACTIVE) + assert page == { + "page_certificate_number": 4816, + "has_details": True, + "module_name": "AWS-LC Cryptographic Module (static)", + "standard": "FIPS 140-3", + "status": "Active", + "historical_reason": None, + "sunset_date": "9/30/2029", + "overall_level": "1", + "caveat": "When operated in approved mode. No assurance of the minimum strength of generated SSPs (e.g., keys)", + "security_level_exceptions": ["Physical security: N/A", "Non-invasive security: N/A"], + "module_type": "Software", + "embodiment": "Multi-Chip Stand Alone", + "vendor": "Amazon Web Services Inc.", + "validation_history": [{"date": "2024-10-01", "type": "Initial", "lab": "atsec information security corporation"}], + } + + +def test_live_historical_and_revoked_pages_parse(): + historical = cmvp.parse_certificate_page(HISTORICAL) + assert (historical["status"], historical["historical_reason"], historical["sunset_date"]) == ( + "Historical", "Moved to historical list due to sunsetting", None) + revoked = cmvp.parse_certificate_page(REVOKED) + assert (revoked["status"], revoked["historical_reason"], revoked["vendor"]) == ("Revoked", "Revoked", "Accellion, Inc.") + + +@pytest.mark.parametrize("number,reason", [ + (4816, None), + (1008, None), + (4282, "historical"), + (1381, "revoked"), + (1001, "sunset_passed"), + (1002, "status_unknown"), + (1003, "status_unknown"), + (1005, "status_unknown"), + (1004, "parse_error"), + (1006, "parse_error"), + (1007, "parse_error"), # names the certificate, but no Details rows + (1009, "not_found"), # an unrelated page + (5000, "not_found"), # the page is for 4816 + (9999, "not_found"), # HTTP 404, as NIST answers an unknown number +]) +def test_verdicts(site, number, reason): + record = check(site, number) + assert record["reason"] == reason, record["reason_detail"] + assert record["validated"] is (reason is None) + + +def test_validated_record(site): + record = check(site, 4816) + assert record["sunset_date"] == "2029-09-30" + assert record["validation_dates"] == ["2024-10-01"] + assert record["url"] == f"{site}/projects/cryptographic-module-validation-program/certificate/4816" + assert record["vendor"] == "Amazon Web Services Inc." + assert _Handler.user_agents == ["paramify-fetchers nist_cmvp_module_validation_status/0.1.0"] + + +def test_server_errors_are_retried_then_fail_as_fetch_error(site): + _Handler.routes[path(4816)] = ([503, 503, 200], ACTIVE) + assert check(site, 4816)["validated"] is True + _Handler.routes[path(4816)] = ([500], "") + record = check(site, 4816) + assert (record["reason"], record["http_status"]) == ("fetch_error", None) + assert _Handler.hits[path(4816)] == 3 + cmvp.RETRIES + 1 + + +def test_one_failed_certificate_does_not_stop_the_others(site): + _Handler.routes[path(4282)] = ([500], "") + evidence, failures = cmvp.collect([("AWS-LC", 4816), (None, 4282), (None, 1381)], site, NOW) + assert [r["reason"] for r in evidence["certificates"]] == [None, "fetch_error", "revoked"] + assert evidence["certificates"][0]["label"] == "AWS-LC" + assert len(failures) == 1 + assert evidence["metadata"]["api_failures"][0]["type"] == "fetch_error" + + +def test_main_writes_summary_and_reports_partial_failure(site, monkeypatch, tmp_path, assert_summary_documented): + status_file = tmp_path / "status.json" + monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) + monkeypatch.setenv("FETCHER_STATUS_FILE", str(status_file)) + monkeypatch.setenv("NIST_CMVP_BASE_URL", site + "/") + monkeypatch.setenv("NIST_CMVP_CERTIFICATES", "AWS-LC=4816, #4282, 1381, 1001, 1004, 9999") + assert cmvp.main() == 1 + evidence = json.loads((tmp_path / "nist_cmvp_module_validation_status.json").read_text()) + summary = evidence["summary"] + assert (summary["certificates_total"], summary["certificates_validated"], summary["certificates_not_validated"]) == (6, 1, 5) + assert summary["not_validated_by_reason"] == { + "historical": 1, "revoked": 1, "sunset_passed": 1, "status_unknown": 0, + "not_found": 1, "parse_error": 1, "fetch_error": 0, + } + assert json.loads(status_file.read_text())["code"] == "partial_failure" + assert_summary_documented("nist_cmvp/module_validation_status", summary) + + +def test_main_exits_zero_when_every_page_was_read(site, monkeypatch, tmp_path): + monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) + monkeypatch.setenv("NIST_CMVP_BASE_URL", site) + monkeypatch.setenv("NIST_CMVP_CERTIFICATES", "4282, 9999") # not validated is a verdict, not a failure + assert cmvp.main() == 0 + assert "metadata" not in json.loads((tmp_path / "nist_cmvp_module_validation_status.json").read_text()) + + +def test_unreachable_base_url_still_writes_json(monkeypatch, tmp_path): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + status_file = tmp_path / "status.json" + monkeypatch.setattr(cmvp, "RETRY_BACKOFF_SECONDS", 0) + monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) + monkeypatch.setenv("FETCHER_STATUS_FILE", str(status_file)) + monkeypatch.setenv("NIST_CMVP_BASE_URL", f"http://127.0.0.1:{port}") + monkeypatch.setenv("NIST_CMVP_CERTIFICATES", "4816, 4282") + assert cmvp.main() == 1 + evidence = json.loads((tmp_path / "nist_cmvp_module_validation_status.json").read_text()) + assert evidence["summary"]["not_validated_by_reason"]["fetch_error"] == 2 + assert evidence["metadata"]["partial_failure"] is True + assert json.loads(status_file.read_text())["code"] == "target_unreachable" + + +@pytest.mark.parametrize("raw,expected", [ + ("4816, 4282", [(None, 4816), (None, 4282)]), + ("AWS-LC=4816;#4282\n", [("AWS-LC", 4816), (None, 4282)]), +]) +def test_parse_certificates(raw, expected): + assert cmvp.parse_certificates(raw) == expected + + +@pytest.mark.parametrize("raw", ["", " , ", "AWS-LC", "48a6"]) +def test_bad_config_fails_with_json(monkeypatch, tmp_path, raw): + status_file = tmp_path / "status.json" + monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) + monkeypatch.setenv("FETCHER_STATUS_FILE", str(status_file)) + monkeypatch.setenv("NIST_CMVP_CERTIFICATES", raw) + assert cmvp.main() == 1 + assert json.loads((tmp_path / "nist_cmvp_module_validation_status.json").read_text())["certificates"] == [] + assert json.loads(status_file.read_text())["code"] == "bad_config" From d6273fd539fadbe2d428c5ef694b794f93f4d28c Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 19:37:03 +0000 Subject: [PATCH 2/5] README: label the NIST CMVP tile "NIST CMVP" Text tile, no logo: NIST reserves the FIPS 140 mark for validated-module vendors and gives no general permission for third-party use of its logo. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WA75NTunK2CFNDjwSpwg8m --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index addf124ca..ddf7347ae 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ If you have a question about this repo, ask DeepWiki. It answers in plain Englis Qualys
Qualys - NIST
NIST CMVP + NIST CMVP From 6649ffbda2ab4b65c246b27307499cfa708fbf96 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 19:41:40 +0000 Subject: [PATCH 3/5] README: use the NIST logo for the NIST CMVP tile Replace the text-only tile with a logo tile that matches the others. The NIST agency wordmark comes from csrc.nist.gov, padded to a square viewBox, with a white variant for GitHub dark theme. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WA75NTunK2CFNDjwSpwg8m --- README.md | 2 +- fetchers/logos/nist_cmvp-dark.svg | 5 +++++ fetchers/logos/nist_cmvp.svg | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 fetchers/logos/nist_cmvp-dark.svg create mode 100644 fetchers/logos/nist_cmvp.svg diff --git a/README.md b/README.md index ddf7347ae..3462f553e 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ If you have a question about this repo, ask DeepWiki. It answers in plain Englis Qualys
Qualys - NIST CMVP + NIST CMVP
NIST CMVP diff --git a/fetchers/logos/nist_cmvp-dark.svg b/fetchers/logos/nist_cmvp-dark.svg new file mode 100644 index 000000000..d609bbc95 --- /dev/null +++ b/fetchers/logos/nist_cmvp-dark.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/fetchers/logos/nist_cmvp.svg b/fetchers/logos/nist_cmvp.svg new file mode 100644 index 000000000..e1292ea26 --- /dev/null +++ b/fetchers/logos/nist_cmvp.svg @@ -0,0 +1,5 @@ + + + + + From 4dab0a7c150349488681e5e4e84a8f4302dcb079 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 19:55:48 +0000 Subject: [PATCH 4/5] nist_cmvp: synthetic test fixtures, one-line category comment CONTRIBUTING.md asks for synthetic data in tests and fixtures. The three certificate pages were trimmed copies of live csrc.nist.gov pages with real vendor names, addresses and labs. They keep the same markup but now carry made-up certificate numbers (7001-7003), names, addresses and labs, and are named synthetic_*.html like the web and zendesk fixtures. The category file's comment is one line, per the onboard-platform comment rule (check_onboarding.py warned on the 6-line block). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WA75NTunK2CFNDjwSpwg8m --- fetchers/_categories/nist_cmvp.yaml | 7 +- .../{cert_4816.html => synthetic_active.html} | 16 ++-- ...rt_4282.html => synthetic_historical.html} | 20 ++--- ...{cert_1381.html => synthetic_revoked.html} | 16 ++-- tests/test_nist_cmvp.py | 78 +++++++++---------- 5 files changed, 66 insertions(+), 71 deletions(-) rename tests/fixtures/nist_cmvp/{cert_4816.html => synthetic_active.html} (86%) rename tests/fixtures/nist_cmvp/{cert_4282.html => synthetic_historical.html} (88%) rename tests/fixtures/nist_cmvp/{cert_1381.html => synthetic_revoked.html} (87%) diff --git a/fetchers/_categories/nist_cmvp.yaml b/fetchers/_categories/nist_cmvp.yaml index 5c6602c4e..9ca898285 100644 --- a/fetchers/_categories/nist_cmvp.yaml +++ b/fetchers/_categories/nist_cmvp.yaml @@ -1,9 +1,4 @@ -# NIST Cryptographic Module Validation Program (CMVP) certificates, read from -# the public certificate pages on csrc.nist.gov. -# -# No secrets and no `auth:` block: the certificate pages are public. No -# `requires:` block either: `requests` is a core dependency (pyproject.toml), -# and HTML is parsed with the standard library. +# Public pages, so no secrets or auth; no `requires:`, since `requests` is core and HTML parsing is stdlib. name: nist_cmvp display_name: NIST CMVP diff --git a/tests/fixtures/nist_cmvp/cert_4816.html b/tests/fixtures/nist_cmvp/synthetic_active.html similarity index 86% rename from tests/fixtures/nist_cmvp/cert_4816.html rename to tests/fixtures/nist_cmvp/synthetic_active.html index 57c62d6e9..005cd5227 100644 --- a/tests/fixtures/nist_cmvp/cert_4816.html +++ b/tests/fixtures/nist_cmvp/synthetic_active.html @@ -1,5 +1,5 @@ - + Cryptographic Module Validation Program | CSRC @@ -7,7 +7,7 @@
-

Certificate #4816

+

Certificate #7001

@@ -22,7 +22,7 @@

Details

Module Name
-
AWS-LC Cryptographic Module (static)
+
Example Crypto Module (static)
Standard
@@ -86,11 +86,11 @@

Details

Vendor

- Amazon Web Services Inc. + Example Vendor A Inc.
- 1918 8th Ave. + 100 Example Street
- Seattle, WA 98121 + Springfield, ST 00001
USA
@@ -116,7 +116,7 @@

Validation History

10/1/2024 Initial - atsec information security corporation + Example Lab A diff --git a/tests/fixtures/nist_cmvp/cert_4282.html b/tests/fixtures/nist_cmvp/synthetic_historical.html similarity index 88% rename from tests/fixtures/nist_cmvp/cert_4282.html rename to tests/fixtures/nist_cmvp/synthetic_historical.html index e1abe4609..cf71d903e 100644 --- a/tests/fixtures/nist_cmvp/cert_4282.html +++ b/tests/fixtures/nist_cmvp/synthetic_historical.html @@ -1,5 +1,5 @@ - + Cryptographic Module Validation Program | CSRC @@ -7,7 +7,7 @@
-

Certificate #4282

+

Certificate #7002

Standard
@@ -88,7 +88,7 @@

Details

Vendor

@@ -112,27 +112,27 @@

Validation History

8/23/2022 Initial - Acumen Security + Example Lab B 3/10/2023 Update - Acumen Security + Example Lab B 5/23/2023 Update - Acumen Security + Example Lab B 1/22/2024 Update - Acumen Security + Example Lab B 7/10/2024 Update - Acumen Security + Example Lab B diff --git a/tests/fixtures/nist_cmvp/cert_1381.html b/tests/fixtures/nist_cmvp/synthetic_revoked.html similarity index 87% rename from tests/fixtures/nist_cmvp/cert_1381.html rename to tests/fixtures/nist_cmvp/synthetic_revoked.html index a81124f42..dd640bf9b 100644 --- a/tests/fixtures/nist_cmvp/cert_1381.html +++ b/tests/fixtures/nist_cmvp/synthetic_revoked.html @@ -1,5 +1,5 @@ - + Cryptographic Module Validation Program | CSRC @@ -7,7 +7,7 @@
-

Certificate #1381

+

Certificate #7003

Standard
@@ -91,11 +91,11 @@

Details

Vendor

- Accellion, Inc. + Example Vendor C, Inc.
- 1900 Embarcadero Road, Suite 207 + 200 Example Road, Suite 2
- Palo Alto, CA 94303 + Springfield, ST 00003
USA
@@ -121,7 +121,7 @@

Validation History

8/12/2010 Initial - UL Verification Services, Inc. + Example Lab C diff --git a/tests/test_nist_cmvp.py b/tests/test_nist_cmvp.py index f21180da6..eb69151e2 100644 --- a/tests/test_nist_cmvp.py +++ b/tests/test_nist_cmvp.py @@ -1,8 +1,8 @@ """nist_cmvp_module_validation_status against a local HTTP server. -The pages under tests/fixtures/nist_cmvp/ are trimmed from the live certificate -pages on csrc.nist.gov (4816 Active, 4282 Historical, 1381 Revoked), as served -on 2026-11-13. The other cases are edits of the 4816 page. +The pages under tests/fixtures/nist_cmvp/ are synthetic, in the markup of the +csrc.nist.gov certificate pages as served on 2026-11-13 (7001 Active, 7002 +Historical, 7003 Revoked). The other cases are edits of the 7001 page. """ from __future__ import annotations @@ -28,14 +28,14 @@ _spec.loader.exec_module(cmvp) NOW = datetime(2026, 11, 13, 18, 0, tzinfo=timezone.utc) -ACTIVE = (FIXTURES / "cert_4816.html").read_text() -HISTORICAL = (FIXTURES / "cert_4282.html").read_text() -REVOKED = (FIXTURES / "cert_1381.html").read_text() +ACTIVE = (FIXTURES / "synthetic_active.html").read_text() +HISTORICAL = (FIXTURES / "synthetic_historical.html").read_text() +REVOKED = (FIXTURES / "synthetic_revoked.html").read_text() def edit(page: str, number: int, **swap: str) -> str: - """The 4816 page renumbered, with each old=new text swap applied.""" - page = page.replace("Certificate #4816", f"Certificate #{number}") + """The 7001 page renumbered, with each old=new text swap applied.""" + page = page.replace("Certificate #7001", f"Certificate #{number}") for old, new in swap.values(): assert old in page, old page = page.replace(old, new) @@ -44,9 +44,9 @@ def edit(page: str, number: int, **swap: str) -> str: STATUS_ROW = '
Status
\n
Active
' PAGES: Dict[int, str] = { - 4816: ACTIVE, - 4282: HISTORICAL, - 1381: REVOKED, + 7001: ACTIVE, + 7002: HISTORICAL, + 7003: REVOKED, # FIPS 140-2 certificates sunset on 9/21/2026; one still shown Active must not pass. 1001: edit(ACTIVE, 1001, std=("FIPS 140-3", "FIPS 140-2"), sunset=("9/30/2029", "9/21/2026")), 1002: edit(ACTIVE, 1002, status=(STATUS_ROW, "")), @@ -110,12 +110,12 @@ def check(site: str, number: int) -> dict: return evidence["certificates"][0] -def test_live_active_page_parses(): +def test_active_page_parses(): page = cmvp.parse_certificate_page(ACTIVE) assert page == { - "page_certificate_number": 4816, + "page_certificate_number": 7001, "has_details": True, - "module_name": "AWS-LC Cryptographic Module (static)", + "module_name": "Example Crypto Module (static)", "standard": "FIPS 140-3", "status": "Active", "historical_reason": None, @@ -125,24 +125,24 @@ def test_live_active_page_parses(): "security_level_exceptions": ["Physical security: N/A", "Non-invasive security: N/A"], "module_type": "Software", "embodiment": "Multi-Chip Stand Alone", - "vendor": "Amazon Web Services Inc.", - "validation_history": [{"date": "2024-10-01", "type": "Initial", "lab": "atsec information security corporation"}], + "vendor": "Example Vendor A Inc.", + "validation_history": [{"date": "2024-10-01", "type": "Initial", "lab": "Example Lab A"}], } -def test_live_historical_and_revoked_pages_parse(): +def test_historical_and_revoked_pages_parse(): historical = cmvp.parse_certificate_page(HISTORICAL) assert (historical["status"], historical["historical_reason"], historical["sunset_date"]) == ( "Historical", "Moved to historical list due to sunsetting", None) revoked = cmvp.parse_certificate_page(REVOKED) - assert (revoked["status"], revoked["historical_reason"], revoked["vendor"]) == ("Revoked", "Revoked", "Accellion, Inc.") + assert (revoked["status"], revoked["historical_reason"], revoked["vendor"]) == ("Revoked", "Revoked", "Example Vendor C, Inc.") @pytest.mark.parametrize("number,reason", [ - (4816, None), + (7001, None), (1008, None), - (4282, "historical"), - (1381, "revoked"), + (7002, "historical"), + (7003, "revoked"), (1001, "sunset_passed"), (1002, "status_unknown"), (1003, "status_unknown"), @@ -151,7 +151,7 @@ def test_live_historical_and_revoked_pages_parse(): (1006, "parse_error"), (1007, "parse_error"), # names the certificate, but no Details rows (1009, "not_found"), # an unrelated page - (5000, "not_found"), # the page is for 4816 + (5000, "not_found"), # the page is for 7001 (9999, "not_found"), # HTTP 404, as NIST answers an unknown number ]) def test_verdicts(site, number, reason): @@ -161,28 +161,28 @@ def test_verdicts(site, number, reason): def test_validated_record(site): - record = check(site, 4816) + record = check(site, 7001) assert record["sunset_date"] == "2029-09-30" assert record["validation_dates"] == ["2024-10-01"] - assert record["url"] == f"{site}/projects/cryptographic-module-validation-program/certificate/4816" - assert record["vendor"] == "Amazon Web Services Inc." + assert record["url"] == f"{site}/projects/cryptographic-module-validation-program/certificate/7001" + assert record["vendor"] == "Example Vendor A Inc." assert _Handler.user_agents == ["paramify-fetchers nist_cmvp_module_validation_status/0.1.0"] def test_server_errors_are_retried_then_fail_as_fetch_error(site): - _Handler.routes[path(4816)] = ([503, 503, 200], ACTIVE) - assert check(site, 4816)["validated"] is True - _Handler.routes[path(4816)] = ([500], "") - record = check(site, 4816) + _Handler.routes[path(7001)] = ([503, 503, 200], ACTIVE) + assert check(site, 7001)["validated"] is True + _Handler.routes[path(7001)] = ([500], "") + record = check(site, 7001) assert (record["reason"], record["http_status"]) == ("fetch_error", None) - assert _Handler.hits[path(4816)] == 3 + cmvp.RETRIES + 1 + assert _Handler.hits[path(7001)] == 3 + cmvp.RETRIES + 1 def test_one_failed_certificate_does_not_stop_the_others(site): - _Handler.routes[path(4282)] = ([500], "") - evidence, failures = cmvp.collect([("AWS-LC", 4816), (None, 4282), (None, 1381)], site, NOW) + _Handler.routes[path(7002)] = ([500], "") + evidence, failures = cmvp.collect([("LIB-A", 7001), (None, 7002), (None, 7003)], site, NOW) assert [r["reason"] for r in evidence["certificates"]] == [None, "fetch_error", "revoked"] - assert evidence["certificates"][0]["label"] == "AWS-LC" + assert evidence["certificates"][0]["label"] == "LIB-A" assert len(failures) == 1 assert evidence["metadata"]["api_failures"][0]["type"] == "fetch_error" @@ -192,7 +192,7 @@ def test_main_writes_summary_and_reports_partial_failure(site, monkeypatch, tmp_ monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) monkeypatch.setenv("FETCHER_STATUS_FILE", str(status_file)) monkeypatch.setenv("NIST_CMVP_BASE_URL", site + "/") - monkeypatch.setenv("NIST_CMVP_CERTIFICATES", "AWS-LC=4816, #4282, 1381, 1001, 1004, 9999") + monkeypatch.setenv("NIST_CMVP_CERTIFICATES", "LIB-A=7001, #7002, 7003, 1001, 1004, 9999") assert cmvp.main() == 1 evidence = json.loads((tmp_path / "nist_cmvp_module_validation_status.json").read_text()) summary = evidence["summary"] @@ -208,7 +208,7 @@ def test_main_writes_summary_and_reports_partial_failure(site, monkeypatch, tmp_ def test_main_exits_zero_when_every_page_was_read(site, monkeypatch, tmp_path): monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) monkeypatch.setenv("NIST_CMVP_BASE_URL", site) - monkeypatch.setenv("NIST_CMVP_CERTIFICATES", "4282, 9999") # not validated is a verdict, not a failure + monkeypatch.setenv("NIST_CMVP_CERTIFICATES", "7002, 9999") # not validated is a verdict, not a failure assert cmvp.main() == 0 assert "metadata" not in json.loads((tmp_path / "nist_cmvp_module_validation_status.json").read_text()) @@ -222,7 +222,7 @@ def test_unreachable_base_url_still_writes_json(monkeypatch, tmp_path): monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) monkeypatch.setenv("FETCHER_STATUS_FILE", str(status_file)) monkeypatch.setenv("NIST_CMVP_BASE_URL", f"http://127.0.0.1:{port}") - monkeypatch.setenv("NIST_CMVP_CERTIFICATES", "4816, 4282") + monkeypatch.setenv("NIST_CMVP_CERTIFICATES", "7001, 7002") assert cmvp.main() == 1 evidence = json.loads((tmp_path / "nist_cmvp_module_validation_status.json").read_text()) assert evidence["summary"]["not_validated_by_reason"]["fetch_error"] == 2 @@ -231,14 +231,14 @@ def test_unreachable_base_url_still_writes_json(monkeypatch, tmp_path): @pytest.mark.parametrize("raw,expected", [ - ("4816, 4282", [(None, 4816), (None, 4282)]), - ("AWS-LC=4816;#4282\n", [("AWS-LC", 4816), (None, 4282)]), + ("7001, 7002", [(None, 7001), (None, 7002)]), + ("LIB-A=7001;#7002\n", [("LIB-A", 7001), (None, 7002)]), ]) def test_parse_certificates(raw, expected): assert cmvp.parse_certificates(raw) == expected -@pytest.mark.parametrize("raw", ["", " , ", "AWS-LC", "48a6"]) +@pytest.mark.parametrize("raw", ["", " , ", "LIB-A", "70a1"]) def test_bad_config_fails_with_json(monkeypatch, tmp_path, raw): status_file = tmp_path / "status.json" monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) From 28fb1684335782a8c268873f610e3bf922f93e0c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 17 Nov 2026 22:16:05 +0000 Subject: [PATCH 5/5] nist_cmvp: do not state a verdict for a certificate the run could not check Addresses review 5947433900 on #355. - A 200 page with no "Certificate #" heading and no Details rows (a maintenance or block page, a redesign) is parse_error, not not_found. not_found stays for HTTP 404/410 and a heading that names another number. - parse_error and fetch_error records have validated: null. When any certificate could not be checked, certificates_validated, certificates_not_validated and the verdict counts are null; the parse_error and fetch_error counts are still written. - HTTP 429 is retried like 5xx, waiting for a numeric Retry-After up to 60 s. A 429 that persists reports rate_limited. http_status is the last attempt's HTTP answer. - target_unreachable only when every certificate is a fetch_error; otherwise partial_failure. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01WA75NTunK2CFNDjwSpwg8m --- CHANGELOG.md | 7 +- fetchers/nist_cmvp/README.md | 25 +++-- .../module_validation_status/fetcher.py | 69 ++++++++++---- .../module_validation_status/fetcher.yaml | 16 ++-- tests/test_nist_cmvp.py | 95 ++++++++++++++++--- 5 files changed, 168 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d9a8bfa8..101558f3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,8 +91,11 @@ schemas and the `paramify` CLI — not the internal code. the module is currently validated: the page is for that number, Status is Active, the standard is FIPS 140-2 or FIPS 140-3, and the sunset date has not passed. Anything else is not validated, with a reason (`historical`, - `revoked`, `sunset_passed`, `status_unknown`, `not_found`, `parse_error` or - `fetch_error`). Maps to KSI-SVC-SIN and KSI-SVC-VRI. No credential. + `revoked`, `sunset_passed`, `status_unknown` or `not_found`). A certificate + the run could not check (`parse_error` or `fetch_error`) has `validated: + null`, the summary's verdict counts are null, and the run exits 1. HTTP 429 + is retried with `Retry-After` and reported as `rate_limited` if it persists. + Maps to KSI-SVC-SIN and KSI-SVC-VRI. No credential. - **`pipeline:` block in `fetcher.yaml`.** A fetcher whose output feeds a Paramify pipeline declares `pipeline: {type, id_scheme}` (`asset_id_field` for a diff --git a/fetchers/nist_cmvp/README.md b/fetchers/nist_cmvp/README.md index c1f0cc8bc..a19ee9166 100644 --- a/fetchers/nist_cmvp/README.md +++ b/fetchers/nist_cmvp/README.md @@ -44,7 +44,9 @@ A certificate is validated only when all of these are true: - Standard is FIPS 140-2 or FIPS 140-3. - The sunset date is today or later (UTC). -Every other certificate is not validated, and `reason` says why: +Every other certificate gets a `reason`. The first five are verdicts: the +certificate is not validated (`validated: false`). The last two mean the run +could not check the certificate, so `validated` is `null`: | Reason | Meaning | |---|---| @@ -52,8 +54,8 @@ Every other certificate is not validated, and `reason` says why: | `revoked` | Status is Revoked. | | `sunset_passed` | Status is Active, but the sunset date is before today. All FIPS 140-2 certificates sunset on 9/21/2026. | | `status_unknown` | Status is missing or has another value, or the standard is not FIPS 140-2 or FIPS 140-3. | -| `not_found` | NIST has no page for the number (HTTP 404), or the page is for a different number. | -| `parse_error` | The page could not be read, for example a missing sunset date. | +| `not_found` | NIST has no page for the number (HTTP 404 or 410), or the page heading names a different number. | +| `parse_error` | The page could not be read, for example a missing sunset date. An HTTP 200 page with no `Certificate #` heading and no Details rows, such as a maintenance or block page, is also a `parse_error`, not `not_found`. | | `fetch_error` | The page could not be fetched after retries. | Each record also carries the module name, vendor, standard, status, historical @@ -61,8 +63,18 @@ reason, sunset date, overall level, module type, embodiment, caveat, security level exceptions and validation history. A `parse_error` or `fetch_error` on any certificate makes the run exit 1, but -the other certificates are still checked and written. The other reasons are -verdicts, so the run still exits 0. +the other certificates are still checked and written. The run reports +`rate_limited` if a certificate was still answered with HTTP 429 after retries, +`target_unreachable` if every certificate was a `fetch_error`, and +`partial_failure` otherwise. The other reasons are verdicts, so the run still +exits 0. + +When any certificate could not be checked, the summary's +`certificates_validated`, `certificates_not_validated` and verdict counts +(`historical`, `revoked`, `sunset_passed`, `status_unknown`, `not_found`) are +`null`. The `parse_error` and `fetch_error` counts are still written. A validator +that reads one of the null counts finds no number and fails, instead of passing +on a partial read. ## Settings @@ -72,4 +84,5 @@ verdicts, so the run still exits 0. | `base_url` (category) | `NIST_CMVP_BASE_URL` | `https://csrc.nist.gov` | Requests go one at a time, with a 30-second timeout. A timeout, connection -error or HTTP 5xx is retried twice. +error, HTTP 429 or HTTP 5xx is retried twice. On HTTP 429 the retry waits for a +numeric `Retry-After` header, up to 60 seconds. diff --git a/fetchers/nist_cmvp/module_validation_status/fetcher.py b/fetchers/nist_cmvp/module_validation_status/fetcher.py index 828d03125..657de82e1 100644 --- a/fetchers/nist_cmvp/module_validation_status/fetcher.py +++ b/fetchers/nist_cmvp/module_validation_status/fetcher.py @@ -5,6 +5,7 @@ import json import logging +import math import os import re import sys @@ -32,12 +33,15 @@ REQUEST_TIMEOUT = 30 RETRIES = 2 RETRY_BACKOFF_SECONDS = 2.0 +# Wait at most this long for a Retry-After header on HTTP 429. +MAX_RETRY_AFTER_SECONDS = 60.0 REASONS = ( "historical", "revoked", "sunset_passed", "status_unknown", "not_found", "parse_error", "fetch_error", ) # Reasons that mean the run could not check a certificate, rather than a verdict on it. COLLECTION_FAILURES = ("parse_error", "fetch_error") +VERDICTS = tuple(r for r in REASONS if r not in COLLECTION_FAILURES) ACCEPTED_STANDARD = re.compile(r"^FIPS\s*140-[23]$", re.IGNORECASE) US_DATE = re.compile(r"^(\d{1,2})/(\d{1,2})/(\d{4})$") CERT_HEADING = re.compile(r"^Certificate\s*#\s*(\d+)$", re.IGNORECASE) @@ -211,7 +215,10 @@ def verdict(number: int, page: Dict[str, Any], today: date) -> Tuple[Optional[st if not page["has_details"]: if page["page_certificate_number"] == number: return "parse_error", "the page names the certificate but has no Details rows" - return "not_found", "the page has no Details section for a certificate" + if page["page_certificate_number"] is None: + # NIST answers an unknown number with HTTP 404. A 200 page with neither part + # is something else (maintenance, block page, redesign), so nothing was checked. + return "parse_error", "the page is not a certificate page: no 'Certificate #' heading and no Details rows" if page["page_certificate_number"] is None: return "parse_error", "the page does not name its certificate number" if page["page_certificate_number"] != number: @@ -254,24 +261,41 @@ def parse_certificates(raw: str) -> List[Tuple[Optional[str], int]]: return entries +def _retry_after(response: requests.Response) -> Optional[float]: + """Seconds from a numeric Retry-After header, capped; None when absent or not a number.""" + try: + delay = float(response.headers.get("Retry-After", "")) + except ValueError: + return None + return max(0.0, min(delay, MAX_RETRY_AFTER_SECONDS)) if math.isfinite(delay) else None + + def fetch_page(session: requests.Session, url: str) -> Tuple[Optional[int], Optional[str], Optional[str]]: - """(http_status, body, error). Retries timeouts, connection errors and 5xx; one request at a time.""" - error = None + """(http_status, body, error). Retries timeouts, connection errors, 429 and 5xx; one request at a time. + + http_status is the last attempt's HTTP answer, or None when that attempt got none. + """ + error: Optional[str] = None + status: Optional[int] = None + wait: Optional[float] = None for attempt in range(RETRIES + 1): if attempt: - time.sleep(RETRY_BACKOFF_SECONDS * attempt) + time.sleep(RETRY_BACKOFF_SECONDS * attempt if wait is None else wait) + status, wait = None, None try: response = session.get(url, timeout=REQUEST_TIMEOUT) except requests.exceptions.RequestException as exc: error = f"GET {url} failed: {type(exc).__name__}: {exc}" continue - if response.status_code >= 500: - error = f"GET {url} returned HTTP {response.status_code}" + status = response.status_code + if status == 429 or status >= 500: + error = f"GET {url} returned HTTP {status}" + wait = _retry_after(response) if status == 429 else None continue - if response.status_code in (200, 404, 410): - return response.status_code, response.text, None - return response.status_code, None, f"GET {url} returned HTTP {response.status_code}" - return None, None, error + if status in (200, 404, 410): + return status, response.text, None + return status, None, f"GET {url} returned HTTP {status}" + return status, None, error def check_certificate( @@ -295,7 +319,8 @@ def check_certificate( reason, detail = "parse_error", f"{type(exc).__name__}: {exc}" history = page.get("validation_history") or [] record.update({ - "validated": reason is None, + # None when the run could not check the certificate: its state is unknown, not "not validated". + "validated": None if reason in COLLECTION_FAILURES else reason is None, "reason": reason, "reason_detail": detail, "module_name": page.get("module_name"), @@ -316,12 +341,18 @@ def check_certificate( def summarize(records: List[Dict[str, Any]]) -> Dict[str, Any]: + """Counts. When any certificate could not be checked, every verdict count is None, so a + validator reading one finds no number and fails instead of passing on a partial read.""" + counts = {reason: sum(1 for r in records if r["reason"] == reason) for reason in REASONS} + complete = not any(counts[reason] for reason in COLLECTION_FAILURES) validated = sum(1 for r in records if r["validated"]) return { "certificates_total": len(records), - "certificates_validated": validated, - "certificates_not_validated": len(records) - validated, - "not_validated_by_reason": {reason: sum(1 for r in records if r["reason"] == reason) for reason in REASONS}, + "certificates_validated": validated if complete else None, + "certificates_not_validated": len(records) - validated if complete else None, + "not_validated_by_reason": { + reason: count if complete or reason not in VERDICTS else None for reason, count in counts.items() + }, } @@ -366,7 +397,7 @@ def main() -> int: try: entries = parse_certificates(os.environ.get("NIST_CMVP_CERTIFICATES", "")) except ConfigError as exc: - evidence = { + evidence: Dict[str, Any] = { "collected_at": now.strftime("%Y-%m-%dT%H:%M:%SZ"), "base_url": base_url, "certificates": [], @@ -381,7 +412,13 @@ def main() -> int: output_path.write_text(json.dumps(evidence, indent=2)) logger.info("Evidence saved to %s", output_path) if failures: - code = "target_unreachable" if len(failures) == len(entries) else "partial_failure" + failed = [r for r in evidence["certificates"] if r["reason"] in COLLECTION_FAILURES] + if any(r["http_status"] == 429 for r in failed): + code = "rate_limited" + elif len(failed) == len(entries) and all(r["reason"] == "fetch_error" for r in failed): + code = "target_unreachable" # the host never gave a usable answer + else: + code = "partial_failure" report_failure(f"{len(failures)} of {len(entries)} certificates could not be checked: " + "; ".join(failures), code) return 1 return 0 diff --git a/fetchers/nist_cmvp/module_validation_status/fetcher.yaml b/fetchers/nist_cmvp/module_validation_status/fetcher.yaml index f17c5ab46..b415997c9 100644 --- a/fetchers/nist_cmvp/module_validation_status/fetcher.yaml +++ b/fetchers/nist_cmvp/module_validation_status/fetcher.yaml @@ -42,19 +42,19 @@ evidence_set: Commands Executed: - GET /projects/cryptographic-module-validation-program/certificate/ — reads the certificate page (per certificate) - Output: nist_cmvp_module_validation_status.json — one record per certificate with its module name, vendor, standard, status, sunset date and verdict. + Output: nist_cmvp_module_validation_status.json — one record per certificate with its module name, vendor, standard, status, sunset date and verdict. A record's validated is null when the certificate could not be checked (parse_error or fetch_error). Summary fields: Is every listed module currently validated? - - certificates_total — Number of certificates checked - - certificates_validated — Number of certificates that are validated: the page is for that number, Status is Active, the standard is FIPS 140-2 or FIPS 140-3, and the sunset date is today or later - - certificates_not_validated — Number of certificates that are not validated - - not_validated_by_reason — Number of not-validated certificates for each reason: + - certificates_total — Number of certificates listed + - certificates_validated — Number of certificates that are validated: the page is for that number, Status is Active, the standard is FIPS 140-2 or FIPS 140-3, and the sunset date is today or later. Null when any certificate could not be checked + - certificates_not_validated — Number of certificates that are not validated. Null when any certificate could not be checked + - not_validated_by_reason — Number of certificates for each reason. The verdict reasons (historical, revoked, sunset_passed, status_unknown, not_found) are null when any certificate could not be checked; parse_error and fetch_error are always counts: - historical (Status is Historical) - revoked (Status is Revoked) - sunset_passed (Status is Active but the sunset date is before today) - status_unknown (Status is missing or another value, or the standard is not FIPS 140-2 or 140-3) - - not_found (NIST has no page for the number, or the page is for another number) - - parse_error (the page could not be read, for example a missing sunset date) - - fetch_error (the page could not be fetched) + - not_found (NIST answered HTTP 404 or 410 for the number, or the page heading names another number) + - parse_error (not checked: the page could not be read, for example a missing sunset date, or an HTTP 200 page with no Certificate heading and no Details rows, such as a maintenance page) + - fetch_error (not checked: the page could not be fetched after retries, including HTTP 429 that persists) diff --git a/tests/test_nist_cmvp.py b/tests/test_nist_cmvp.py index eb69151e2..5e2d7f67a 100644 --- a/tests/test_nist_cmvp.py +++ b/tests/test_nist_cmvp.py @@ -14,7 +14,7 @@ from datetime import datetime, timezone from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path -from typing import Dict, Tuple +from typing import Any, Dict, List, Tuple import pytest @@ -56,11 +56,13 @@ def edit(page: str, number: int, **swap: str) -> str: 1006: edit(ACTIVE, 1006, heading=("

Certificate #1006

", "")), 1007: "

Certificate #1007

Something went wrong", 1009: "

Search results

No modules match.

", + 1010: "

Certificate #7001

Something went wrong", # heading names another number 1008: edit(ACTIVE, 1008, sunset=("9/30/2029", "11/13/2026")), # sunset today still counts 5000: ACTIVE, # serves another certificate's page } -Routes = Dict[str, Tuple[int, str]] +Routes = Dict[str, Tuple[Any, ...]] +MAINTENANCE = "Scheduled maintenance

We'll be back soon

" class _Handler(BaseHTTPRequestHandler): @@ -71,12 +73,14 @@ class _Handler(BaseHTTPRequestHandler): def do_GET(self) -> None: # noqa: N802 _Handler.hits[self.path] = _Handler.hits.get(self.path, 0) + 1 _Handler.user_agents.append(self.headers.get("User-Agent")) - status, body = self.routes.get(self.path, (404, "Page Not Found | CSRC")) + status, body, *extra = self.routes.get(self.path, (404, "Page Not Found | CSRC")) if isinstance(status, list): # a sequence of statuses, one per request status = status.pop(0) if len(status) > 1 else status[0] data = body.encode() self.send_response(status) self.send_header("Content-Type", "text/html; charset=utf-8") + for name, value in (extra[0] if extra else {}).items(): + self.send_header(name, value) self.send_header("Content-Length", str(len(data))) self.end_headers() self.wfile.write(data) @@ -150,14 +154,16 @@ def test_historical_and_revoked_pages_parse(): (1004, "parse_error"), (1006, "parse_error"), (1007, "parse_error"), # names the certificate, but no Details rows - (1009, "not_found"), # an unrelated page + (1009, "parse_error"), # a 200 page that is not a certificate page: nothing was checked + (1010, "not_found"), # the heading names 7001 (5000, "not_found"), # the page is for 7001 (9999, "not_found"), # HTTP 404, as NIST answers an unknown number ]) def test_verdicts(site, number, reason): record = check(site, number) assert record["reason"] == reason, record["reason_detail"] - assert record["validated"] is (reason is None) + # A certificate the run could not check has no known validation state. + assert record["validated"] is (None if reason in cmvp.COLLECTION_FAILURES else reason is None) def test_validated_record(site): @@ -174,17 +180,72 @@ def test_server_errors_are_retried_then_fail_as_fetch_error(site): assert check(site, 7001)["validated"] is True _Handler.routes[path(7001)] = ([500], "") record = check(site, 7001) - assert (record["reason"], record["http_status"]) == ("fetch_error", None) + assert (record["reason"], record["http_status"], record["validated"]) == ("fetch_error", 500, None) assert _Handler.hits[path(7001)] == 3 + cmvp.RETRIES + 1 +@pytest.fixture +def sleeps(monkeypatch) -> List[float]: + waits: List[float] = [] + monkeypatch.setattr(cmvp.time, "sleep", waits.append) + return waits + + +@pytest.mark.parametrize("retry_after,wait", [("7", 7.0), ("3600", 60.0), ("soon", 0)]) +def test_rate_limit_is_retried_after_retry_after(site, sleeps, retry_after, wait): + _Handler.routes[path(7001)] = ([429, 200], ACTIVE, {"Retry-After": retry_after}) + assert check(site, 7001)["validated"] is True + assert _Handler.hits[path(7001)] == 2 + assert sleeps == [wait] + + +def test_rate_limit_that_persists_reports_rate_limited(site, sleeps, monkeypatch, tmp_path): + status_file = tmp_path / "status.json" + _Handler.routes[path(7002)] = ([429], "", {"Retry-After": "1"}) + monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) + monkeypatch.setenv("FETCHER_STATUS_FILE", str(status_file)) + monkeypatch.setenv("NIST_CMVP_BASE_URL", site) + monkeypatch.setenv("NIST_CMVP_CERTIFICATES", "7001, 7002") + assert cmvp.main() == 1 + record = json.loads((tmp_path / "nist_cmvp_module_validation_status.json").read_text())["certificates"][1] + assert (record["reason"], record["http_status"], record["validated"]) == ("fetch_error", 429, None) + assert _Handler.hits[path(7002)] == cmvp.RETRIES + 1 + assert json.loads(status_file.read_text())["code"] == "rate_limited" + + +def test_a_page_that_is_not_a_certificate_page_fails_the_run(site, monkeypatch, tmp_path): + """A maintenance or block page served with HTTP 200 is not evidence that NIST has no such certificate.""" + status_file = tmp_path / "status.json" + _Handler.routes = {path(4816): (200, MAINTENANCE), path(4282): (200, MAINTENANCE)} + monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) + monkeypatch.setenv("FETCHER_STATUS_FILE", str(status_file)) + monkeypatch.setenv("NIST_CMVP_BASE_URL", site) + monkeypatch.setenv("NIST_CMVP_CERTIFICATES", "4816, 4282") + assert cmvp.main() == 1 + evidence = json.loads((tmp_path / "nist_cmvp_module_validation_status.json").read_text()) + assert [r["reason"] for r in evidence["certificates"]] == ["parse_error", "parse_error"] + assert evidence["metadata"]["partial_failure"] is True + assert evidence["summary"]["not_validated_by_reason"]["not_found"] is None + # The host answered, so this is not target_unreachable. + assert json.loads(status_file.read_text())["code"] == "partial_failure" + + def test_one_failed_certificate_does_not_stop_the_others(site): - _Handler.routes[path(7002)] = ([500], "") + _Handler.routes[path(7002)] = ([503], "") evidence, failures = cmvp.collect([("LIB-A", 7001), (None, 7002), (None, 7003)], site, NOW) assert [r["reason"] for r in evidence["certificates"]] == [None, "fetch_error", "revoked"] + assert [r["validated"] for r in evidence["certificates"]] == [True, None, False] assert evidence["certificates"][0]["label"] == "LIB-A" assert len(failures) == 1 assert evidence["metadata"]["api_failures"][0]["type"] == "fetch_error" + # 7002 was never read, so no verdict count can be stated; a validator then finds no number. + summary = evidence["summary"] + assert (summary["certificates_total"], summary["certificates_validated"], summary["certificates_not_validated"]) == ( + 3, None, None) + assert summary["not_validated_by_reason"] == { + "historical": None, "revoked": None, "sunset_passed": None, "status_unknown": None, + "not_found": None, "parse_error": 0, "fetch_error": 1, + } def test_main_writes_summary_and_reports_partial_failure(site, monkeypatch, tmp_path, assert_summary_documented): @@ -196,10 +257,12 @@ def test_main_writes_summary_and_reports_partial_failure(site, monkeypatch, tmp_ assert cmvp.main() == 1 evidence = json.loads((tmp_path / "nist_cmvp_module_validation_status.json").read_text()) summary = evidence["summary"] - assert (summary["certificates_total"], summary["certificates_validated"], summary["certificates_not_validated"]) == (6, 1, 5) + # 1004 could not be read, so only the collection-failure counts are stated. + assert (summary["certificates_total"], summary["certificates_validated"], summary["certificates_not_validated"]) == ( + 6, None, None) assert summary["not_validated_by_reason"] == { - "historical": 1, "revoked": 1, "sunset_passed": 1, "status_unknown": 0, - "not_found": 1, "parse_error": 1, "fetch_error": 0, + "historical": None, "revoked": None, "sunset_passed": None, "status_unknown": None, + "not_found": None, "parse_error": 1, "fetch_error": 0, } assert json.loads(status_file.read_text())["code"] == "partial_failure" assert_summary_documented("nist_cmvp/module_validation_status", summary) @@ -208,9 +271,17 @@ def test_main_writes_summary_and_reports_partial_failure(site, monkeypatch, tmp_ def test_main_exits_zero_when_every_page_was_read(site, monkeypatch, tmp_path): monkeypatch.setenv("EVIDENCE_DIR", str(tmp_path)) monkeypatch.setenv("NIST_CMVP_BASE_URL", site) - monkeypatch.setenv("NIST_CMVP_CERTIFICATES", "7002, 9999") # not validated is a verdict, not a failure + monkeypatch.setenv("NIST_CMVP_CERTIFICATES", "LIB-A=7001, #7002, 7003, 1001, 1003, 9999") # verdicts, not failures assert cmvp.main() == 0 - assert "metadata" not in json.loads((tmp_path / "nist_cmvp_module_validation_status.json").read_text()) + evidence = json.loads((tmp_path / "nist_cmvp_module_validation_status.json").read_text()) + assert "metadata" not in evidence + summary = evidence["summary"] + assert (summary["certificates_total"], summary["certificates_validated"], summary["certificates_not_validated"]) == ( + 6, 1, 5) + assert summary["not_validated_by_reason"] == { + "historical": 1, "revoked": 1, "sunset_passed": 1, "status_unknown": 1, + "not_found": 1, "parse_error": 0, "fetch_error": 0, + } def test_unreachable_base_url_still_writes_json(monkeypatch, tmp_path):