From 8b103622e508ec1b306697a94257150f6dc19d2e Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Tue, 3 Mar 2026 18:28:25 +0000 Subject: [PATCH 01/12] [NRL-1922] Fix sonarqube issue with use of wget --- terraform/bastion/scripts/user-data.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/terraform/bastion/scripts/user-data.sh b/terraform/bastion/scripts/user-data.sh index 624793c97..7070d3159 100644 --- a/terraform/bastion/scripts/user-data.sh +++ b/terraform/bastion/scripts/user-data.sh @@ -4,12 +4,14 @@ set -o errexit -o nounset -o pipefail ASDF_VERSION="v0.18.0" +export DEBIAN_FRONTEND=noninteractive + sudo apt update && \ - sudo apt upgrade -y && \ - sudo apt install -y \ + sudo -E apt upgrade -y && \ + sudo -E apt install -y \ git \ jq \ - wget \ + curl \ build-essential \ unzip \ gnupg \ @@ -42,11 +44,12 @@ output = json " >> /home/nrlf_ops/.aws/config' # Install ASDF -wget https://github.com/asdf-vm/asdf/releases/download/${ASDF_VERSION}/asdf-${ASDF_VERSION}-linux-amd64.tar.gz -O asdf.tar.gz && \ - tar -xzf asdf.tar.gz && \ - mv asdf /usr/bin/asdf && \ - chmod 755 /usr/bin/asdf && \ - rm asdf.tar.gz +curl --location --silent --show-error --fail --output asdf.tar.gz \ + https://github.com/asdf-vm/asdf/releases/download/${ASDF_VERSION}/asdf-${ASDF_VERSION}-linux-amd64.tar.gz && \ + tar -xzf asdf.tar.gz && \ + mv asdf /usr/bin/asdf && \ + chmod 755 /usr/bin/asdf && \ + rm asdf.tar.gz # Clone NRLF into nrlf_ops home directory sudo -u nrlf_ops git clone https://github.com/NHSDigital/NRLF.git /home/nrlf_ops/NRLF From 6146fa847a0a4b8e32c6bea469b3e3bc067cd4d4 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Wed, 4 Mar 2026 07:45:16 +0000 Subject: [PATCH 02/12] [NRL-1922] Add NOSONAR for S3 bucket logging issue --- terraform/account-wide-infrastructure/modules/glue/s3.tf | 6 +++--- .../modules/metadata-bucket/s3.tf | 2 +- .../modules/permissions-store-bucket/s3.tf | 2 +- .../modules/truststore-bucket/s3.tf | 2 +- terraform/account-wide-infrastructure/prod/aws-backup.tf | 2 +- terraform/account-wide-infrastructure/test/aws-backup.tf | 2 +- terraform/infrastructure/modules/firehose/s3.tf | 2 +- .../infrastructure/modules/permissions-store-bucket/s3.tf | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/terraform/account-wide-infrastructure/modules/glue/s3.tf b/terraform/account-wide-infrastructure/modules/glue/s3.tf index cff5d1274..6e7d9a178 100644 --- a/terraform/account-wide-infrastructure/modules/glue/s3.tf +++ b/terraform/account-wide-infrastructure/modules/glue/s3.tf @@ -1,5 +1,5 @@ # S3 Bucket for Raw Data -resource "aws_s3_bucket" "source-data-bucket" { +resource "aws_s3_bucket" "source-data-bucket" { # NOSONAR (S6258) - Logging not required for this bucket bucket = "${var.name_prefix}-source-data-bucket" } @@ -74,7 +74,7 @@ resource "aws_s3_bucket_versioning" "source-data-bucket-versioning" { # S3 Bucket for Processed Data -resource "aws_s3_bucket" "target-data-bucket" { +resource "aws_s3_bucket" "target-data-bucket" { # NOSONAR (S6258) - Logging not required for this bucket bucket = "${var.name_prefix}-target-data-bucket" } @@ -127,7 +127,7 @@ resource "aws_s3_bucket_public_access_block" "target-data-bucket-public-access-b } # S3 Bucket for Code -resource "aws_s3_bucket" "code-bucket" { +resource "aws_s3_bucket" "code-bucket" { # NOSONAR (S6258) - Logging not required for this bucket bucket = "${var.name_prefix}-code-bucket" } diff --git a/terraform/account-wide-infrastructure/modules/metadata-bucket/s3.tf b/terraform/account-wide-infrastructure/modules/metadata-bucket/s3.tf index a9f50439c..0a29791a4 100644 --- a/terraform/account-wide-infrastructure/modules/metadata-bucket/s3.tf +++ b/terraform/account-wide-infrastructure/modules/metadata-bucket/s3.tf @@ -1,4 +1,4 @@ -resource "aws_s3_bucket" "metadata_bucket" { +resource "aws_s3_bucket" "metadata_bucket" { # NOSONAR (S6258) - Logging not required for this bucket bucket = "${var.name_prefix}-metadata" force_destroy = false } diff --git a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf index 2c7f43d1e..3711331b2 100644 --- a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf +++ b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf @@ -1,4 +1,4 @@ -resource "aws_s3_bucket" "authorization-store" { +resource "aws_s3_bucket" "authorization-store" { # NOSONAR (S6258) - Logging not required for this bucket bucket = "${var.name_prefix}-authorization-store" force_destroy = var.enable_bucket_force_destroy diff --git a/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf b/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf index 1f7bd3e81..c6df13a7d 100644 --- a/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf +++ b/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf @@ -1,4 +1,4 @@ -resource "aws_s3_bucket" "api_truststore" { +resource "aws_s3_bucket" "api_truststore" { # NOSONAR (S6258) - Logging not required for this bucket bucket = "${var.name_prefix}-api-truststore" force_destroy = var.enable_bucket_force_destroy tags = { diff --git a/terraform/account-wide-infrastructure/prod/aws-backup.tf b/terraform/account-wide-infrastructure/prod/aws-backup.tf index e86fed016..2e7095098 100644 --- a/terraform/account-wide-infrastructure/prod/aws-backup.tf +++ b/terraform/account-wide-infrastructure/prod/aws-backup.tf @@ -1,5 +1,5 @@ -resource "aws_s3_bucket" "backup_reports" { +resource "aws_s3_bucket" "backup_reports" { # NOSONAR (S6258) - Logging not required for this bucket bucket_prefix = "${local.prefix}-backup-reports" } diff --git a/terraform/account-wide-infrastructure/test/aws-backup.tf b/terraform/account-wide-infrastructure/test/aws-backup.tf index 7bd80f364..aa8a22bdf 100644 --- a/terraform/account-wide-infrastructure/test/aws-backup.tf +++ b/terraform/account-wide-infrastructure/test/aws-backup.tf @@ -1,5 +1,5 @@ -resource "aws_s3_bucket" "backup_reports" { +resource "aws_s3_bucket" "backup_reports" { # NOSONAR (S6258) - Logging not required for this bucket bucket_prefix = "${local.prefix}-backup-reports" } diff --git a/terraform/infrastructure/modules/firehose/s3.tf b/terraform/infrastructure/modules/firehose/s3.tf index 121bfc04b..960d2e358 100644 --- a/terraform/infrastructure/modules/firehose/s3.tf +++ b/terraform/infrastructure/modules/firehose/s3.tf @@ -1,4 +1,4 @@ -resource "aws_s3_bucket" "firehose" { +resource "aws_s3_bucket" "firehose" { # NOSONAR (S6258) - Logging not required for this bucket bucket = "${var.prefix}-firehose" force_destroy = true } diff --git a/terraform/infrastructure/modules/permissions-store-bucket/s3.tf b/terraform/infrastructure/modules/permissions-store-bucket/s3.tf index ad37cff7d..071b87320 100644 --- a/terraform/infrastructure/modules/permissions-store-bucket/s3.tf +++ b/terraform/infrastructure/modules/permissions-store-bucket/s3.tf @@ -1,4 +1,4 @@ -resource "aws_s3_bucket" "authorization-store" { +resource "aws_s3_bucket" "authorization-store" { # NOSONAR (S6258) - Logging not required for this bucket bucket = "${var.name_prefix}-authorization-store" force_destroy = var.enable_bucket_force_destroy From 2f36570e4f9b34007023873bcc7486c6e528a74c Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Wed, 4 Mar 2026 07:55:23 +0000 Subject: [PATCH 03/12] [NRL-1922] Add security options to curl usage in bastion user_data.sh --- terraform/bastion/scripts/user-data.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/terraform/bastion/scripts/user-data.sh b/terraform/bastion/scripts/user-data.sh index 7070d3159..e7b5eb79f 100644 --- a/terraform/bastion/scripts/user-data.sh +++ b/terraform/bastion/scripts/user-data.sh @@ -44,7 +44,8 @@ output = json " >> /home/nrlf_ops/.aws/config' # Install ASDF -curl --location --silent --show-error --fail --output asdf.tar.gz \ +curl --silent --show-error --fail \ + --location --proto '=https' --tlsv1.2 --output asdf.tar.gz \ https://github.com/asdf-vm/asdf/releases/download/${ASDF_VERSION}/asdf-${ASDF_VERSION}-linux-amd64.tar.gz && \ tar -xzf asdf.tar.gz && \ mv asdf /usr/bin/asdf && \ From d74cf2511f97ade46990077ec98c579840ed2300 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Wed, 4 Mar 2026 08:05:17 +0000 Subject: [PATCH 04/12] [NRL-1922] Add NOSONAR for 'apt install -y' in ci-build Dockerfile --- Dockerfile.ci-build | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile.ci-build b/Dockerfile.ci-build index b13beca43..a6d9d66c6 100644 --- a/Dockerfile.ci-build +++ b/Dockerfile.ci-build @@ -1,5 +1,6 @@ FROM ubuntu:22.04 + RUN apt update && \ apt upgrade -y && \ DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y \ @@ -30,7 +31,8 @@ RUN apt update && \ zip \ zlib1g-dev && \ apt clean && \ - rm -rf /var/lib/apt/lists/* + rm -rf /var/lib/apt/lists/* # NOSONAR (S6500) - Auto installing the defined packages is acceptable here + WORKDIR /root RUN git clone https://github.com/asdf-vm/asdf.git ~/.asdf --branch v0.13.1 && \ From adf433867ef855e3e9fef6d65b215128cb307d94 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Wed, 4 Mar 2026 08:06:27 +0000 Subject: [PATCH 05/12] [NRL-1922] Make sure auto-tz for bastion is UTC --- terraform/bastion/scripts/user-data.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/bastion/scripts/user-data.sh b/terraform/bastion/scripts/user-data.sh index e7b5eb79f..485c58063 100644 --- a/terraform/bastion/scripts/user-data.sh +++ b/terraform/bastion/scripts/user-data.sh @@ -5,6 +5,7 @@ set -o errexit -o nounset -o pipefail ASDF_VERSION="v0.18.0" export DEBIAN_FRONTEND=noninteractive +export TZ=Etc/UTC sudo apt update && \ sudo -E apt upgrade -y && \ From 5d44a74c1baeb3090bb1561a7e8db86f7cf42bf7 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Wed, 4 Mar 2026 08:12:09 +0000 Subject: [PATCH 06/12] [NRL-1922] Add NOSONAR for S3 bucket logging issue --- terraform/account-wide-infrastructure/dev/aws-backup.tf | 2 +- terraform/account-wide-infrastructure/mgmt/s3.tf | 2 +- terraform/account-wide-infrastructure/modules/athena/s3.tf | 2 +- terraform/account-wide-infrastructure/modules/glue/s3.tf | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/terraform/account-wide-infrastructure/dev/aws-backup.tf b/terraform/account-wide-infrastructure/dev/aws-backup.tf index b4e9b27a1..be24d6c6a 100644 --- a/terraform/account-wide-infrastructure/dev/aws-backup.tf +++ b/terraform/account-wide-infrastructure/dev/aws-backup.tf @@ -1,5 +1,5 @@ -resource "aws_s3_bucket" "backup_reports" { +resource "aws_s3_bucket" "backup_reports" { # NOSONAR (S6258) - Logging not required for this bucket bucket_prefix = "${local.prefix}-backup-reports" } diff --git a/terraform/account-wide-infrastructure/mgmt/s3.tf b/terraform/account-wide-infrastructure/mgmt/s3.tf index 0b61531cf..8a80dd1ad 100644 --- a/terraform/account-wide-infrastructure/mgmt/s3.tf +++ b/terraform/account-wide-infrastructure/mgmt/s3.tf @@ -1,4 +1,4 @@ -resource "aws_s3_bucket" "ci_data" { +resource "aws_s3_bucket" "ci_data" { # NOSONAR (S6258) - Logging not required for this bucket bucket = "${local.prefix}--ci-data" } diff --git a/terraform/account-wide-infrastructure/modules/athena/s3.tf b/terraform/account-wide-infrastructure/modules/athena/s3.tf index 294e7bd62..787f09473 100644 --- a/terraform/account-wide-infrastructure/modules/athena/s3.tf +++ b/terraform/account-wide-infrastructure/modules/athena/s3.tf @@ -1,4 +1,4 @@ -resource "aws_s3_bucket" "athena" { +resource "aws_s3_bucket" "athena" { # NOSONAR (S6258) - Logging not required for this bucket bucket = "${var.name_prefix}-athena" } diff --git a/terraform/account-wide-infrastructure/modules/glue/s3.tf b/terraform/account-wide-infrastructure/modules/glue/s3.tf index 6e7d9a178..7b4445cb4 100644 --- a/terraform/account-wide-infrastructure/modules/glue/s3.tf +++ b/terraform/account-wide-infrastructure/modules/glue/s3.tf @@ -68,7 +68,7 @@ resource "aws_s3_bucket_lifecycle_configuration" "source-data-bucket-lifecycle" resource "aws_s3_bucket_versioning" "source-data-bucket-versioning" { bucket = aws_s3_bucket.source-data-bucket.id versioning_configuration { - status = "Disabled" + status = "Disabled" # NOSONAR (S6252) - Versioning is not required for this bucket } } From 1f0965ebd45729bd35795a48954ef1a87da0583e Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Wed, 4 Mar 2026 08:12:34 +0000 Subject: [PATCH 07/12] [NRL-1922] Enforce https when using curl in workflows --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1fd1d28a8..68268ee25 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -60,7 +60,7 @@ jobs: run: | DOWNLOAD_URL="https://github.com/anchore/syft/releases/download/v${{ env.SYFT_VERSION }}/syft_${{ env.SYFT_VERSION }}_linux_${{ steps.os-arch.outputs.arch }}.tar.gz" echo "Downloading: ${DOWNLOAD_URL}" - curl -L -o syft.tar.gz "${DOWNLOAD_URL}" + curl --proto '=https' --tlsv1.2 --location --output syft.tar.gz "${DOWNLOAD_URL}" tar -xzf syft.tar.gz chmod +x syft # Add to PATH for subsequent steps From ee7c21e00e9f20888e1409919e296c57da8fa978 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Wed, 4 Mar 2026 08:15:13 +0000 Subject: [PATCH 08/12] [NRL-1922] Set commit as version for upload-release-action --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 68268ee25..c019d2905 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -111,7 +111,7 @@ jobs: - name: Upload SBOM to release if: ${{ github.event.release.tag_name }} - uses: svenstaro/upload-release-action@v2.11.3 + uses: svenstaro/upload-release-action@b98a3b12e86552593f3e4e577ca8a62aa2f3f22b # v2.11.4 with: file: sbom.spdx.json asset_name: sbom-${{ github.event.release.tag_name }} From a0bccf66cbaa146bb0bdcacf462de4a5934413ed Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Wed, 4 Mar 2026 08:59:29 +0000 Subject: [PATCH 09/12] [NRL-1922] Re-organise all bucket policies to satistfy sonarqube --- .../dev/aws-backup.tf | 64 +++++++++++-------- .../modules/athena/s3.tf | 19 ++++-- .../modules/glue/s3.tf | 6 +- .../modules/permissions-store-bucket/s3.tf | 38 +++++------ .../prod/aws-backup.tf | 54 +++++++++------- .../test/aws-backup.tf | 54 +++++++++------- .../infrastructure/modules/firehose/s3.tf | 20 +++--- .../modules/permissions-store-bucket/s3.tf | 26 ++++++++ 8 files changed, 175 insertions(+), 106 deletions(-) diff --git a/terraform/account-wide-infrastructure/dev/aws-backup.tf b/terraform/account-wide-infrastructure/dev/aws-backup.tf index be24d6c6a..d711d0a71 100644 --- a/terraform/account-wide-infrastructure/dev/aws-backup.tf +++ b/terraform/account-wide-infrastructure/dev/aws-backup.tf @@ -3,37 +3,20 @@ resource "aws_s3_bucket" "backup_reports" { # NOSONAR (S6258) - Logging not requ bucket_prefix = "${local.prefix}-backup-reports" } -resource "aws_s3_bucket_public_access_block" "backup_reports" { - bucket = aws_s3_bucket.backup_reports.id - - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true -} - -resource "aws_s3_bucket_server_side_encryption_configuration" "backup_reports" { - bucket = aws_s3_bucket.backup_reports.bucket - - rule { - apply_server_side_encryption_by_default { - sse_algorithm = "AES256" - } - } -} - -resource "aws_s3_bucket_policy" "backup_reports_bucket_policy" { +resource "aws_s3_bucket_policy" "backup_reports_https_only" { bucket = aws_s3_bucket.backup_reports.id policy = jsonencode({ Version = "2012-10-17" - Id = "backup_reports_bucket_policy" + Id = "backup_reports_https_only_policy" Statement = [ { - Sid = "HTTPSOnly" - Effect = "Deny" - Principal = "*" - Action = "s3:*" + Sid = "HTTPSOnly" + Effect = "Deny" + Principal = { + "AWS" : "*" + } + Action = "s3:*" Resource = [ aws_s3_bucket.backup_reports.arn, "${aws_s3_bucket.backup_reports.arn}/*", @@ -43,7 +26,18 @@ resource "aws_s3_bucket_policy" "backup_reports_bucket_policy" { "aws:SecureTransport" = "false" } } - }, + } + ] + }) +} + +resource "aws_s3_bucket_policy" "backup_reports_write_access" { + bucket = aws_s3_bucket.backup_reports.id + + policy = jsonencode({ + Version = "2012-10-17" + Id = "backup_reports_write_access_policy" + Statement = [ { Sid = "AllowBackupReportsWrite" Effect = "Allow" @@ -64,6 +58,24 @@ resource "aws_s3_bucket_policy" "backup_reports_bucket_policy" { }) } +resource "aws_s3_bucket_public_access_block" "backup_reports" { + bucket = aws_s3_bucket.backup_reports.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "backup_reports" { + bucket = aws_s3_bucket.backup_reports.bucket + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} resource "aws_s3_bucket_ownership_controls" "backup_reports" { bucket = aws_s3_bucket.backup_reports.id diff --git a/terraform/account-wide-infrastructure/modules/athena/s3.tf b/terraform/account-wide-infrastructure/modules/athena/s3.tf index 787f09473..a5464896e 100644 --- a/terraform/account-wide-infrastructure/modules/athena/s3.tf +++ b/terraform/account-wide-infrastructure/modules/athena/s3.tf @@ -2,12 +2,12 @@ resource "aws_s3_bucket" "athena" { # NOSONAR (S6258) - Logging not required for bucket = "${var.name_prefix}-athena" } -resource "aws_s3_bucket_policy" "athena" { - bucket = "${var.name_prefix}-athena" +resource "aws_s3_bucket_policy" "athena-https-only" { + bucket = aws_s3_bucket.athena.id policy = jsonencode({ Version = "2012-10-17" - Id = "athena-policy" + Id = "athena-https-only-policy" Statement = [ { Sid = "HTTPSOnly" @@ -25,7 +25,18 @@ resource "aws_s3_bucket_policy" "athena" { "aws:SecureTransport" = "false" } } - }, + } + ] + }) +} + +resource "aws_s3_bucket_policy" "athena-access" { + bucket = aws_s3_bucket.athena.id + + policy = jsonencode({ + Version = "2012-10-17" + Id = "athena-access-policy" + Statement = [ { Sid : "AllowAthenaAccess", Effect : "Allow", diff --git a/terraform/account-wide-infrastructure/modules/glue/s3.tf b/terraform/account-wide-infrastructure/modules/glue/s3.tf index 7b4445cb4..5656d8564 100644 --- a/terraform/account-wide-infrastructure/modules/glue/s3.tf +++ b/terraform/account-wide-infrastructure/modules/glue/s3.tf @@ -4,7 +4,7 @@ resource "aws_s3_bucket" "source-data-bucket" { # NOSONAR (S6258) - Logging not } resource "aws_s3_bucket_policy" "source-data-bucket" { - bucket = "${var.name_prefix}-source-data-bucket" + bucket = aws_s3_bucket.source-data-bucket.id policy = jsonencode({ Version = "2012-10-17" @@ -79,7 +79,7 @@ resource "aws_s3_bucket" "target-data-bucket" { # NOSONAR (S6258) - Logging not } resource "aws_s3_bucket_policy" "target-data-bucket" { - bucket = "${var.name_prefix}-target-data-bucket" + bucket = aws_s3_bucket.target-data-bucket.id policy = jsonencode({ Version = "2012-10-17" @@ -132,7 +132,7 @@ resource "aws_s3_bucket" "code-bucket" { # NOSONAR (S6258) - Logging not require } resource "aws_s3_bucket_policy" "code-bucket" { - bucket = "${var.name_prefix}-code-bucket" + bucket = aws_s3_bucket.code-bucket.id policy = jsonencode({ Version = "2012-10-17" diff --git a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf index 3711331b2..3693cf6c4 100644 --- a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf +++ b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf @@ -9,25 +9,6 @@ resource "aws_s3_bucket" "authorization-store" { # NOSONAR (S6258) - Logging not } } -resource "aws_s3_bucket_public_access_block" "authorization-store-public-access-block" { - bucket = aws_s3_bucket.authorization-store.id - - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true -} - -resource "aws_s3_bucket_server_side_encryption_configuration" "authorization-store" { - bucket = aws_s3_bucket.authorization-store.bucket - - rule { - apply_server_side_encryption_by_default { - sse_algorithm = "AES256" - } - } -} - resource "aws_s3_bucket_policy" "authorization_store_bucket_policy" { bucket = aws_s3_bucket.authorization-store.id @@ -54,6 +35,25 @@ resource "aws_s3_bucket_policy" "authorization_store_bucket_policy" { }) } +resource "aws_s3_bucket_public_access_block" "authorization-store-public-access-block" { + bucket = aws_s3_bucket.authorization-store.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "authorization-store" { + bucket = aws_s3_bucket.authorization-store.bucket + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + resource "aws_s3_bucket_versioning" "authorization-store" { bucket = aws_s3_bucket.authorization-store.id versioning_configuration { diff --git a/terraform/account-wide-infrastructure/prod/aws-backup.tf b/terraform/account-wide-infrastructure/prod/aws-backup.tf index 2e7095098..5b2509b42 100644 --- a/terraform/account-wide-infrastructure/prod/aws-backup.tf +++ b/terraform/account-wide-infrastructure/prod/aws-backup.tf @@ -3,31 +3,12 @@ resource "aws_s3_bucket" "backup_reports" { # NOSONAR (S6258) - Logging not requ bucket_prefix = "${local.prefix}-backup-reports" } -resource "aws_s3_bucket_public_access_block" "backup_reports" { - bucket = aws_s3_bucket.backup_reports.id - - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true -} - -resource "aws_s3_bucket_server_side_encryption_configuration" "backup_reports" { - bucket = aws_s3_bucket.backup_reports.bucket - - rule { - apply_server_side_encryption_by_default { - sse_algorithm = "AES256" - } - } -} - -resource "aws_s3_bucket_policy" "backup_reports_bucket_policy" { +resource "aws_s3_bucket_policy" "backup_reports_https_only" { bucket = aws_s3_bucket.backup_reports.id policy = jsonencode({ Version = "2012-10-17" - Id = "backup_reports_bucket_policy" + Id = "backup_reports_https_only_policy" Statement = [ { Sid = "HTTPSOnly" @@ -43,7 +24,18 @@ resource "aws_s3_bucket_policy" "backup_reports_bucket_policy" { "aws:SecureTransport" = "false" } } - }, + } + ] + }) +} + +resource "aws_s3_bucket_policy" "backup_reports_read_access" { + bucket = aws_s3_bucket.backup_reports.id + + policy = jsonencode({ + Version = "2012-10-17" + Id = "backup_reports_read_access" + Statement = [ { Sid = "AllowBackupReportsWrite" Effect = "Allow" @@ -64,6 +56,24 @@ resource "aws_s3_bucket_policy" "backup_reports_bucket_policy" { }) } +resource "aws_s3_bucket_public_access_block" "backup_reports" { + bucket = aws_s3_bucket.backup_reports.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "backup_reports" { + bucket = aws_s3_bucket.backup_reports.bucket + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} resource "aws_s3_bucket_ownership_controls" "backup_reports" { bucket = aws_s3_bucket.backup_reports.id diff --git a/terraform/account-wide-infrastructure/test/aws-backup.tf b/terraform/account-wide-infrastructure/test/aws-backup.tf index aa8a22bdf..00429012b 100644 --- a/terraform/account-wide-infrastructure/test/aws-backup.tf +++ b/terraform/account-wide-infrastructure/test/aws-backup.tf @@ -3,31 +3,12 @@ resource "aws_s3_bucket" "backup_reports" { # NOSONAR (S6258) - Logging not requ bucket_prefix = "${local.prefix}-backup-reports" } -resource "aws_s3_bucket_public_access_block" "backup_reports" { - bucket = aws_s3_bucket.backup_reports.id - - block_public_acls = true - block_public_policy = true - ignore_public_acls = true - restrict_public_buckets = true -} - -resource "aws_s3_bucket_server_side_encryption_configuration" "backup_reports" { - bucket = aws_s3_bucket.backup_reports.bucket - - rule { - apply_server_side_encryption_by_default { - sse_algorithm = "AES256" - } - } -} - -resource "aws_s3_bucket_policy" "backup_reports_bucket_policy" { +resource "aws_s3_bucket_policy" "backup_reports_https_only" { bucket = aws_s3_bucket.backup_reports.id policy = jsonencode({ Version = "2012-10-17" - Id = "backup_reports_bucket_policy" + Id = "backup_reports_https_only_policy" Statement = [ { Sid = "HTTPSOnly" @@ -43,7 +24,18 @@ resource "aws_s3_bucket_policy" "backup_reports_bucket_policy" { "aws:SecureTransport" = "false" } } - }, + } + ] + }) +} + +resource "aws_s3_bucket_policy" "backup_reports_read_access" { + bucket = aws_s3_bucket.backup_reports.id + + policy = jsonencode({ + Version = "2012-10-17" + Id = "backup_reports_read_access" + Statement = [ { Sid = "AllowBackupReportsWrite" Effect = "Allow" @@ -64,6 +56,24 @@ resource "aws_s3_bucket_policy" "backup_reports_bucket_policy" { }) } +resource "aws_s3_bucket_public_access_block" "backup_reports" { + bucket = aws_s3_bucket.backup_reports.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "backup_reports" { + bucket = aws_s3_bucket.backup_reports.bucket + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} resource "aws_s3_bucket_ownership_controls" "backup_reports" { bucket = aws_s3_bucket.backup_reports.id diff --git a/terraform/infrastructure/modules/firehose/s3.tf b/terraform/infrastructure/modules/firehose/s3.tf index 960d2e358..52bc4e729 100644 --- a/terraform/infrastructure/modules/firehose/s3.tf +++ b/terraform/infrastructure/modules/firehose/s3.tf @@ -3,16 +3,6 @@ resource "aws_s3_bucket" "firehose" { # NOSONAR (S6258) - Logging not required f force_destroy = true } -resource "aws_s3_bucket_server_side_encryption_configuration" "firehose" { - bucket = aws_s3_bucket.firehose.id - rule { - apply_server_side_encryption_by_default { - kms_master_key_id = aws_kms_key.firehose.arn - sse_algorithm = "aws:kms" - } - } -} - resource "aws_s3_bucket_policy" "firehose-policy" { bucket = aws_s3_bucket.firehose.id @@ -39,6 +29,16 @@ resource "aws_s3_bucket_policy" "firehose-policy" { }) } +resource "aws_s3_bucket_server_side_encryption_configuration" "firehose" { + bucket = aws_s3_bucket.firehose.id + rule { + apply_server_side_encryption_by_default { + kms_master_key_id = aws_kms_key.firehose.arn + sse_algorithm = "aws:kms" + } + } +} + resource "aws_s3_bucket_public_access_block" "firehose-public-access-block" { bucket = aws_s3_bucket.firehose.id diff --git a/terraform/infrastructure/modules/permissions-store-bucket/s3.tf b/terraform/infrastructure/modules/permissions-store-bucket/s3.tf index 071b87320..dccdccda9 100644 --- a/terraform/infrastructure/modules/permissions-store-bucket/s3.tf +++ b/terraform/infrastructure/modules/permissions-store-bucket/s3.tf @@ -8,6 +8,32 @@ resource "aws_s3_bucket" "authorization-store" { # NOSONAR (S6258) - Logging not } } +resource "aws_s3_bucket_policy" "authorization_store_https_only" { + bucket = aws_s3_bucket.authorization-store.id + + policy = jsonencode({ + Version = "2012-10-17" + Id = "authorization_store_https_only_policy" + Statement = [ + { + Sid = "HTTPSOnly" + Effect = "Deny" + Principal = "*" + Action = "s3:*" + Resource = [ + aws_s3_bucket.authorization-store.arn, + "${aws_s3_bucket.authorization-store.arn}/*", + ] + Condition = { + Bool = { + "aws:SecureTransport" = "false" + } + } + }, + ] + }) +} + resource "aws_s3_bucket_public_access_block" "authorization-store-public-access-block" { bucket = aws_s3_bucket.authorization-store.id From ee0e104be2fbf9cd7f1d1d663aba48a323dadc8a Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Wed, 11 Mar 2026 15:53:59 +0000 Subject: [PATCH 10/12] [NRL-1922] Resolve sonarcube issues with default and hard-coded props --- .../modules/glue/src/pipeline.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py b/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py index 6082a99d5..7831fb387 100644 --- a/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py +++ b/terraform/account-wide-infrastructure/modules/glue/src/pipeline.py @@ -1,7 +1,10 @@ +import os import time import boto3 +AWS_REGION = os.getenv("AWS_REGION", "eu-west-2") + class LogPipeline: def __init__( @@ -13,8 +16,8 @@ def __init__( target_path, host_prefixes, job_name, - partition_cols=[], - transformations=[], + partition_cols=None, + transformations=None, ): """Initialize Glue context, Spark session, logger, and paths""" self.glue_context = glue_context @@ -23,12 +26,12 @@ def __init__( self.source_path = source_path self.target_path = target_path self.host_prefixes = host_prefixes - self.partition_cols = partition_cols - self.transformations = transformations + self.partition_cols = partition_cols if partition_cols else [] + self.transformations = transformations if transformations else [] self.glue = boto3.client( service_name="glue", - region_name="eu-west-2", - endpoint_url="https://glue.eu-west-2.amazonaws.com", + region_name=AWS_REGION, + endpoint_url=f"https://glue.{AWS_REGION}.amazonaws.com", ) self.job_name = job_name self.name_prefix = "-".join(job_name.split("-")[:4]) From 0b856531b2a13d82a5406a7677b3ef1c86a53b7c Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Wed, 11 Mar 2026 15:55:43 +0000 Subject: [PATCH 11/12] [NRL-1922] Remove old TODO --- layer/nrlf/core/validators.py | 1 - 1 file changed, 1 deletion(-) diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index 0f1c1a81f..e213c5bfe 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -41,7 +41,6 @@ def validate_type(type_: Optional[RequestQueryType], pointer_types: List[str]) - return type_.root in pointer_types -# TODO - Validate category is in set permissions once permissioning by category is done. def validate_category(categories: Optional[RequestQueryCategory]) -> bool: """ Validates if the given category is valid. From 0c47f751527e7c8c692dbbf7f00247ad3b9c1fe3 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Thu, 12 Mar 2026 07:58:38 +0000 Subject: [PATCH 12/12] [NRL-1922] Add initial unit tests for changes to glue pipeline.py --- pyproject.toml | 2 +- .../modules/glue/tests/test_pipeline.py | 56 +++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 terraform/account-wide-infrastructure/modules/glue/tests/test_pipeline.py diff --git a/pyproject.toml b/pyproject.toml index 4e8612ad6..3b207d4ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -104,7 +104,7 @@ env = [ "AUTH_STORE=auth-store", "TABLE_NAME=unit-test-document-pointer" ] -pythonpath = [".", "./scripts"] +pythonpath = [".", "./scripts", "./terraform/account-wide-infrastructure/modules/glue/src"] [tool.datamodel-codegen] target-python-version = "3.12" diff --git a/terraform/account-wide-infrastructure/modules/glue/tests/test_pipeline.py b/terraform/account-wide-infrastructure/modules/glue/tests/test_pipeline.py new file mode 100644 index 000000000..68d7c6e42 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/glue/tests/test_pipeline.py @@ -0,0 +1,56 @@ +from moto import mock_aws +from pipeline import LogPipeline + + +@mock_aws +def test_pipeline_init_defaults(): + glue_context = "mock_glue_context" + spark = "mock_spark_session" + logger = "mock_logger" + source_path = "s3://mock-source-path" + target_path = "s3://mock-target-path" + host_prefixes = ["host1", "host2"] + job_name = "test-job-name" + + pipeline = LogPipeline( + glue_context, spark, logger, source_path, target_path, host_prefixes, job_name + ) + + assert pipeline.glue_context == glue_context + assert pipeline.spark == spark + assert pipeline.logger == logger + assert pipeline.source_path == source_path + assert pipeline.target_path == target_path + assert pipeline.host_prefixes == host_prefixes + assert pipeline.job_name == job_name + assert pipeline.name_prefix == "test-job-name" + assert pipeline.partition_cols == [] + assert pipeline.transformations == [] + + +@mock_aws +def test_pipeline_init_with_custom_values(): + glue_context = "mock_glue_context" + spark = "mock_spark_session" + logger = "mock_logger" + source_path = "s3://mock-source-path" + target_path = "s3://mock-target-path" + host_prefixes = ["host1", "host2"] + job_name = "test-job-name" + partition_cols = ["col1", "col2"] + transformations = ["transformation1", "transformation2"] + + pipeline = LogPipeline( + glue_context, + spark, + logger, + source_path, + target_path, + host_prefixes, + job_name, + partition_cols=partition_cols, + transformations=transformations, + ) + + assert pipeline.partition_cols == partition_cols + assert pipeline.transformations == transformations