From c40ae8f8c49874b5a4a2ca011e21d93f7b3d287c Mon Sep 17 00:00:00 2001 From: "Richard J. Reyes" Date: Thu, 9 Jul 2026 19:14:25 -0700 Subject: [PATCH 01/11] Add canonical Python container for NIST scanner --- Dockerfile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1f1398a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + MPLBACKEND=Agg \ + PIP_NO_CACHE_DIR=1 + +WORKDIR /app + +COPY requirements.txt ./ +RUN python -m pip install --upgrade pip \ + && python -m pip install --requirement requirements.txt + +COPY . . + +CMD ["python", "scripts/run_feii_bin_stability.py", "--null-n", "100"] From 39bda1bea3d753291278547f3c4df0cafe407df4 Mon Sep 17 00:00:00 2001 From: "Richard J. Reyes" Date: Thu, 9 Jul 2026 19:14:32 -0700 Subject: [PATCH 02/11] Add independent R reproduction container for NIST scanner --- Dockerfile.r | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 Dockerfile.r diff --git a/Dockerfile.r b/Dockerfile.r new file mode 100644 index 0000000..a3e05bd --- /dev/null +++ b/Dockerfile.r @@ -0,0 +1,14 @@ +FROM rocker/r-ver:4.4.2 + +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update \ + && apt-get install -y --no-install-recommends libgmp-dev \ + && rm -rf /var/lib/apt/lists/* + +RUN Rscript -e "install.packages(c('jsonlite', 'gmp', 'testthat'), repos='https://cloud.r-project.org', Ncpus=2)" + +WORKDIR /app +COPY . . + +CMD ["Rscript", "tests/testthat.R"] From 6825c27ab8b0229d79f88051fdfd7a6ac9c51cc6 Mon Sep 17 00:00:00 2001 From: "Richard J. Reyes" Date: Thu, 9 Jul 2026 19:14:36 -0700 Subject: [PATCH 03/11] Add NIST Docker build exclusions --- .dockerignore | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..408a0f3 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +.git +.github +__pycache__/ +*.py[cod] +.pytest_cache/ +.Rhistory +.RData +.Rproj.user/ +.venv/ +venv/ +outputs/ +outputs_*/ +outputs_r/ +tables_r/ From d62ed32c7b3933ca382e2aaf3eedd7b778804187 Mon Sep 17 00:00:00 2001 From: "Richard J. Reyes" Date: Thu, 9 Jul 2026 19:14:45 -0700 Subject: [PATCH 04/11] Add NIST Python, R, and container CI --- .github/workflows/container-ci.yml | 66 ++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 .github/workflows/container-ci.yml diff --git a/.github/workflows/container-ci.yml b/.github/workflows/container-ci.yml new file mode 100644 index 0000000..a8975f7 --- /dev/null +++ b/.github/workflows/container-ci.yml @@ -0,0 +1,66 @@ +name: Python, R, and container CI + +on: + push: + branches: [main] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: container-ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + python-canonical: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + python -m pip install --requirement requirements.txt + - name: Run scanner tests + run: python -m pytest -q + - name: Run a quick canonical Fe II scan + run: | + python scripts/nist_wct_log_spectral_scan_FIXED.py \ + --csv data/Fe_lines.csv \ + --ion 2 \ + --bins 160 \ + --null-n 10 \ + --min-lines 100 \ + --out-dir /tmp/nist-preview + + python-container: + runs-on: ubuntu-24.04 + needs: python-canonical + steps: + - uses: actions/checkout@v4 + - name: Build canonical Python image + run: docker build --tag nist-python-ci . + - name: Run a quick scan in the Python image + run: | + docker run --rm nist-python-ci \ + python scripts/nist_wct_log_spectral_scan_FIXED.py \ + --csv data/Fe_lines.csv \ + --ion 2 \ + --bins 160 \ + --null-n 5 \ + --min-lines 100 \ + --out-dir /tmp/nist-preview + + r-container: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - name: Build independent R image + run: docker build --file Dockerfile.r --tag nist-r-ci . + - name: Run R test suite + run: docker run --rm nist-r-ci From a15c453ce2d42cfc0992fdebeb709534ef1ee29e Mon Sep 17 00:00:00 2001 From: "Richard J. Reyes" Date: Thu, 9 Jul 2026 19:16:11 -0700 Subject: [PATCH 05/11] Keep canonical NIST outputs in container parity tests --- .dockerignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.dockerignore b/.dockerignore index 408a0f3..11257b0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,7 +8,6 @@ __pycache__/ .Rproj.user/ .venv/ venv/ -outputs/ outputs_*/ outputs_r/ tables_r/ From e3b665e8826b8cf1a39703bb19e0684519598bef Mon Sep 17 00:00:00 2001 From: "Richard J. Reyes" Date: Thu, 9 Jul 2026 19:21:48 -0700 Subject: [PATCH 06/11] Make NIST R test failures visible in CI logs --- .github/workflows/container-ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container-ci.yml b/.github/workflows/container-ci.yml index a8975f7..6485f27 100644 --- a/.github/workflows/container-ci.yml +++ b/.github/workflows/container-ci.yml @@ -61,6 +61,9 @@ jobs: steps: - uses: actions/checkout@v4 - name: Build independent R image - run: docker build --file Dockerfile.r --tag nist-r-ci . + run: docker build --quiet --file Dockerfile.r --tag nist-r-ci . - name: Run R test suite - run: docker run --rm nist-r-ci + shell: bash + run: | + set -o pipefail + docker run --rm nist-r-ci 2>&1 | tail -n 250 From 8920232f4793e255d5e6345eb2f2a646c8d9f5c6 Mon Sep 17 00:00:00 2001 From: "Richard J. Reyes" Date: Thu, 9 Jul 2026 19:23:48 -0700 Subject: [PATCH 07/11] Upload complete NIST R test diagnostics --- .github/workflows/container-ci.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container-ci.yml b/.github/workflows/container-ci.yml index 6485f27..89dd185 100644 --- a/.github/workflows/container-ci.yml +++ b/.github/workflows/container-ci.yml @@ -63,7 +63,21 @@ jobs: - name: Build independent R image run: docker build --quiet --file Dockerfile.r --tag nist-r-ci . - name: Run R test suite + id: r_tests shell: bash run: | - set -o pipefail - docker run --rm nist-r-ci 2>&1 | tail -n 250 + set +e + docker run --rm nist-r-ci > r-test.log 2>&1 + status=$? + echo "status=$status" >> "$GITHUB_OUTPUT" + tail -n 250 r-test.log + exit 0 + - name: Upload R test diagnostics + if: always() + uses: actions/upload-artifact@v4 + with: + name: nist-r-test-log-${{ github.sha }} + if-no-files-found: error + path: r-test.log + - name: Enforce R test result + run: test "${{ steps.r_tests.outputs.status }}" -eq 0 From 51a9b3ec444609ec4bbbc5ec741593813af5d95d Mon Sep 17 00:00:00 2001 From: "Richard J. Reyes" Date: Thu, 9 Jul 2026 19:26:08 -0700 Subject: [PATCH 08/11] Fail closed when NIST R dependencies are unavailable --- Dockerfile.r | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile.r b/Dockerfile.r index a3e05bd..ee972ad 100644 --- a/Dockerfile.r +++ b/Dockerfile.r @@ -3,10 +3,15 @@ FROM rocker/r-ver:4.4.2 ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ - && apt-get install -y --no-install-recommends libgmp-dev \ + && apt-get install -y --no-install-recommends \ + build-essential \ + libcurl4-openssl-dev \ + libgmp-dev \ + libssl-dev \ + libxml2-dev \ && rm -rf /var/lib/apt/lists/* -RUN Rscript -e "install.packages(c('jsonlite', 'gmp', 'testthat'), repos='https://cloud.r-project.org', Ncpus=2)" +RUN Rscript -e "install.packages(c('jsonlite', 'gmp', 'testthat'), repos='https://cloud.r-project.org', Ncpus=2); stopifnot(requireNamespace('jsonlite', quietly=TRUE), requireNamespace('gmp', quietly=TRUE), requireNamespace('testthat', quietly=TRUE))" WORKDIR /app COPY . . From bb771c9ab793bf5992ac24c2bc1773aeb90e7485 Mon Sep 17 00:00:00 2001 From: "Richard J. Reyes" Date: Thu, 9 Jul 2026 19:33:59 -0700 Subject: [PATCH 09/11] Use distro R packages for reliable container builds --- Dockerfile.r | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Dockerfile.r b/Dockerfile.r index ee972ad..b166b0f 100644 --- a/Dockerfile.r +++ b/Dockerfile.r @@ -2,16 +2,18 @@ FROM rocker/r-ver:4.4.2 ENV DEBIAN_FRONTEND=noninteractive +# Use distribution-built R packages instead of compiling the complete testthat +# dependency tree from CRAN during every image build. This removes transient +# source-build failures while retaining the repository-pinned R interpreter. RUN apt-get update \ && apt-get install -y --no-install-recommends \ - build-essential \ - libcurl4-openssl-dev \ - libgmp-dev \ - libssl-dev \ - libxml2-dev \ + libgmp-dev \ + r-cran-gmp \ + r-cran-jsonlite \ + r-cran-testthat \ && rm -rf /var/lib/apt/lists/* -RUN Rscript -e "install.packages(c('jsonlite', 'gmp', 'testthat'), repos='https://cloud.r-project.org', Ncpus=2); stopifnot(requireNamespace('jsonlite', quietly=TRUE), requireNamespace('gmp', quietly=TRUE), requireNamespace('testthat', quietly=TRUE))" +RUN Rscript -e "required <- c('jsonlite', 'gmp', 'testthat'); missing <- required[!vapply(required, requireNamespace, logical(1), quietly=TRUE)]; if (length(missing)) stop('missing R packages: ', paste(missing, collapse=', '))" WORKDIR /app COPY . . From 1d03bc2972e3323c304709a618bd4aca118755f0 Mon Sep 17 00:00:00 2001 From: "Richard J. Reyes" Date: Thu, 9 Jul 2026 19:35:55 -0700 Subject: [PATCH 10/11] Capture R image build diagnostics --- .github/workflows/container-ci.yml | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/.github/workflows/container-ci.yml b/.github/workflows/container-ci.yml index 89dd185..f6b620e 100644 --- a/.github/workflows/container-ci.yml +++ b/.github/workflows/container-ci.yml @@ -61,9 +61,19 @@ jobs: steps: - uses: actions/checkout@v4 - name: Build independent R image - run: docker build --quiet --file Dockerfile.r --tag nist-r-ci . + id: r_build + shell: bash + run: | + set +e + set -o pipefail + docker build --progress=plain --file Dockerfile.r --tag nist-r-ci . \ + 2>&1 | tee r-build.log + status=${PIPESTATUS[0]} + echo "status=$status" >> "$GITHUB_OUTPUT" + exit 0 - name: Run R test suite id: r_tests + if: steps.r_build.outputs.status == '0' shell: bash run: | set +e @@ -72,12 +82,17 @@ jobs: echo "status=$status" >> "$GITHUB_OUTPUT" tail -n 250 r-test.log exit 0 - - name: Upload R test diagnostics + - name: Upload R diagnostics if: always() uses: actions/upload-artifact@v4 with: - name: nist-r-test-log-${{ github.sha }} - if-no-files-found: error - path: r-test.log - - name: Enforce R test result - run: test "${{ steps.r_tests.outputs.status }}" -eq 0 + name: nist-r-diagnostics-${{ github.sha }} + if-no-files-found: warn + path: | + r-build.log + r-test.log + - name: Enforce R image and test results + shell: bash + run: | + test "${{ steps.r_build.outputs.status }}" -eq 0 + test "${{ steps.r_tests.outputs.status }}" -eq 0 From 3c98994c52d724592bfa4246116b968575f396b0 Mon Sep 17 00:00:00 2001 From: "Richard J. Reyes" Date: Thu, 9 Jul 2026 19:37:52 -0700 Subject: [PATCH 11/11] Install R dependencies with Rocker package tooling --- Dockerfile.r | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/Dockerfile.r b/Dockerfile.r index b166b0f..a59db33 100644 --- a/Dockerfile.r +++ b/Dockerfile.r @@ -2,17 +2,18 @@ FROM rocker/r-ver:4.4.2 ENV DEBIAN_FRONTEND=noninteractive -# Use distribution-built R packages instead of compiling the complete testthat -# dependency tree from CRAN during every image build. This removes transient -# source-build failures while retaining the repository-pinned R interpreter. +# Rocker versioned images pin both R and the CRAN snapshot. Install only system +# libraries with apt; install R packages into the Rocker R library with +# install2.r so packages cannot be pulled from Ubuntu's separate R runtime. RUN apt-get update \ - && apt-get install -y --no-install-recommends \ - libgmp-dev \ - r-cran-gmp \ - r-cran-jsonlite \ - r-cran-testthat \ + && apt-get install -y --no-install-recommends libgmp-dev \ && rm -rf /var/lib/apt/lists/* +RUN install2.r --error --skipinstalled --ncpus -1 \ + jsonlite \ + gmp \ + testthat + RUN Rscript -e "required <- c('jsonlite', 'gmp', 'testthat'); missing <- required[!vapply(required, requireNamespace, logical(1), quietly=TRUE)]; if (length(missing)) stop('missing R packages: ', paste(missing, collapse=', '))" WORKDIR /app