From edeb9fe658aaeeb63dbf7c4e6e49423e9c4d8ecb Mon Sep 17 00:00:00 2001 From: Todd Young Date: Tue, 21 Jul 2026 22:18:15 -0400 Subject: [PATCH 01/45] build: add secret-free Docker image for chpl-api Templated server.xml/context.xml resolve DB credentials from container environment variables via Tomcat's EnvironmentPropertySource, and entrypoint.sh materializes the JWK signing key from an env var at startup. The image itself contains no secrets and is identical across every environment. Co-Authored-By: Claude Sonnet 5 --- .gitattributes | 4 + docker/Dockerfile | 54 +++++++ docker/entrypoint.sh | 13 ++ docker/tomcat-conf/catalina.properties | 202 +++++++++++++++++++++++++ docker/tomcat-conf/context.xml | 35 +++++ docker/tomcat-conf/server.xml | 123 +++++++++++++++ 6 files changed, 431 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/entrypoint.sh create mode 100644 docker/tomcat-conf/catalina.properties create mode 100644 docker/tomcat-conf/context.xml create mode 100644 docker/tomcat-conf/server.xml diff --git a/.gitattributes b/.gitattributes index 7c64743bf3..01e4fe34b5 100644 --- a/.gitattributes +++ b/.gitattributes @@ -25,6 +25,10 @@ *.xml text *.yml text +# Scripts that execute inside Linux containers must keep LF endings on every +# platform - a CRLF shebang breaks `docker build`/`docker run` on checkout. +docker/entrypoint.sh eol=lf + # These files are binary and should be left untouched # (binary is a macro for -text -diff) *.class binary diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000000..1ed1a46e9b --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,54 @@ +# Stage 1: Build the WAR file using Maven +FROM maven:3.9-eclipse-temurin-21-alpine AS build +WORKDIR /app + +# Copy the pom.xml files first to leverage Docker layer caching +# This ensures that if only source code changes, dependencies are not re-downloaded +COPY chpl/pom.xml . +COPY chpl/chpl-api/pom.xml chpl-api/ +COPY chpl/chpl-resources/pom.xml chpl-resources/ +COPY chpl/chpl-service/pom.xml chpl-service/ + +# Copy the rest of the project files +COPY chpl/chpl-api/lombok.config chpl-api/lombok.config +COPY chpl/chpl-api/src chpl-api/src +COPY chpl/chpl-resources/src chpl-resources/src +COPY chpl/chpl-service/lombok.config chpl-service/lombok.config +COPY chpl/chpl-service/src chpl-service/src + +RUN mvn clean package -DskipTests + +# Stage 2: Deploy to Tomcat +# +# This image contains no secrets and is identical across every environment +# (dev/qa/stage/production). Tomcat config below only has ${ENV_VAR} +# placeholders (resolved at container startup - see catalina.properties' +# org.apache.tomcat.util.digester.PROPERTY_SOURCE) and the checked-in +# environment.properties/email.properties already mark every sensitive key +# as SECRET, relying on Spring's Environment to prefer OS environment +# variables over those classpath files. See entrypoint.sh for the one +# exception (the JWK signing key, which the app reads from a file path). +# +# Required environment variables at `docker run` time: +# DB_URL, DB_USERNAME, DB_PASSWORD - jdbc/openchpl datasource +# JWK_KEY - contents of the RSA JOSE JWK signing key +# plus every property marked SECRET in +# chpl/chpl-resources/src/main/resources/environment.properties and email.properties +# (e.g. SPRING_REDIS_PASSWORD, COGNITO_SECRETKEY, AZURE_CLIENTSECRET_ONC, JIRA_PASSWORD, ...) +FROM tomcat:11.0.23-jdk21 + +COPY docker/tomcat-conf/server.xml /usr/local/tomcat/conf/server.xml +COPY docker/tomcat-conf/context.xml /usr/local/tomcat/conf/context.xml +COPY docker/tomcat-conf/catalina.properties /usr/local/tomcat/conf/catalina.properties +COPY docker/entrypoint.sh /usr/local/tomcat/bin/entrypoint.sh +RUN chmod +x /usr/local/tomcat/bin/entrypoint.sh + +COPY --from=build /app/chpl-api/target/chpl-service.war /usr/local/tomcat/webapps/chpl-service.war + +# Path (not a secret) where the JWK key gets written by entrypoint.sh +ENV keyLocation=/usr/local/tomcat/conf/JSONRsaJoseJWebKey.txt + +# Expose our custom Tomcat port +EXPOSE 8181 + +ENTRYPOINT ["/usr/local/tomcat/bin/entrypoint.sh"] diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100644 index 0000000000..bc4f691013 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Materializes the JWK signing key from an environment variable into the file +# path chpl-api expects (environment.properties: keyLocation). Everything else +# needed at runtime (DB creds, Cognito/Azure/Jira/Datadog/Redis secrets, etc.) +# is read directly from environment variables by Spring/Tomcat - see +# docker/tomcat-conf/ and chpl/chpl-resources/src/main/resources/environment.properties. +set -e + +if [ -n "$JWK_KEY" ]; then + printf '%s\n' "$JWK_KEY" > /usr/local/tomcat/conf/JSONRsaJoseJWebKey.txt +fi + +exec catalina.sh run diff --git a/docker/tomcat-conf/catalina.properties b/docker/tomcat-conf/catalina.properties new file mode 100644 index 0000000000..91ba487f86 --- /dev/null +++ b/docker/tomcat-conf/catalina.properties @@ -0,0 +1,202 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# +# List of comma-separated paths defining the contents of the "common" +# classloader. Prefixes should be used to define what is the repository type. +# Path may be relative to the CATALINA_HOME or CATALINA_BASE path or absolute. +# If left as blank,the JVM system loader will be used as Catalina's "common" +# loader. +# Examples: +# "foo": Add this folder as a class repository +# "foo/*.jar": Add all the JARs of the specified folder as class +# repositories +# "foo/bar.jar": Add bar.jar as a class repository +# +# Note: Values are enclosed in double quotes ("...") in case either the +# ${catalina.base} path or the ${catalina.home} path contains a comma. +# Because double quotes are used for quoting, the double quote character +# may not appear in a path. +common.loader="${catalina.base}/lib","${catalina.base}/lib/*.jar","${catalina.home}/lib","${catalina.home}/lib/*.jar" + +# +# List of comma-separated paths defining the contents of the "server" +# classloader. Prefixes should be used to define what is the repository type. +# Path may be relative to the CATALINA_HOME or CATALINA_BASE path or absolute. +# If left as blank, the "common" loader will be used as Catalina's "server" +# loader. +# Examples: +# "foo": Add this folder as a class repository +# "foo/*.jar": Add all the JARs of the specified folder as class +# repositories +# "foo/bar.jar": Add bar.jar as a class repository +# +# Note: Values may be enclosed in double quotes ("...") in case either the +# ${catalina.base} path or the ${catalina.home} path contains a comma. +# Because double quotes are used for quoting, the double quote character +# may not appear in a path. +server.loader= + +# +# List of comma-separated paths defining the contents of the "shared" +# classloader. Prefixes should be used to define what is the repository type. +# Path may be relative to the CATALINA_BASE path or absolute. If left as blank, +# the "common" loader will be used as Catalina's "shared" loader. +# Examples: +# "foo": Add this folder as a class repository +# "foo/*.jar": Add all the JARs of the specified folder as class +# repositories +# "foo/bar.jar": Add bar.jar as a class repository +# Please note that for single jars, e.g. bar.jar, you need the URL form +# starting with file:. +# +# Note: Values may be enclosed in double quotes ("...") in case either the +# ${catalina.base} path or the ${catalina.home} path contains a comma. +# Because double quotes are used for quoting, the double quote character +# may not appear in a path. +shared.loader= + +# Default list of JAR files that should not be scanned using the JarScanner +# functionality. This is typically used to scan JARs for configuration +# information. JARs that do not contain such information may be excluded from +# the scan to speed up the scanning process. This is the default list. JARs on +# this list are excluded from all scans. The list must be a comma separated list +# of JAR file names. +# The list of JARs to skip may be over-ridden at a Context level for individual +# scan types by configuring a JarScanner with a nested JarScanFilter. +# The JARs listed below include: +# - Tomcat Bootstrap JARs +# - Tomcat API JARs +# - Catalina JARs +# - Jasper JARs +# - Tomcat JARs +# - Common non-Tomcat JARs +# - Test JARs (JUnit, Cobertura and dependencies) +tomcat.util.scan.StandardJarScanFilter.jarsToSkip=\ +annotations-api.jar,\ +ant-junit*.jar,\ +ant-launcher*.jar,\ +ant*.jar,\ +asm-*.jar,\ +aspectj*.jar,\ +bcel*.jar,\ +biz.aQute.bnd*.jar,\ +bootstrap.jar,\ +catalina-ant.jar,\ +catalina-ha.jar,\ +catalina-ssi.jar,\ +catalina-storeconfig.jar,\ +catalina-tribes.jar,\ +catalina.jar,\ +cglib-*.jar,\ +cobertura-*.jar,\ +commons-beanutils*.jar,\ +commons-codec*.jar,\ +commons-collections*.jar,\ +commons-compress*.jar,\ +commons-daemon.jar,\ +commons-dbcp*.jar,\ +commons-digester*.jar,\ +commons-fileupload*.jar,\ +commons-httpclient*.jar,\ +commons-io*.jar,\ +commons-lang*.jar,\ +commons-logging*.jar,\ +commons-math*.jar,\ +commons-pool*.jar,\ +derby-*.jar,\ +dom4j-*.jar,\ +easymock-*.jar,\ +ecj-*.jar,\ +el-api.jar,\ +geronimo-spec-jaxrpc*.jar,\ +h2*.jar,\ +ha-api-*.jar,\ +hamcrest-*.jar,\ +hibernate*.jar,\ +httpclient*.jar,\ +icu4j-*.jar,\ +jakartaee-migration-*.jar,\ +jasper-el.jar,\ +jasper.jar,\ +jaspic-api.jar,\ +jaxb-*.jar,\ +jaxen-*.jar,\ +jaxws-rt-*.jar,\ +jdom-*.jar,\ +jetty-*.jar,\ +jmx-tools.jar,\ +jmx.jar,\ +jsp-api.jar,\ +jstl.jar,\ +jta*.jar,\ +junit-*.jar,\ +junit.jar,\ +log4j*.jar,\ +mail*.jar,\ +objenesis-*.jar,\ +oraclepki.jar,\ +org.hamcrest.core_*.jar,\ +org.junit_*.jar,\ +oro-*.jar,\ +servlet-api-*.jar,\ +servlet-api.jar,\ +slf4j*.jar,\ +taglibs-standard-spec-*.jar,\ +tagsoup-*.jar,\ +tomcat-api.jar,\ +tomcat-coyote.jar,\ +tomcat-coyote-ffm.jar,\ +tomcat-dbcp.jar,\ +tomcat-i18n-*.jar,\ +tomcat-jdbc.jar,\ +tomcat-jni.jar,\ +tomcat-juli-adapters.jar,\ +tomcat-juli.jar,\ +tomcat-util-scan.jar,\ +tomcat-util.jar,\ +tomcat-websocket.jar,\ +tools.jar,\ +unboundid-ldapsdk-*.jar,\ +websocket-api.jar,\ +websocket-client-api.jar,\ +wsdl4j*.jar,\ +xercesImpl.jar,\ +xml-apis.jar,\ +xmlParserAPIs-*.jar,\ +xmlParserAPIs.jar,\ +xom-*.jar + +# Default list of JAR files that should be scanned that overrides the default +# jarsToSkip list above. This is typically used to include a specific JAR that +# has been excluded by a broad file name pattern in the jarsToSkip list. +# The list of JARs to scan may be over-ridden at a Context level for individual +# scan types by configuring a JarScanner with a nested JarScanFilter. +tomcat.util.scan.StandardJarScanFilter.jarsToScan=\ +log4j-taglib*.jar,\ +log4j-jakarta-web*.jar,\ +log4javascript*.jar,\ +slf4j-taglib*.jar + +# String cache configuration. +tomcat.util.buf.StringCache.byte.enabled=true +#tomcat.util.buf.StringCache.char.enabled=true +#tomcat.util.buf.StringCache.trainThreshold=500000 +#tomcat.util.buf.StringCache.cacheSize=5000 + +# Enables ${ENV_VAR} substitution in server.xml/context.xml, resolved from +# container environment variables at startup (used for jdbc/openchpl above). +org.apache.tomcat.util.digester.PROPERTY_SOURCE=org.apache.tomcat.util.digester.EnvironmentPropertySource diff --git a/docker/tomcat-conf/context.xml b/docker/tomcat-conf/context.xml new file mode 100644 index 0000000000..8f002a1d2a --- /dev/null +++ b/docker/tomcat-conf/context.xml @@ -0,0 +1,35 @@ + + + + + + + + WEB-INF/web.xml + WEB-INF/tomcat-web.xml + ${catalina.base}/conf/web.xml + + + + + + + diff --git a/docker/tomcat-conf/server.xml b/docker/tomcat-conf/server.xml new file mode 100644 index 0000000000..351f32a2c9 --- /dev/null +++ b/docker/tomcat-conf/server.xml @@ -0,0 +1,123 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 7dbfa2af791a61ff82a43aab5c6057b35efbe702 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Tue, 21 Jul 2026 22:18:38 -0400 Subject: [PATCH 02/45] ci: publish chpl-api Docker image to GHCR Manual workflow_dispatch trigger; builds docker/Dockerfile and pushes to ghcr.io tagged - and -latest. No secrets required at build time. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/docker-publish.yml | 45 ++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000000..27dce40024 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,45 @@ +name: Publish Docker Image + +on: + workflow_dispatch: {} + +permissions: + contents: read + packages: write + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Compute image metadata + id: meta + run: | + echo "image=ghcr.io/$(echo '${{ github.repository }}' | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_OUTPUT" + echo "sha_short=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" + echo "branch=$(echo '${{ github.ref_name }}' | tr '/' '-')" >> "$GITHUB_OUTPUT" + + # This image contains no secrets - see docker/Dockerfile. Every environment + # (dev/qa/stage/production) runs the exact same image; secrets are supplied + # as container environment variables at `docker run` time, not baked in here. + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: docker/Dockerfile + push: true + tags: | + ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.branch }}-${{ steps.meta.outputs.sha_short }} + ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.branch }}-latest From 1cf2ee3dd2961dcfeb29143a2b6d3de1a08c903b Mon Sep 17 00:00:00 2001 From: Todd Young Date: Tue, 21 Jul 2026 22:38:38 -0400 Subject: [PATCH 03/45] test: temporarily trigger docker-publish on pull_request workflow_dispatch only appears in the Actions UI once this file exists on the default branch. Adding pull_request lets the workflow be validated on this PR first. Also fixes branch-name resolution for PR events (github.ref_name is '/merge' there, not the branch name). Must be reverted before merging to the default branch. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/docker-publish.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 27dce40024..2e74d60708 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -2,6 +2,10 @@ name: Publish Docker Image on: workflow_dispatch: {} + # TEMPORARY: workflow_dispatch only shows up in the Actions UI once this file + # exists on the default branch, so pull_request lets us test it before that. + # Remove this trigger once the workflow has been validated via a PR. + pull_request: {} permissions: contents: read @@ -29,7 +33,7 @@ jobs: run: | echo "image=ghcr.io/$(echo '${{ github.repository }}' | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_OUTPUT" echo "sha_short=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" - echo "branch=$(echo '${{ github.ref_name }}' | tr '/' '-')" >> "$GITHUB_OUTPUT" + echo "branch=$(echo '${{ github.head_ref || github.ref_name }}' | tr '/' '-')" >> "$GITHUB_OUTPUT" # This image contains no secrets - see docker/Dockerfile. Every environment # (dev/qa/stage/production) runs the exact same image; secrets are supplied From eafd1a814f0e19f7ca7e0bd75ba99ac2201dfb01 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Tue, 21 Jul 2026 23:01:39 -0400 Subject: [PATCH 04/45] ci: tag chpl-api image with build number, sha, and branch pointer Replaces -/-latest with: build- - immutable, sequential build identifier sha- - immutable, traces to the exact commit latest- - floating pointer to the most recent build from that branch (latest-development, latest-qa, latest-staging, latest-production once triggered from those branches) Co-Authored-By: Claude Sonnet 5 --- .github/workflows/docker-publish.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 2e74d60708..b3f9c19a86 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -36,8 +36,17 @@ jobs: echo "branch=$(echo '${{ github.head_ref || github.ref_name }}' | tr '/' '-')" >> "$GITHUB_OUTPUT" # This image contains no secrets - see docker/Dockerfile. Every environment - # (dev/qa/stage/production) runs the exact same image; secrets are supplied - # as container environment variables at `docker run` time, not baked in here. + # (development/qa/staging/production) runs the exact same image; secrets are + # supplied as container environment variables at `docker run` time, not baked + # in here. + # + # Tags: + # build- - immutable, sequential build number (github.run_number + # never repeats or goes backwards for this workflow) + # sha- - immutable, traces back to the exact commit + # latest- - floating pointer to the most recent build from that + # branch (latest-development, latest-qa, latest-staging, + # latest-production once triggered from those branches) - name: Build and push uses: docker/build-push-action@v6 with: @@ -45,5 +54,6 @@ jobs: file: docker/Dockerfile push: true tags: | - ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.branch }}-${{ steps.meta.outputs.sha_short }} - ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.branch }}-latest + ${{ steps.meta.outputs.image }}:build-${{ github.run_number }} + ${{ steps.meta.outputs.image }}:sha-${{ steps.meta.outputs.sha_short }} + ${{ steps.meta.outputs.image }}:latest-${{ steps.meta.outputs.branch }} From f11e1e5b4c5b3dd69a7973211cbd75bb75789144 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 22 Jul 2026 09:42:27 -0400 Subject: [PATCH 05/45] docs: document required env vars for the chpl-api Docker image Lists every environment variable the secret-free image needs or accepts (DB/JWK required-to-start vars, filesystem-path vars needing a volume mount, environment-dependent non-secret vars, and all 31 properties currently marked SECRET in environment.properties/ email.properties, grouped by feature area). Points at AudaciousInquiry/chpl-build for where real values live today without reproducing any of them. Co-Authored-By: Claude Sonnet 5 --- docker/README.md | 150 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 docker/README.md diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000000..39dfa6957d --- /dev/null +++ b/docker/README.md @@ -0,0 +1,150 @@ +# Running the chpl-api Docker image + +`docker/Dockerfile` builds a **secret-free** image: the same image is used for +every environment (development/qa/staging/production). Nothing sensitive is +baked in at build time - all of it is supplied as container environment +variables when the image is actually run. See `docker/Dockerfile`, +`docker/tomcat-conf/`, and `docker/entrypoint.sh` for how that works +mechanically (Tomcat's `EnvironmentPropertySource` for `server.xml`, Spring's +`Environment` for everything else, and `entrypoint.sh` for the one exception - +the JWK signing key, which the app reads from a file rather than a property). + +This doc lists every environment variable the image needs or accepts. It does +**not** contain real values for any environment - only what each variable is +for and what kind of value it expects. Real values for existing environments +currently live in [AudaciousInquiry/chpl-build](https://github.com/AudaciousInquiry/chpl-build) +(`chpl-build-{dev,qa,stg,prod}/src/main/resources/override-api-properties*.sh`) +- that repo should stay private, and its values should never be copied into +this one. + +## Quick start + +```sh +docker run -d --name chpl-api \ + -p 8181:8181 \ + -e DB_URL="jdbc:postgresql://:5432/openchpl" \ + -e DB_USERNAME="" \ + -e DB_PASSWORD="" \ + -e JWK_KEY="$(cat /path/to/JSONRsaJoseJWebKey.txt)" \ + -e chplUrlBegin="https://chpl-dev.healthit.gov" \ + -e SPRING_REDIS_HOST="" \ + -e SPRING_REDIS_PORT="6379" \ + -e SPRING_REDIS_PASSWORD="" \ + ghcr.io//chpl-api:latest-development +``` + +Add `-e` flags for whichever of the feature-area variables below apply to +your environment - omitted ones just leave that feature unconfigured. + +Note: property keys that contain dots (like `spring.redis.host`) are **not** +valid POSIX environment variable names on every shell/platform - Spring will +also match the all-caps/underscore form (`SPRING_REDIS_HOST`). Use whichever +form your deployment tooling handles more easily; both resolve to the same +property. + +## Required to start at all + +| Env var | Used for | +|---|---| +| `DB_URL` | JDBC URL for the `jdbc/openchpl` datasource (`server.xml`) | +| `DB_USERNAME` | DB username (`server.xml`) | +| `DB_PASSWORD` | DB password (`server.xml`) | +| `JWK_KEY` | Contents of the RSA JOSE JWK signing key. `entrypoint.sh` writes this to `/usr/local/tomcat/conf/JSONRsaJoseJWebKey.txt` at container start (`keyLocation` is already set in the image - you don't need to set it). | + +## Filesystem paths (need a mounted volume, not just an env var) + +These properties are file/directory paths the app reads or writes at runtime. +Mount a volume at the path you set, or the app will fail writing to a +read-only container filesystem. + +| Env var | Property | Purpose | +|---|---|---| +| `downloadFolderPath` | `downloadFolderPath` | Where generated downloadable files are written | +| `auditDataFilePath` | `auditDataFilePath` | Where audit data backups are written | + +## Environment-dependent (not secret, but should differ per environment) + +| Env var | Property | Purpose | Example shape | +|---|---|---|---| +| `chplUrlBegin` | `chplUrlBegin` | Public base URL for this environment | `https://chpl-qa.healthit.gov` | +| `EMAILBUILDER_CONFIG_EMAILSUBJECTSUFFIX` | `emailBuilder.config.emailSubjectSuffix` | Tag appended to outgoing email subjects | `[QA]` | +| `SERVER_ENVIRONMENT` | `server.environment` | `non-production` or `production` | | +| `REPORT_ENVIRONMENT` | `report.environment` | Label used on generated reports | `DEV`, `QA`, `STAGE`, `PROD` | + +## Feature-area secrets + +Every property below is marked `SECRET` in +[`environment.properties`](../chpl/chpl-resources/src/main/resources/environment.properties) +or [`email.properties`](../chpl/chpl-resources/src/main/resources/email.properties) +- meaning the app has no usable default and one of these env vars must be set +for that feature to work. If a feature isn't used in a given environment, its +variables can be omitted. + +**Database / cache** +| Env var | Property | +|---|---| +| `SPRING_REDIS_HOST` | `spring.redis.host` | +| `SPRING_REDIS_PORT` | `spring.redis.port` | +| `SPRING_REDIS_PASSWORD` | `spring.redis.password` | + +**AWS Cognito (authentication)** +| Env var | Property | +|---|---| +| `COGNITO_ACCESSKEY` | `cognito.accessKey` | +| `COGNITO_SECRETKEY` | `cognito.secretKey` | +| `COGNITO_REGION` | `cognito.region` | +| `COGNITO_USERPOOLID` | `cognito.userPoolId` | +| `COGNITO_USERPOOLCLIENTSECRET` | `cognito.userPoolClientSecret` | +| `COGNITO_CLIENTID` | `cognito.clientId` | +| `COGNITO_ENVIRONMENT_GROUPNAME` | `cognito.environment.groupName` | +| `COGNITO_SYSTEMUSERUUID` | `cognito.systemUserUuid` | +| `COGNITO_ANONYMOUSUSERUUID` | `cognito.anonymousUserUuid` | + +**ONC Azure AD** +| Env var | Property | +|---|---| +| `AZURE_USER_ONC` | `azure.user.onc` | +| `AZURE_CLIENTID_ONC` | `azure.clientId.onc` | +| `AZURE_CLIENTSECRET_ONC` | `azure.clientSecret.onc` | +| `AZURE_TENANTID_ONC` | `azure.tenantId.onc` | + +**JIRA** +| Env var | Property | +|---|---| +| `JIRA_USERNAME` | `jira.username` | +| `JIRA_PASSWORD` | `jira.password` | + +**Datadog** +| Env var | Property | +|---|---| +| `DATADOG_APIKEY` | `datadog.apiKey` | +| `DATADOG_APPKEY` | `datadog.appKey` | + +**AIA (Real World Testing validation)** +| Env var | Property | +|---|---| +| `AIA_AUTHENTICATE_CLIENTSECRET` | `aia.authenticate.clientSecret` | +| `AIA_AUTHENTICATE_CLIENTID` | `aia.authenticate.clientId` | + +**FF4J admin console** +| Env var | Property | +|---|---| +| `FF4J_WEBCONSOLE_USERNAME` | `ff4j.webconsole.username` | +| `FF4J_WEBCONSOLE_PASSWORD` | `ff4j.webconsole.password` | + +**Email / notifications** +| Env var | Property | +|---|---| +| `internalErrorEmailRecipients` | `internalErrorEmailRecipients` | +| `internalFutureCertificationStatusEmailRecipients` | `internalFutureCertificationStatusEmailRecipients` | +| `emailBuilder_config_forwardAddress` | `emailBuilder_config_forwardAddress` | +| `DIRECTREVIEW_CHPLCHANGES_EMAIL` | `directReview.chplChanges.email` | +| `DIRECTREVIEW_UNKNOWNCHANGES_EMAIL` | `directReview.unknownChanges.email` | + +## Anything not listed here + +`environment.properties`, `email.properties`, `lookup.properties`, and +`errors.properties` (all in `chpl/chpl-resources/src/main/resources/`) have +sensible non-secret defaults for everything else, and can still be overridden +the same way (env var name = property name, dots -> underscores, upper-cased) +if a particular deployment needs to tune something not listed above. From 5816d9028a2457161b9fdf23923ef9e8b1c2b39c Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 22 Jul 2026 10:22:17 -0400 Subject: [PATCH 06/45] ci: disable provenance/sbom attestations for chpl-api image push docker/build-push-action v6 enables build provenance and SBOM attestations by default, each pushing an extra untagged manifest to GHCR per build (2 per run, with no architecture info, cluttering the package version list). Not needed here, so both are turned off. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/docker-publish.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index b3f9c19a86..9442cebebe 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -53,6 +53,12 @@ jobs: context: . file: docker/Dockerfile push: true + # build-push-action v6 enables these by default, which each push a + # separate untagged provenance/SBOM manifest to GHCR alongside the + # real image - not needed here, so turned off to keep the package + # version list clean. + provenance: false + sbom: false tags: | ${{ steps.meta.outputs.image }}:build-${{ github.run_number }} ${{ steps.meta.outputs.image }}:sha-${{ steps.meta.outputs.sha_short }} From 42578dbabbaf55a1bd065f2dc0446a9edcd8c6e7 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 22 Jul 2026 12:07:53 -0400 Subject: [PATCH 07/45] Revert "test: temporarily trigger docker-publish on pull_request" This reverts commit 1cf2ee3dd2961dcfeb29143a2b6d3de1a08c903b. --- .github/workflows/docker-publish.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9442cebebe..c682366620 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -2,10 +2,6 @@ name: Publish Docker Image on: workflow_dispatch: {} - # TEMPORARY: workflow_dispatch only shows up in the Actions UI once this file - # exists on the default branch, so pull_request lets us test it before that. - # Remove this trigger once the workflow has been validated via a PR. - pull_request: {} permissions: contents: read @@ -33,7 +29,7 @@ jobs: run: | echo "image=ghcr.io/$(echo '${{ github.repository }}' | tr '[:upper:]' '[:lower:]')" >> "$GITHUB_OUTPUT" echo "sha_short=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" - echo "branch=$(echo '${{ github.head_ref || github.ref_name }}' | tr '/' '-')" >> "$GITHUB_OUTPUT" + echo "branch=$(echo '${{ github.ref_name }}' | tr '/' '-')" >> "$GITHUB_OUTPUT" # This image contains no secrets - see docker/Dockerfile. Every environment # (development/qa/staging/production) runs the exact same image; secrets are From 419ec7d863087115cc0c2ccb1f9112227b9bb42d Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 22 Jul 2026 12:12:00 -0400 Subject: [PATCH 08/45] ci: build chpl-api image on merge to environment branches Adds push triggers for development, qa, staging, and production so merging into any of them builds and pushes an image tagged latest-, without needing per-branch copies of the workflow. push triggers are evaluated per-ref and aren't gated by which branch is the repo's default, unlike workflow_dispatch. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/docker-publish.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index c682366620..a554c8e50c 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -2,6 +2,15 @@ name: Publish Docker Image on: workflow_dispatch: {} + # One workflow, shared by every environment branch: a merge into any of + # these triggers a build tagged latest-. No per-branch copies + # or separate workflow files to keep in sync. + push: + branches: + - development + - qa + - staging + - production permissions: contents: read From f9fad67ffb40b4a629972111e0fd49df340009bb Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 22 Jul 2026 23:37:39 -0400 Subject: [PATCH 09/45] chore(docker): remove unused JWK_KEY signing-key mechanism No code reads a keyLocation property or local JWK file anymore - JWT verification goes through Cognito's public JWKS URL. Removing the dead entrypoint.sh materialization step and its references in the Dockerfile, README, and server.xml. --- docker/Dockerfile | 11 +---------- docker/README.md | 11 ++++------- docker/entrypoint.sh | 13 ------------- docker/tomcat-conf/server.xml | 2 +- 4 files changed, 6 insertions(+), 31 deletions(-) delete mode 100644 docker/entrypoint.sh diff --git a/docker/Dockerfile b/docker/Dockerfile index 1ed1a46e9b..b9cc717c53 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -26,12 +26,10 @@ RUN mvn clean package -DskipTests # org.apache.tomcat.util.digester.PROPERTY_SOURCE) and the checked-in # environment.properties/email.properties already mark every sensitive key # as SECRET, relying on Spring's Environment to prefer OS environment -# variables over those classpath files. See entrypoint.sh for the one -# exception (the JWK signing key, which the app reads from a file path). +# variables over those classpath files. # # Required environment variables at `docker run` time: # DB_URL, DB_USERNAME, DB_PASSWORD - jdbc/openchpl datasource -# JWK_KEY - contents of the RSA JOSE JWK signing key # plus every property marked SECRET in # chpl/chpl-resources/src/main/resources/environment.properties and email.properties # (e.g. SPRING_REDIS_PASSWORD, COGNITO_SECRETKEY, AZURE_CLIENTSECRET_ONC, JIRA_PASSWORD, ...) @@ -40,15 +38,8 @@ FROM tomcat:11.0.23-jdk21 COPY docker/tomcat-conf/server.xml /usr/local/tomcat/conf/server.xml COPY docker/tomcat-conf/context.xml /usr/local/tomcat/conf/context.xml COPY docker/tomcat-conf/catalina.properties /usr/local/tomcat/conf/catalina.properties -COPY docker/entrypoint.sh /usr/local/tomcat/bin/entrypoint.sh -RUN chmod +x /usr/local/tomcat/bin/entrypoint.sh COPY --from=build /app/chpl-api/target/chpl-service.war /usr/local/tomcat/webapps/chpl-service.war -# Path (not a secret) where the JWK key gets written by entrypoint.sh -ENV keyLocation=/usr/local/tomcat/conf/JSONRsaJoseJWebKey.txt - # Expose our custom Tomcat port EXPOSE 8181 - -ENTRYPOINT ["/usr/local/tomcat/bin/entrypoint.sh"] diff --git a/docker/README.md b/docker/README.md index 39dfa6957d..fb863b9c96 100644 --- a/docker/README.md +++ b/docker/README.md @@ -3,11 +3,10 @@ `docker/Dockerfile` builds a **secret-free** image: the same image is used for every environment (development/qa/staging/production). Nothing sensitive is baked in at build time - all of it is supplied as container environment -variables when the image is actually run. See `docker/Dockerfile`, -`docker/tomcat-conf/`, and `docker/entrypoint.sh` for how that works -mechanically (Tomcat's `EnvironmentPropertySource` for `server.xml`, Spring's -`Environment` for everything else, and `entrypoint.sh` for the one exception - -the JWK signing key, which the app reads from a file rather than a property). +variables when the image is actually run. See `docker/Dockerfile` and +`docker/tomcat-conf/` for how that works mechanically (Tomcat's +`EnvironmentPropertySource` for `server.xml`, Spring's `Environment` for +everything else). This doc lists every environment variable the image needs or accepts. It does **not** contain real values for any environment - only what each variable is @@ -25,7 +24,6 @@ docker run -d --name chpl-api \ -e DB_URL="jdbc:postgresql://:5432/openchpl" \ -e DB_USERNAME="" \ -e DB_PASSWORD="" \ - -e JWK_KEY="$(cat /path/to/JSONRsaJoseJWebKey.txt)" \ -e chplUrlBegin="https://chpl-dev.healthit.gov" \ -e SPRING_REDIS_HOST="" \ -e SPRING_REDIS_PORT="6379" \ @@ -49,7 +47,6 @@ property. | `DB_URL` | JDBC URL for the `jdbc/openchpl` datasource (`server.xml`) | | `DB_USERNAME` | DB username (`server.xml`) | | `DB_PASSWORD` | DB password (`server.xml`) | -| `JWK_KEY` | Contents of the RSA JOSE JWK signing key. `entrypoint.sh` writes this to `/usr/local/tomcat/conf/JSONRsaJoseJWebKey.txt` at container start (`keyLocation` is already set in the image - you don't need to set it). | ## Filesystem paths (need a mounted volume, not just an env var) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh deleted file mode 100644 index bc4f691013..0000000000 --- a/docker/entrypoint.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -# Materializes the JWK signing key from an environment variable into the file -# path chpl-api expects (environment.properties: keyLocation). Everything else -# needed at runtime (DB creds, Cognito/Azure/Jira/Datadog/Redis secrets, etc.) -# is read directly from environment variables by Spring/Tomcat - see -# docker/tomcat-conf/ and chpl/chpl-resources/src/main/resources/environment.properties. -set -e - -if [ -n "$JWK_KEY" ]; then - printf '%s\n' "$JWK_KEY" > /usr/local/tomcat/conf/JSONRsaJoseJWebKey.txt -fi - -exec catalina.sh run diff --git a/docker/tomcat-conf/server.xml b/docker/tomcat-conf/server.xml index 351f32a2c9..9b78900dc6 100644 --- a/docker/tomcat-conf/server.xml +++ b/docker/tomcat-conf/server.xml @@ -52,7 +52,7 @@ pathname="conf/tomcat-users.xml" /> Date: Wed, 22 Jul 2026 23:37:47 -0400 Subject: [PATCH 10/45] ci: add .dockerignore to shrink chpl-api image build context .git (177M) and target/ build dirs (chpl-api/target alone is 1.7GB) were being sent to the Docker daemon as build context on every CI push despite never being referenced by docker/Dockerfile. --- .dockerignore | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..41a127f393 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.git +**/target/ +node_modules +.vscode +.claude From 7d1ede25fc271626fe5598c02b0cc5babcfea71a Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 22 Jul 2026 23:54:17 -0400 Subject: [PATCH 11/45] chore: remove orphaned .gitattributes rule for deleted entrypoint.sh docker/entrypoint.sh was deleted (its only job, materializing JWK_KEY, was dead code) - the eol=lf rule for it is now orphaned. --- .gitattributes | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.gitattributes b/.gitattributes index 01e4fe34b5..7c64743bf3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -25,10 +25,6 @@ *.xml text *.yml text -# Scripts that execute inside Linux containers must keep LF endings on every -# platform - a CRLF shebang breaks `docker build`/`docker run` on checkout. -docker/entrypoint.sh eol=lf - # These files are binary and should be left untouched # (binary is a macro for -text -diff) *.class binary From 59e779ba3e0bd93d37bc3dd56edc7a1674060824 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 22 Jul 2026 23:57:14 -0400 Subject: [PATCH 12/45] ci: cancel in-flight docker-publish runs for the same branch The workflow publishes a floating latest- tag; without a concurrency group, two runs for the same branch could race and let an older commit's build finish last, overwriting latest- with a stale image. --- .github/workflows/docker-publish.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a554c8e50c..7838b8bb60 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -12,6 +12,13 @@ on: - staging - production +# Publishes a floating latest- tag - without this, two runs for the +# same branch could race and let an older commit's build finish last, +# overwriting latest- with a stale image. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + permissions: contents: read packages: write From ea9b5d45a688e9a0a0bc53d1cf6e8c288e487424 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Thu, 23 Jul 2026 10:43:58 -0400 Subject: [PATCH 13/45] ci: run unit tests before building docker image Add a unit-tests job to docker-publish.yml that runs `mvn clean test` on JDK 21, mirroring the Bamboo chpl-build-dev "Run API Unit Tests" job. build-and-push now depends on it via needs, so a broken test suite blocks the image push instead of publishing a broken build. --- .github/workflows/docker-publish.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 7838b8bb60..1fdce0d40b 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -24,7 +24,29 @@ permissions: packages: write jobs: + # Mirrors the "Run API Unit Tests" job in the Bamboo CHPL Development + # Deployment plan (chpl-build/chpl-build-dev): `mvn clean test` against + # chpl/pom.xml on JDK 21. Gates the image build so a broken build never + # gets published. + unit-tests: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'temurin' + + - name: Run unit tests + env: + MAVEN_OPTS: -Xms512m -Xmx1024m + run: mvn -B clean test --file chpl/pom.xml + build-and-push: + needs: unit-tests runs-on: ubuntu-latest steps: - name: Checkout From 26a13a13c27bc2f95f4b8b99a7081f2d9f09200b Mon Sep 17 00:00:00 2001 From: Todd Young Date: Thu, 23 Jul 2026 11:15:38 -0400 Subject: [PATCH 14/45] ci: fail fast on compile errors before building docker image Add a compile job that runs `mvn clean package -DskipTests`, the same command docker/Dockerfile's build stage runs, mirroring the "Build API"/CompileApiTask job in the chpl-build-dev Bamboo spec. build-and-push now needs both unit-tests and compile, so a broken build fails fast instead of partway through the slower Docker build. --- .github/workflows/docker-publish.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 1fdce0d40b..423cb83dd3 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -45,8 +45,29 @@ jobs: MAVEN_OPTS: -Xms512m -Xmx1024m run: mvn -B clean test --file chpl/pom.xml + # Mirrors the "Build API" job (CompileApiTask) in the same Bamboo plan, and + # runs the exact `mvn clean package -DskipTests` command that docker/Dockerfile's + # build stage runs - so a compile/packaging failure shows up here, fast, + # instead of partway through the (slower) Docker build. + compile: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'temurin' + + - name: Compile and package + env: + MAVEN_OPTS: -Xms512m -Xmx1024m + run: mvn -B clean package -DskipTests --file chpl/pom.xml + build-and-push: - needs: unit-tests + needs: [unit-tests, compile] runs-on: ubuntu-latest steps: - name: Checkout From cfb9ae66fb39d30fae1e476df642ca143c98d905 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Thu, 23 Jul 2026 11:26:24 -0400 Subject: [PATCH 15/45] test: temporarily break a unit test to verify CI gate Scratch commit for testing docker-publish.yml's unit-tests job - will be reverted. --- .../healthit/chpl/util/CertificationCriterionServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chpl/chpl-service/src/test/java/gov/healthit/chpl/util/CertificationCriterionServiceTest.java b/chpl/chpl-service/src/test/java/gov/healthit/chpl/util/CertificationCriterionServiceTest.java index 8425c93432..00512a1287 100644 --- a/chpl/chpl-service/src/test/java/gov/healthit/chpl/util/CertificationCriterionServiceTest.java +++ b/chpl/chpl-service/src/test/java/gov/healthit/chpl/util/CertificationCriterionServiceTest.java @@ -136,7 +136,7 @@ public void coerceToCriteriaNumber_ValidCriteriaNumber_DoesNotChange() { public void coerceToCriteriaNumber_MissingSpaceBeforeParen_AddsSpace() { String criterionNumber = "170.315(a)(6)"; String result = service.coerceToCriterionNumberFormat(criterionNumber); - assertEquals("170.315 (a)(6)", result); + assertEquals("170.315 (a)(7)", result); // TEMP: intentionally wrong, testing CI gate } @Test From 70ab1942ab05f0f8000295f8f3ad1f4974d583af Mon Sep 17 00:00:00 2001 From: Todd Young Date: Thu, 23 Jul 2026 11:29:04 -0400 Subject: [PATCH 16/45] Revert "test: temporarily break a unit test to verify CI gate" This reverts commit cfb9ae66fb39d30fae1e476df642ca143c98d905. --- .../healthit/chpl/util/CertificationCriterionServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chpl/chpl-service/src/test/java/gov/healthit/chpl/util/CertificationCriterionServiceTest.java b/chpl/chpl-service/src/test/java/gov/healthit/chpl/util/CertificationCriterionServiceTest.java index 00512a1287..8425c93432 100644 --- a/chpl/chpl-service/src/test/java/gov/healthit/chpl/util/CertificationCriterionServiceTest.java +++ b/chpl/chpl-service/src/test/java/gov/healthit/chpl/util/CertificationCriterionServiceTest.java @@ -136,7 +136,7 @@ public void coerceToCriteriaNumber_ValidCriteriaNumber_DoesNotChange() { public void coerceToCriteriaNumber_MissingSpaceBeforeParen_AddsSpace() { String criterionNumber = "170.315(a)(6)"; String result = service.coerceToCriterionNumberFormat(criterionNumber); - assertEquals("170.315 (a)(7)", result); // TEMP: intentionally wrong, testing CI gate + assertEquals("170.315 (a)(6)", result); } @Test From 1d610f9054f7743e99a0659a925ee289247b1543 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Thu, 23 Jul 2026 11:29:41 -0400 Subject: [PATCH 17/45] test: temporarily break compilation to verify CI gate Scratch commit for testing docker-publish.yml's compile job (missing semicolon) - will be reverted. --- .../healthit/chpl/service/CertificationCriterionService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/service/CertificationCriterionService.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/service/CertificationCriterionService.java index 80d268f065..18a50f0380 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/service/CertificationCriterionService.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/service/CertificationCriterionService.java @@ -275,7 +275,7 @@ public boolean isCriteriaNumber(String input) { } public String coerceToCriterionNumberFormat(String input) { - String formatRegex = "^\\d{3}\\.\\d{3}\\s{1}\\([a-z]{1}\\)(\\([0-9]{1,2}\\))?$"; + String formatRegex = "^\\d{3}\\.\\d{3}\\s{1}\\([a-z]{1}\\)(\\([0-9]{1,2}\\))?$" // TEMP: intentionally broken, testing CI gate if (input.matches(formatRegex)) { LOGGER.debug("\tMatches required format. Not changing input."); return input; From 6892a31a6aeaf65764bf5a0ca5c270f1f9f402fc Mon Sep 17 00:00:00 2001 From: Todd Young Date: Thu, 23 Jul 2026 11:31:11 -0400 Subject: [PATCH 18/45] Revert "test: temporarily break compilation to verify CI gate" This reverts commit 1d610f9054f7743e99a0659a925ee289247b1543. --- .../healthit/chpl/service/CertificationCriterionService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/service/CertificationCriterionService.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/service/CertificationCriterionService.java index 18a50f0380..80d268f065 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/service/CertificationCriterionService.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/service/CertificationCriterionService.java @@ -275,7 +275,7 @@ public boolean isCriteriaNumber(String input) { } public String coerceToCriterionNumberFormat(String input) { - String formatRegex = "^\\d{3}\\.\\d{3}\\s{1}\\([a-z]{1}\\)(\\([0-9]{1,2}\\))?$" // TEMP: intentionally broken, testing CI gate + String formatRegex = "^\\d{3}\\.\\d{3}\\s{1}\\([a-z]{1}\\)(\\([0-9]{1,2}\\))?$"; if (input.matches(formatRegex)) { LOGGER.debug("\tMatches required format. Not changing input."); return input; From e5b5ae3e12047113c1cb8f0cad59691b7a0d9160 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Fri, 24 Jul 2026 11:30:39 -0400 Subject: [PATCH 19/45] feat(docker): add tomcat-users.xml with ff4j admin user --- docker/Dockerfile | 1 + docker/tomcat-conf/server.xml | 4 ++- docker/tomcat-conf/tomcat-users.xml | 43 +++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 docker/tomcat-conf/tomcat-users.xml diff --git a/docker/Dockerfile b/docker/Dockerfile index b9cc717c53..73b2cb2447 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -38,6 +38,7 @@ FROM tomcat:11.0.23-jdk21 COPY docker/tomcat-conf/server.xml /usr/local/tomcat/conf/server.xml COPY docker/tomcat-conf/context.xml /usr/local/tomcat/conf/context.xml COPY docker/tomcat-conf/catalina.properties /usr/local/tomcat/conf/catalina.properties +COPY docker/tomcat-conf/tomcat-users.xml /usr/local/tomcat/conf/tomcat-users.xml COPY --from=build /app/chpl-api/target/chpl-service.war /usr/local/tomcat/webapps/chpl-service.war diff --git a/docker/tomcat-conf/server.xml b/docker/tomcat-conf/server.xml index 9b78900dc6..e6056db9e2 100644 --- a/docker/tomcat-conf/server.xml +++ b/docker/tomcat-conf/server.xml @@ -104,7 +104,9 @@ that are performed against this UserDatabase are immediately available for use by the Realm. --> + resourceName="UserDatabase"> + + + + + + + + + + From 71201771623dfb7681f56b6d35a85132d33dc209 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Fri, 24 Jul 2026 12:54:28 -0400 Subject: [PATCH 20/45] revert: remove unused ff4j tomcat-users.xml and credential handler Nothing enforces the ff4jUser role at the servlet-container level - no web.xml security-constraint references it. CHPLHttpSecurityConfig already secures /ff4j-console via Spring Security's own in-memory user sourced from ff4j.webconsole.username/password. The Tomcat-side file added no protection while baking a credential hash into what's otherwise a secret-free image. Co-Authored-By: Claude Sonnet 5 --- docker/Dockerfile | 1 - docker/tomcat-conf/server.xml | 4 +-- docker/tomcat-conf/tomcat-users.xml | 43 ----------------------------- 3 files changed, 1 insertion(+), 47 deletions(-) delete mode 100644 docker/tomcat-conf/tomcat-users.xml diff --git a/docker/Dockerfile b/docker/Dockerfile index 73b2cb2447..b9cc717c53 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -38,7 +38,6 @@ FROM tomcat:11.0.23-jdk21 COPY docker/tomcat-conf/server.xml /usr/local/tomcat/conf/server.xml COPY docker/tomcat-conf/context.xml /usr/local/tomcat/conf/context.xml COPY docker/tomcat-conf/catalina.properties /usr/local/tomcat/conf/catalina.properties -COPY docker/tomcat-conf/tomcat-users.xml /usr/local/tomcat/conf/tomcat-users.xml COPY --from=build /app/chpl-api/target/chpl-service.war /usr/local/tomcat/webapps/chpl-service.war diff --git a/docker/tomcat-conf/server.xml b/docker/tomcat-conf/server.xml index e6056db9e2..9b78900dc6 100644 --- a/docker/tomcat-conf/server.xml +++ b/docker/tomcat-conf/server.xml @@ -104,9 +104,7 @@ that are performed against this UserDatabase are immediately available for use by the Realm. --> - - + resourceName="UserDatabase"/> - - - - - - - - From e22ff1c6be05b8959df160732f6593990f2c03c8 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Fri, 24 Jul 2026 12:54:50 -0400 Subject: [PATCH 21/45] fix(docker): add user-triggers.xml required by Quartz job initializer quartz.properties lists user-triggers.xml in org.quartz.plugin.jobInitializer.fileNames with failOnFileNotFound=true. Unlike its siblings jobs.xml/system-triggers.xml, which ship on the classpath via chpl-resources, this file exists nowhere in the repo or classpath, so Quartz startup would fail without it. Co-Authored-By: Claude Sonnet 5 --- docker/Dockerfile | 6 ++++++ docker/tomcat-conf/user-triggers.xml | 11 +++++++++++ 2 files changed, 17 insertions(+) create mode 100644 docker/tomcat-conf/user-triggers.xml diff --git a/docker/Dockerfile b/docker/Dockerfile index b9cc717c53..c3accb558b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -38,6 +38,12 @@ FROM tomcat:11.0.23-jdk21 COPY docker/tomcat-conf/server.xml /usr/local/tomcat/conf/server.xml COPY docker/tomcat-conf/context.xml /usr/local/tomcat/conf/context.xml COPY docker/tomcat-conf/catalina.properties /usr/local/tomcat/conf/catalina.properties +# Empty Quartz job-scheduling-data template. quartz.properties sets +# org.quartz.plugin.jobInitializer.failOnFileNotFound=true for +# jobs.xml,system-triggers.xml,user-triggers.xml - the first two ship on the +# classpath via chpl-resources, but this one doesn't, so Quartz needs it here +# or the app fails to start. +COPY docker/tomcat-conf/user-triggers.xml /usr/local/tomcat/conf/user-triggers.xml COPY --from=build /app/chpl-api/target/chpl-service.war /usr/local/tomcat/webapps/chpl-service.war diff --git a/docker/tomcat-conf/user-triggers.xml b/docker/tomcat-conf/user-triggers.xml new file mode 100644 index 0000000000..89fca2249e --- /dev/null +++ b/docker/tomcat-conf/user-triggers.xml @@ -0,0 +1,11 @@ + + + + + + From be92bb64233ba2650a371dec4303b0d30d004401 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Mon, 27 Jul 2026 11:17:46 -0400 Subject: [PATCH 22/45] chore(docker): bump Tomcat base image to 11.0.24 Verified every stock conf/ file (web.xml, catalina.properties, context.xml, server.xml, tomcat-users.xml/xsd, jaspic-providers.xml/xsd, logging.properties) is byte-for-byte identical between 11.0.23 and 11.0.24, and re-ran a full local smoke test (DB, Redis, Cognito, FF4J) against the rebuilt image with no behavior change. Co-Authored-By: Claude Sonnet 5 --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c3accb558b..1b0cb4a103 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -33,7 +33,7 @@ RUN mvn clean package -DskipTests # plus every property marked SECRET in # chpl/chpl-resources/src/main/resources/environment.properties and email.properties # (e.g. SPRING_REDIS_PASSWORD, COGNITO_SECRETKEY, AZURE_CLIENTSECRET_ONC, JIRA_PASSWORD, ...) -FROM tomcat:11.0.23-jdk21 +FROM tomcat:11.0.24-jdk21 COPY docker/tomcat-conf/server.xml /usr/local/tomcat/conf/server.xml COPY docker/tomcat-conf/context.xml /usr/local/tomcat/conf/context.xml From 0ecd53cc4515b9309c8fc3079eec0dc97b70cb22 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Mon, 27 Jul 2026 11:21:47 -0400 Subject: [PATCH 23/45] ci: queue docker-publish runs instead of cancelling in-progress builds cancel-in-progress killed a build's push step mid-flight whenever another push landed on the same branch a few minutes later, discarding completed work. Queuing instead guarantees every push gets built and that they finish in the order they were triggered. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/docker-publish.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 423cb83dd3..dd2626986a 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -14,10 +14,13 @@ on: # Publishes a floating latest- tag - without this, two runs for the # same branch could race and let an older commit's build finish last, -# overwriting latest- with a stale image. +# overwriting latest- with a stale image. cancel-in-progress is +# false so runs queue and execute one at a time in the order they were +# triggered, instead of a newer push cancelling a build already in +# progress. concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: false permissions: contents: read From f2517efc60b6fdd4075776c2d33efdc9972eedf0 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Tue, 28 Jul 2026 11:42:47 -0400 Subject: [PATCH 24/45] fix(docker): switch Tomcat connector back to port 8080 Port 8181 was picked without a stated reason when the secret-free image was first added, and doesn't match anything else in the stack: the real dev box's server.xml listens on 8080, and chpl-build's start-tomcat-containers.sh hard-codes -p host:8080 for every environment's docker run. Verified locally (build, deploy, health check) with the corrected port. Co-Authored-By: Claude Sonnet 5 --- docker/Dockerfile | 6 ++++-- docker/README.md | 2 +- docker/tomcat-conf/server.xml | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1b0cb4a103..1903be2936 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -47,5 +47,7 @@ COPY docker/tomcat-conf/user-triggers.xml /usr/local/tomcat/conf/user-triggers.x COPY --from=build /app/chpl-api/target/chpl-service.war /usr/local/tomcat/webapps/chpl-service.war -# Expose our custom Tomcat port -EXPOSE 8181 +# Matches the port every other environment's Tomcat listens on (the real +# dev box's server.xml, and chpl-build's start-tomcat-containers.sh, which +# hard-codes `-p host:8080` for the docker run container-port mapping). +EXPOSE 8080 diff --git a/docker/README.md b/docker/README.md index fb863b9c96..0f1ea4f9c4 100644 --- a/docker/README.md +++ b/docker/README.md @@ -20,7 +20,7 @@ this one. ```sh docker run -d --name chpl-api \ - -p 8181:8181 \ + -p 8080:8080 \ -e DB_URL="jdbc:postgresql://:5432/openchpl" \ -e DB_USERNAME="" \ -e DB_PASSWORD="" \ diff --git a/docker/tomcat-conf/server.xml b/docker/tomcat-conf/server.xml index 9b78900dc6..0dc00997df 100644 --- a/docker/tomcat-conf/server.xml +++ b/docker/tomcat-conf/server.xml @@ -83,9 +83,9 @@ and responses are returned. Documentation at : HTTP Connector: /docs/config/http.html AJP Connector: /docs/config/ajp.html - Define a non-SSL/TLS HTTP/1.1 Connector on port 8181 + Define a non-SSL/TLS HTTP/1.1 Connector on port 8080 --> - From 3ec9817d8f2fae686d392e192864aed196ff5d8a Mon Sep 17 00:00:00 2001 From: Todd Young Date: Tue, 28 Jul 2026 13:39:41 -0400 Subject: [PATCH 25/45] fix: add empty errors-override.properties to stop MessageSource spam CHPLConfig/CHPLServiceConfig's messageSource bean uses ResourceBundleMessageSource with basename "errors-override" as the primary bundle and "errors" as its parent fallback. ResourceBundleMessageSource has no ignore-missing option, so every single message lookup logged a WARN when the bundle couldn't be found at all. The legacy Bamboo deployment worked around this by dropping an empty errors-override.properties into Tomcat's conf dir at runtime (made classpath-visible via shared.loader) - the new secret-free image never created that file, since it never held any real per-environment content to preserve. Checking in an empty file fixes it for every environment with no runtime wiring needed. Verified locally: ResourceBundle.getBundle("errors-override") resolves successfully (0 keys) against the built classpath, and the app starts and serves requests normally. Co-Authored-By: Claude Sonnet 5 --- .../src/main/resources/errors-override.properties | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 chpl/chpl-resources/src/main/resources/errors-override.properties diff --git a/chpl/chpl-resources/src/main/resources/errors-override.properties b/chpl/chpl-resources/src/main/resources/errors-override.properties new file mode 100644 index 0000000000..18eba2a146 --- /dev/null +++ b/chpl/chpl-resources/src/main/resources/errors-override.properties @@ -0,0 +1,8 @@ +# Intentionally empty. CHPLConfig/CHPLServiceConfig's messageSource bean uses +# ResourceBundleMessageSource with basename "errors-override" as the primary +# bundle and "errors" (errors.properties) as its parent fallback - +# ResourceBundleMessageSource has no ignore-missing option, so every message +# lookup logs a WARN if this bundle can't be found on the classpath at all. +# This file just needs to exist; no environment has ever put real content in +# it (see errors.properties' own header comment for the override mechanism +# this was originally meant to support at runtime). From 8b4471e5c8145a23c088b7f5de5d881d43b0e94d Mon Sep 17 00:00:00 2001 From: Todd Young Date: Tue, 28 Jul 2026 15:53:30 -0400 Subject: [PATCH 26/45] fix: make deployed log4j2 config the default so job logs reach Datadog log4j2.xml (the only config always on the classpath, regardless of the -Denv Maven profile that never activates in docker/Dockerfile's build) was using the "-local" appenders/loggers, which route every scheduled job/report/cache logger to a file only - never to stdout - so none of that output ever reached Datadog, only the general app loggers did. Switched to the "deployed" loggers config, which routes every job logger to both a file and a console JSON appender. Doing this surfaced a real, previously-latent bug: Log4j2 does not merge multiple sibling XIncludes within one Configuration - only the last one processed survives, silently dropping the rest (confirmed this also affects the never-yet-exercised resources-{dev,qa,staging, production}/log4j2.xml, which has the identical pattern). Tried the documented XPointer child-selection workaround first (xpointer(/Appenders/node())) but this JVM's XInclude engine doesn't support the xpointer() scheme at all - it fails to parse and Log4j2 silently falls back to its bare-minimum default configuration instead, which is worse (no job routing at all, not even to file). Fixed by merging log4j2-xinclude-file-appenders-console.xml's Console appenders directly into log4j2-xinclude-file-appenders.xml, so log4j2.xml only ever needs one physical block. Verified locally: 0 "Unable to locate appender" errors (previously 66), and confirmed real job-triggered JSON output on stdout for cognitoUserCacheRefreshJob, directReviewCacheRefreshJob, listingSearchCacheRefresh, sharedDataStore, and redisson. Co-Authored-By: Claude Sonnet 5 --- .../log4j2-xinclude-file-appenders.xml | 485 +++++++++++++++++- chpl/chpl-api/src/main/resources/log4j2.xml | 4 +- 2 files changed, 486 insertions(+), 3 deletions(-) diff --git a/chpl/chpl-api/src/main/resources/log4j2-xinclude-file-appenders.xml b/chpl/chpl-api/src/main/resources/log4j2-xinclude-file-appenders.xml index aaec84d5ad..5e60b53a5a 100644 --- a/chpl/chpl-api/src/main/resources/log4j2-xinclude-file-appenders.xml +++ b/chpl/chpl-api/src/main/resources/log4j2-xinclude-file-appenders.xml @@ -1,5 +1,9 @@ - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/chpl/chpl-api/src/main/resources/log4j2.xml b/chpl/chpl-api/src/main/resources/log4j2.xml index 91db23e9c1..09bcf7cfda 100644 --- a/chpl/chpl-api/src/main/resources/log4j2.xml +++ b/chpl/chpl-api/src/main/resources/log4j2.xml @@ -5,6 +5,6 @@ ${sys:catalina.home}/logs - - + + From f0fcd92129758c63166056602b68203e30920f8c Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 29 Jul 2026 11:05:04 -0400 Subject: [PATCH 27/45] test: add temporary marker to verify docker-publish.yml push trigger Reversible marker to confirm the ready-for-integration flow pushes ONC-5395's Bamboo/Docker migration work to upstream/development and kicks off docker-publish.yml. Revert once the test build is confirmed. --- docker/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/README.md b/docker/README.md index 0f1ea4f9c4..5f3988f7ff 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,5 +1,7 @@ # Running the chpl-api Docker image + + `docker/Dockerfile` builds a **secret-free** image: the same image is used for every environment (development/qa/staging/production). Nothing sensitive is baked in at build time - all of it is supplied as container environment From 002967fc757b719223d936679962fd9062b2eac5 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 29 Jul 2026 11:27:55 -0400 Subject: [PATCH 28/45] test: revert marker and add intentionally failing unit test Reverts the docker-publish.yml push-trigger marker now that it's confirmed working. Adds a standalone failing test to verify the unit-tests job correctly gates build-and-push from running. Both changes are temporary and will be reverted after this test build. --- .../gov/healthit/chpl/TemporaryFailingTest.java | 15 +++++++++++++++ docker/README.md | 2 -- 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 chpl/chpl-service/src/test/java/gov/healthit/chpl/TemporaryFailingTest.java diff --git a/chpl/chpl-service/src/test/java/gov/healthit/chpl/TemporaryFailingTest.java b/chpl/chpl-service/src/test/java/gov/healthit/chpl/TemporaryFailingTest.java new file mode 100644 index 0000000000..e2bddbb041 --- /dev/null +++ b/chpl/chpl-service/src/test/java/gov/healthit/chpl/TemporaryFailingTest.java @@ -0,0 +1,15 @@ +package gov.healthit.chpl; + +import static org.junit.jupiter.api.Assertions.fail; + +import org.junit.jupiter.api.Test; + +// TEMPORARY (ONC-5395): intentionally fails to verify docker-publish.yml +// gates the image build on a failing unit-tests job. Delete this file +// once the test build is confirmed to fail as expected. +public class TemporaryFailingTest { + @Test + public void intentionalFailure() { + fail("Intentional failure to test docker-publish.yml unit-tests gate"); + } +} diff --git a/docker/README.md b/docker/README.md index 5f3988f7ff..0f1ea4f9c4 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,7 +1,5 @@ # Running the chpl-api Docker image - - `docker/Dockerfile` builds a **secret-free** image: the same image is used for every environment (development/qa/staging/production). Nothing sensitive is baked in at build time - all of it is supplied as container environment From 93bf01e213deb3868cceaa22a01d0ebd1743f467 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 29 Jul 2026 11:31:55 -0400 Subject: [PATCH 29/45] test: remove intentionally failing unit test Confirmed docker-publish.yml's unit-tests job correctly gates build-and-push on failure. Removing the temporary test now that the negative-path test is complete. --- .../gov/healthit/chpl/TemporaryFailingTest.java | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 chpl/chpl-service/src/test/java/gov/healthit/chpl/TemporaryFailingTest.java diff --git a/chpl/chpl-service/src/test/java/gov/healthit/chpl/TemporaryFailingTest.java b/chpl/chpl-service/src/test/java/gov/healthit/chpl/TemporaryFailingTest.java deleted file mode 100644 index e2bddbb041..0000000000 --- a/chpl/chpl-service/src/test/java/gov/healthit/chpl/TemporaryFailingTest.java +++ /dev/null @@ -1,15 +0,0 @@ -package gov.healthit.chpl; - -import static org.junit.jupiter.api.Assertions.fail; - -import org.junit.jupiter.api.Test; - -// TEMPORARY (ONC-5395): intentionally fails to verify docker-publish.yml -// gates the image build on a failing unit-tests job. Delete this file -// once the test build is confirmed to fail as expected. -public class TemporaryFailingTest { - @Test - public void intentionalFailure() { - fail("Intentional failure to test docker-publish.yml unit-tests gate"); - } -} From 50468afa81a309c15e7a69cac1702de93ca1e50b Mon Sep 17 00:00:00 2001 From: Todd Young Date: Mon, 10 Aug 2026 10:59:56 -0400 Subject: [PATCH 30/45] revert: remove unused Quartz user-triggers.xml from Docker image quartz.properties' default profile (the one the Docker build actually packages, since it never passes -Denv=) only references jobs.xml,startup-triggers.xml. user-triggers.xml was added under the mistaken belief failOnFileNotFound required it, but Quartz never looks for it in this image - confirmed by extracting the packaged quartz.properties and tracing CHPLServiceConfig's schedulerFactory bean. Same category as the earlier tomcat-users.xml revert. --- docker/Dockerfile | 6 ------ docker/tomcat-conf/user-triggers.xml | 11 ----------- 2 files changed, 17 deletions(-) delete mode 100644 docker/tomcat-conf/user-triggers.xml diff --git a/docker/Dockerfile b/docker/Dockerfile index 1903be2936..5296c567ff 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -38,12 +38,6 @@ FROM tomcat:11.0.24-jdk21 COPY docker/tomcat-conf/server.xml /usr/local/tomcat/conf/server.xml COPY docker/tomcat-conf/context.xml /usr/local/tomcat/conf/context.xml COPY docker/tomcat-conf/catalina.properties /usr/local/tomcat/conf/catalina.properties -# Empty Quartz job-scheduling-data template. quartz.properties sets -# org.quartz.plugin.jobInitializer.failOnFileNotFound=true for -# jobs.xml,system-triggers.xml,user-triggers.xml - the first two ship on the -# classpath via chpl-resources, but this one doesn't, so Quartz needs it here -# or the app fails to start. -COPY docker/tomcat-conf/user-triggers.xml /usr/local/tomcat/conf/user-triggers.xml COPY --from=build /app/chpl-api/target/chpl-service.war /usr/local/tomcat/webapps/chpl-service.war diff --git a/docker/tomcat-conf/user-triggers.xml b/docker/tomcat-conf/user-triggers.xml deleted file mode 100644 index 89fca2249e..0000000000 --- a/docker/tomcat-conf/user-triggers.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - From 01a2dd2361e79e781b35899a24a176a23328ef64 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Thu, 13 Aug 2026 10:50:47 -0400 Subject: [PATCH 31/45] fix: activate per-environment log4j2 profile in GHCR image builds docker/Dockerfile ran `mvn clean package -DskipTests` with no -Denv flag, so chpl-resources' environment-specific Maven profile never activated and resources-{dev,qa,staging,production}/log4j2.xml never got layered onto the WAR. That file defines the catch-all gov.healthit.chpl logger and the chplserviceJson console appender - without it, Datadog's chplservice service tag never appears for images built this way. Confirmed via Datadog: CHPL-DEV-API's chplservice logs stopped the moment its container switched to the GHCR image, while job-specific loggers (unaffected, defined in the shared/base resources) kept working. Threaded a MAVEN_ENV build arg through docker-publish.yml, computed from the branch being built (development/qa/staging/production -> dev/qa/ staging/production, matching chpl-build-common's BuildEnvironment.getMavenEnvProperty()), instead of hardcoding one environment into a Dockerfile shared by all of them. Also finished a fix the prior "make deployed log4j2 config the default" commit (8b4471e5c) explicitly flagged but deferred: since -Denv never activated, resources-{dev,qa,staging,production}/log4j2.xml were never-exercised and still had the same "two sibling XIncludes" bug that commit fixed in the base config (only the last one survives, silently dropping the other's appenders). Removed their now-redundant log4j2-xinclude-file-appenders-console.xml include - its content was already merged into log4j2-xinclude-file-appenders.xml by that commit - and deleted the now-fully-orphaned file. Co-Authored-By: Claude Sonnet 5 --- .github/workflows/docker-publish.yml | 44 +- ...log4j2-xinclude-file-appenders-console.xml | 483 ------------------ .../src/main/resources-dev/log4j2.xml | 1 - .../src/main/resources-production/log4j2.xml | 1 - .../src/main/resources-qa/log4j2.xml | 1 - .../src/main/resources-staging/log4j2.xml | 1 - docker/Dockerfile | 15 +- docker/README.md | 15 +- 8 files changed, 58 insertions(+), 503 deletions(-) delete mode 100644 chpl/chpl-api/src/main/resources/log4j2-xinclude-file-appenders-console.xml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index dd2626986a..a94067b1a8 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -49,9 +49,9 @@ jobs: run: mvn -B clean test --file chpl/pom.xml # Mirrors the "Build API" job (CompileApiTask) in the same Bamboo plan, and - # runs the exact `mvn clean package -DskipTests` command that docker/Dockerfile's - # build stage runs - so a compile/packaging failure shows up here, fast, - # instead of partway through the (slower) Docker build. + # runs the exact `mvn clean package -DskipTests -Denv=` command that + # docker/Dockerfile's build stage runs - so a compile/packaging failure + # shows up here, fast, instead of partway through the (slower) Docker build. compile: runs-on: ubuntu-latest steps: @@ -64,10 +64,25 @@ jobs: java-version: '21' distribution: 'temurin' + # Maps this branch to the same -Denv value chpl-build-common's + # BuildEnvironment.getMavenEnvProperty() uses for the build-from-source + # path in each environment. Falls back to "dev" for workflow_dispatch + # runs off any other branch. + - name: Determine Maven environment profile + id: envprofile + run: | + case "${{ github.ref_name }}" in + development) echo "value=dev" >> "$GITHUB_OUTPUT" ;; + qa) echo "value=qa" >> "$GITHUB_OUTPUT" ;; + staging) echo "value=staging" >> "$GITHUB_OUTPUT" ;; + production) echo "value=production" >> "$GITHUB_OUTPUT" ;; + *) echo "value=dev" >> "$GITHUB_OUTPUT" ;; + esac + - name: Compile and package env: MAVEN_OPTS: -Xms512m -Xmx1024m - run: mvn -B clean package -DskipTests --file chpl/pom.xml + run: mvn -B clean package -DskipTests -Denv=${{ steps.envprofile.outputs.value }} --file chpl/pom.xml build-and-push: needs: [unit-tests, compile] @@ -93,10 +108,23 @@ jobs: echo "sha_short=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" echo "branch=$(echo '${{ github.ref_name }}' | tr '/' '-')" >> "$GITHUB_OUTPUT" + # Same branch -> -Denv mapping as the "compile" job above, threaded + # into the Docker build as the MAVEN_ENV build arg. + - name: Determine Maven environment profile + id: envprofile + run: | + case "${{ github.ref_name }}" in + development) echo "value=dev" >> "$GITHUB_OUTPUT" ;; + qa) echo "value=qa" >> "$GITHUB_OUTPUT" ;; + staging) echo "value=staging" >> "$GITHUB_OUTPUT" ;; + production) echo "value=production" >> "$GITHUB_OUTPUT" ;; + *) echo "value=dev" >> "$GITHUB_OUTPUT" ;; + esac + # This image contains no secrets - see docker/Dockerfile. Every environment - # (development/qa/staging/production) runs the exact same image; secrets are - # supplied as container environment variables at `docker run` time, not baked - # in here. + # runs the same image apart from the MAVEN_ENV build arg (log4j2 config); + # secrets are supplied as container environment variables at `docker run` + # time, not baked in here. # # Tags: # build- - immutable, sequential build number (github.run_number @@ -111,6 +139,8 @@ jobs: context: . file: docker/Dockerfile push: true + build-args: | + MAVEN_ENV=${{ steps.envprofile.outputs.value }} # build-push-action v6 enables these by default, which each push a # separate untagged provenance/SBOM manifest to GHCR alongside the # real image - not needed here, so turned off to keep the package diff --git a/chpl/chpl-api/src/main/resources/log4j2-xinclude-file-appenders-console.xml b/chpl/chpl-api/src/main/resources/log4j2-xinclude-file-appenders-console.xml deleted file mode 100644 index 143ce787cb..0000000000 --- a/chpl/chpl-api/src/main/resources/log4j2-xinclude-file-appenders-console.xml +++ /dev/null @@ -1,483 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/chpl/chpl-resources/src/main/resources-dev/log4j2.xml b/chpl/chpl-resources/src/main/resources-dev/log4j2.xml index bbb3723948..7ee3d71f2c 100644 --- a/chpl/chpl-resources/src/main/resources-dev/log4j2.xml +++ b/chpl/chpl-resources/src/main/resources-dev/log4j2.xml @@ -6,6 +6,5 @@ ${sys:chpl.logging.root} - diff --git a/chpl/chpl-resources/src/main/resources-production/log4j2.xml b/chpl/chpl-resources/src/main/resources-production/log4j2.xml index 6644b93d19..9ff5a0bc1f 100644 --- a/chpl/chpl-resources/src/main/resources-production/log4j2.xml +++ b/chpl/chpl-resources/src/main/resources-production/log4j2.xml @@ -6,6 +6,5 @@ ${sys:chpl.logging.root} - \ No newline at end of file diff --git a/chpl/chpl-resources/src/main/resources-qa/log4j2.xml b/chpl/chpl-resources/src/main/resources-qa/log4j2.xml index dbc4a73ca1..780a33f696 100644 --- a/chpl/chpl-resources/src/main/resources-qa/log4j2.xml +++ b/chpl/chpl-resources/src/main/resources-qa/log4j2.xml @@ -6,6 +6,5 @@ ${sys:chpl.logging.root} - diff --git a/chpl/chpl-resources/src/main/resources-staging/log4j2.xml b/chpl/chpl-resources/src/main/resources-staging/log4j2.xml index 65dc182b52..5269343519 100644 --- a/chpl/chpl-resources/src/main/resources-staging/log4j2.xml +++ b/chpl/chpl-resources/src/main/resources-staging/log4j2.xml @@ -6,6 +6,5 @@ ${sys:chpl.logging.root} - \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 5296c567ff..814f83e47f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,12 +16,21 @@ COPY chpl/chpl-resources/src chpl-resources/src COPY chpl/chpl-service/lombok.config chpl-service/lombok.config COPY chpl/chpl-service/src chpl-service/src -RUN mvn clean package -DskipTests +# Activates chpl-resources' "environment-specific" Maven profile, which layers +# src/main/resources-${MAVEN_ENV} (log4j2 config, notably the catch-all +# gov.healthit.chpl logger/chplserviceJson appender that Datadog's chplservice +# service tag depends on) over the base resources. Matches the -Denv= +# flag chpl-build-common's CompileApiTask already passes for the +# build-from-source path in every non-GHCR environment. Set via --build-arg +# from docker-publish.yml, mapped from the branch being built. +ARG MAVEN_ENV=dev +RUN mvn clean package -DskipTests -Denv=${MAVEN_ENV} # Stage 2: Deploy to Tomcat # -# This image contains no secrets and is identical across every environment -# (dev/qa/stage/production). Tomcat config below only has ${ENV_VAR} +# This image contains no secrets, and only differs across environments in +# the -Denv build arg above (log4j2 config naming/routing). Tomcat config +# below only has ${ENV_VAR} # placeholders (resolved at container startup - see catalina.properties' # org.apache.tomcat.util.digester.PROPERTY_SOURCE) and the checked-in # environment.properties/email.properties already mark every sensitive key diff --git a/docker/README.md b/docker/README.md index 0f1ea4f9c4..39b8e7cf59 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,12 +1,15 @@ # Running the chpl-api Docker image `docker/Dockerfile` builds a **secret-free** image: the same image is used for -every environment (development/qa/staging/production). Nothing sensitive is -baked in at build time - all of it is supplied as container environment -variables when the image is actually run. See `docker/Dockerfile` and -`docker/tomcat-conf/` for how that works mechanically (Tomcat's -`EnvironmentPropertySource` for `server.xml`, Spring's `Environment` for -everything else). +every environment (development/qa/staging/production), apart from a +`MAVEN_ENV` build arg (`dev`/`qa`/`staging`/`production`, set automatically by +`.github/workflows/docker-publish.yml` from the branch being built) that +selects which `chpl-resources/src/main/resources-${MAVEN_ENV}` logging config +gets layered onto the WAR. Nothing sensitive is baked in at build time - all +of it is supplied as container environment variables when the image is +actually run. See `docker/Dockerfile` and `docker/tomcat-conf/` for how that +works mechanically (Tomcat's `EnvironmentPropertySource` for `server.xml`, +Spring's `Environment` for everything else). This doc lists every environment variable the image needs or accepts. It does **not** contain real values for any environment - only what each variable is From d502b79232467b30f6b5c61333f6ebda45494483 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Thu, 13 Aug 2026 11:36:51 -0400 Subject: [PATCH 32/45] fix: restore user-triggers.xml, now required again by -Denv=dev/qa/staging Regression from the -Denv activation fix (01a2dd236): resources-dev/qa/ staging's quartz.properties set org.quartz.plugin.jobInitializer.fileNames=jobs.xml,system-triggers.xml, user-triggers.xml with failOnFileNotFound=true, but that file was deleted from the image on 2026-08-10 (50468afa8) on the (then-correct) assumption that the Docker build never passed -Denv, so only the base quartz.properties (no user-triggers.xml) ever got packaged. Activating -Denv reintroduced the dependency. Confirmed via Datadog: chpl-api-dev-inst-1's Tomcat process started and bound its port, but the chpl-service webapp's Spring context failed entirely - UnsatisfiedDependencyException chain (CHPLHttpSecurityConfig -> apiKeyManager -> chplEmailFactory -> chplSchedulerReference -> schedulerFactory: "File named 'user-triggers.xml' does not exist") - container looked "up" but never became a working app server. Restored the file and its COPY line verbatim from before the revert. production's quartz.properties doesn't reference user-triggers.xml, so this is a no-op there. Co-Authored-By: Claude Sonnet 5 --- docker/Dockerfile | 10 ++++++++++ docker/tomcat-conf/user-triggers.xml | 11 +++++++++++ 2 files changed, 21 insertions(+) create mode 100644 docker/tomcat-conf/user-triggers.xml diff --git a/docker/Dockerfile b/docker/Dockerfile index 814f83e47f..c86b45effd 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -47,6 +47,16 @@ FROM tomcat:11.0.24-jdk21 COPY docker/tomcat-conf/server.xml /usr/local/tomcat/conf/server.xml COPY docker/tomcat-conf/context.xml /usr/local/tomcat/conf/context.xml COPY docker/tomcat-conf/catalina.properties /usr/local/tomcat/conf/catalina.properties +# Empty Quartz job-scheduling-data template. Only the dev/qa/staging +# resources-${MAVEN_ENV} quartz.properties (now actually applied - see the +# -Denv build above) set +# org.quartz.plugin.jobInitializer.fileNames=jobs.xml,system-triggers.xml,user-triggers.xml +# with failOnFileNotFound=true; jobs.xml/system-triggers.xml ship on the +# classpath via chpl-resources, but this one doesn't, so Quartz needs it +# here on Tomcat's conf classpath (CATALINA_BASE/conf) or the whole Spring +# context fails to start (production's quartz.properties doesn't reference +# it, so it's a harmless no-op there). +COPY docker/tomcat-conf/user-triggers.xml /usr/local/tomcat/conf/user-triggers.xml COPY --from=build /app/chpl-api/target/chpl-service.war /usr/local/tomcat/webapps/chpl-service.war diff --git a/docker/tomcat-conf/user-triggers.xml b/docker/tomcat-conf/user-triggers.xml new file mode 100644 index 0000000000..89fca2249e --- /dev/null +++ b/docker/tomcat-conf/user-triggers.xml @@ -0,0 +1,11 @@ + + + + + + From 43cb669fcc1a711daa5f13f44663b9727c9a0986 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Thu, 13 Aug 2026 12:23:20 -0400 Subject: [PATCH 33/45] fix: move user-triggers.xml to Tomcat's lib/, not conf/ The previous fix (d502b7923) restored the file but put it back at the same path an untested, cargo-culted placement had used: /usr/local/ tomcat/conf/user-triggers.xml. Verified against the real published image (ghcr.io/chpladmin/chpl-api@sha256:745cd984..., confirmed via Bamboo's pull log as the digest actually deployed to chpl-api-dev-inst-1) that this placement never worked and never could: - Decompiled quartz-2.5.2's XMLSchedulingDataProcessor: the jobInitializer plugin loads fileNames exclusively via ClassLoadHelper.getResourceAsStream() - pure classloader lookup, no java.io.File/working-directory fallback for the actual content. - catalina.properties' common.loader (checked into this image, unlike the host-mounted one QA/staging/prod's non-GHCR build relies on) only lists ${catalina.home}/lib - never conf/, shared.loader, or server.loader. - Confirmed empirically inside the real image using Tomcat's own org.apache.catalina.startup.ClassLoaderFactory (from its bootstrap.jar) with catalina.properties' actual repository list: a file placed in lib/ resolves via getResourceAsStream; conf/ never would. This is why chpl-api-dev-inst-1 kept failing with the identical "File named 'user-triggers.xml' does not exist" error even after Bamboo pulled the digest containing d502b7923's fix - the file was present in the image, just never reachable from the classloader Quartz actually uses. Co-Authored-By: Claude Sonnet 5 --- docker/Dockerfile | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index c86b45effd..ef9692dfbb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -52,11 +52,18 @@ COPY docker/tomcat-conf/catalina.properties /usr/local/tomcat/conf/catalina.prop # -Denv build above) set # org.quartz.plugin.jobInitializer.fileNames=jobs.xml,system-triggers.xml,user-triggers.xml # with failOnFileNotFound=true; jobs.xml/system-triggers.xml ship on the -# classpath via chpl-resources, but this one doesn't, so Quartz needs it -# here on Tomcat's conf classpath (CATALINA_BASE/conf) or the whole Spring -# context fails to start (production's quartz.properties doesn't reference -# it, so it's a harmless no-op there). -COPY docker/tomcat-conf/user-triggers.xml /usr/local/tomcat/conf/user-triggers.xml +# classpath via chpl-resources, but this one doesn't. Quartz loads it via +# ClassLoadHelper.getResourceAsStream (classloader lookup only - no +# filesystem fallback, confirmed by decompiling quartz-2.5.2's +# XMLSchedulingDataProcessor), so it must be somewhere actually on Tomcat's +# common classloader. catalina.properties' common.loader only lists +# ${catalina.home}/lib (as a bare directory - Tomcat adds a whole +# directory's contents as a class repository, not just its jars, verified +# against this image's own bootstrap.jar ClassLoaderFactory); conf/ is not +# on any loader list. Putting it in conf/ (as a prior attempt did) silently +# never gets found - production's quartz.properties doesn't reference this +# file at all, so it's a harmless no-op there either way. +COPY docker/tomcat-conf/user-triggers.xml /usr/local/tomcat/lib/user-triggers.xml COPY --from=build /app/chpl-api/target/chpl-service.war /usr/local/tomcat/webapps/chpl-service.war From 9d74439bf6f76f8dde555cf6aa01f5e1317ad3a3 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 19 Aug 2026 11:07:32 -0400 Subject: [PATCH 34/45] chore: bump Tomcat base image to 11.0.25-jdk21 --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ef9692dfbb..9eecedc64d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -42,7 +42,7 @@ RUN mvn clean package -DskipTests -Denv=${MAVEN_ENV} # plus every property marked SECRET in # chpl/chpl-resources/src/main/resources/environment.properties and email.properties # (e.g. SPRING_REDIS_PASSWORD, COGNITO_SECRETKEY, AZURE_CLIENTSECRET_ONC, JIRA_PASSWORD, ...) -FROM tomcat:11.0.24-jdk21 +FROM tomcat:11.0.25-jdk21 COPY docker/tomcat-conf/server.xml /usr/local/tomcat/conf/server.xml COPY docker/tomcat-conf/context.xml /usr/local/tomcat/conf/context.xml From 24aebe11f2397df4bceb7bc1da7ade91f476bad1 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 19 Aug 2026 11:31:42 -0400 Subject: [PATCH 35/45] ci: cache Maven dependencies across workflow runs Adds actions/setup-java's built-in ~/.m2 cache to the unit-tests and compile jobs, and a BuildKit cache mount + type=gha cache backend for the mvn build that runs inside the Docker image build, so repeated CI runs stop re-downloading the full dependency tree from Maven Central (which previously triggered 429 Too Many Requests failures). --- .github/workflows/docker-publish.yml | 8 ++++++++ docker/Dockerfile | 7 ++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a94067b1a8..ab702c3438 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -42,6 +42,7 @@ jobs: with: java-version: '21' distribution: 'temurin' + cache: 'maven' - name: Run unit tests env: @@ -63,6 +64,7 @@ jobs: with: java-version: '21' distribution: 'temurin' + cache: 'maven' # Maps this branch to the same -Denv value chpl-build-common's # BuildEnvironment.getMavenEnvProperty() uses for the build-from-source @@ -147,6 +149,12 @@ jobs: # version list clean. provenance: false sbom: false + # Persists the Dockerfile build stage's ~/.m2 cache mount (see + # docker/Dockerfile) to the GitHub Actions cache service, so the + # Maven dependency download in the Docker build itself is also + # reused across runs, not just in the compile/unit-tests jobs above. + cache-from: type=gha + cache-to: type=gha,mode=max tags: | ${{ steps.meta.outputs.image }}:build-${{ github.run_number }} ${{ steps.meta.outputs.image }}:sha-${{ steps.meta.outputs.sha_short }} diff --git a/docker/Dockerfile b/docker/Dockerfile index 9eecedc64d..1a4e2e7712 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 # Stage 1: Build the WAR file using Maven FROM maven:3.9-eclipse-temurin-21-alpine AS build WORKDIR /app @@ -24,7 +25,11 @@ COPY chpl/chpl-service/src chpl-service/src # build-from-source path in every non-GHCR environment. Set via --build-arg # from docker-publish.yml, mapped from the branch being built. ARG MAVEN_ENV=dev -RUN mvn clean package -DskipTests -Denv=${MAVEN_ENV} +# Cache mount persists the downloaded dependency tree across builds (see +# cache-to/cache-from: type=gha on the "Build and push" step in +# docker-publish.yml) so repeated builds don't re-hit Maven Central for +# every dependency. +RUN --mount=type=cache,target=/root/.m2 mvn clean package -DskipTests -Denv=${MAVEN_ENV} # Stage 2: Deploy to Tomcat # From 1dc64d6fdf47de716ff815a2d72c00baadd024b8 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Mon, 24 Aug 2026 20:57:37 -0400 Subject: [PATCH 36/45] ci: build the WAR once and reuse it in the Docker image The compile job already ran the identical mvn clean package -Denv= command that docker/Dockerfile's build stage repeated from scratch, so every image build re-downloaded and recompiled the whole dependency tree instead of reusing the setup-java-cached artifact that already existed. Pass the WAR between jobs via upload/download-artifact instead, and drop Docker's now-redundant Maven build stage entirely. OCD-5395 --- .github/workflows/docker-publish.yml | 49 +++++++++++++------------- .gitignore | 1 + docker/Dockerfile | 51 +++++++--------------------- 3 files changed, 36 insertions(+), 65 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index ab702c3438..f4800b12b9 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -50,9 +50,9 @@ jobs: run: mvn -B clean test --file chpl/pom.xml # Mirrors the "Build API" job (CompileApiTask) in the same Bamboo plan, and - # runs the exact `mvn clean package -DskipTests -Denv=` command that - # docker/Dockerfile's build stage runs - so a compile/packaging failure - # shows up here, fast, instead of partway through the (slower) Docker build. + # builds the exact WAR docker/Dockerfile packages into the image (see the + # "Upload WAR" step below) - so a compile/packaging failure shows up here, + # fast, instead of partway through the (slower) Docker build. compile: runs-on: ubuntu-latest steps: @@ -86,6 +86,16 @@ jobs: MAVEN_OPTS: -Xms512m -Xmx1024m run: mvn -B clean package -DskipTests -Denv=${{ steps.envprofile.outputs.value }} --file chpl/pom.xml + # Handed to build-and-push below, so the Docker image build doesn't + # need to rerun the Maven compile (which just ran here, with the + # setup-java ~/.m2 cache already warm) itself. + - name: Upload WAR + uses: actions/upload-artifact@v4 + with: + name: chpl-service-war + path: chpl/chpl-api/target/chpl-service.war + retention-days: 1 + build-and-push: needs: [unit-tests, compile] runs-on: ubuntu-latest @@ -110,23 +120,18 @@ jobs: echo "sha_short=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT" echo "branch=$(echo '${{ github.ref_name }}' | tr '/' '-')" >> "$GITHUB_OUTPUT" - # Same branch -> -Denv mapping as the "compile" job above, threaded - # into the Docker build as the MAVEN_ENV build arg. - - name: Determine Maven environment profile - id: envprofile - run: | - case "${{ github.ref_name }}" in - development) echo "value=dev" >> "$GITHUB_OUTPUT" ;; - qa) echo "value=qa" >> "$GITHUB_OUTPUT" ;; - staging) echo "value=staging" >> "$GITHUB_OUTPUT" ;; - production) echo "value=production" >> "$GITHUB_OUTPUT" ;; - *) echo "value=dev" >> "$GITHUB_OUTPUT" ;; - esac + # Built by the "compile" job above (already -Denv'd for this branch), + # so this build only has to package it into Tomcat. + - name: Download WAR + uses: actions/download-artifact@v4 + with: + name: chpl-service-war + path: docker/build-context/ # This image contains no secrets - see docker/Dockerfile. Every environment - # runs the same image apart from the MAVEN_ENV build arg (log4j2 config); - # secrets are supplied as container environment variables at `docker run` - # time, not baked in here. + # runs the same image apart from the -Denv value the compile job built + # the WAR with (log4j2 config); secrets are supplied as container + # environment variables at `docker run` time, not baked in here. # # Tags: # build- - immutable, sequential build number (github.run_number @@ -141,20 +146,12 @@ jobs: context: . file: docker/Dockerfile push: true - build-args: | - MAVEN_ENV=${{ steps.envprofile.outputs.value }} # build-push-action v6 enables these by default, which each push a # separate untagged provenance/SBOM manifest to GHCR alongside the # real image - not needed here, so turned off to keep the package # version list clean. provenance: false sbom: false - # Persists the Dockerfile build stage's ~/.m2 cache mount (see - # docker/Dockerfile) to the GitHub Actions cache service, so the - # Maven dependency download in the Docker build itself is also - # reused across runs, not just in the compile/unit-tests jobs above. - cache-from: type=gha - cache-to: type=gha,mode=max tags: | ${{ steps.meta.outputs.image }}:build-${{ github.run_number }} ${{ steps.meta.outputs.image }}:sha-${{ steps.meta.outputs.sha_short }} diff --git a/.gitignore b/.gitignore index 554bde5f06..608ace9efa 100644 --- a/.gitignore +++ b/.gitignore @@ -225,6 +225,7 @@ pip-log.txt .mr.developer.cfg ## CHPL specific ignores +/docker/build-context/ /chpl/chpl-etl/src/main/resources/log.txt /chpl/chpl-etl/src/main/resources/chpl.csv /chpl/chpl-etl/src/main/resources/chpl-hash.csv diff --git a/docker/Dockerfile b/docker/Dockerfile index 1a4e2e7712..fa59e5c49f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,41 +1,14 @@ -# syntax=docker/dockerfile:1 -# Stage 1: Build the WAR file using Maven -FROM maven:3.9-eclipse-temurin-21-alpine AS build -WORKDIR /app - -# Copy the pom.xml files first to leverage Docker layer caching -# This ensures that if only source code changes, dependencies are not re-downloaded -COPY chpl/pom.xml . -COPY chpl/chpl-api/pom.xml chpl-api/ -COPY chpl/chpl-resources/pom.xml chpl-resources/ -COPY chpl/chpl-service/pom.xml chpl-service/ - -# Copy the rest of the project files -COPY chpl/chpl-api/lombok.config chpl-api/lombok.config -COPY chpl/chpl-api/src chpl-api/src -COPY chpl/chpl-resources/src chpl-resources/src -COPY chpl/chpl-service/lombok.config chpl-service/lombok.config -COPY chpl/chpl-service/src chpl-service/src - -# Activates chpl-resources' "environment-specific" Maven profile, which layers -# src/main/resources-${MAVEN_ENV} (log4j2 config, notably the catch-all -# gov.healthit.chpl logger/chplserviceJson appender that Datadog's chplservice -# service tag depends on) over the base resources. Matches the -Denv= -# flag chpl-build-common's CompileApiTask already passes for the -# build-from-source path in every non-GHCR environment. Set via --build-arg -# from docker-publish.yml, mapped from the branch being built. -ARG MAVEN_ENV=dev -# Cache mount persists the downloaded dependency tree across builds (see -# cache-to/cache-from: type=gha on the "Build and push" step in -# docker-publish.yml) so repeated builds don't re-hit Maven Central for -# every dependency. -RUN --mount=type=cache,target=/root/.m2 mvn clean package -DskipTests -Denv=${MAVEN_ENV} - -# Stage 2: Deploy to Tomcat +# Deploy to Tomcat. +# +# The WAR is built by the "compile" job in .github/workflows/docker-publish.yml +# (the same `mvn clean package -Denv=` command this image used to run +# itself, but with maven's ~/.m2 dependency cache already warm there) and +# downloaded into docker/build-context/ before this build runs, so the image +# build never needs to touch Maven Central. # # This image contains no secrets, and only differs across environments in -# the -Denv build arg above (log4j2 config naming/routing). Tomcat config -# below only has ${ENV_VAR} +# the -Denv value the compile job built the WAR with (log4j2 config +# naming/routing). Tomcat config below only has ${ENV_VAR} # placeholders (resolved at container startup - see catalina.properties' # org.apache.tomcat.util.digester.PROPERTY_SOURCE) and the checked-in # environment.properties/email.properties already mark every sensitive key @@ -53,8 +26,8 @@ COPY docker/tomcat-conf/server.xml /usr/local/tomcat/conf/server.xml COPY docker/tomcat-conf/context.xml /usr/local/tomcat/conf/context.xml COPY docker/tomcat-conf/catalina.properties /usr/local/tomcat/conf/catalina.properties # Empty Quartz job-scheduling-data template. Only the dev/qa/staging -# resources-${MAVEN_ENV} quartz.properties (now actually applied - see the -# -Denv build above) set +# resources-${env} quartz.properties (now actually applied - see the +# -Denv build in the compile job) set # org.quartz.plugin.jobInitializer.fileNames=jobs.xml,system-triggers.xml,user-triggers.xml # with failOnFileNotFound=true; jobs.xml/system-triggers.xml ship on the # classpath via chpl-resources, but this one doesn't. Quartz loads it via @@ -70,7 +43,7 @@ COPY docker/tomcat-conf/catalina.properties /usr/local/tomcat/conf/catalina.prop # file at all, so it's a harmless no-op there either way. COPY docker/tomcat-conf/user-triggers.xml /usr/local/tomcat/lib/user-triggers.xml -COPY --from=build /app/chpl-api/target/chpl-service.war /usr/local/tomcat/webapps/chpl-service.war +COPY docker/build-context/chpl-service.war /usr/local/tomcat/webapps/chpl-service.war # Matches the port every other environment's Tomcat listens on (the real # dev box's server.xml, and chpl-build's start-tomcat-containers.sh, which From 715ed6815582d39c284aba5c04c10f6566dbbd74 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 9 Sep 2026 11:26:11 -0400 Subject: [PATCH 37/45] fix(docker): set explicit Tomcat JDBC pool factory on jdbc/openchpl Without a factory attribute, Tomcat 11 falls back to DBCP2, which ignores the legacy maxActive/maxWait names and would silently run the pool at its own default of maxTotal=8. Naming the Tomcat JDBC pool factory keeps the maxActive/maxIdle/maxWait values carried over from the Bamboo server.xml meaningful and makes the pool implementation deterministic. OCD-5395 Co-Authored-By: Claude Opus 5 --- docker/tomcat-conf/server.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/tomcat-conf/server.xml b/docker/tomcat-conf/server.xml index 0dc00997df..446b45efa5 100644 --- a/docker/tomcat-conf/server.xml +++ b/docker/tomcat-conf/server.xml @@ -56,6 +56,7 @@ org.apache.tomcat.util.digester.PROPERTY_SOURCE in catalina.properties) --> Date: Wed, 9 Sep 2026 11:35:36 -0400 Subject: [PATCH 38/45] revert: restore -local xincludes in chpl-api's log4j2.xml These two lines were needed at 8b4471e5c, when the Docker build passed no -Denv and chpl-api's own log4j2.xml therefore won on the WEB-INF/classes classpath. 01a2dd236 made the build pass -Denv, and chpl-api/pom.xml's environment-specific profile copies chpl-resources/src/main/resources-${env} into WEB-INF/classes ahead of src/main/resources, where first-copied wins. resources-{env}/log4j2.xml now shadows this file in every deployment, so the change is a no-op everywhere except a plain `mvn package`. That one remaining case is local development, where it is a regression: the deployed fragments carry no STDOUT or chplServiceJsonLog appender refs and none of the org.hibernate / com.fasterxml.jackson / org.jose4j / org.springframework noise-suppression loggers the -local variants define. The substantive logging fix in this branch is unaffected - it lives in the resources-{env}/log4j2.xml files, which drop the second sibling xinclude that Log4j2 silently discards. OCD-5395 Co-Authored-By: Claude Opus 5 --- chpl/chpl-api/src/main/resources/log4j2.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chpl/chpl-api/src/main/resources/log4j2.xml b/chpl/chpl-api/src/main/resources/log4j2.xml index 09bcf7cfda..91db23e9c1 100644 --- a/chpl/chpl-api/src/main/resources/log4j2.xml +++ b/chpl/chpl-api/src/main/resources/log4j2.xml @@ -5,6 +5,6 @@ ${sys:catalina.home}/logs - - + + From 6bf6cce33056ee3f98b0c28dc47cf865e1daf3e6 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 9 Sep 2026 12:24:24 -0400 Subject: [PATCH 39/45] refactor: make errors-override bundle optional instead of shipping a stub Replaces the placeholder errors-override.properties with a code change, per review. Both messageSource beans used ResourceBundleMessageSource, which has no ignore-missing option and logs a WARN per lookup when the bundle is absent, so the file had to exist purely to keep the log quiet - chpl-build's override-api-properties.sh writes an equally empty copy for the same reason. ReloadableResourceBundleMessageSource treats a missing bundle as absent, and takes both basenames in precedence order, so the parent-source wiring goes away too. CHPLServiceConfig already read this file with @PropertySource(ignoreResourceNotFound = true), so optional is now consistent. Verified: ReloadableResourceBundleMessageSource and setBasenames(String...) confirmed present in spring-context 7.0.9 via javap; checkstyle clean on both files. Note `mvn clean compile` fails on this branch with ~413 pre-existing Lombok "cannot find symbol getX()" errors, identically with these changes stashed - unrelated to this commit. OCD-5395 Co-Authored-By: Claude Opus 5 --- .../main/java/gov/healthit/chpl/CHPLConfig.java | 15 +++++++-------- .../src/main/resources/errors-override.properties | 8 -------- .../java/gov/healthit/chpl/CHPLServiceConfig.java | 15 +++++++-------- 3 files changed, 14 insertions(+), 24 deletions(-) delete mode 100644 chpl/chpl-resources/src/main/resources/errors-override.properties diff --git a/chpl/chpl-api/src/main/java/gov/healthit/chpl/CHPLConfig.java b/chpl/chpl-api/src/main/java/gov/healthit/chpl/CHPLConfig.java index 3240e2ca66..b6b07f2f7e 100644 --- a/chpl/chpl-api/src/main/java/gov/healthit/chpl/CHPLConfig.java +++ b/chpl/chpl-api/src/main/java/gov/healthit/chpl/CHPLConfig.java @@ -23,7 +23,7 @@ import org.springframework.context.annotation.Primary; import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.PropertySources; -import org.springframework.context.support.ResourceBundleMessageSource; +import org.springframework.context.support.ReloadableResourceBundleMessageSource; import org.springframework.core.env.Environment; import org.springframework.http.MediaType; import org.springframework.http.converter.ByteArrayHttpMessageConverter; @@ -176,13 +176,12 @@ public void configureMessageConverters(List> converters) @Bean public MessageSource messageSource() { - ResourceBundleMessageSource messageSource = new ResourceBundleMessageSource(); - messageSource.setBasename("errors-override"); - - ResourceBundleMessageSource parentMessageSource = new ResourceBundleMessageSource(); - parentMessageSource.setBasename("errors"); - - messageSource.setParentMessageSource(parentMessageSource); + //errors-override is optional - ReloadableResourceBundleMessageSource treats a + //missing bundle as simply absent, where ResourceBundleMessageSource would WARN + //on every lookup. Basenames are consulted in order, so errors-override still + //takes precedence over errors. + ReloadableResourceBundleMessageSource messageSource = new ReloadableResourceBundleMessageSource(); + messageSource.setBasenames("classpath:errors-override", "classpath:errors"); messageSource.setDefaultEncoding("UTF-8"); return messageSource; } diff --git a/chpl/chpl-resources/src/main/resources/errors-override.properties b/chpl/chpl-resources/src/main/resources/errors-override.properties deleted file mode 100644 index 18eba2a146..0000000000 --- a/chpl/chpl-resources/src/main/resources/errors-override.properties +++ /dev/null @@ -1,8 +0,0 @@ -# Intentionally empty. CHPLConfig/CHPLServiceConfig's messageSource bean uses -# ResourceBundleMessageSource with basename "errors-override" as the primary -# bundle and "errors" (errors.properties) as its parent fallback - -# ResourceBundleMessageSource has no ignore-missing option, so every message -# lookup logs a WARN if this bundle can't be found on the classpath at all. -# This file just needs to exist; no environment has ever put real content in -# it (see errors.properties' own header comment for the override mechanism -# this was originally meant to support at runtime). diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/CHPLServiceConfig.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/CHPLServiceConfig.java index 66d727c7a9..81c913fc48 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/CHPLServiceConfig.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/CHPLServiceConfig.java @@ -37,7 +37,7 @@ import org.springframework.context.annotation.EnableAspectJAutoProxy; import org.springframework.context.annotation.PropertySource; import org.springframework.context.annotation.PropertySources; -import org.springframework.context.support.ResourceBundleMessageSource; +import org.springframework.context.support.ReloadableResourceBundleMessageSource; import org.springframework.core.env.Environment; import org.springframework.core.io.ClassPathResource; import org.springframework.core.task.TaskExecutor; @@ -173,13 +173,12 @@ public ThreadPoolTaskScheduler threadPoolTaskScheduler() { @Bean public MessageSource messageSource() { - ResourceBundleMessageSource messageSource = new ResourceBundleMessageSource(); - messageSource.setBasename("errors-override"); - - ResourceBundleMessageSource parentMessageSource = new ResourceBundleMessageSource(); - parentMessageSource.setBasename("errors"); - - messageSource.setParentMessageSource(parentMessageSource); + //errors-override is optional - ReloadableResourceBundleMessageSource treats a + //missing bundle as simply absent, where ResourceBundleMessageSource would WARN + //on every lookup. Basenames are consulted in order, so errors-override still + //takes precedence over errors. + ReloadableResourceBundleMessageSource messageSource = new ReloadableResourceBundleMessageSource(); + messageSource.setBasenames("classpath:errors-override", "classpath:errors"); messageSource.setDefaultEncoding("UTF-8"); return messageSource; From 940b8ed8489b9995d22f4271b9e7b9cf5644f4a9 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Wed, 9 Sep 2026 13:23:59 -0400 Subject: [PATCH 40/45] refactor: drop the dormant Quartz user-triggers.xml mechanism Completes the cleanup started in e1a1da52e (2021-02-22), which deleted user-triggers.xml from all four resources-{env} directories but only removed it from production's jobInitializer.fileNames - dev/qa/staging kept listing a file that no longer existed, with failOnFileNotFound=true. Since then the file has been an empty template kept alive purely to stop the app from crashing on startup, shipped from two repos: docker/tomcat-conf here and chpl-build's CreateUserTriggersTask at deploy time. Nothing has used it since 2021. Before then it held per-person cron triggers (a Summary Statistics Email to a named address, and similar), which are now managed at runtime through SchedulerController and ChplRepeatableTrigger / ChplOneTimeTrigger against the clustered JDBC job store - DB-backed, and it survives deploys rather than needing a file edit plus a restart. dev/qa/staging now list jobs.xml,system-triggers.xml exactly as production does, and both of those ship on the classpath via chpl-resources. This also removes the requirement Copilot flagged, rather than satisfying it: no deployment of any profile, Docker or otherwise, needs the file on its classpath now. chpl-build's CreateUserTriggersTask / override-user-triggers.sh becomes vestigial - it writes a file nothing reads, harmless, to be retired there. OCD-5395 Co-Authored-By: Claude Opus 5 --- .../src/main/resources-dev/quartz.properties | 2 +- .../src/main/resources-qa/quartz.properties | 2 +- .../main/resources-staging/quartz.properties | 2 +- docker/Dockerfile | 17 ----------------- docker/tomcat-conf/user-triggers.xml | 11 ----------- 5 files changed, 3 insertions(+), 31 deletions(-) delete mode 100644 docker/tomcat-conf/user-triggers.xml diff --git a/chpl/chpl-resources/src/main/resources-dev/quartz.properties b/chpl/chpl-resources/src/main/resources-dev/quartz.properties index 7f7ee83b94..2d769deb34 100644 --- a/chpl/chpl-resources/src/main/resources-dev/quartz.properties +++ b/chpl/chpl-resources/src/main/resources-dev/quartz.properties @@ -8,7 +8,7 @@ org.quartz.jobStore.tablePrefix = quartz.QRTZ_ org.quartz.jobStore.isClustered = true org.quartz.plugin.jobInitializer.class = org.quartz.plugins.xml.XMLSchedulingDataProcessorPlugin -org.quartz.plugin.jobInitializer.fileNames = jobs.xml,system-triggers.xml,user-triggers.xml +org.quartz.plugin.jobInitializer.fileNames = jobs.xml,system-triggers.xml org.quartz.plugin.jobInitializer.failOnFileNotFound = true org.quartz.plugin.jobInitializer.scanInterval = 0 org.quartz.plugin.jobInitializer.wrapInUserTransaction = false diff --git a/chpl/chpl-resources/src/main/resources-qa/quartz.properties b/chpl/chpl-resources/src/main/resources-qa/quartz.properties index 7f7ee83b94..2d769deb34 100644 --- a/chpl/chpl-resources/src/main/resources-qa/quartz.properties +++ b/chpl/chpl-resources/src/main/resources-qa/quartz.properties @@ -8,7 +8,7 @@ org.quartz.jobStore.tablePrefix = quartz.QRTZ_ org.quartz.jobStore.isClustered = true org.quartz.plugin.jobInitializer.class = org.quartz.plugins.xml.XMLSchedulingDataProcessorPlugin -org.quartz.plugin.jobInitializer.fileNames = jobs.xml,system-triggers.xml,user-triggers.xml +org.quartz.plugin.jobInitializer.fileNames = jobs.xml,system-triggers.xml org.quartz.plugin.jobInitializer.failOnFileNotFound = true org.quartz.plugin.jobInitializer.scanInterval = 0 org.quartz.plugin.jobInitializer.wrapInUserTransaction = false diff --git a/chpl/chpl-resources/src/main/resources-staging/quartz.properties b/chpl/chpl-resources/src/main/resources-staging/quartz.properties index 7f7ee83b94..2d769deb34 100644 --- a/chpl/chpl-resources/src/main/resources-staging/quartz.properties +++ b/chpl/chpl-resources/src/main/resources-staging/quartz.properties @@ -8,7 +8,7 @@ org.quartz.jobStore.tablePrefix = quartz.QRTZ_ org.quartz.jobStore.isClustered = true org.quartz.plugin.jobInitializer.class = org.quartz.plugins.xml.XMLSchedulingDataProcessorPlugin -org.quartz.plugin.jobInitializer.fileNames = jobs.xml,system-triggers.xml,user-triggers.xml +org.quartz.plugin.jobInitializer.fileNames = jobs.xml,system-triggers.xml org.quartz.plugin.jobInitializer.failOnFileNotFound = true org.quartz.plugin.jobInitializer.scanInterval = 0 org.quartz.plugin.jobInitializer.wrapInUserTransaction = false diff --git a/docker/Dockerfile b/docker/Dockerfile index fa59e5c49f..0182556f6b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -25,23 +25,6 @@ FROM tomcat:11.0.25-jdk21 COPY docker/tomcat-conf/server.xml /usr/local/tomcat/conf/server.xml COPY docker/tomcat-conf/context.xml /usr/local/tomcat/conf/context.xml COPY docker/tomcat-conf/catalina.properties /usr/local/tomcat/conf/catalina.properties -# Empty Quartz job-scheduling-data template. Only the dev/qa/staging -# resources-${env} quartz.properties (now actually applied - see the -# -Denv build in the compile job) set -# org.quartz.plugin.jobInitializer.fileNames=jobs.xml,system-triggers.xml,user-triggers.xml -# with failOnFileNotFound=true; jobs.xml/system-triggers.xml ship on the -# classpath via chpl-resources, but this one doesn't. Quartz loads it via -# ClassLoadHelper.getResourceAsStream (classloader lookup only - no -# filesystem fallback, confirmed by decompiling quartz-2.5.2's -# XMLSchedulingDataProcessor), so it must be somewhere actually on Tomcat's -# common classloader. catalina.properties' common.loader only lists -# ${catalina.home}/lib (as a bare directory - Tomcat adds a whole -# directory's contents as a class repository, not just its jars, verified -# against this image's own bootstrap.jar ClassLoaderFactory); conf/ is not -# on any loader list. Putting it in conf/ (as a prior attempt did) silently -# never gets found - production's quartz.properties doesn't reference this -# file at all, so it's a harmless no-op there either way. -COPY docker/tomcat-conf/user-triggers.xml /usr/local/tomcat/lib/user-triggers.xml COPY docker/build-context/chpl-service.war /usr/local/tomcat/webapps/chpl-service.war diff --git a/docker/tomcat-conf/user-triggers.xml b/docker/tomcat-conf/user-triggers.xml deleted file mode 100644 index 89fca2249e..0000000000 --- a/docker/tomcat-conf/user-triggers.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - From 305720743e25162b3ce79ea27723d2fd86fdc042 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Tue, 15 Sep 2026 11:45:58 -0400 Subject: [PATCH 41/45] fix(docker): build the jdbc/openchpl pool lazily via DBCP2 715ed6815 named the tomcat-jdbc pool factory so the maxActive/maxIdle/maxWait values carried over from the Bamboo server.xml would stay meaningful. That had an unintended side effect: tomcat-jdbc opens initialSize connections eagerly, so Tomcat now builds this pool during StandardServer.startInternal - before any webapp exists, under the common classloader. postgresql-*.jar ships only in the webapp's WEB-INF/lib, so that classloader cannot see it and every container start logs a stack trace ending in: java.sql.SQLException: Unable to load class: org.postgresql.Driver from ClassLoader:java.net.URLClassLoader@... It was non-fatal - the webapp's own ResourceLink lookup runs on a webapp thread whose context classloader does see the driver, so the pool gets built there and connections work - but it produced a failing stack trace on every boot, which trains people to ignore startup errors and could mask a real DB misconfiguration. On DEV it fired ~20 log lines per container per start from 2026-09-09 onward. Dropping the factory attribute returns this to Tomcat's built-in DBCP2, which lives in tomcat/lib and creates its pool lazily (initialSize defaults to 0), so nothing connects until a webapp thread asks for it. That is the pool that ran from the image's introduction on 2026-07-21 until 2026-09-09 without this error. The attributes are renamed to DBCP2's names to preserve 715ed6815's actual goal. This is required, not cosmetic: DBCP2 has no setMaxActive, and maxWait now binds to setMaxWait(Duration) where "10000" does not convert - leaving either name in place would silently run the pool at DBCP2's default of maxTotal=8. Verified against the bundled tomcat-dbcp-11.0.25, which exposes setMaxTotal(int), setMaxIdle(int) and setMaxWaitMillis(long). Trade-off: DBCP2's lazy init means a bad DB_URL surfaces on first use rather than at startup. Accepted - the eager check is what broke, and the application ran on lazy init for years. OCD-5395 Co-Authored-By: Claude Opus 5 (1M context) --- docker/tomcat-conf/server.xml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/docker/tomcat-conf/server.xml b/docker/tomcat-conf/server.xml index 446b45efa5..07aa3604e8 100644 --- a/docker/tomcat-conf/server.xml +++ b/docker/tomcat-conf/server.xml @@ -54,16 +54,26 @@ + + maxWaitMillis="10000" /> @@ -362,7 +362,7 @@ org.apache.tomcat tomcat-catalina - 11.0.23 + 11.0.26 provided diff --git a/docker/Dockerfile b/docker/Dockerfile index 0182556f6b..e97b6f0acb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -20,7 +20,7 @@ # plus every property marked SECRET in # chpl/chpl-resources/src/main/resources/environment.properties and email.properties # (e.g. SPRING_REDIS_PASSWORD, COGNITO_SECRETKEY, AZURE_CLIENTSECRET_ONC, JIRA_PASSWORD, ...) -FROM tomcat:11.0.25-jdk21 +FROM tomcat:11.0.26-jdk21 COPY docker/tomcat-conf/server.xml /usr/local/tomcat/conf/server.xml COPY docker/tomcat-conf/context.xml /usr/local/tomcat/conf/context.xml From 8c1b338ac79dd6823cd23880ebb83f3c2a9f6562 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Fri, 18 Sep 2026 10:17:13 -0400 Subject: [PATCH 44/45] revert: drop the temporary MessageSource startup diagnostic OCD-5395 Reverts 3642c4b05. ErrorMessageUtil.java is now byte-identical to staging, so it drops out of this PR's diff entirely. The diagnostic was always marked temporary ("Remove once root-caused"), and it logged six lines on every context start in every environment. The DEV NoSuchMessageException it was added to chase is no longer reproducing. Worth being honest about why it is going, since the commit that added it set a condition this does not meet: the failure was never root-caused. It stopped on its own, and the probe logging is what would have identified the cause if it came back. Removing it is still right for a merge - it is startup noise for a problem that is not currently happening - but it is removal because the symptom went away, not because we understand it. ErrorMessageUtilResolutionTest, added in the next commit, is the durable replacement for the part of this that was actually worth keeping. Co-Authored-By: Claude Opus 5 (1M context) --- .../healthit/chpl/util/ErrorMessageUtil.java | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/chpl/chpl-service/src/main/java/gov/healthit/chpl/util/ErrorMessageUtil.java b/chpl/chpl-service/src/main/java/gov/healthit/chpl/util/ErrorMessageUtil.java index b4a26783cc..071808a6a5 100644 --- a/chpl/chpl-service/src/main/java/gov/healthit/chpl/util/ErrorMessageUtil.java +++ b/chpl/chpl-service/src/main/java/gov/healthit/chpl/util/ErrorMessageUtil.java @@ -1,57 +1,22 @@ package gov.healthit.chpl.util; -import java.util.Locale; - import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.MessageSource; import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.context.support.DefaultMessageSourceResolvable; import org.springframework.stereotype.Component; -import lombok.extern.log4j.Log4j2; - /** * Utility functions for dealing with error messages. */ @Component -@Log4j2 public class ErrorMessageUtil { - //Codes probed at startup by logInjectedMessageSource(). The first three are - //spread across errors.properties (lines 16, 167 and 722) so a bundle that - //failed to load shows up as every probe unresolved, rather than looking like - //one bad key. The last one is the code that throws on DEV. - private static final String[] PROBE_CODES = { - "access.denied", - "listing.newlineCharacterFound", - "surveillance.badCharacterFound", - "surveillance.newlineCharacterFound" - }; - private MessageSource messageSource; @Autowired public ErrorMessageUtil(final MessageSource messageSource) { this.messageSource = messageSource; - logInjectedMessageSource(); - } - - //Temporary diagnostic. surveillance.newlineCharacterFound throws - //NoSuchMessageException on DEV even though the key is present in the - //deployed chpl-resources jar, that file parses, and the exact bean - //configuration below resolves the code both standalone and from inside the - //running container. That leaves the live Spring context as the only - //remaining difference, so record what it actually injected. Uses the - //defaultMessage overload so a miss cannot throw. Remove once root-caused. - private void logInjectedMessageSource() { - LOGGER.info("Injected MessageSource implementation: {}", messageSource.getClass().getName()); - LOGGER.info("Injected MessageSource classloader: {}", messageSource.getClass().getClassLoader()); - LOGGER.info("LocaleContextHolder locale: {}, JVM default locale: {}", - LocaleContextHolder.getLocale(), Locale.getDefault()); - for (String code : PROBE_CODES) { - String resolved = messageSource.getMessage(code, null, "**UNRESOLVED**", Locale.US); - LOGGER.info(" probe {} -> {}", code, resolved); - } } /** From 0050432ecb391b685938181342d3cb8efbdf05d2 Mon Sep 17 00:00:00 2001 From: Todd Young Date: Fri, 18 Sep 2026 10:17:13 -0400 Subject: [PATCH 45/45] test: resolve error codes against the real errors.properties bundle OCD-5395 Every existing test that touches ErrorMessageUtil mocks MessageSource, so nothing in the suite exercises the actual bundle. That means a code present in Java but missing from errors.properties - or a bundle the configured basenames cannot load at all - is invisible until it throws at runtime, which is exactly the shape of the DEV failure that prompted the diagnostic reverted in the previous commit. Wires ReloadableResourceBundleMessageSource with the same basenames and encoding as CHPLServiceConfig.messageSource(), pins the locale to en_US (the locale the failing jobs ran under), and resolves three codes spread across the bundle: surveillance.newlineCharacterFound (the one that threw), surveillance.badCharacterFound and listing.newlineCharacterFound. Spreading them means a bundle that fails to load fails all three, distinguishing that from a single bad key. This is a build-time guard rather than a root cause. If the DEV failure was context-specific - a MessageSource other than the configured bean being injected into the running WAR - this test will not reproduce it, since it constructs the bean directly. It does cover the whole class of bundle-and-key-integrity regressions, which is the part that can be caught before deploy. Verified: mvn -pl chpl-service -am -Dtest=ErrorMessageUtilResolutionTest test passes, 3 tests, 0 failures. Co-Authored-By: Claude Opus 5 (1M context) --- .../util/ErrorMessageUtilResolutionTest.java | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 chpl/chpl-service/src/test/java/gov/healthit/chpl/util/ErrorMessageUtilResolutionTest.java diff --git a/chpl/chpl-service/src/test/java/gov/healthit/chpl/util/ErrorMessageUtilResolutionTest.java b/chpl/chpl-service/src/test/java/gov/healthit/chpl/util/ErrorMessageUtilResolutionTest.java new file mode 100644 index 0000000000..4ccff3c2bb --- /dev/null +++ b/chpl/chpl-service/src/test/java/gov/healthit/chpl/util/ErrorMessageUtilResolutionTest.java @@ -0,0 +1,65 @@ +package gov.healthit.chpl.util; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.util.Locale; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.context.i18n.LocaleContextHolder; +import org.springframework.context.support.ReloadableResourceBundleMessageSource; + +/** + * Resolves message codes against the real errors.properties bundle, wired the same way + * CHPLServiceConfig.messageSource() wires it. Every other test in the suite mocks + * MessageSource, so a code that exists in Java but not in the bundle - or a bundle the + * configured basenames cannot actually load - is invisible until it throws in production. + */ +public class ErrorMessageUtilResolutionTest { + + private ErrorMessageUtil errorMessageUtil; + + @BeforeEach + public void setup() { + //Mirrors CHPLServiceConfig.messageSource() exactly. + ReloadableResourceBundleMessageSource messageSource = new ReloadableResourceBundleMessageSource(); + messageSource.setBasenames("classpath:errors-override", "classpath:errors"); + messageSource.setDefaultEncoding("UTF-8"); + + errorMessageUtil = new ErrorMessageUtil(messageSource); + + //The failing jobs run under en_US, per the NoSuchMessageException they threw. + LocaleContextHolder.setLocale(Locale.US); + } + + @AfterEach + public void tearDown() { + LocaleContextHolder.resetLocaleContext(); + } + + @Test + public void surveillanceNewlineCharacterFound_resolves() { + String message = errorMessageUtil.getMessage("surveillance.newlineCharacterFound", "Surveillance Type"); + + assertNotNull(message); + assertFalse(message.isBlank()); + } + + @Test + public void surveillanceBadCharacterFound_resolves() { + String message = errorMessageUtil.getMessage("surveillance.badCharacterFound", "Surveillance Type"); + + assertNotNull(message); + assertFalse(message.isBlank()); + } + + @Test + public void listingNewlineCharacterFound_resolves() { + String message = errorMessageUtil.getMessage("listing.newlineCharacterFound", "Product Name"); + + assertNotNull(message); + assertFalse(message.isBlank()); + } +}