diff --git a/Dockerfile b/Dockerfile index db48dda..dd6a55d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ ARG VERSION=2.2.0 ARG JAVA=17 +ARG OPENTELEMETRY_AGENT_VERSION=2.23.0 FROM alpine:3.23 as builder @@ -19,7 +20,10 @@ ARG MAVEN_PROXY_PASSWORD ARG POSTGRESQL_VERSION ARG MYSQL_VERSION -ARG JMX_PROMETHEUS_VERSION=1.0.1 + +# --- OpenTelemetry Java Agent version argument --- +# Re-declare to use in this stage (inherits the value from global) +ARG OPENTELEMETRY_AGENT_VERSION RUN apk add --no-cache \ bash \ @@ -43,6 +47,8 @@ FROM alpine:3.23 # Re-declare to use in this stage (inherits the value from global) ARG VERSION ARG JAVA +ARG OPENTELEMETRY_AGENT_VERSION +ENV OPENTELEMETRY_AGENT_VERSION=${OPENTELEMETRY_AGENT_VERSION} ENV DB_DRIVER= ENV DB_URL= @@ -59,11 +65,19 @@ ENV WAIT_FOR_TIMEOUT=30 ENV TZ=UTC ENV DEBUG=false ENV JAVA_OPTS="" -ENV JMX_PROMETHEUS=false -ENV JMX_PROMETHEUS_CONF=/camunda/javaagent/prometheus-jmx.yml -ENV JMX_PROMETHEUS_PORT=9404 -EXPOSE 8080 8000 9404 +# OpenTelemetry default exporter settings (all exporters disabled, user must configure) +# Note: The OTEL agent is loaded via server-specific variables (PREPEND_JAVA_OPTS for WildFly, +# CATALINA_OPTS for Tomcat) to avoid affecting CLI tools like jboss-cli.sh +ENV OTEL_SERVICE_NAME=cibseven \ + OTEL_JMX_CONFIG=/camunda/javaagent/jmx_config.yaml,/camunda/javaagent/jmx_custom_config.yaml \ + OTEL_JAVAAGENT_LOGGING=application \ + OTEL_METRICS_EXPORTER=none \ + OTEL_LOGS_EXPORTER=none \ + OTEL_TRACES_EXPORTER=none \ + OTEL_EXPORTER_PROMETHEUS_PORT=9464 + +EXPOSE 8080 8000 9464 # Downgrading wait-for-it is necessary until this PR is merged # https://github.com/vishnubob/wait-for-it/pull/68 @@ -89,3 +103,7 @@ ENTRYPOINT ["/sbin/tini", "--"] CMD ["./cibseven.sh"] COPY --chown=camunda:camunda --from=builder /camunda . + +# --- Add JMX config files (ensure these are present in your build context) --- +COPY --chown=camunda:camunda opentelemetry/jmx_config.yaml /camunda/javaagent/jmx_config.yaml +COPY --chown=camunda:camunda opentelemetry/jmx_custom_config.yaml /camunda/javaagent/jmx_custom_config.yaml diff --git a/README.md b/README.md index 5d6770a..3f8bae1 100644 --- a/README.md +++ b/README.md @@ -107,8 +107,7 @@ when provided: * `DB_URL` * `DB_PASSWORD_FILE` -The `JMX_PROMETHEUS` configuration is not supported, and while `DEBUG` can be -used to enable debug output, it doesn't start a debug socket. +While `DEBUG` can be used to enable debug output, it doesn't start a debug socket. `run` supports different startup options to choose whether or not to enable the WebApps, the REST API or Swagger UI. By default, all three are enabled. @@ -278,14 +277,76 @@ To enable JPDA inside the container, you can set the environment variable container on port `8000` to debug your application. This is only supported for `wildfly` and `tomcat` distributions. -### Prometheus JMX Exporter +### OpenTelemetry Agent -To enable Prometheus JMX Exporter inside the container, you can set the -environment variable `JMX_PROMETHEUS=true` on startup of the container. -This will allow you to get metrics in Prometheus format at `:9404/metrics`. -For configuring exporter you need attach your configuration as a container volume -at `/camunda/javaagent/prometheus-jmx.yml`. This is only supported for `wildfly` -and `tomcat` distributions. +The CIB seven Docker images come with OpenTelemetry Java-Agent pre-installed. The agent automatically instruments your application to generate telemetry data (metrics, traces, and logs), but all exporters are disabled by default. You need to configure at least one exporter to provide telemetry data. + +#### Configuration + +The OpenTelemetry Agent can be configured using environment variables. + +**Pre-configured Environment Variables (from Dockerfile):** + +* `OTEL_SERVICE_NAME`: Service name for telemetry data (default: `cibseven`) +* `OTEL_METRICS_EXPORTER`: Configure metrics exporter (default: `none`, examples: `prometheus`, `otlp`) +* `OTEL_TRACES_EXPORTER`: Configure traces exporter (default: `none`, example: `otlp`) +* `OTEL_LOGS_EXPORTER`: Configure logs exporter (default: `none`, example: `otlp`) - **Note:** CIB seven uses a logging framework for application logging, so this is typically not needed +* `OTEL_EXPORTER_PROMETHEUS_PORT`: Port for Prometheus metrics exporter (default: `9464`) +* `OTEL_JAVAAGENT_LOGGING`: OpenTelemetry agent logging mode (default: `application`) +* `OTEL_JMX_CONFIG`: JMX metrics configuration files (default: `/camunda/javaagent/jmx_config.yaml,/camunda/javaagent/jmx_custom_config.yaml`) + +**Additional Configuration for Metrics Export:** + +* **Prometheus:** Set `OTEL_METRICS_EXPORTER=prometheus` (metrics available at port 9464) +* **OTLP:** Set `OTEL_METRICS_EXPORTER=otlp` and `OTEL_EXPORTER_OTLP_ENDPOINT=http://your-collector:4318` + +**Additional Configuration for Trace Export:** + +* Set `OTEL_TRACES_EXPORTER=otlp` +* Set `OTEL_EXPORTER_OTLP_ENDPOINT=http://your-collector:4318` + +For a complete list of available configuration options, see the [OpenTelemetry Configuration](https://opentelemetry.io/docs/concepts/sdk-configuration/) documentation. + + +#### JMX Metrics + +The OpenTelemetry Agent provides access to JVM metrics through JMX. By default, the image includes extended JVM metrics (CPU, memory, GC, threads, file descriptors). + +You can add custom JMX metrics by mounting your own configuration file to `/camunda/javaagent/jmx_custom_config.yaml`. For configuration syntax examples, see the pre-configured `opentelemetry/jmx_config.yaml` in this repository or the [OpenTelemetry JMX Metrics documentation](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/main/instrumentation/jmx-metrics). + +```bash +docker run -d --name cibseven -p 8080:8080 -p 9464:9464 \ + -e OTEL_METRICS_EXPORTER=prometheus \ + -e OTEL_EXPORTER_PROMETHEUS_PORT=9464 \ + -v $(pwd)/my_custom_jmx_config.yaml:/camunda/javaagent/jmx_custom_config.yaml \ + cibseven/cibseven:latest +``` + +**Note:** Create your own `my_custom_jmx_config.yaml` file before mounting it. The image already contains a default `/camunda/javaagent/jmx_custom_config.yaml` which you can override. + +#### Example with Docker Compose + +See the `test/docker-compose.yml` file for a complete example. Use the `camunda-opentelemetry` service that includes: +- CIB seven with Prometheus metrics exporter and OTLP traces +- OpenTelemetry Collector for receiving and processing telemetry data + +```bash +cd test +docker-compose up camunda-opentelemetry opentelemetry-collector +``` + +This will start CIB seven with metrics available at `http://localhost:9464/metrics` and an OpenTelemetry Collector that receives telemetry data on port 4318. + +#### Custom JMX Configuration + +You can create custom JMX metric rules by creating a YAML configuration file. See `opentelemetry/jmx_config.yaml` for an example that collects extended JVM metrics including CPU, memory, and file descriptor usage. + +#### Further Reading + +* [OpenTelemetry Java Agent Documentation](https://opentelemetry.io/docs/instrumentation/java/automatic/) +* [OpenTelemetry Java Agent GitHub Repository](https://github.com/open-telemetry/opentelemetry-java-instrumentation) +* [OpenTelemetry Configuration](https://opentelemetry.io/docs/concepts/sdk-configuration/) +* [JMX Metrics Collection](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/main/instrumentation/jmx-metrics) ### Change timezone diff --git a/cibseven-run.sh b/cibseven-run.sh index 3330032..0462207 100755 --- a/cibseven-run.sh +++ b/cibseven-run.sh @@ -30,6 +30,10 @@ if [[ -z "${SPRING_DATASOURCE_URL:-}" && -n "${DB_URL:-}" ]]; then export SPRING_DATASOURCE_URL="${DB_URL}" fi +# OpenTelemetry Agent configuration +# Load the agent via JAVA_OPTS instead of JAVA_TOOL_OPTIONS +export JAVA_OPTS="${JAVA_OPTS:-} -javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar" + CMD="/camunda/internal/run.sh start" wait_for_it diff --git a/cibseven-tomcat.sh b/cibseven-tomcat.sh index a39cca4..a72b73a 100755 --- a/cibseven-tomcat.sh +++ b/cibseven-tomcat.sh @@ -52,11 +52,9 @@ if [ "${DEBUG}" = "true" ]; then CMD+=" jpda" fi -if [ "$JMX_PROMETHEUS" = "true" ] ; then - echo "Enabling Prometheus JMX Exporter on port ${JMX_PROMETHEUS_PORT}" - [ ! -f "$JMX_PROMETHEUS_CONF" ] && touch "$JMX_PROMETHEUS_CONF" - export CATALINA_OPTS="${CATALINA_OPTS:=} -javaagent:/camunda/javaagent/jmx_prometheus_javaagent.jar=${JMX_PROMETHEUS_PORT}:${JMX_PROMETHEUS_CONF}" -fi +# OpenTelemetry Agent configuration +# Load the agent via CATALINA_OPTS (Tomcat-specific) instead of JAVA_TOOL_OPTIONS +export CATALINA_OPTS="${CATALINA_OPTS:-} -javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar" CMD+=" run" diff --git a/cibseven-wildfly.sh b/cibseven-wildfly.sh index dd2c6dd..d9fe9f4 100755 --- a/cibseven-wildfly.sh +++ b/cibseven-wildfly.sh @@ -54,12 +54,8 @@ if [ -z "$SKIP_DB_CONFIG" ]; then modify_datasource fi -if [ "$JMX_PROMETHEUS" = "true" ] ; then - # See https://issues.jboss.org/browse/LOGMGR-218 - export JBOSS_MODULES_SYSTEM_PKGS=${JBOSS_MODULES_SYSTEM_PKGS:-"org.jboss.byteman,org.jboss.logmanager"} -else - export JBOSS_MODULES_SYSTEM_PKGS=${JBOSS_MODULES_SYSTEM_PKGS:-"org.jboss.byteman"} -fi +# See https://issues.jboss.org/browse/LOGMGR-218 +export JBOSS_MODULES_SYSTEM_PKGS=${JBOSS_MODULES_SYSTEM_PKGS:-"org.jboss.byteman,org.jboss.logmanager"} # Ensure wildfly binds to public interface, preferes IPv4 and runs in the background export PREPEND_JAVA_OPTS="-Djboss.bind.address=0.0.0.0 -Djboss.bind.address.management=0.0.0.0 -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Djboss.modules.system.pkgs=${JBOSS_MODULES_SYSTEM_PKGS}" @@ -72,15 +68,14 @@ if [ "${DEBUG}" = "true" ]; then CMD+=" --debug *:8000" fi -if [ "$JMX_PROMETHEUS" = "true" ] ; then - echo "Enabling Prometheus JMX Exporter on port ${JMX_PROMETHEUS_PORT}" - [ ! -f "$JMX_PROMETHEUS_CONF" ] && touch "$JMX_PROMETHEUS_CONF" - # See https://github.com/prometheus/jmx_exporter/issues/344 - LOG_MANAGER_PATH=$(find /camunda/modules -name "jboss-logmanager*.jar") - COMMON_PATH=$(find /camunda/modules -name "wildfly-common*.jar") - export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -Dsun.util.logging.disableCallerCheck=true -Djava.util.logging.manager=org.jboss.logmanager.LogManager -Xbootclasspath/a:$LOG_MANAGER_PATH:$COMMON_PATH" - export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -javaagent:/camunda/javaagent/jmx_prometheus_javaagent.jar=${JMX_PROMETHEUS_PORT}:${JMX_PROMETHEUS_CONF}" -fi +# JBoss LogManager configuration for OpenTelemetry +# See https://github.com/prometheus/jmx_exporter/issues/344 +LOG_MANAGER_PATH=$(find /camunda/modules -name "jboss-logmanager*.jar") +COMMON_PATH=$(find /camunda/modules -name "wildfly-common*.jar") +SMALLRYE_OS_PATH=$(find /camunda/modules -name "smallrye-common-os*.jar") +SMALLRYE_NET_PATH=$(find /camunda/modules -name "smallrye-common-net*.jar") +export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -Dsun.util.logging.disableCallerCheck=true -Djava.util.logging.manager=org.jboss.logmanager.LogManager -Xbootclasspath/a:$LOG_MANAGER_PATH:$COMMON_PATH:$SMALLRYE_OS_PATH:$SMALLRYE_NET_PATH" +export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar" wait_for_it diff --git a/download.sh b/download.sh index 178e633..c593870 100755 --- a/download.sh +++ b/download.sh @@ -91,6 +91,12 @@ mvn dependency:copy -B \ -Dartifact="org.postgresql:postgresql:${POSTGRESQL_VERSION}:jar" \ -DoutputDirectory=/tmp/ +# download OpenTelemetry Java Agent +mvn dependency:copy -B \ + $PROXY \ + -Dartifact="io.opentelemetry.javaagent:opentelemetry-javaagent:${OPENTELEMETRY_AGENT_VERSION}:jar" \ + -DoutputDirectory=/tmp/ + case ${DISTRO} in wildfly*) cat <<-EOF > batch.cli @@ -120,12 +126,6 @@ EOF ;; esac -# download Prometheus JMX Exporter. -# Details on https://blog.camunda.com/post/2019/06/camunda-bpm-on-kubernetes/ -mvn dependency:copy -B \ - $PROXY \ - -Dartifact="io.prometheus.jmx:jmx_prometheus_javaagent:${JMX_PROMETHEUS_VERSION}:jar" \ - -DoutputDirectory=/tmp/ - +# Install OpenTelemetry Java Agent for all distributions mkdir -p /camunda/javaagent -cp /tmp/jmx_prometheus_javaagent-${JMX_PROMETHEUS_VERSION}.jar /camunda/javaagent/jmx_prometheus_javaagent.jar +cp /tmp/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar /camunda/javaagent/ diff --git a/opentelemetry/jmx_config.yaml b/opentelemetry/jmx_config.yaml new file mode 100644 index 0000000..ecaf443 --- /dev/null +++ b/opentelemetry/jmx_config.yaml @@ -0,0 +1,80 @@ +rules: + # Config from: https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/main/instrumentation/jmx-metrics/library/src/main/resources/jmx/rules/jvm.yaml + # Extended for CIB: MaxFileDescriptorCount, TotalSwapSpaceSize, FreeSwapSpaceSize, TotalPhysicalMemorySize, FreePhysicalMemorySize, CommittedVirtualMemorySize + - bean: java.lang:type=OperatingSystem + prefix: os. + dropNegativeValues: true + mapping: + # os.cpu.count + AvailableProcessors: + metric: cpu.count + type: updowncounter + unit: "{cpu}" + desc: Number of processors available to the Java virtual machine. + # os.cpu.time + ProcessCpuTime: + metric: cpu.time + type: counter + sourceUnit: ns + unit: s + desc: CPU time used by the process as reported by the JVM. + # os.cpu.recent_utilization + ProcessCpuLoad: + metric: cpu.recent_utilization + type: gauge + unit: '1' + desc: Recent CPU utilization for the process as reported by the JVM. + # os.system.cpu.load_1m (experimental) + SystemLoadAverage: + metric: system.cpu.load_1m + type: gauge + unit: "{run_queue_item}" + desc: Average CPU load of the whole system for the last minute as reported by the JVM. + # os.system.cpu.utilization (experimental) + SystemCpuLoad: + metric: system.cpu.utilization + type: gauge + unit: '1' + desc: Recent CPU utilization for the whole system as reported by the JVM. + # os.file_descriptor.open.count (experimental) + OpenFileDescriptorCount: + metric: file_descriptor.open.count + type: updowncounter + unit: "{file_descriptor}" + desc: Number of open file descriptors as reported by the JVM. + # os.file_descriptor.max.count + MaxFileDescriptorCount: + metric: file_descriptor.max.count + type: updowncounter + unit: "{file_descriptor}" + desc: Number of max file descriptors as reported by the JVM. + # virtual_memory.committed.size + CommittedVirtualMemorySize: + metric: virtual_memory.committed.size + type: gauge + unit: 'By' + desc: Size of CommittedVirtualMemorySize. + # os.physical_memory.free.size + FreePhysicalMemorySize: + metric: physical_memory.free.size + type: gauge + unit: 'By' + desc: Size of FreePhysicalMemorySize. + # os.physical_memory.total.size + TotalPhysicalMemorySize: + metric: physical_memory.total.size + type: gauge + unit: 'By' + desc: Size of TotalPhysicalMemorySize. + # os.swap_space.free.size + FreeSwapSpaceSize: + metric: swap_space.free.size + type: gauge + unit: 'By' + desc: Size of FreeSwapSpaceSize. + # os.swap_space.total.size + TotalSwapSpaceSize: + metric: swap_space.total.size + type: gauge + unit: 'By' + desc: Size of TotalSwapSpaceSize. \ No newline at end of file diff --git a/opentelemetry/jmx_custom_config.yaml b/opentelemetry/jmx_custom_config.yaml new file mode 100644 index 0000000..3ba9c1c --- /dev/null +++ b/opentelemetry/jmx_custom_config.yaml @@ -0,0 +1,6 @@ +# Custom JMX Metrics Configuration +# +# This file can be overridden by mounting your own configuration: +# -v $(pwd)/my_custom_jmx_config.yaml:/camunda/javaagent/jmx_custom_config.yaml +# +# For configuration syntax and examples, see jmx_config.yaml diff --git a/test/docker-compose.yml b/test/docker-compose.yml index c688f37..f65098b 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -46,18 +46,6 @@ services: - "8000:8000" restart: unless-stopped - camunda-prometheus-jmx: - image: cibseven/cibseven:${IMAGE_TAG:-${DISTRO}-${PLATFORM}} - platform: linux/${PLATFORM} - environment: - - JMX_PROMETHEUS=true - ports: - - "8080:8080" - - "9404:9404" - volumes: - - $PWD/prometheus-jmx.yml:/camunda/javaagent/prometheus-jmx.yml - restart: unless-stopped - camunda-password-file: image: cibseven/cibseven:${IMAGE_TAG:-${DISTRO}-${PLATFORM}} platform: linux/${PLATFORM} @@ -88,3 +76,28 @@ services: environment: - POSTGRES_USER=camunda - POSTGRES_PASSWORD=camunda + + camunda-opentelemetry: + image: cibseven/cibseven:${IMAGE_TAG:-${DISTRO}-${PLATFORM}} + platform: linux/${PLATFORM} + environment: + - OTEL_METRICS_EXPORTER=prometheus + - OTEL_LOGS_EXPORTER=none + - OTEL_TRACES_EXPORTER=otlp + - OTEL_EXPORTER_PROMETHEUS_PORT=9464 + - OTEL_EXPORTER_OTLP_ENDPOINT=http://opentelemetry-collector:4318 + ports: + - "8080:8080" + - "9464:9464" + depends_on: + - opentelemetry-collector + restart: unless-stopped + + opentelemetry-collector: + image: otel/opentelemetry-collector:0.145.0 + command: ["--config=/etc/otel-collector-config.yml"] + volumes: + - ./otel-collector-config.yml:/etc/otel-collector-config.yml + ports: + - "4318:4318" + restart: unless-stopped \ No newline at end of file diff --git a/test/otel-collector-config.yml b/test/otel-collector-config.yml new file mode 100644 index 0000000..de75754 --- /dev/null +++ b/test/otel-collector-config.yml @@ -0,0 +1,31 @@ +receivers: + otlp: + protocols: + http: + endpoint: "0.0.0.0:4318" + + prometheus: + config: + scrape_configs: + - job_name: 'cibseven' + scrape_interval: 30s + static_configs: + - targets: ['camunda-opentelemetry:9464'] + labels: + service: 'cibseven' + +exporters: + debug: + verbosity: detailed + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug] + metrics: + receivers: [otlp, prometheus] + exporters: [debug] + logs: + receivers: [otlp] + exporters: [debug] diff --git a/test/prometheus-jmx.yml b/test/prometheus-jmx.yml deleted file mode 100644 index 54ee51a..0000000 --- a/test/prometheus-jmx.yml +++ /dev/null @@ -1,5 +0,0 @@ -startDelaySeconds: 0 -ssl: false - -rules: - - pattern: "java.lang:type=ClassLoading" diff --git a/test/test-debug.sh b/test/test-debug.sh index 3009797..a66a4fe 100755 --- a/test/test-debug.sh +++ b/test/test-debug.sh @@ -8,7 +8,7 @@ test "${DISTRO}" = "run" && _log "skipping test of DEBUG socket: not supported f start_container -poll_log "Listening for transport dt_socket at address: 8000" "ERROR" || _exit 1 "JPDA not started" +poll_log "Listening for transport dt_socket at address: 8000" "ERROR(?!.*stderr.*SLF4J)" || _exit 1 "JPDA not started" timeout 1 bash -c 'cat < /dev/null > /dev/tcp/localhost/8000' || _exit 2 "JPDA port not open" diff --git a/test/test-opentelemetry-run.sh b/test/test-opentelemetry-run.sh new file mode 100755 index 0000000..38d6cf6 --- /dev/null +++ b/test/test-opentelemetry-run.sh @@ -0,0 +1,22 @@ +#!/bin/bash -eu + +SERVICE=${1} + +source test_helper.sh + +start_container + +WAIT=10 poll_log "starting to acquire jobs" "Application run failed" || _exit 1 "Server not started" + +_log "Server started" + +# Test OpenTelemetry metrics endpoint +_log "Testing OpenTelemetry metrics endpoint" +curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 3 "OpenTelemetry metrics not available" +_log "OpenTelemetry metrics available" + +# Verify JMX metrics from opentelemetry/jmx_config.yaml are present +assert_jmx_metrics "http://localhost:9464/metrics" || _exit 4 "Not all JMX metrics from config are exposed" +_log "All JMX metrics from config are present" + +_exit 0 "Test successful" diff --git a/test/test-opentelemetry-tomcat.sh b/test/test-opentelemetry-tomcat.sh new file mode 100755 index 0000000..d263957 --- /dev/null +++ b/test/test-opentelemetry-tomcat.sh @@ -0,0 +1,22 @@ +#!/bin/bash -eu + +SERVICE=${1} + +source test_helper.sh + +start_container + +poll_log "org.apache.catalina.startup.Catalina.start Server startup in" "^SEVERE" || _exit 1 "Server not started" + +_log "Server started" + +# Test OpenTelemetry metrics endpoint +_log "Testing OpenTelemetry metrics endpoint" +curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 3 "OpenTelemetry metrics not available" +_log "OpenTelemetry metrics available" + +# Verify JMX metrics from opentelemetry/jmx_config.yaml are present +assert_jmx_metrics "http://localhost:9464/metrics" || _exit 4 "Not all JMX metrics from config are exposed" +_log "All JMX metrics from config are present" + +_exit 0 "Test successful" \ No newline at end of file diff --git a/test/test-opentelemetry-wildfly.sh b/test/test-opentelemetry-wildfly.sh new file mode 100755 index 0000000..f9d5f78 --- /dev/null +++ b/test/test-opentelemetry-wildfly.sh @@ -0,0 +1,22 @@ +#!/bin/bash -eu + +SERVICE=${1} + +source test_helper.sh + +start_container + +poll_log 'started in' 'started (with errors) in' || _exit 1 "Server not started" + +_log "Server started" + +# Test OpenTelemetry metrics endpoint +_log "Testing OpenTelemetry metrics endpoint" +curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 3 "OpenTelemetry metrics not available" +_log "OpenTelemetry metrics available" + +# Verify JMX metrics from opentelemetry/jmx_config.yaml are present +assert_jmx_metrics "http://localhost:9464/metrics" || _exit 4 "Not all JMX metrics from config are exposed" +_log "All JMX metrics from config are present" + +_exit 0 "Test successful" \ No newline at end of file diff --git a/test/test-prometheus-jmx-run.sh b/test/test-prometheus-jmx-run.sh deleted file mode 100755 index 4af92d1..0000000 --- a/test/test-prometheus-jmx-run.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -eu - -SERVICE=${1} - -source test_helper.sh - -_log "skipping test of JMX Prometheus exporter: not supported for cibseven-run" diff --git a/test/test-prometheus-jmx-tomcat.sh b/test/test-prometheus-jmx-tomcat.sh deleted file mode 100755 index 88439f4..0000000 --- a/test/test-prometheus-jmx-tomcat.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -eu - -SERVICE=${1} - -source test_helper.sh - -start_container - -poll_log "org.apache.catalina.startup.Catalina.start Server startup in" "^SEVERE" || _exit 1 "Server not started" - -_log "Server started" - -grep_log "Enabling Prometheus JMX Exporter on port" || _exit 2 "Prometheus JMX Exporter not enabled" - -curl -s http://localhost:9404/metrics | grep -q "jvm_classes_currently_loaded" || _exit 3 "Prometheus metrics not available" - -_exit 0 "Test successfull" diff --git a/test/test-prometheus-jmx-wildfly.sh b/test/test-prometheus-jmx-wildfly.sh deleted file mode 100755 index ec0ab21..0000000 --- a/test/test-prometheus-jmx-wildfly.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -eu - -SERVICE=${1} - -source test_helper.sh - -start_container - -poll_log 'started in' 'started (with errors) in' || _exit 1 "Server not started" - -_log "Server started" - -grep_log "Enabling Prometheus JMX Exporter on port" || _exit 2 "Prometheus JMX Exporter not enabled" - -curl -s http://localhost:9404/metrics | grep -q "jvm_classes_currently_loaded" || _exit 3 "Prometheus metrics not available" - -_exit 0 "Test successfull" diff --git a/test/test.sh b/test/test.sh index 1a4828f..fbd604d 100755 --- a/test/test.sh +++ b/test/test.sh @@ -13,12 +13,12 @@ fi source test_helper.sh -docker-compose up --force-recreate -d postgres mysql +docker-compose up --force-recreate -d postgres mysql opentelemetry-collector ./test-${DISTRO}.sh camunda ./test-${DISTRO}.sh camunda-mysql ./test-${DISTRO}.sh camunda-postgres ./test-${DISTRO}.sh camunda-password-file -./test-prometheus-jmx-${DISTRO}.sh camunda-prometheus-jmx +./test-opentelemetry-${DISTRO}.sh camunda-opentelemetry ./test-debug.sh camunda-debug docker-compose down -v cd - diff --git a/test/test_helper.sh b/test/test_helper.sh index 2c6d458..728801e 100644 --- a/test/test_helper.sh +++ b/test/test_helper.sh @@ -83,3 +83,60 @@ function test_encoding { curl --fail -w "\n" http://localhost:8080/engine-rest/deployment/create -F deployment-name=testEncoding -F testEncoding.bpmn=@testEncoding.bpmn curl --fail -w "\n" -H "Content-Type: application/json" -d '{}' http://localhost:8080/engine-rest/process-definition/key/testEncoding/start } + +# Expected Prometheus metric names from opentelemetry/jmx_config.yaml as exported by the OTel agent +# (prefix os. + mapping metrics; agent adds unit suffixes: _ratio, _seconds_total, _bytes). +EXPECTED_JMX_METRICS=( + os_cpu_count + os_cpu_time_seconds_total + os_cpu_recent_utilization_ratio + os_system_cpu_load_1m + os_system_cpu_utilization_ratio + os_file_descriptor_open_count + os_file_descriptor_max_count + os_virtual_memory_committed_size_bytes + os_physical_memory_free_size_bytes + os_physical_memory_total_size_bytes + os_swap_space_free_size_bytes + os_swap_space_total_size_bytes +) + +# Verifies that all expected JMX metrics are present on the Prometheus metrics endpoint. +# Polls until all metrics appear (JMX metrics are created asynchronously by the agent after server start) or timeout. +# Usage: assert_jmx_metrics [metrics_url] +# Default metrics_url: http://localhost:9464/metrics +JMX_METRICS_RETRIES=10 +JMX_METRICS_WAIT=60 + +function assert_jmx_metrics { + local metrics_url="${1:-http://localhost:9464/metrics}" + local metrics_output + local missing="" + local attempt + + for attempt in $(seq 1 $JMX_METRICS_RETRIES); do + _log "Checking JMX metrics (attempt $attempt/$JMX_METRICS_RETRIES)" + metrics_output=$(curl -s --fail "$metrics_url") || { _log "Failed to fetch $metrics_url"; return 1; } + missing="" + + for prometheus_name in "${EXPECTED_JMX_METRICS[@]}"; do + if ! echo "$metrics_output" | grep -qE "(^|[[:space:]])${prometheus_name}([{\s]|$)"; then + [ -n "$missing" ] && missing="$missing, " + missing="${missing}${prometheus_name}" + fi + done + + if [ -z "$missing" ]; then + _log "All JMX metrics present" + return 0 + fi + + if [ $attempt -lt $JMX_METRICS_RETRIES ]; then + _log "Waiting $JMX_METRICS_WAIT s for JMX metrics (missing: $missing)" + sleep $JMX_METRICS_WAIT + fi + done + + _log "Missing expected JMX metrics after ${JMX_METRICS_RETRIES} attempts: $missing" + return 1 +}