From e8e156272b80a7efc6757140f596e95ab191fd12 Mon Sep 17 00:00:00 2001 From: Oleg Skrypnyuk Date: Tue, 2 Sep 2025 18:07:03 +0200 Subject: [PATCH 01/44] feat(opentelemetry): add OpenTelemetry configuration and JMX metrics support --- Dockerfile | 27 ++++++++++++- opentelemetry/compose.yml | 31 ++++++++++++++ opentelemetry/jmx_config.yaml | 51 ++++++++++++++++++++++++ opentelemetry/jmx_custom_config.yaml | 0 opentelemetry/otel-collector-config.yaml | 21 ++++++++++ 5 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 opentelemetry/compose.yml create mode 100644 opentelemetry/jmx_config.yaml create mode 100644 opentelemetry/jmx_custom_config.yaml create mode 100644 opentelemetry/otel-collector-config.yaml diff --git a/Dockerfile b/Dockerfile index 436520d..1f60eb6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,9 @@ ARG MYSQL_VERSION ARG JMX_PROMETHEUS_VERSION=1.0.1 +# --- OpenTelemetry Java Agent version argument --- +ARG OPENTELEMETRY_AGENT_VERSION=2.19.0 + RUN apk add --no-cache \ bash \ ca-certificates \ @@ -30,12 +33,18 @@ COPY settings.xml download.sh cibseven-run.sh cibseven-tomcat.sh cibseven-wildfl RUN /tmp/download.sh COPY wait_for_it-lib.sh /camunda/ +# --- Create javaagent directory and download the OpenTelemetry agent --- +RUN mkdir -p /camunda/javaagent && \ + wget -O /camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar \ + https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v${OPENTELEMETRY_AGENT_VERSION}/opentelemetry-javaagent.jar ##### FINAL IMAGE ##### FROM alpine:3.22 ARG VERSION=2.1.0 +ARG OPENTELEMETRY_AGENT_VERSION=2.19.0 +ENV OPENTELEMETRY_AGENT_VERSION=$OPENTELEMETRY_AGENT_VERSION ENV DB_DRIVER= ENV DB_URL= @@ -52,11 +61,23 @@ ENV WAIT_FOR_TIMEOUT=30 ENV TZ=UTC ENV DEBUG=false ENV JAVA_OPTS="" + +# --- Use OpenTelemetry agent by default --- +ENV JAVA_TOOL_OPTIONS="-javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar" + ENV JMX_PROMETHEUS=false ENV JMX_PROMETHEUS_CONF=/camunda/javaagent/prometheus-jmx.yml ENV JMX_PROMETHEUS_PORT=9404 -EXPOSE 8080 8000 9404 +# OpenTelemetry default exporter settings (all exporters disabled, user must configure) +ENV OTEL_SERVICE_NAME=cibseven-java17 \ + OTEL_JMX_CONFIG=/camunda/javaagent/jmx_config.yaml,/camunda/javaagent/jmx_custom_config.yaml \ + OTEL_METRICS_EXPORTER=none \ + OTEL_LOGS_EXPORTER=none \ + OTEL_TRACES_EXPORTER=none \ + OTEL_EXPORTER_PROMETHEUS_PORT=9464 + +EXPOSE 8080 8000 9404 9464 # Downgrading wait-for-it is necessary until this PR is merged # https://github.com/vishnubob/wait-for-it/pull/68 @@ -81,3 +102,7 @@ ENTRYPOINT ["/sbin/tini", "--"] CMD ["./cibseven.sh"] COPY --chown=camunda:camunda --from=builder /camunda . + +# --- Add JMX config files (ensure these are present in your build context) --- +COPY opentelemetry/jmx_config.yaml /camunda/javaagent/jmx_config.yaml +COPY opentelemetry/jmx_custom_config.yaml /camunda/javaagent/jmx_custom_config.yaml diff --git a/opentelemetry/compose.yml b/opentelemetry/compose.yml new file mode 100644 index 0000000..9909c5c --- /dev/null +++ b/opentelemetry/compose.yml @@ -0,0 +1,31 @@ +services: + cibseven: + image: cibseven/cibseven:tomcat-amd64 + container_name: cibseven + environment: + OTEL_METRICS_EXPORTER: "otlp" + OTEL_LOGS_EXPORTER: "otlp" + OTEL_TRACES_EXPORTER: "otlp" + OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4318" + ports: + - 8080:8080 + - 9464:9464 + depends_on: + otel-collector: + condition: service_started + networks: + - otelnet + + otel-collector: + image: otel/opentelemetry-collector:latest + container_name: otel-collector + ports: + - "4318:4318" + command: ["--config=/etc/otel-collector-config.yaml"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + networks: + - otelnet + +networks: + otelnet: diff --git a/opentelemetry/jmx_config.yaml b/opentelemetry/jmx_config.yaml new file mode 100644 index 0000000..87ef6f8 --- /dev/null +++ b/opentelemetry/jmx_config.yaml @@ -0,0 +1,51 @@ +rules: + - bean: java.lang:type=OperatingSystem + mapping: + AvailableProcessors: + metric: java_lang_operatingsystem_availableprocessors + type: updowncounter + desc: java_lang_operatingsystem_availableprocessors + ProcessCpuLoad: + metric: java_lang_operatingsystem_processcpuload + type: updowncounter + desc: java_lang_operatingsystem_processcpuload. The recent cpu usage for the Java Virtual Machine process. + SystemCpuLoad: + metric: java_lang_operatingsystem_systemcpuload + type: updowncounter + desc: java_lang_operatingsystem_systemcpuload. The recent cpu usage for the whole system. + ProcessCpuTime: + metric: java_lang_operatingsystem_processcputime + type: updowncounter + desc: java_lang_operatingsystem_processcputime + SystemLoadAverage: + metric: java_lang_operatingsystem_systemloadaverage + type: updowncounter + desc: java_lang_operatingsystem_systemloadaverage + CommittedVirtualMemorySize: + metric: java_lang_operatingsystem_committedvirtualmemorysize + type: updowncounter + desc: java_lang_operatingsystem_committedvirtualmemorysize + FreePhysicalMemorySize: + metric: java_lang_operatingsystem_freephysicalmemorysize + type: updowncounter + desc: java_lang_operatingsystem_freephysicalmemorysize + TotalPhysicalMemorySize: + metric: java_lang_operatingsystem_totalphysicalmemorysize + type: updowncounter + desc: java_lang_operatingsystem_totalphysicalmemorysize + FreeSwapSpaceSize: + metric: java_lang_operatingsystem_freeswapspacesize + type: updowncounter + desc: java_lang_operatingsystem_freeswapspacesize + TotalSwapSpaceSize: + metric: java_lang_operatingsystem_totalswapspacesize + type: updowncounter + desc: java_lang_operatingsystem_totalswapspacesize + OpenFileDescriptorCount: + metric: java_lang_operatingsystem_openfiledescriptorcount + type: updowncounter + desc: java_lang_operatingsystem_openfiledescriptorcount + MaxFileDescriptorCount: + metric: java_lang_operatingsystem_maxfiledescriptorcount + type: updowncounter + desc: java_lang_operatingsystem_maxfiledescriptorcount \ No newline at end of file diff --git a/opentelemetry/jmx_custom_config.yaml b/opentelemetry/jmx_custom_config.yaml new file mode 100644 index 0000000..e69de29 diff --git a/opentelemetry/otel-collector-config.yaml b/opentelemetry/otel-collector-config.yaml new file mode 100644 index 0000000..7615966 --- /dev/null +++ b/opentelemetry/otel-collector-config.yaml @@ -0,0 +1,21 @@ +receivers: + otlp: + protocols: + http: + endpoint: "0.0.0.0:4318" + +exporters: + debug: + verbosity: detailed + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug] + metrics: + receivers: [otlp] + exporters: [debug] + logs: + receivers: [otlp] + exporters: [debug] From f99ed98499f8f75da1964d789f40e03e342693a2 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 14 Jan 2026 08:08:04 +0100 Subject: [PATCH 02/44] feat(opentelemetry): update OpenTelemetry version and settings, tests, etc. --- Dockerfile | 15 +-- README.md | 62 +++++++++-- cibseven-tomcat.sh | 5 - cibseven-wildfly.sh | 24 ++-- download.sh | 17 +-- opentelemetry/compose.yml | 31 ------ opentelemetry/jmx_config.yaml | 103 +++++++++++------- test/docker-compose.yml | 38 +++++-- .../otel-collector-config.yml | 42 +++---- test/prometheus-jmx.yml | 5 - test/test-prometheus-jmx-run.sh | 7 -- test/test-prometheus-jmx-tomcat.sh | 17 --- test/test-prometheus-jmx-wildfly.sh | 17 --- test/test-run.sh | 5 + test/test-tomcat.sh | 5 + test/test-wildfly.sh | 5 + test/test.sh | 1 - 17 files changed, 202 insertions(+), 197 deletions(-) delete mode 100644 opentelemetry/compose.yml rename opentelemetry/otel-collector-config.yaml => test/otel-collector-config.yml (94%) delete mode 100644 test/prometheus-jmx.yml delete mode 100755 test/test-prometheus-jmx-run.sh delete mode 100755 test/test-prometheus-jmx-tomcat.sh delete mode 100755 test/test-prometheus-jmx-wildfly.sh diff --git a/Dockerfile b/Dockerfile index 1f60eb6..a3e9f05 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,10 +15,9 @@ ARG MAVEN_PROXY_PASSWORD ARG POSTGRESQL_VERSION ARG MYSQL_VERSION -ARG JMX_PROMETHEUS_VERSION=1.0.1 # --- OpenTelemetry Java Agent version argument --- -ARG OPENTELEMETRY_AGENT_VERSION=2.19.0 +ARG OPENTELEMETRY_AGENT_VERSION=2.23.0 RUN apk add --no-cache \ bash \ @@ -33,17 +32,12 @@ COPY settings.xml download.sh cibseven-run.sh cibseven-tomcat.sh cibseven-wildfl RUN /tmp/download.sh COPY wait_for_it-lib.sh /camunda/ -# --- Create javaagent directory and download the OpenTelemetry agent --- -RUN mkdir -p /camunda/javaagent && \ - wget -O /camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar \ - https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v${OPENTELEMETRY_AGENT_VERSION}/opentelemetry-javaagent.jar - ##### FINAL IMAGE ##### FROM alpine:3.22 ARG VERSION=2.1.0 -ARG OPENTELEMETRY_AGENT_VERSION=2.19.0 +ARG OPENTELEMETRY_AGENT_VERSION=2.23.0 ENV OPENTELEMETRY_AGENT_VERSION=$OPENTELEMETRY_AGENT_VERSION ENV DB_DRIVER= @@ -65,12 +59,9 @@ ENV JAVA_OPTS="" # --- Use OpenTelemetry agent by default --- ENV JAVA_TOOL_OPTIONS="-javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar" -ENV JMX_PROMETHEUS=false -ENV JMX_PROMETHEUS_CONF=/camunda/javaagent/prometheus-jmx.yml -ENV JMX_PROMETHEUS_PORT=9404 # OpenTelemetry default exporter settings (all exporters disabled, user must configure) -ENV OTEL_SERVICE_NAME=cibseven-java17 \ +ENV OTEL_SERVICE_NAME=cibseven \ OTEL_JMX_CONFIG=/camunda/javaagent/jmx_config.yaml,/camunda/javaagent/jmx_custom_config.yaml \ OTEL_METRICS_EXPORTER=none \ OTEL_LOGS_EXPORTER=none \ diff --git a/README.md b/README.md index 5d6770a..9f3c800 100644 --- a/README.md +++ b/README.md @@ -107,8 +107,7 @@ when provided: * `DB_URL` * `DB_PASSWORD_FILE` -The `JMX_PROMETHEUS` configuration is not supported, and while `DEBUG` can be -used to enable debug output, it doesn't start a debug socket. +While `DEBUG` can be used to enable debug output, it doesn't start a debug socket. `run` supports different startup options to choose whether or not to enable the WebApps, the REST API or Swagger UI. By default, all three are enabled. @@ -278,14 +277,59 @@ To enable JPDA inside the container, you can set the environment variable container on port `8000` to debug your application. This is only supported for `wildfly` and `tomcat` distributions. -### Prometheus JMX Exporter +### OpenTelemetry Agent -To enable Prometheus JMX Exporter inside the container, you can set the -environment variable `JMX_PROMETHEUS=true` on startup of the container. -This will allow you to get metrics in Prometheus format at `:9404/metrics`. -For configuring exporter you need attach your configuration as a container volume -at `/camunda/javaagent/prometheus-jmx.yml`. This is only supported for `wildfly` -and `tomcat` distributions. +The CIB seven Docker images come with OpenTelemetry Java-Agent pre-installed. The agent automatically instruments your application to generate telemetry data (metrics, traces, and logs), but all exporters are disabled by default. You need to configure at least one exporter to provide telemetry data. + +#### Configuration + +The OpenTelemetry Agent can be configured using environment variables. + +**Available Environment Variables:** + +* `OTEL_SERVICE_NAME`: Service name for telemetry data (default: `cibseven`) +* `OTEL_METRICS_EXPORTER`: Configure metrics exporter (e.g., `prometheus`, `otlp`) +* `OTEL_TRACES_EXPORTER`: Configure traces exporter (e.g., `otlp`, `jaeger`) +* `OTEL_LOGS_EXPORTER`: Configure logs exporter (e.g., `otlp`) - **Note:** CIB seven uses log4j2 for application logging, so this is typically not needed +* `OTEL_EXPORTER_PROMETHEUS_PORT`: Port for Prometheus metrics exporter (default: `9464`) +* `OTEL_EXPORTER_OTLP_ENDPOINT`: Endpoint for OTLP exporter (e.g., `http://otel-collector:4318`) + + +#### JMX Metrics + +The OpenTelemetry Agent provides access to JVM metrics through JMX. You can customize JMX metrics collection by mounting a custom configuration file: + +```bash +docker run -d --name cibseven -p 8080:8080 -p 9464:9464 \ + -e OTEL_METRICS_EXPORTER=prometheus \ + -e OTEL_EXPORTER_PROMETHEUS_PORT=9464 \ + -v $(pwd)/custom_jmx_config.yaml:/camunda/javaagent/jmx_custom_config.yaml \ + cibseven/cibseven:latest +``` + +#### Example with Docker Compose + +See the `test/docker-compose.yml` file for a complete example. Use the `camunda-opentelemetry` service that includes: +- CIB seven with Prometheus metrics exporter and OTLP traces +- OpenTelemetry Collector for receiving and processing telemetry data + +```bash +cd test +docker-compose up camunda-opentelemetry opentelemetry-collector +``` + +This will start CIB seven with metrics available at `http://localhost:9464/metrics` and an OpenTelemetry Collector that receives telemetry data on port 4318. + +#### Custom JMX Configuration + +You can create custom JMX metric rules by creating a YAML configuration file. See `opentelemetry/jmx_config.yaml` for an example that collects extended JVM metrics including CPU, memory, and file descriptor usage. + +#### Further Reading + +* [OpenTelemetry Java Agent Documentation](https://opentelemetry.io/docs/instrumentation/java/automatic/) +* [OpenTelemetry Java Agent GitHub Repository](https://github.com/open-telemetry/opentelemetry-java-instrumentation) +* [OpenTelemetry Configuration](https://opentelemetry.io/docs/concepts/sdk-configuration/) +* [JMX Metrics Collection](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/main/instrumentation/jmx-metrics) ### Change timezone diff --git a/cibseven-tomcat.sh b/cibseven-tomcat.sh index a39cca4..bf6ee0f 100755 --- a/cibseven-tomcat.sh +++ b/cibseven-tomcat.sh @@ -52,11 +52,6 @@ if [ "${DEBUG}" = "true" ]; then CMD+=" jpda" fi -if [ "$JMX_PROMETHEUS" = "true" ] ; then - echo "Enabling Prometheus JMX Exporter on port ${JMX_PROMETHEUS_PORT}" - [ ! -f "$JMX_PROMETHEUS_CONF" ] && touch "$JMX_PROMETHEUS_CONF" - export CATALINA_OPTS="${CATALINA_OPTS:=} -javaagent:/camunda/javaagent/jmx_prometheus_javaagent.jar=${JMX_PROMETHEUS_PORT}:${JMX_PROMETHEUS_CONF}" -fi CMD+=" run" diff --git a/cibseven-wildfly.sh b/cibseven-wildfly.sh index dd2c6dd..8415581 100755 --- a/cibseven-wildfly.sh +++ b/cibseven-wildfly.sh @@ -54,15 +54,18 @@ if [ -z "$SKIP_DB_CONFIG" ]; then modify_datasource fi -if [ "$JMX_PROMETHEUS" = "true" ] ; then - # See https://issues.jboss.org/browse/LOGMGR-218 - export JBOSS_MODULES_SYSTEM_PKGS=${JBOSS_MODULES_SYSTEM_PKGS:-"org.jboss.byteman,org.jboss.logmanager"} -else - export JBOSS_MODULES_SYSTEM_PKGS=${JBOSS_MODULES_SYSTEM_PKGS:-"org.jboss.byteman"} -fi +# See https://issues.jboss.org/browse/LOGMGR-218 +export JBOSS_MODULES_SYSTEM_PKGS=${JBOSS_MODULES_SYSTEM_PKGS:-"org.jboss.byteman,org.jboss.logmanager"} # Ensure wildfly binds to public interface, preferes IPv4 and runs in the background export PREPEND_JAVA_OPTS="-Djboss.bind.address=0.0.0.0 -Djboss.bind.address.management=0.0.0.0 -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Djboss.modules.system.pkgs=${JBOSS_MODULES_SYSTEM_PKGS}" + +# OpenTelemetry Agent JMX configuration +# See https://github.com/prometheus/jmx_exporter/issues/344 +LOG_MANAGER_PATH=$(find /camunda/modules -name "jboss-logmanager*.jar") +COMMON_PATH=$(find /camunda/modules -name "wildfly-common*.jar") +export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -Dsun.util.logging.disableCallerCheck=true -Djava.util.logging.manager=org.jboss.logmanager.LogManager -Xbootclasspath/a:$LOG_MANAGER_PATH:$COMMON_PATH -Dotel.javaagent.logging=application" +export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -Dsun.util.logging.disableCallerCheck=true -Djava.util.logging.manager=org.jboss.logmanager.LogManager -Xbootclasspath/a:$LOG_MANAGER_PATH:$COMMON_PATH -Dotel.javaagent.logging=application" export LAUNCH_JBOSS_IN_BACKGROUND=TRUE CMD="/camunda/bin/standalone.sh" @@ -72,15 +75,6 @@ if [ "${DEBUG}" = "true" ]; then CMD+=" --debug *:8000" fi -if [ "$JMX_PROMETHEUS" = "true" ] ; then - echo "Enabling Prometheus JMX Exporter on port ${JMX_PROMETHEUS_PORT}" - [ ! -f "$JMX_PROMETHEUS_CONF" ] && touch "$JMX_PROMETHEUS_CONF" - # See https://github.com/prometheus/jmx_exporter/issues/344 - LOG_MANAGER_PATH=$(find /camunda/modules -name "jboss-logmanager*.jar") - COMMON_PATH=$(find /camunda/modules -name "wildfly-common*.jar") - export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -Dsun.util.logging.disableCallerCheck=true -Djava.util.logging.manager=org.jboss.logmanager.LogManager -Xbootclasspath/a:$LOG_MANAGER_PATH:$COMMON_PATH" - export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -javaagent:/camunda/javaagent/jmx_prometheus_javaagent.jar=${JMX_PROMETHEUS_PORT}:${JMX_PROMETHEUS_CONF}" -fi wait_for_it diff --git a/download.sh b/download.sh index 178e633..350f036 100755 --- a/download.sh +++ b/download.sh @@ -91,6 +91,12 @@ mvn dependency:copy -B \ -Dartifact="org.postgresql:postgresql:${POSTGRESQL_VERSION}:jar" \ -DoutputDirectory=/tmp/ +# download OpenTelemetry Java Agent +mvn dependency:copy -B \ + $PROXY \ + -Dartifact="io.opentelemetry.javaagent:opentelemetry-javaagent:${OPENTELEMETRY_AGENT_VERSION}:jar" \ + -DoutputDirectory=/tmp/ + case ${DISTRO} in wildfly*) cat <<-EOF > batch.cli @@ -120,12 +126,7 @@ EOF ;; esac -# download Prometheus JMX Exporter. -# Details on https://blog.camunda.com/post/2019/06/camunda-bpm-on-kubernetes/ -mvn dependency:copy -B \ - $PROXY \ - -Dartifact="io.prometheus.jmx:jmx_prometheus_javaagent:${JMX_PROMETHEUS_VERSION}:jar" \ - -DoutputDirectory=/tmp/ - +# Install OpenTelemetry Java Agent for all distributions mkdir -p /camunda/javaagent -cp /tmp/jmx_prometheus_javaagent-${JMX_PROMETHEUS_VERSION}.jar /camunda/javaagent/jmx_prometheus_javaagent.jar +cp /tmp/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar /camunda/javaagent/ + diff --git a/opentelemetry/compose.yml b/opentelemetry/compose.yml deleted file mode 100644 index 9909c5c..0000000 --- a/opentelemetry/compose.yml +++ /dev/null @@ -1,31 +0,0 @@ -services: - cibseven: - image: cibseven/cibseven:tomcat-amd64 - container_name: cibseven - environment: - OTEL_METRICS_EXPORTER: "otlp" - OTEL_LOGS_EXPORTER: "otlp" - OTEL_TRACES_EXPORTER: "otlp" - OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4318" - ports: - - 8080:8080 - - 9464:9464 - depends_on: - otel-collector: - condition: service_started - networks: - - otelnet - - otel-collector: - image: otel/opentelemetry-collector:latest - container_name: otel-collector - ports: - - "4318:4318" - command: ["--config=/etc/otel-collector-config.yaml"] - volumes: - - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml - networks: - - otelnet - -networks: - otelnet: diff --git a/opentelemetry/jmx_config.yaml b/opentelemetry/jmx_config.yaml index 87ef6f8..ecaf443 100644 --- a/opentelemetry/jmx_config.yaml +++ b/opentelemetry/jmx_config.yaml @@ -1,51 +1,80 @@ rules: + # Config from: https://github.com/open-telemetry/opentelemetry-java-instrumentation/blob/main/instrumentation/jmx-metrics/library/src/main/resources/jmx/rules/jvm.yaml + # Extended for CIB: MaxFileDescriptorCount, TotalSwapSpaceSize, FreeSwapSpaceSize, TotalPhysicalMemorySize, FreePhysicalMemorySize, CommittedVirtualMemorySize - bean: java.lang:type=OperatingSystem + prefix: os. + dropNegativeValues: true mapping: + # os.cpu.count AvailableProcessors: - metric: java_lang_operatingsystem_availableprocessors + metric: cpu.count type: updowncounter - desc: java_lang_operatingsystem_availableprocessors + unit: "{cpu}" + desc: Number of processors available to the Java virtual machine. + # os.cpu.time + ProcessCpuTime: + metric: cpu.time + type: counter + sourceUnit: ns + unit: s + desc: CPU time used by the process as reported by the JVM. + # os.cpu.recent_utilization ProcessCpuLoad: - metric: java_lang_operatingsystem_processcpuload - type: updowncounter - desc: java_lang_operatingsystem_processcpuload. The recent cpu usage for the Java Virtual Machine process. + metric: cpu.recent_utilization + type: gauge + unit: '1' + desc: Recent CPU utilization for the process as reported by the JVM. + # os.system.cpu.load_1m (experimental) + SystemLoadAverage: + metric: system.cpu.load_1m + type: gauge + unit: "{run_queue_item}" + desc: Average CPU load of the whole system for the last minute as reported by the JVM. + # os.system.cpu.utilization (experimental) SystemCpuLoad: - metric: java_lang_operatingsystem_systemcpuload - type: updowncounter - desc: java_lang_operatingsystem_systemcpuload. The recent cpu usage for the whole system. - ProcessCpuTime: - metric: java_lang_operatingsystem_processcputime + metric: system.cpu.utilization + type: gauge + unit: '1' + desc: Recent CPU utilization for the whole system as reported by the JVM. + # os.file_descriptor.open.count (experimental) + OpenFileDescriptorCount: + metric: file_descriptor.open.count type: updowncounter - desc: java_lang_operatingsystem_processcputime - SystemLoadAverage: - metric: java_lang_operatingsystem_systemloadaverage + unit: "{file_descriptor}" + desc: Number of open file descriptors as reported by the JVM. + # os.file_descriptor.max.count + MaxFileDescriptorCount: + metric: file_descriptor.max.count type: updowncounter - desc: java_lang_operatingsystem_systemloadaverage + unit: "{file_descriptor}" + desc: Number of max file descriptors as reported by the JVM. + # virtual_memory.committed.size CommittedVirtualMemorySize: - metric: java_lang_operatingsystem_committedvirtualmemorysize - type: updowncounter - desc: java_lang_operatingsystem_committedvirtualmemorysize + metric: virtual_memory.committed.size + type: gauge + unit: 'By' + desc: Size of CommittedVirtualMemorySize. + # os.physical_memory.free.size FreePhysicalMemorySize: - metric: java_lang_operatingsystem_freephysicalmemorysize - type: updowncounter - desc: java_lang_operatingsystem_freephysicalmemorysize + metric: physical_memory.free.size + type: gauge + unit: 'By' + desc: Size of FreePhysicalMemorySize. + # os.physical_memory.total.size TotalPhysicalMemorySize: - metric: java_lang_operatingsystem_totalphysicalmemorysize - type: updowncounter - desc: java_lang_operatingsystem_totalphysicalmemorysize + metric: physical_memory.total.size + type: gauge + unit: 'By' + desc: Size of TotalPhysicalMemorySize. + # os.swap_space.free.size FreeSwapSpaceSize: - metric: java_lang_operatingsystem_freeswapspacesize - type: updowncounter - desc: java_lang_operatingsystem_freeswapspacesize + metric: swap_space.free.size + type: gauge + unit: 'By' + desc: Size of FreeSwapSpaceSize. + # os.swap_space.total.size TotalSwapSpaceSize: - metric: java_lang_operatingsystem_totalswapspacesize - type: updowncounter - desc: java_lang_operatingsystem_totalswapspacesize - OpenFileDescriptorCount: - metric: java_lang_operatingsystem_openfiledescriptorcount - type: updowncounter - desc: java_lang_operatingsystem_openfiledescriptorcount - MaxFileDescriptorCount: - metric: java_lang_operatingsystem_maxfiledescriptorcount - type: updowncounter - desc: java_lang_operatingsystem_maxfiledescriptorcount \ No newline at end of file + metric: swap_space.total.size + type: gauge + unit: 'By' + desc: Size of TotalSwapSpaceSize. \ No newline at end of file diff --git a/test/docker-compose.yml b/test/docker-compose.yml index b09e0e1..db5c3d3 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -46,18 +46,6 @@ services: - "8000:8000" restart: unless-stopped - camunda-prometheus-jmx: - image: cibseven/cibseven:${DISTRO}-${PLATFORM} - platform: linux/${PLATFORM} - environment: - - JMX_PROMETHEUS=true - ports: - - "8080:8080" - - "9404:9404" - volumes: - - $PWD/prometheus-jmx.yml:/camunda/javaagent/prometheus-jmx.yml - restart: unless-stopped - camunda-password-file: image: cibseven/cibseven:${DISTRO}-${PLATFORM} platform: linux/${PLATFORM} @@ -88,3 +76,29 @@ services: environment: - POSTGRES_USER=camunda - POSTGRES_PASSWORD=camunda + + camunda-opentelemetry: + image: cibseven/cibseven:${DISTRO}-${PLATFORM} + platform: linux/${PLATFORM} + environment: + - OTEL_METRICS_EXPORTER=prometheus + - OTEL_LOGS_EXPORTER=none + - OTEL_TRACES_EXPORTER=otlp + - OTEL_EXPORTER_PROMETHEUS_PORT=9464 + - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + ports: + - "8080:8080" + - "9464:9464" + depends_on: + - otel-collector + restart: unless-stopped + + opentelemetry-collector: + image: otel/opentelemetry-collector:latest + container_name: otel-collector + command: ["--config=/etc/otel-collector-config.yaml"] + volumes: + - ./otel-collector-config.yml:/etc/otelcol/config.yaml + ports: + - "4318:4318" + restart: unless-stopped \ No newline at end of file diff --git a/opentelemetry/otel-collector-config.yaml b/test/otel-collector-config.yml similarity index 94% rename from opentelemetry/otel-collector-config.yaml rename to test/otel-collector-config.yml index 7615966..269e564 100644 --- a/opentelemetry/otel-collector-config.yaml +++ b/test/otel-collector-config.yml @@ -1,21 +1,21 @@ -receivers: - otlp: - protocols: - http: - endpoint: "0.0.0.0:4318" - -exporters: - debug: - verbosity: detailed - -service: - pipelines: - traces: - receivers: [otlp] - exporters: [debug] - metrics: - receivers: [otlp] - exporters: [debug] - logs: - receivers: [otlp] - exporters: [debug] +receivers: + otlp: + protocols: + http: + endpoint: "0.0.0.0:4318" + +exporters: + debug: + verbosity: detailed + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug] + metrics: + receivers: [otlp] + exporters: [debug] + logs: + receivers: [otlp] + exporters: [debug] diff --git a/test/prometheus-jmx.yml b/test/prometheus-jmx.yml deleted file mode 100644 index 54ee51a..0000000 --- a/test/prometheus-jmx.yml +++ /dev/null @@ -1,5 +0,0 @@ -startDelaySeconds: 0 -ssl: false - -rules: - - pattern: "java.lang:type=ClassLoading" diff --git a/test/test-prometheus-jmx-run.sh b/test/test-prometheus-jmx-run.sh deleted file mode 100755 index 4af92d1..0000000 --- a/test/test-prometheus-jmx-run.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -eu - -SERVICE=${1} - -source test_helper.sh - -_log "skipping test of JMX Prometheus exporter: not supported for cibseven-run" diff --git a/test/test-prometheus-jmx-tomcat.sh b/test/test-prometheus-jmx-tomcat.sh deleted file mode 100755 index 88439f4..0000000 --- a/test/test-prometheus-jmx-tomcat.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -eu - -SERVICE=${1} - -source test_helper.sh - -start_container - -poll_log "org.apache.catalina.startup.Catalina.start Server startup in" "^SEVERE" || _exit 1 "Server not started" - -_log "Server started" - -grep_log "Enabling Prometheus JMX Exporter on port" || _exit 2 "Prometheus JMX Exporter not enabled" - -curl -s http://localhost:9404/metrics | grep -q "jvm_classes_currently_loaded" || _exit 3 "Prometheus metrics not available" - -_exit 0 "Test successfull" diff --git a/test/test-prometheus-jmx-wildfly.sh b/test/test-prometheus-jmx-wildfly.sh deleted file mode 100755 index ec0ab21..0000000 --- a/test/test-prometheus-jmx-wildfly.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -eu - -SERVICE=${1} - -source test_helper.sh - -start_container - -poll_log 'started in' 'started (with errors) in' || _exit 1 "Server not started" - -_log "Server started" - -grep_log "Enabling Prometheus JMX Exporter on port" || _exit 2 "Prometheus JMX Exporter not enabled" - -curl -s http://localhost:9404/metrics | grep -q "jvm_classes_currently_loaded" || _exit 3 "Prometheus metrics not available" - -_exit 0 "Test successfull" diff --git a/test/test-run.sh b/test/test-run.sh index 9cdfae9..7ecfa7f 100755 --- a/test/test-run.sh +++ b/test/test-run.sh @@ -24,4 +24,9 @@ _log "Login successfull" # Disabled encoding test ... # test_encoding || _exit 7 "Wrong encoding detected" +# Test OpenTelemetry metrics endpoint +_log "Testing OpenTelemetry metrics endpoint" +curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 8 "OpenTelemetry metrics not available" +_log "OpenTelemetry metrics available" + _exit 0 "Test successfull" diff --git a/test/test-tomcat.sh b/test/test-tomcat.sh index 62df972..22d9e85 100755 --- a/test/test-tomcat.sh +++ b/test/test-tomcat.sh @@ -23,4 +23,9 @@ _log "Login successfull" test_encoding || _exit 7 "Wrong encoding detected" +# Test OpenTelemetry metrics endpoint +_log "Testing OpenTelemetry metrics endpoint" +curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 8 "OpenTelemetry metrics not available" +_log "OpenTelemetry metrics available" + _exit 0 "Test successfull" diff --git a/test/test-wildfly.sh b/test/test-wildfly.sh index f982f34..6a6b019 100755 --- a/test/test-wildfly.sh +++ b/test/test-wildfly.sh @@ -23,4 +23,9 @@ _log "Login successfull" test_encoding || _exit 7 "Wrong encoding detected" +# Test OpenTelemetry metrics endpoint +_log "Testing OpenTelemetry metrics endpoint" +curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 8 "OpenTelemetry metrics not available" +_log "OpenTelemetry metrics available" + _exit 0 "Test successfull" diff --git a/test/test.sh b/test/test.sh index bb8a4b9..c7e3dce 100755 --- a/test/test.sh +++ b/test/test.sh @@ -11,7 +11,6 @@ docker-compose up --force-recreate -d postgres mysql ./test-${DISTRO}.sh camunda-mysql ./test-${DISTRO}.sh camunda-postgres ./test-${DISTRO}.sh camunda-password-file -./test-prometheus-jmx-${DISTRO}.sh camunda-prometheus-jmx ./test-debug.sh camunda-debug docker-compose down -v cd - From 39cacee99ac380ac73a2092d16c8ed5958ecdd13 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 14 Jan 2026 13:29:08 +0100 Subject: [PATCH 03/44] fix(docker-compose): rename otel-collector to opentelemetry-collector and update endpoint --- test/docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/docker-compose.yml b/test/docker-compose.yml index db5c3d3..2b1aa69 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -85,17 +85,17 @@ services: - OTEL_LOGS_EXPORTER=none - OTEL_TRACES_EXPORTER=otlp - OTEL_EXPORTER_PROMETHEUS_PORT=9464 - - OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318 + - OTEL_EXPORTER_OTLP_ENDPOINT=http://opentelemetry-collector:4318 ports: - "8080:8080" - "9464:9464" depends_on: - - otel-collector + - opentelemetry-collector restart: unless-stopped opentelemetry-collector: image: otel/opentelemetry-collector:latest - container_name: otel-collector + container_name: opentelemetry-collector command: ["--config=/etc/otel-collector-config.yaml"] volumes: - ./otel-collector-config.yml:/etc/otelcol/config.yaml From 2ce500135a67d093813a64d012cc62a45065600a Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 14 Jan 2026 13:44:07 +0100 Subject: [PATCH 04/44] feat(tests): add OpenTelemetry test scripts for various services --- test/test-opentelemetry-run.sh | 32 ++++++++++++++++++++++++++++++ test/test-opentelemetry-tomcat.sh | 31 +++++++++++++++++++++++++++++ test/test-opentelemetry-wildfly.sh | 31 +++++++++++++++++++++++++++++ test/test-run.sh | 4 ---- test/test-tomcat.sh | 4 ---- test/test-wildfly.sh | 4 ---- test/test.sh | 3 ++- 7 files changed, 96 insertions(+), 13 deletions(-) create mode 100644 test/test-opentelemetry-run.sh create mode 100644 test/test-opentelemetry-tomcat.sh create mode 100644 test/test-opentelemetry-wildfly.sh diff --git a/test/test-opentelemetry-run.sh b/test/test-opentelemetry-run.sh new file mode 100644 index 0000000..7ecfa7f --- /dev/null +++ b/test/test-opentelemetry-run.sh @@ -0,0 +1,32 @@ +#!/bin/bash -eu + +SERVICE=${1} + +source test_helper.sh + +start_container + +WAIT=10 poll_log "starting to acquire jobs" "Application run failed" || _exit 1 "Server not started" + +_log "Server started" + +create_user || echo "Unable to create demo user (maybe it already exists)" + +_log "User created" + +test_login admin || _exit 3 "Unable to login to admin" +test_login cockpit || _exit 4 "Unable to login to cockpit" +test_login tasklist || _exit 5 "Unable to login to tasklist" +test_login_webapp || _exit 6 "Unable to login to webapp" + +_log "Login successfull" + +# Disabled encoding test ... +# test_encoding || _exit 7 "Wrong encoding detected" + +# Test OpenTelemetry metrics endpoint +_log "Testing OpenTelemetry metrics endpoint" +curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 8 "OpenTelemetry metrics not available" +_log "OpenTelemetry metrics available" + +_exit 0 "Test successfull" diff --git a/test/test-opentelemetry-tomcat.sh b/test/test-opentelemetry-tomcat.sh new file mode 100644 index 0000000..22d9e85 --- /dev/null +++ b/test/test-opentelemetry-tomcat.sh @@ -0,0 +1,31 @@ +#!/bin/bash -eu + +SERVICE=${1} + +source test_helper.sh + +start_container + +poll_log "org.apache.catalina.startup.Catalina.start Server startup in" "^SEVERE" || _exit 1 "Server not started" + +_log "Server started" + +grep_log "Deployment.*camunda-invoice.*finished" || _exit 2 "Process application not deployed" + +_log "Process application deployed" + +test_login admin || _exit 3 "Unable to login to admin" +test_login cockpit || _exit 4 "Unable to login to cockpit" +test_login tasklist || _exit 5 "Unable to login to tasklist" +test_login_webapp || _exit 6 "Unable to login to webapp" + +_log "Login successfull" + +test_encoding || _exit 7 "Wrong encoding detected" + +# Test OpenTelemetry metrics endpoint +_log "Testing OpenTelemetry metrics endpoint" +curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 8 "OpenTelemetry metrics not available" +_log "OpenTelemetry metrics available" + +_exit 0 "Test successfull" diff --git a/test/test-opentelemetry-wildfly.sh b/test/test-opentelemetry-wildfly.sh new file mode 100644 index 0000000..ad982dc --- /dev/null +++ b/test/test-opentelemetry-wildfly.sh @@ -0,0 +1,31 @@ +#!/bin/bash -eu + +SERVICE=${1} + +source test_helper.sh + +start_container + +poll_log "WFLYSRV0025" "^SEVERE" || _exit 1 "Server not started" + +_log "Server started" + +grep_log "Deployment.*camunda-invoice.*finished" || _exit 2 "Process application not deployed" + +_log "Process application deployed" + +test_login admin || _exit 3 "Unable to login to admin" +test_login cockpit || _exit 4 "Unable to login to cockpit" +test_login tasklist || _exit 5 "Unable to login to tasklist" +test_login_webapp || _exit 6 "Unable to login to webapp" + +_log "Login successfull" + +test_encoding || _exit 7 "Wrong encoding detected" + +# Test OpenTelemetry metrics endpoint +_log "Testing OpenTelemetry metrics endpoint" +curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 8 "OpenTelemetry metrics not available" +_log "OpenTelemetry metrics available" + +_exit 0 "Test successfull" diff --git a/test/test-run.sh b/test/test-run.sh index 7ecfa7f..9f3e026 100755 --- a/test/test-run.sh +++ b/test/test-run.sh @@ -24,9 +24,5 @@ _log "Login successfull" # Disabled encoding test ... # test_encoding || _exit 7 "Wrong encoding detected" -# Test OpenTelemetry metrics endpoint -_log "Testing OpenTelemetry metrics endpoint" -curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 8 "OpenTelemetry metrics not available" -_log "OpenTelemetry metrics available" _exit 0 "Test successfull" diff --git a/test/test-tomcat.sh b/test/test-tomcat.sh index 22d9e85..560c38d 100755 --- a/test/test-tomcat.sh +++ b/test/test-tomcat.sh @@ -23,9 +23,5 @@ _log "Login successfull" test_encoding || _exit 7 "Wrong encoding detected" -# Test OpenTelemetry metrics endpoint -_log "Testing OpenTelemetry metrics endpoint" -curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 8 "OpenTelemetry metrics not available" -_log "OpenTelemetry metrics available" _exit 0 "Test successfull" diff --git a/test/test-wildfly.sh b/test/test-wildfly.sh index 6a6b019..02f28fd 100755 --- a/test/test-wildfly.sh +++ b/test/test-wildfly.sh @@ -23,9 +23,5 @@ _log "Login successfull" test_encoding || _exit 7 "Wrong encoding detected" -# Test OpenTelemetry metrics endpoint -_log "Testing OpenTelemetry metrics endpoint" -curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 8 "OpenTelemetry metrics not available" -_log "OpenTelemetry metrics available" _exit 0 "Test successfull" diff --git a/test/test.sh b/test/test.sh index c7e3dce..2c5ca82 100755 --- a/test/test.sh +++ b/test/test.sh @@ -6,11 +6,12 @@ cd ${DIR} source test_helper.sh -docker-compose up --force-recreate -d postgres mysql +docker-compose up --force-recreate -d postgres mysql opentelemetry-collector ./test-${DISTRO}.sh camunda ./test-${DISTRO}.sh camunda-mysql ./test-${DISTRO}.sh camunda-postgres ./test-${DISTRO}.sh camunda-password-file +./test-opentelemetry-${DISTRO}.sh camunda-opentelemetry ./test-debug.sh camunda-debug docker-compose down -v cd - From 4ba32ab79fea22838bc402f57450c50773dfa25c Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 14 Jan 2026 14:21:51 +0100 Subject: [PATCH 05/44] chore(tests): update permissions for OpenTelemetry test scripts --- test/test-opentelemetry-run.sh | 0 test/test-opentelemetry-tomcat.sh | 0 test/test-opentelemetry-wildfly.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 test/test-opentelemetry-run.sh mode change 100644 => 100755 test/test-opentelemetry-tomcat.sh mode change 100644 => 100755 test/test-opentelemetry-wildfly.sh diff --git a/test/test-opentelemetry-run.sh b/test/test-opentelemetry-run.sh old mode 100644 new mode 100755 diff --git a/test/test-opentelemetry-tomcat.sh b/test/test-opentelemetry-tomcat.sh old mode 100644 new mode 100755 diff --git a/test/test-opentelemetry-wildfly.sh b/test/test-opentelemetry-wildfly.sh old mode 100644 new mode 100755 From 5f99907254be02a480729f0c61e329ab24cbad64 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 14 Jan 2026 14:48:32 +0100 Subject: [PATCH 06/44] refactor(tests): streamline OpenTelemetry test scripts by removing redundant login and user creation steps --- test/test-opentelemetry-run.sh | 14 -------------- test/test-opentelemetry-tomcat.sh | 17 ++--------------- test/test-opentelemetry-wildfly.sh | 19 +++---------------- 3 files changed, 5 insertions(+), 45 deletions(-) diff --git a/test/test-opentelemetry-run.sh b/test/test-opentelemetry-run.sh index 7ecfa7f..0a60f3d 100755 --- a/test/test-opentelemetry-run.sh +++ b/test/test-opentelemetry-run.sh @@ -10,20 +10,6 @@ WAIT=10 poll_log "starting to acquire jobs" "Application run failed" || _exit 1 _log "Server started" -create_user || echo "Unable to create demo user (maybe it already exists)" - -_log "User created" - -test_login admin || _exit 3 "Unable to login to admin" -test_login cockpit || _exit 4 "Unable to login to cockpit" -test_login tasklist || _exit 5 "Unable to login to tasklist" -test_login_webapp || _exit 6 "Unable to login to webapp" - -_log "Login successfull" - -# Disabled encoding test ... -# test_encoding || _exit 7 "Wrong encoding detected" - # Test OpenTelemetry metrics endpoint _log "Testing OpenTelemetry metrics endpoint" curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 8 "OpenTelemetry metrics not available" diff --git a/test/test-opentelemetry-tomcat.sh b/test/test-opentelemetry-tomcat.sh index 22d9e85..84be1bb 100755 --- a/test/test-opentelemetry-tomcat.sh +++ b/test/test-opentelemetry-tomcat.sh @@ -10,22 +10,9 @@ poll_log "org.apache.catalina.startup.Catalina.start Server startup in" "^SEVERE _log "Server started" -grep_log "Deployment.*camunda-invoice.*finished" || _exit 2 "Process application not deployed" - -_log "Process application deployed" - -test_login admin || _exit 3 "Unable to login to admin" -test_login cockpit || _exit 4 "Unable to login to cockpit" -test_login tasklist || _exit 5 "Unable to login to tasklist" -test_login_webapp || _exit 6 "Unable to login to webapp" - -_log "Login successfull" - -test_encoding || _exit 7 "Wrong encoding detected" - # Test OpenTelemetry metrics endpoint _log "Testing OpenTelemetry metrics endpoint" -curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 8 "OpenTelemetry metrics not available" +curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 3 "OpenTelemetry metrics not available" _log "OpenTelemetry metrics available" -_exit 0 "Test successfull" +_exit 0 "Test successfull" \ No newline at end of file diff --git a/test/test-opentelemetry-wildfly.sh b/test/test-opentelemetry-wildfly.sh index ad982dc..0c862dc 100755 --- a/test/test-opentelemetry-wildfly.sh +++ b/test/test-opentelemetry-wildfly.sh @@ -6,26 +6,13 @@ source test_helper.sh start_container -poll_log "WFLYSRV0025" "^SEVERE" || _exit 1 "Server not started" +poll_log 'started in' 'started (with errors) in' || _exit 1 "Server not started" _log "Server started" -grep_log "Deployment.*camunda-invoice.*finished" || _exit 2 "Process application not deployed" - -_log "Process application deployed" - -test_login admin || _exit 3 "Unable to login to admin" -test_login cockpit || _exit 4 "Unable to login to cockpit" -test_login tasklist || _exit 5 "Unable to login to tasklist" -test_login_webapp || _exit 6 "Unable to login to webapp" - -_log "Login successfull" - -test_encoding || _exit 7 "Wrong encoding detected" - # Test OpenTelemetry metrics endpoint _log "Testing OpenTelemetry metrics endpoint" -curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 8 "OpenTelemetry metrics not available" +curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 3 "OpenTelemetry metrics not available" _log "OpenTelemetry metrics available" -_exit 0 "Test successfull" +_exit 0 "Test successfull" \ No newline at end of file From 272507c324a9d7a3a384bb198e03fc21bb9e0bfe Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 14 Jan 2026 18:06:07 +0100 Subject: [PATCH 07/44] chore(Dockerfile): add OPENTELEMETRY_AGENT_VERSION argument and environment variable --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index c6eecf0..5ba92d1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,6 +46,8 @@ FROM alpine:3.23 # Re-declare to use in this stage (inherits the value from global) ARG VERSION ARG JAVA +ARG OPENTELEMETRY_AGENT_VERSION +ENV OPENTELEMETRY_AGENT_VERSION=${OPENTELEMETRY_AGENT_VERSION} ENV DB_DRIVER= ENV DB_URL= From 02d1ef2a43fd588f437db149528efbfbfb3b8942 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 14 Jan 2026 18:49:29 +0100 Subject: [PATCH 08/44] fix(cibseven-wildfly.sh): remove duplicate PREPEND_JAVA_OPTS export statement --- cibseven-wildfly.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/cibseven-wildfly.sh b/cibseven-wildfly.sh index 8415581..bd74e6a 100755 --- a/cibseven-wildfly.sh +++ b/cibseven-wildfly.sh @@ -65,7 +65,6 @@ export PREPEND_JAVA_OPTS="-Djboss.bind.address=0.0.0.0 -Djboss.bind.address.mana LOG_MANAGER_PATH=$(find /camunda/modules -name "jboss-logmanager*.jar") COMMON_PATH=$(find /camunda/modules -name "wildfly-common*.jar") export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -Dsun.util.logging.disableCallerCheck=true -Djava.util.logging.manager=org.jboss.logmanager.LogManager -Xbootclasspath/a:$LOG_MANAGER_PATH:$COMMON_PATH -Dotel.javaagent.logging=application" -export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -Dsun.util.logging.disableCallerCheck=true -Djava.util.logging.manager=org.jboss.logmanager.LogManager -Xbootclasspath/a:$LOG_MANAGER_PATH:$COMMON_PATH -Dotel.javaagent.logging=application" export LAUNCH_JBOSS_IN_BACKGROUND=TRUE CMD="/camunda/bin/standalone.sh" From 94395ea91e47ebd59e41c3e45c485d281c149895 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 14 Jan 2026 18:55:33 +0100 Subject: [PATCH 09/44] fix(docker-compose): update image tag syntax for camunda-opentelemetry service --- test/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/docker-compose.yml b/test/docker-compose.yml index fbbcbbf..431b2bf 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -78,7 +78,7 @@ services: - POSTGRES_PASSWORD=camunda camunda-opentelemetry: - image: cibseven/cibseven:${DISTRO}-${PLATFORM} + image: cibseven/cibseven:${IMAGE_TAG:-${DISTRO}-${PLATFORM}} platform: linux/${PLATFORM} environment: - OTEL_METRICS_EXPORTER=prometheus From 5a16982cb25e8a0154575714e4cbe18346244ff2 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 15 Jan 2026 06:55:08 +0100 Subject: [PATCH 10/44] chore(Dockerfile): disable default OpenTelemetry instrumentation for test --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 5ba92d1..c059470 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,6 +71,7 @@ ENV JAVA_TOOL_OPTIONS="-javaagent:/camunda/javaagent/opentelemetry-javaagent-${O # OpenTelemetry default exporter settings (all exporters disabled, user must configure) ENV OTEL_SERVICE_NAME=cibseven \ + OTEL_INSTRUMENTATION_COMMON_DEFAULT_ENABLED=false \ OTEL_JMX_CONFIG=/camunda/javaagent/jmx_config.yaml,/camunda/javaagent/jmx_custom_config.yaml \ OTEL_METRICS_EXPORTER=none \ OTEL_LOGS_EXPORTER=none \ From b2c4187ffeb9fec3e0efa6395424f6eaa06b663b Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 15 Jan 2026 07:12:52 +0100 Subject: [PATCH 11/44] chore(Dockerfile): comment out default OpenTelemetry agent configuration --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c059470..3f17895 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,12 +66,11 @@ ENV DEBUG=false ENV JAVA_OPTS="" # --- Use OpenTelemetry agent by default --- -ENV JAVA_TOOL_OPTIONS="-javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar" +#ENV JAVA_TOOL_OPTIONS="-javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar" # OpenTelemetry default exporter settings (all exporters disabled, user must configure) ENV OTEL_SERVICE_NAME=cibseven \ - OTEL_INSTRUMENTATION_COMMON_DEFAULT_ENABLED=false \ OTEL_JMX_CONFIG=/camunda/javaagent/jmx_config.yaml,/camunda/javaagent/jmx_custom_config.yaml \ OTEL_METRICS_EXPORTER=none \ OTEL_LOGS_EXPORTER=none \ From cedaa3359767ff12f9b341dc5fb0f1b66c8bc0c3 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 15 Jan 2026 07:44:49 +0100 Subject: [PATCH 12/44] chore(Dockerfile): enable OpenTelemetry agent without jmx-instrumentation --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3f17895..7367c35 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,12 +66,11 @@ ENV DEBUG=false ENV JAVA_OPTS="" # --- Use OpenTelemetry agent by default --- -#ENV JAVA_TOOL_OPTIONS="-javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar" +ENV JAVA_TOOL_OPTIONS="-javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar" # OpenTelemetry default exporter settings (all exporters disabled, user must configure) ENV OTEL_SERVICE_NAME=cibseven \ - OTEL_JMX_CONFIG=/camunda/javaagent/jmx_config.yaml,/camunda/javaagent/jmx_custom_config.yaml \ OTEL_METRICS_EXPORTER=none \ OTEL_LOGS_EXPORTER=none \ OTEL_TRACES_EXPORTER=none \ From 3c400fc551eb57a34d6462d621e7cc27edcb259c Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 15 Jan 2026 08:24:58 +0100 Subject: [PATCH 13/44] chore(Dockerfile): add JMX configuration and enable debug for OpenTelemetry agent --- Dockerfile | 2 ++ docker-compose.yml | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile index 7367c35..05198f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,6 +71,8 @@ ENV JAVA_TOOL_OPTIONS="-javaagent:/camunda/javaagent/opentelemetry-javaagent-${O # OpenTelemetry default exporter settings (all exporters disabled, user must configure) ENV OTEL_SERVICE_NAME=cibseven \ + OTEL_JMX_CONFIG=/camunda/javaagent/jmx_config.yaml,/camunda/javaagent/jmx_custom_config.yaml \ + OTEL_JAVAAGENT_DEBUG=true \ OTEL_METRICS_EXPORTER=none \ OTEL_LOGS_EXPORTER=none \ OTEL_TRACES_EXPORTER=none \ diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..ce84115 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,16 @@ +services: + + + camunda-opentelemetry: + image: cibseven-otel-wildfly:latest + + environment: + - OTEL_METRICS_EXPORTER=prometheus + - OTEL_LOGS_EXPORTER=none + - OTEL_TRACES_EXPORTER=otlp + - OTEL_EXPORTER_PROMETHEUS_PORT=9464 + - OTEL_EXPORTER_OTLP_ENDPOINT=http://opentelemetry-collector:4318 + ports: + - "8080:8080" + - "9464:9464" + restart: unless-stopped From 15cd57ad0c8c3e0c36e31a30392247af020cc778 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 15 Jan 2026 08:51:53 +0100 Subject: [PATCH 14/44] fix(cibseven-wildfly.sh): disable OpenTelemetry agent for CLI tool to resolve ARM64 issues --- cibseven-wildfly.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cibseven-wildfly.sh b/cibseven-wildfly.sh index bd74e6a..1c05e84 100755 --- a/cibseven-wildfly.sh +++ b/cibseven-wildfly.sh @@ -31,7 +31,8 @@ embed-server run-batch stop-embedded-server EOF - /camunda/bin/jboss-cli.sh --file=batch.cli + # Disable OpenTelemetry agent for CLI tool (not needed and causes issues on ARM64) + JAVA_TOOL_OPTIONS="" /camunda/bin/jboss-cli.sh --file=batch.cli rm -rf /camunda/standalone/configuration/standalone_xml_history/current/* batch.cli } From d5a8431abb885e98b095ac50d7903f709ed086aa Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 15 Jan 2026 09:16:47 +0100 Subject: [PATCH 15/44] chore(Dockerfile): disable OpenTelemetry agent debug mode for cleaner configuration --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 05198f1..5ba92d1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,7 +72,6 @@ ENV JAVA_TOOL_OPTIONS="-javaagent:/camunda/javaagent/opentelemetry-javaagent-${O # OpenTelemetry default exporter settings (all exporters disabled, user must configure) ENV OTEL_SERVICE_NAME=cibseven \ OTEL_JMX_CONFIG=/camunda/javaagent/jmx_config.yaml,/camunda/javaagent/jmx_custom_config.yaml \ - OTEL_JAVAAGENT_DEBUG=true \ OTEL_METRICS_EXPORTER=none \ OTEL_LOGS_EXPORTER=none \ OTEL_TRACES_EXPORTER=none \ From 573c32bfdbeaa7cd8fb5f2934ad3a638382c852b Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 15 Jan 2026 09:44:33 +0100 Subject: [PATCH 16/44] fix(test_helper.sh): increase wait time from 5 to 10 seconds for improved stability --- test/test_helper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_helper.sh b/test/test_helper.sh index 2c6d458..b510acf 100644 --- a/test/test_helper.sh +++ b/test/test_helper.sh @@ -1,6 +1,6 @@ #!/bin/bash -eu RETRIES=100 -WAIT=5 +WAIT=10 GHA=${GITHUB_ACTIONS:-false} if [ "${GHA}" = "true" ]; then From 11a8fd35b97fd324e62e68e24fd860c3ffb3367a Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 15 Jan 2026 12:06:45 +0100 Subject: [PATCH 17/44] chore(scripts): update OpenTelemetry agent configuration for Java options across scripts --- Dockerfile | 6 ++---- cibseven-run.sh | 4 ++++ cibseven-tomcat.sh | 4 ++++ cibseven-wildfly.sh | 9 +++++---- 4 files changed, 15 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5ba92d1..68d0603 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,11 +65,9 @@ ENV TZ=UTC ENV DEBUG=false ENV JAVA_OPTS="" -# --- Use OpenTelemetry agent by default --- -ENV JAVA_TOOL_OPTIONS="-javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar" - - # OpenTelemetry default exporter settings (all exporters disabled, user must configure) +# Note: The OTEL agent is loaded via server-specific variables (PREPEND_JAVA_OPTS for WildFly, +# CATALINA_OPTS for Tomcat) to avoid affecting CLI tools like jboss-cli.sh ENV OTEL_SERVICE_NAME=cibseven \ OTEL_JMX_CONFIG=/camunda/javaagent/jmx_config.yaml,/camunda/javaagent/jmx_custom_config.yaml \ OTEL_METRICS_EXPORTER=none \ diff --git a/cibseven-run.sh b/cibseven-run.sh index 3330032..0462207 100755 --- a/cibseven-run.sh +++ b/cibseven-run.sh @@ -30,6 +30,10 @@ if [[ -z "${SPRING_DATASOURCE_URL:-}" && -n "${DB_URL:-}" ]]; then export SPRING_DATASOURCE_URL="${DB_URL}" fi +# OpenTelemetry Agent configuration +# Load the agent via JAVA_OPTS instead of JAVA_TOOL_OPTIONS +export JAVA_OPTS="${JAVA_OPTS:-} -javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar" + CMD="/camunda/internal/run.sh start" wait_for_it diff --git a/cibseven-tomcat.sh b/cibseven-tomcat.sh index bf6ee0f..66b5247 100755 --- a/cibseven-tomcat.sh +++ b/cibseven-tomcat.sh @@ -45,6 +45,10 @@ if [ -z "$SKIP_DB_CONFIG" ]; then "${CATALINA_HOME}/conf/server.xml" fi +# OpenTelemetry Agent configuration +# Load the agent via CATALINA_OPTS (Tomcat-specific) instead of JAVA_TOOL_OPTIONS +export CATALINA_OPTS="${CATALINA_OPTS:-} -javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar" + CMD="${CATALINA_HOME}/bin/catalina.sh" if [ "${DEBUG}" = "true" ]; then echo "Enabling debug mode, JPDA accesible under port 8000" diff --git a/cibseven-wildfly.sh b/cibseven-wildfly.sh index 1c05e84..3009889 100755 --- a/cibseven-wildfly.sh +++ b/cibseven-wildfly.sh @@ -31,8 +31,7 @@ embed-server run-batch stop-embedded-server EOF - # Disable OpenTelemetry agent for CLI tool (not needed and causes issues on ARM64) - JAVA_TOOL_OPTIONS="" /camunda/bin/jboss-cli.sh --file=batch.cli + /camunda/bin/jboss-cli.sh --file=batch.cli rm -rf /camunda/standalone/configuration/standalone_xml_history/current/* batch.cli } @@ -61,11 +60,13 @@ export JBOSS_MODULES_SYSTEM_PKGS=${JBOSS_MODULES_SYSTEM_PKGS:-"org.jboss.byteman # Ensure wildfly binds to public interface, preferes IPv4 and runs in the background export PREPEND_JAVA_OPTS="-Djboss.bind.address=0.0.0.0 -Djboss.bind.address.management=0.0.0.0 -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Djboss.modules.system.pkgs=${JBOSS_MODULES_SYSTEM_PKGS}" -# OpenTelemetry Agent JMX configuration +# OpenTelemetry Agent configuration +# Load the agent via PREPEND_JAVA_OPTS (WildFly-specific) instead of JAVA_TOOL_OPTIONS +# to avoid affecting CLI tools like jboss-cli.sh # See https://github.com/prometheus/jmx_exporter/issues/344 LOG_MANAGER_PATH=$(find /camunda/modules -name "jboss-logmanager*.jar") COMMON_PATH=$(find /camunda/modules -name "wildfly-common*.jar") -export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -Dsun.util.logging.disableCallerCheck=true -Djava.util.logging.manager=org.jboss.logmanager.LogManager -Xbootclasspath/a:$LOG_MANAGER_PATH:$COMMON_PATH -Dotel.javaagent.logging=application" +export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar -Dsun.util.logging.disableCallerCheck=true -Djava.util.logging.manager=org.jboss.logmanager.LogManager -Xbootclasspath/a:$LOG_MANAGER_PATH:$COMMON_PATH -Dotel.javaagent.logging=application" export LAUNCH_JBOSS_IN_BACKGROUND=TRUE CMD="/camunda/bin/standalone.sh" From a3f4048f89117f95cc3fbe9f8b7c84a2cb65a425 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 15 Jan 2026 12:38:38 +0100 Subject: [PATCH 18/44] chore(docker-compose): remove obsolete docker-compose.yml file --- Dockerfile | 1 + docker-compose.yml | 16 ---------------- test/test-run.sh | 1 - test/test-tomcat.sh | 1 - test/test-wildfly.sh | 1 - test/test_helper.sh | 2 +- 6 files changed, 2 insertions(+), 20 deletions(-) delete mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile index 68d0603..3ffdc6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,6 +39,7 @@ COPY settings.xml download.sh cibseven-run.sh cibseven-tomcat.sh cibseven-wildfl RUN /tmp/download.sh COPY wait_for_it-lib.sh /camunda/ + ##### FINAL IMAGE ##### FROM alpine:3.23 diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index ce84115..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,16 +0,0 @@ -services: - - - camunda-opentelemetry: - image: cibseven-otel-wildfly:latest - - environment: - - OTEL_METRICS_EXPORTER=prometheus - - OTEL_LOGS_EXPORTER=none - - OTEL_TRACES_EXPORTER=otlp - - OTEL_EXPORTER_PROMETHEUS_PORT=9464 - - OTEL_EXPORTER_OTLP_ENDPOINT=http://opentelemetry-collector:4318 - ports: - - "8080:8080" - - "9464:9464" - restart: unless-stopped diff --git a/test/test-run.sh b/test/test-run.sh index 9f3e026..9cdfae9 100755 --- a/test/test-run.sh +++ b/test/test-run.sh @@ -24,5 +24,4 @@ _log "Login successfull" # Disabled encoding test ... # test_encoding || _exit 7 "Wrong encoding detected" - _exit 0 "Test successfull" diff --git a/test/test-tomcat.sh b/test/test-tomcat.sh index 560c38d..62df972 100755 --- a/test/test-tomcat.sh +++ b/test/test-tomcat.sh @@ -23,5 +23,4 @@ _log "Login successfull" test_encoding || _exit 7 "Wrong encoding detected" - _exit 0 "Test successfull" diff --git a/test/test-wildfly.sh b/test/test-wildfly.sh index 02f28fd..f982f34 100755 --- a/test/test-wildfly.sh +++ b/test/test-wildfly.sh @@ -23,5 +23,4 @@ _log "Login successfull" test_encoding || _exit 7 "Wrong encoding detected" - _exit 0 "Test successfull" diff --git a/test/test_helper.sh b/test/test_helper.sh index b510acf..2c6d458 100644 --- a/test/test_helper.sh +++ b/test/test_helper.sh @@ -1,6 +1,6 @@ #!/bin/bash -eu RETRIES=100 -WAIT=10 +WAIT=5 GHA=${GITHUB_ACTIONS:-false} if [ "${GHA}" = "true" ]; then From bfbd5358c3f87fe1dfc3eb0e5ddfd87734bc6662 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 15 Jan 2026 12:52:04 +0100 Subject: [PATCH 19/44] refactor(scripts): reorganize OpenTelemetry agent configuration in cibseven-tomcat.sh and cibseven-wildfly.sh for clarity and consistency --- cibseven-tomcat.sh | 7 +++---- cibseven-wildfly.sh | 13 ++++++------- download.sh | 1 - 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/cibseven-tomcat.sh b/cibseven-tomcat.sh index 66b5247..a72b73a 100755 --- a/cibseven-tomcat.sh +++ b/cibseven-tomcat.sh @@ -45,10 +45,6 @@ if [ -z "$SKIP_DB_CONFIG" ]; then "${CATALINA_HOME}/conf/server.xml" fi -# OpenTelemetry Agent configuration -# Load the agent via CATALINA_OPTS (Tomcat-specific) instead of JAVA_TOOL_OPTIONS -export CATALINA_OPTS="${CATALINA_OPTS:-} -javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar" - CMD="${CATALINA_HOME}/bin/catalina.sh" if [ "${DEBUG}" = "true" ]; then echo "Enabling debug mode, JPDA accesible under port 8000" @@ -56,6 +52,9 @@ if [ "${DEBUG}" = "true" ]; then CMD+=" jpda" fi +# OpenTelemetry Agent configuration +# Load the agent via CATALINA_OPTS (Tomcat-specific) instead of JAVA_TOOL_OPTIONS +export CATALINA_OPTS="${CATALINA_OPTS:-} -javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar" CMD+=" run" diff --git a/cibseven-wildfly.sh b/cibseven-wildfly.sh index 3009889..25d72e6 100755 --- a/cibseven-wildfly.sh +++ b/cibseven-wildfly.sh @@ -60,13 +60,6 @@ export JBOSS_MODULES_SYSTEM_PKGS=${JBOSS_MODULES_SYSTEM_PKGS:-"org.jboss.byteman # Ensure wildfly binds to public interface, preferes IPv4 and runs in the background export PREPEND_JAVA_OPTS="-Djboss.bind.address=0.0.0.0 -Djboss.bind.address.management=0.0.0.0 -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Djboss.modules.system.pkgs=${JBOSS_MODULES_SYSTEM_PKGS}" -# OpenTelemetry Agent configuration -# Load the agent via PREPEND_JAVA_OPTS (WildFly-specific) instead of JAVA_TOOL_OPTIONS -# to avoid affecting CLI tools like jboss-cli.sh -# See https://github.com/prometheus/jmx_exporter/issues/344 -LOG_MANAGER_PATH=$(find /camunda/modules -name "jboss-logmanager*.jar") -COMMON_PATH=$(find /camunda/modules -name "wildfly-common*.jar") -export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar -Dsun.util.logging.disableCallerCheck=true -Djava.util.logging.manager=org.jboss.logmanager.LogManager -Xbootclasspath/a:$LOG_MANAGER_PATH:$COMMON_PATH -Dotel.javaagent.logging=application" export LAUNCH_JBOSS_IN_BACKGROUND=TRUE CMD="/camunda/bin/standalone.sh" @@ -76,6 +69,12 @@ if [ "${DEBUG}" = "true" ]; then CMD+=" --debug *:8000" fi +# JBoss LogManager configuration for OpenTelemetry +# See https://github.com/prometheus/jmx_exporter/issues/344 +LOG_MANAGER_PATH=$(find /camunda/modules -name "jboss-logmanager*.jar") +COMMON_PATH=$(find /camunda/modules -name "wildfly-common*.jar") +export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -Dsun.util.logging.disableCallerCheck=true -Djava.util.logging.manager=org.jboss.logmanager.LogManager -Xbootclasspath/a:$LOG_MANAGER_PATH:$COMMON_PATH" +export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar -Dotel.javaagent.logging=application" wait_for_it diff --git a/download.sh b/download.sh index 350f036..c593870 100755 --- a/download.sh +++ b/download.sh @@ -129,4 +129,3 @@ esac # Install OpenTelemetry Java Agent for all distributions mkdir -p /camunda/javaagent cp /tmp/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar /camunda/javaagent/ - From a21bfc7ca062e33af4197b4bc5588a06a0edbf4d Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 15 Jan 2026 12:57:27 +0100 Subject: [PATCH 20/44] fix(cibseven-wildfly.sh, docker-compose): correct OpenTelemetry configuration file path and remove unnecessary newline --- cibseven-wildfly.sh | 1 - test/docker-compose.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cibseven-wildfly.sh b/cibseven-wildfly.sh index 25d72e6..7694727 100755 --- a/cibseven-wildfly.sh +++ b/cibseven-wildfly.sh @@ -59,7 +59,6 @@ export JBOSS_MODULES_SYSTEM_PKGS=${JBOSS_MODULES_SYSTEM_PKGS:-"org.jboss.byteman # Ensure wildfly binds to public interface, preferes IPv4 and runs in the background export PREPEND_JAVA_OPTS="-Djboss.bind.address=0.0.0.0 -Djboss.bind.address.management=0.0.0.0 -Djava.net.preferIPv4Stack=true -Djava.awt.headless=true -Djboss.modules.system.pkgs=${JBOSS_MODULES_SYSTEM_PKGS}" - export LAUNCH_JBOSS_IN_BACKGROUND=TRUE CMD="/camunda/bin/standalone.sh" diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 431b2bf..bf8cae8 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -98,7 +98,7 @@ services: container_name: opentelemetry-collector command: ["--config=/etc/otel-collector-config.yaml"] volumes: - - ./otel-collector-config.yml:/etc/otelcol/config.yaml + - ./otel-collector-config.yml:/etc/otel-collector-config.yaml ports: - "4318:4318" restart: unless-stopped \ No newline at end of file From 6066eb6a63f7c94effbba55418440a66a881a001 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 15 Jan 2026 13:46:44 +0100 Subject: [PATCH 21/44] chore(Dockerfile): set OpenTelemetry log level to error --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 3ffdc6e..58e1e61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,6 +71,7 @@ ENV JAVA_OPTS="" # CATALINA_OPTS for Tomcat) to avoid affecting CLI tools like jboss-cli.sh ENV OTEL_SERVICE_NAME=cibseven \ OTEL_JMX_CONFIG=/camunda/javaagent/jmx_config.yaml,/camunda/javaagent/jmx_custom_config.yaml \ + OTEL_LOG_LEVEL=error \ OTEL_METRICS_EXPORTER=none \ OTEL_LOGS_EXPORTER=none \ OTEL_TRACES_EXPORTER=none \ From 37fba8cc694a918ce6cb15dab49ec33aefef3e98 Mon Sep 17 00:00:00 2001 From: Oleg Skrypnyuk Date: Thu, 15 Jan 2026 17:05:44 +0100 Subject: [PATCH 22/44] fix(test-opentelemetry-wildfly.sh): add JAVA_OPTS export for WildFly startup timeout --- test/test-opentelemetry-wildfly.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test-opentelemetry-wildfly.sh b/test/test-opentelemetry-wildfly.sh index 0c862dc..97d69c5 100755 --- a/test/test-opentelemetry-wildfly.sh +++ b/test/test-opentelemetry-wildfly.sh @@ -4,6 +4,12 @@ SERVICE=${1} source test_helper.sh +# Increase timeout for WildFly startup +# Can be overridden by setting JAVA_OPTS before running script +export JAVA_OPTS="${JAVA_OPTS:--Djboss.as.management. blocking.timeout=600 -Xms1024m -Xmx2048m -XX:MetaspaceSize=256M}" + +_log "Using JAVA_OPTS: ${JAVA_OPTS}" + start_container poll_log 'started in' 'started (with errors) in' || _exit 1 "Server not started" From 703b713fdeb07263a51cf195eea6935448efb63a Mon Sep 17 00:00:00 2001 From: Fernando Mambrilla Garcia Date: Wed, 21 Jan 2026 14:41:53 +0100 Subject: [PATCH 23/44] fix(test): fix -Djboss.as.management.blocking.timeout flag --- test/test-opentelemetry-wildfly.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-opentelemetry-wildfly.sh b/test/test-opentelemetry-wildfly.sh index 97d69c5..be07a18 100755 --- a/test/test-opentelemetry-wildfly.sh +++ b/test/test-opentelemetry-wildfly.sh @@ -6,7 +6,7 @@ source test_helper.sh # Increase timeout for WildFly startup # Can be overridden by setting JAVA_OPTS before running script -export JAVA_OPTS="${JAVA_OPTS:--Djboss.as.management. blocking.timeout=600 -Xms1024m -Xmx2048m -XX:MetaspaceSize=256M}" +export JAVA_OPTS="${JAVA_OPTS:--Djboss.as.management.blocking.timeout=600 -Xms1024m -Xmx2048m -XX:MetaspaceSize=256M}" _log "Using JAVA_OPTS: ${JAVA_OPTS}" From 47f29a79e41a42ec914f66dcbefd7c90af0668dc Mon Sep 17 00:00:00 2001 From: Fernando Mambrilla Garcia Date: Wed, 21 Jan 2026 14:53:38 +0100 Subject: [PATCH 24/44] feat(test-opentelemetry-wildfly.sh): otel use direct console output * direct console output instead of slf4j * once wildfly's logging is reade, otel uses proper logging framework --- test/test-opentelemetry-wildfly.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-opentelemetry-wildfly.sh b/test/test-opentelemetry-wildfly.sh index be07a18..b342143 100755 --- a/test/test-opentelemetry-wildfly.sh +++ b/test/test-opentelemetry-wildfly.sh @@ -6,7 +6,7 @@ source test_helper.sh # Increase timeout for WildFly startup # Can be overridden by setting JAVA_OPTS before running script -export JAVA_OPTS="${JAVA_OPTS:--Djboss.as.management.blocking.timeout=600 -Xms1024m -Xmx2048m -XX:MetaspaceSize=256M}" +export JAVA_OPTS="${JAVA_OPTS:--Djboss.as.management.blocking.timeout=600 -Xms1024m -Xmx2048m -XX:MetaspaceSize=256M -Dotel.javaagent.logging=simple}" _log "Using JAVA_OPTS: ${JAVA_OPTS}" From 2244a4ca6c2218edaeea98fc7d8f9a5dfdf16a13 Mon Sep 17 00:00:00 2001 From: Fernando Mambrilla Garcia Date: Wed, 21 Jan 2026 16:11:57 +0100 Subject: [PATCH 25/44] revert(test-opentelemetry-wildfly.sh): get rid of delay at startup --- test/test-opentelemetry-wildfly.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-opentelemetry-wildfly.sh b/test/test-opentelemetry-wildfly.sh index b342143..c18c6b5 100755 --- a/test/test-opentelemetry-wildfly.sh +++ b/test/test-opentelemetry-wildfly.sh @@ -6,7 +6,7 @@ source test_helper.sh # Increase timeout for WildFly startup # Can be overridden by setting JAVA_OPTS before running script -export JAVA_OPTS="${JAVA_OPTS:--Djboss.as.management.blocking.timeout=600 -Xms1024m -Xmx2048m -XX:MetaspaceSize=256M -Dotel.javaagent.logging=simple}" +export JAVA_OPTS="${JAVA_OPTS:--Dotel.javaagent.logging=simple -Xms1024m -Xmx2048m -XX:MetaspaceSize=256M}" _log "Using JAVA_OPTS: ${JAVA_OPTS}" From db032ef914cbad74015ce2412e2e5014f5953eb6 Mon Sep 17 00:00:00 2001 From: Fernando Mambrilla Garcia Date: Wed, 21 Jan 2026 16:47:31 +0100 Subject: [PATCH 26/44] chore(test-opentelemetry-wildfly): add JAVA_OPTS env to docker-compose * remove export at test-opentelemetry-wildfly script --- test/docker-compose.yml | 1 + test/test-opentelemetry-wildfly.sh | 6 ------ 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/test/docker-compose.yml b/test/docker-compose.yml index bf8cae8..781be42 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -86,6 +86,7 @@ services: - OTEL_TRACES_EXPORTER=otlp - OTEL_EXPORTER_PROMETHEUS_PORT=9464 - OTEL_EXPORTER_OTLP_ENDPOINT=http://opentelemetry-collector:4318 + - JAVA_OPTS=-Dotel.javaagent.logging=simple ports: - "8080:8080" - "9464:9464" diff --git a/test/test-opentelemetry-wildfly.sh b/test/test-opentelemetry-wildfly.sh index c18c6b5..0c862dc 100755 --- a/test/test-opentelemetry-wildfly.sh +++ b/test/test-opentelemetry-wildfly.sh @@ -4,12 +4,6 @@ SERVICE=${1} source test_helper.sh -# Increase timeout for WildFly startup -# Can be overridden by setting JAVA_OPTS before running script -export JAVA_OPTS="${JAVA_OPTS:--Dotel.javaagent.logging=simple -Xms1024m -Xmx2048m -XX:MetaspaceSize=256M}" - -_log "Using JAVA_OPTS: ${JAVA_OPTS}" - start_container poll_log 'started in' 'started (with errors) in' || _exit 1 "Server not started" From bb91bdba090e8a1cfe63a564fb03ae4ba37767ad Mon Sep 17 00:00:00 2001 From: Fernando Mambrilla Garcia Date: Mon, 26 Jan 2026 10:35:31 +0100 Subject: [PATCH 27/44] chore(test-opentelemetry-wildfly): replace JAVA_OPTS by JAVA_TOOLS_OPTIONS --- test/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 781be42..0b9897f 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -86,7 +86,7 @@ services: - OTEL_TRACES_EXPORTER=otlp - OTEL_EXPORTER_PROMETHEUS_PORT=9464 - OTEL_EXPORTER_OTLP_ENDPOINT=http://opentelemetry-collector:4318 - - JAVA_OPTS=-Dotel.javaagent.logging=simple + - JAVA_TOOL_OPTIONS=-Dotel.javaagent.logging=simple ports: - "8080:8080" - "9464:9464" From 4d892376de553bc9c5a363ab5c98aa37bcb1f9aa Mon Sep 17 00:00:00 2001 From: Fernando Mambrilla Garcia Date: Mon, 26 Jan 2026 15:41:13 +0100 Subject: [PATCH 28/44] chore(test-debug): apply JAVA_TOOLS_OPTIONS flag --- test/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 0b9897f..ac7908d 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -41,6 +41,7 @@ services: platform: linux/${PLATFORM} environment: - DEBUG=true + - JAVA_TOOL_OPTIONS=-Dotel.javaagent.logging=simple ports: - "8080:8080" - "8000:8000" From f867c3c7f32045ecfff6bf445155107891dd9375 Mon Sep 17 00:00:00 2001 From: Fernando Mambrilla Garcia Date: Mon, 26 Jan 2026 16:04:03 +0100 Subject: [PATCH 29/44] chore(test-debug): warn logging level for otel --- test/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/test/docker-compose.yml b/test/docker-compose.yml index ac7908d..e642593 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -41,6 +41,7 @@ services: platform: linux/${PLATFORM} environment: - DEBUG=true + - OTEL_JAVAAGENT_LOGGING_LEVEL=warn - JAVA_TOOL_OPTIONS=-Dotel.javaagent.logging=simple ports: - "8080:8080" From 4a2e74d2fe270e7551d585367d74f0084e74fc7c Mon Sep 17 00:00:00 2001 From: Fernando Mambrilla Garcia Date: Wed, 28 Jan 2026 10:41:57 +0100 Subject: [PATCH 30/44] revert(test): revert docker-compose --- test/docker-compose.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/test/docker-compose.yml b/test/docker-compose.yml index e642593..bf8cae8 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -41,8 +41,6 @@ services: platform: linux/${PLATFORM} environment: - DEBUG=true - - OTEL_JAVAAGENT_LOGGING_LEVEL=warn - - JAVA_TOOL_OPTIONS=-Dotel.javaagent.logging=simple ports: - "8080:8080" - "8000:8000" @@ -88,7 +86,6 @@ services: - OTEL_TRACES_EXPORTER=otlp - OTEL_EXPORTER_PROMETHEUS_PORT=9464 - OTEL_EXPORTER_OTLP_ENDPOINT=http://opentelemetry-collector:4318 - - JAVA_TOOL_OPTIONS=-Dotel.javaagent.logging=simple ports: - "8080:8080" - "9464:9464" From 6f6a395e7bb0d10109b349ed6db85b25bbcdfad3 Mon Sep 17 00:00:00 2001 From: Fernando Mambrilla Garcia Date: Wed, 28 Jan 2026 10:43:46 +0100 Subject: [PATCH 31/44] chore(test-debug): use negative lookahead to prevent exit on error --- test/test-debug.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-debug.sh b/test/test-debug.sh index 3009797..a66a4fe 100755 --- a/test/test-debug.sh +++ b/test/test-debug.sh @@ -8,7 +8,7 @@ test "${DISTRO}" = "run" && _log "skipping test of DEBUG socket: not supported f start_container -poll_log "Listening for transport dt_socket at address: 8000" "ERROR" || _exit 1 "JPDA not started" +poll_log "Listening for transport dt_socket at address: 8000" "ERROR(?!.*stderr.*SLF4J)" || _exit 1 "JPDA not started" timeout 1 bash -c 'cat < /dev/null > /dev/tcp/localhost/8000' || _exit 2 "JPDA port not open" From 61322940d5b21cdb1264b4615b6f7dfb426260e0 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 3 Feb 2026 07:13:26 +0100 Subject: [PATCH 32/44] docs(README): update JMX metrics section and add custom configuration instructions; add custom JMX metrics config file --- README.md | 8 ++++++-- opentelemetry/jmx_custom_config.yaml | 6 ++++++ test/otel-collector-config.yml | 12 +++++++++++- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9f3c800..d194424 100644 --- a/README.md +++ b/README.md @@ -297,16 +297,20 @@ The OpenTelemetry Agent can be configured using environment variables. #### JMX Metrics -The OpenTelemetry Agent provides access to JVM metrics through JMX. You can customize JMX metrics collection by mounting a custom configuration file: +The OpenTelemetry Agent provides access to JVM metrics through JMX. By default, the image includes extended JVM metrics (CPU, memory, GC, threads, file descriptors). + +You can add custom JMX metrics by mounting your own configuration file to `/camunda/javaagent/jmx_custom_config.yaml`. For configuration syntax examples, see the pre-configured `opentelemetry/jmx_config.yaml` in this repository or the [OpenTelemetry JMX Metrics documentation](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/main/instrumentation/jmx-metrics). ```bash docker run -d --name cibseven -p 8080:8080 -p 9464:9464 \ -e OTEL_METRICS_EXPORTER=prometheus \ -e OTEL_EXPORTER_PROMETHEUS_PORT=9464 \ - -v $(pwd)/custom_jmx_config.yaml:/camunda/javaagent/jmx_custom_config.yaml \ + -v $(pwd)/my_custom_jmx_config.yaml:/camunda/javaagent/jmx_custom_config.yaml \ cibseven/cibseven:latest ``` +**Note:** Create your own `my_custom_jmx_config.yaml` file before mounting it. The image already contains a default `/camunda/javaagent/jmx_custom_config.yaml` which you can override. + #### Example with Docker Compose See the `test/docker-compose.yml` file for a complete example. Use the `camunda-opentelemetry` service that includes: diff --git a/opentelemetry/jmx_custom_config.yaml b/opentelemetry/jmx_custom_config.yaml index e69de29..3ba9c1c 100644 --- a/opentelemetry/jmx_custom_config.yaml +++ b/opentelemetry/jmx_custom_config.yaml @@ -0,0 +1,6 @@ +# Custom JMX Metrics Configuration +# +# This file can be overridden by mounting your own configuration: +# -v $(pwd)/my_custom_jmx_config.yaml:/camunda/javaagent/jmx_custom_config.yaml +# +# For configuration syntax and examples, see jmx_config.yaml diff --git a/test/otel-collector-config.yml b/test/otel-collector-config.yml index 269e564..de75754 100644 --- a/test/otel-collector-config.yml +++ b/test/otel-collector-config.yml @@ -3,6 +3,16 @@ receivers: protocols: http: endpoint: "0.0.0.0:4318" + + prometheus: + config: + scrape_configs: + - job_name: 'cibseven' + scrape_interval: 30s + static_configs: + - targets: ['camunda-opentelemetry:9464'] + labels: + service: 'cibseven' exporters: debug: @@ -14,7 +24,7 @@ service: receivers: [otlp] exporters: [debug] metrics: - receivers: [otlp] + receivers: [otlp, prometheus] exporters: [debug] logs: receivers: [otlp] From 1bf9d267576043d2bb1dd08f6e959915781d0c4a Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 3 Feb 2026 07:20:16 +0100 Subject: [PATCH 33/44] docs(README): update environment variable defaults for OpenTelemetry configuration --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d194424..c1b6401 100644 --- a/README.md +++ b/README.md @@ -288,11 +288,13 @@ The OpenTelemetry Agent can be configured using environment variables. **Available Environment Variables:** * `OTEL_SERVICE_NAME`: Service name for telemetry data (default: `cibseven`) -* `OTEL_METRICS_EXPORTER`: Configure metrics exporter (e.g., `prometheus`, `otlp`) -* `OTEL_TRACES_EXPORTER`: Configure traces exporter (e.g., `otlp`, `jaeger`) -* `OTEL_LOGS_EXPORTER`: Configure logs exporter (e.g., `otlp`) - **Note:** CIB seven uses log4j2 for application logging, so this is typically not needed +* `OTEL_METRICS_EXPORTER`: Configure metrics exporter (default: `none`, examples: `prometheus`, `otlp`) +* `OTEL_TRACES_EXPORTER`: Configure traces exporter (default: `none`, example: `otlp`) +* `OTEL_LOGS_EXPORTER`: Configure logs exporter (default: `none`, example: `otlp`) - **Note:** CIB seven uses log4j2 for application logging, so this is typically not needed * `OTEL_EXPORTER_PROMETHEUS_PORT`: Port for Prometheus metrics exporter (default: `9464`) -* `OTEL_EXPORTER_OTLP_ENDPOINT`: Endpoint for OTLP exporter (e.g., `http://otel-collector:4318`) +* `OTEL_EXPORTER_OTLP_ENDPOINT`: Endpoint for OTLP exporter (default: `http://localhost:4318`, example: `http://otel-collector:4318`) +* `OTEL_LOG_LEVEL`: OpenTelemetry agent log level (default: `error`, options: `error`, `warn`, `info`, `debug`) +* `OTEL_JMX_CONFIG`: JMX metrics configuration files (default: `/camunda/javaagent/jmx_config.yaml,/camunda/javaagent/jmx_custom_config.yaml`) #### JMX Metrics From 72a0ce4460d2900e073cab2e13ebf40f28f09514 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 3 Feb 2026 08:33:17 +0100 Subject: [PATCH 34/44] removed old prometheus-jmx port --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 58e1e61..ea7a7c5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,7 +77,7 @@ ENV OTEL_SERVICE_NAME=cibseven \ OTEL_TRACES_EXPORTER=none \ OTEL_EXPORTER_PROMETHEUS_PORT=9464 -EXPOSE 8080 8000 9404 9464 +EXPOSE 8080 8000 9464 # Downgrading wait-for-it is necessary until this PR is merged # https://github.com/vishnubob/wait-for-it/pull/68 From 26bfdbd1b25aecdb247e61982235f0f6a518735c Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 5 Feb 2026 11:17:14 +0100 Subject: [PATCH 35/44] chore: update OpenTelemetry configuration and fix logging in test scripts --- Dockerfile | 5 +++-- README.md | 2 +- cibseven-wildfly.sh | 2 +- test/docker-compose.yml | 4 ++-- test/test-opentelemetry-run.sh | 4 ++-- test/test-opentelemetry-tomcat.sh | 2 +- test/test-opentelemetry-wildfly.sh | 2 +- 7 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index ea7a7c5..f80bdea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,6 +72,7 @@ ENV JAVA_OPTS="" ENV OTEL_SERVICE_NAME=cibseven \ OTEL_JMX_CONFIG=/camunda/javaagent/jmx_config.yaml,/camunda/javaagent/jmx_custom_config.yaml \ OTEL_LOG_LEVEL=error \ + OTEL_JAVAAGENT_LOGGING=application \ OTEL_METRICS_EXPORTER=none \ OTEL_LOGS_EXPORTER=none \ OTEL_TRACES_EXPORTER=none \ @@ -105,5 +106,5 @@ CMD ["./cibseven.sh"] COPY --chown=camunda:camunda --from=builder /camunda . # --- Add JMX config files (ensure these are present in your build context) --- -COPY opentelemetry/jmx_config.yaml /camunda/javaagent/jmx_config.yaml -COPY opentelemetry/jmx_custom_config.yaml /camunda/javaagent/jmx_custom_config.yaml +COPY --chown=camunda:camunda opentelemetry/jmx_config.yaml /camunda/javaagent/jmx_config.yaml +COPY --chown=camunda:camunda opentelemetry/jmx_custom_config.yaml /camunda/javaagent/jmx_custom_config.yaml diff --git a/README.md b/README.md index c1b6401..1188221 100644 --- a/README.md +++ b/README.md @@ -290,7 +290,7 @@ The OpenTelemetry Agent can be configured using environment variables. * `OTEL_SERVICE_NAME`: Service name for telemetry data (default: `cibseven`) * `OTEL_METRICS_EXPORTER`: Configure metrics exporter (default: `none`, examples: `prometheus`, `otlp`) * `OTEL_TRACES_EXPORTER`: Configure traces exporter (default: `none`, example: `otlp`) -* `OTEL_LOGS_EXPORTER`: Configure logs exporter (default: `none`, example: `otlp`) - **Note:** CIB seven uses log4j2 for application logging, so this is typically not needed +* `OTEL_LOGS_EXPORTER`: Configure logs exporter (default: `none`, example: `otlp`) - **Note:** CIB seven uses a logging framework for application logging, so this is typically not needed * `OTEL_EXPORTER_PROMETHEUS_PORT`: Port for Prometheus metrics exporter (default: `9464`) * `OTEL_EXPORTER_OTLP_ENDPOINT`: Endpoint for OTLP exporter (default: `http://localhost:4318`, example: `http://otel-collector:4318`) * `OTEL_LOG_LEVEL`: OpenTelemetry agent log level (default: `error`, options: `error`, `warn`, `info`, `debug`) diff --git a/cibseven-wildfly.sh b/cibseven-wildfly.sh index 7694727..06e590c 100755 --- a/cibseven-wildfly.sh +++ b/cibseven-wildfly.sh @@ -73,7 +73,7 @@ fi LOG_MANAGER_PATH=$(find /camunda/modules -name "jboss-logmanager*.jar") COMMON_PATH=$(find /camunda/modules -name "wildfly-common*.jar") export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -Dsun.util.logging.disableCallerCheck=true -Djava.util.logging.manager=org.jboss.logmanager.LogManager -Xbootclasspath/a:$LOG_MANAGER_PATH:$COMMON_PATH" -export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar -Dotel.javaagent.logging=application" +export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar" wait_for_it diff --git a/test/docker-compose.yml b/test/docker-compose.yml index bf8cae8..47740f2 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -96,9 +96,9 @@ services: opentelemetry-collector: image: otel/opentelemetry-collector:latest container_name: opentelemetry-collector - command: ["--config=/etc/otel-collector-config.yaml"] + command: ["--config=/etc/otel-collector-config.yml"] volumes: - - ./otel-collector-config.yml:/etc/otel-collector-config.yaml + - ./otel-collector-config.yml:/etc/otel-collector-config.yml ports: - "4318:4318" restart: unless-stopped \ No newline at end of file diff --git a/test/test-opentelemetry-run.sh b/test/test-opentelemetry-run.sh index 0a60f3d..984d7fe 100755 --- a/test/test-opentelemetry-run.sh +++ b/test/test-opentelemetry-run.sh @@ -12,7 +12,7 @@ _log "Server started" # Test OpenTelemetry metrics endpoint _log "Testing OpenTelemetry metrics endpoint" -curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 8 "OpenTelemetry metrics not available" +curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 3 "OpenTelemetry metrics not available" _log "OpenTelemetry metrics available" -_exit 0 "Test successfull" +_exit 0 "Test successful" diff --git a/test/test-opentelemetry-tomcat.sh b/test/test-opentelemetry-tomcat.sh index 84be1bb..eceb3c5 100755 --- a/test/test-opentelemetry-tomcat.sh +++ b/test/test-opentelemetry-tomcat.sh @@ -15,4 +15,4 @@ _log "Testing OpenTelemetry metrics endpoint" curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 3 "OpenTelemetry metrics not available" _log "OpenTelemetry metrics available" -_exit 0 "Test successfull" \ No newline at end of file +_exit 0 "Test successful" \ No newline at end of file diff --git a/test/test-opentelemetry-wildfly.sh b/test/test-opentelemetry-wildfly.sh index 0c862dc..a4c651b 100755 --- a/test/test-opentelemetry-wildfly.sh +++ b/test/test-opentelemetry-wildfly.sh @@ -15,4 +15,4 @@ _log "Testing OpenTelemetry metrics endpoint" curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 3 "OpenTelemetry metrics not available" _log "OpenTelemetry metrics available" -_exit 0 "Test successfull" \ No newline at end of file +_exit 0 "Test successful" \ No newline at end of file From 39b1840285bf696556953a1f9ed74275654ecda5 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 5 Feb 2026 13:32:17 +0100 Subject: [PATCH 36/44] chore(Dockerfile): remove OTEL_LOG_LEVEL environment variable from OpenTelemetry configuration to use standard --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f80bdea..dd6a55d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,7 +71,6 @@ ENV JAVA_OPTS="" # CATALINA_OPTS for Tomcat) to avoid affecting CLI tools like jboss-cli.sh ENV OTEL_SERVICE_NAME=cibseven \ OTEL_JMX_CONFIG=/camunda/javaagent/jmx_config.yaml,/camunda/javaagent/jmx_custom_config.yaml \ - OTEL_LOG_LEVEL=error \ OTEL_JAVAAGENT_LOGGING=application \ OTEL_METRICS_EXPORTER=none \ OTEL_LOGS_EXPORTER=none \ From d3adc373cd1644a3e48bf700b818f05ede5b3c92 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 10 Feb 2026 15:38:33 +0100 Subject: [PATCH 37/44] docs(README): enhance OpenTelemetry environment variable section with pre-configured options and additional configuration instructions for metrics and trace exports --- README.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1188221..3f8bae1 100644 --- a/README.md +++ b/README.md @@ -285,17 +285,28 @@ The CIB seven Docker images come with OpenTelemetry Java-Agent pre-installed. Th The OpenTelemetry Agent can be configured using environment variables. -**Available Environment Variables:** +**Pre-configured Environment Variables (from Dockerfile):** * `OTEL_SERVICE_NAME`: Service name for telemetry data (default: `cibseven`) * `OTEL_METRICS_EXPORTER`: Configure metrics exporter (default: `none`, examples: `prometheus`, `otlp`) * `OTEL_TRACES_EXPORTER`: Configure traces exporter (default: `none`, example: `otlp`) * `OTEL_LOGS_EXPORTER`: Configure logs exporter (default: `none`, example: `otlp`) - **Note:** CIB seven uses a logging framework for application logging, so this is typically not needed * `OTEL_EXPORTER_PROMETHEUS_PORT`: Port for Prometheus metrics exporter (default: `9464`) -* `OTEL_EXPORTER_OTLP_ENDPOINT`: Endpoint for OTLP exporter (default: `http://localhost:4318`, example: `http://otel-collector:4318`) -* `OTEL_LOG_LEVEL`: OpenTelemetry agent log level (default: `error`, options: `error`, `warn`, `info`, `debug`) +* `OTEL_JAVAAGENT_LOGGING`: OpenTelemetry agent logging mode (default: `application`) * `OTEL_JMX_CONFIG`: JMX metrics configuration files (default: `/camunda/javaagent/jmx_config.yaml,/camunda/javaagent/jmx_custom_config.yaml`) +**Additional Configuration for Metrics Export:** + +* **Prometheus:** Set `OTEL_METRICS_EXPORTER=prometheus` (metrics available at port 9464) +* **OTLP:** Set `OTEL_METRICS_EXPORTER=otlp` and `OTEL_EXPORTER_OTLP_ENDPOINT=http://your-collector:4318` + +**Additional Configuration for Trace Export:** + +* Set `OTEL_TRACES_EXPORTER=otlp` +* Set `OTEL_EXPORTER_OTLP_ENDPOINT=http://your-collector:4318` + +For a complete list of available configuration options, see the [OpenTelemetry Configuration](https://opentelemetry.io/docs/concepts/sdk-configuration/) documentation. + #### JMX Metrics From b10dbaf5ecbf4d5f088fafe3e427925e19c40fd0 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 10 Feb 2026 15:43:48 +0100 Subject: [PATCH 38/44] chore(docker-compose): update OpenTelemetry Collector image to version 0.145.0 --- test/docker-compose.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 47740f2..f65098b 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -94,8 +94,7 @@ services: restart: unless-stopped opentelemetry-collector: - image: otel/opentelemetry-collector:latest - container_name: opentelemetry-collector + image: otel/opentelemetry-collector:0.145.0 command: ["--config=/etc/otel-collector-config.yml"] volumes: - ./otel-collector-config.yml:/etc/otel-collector-config.yml From c6c54c0187f72297f18bcecd12897bc9a9367316 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 4 Mar 2026 10:02:22 +0100 Subject: [PATCH 39/44] added metrics check --- test/test-opentelemetry-run.sh | 4 +++ test/test-opentelemetry-tomcat.sh | 4 +++ test/test-opentelemetry-wildfly.sh | 4 +++ test/test_helper.sh | 40 ++++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+) diff --git a/test/test-opentelemetry-run.sh b/test/test-opentelemetry-run.sh index 984d7fe..38d6cf6 100755 --- a/test/test-opentelemetry-run.sh +++ b/test/test-opentelemetry-run.sh @@ -15,4 +15,8 @@ _log "Testing OpenTelemetry metrics endpoint" curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 3 "OpenTelemetry metrics not available" _log "OpenTelemetry metrics available" +# Verify JMX metrics from opentelemetry/jmx_config.yaml are present +assert_jmx_metrics "http://localhost:9464/metrics" || _exit 4 "Not all JMX metrics from config are exposed" +_log "All JMX metrics from config are present" + _exit 0 "Test successful" diff --git a/test/test-opentelemetry-tomcat.sh b/test/test-opentelemetry-tomcat.sh index eceb3c5..d263957 100755 --- a/test/test-opentelemetry-tomcat.sh +++ b/test/test-opentelemetry-tomcat.sh @@ -15,4 +15,8 @@ _log "Testing OpenTelemetry metrics endpoint" curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 3 "OpenTelemetry metrics not available" _log "OpenTelemetry metrics available" +# Verify JMX metrics from opentelemetry/jmx_config.yaml are present +assert_jmx_metrics "http://localhost:9464/metrics" || _exit 4 "Not all JMX metrics from config are exposed" +_log "All JMX metrics from config are present" + _exit 0 "Test successful" \ No newline at end of file diff --git a/test/test-opentelemetry-wildfly.sh b/test/test-opentelemetry-wildfly.sh index a4c651b..f9d5f78 100755 --- a/test/test-opentelemetry-wildfly.sh +++ b/test/test-opentelemetry-wildfly.sh @@ -15,4 +15,8 @@ _log "Testing OpenTelemetry metrics endpoint" curl -s http://localhost:9464/metrics | grep -q "target_info" || _exit 3 "OpenTelemetry metrics not available" _log "OpenTelemetry metrics available" +# Verify JMX metrics from opentelemetry/jmx_config.yaml are present +assert_jmx_metrics "http://localhost:9464/metrics" || _exit 4 "Not all JMX metrics from config are exposed" +_log "All JMX metrics from config are present" + _exit 0 "Test successful" \ No newline at end of file diff --git a/test/test_helper.sh b/test/test_helper.sh index 2c6d458..2d7d5c9 100644 --- a/test/test_helper.sh +++ b/test/test_helper.sh @@ -83,3 +83,43 @@ function test_encoding { curl --fail -w "\n" http://localhost:8080/engine-rest/deployment/create -F deployment-name=testEncoding -F testEncoding.bpmn=@testEncoding.bpmn curl --fail -w "\n" -H "Content-Type: application/json" -d '{}' http://localhost:8080/engine-rest/process-definition/key/testEncoding/start } + +# Expected Prometheus metric names from opentelemetry/jmx_config.yaml (prefix os. + mapping metrics, dots -> underscores). +EXPECTED_JMX_METRICS=( + os_cpu_count + os_cpu_time + os_cpu_recent_utilization + os_system_cpu_load_1m + os_system_cpu_utilization + os_file_descriptor_open_count + os_file_descriptor_max_count + os_virtual_memory_committed_size + os_physical_memory_free_size + os_physical_memory_total_size + os_swap_space_free_size + os_swap_space_total_size +) + +# Verifies that all expected JMX metrics are present on the Prometheus metrics endpoint. +# Usage: assert_jmx_metrics [metrics_url] +# Default metrics_url: http://localhost:9464/metrics +function assert_jmx_metrics { + local metrics_url="${1:-http://localhost:9464/metrics}" + local metrics_output + local missing="" + + metrics_output=$(curl -s --fail "$metrics_url") || { _log "Failed to fetch $metrics_url"; return 1; } + + for prometheus_name in "${EXPECTED_JMX_METRICS[@]}"; do + if ! echo "$metrics_output" | grep -qE "(^|[[:space:]])${prometheus_name}([{\s]|$)"; then + [ -n "$missing" ] && missing="$missing, " + missing="${missing}${prometheus_name}" + fi + done + + if [ -n "$missing" ]; then + _log "Missing expected JMX metrics: $missing" + return 1 + fi + return 0 +} From 30173c0a1b25476a0d5af705443b02e1d779fbfb Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 4 Mar 2026 10:43:21 +0100 Subject: [PATCH 40/44] added polling mechanism for metrics-test --- test/test_helper.sh | 57 +++++++++++++++++++++++++++++---------------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/test/test_helper.sh b/test/test_helper.sh index 2d7d5c9..eed2496 100644 --- a/test/test_helper.sh +++ b/test/test_helper.sh @@ -84,42 +84,59 @@ function test_encoding { curl --fail -w "\n" -H "Content-Type: application/json" -d '{}' http://localhost:8080/engine-rest/process-definition/key/testEncoding/start } -# Expected Prometheus metric names from opentelemetry/jmx_config.yaml (prefix os. + mapping metrics, dots -> underscores). +# Expected Prometheus metric names from opentelemetry/jmx_config.yaml as exported by the OTel agent +# (prefix os. + mapping metrics; agent adds unit suffixes: _ratio, _seconds_total, _bytes). EXPECTED_JMX_METRICS=( os_cpu_count - os_cpu_time - os_cpu_recent_utilization + os_cpu_time_seconds_total + os_cpu_recent_utilization_ratio os_system_cpu_load_1m - os_system_cpu_utilization + os_system_cpu_utilization_ratio os_file_descriptor_open_count os_file_descriptor_max_count - os_virtual_memory_committed_size - os_physical_memory_free_size - os_physical_memory_total_size - os_swap_space_free_size - os_swap_space_total_size + os_virtual_memory_committed_size_bytes + os_physical_memory_free_size_bytes + os_physical_memory_total_size_bytes + os_swap_space_free_size_bytes + os_swap_space_total_size_bytes ) # Verifies that all expected JMX metrics are present on the Prometheus metrics endpoint. +# Polls until all metrics appear (JMX metrics are created asynchronously by the agent after server start) or timeout. # Usage: assert_jmx_metrics [metrics_url] # Default metrics_url: http://localhost:9464/metrics +JMX_METRICS_RETRIES=10 +JMX_METRICS_WAIT=3 + function assert_jmx_metrics { local metrics_url="${1:-http://localhost:9464/metrics}" local metrics_output local missing="" + local attempt + + for attempt in $(seq 1 $JMX_METRICS_RETRIES); do + _log "Checking JMX metrics (attempt $attempt/$JMX_METRICS_RETRIES)" + metrics_output=$(curl -s --fail "$metrics_url") || { _log "Failed to fetch $metrics_url"; return 1; } + missing="" + + for prometheus_name in "${EXPECTED_JMX_METRICS[@]}"; do + if ! echo "$metrics_output" | grep -qE "(^|[[:space:]])${prometheus_name}([{\s]|$)"; then + [ -n "$missing" ] && missing="$missing, " + missing="${missing}${prometheus_name}" + fi + done + + if [ -z "$missing" ]; then + _log "All JMX metrics present" + return 0 + fi - metrics_output=$(curl -s --fail "$metrics_url") || { _log "Failed to fetch $metrics_url"; return 1; } - - for prometheus_name in "${EXPECTED_JMX_METRICS[@]}"; do - if ! echo "$metrics_output" | grep -qE "(^|[[:space:]])${prometheus_name}([{\s]|$)"; then - [ -n "$missing" ] && missing="$missing, " - missing="${missing}${prometheus_name}" + if [ $attempt -lt $JMX_METRICS_RETRIES ]; then + _log "Waiting $JMX_METRICS_WAIT s for JMX metrics (missing: $missing)" + sleep $JMX_METRICS_WAIT fi done - if [ -n "$missing" ]; then - _log "Missing expected JMX metrics: $missing" - return 1 - fi - return 0 + _log "Missing expected JMX metrics after ${JMX_METRICS_RETRIES} attempts: $missing" + return 1 } From 6d0f428d9e7f7c84e0161570f7420a09f20f6667 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 4 Mar 2026 11:21:44 +0100 Subject: [PATCH 41/44] increased wait-time for jmx-tests --- test/test_helper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_helper.sh b/test/test_helper.sh index eed2496..728801e 100644 --- a/test/test_helper.sh +++ b/test/test_helper.sh @@ -106,7 +106,7 @@ EXPECTED_JMX_METRICS=( # Usage: assert_jmx_metrics [metrics_url] # Default metrics_url: http://localhost:9464/metrics JMX_METRICS_RETRIES=10 -JMX_METRICS_WAIT=3 +JMX_METRICS_WAIT=60 function assert_jmx_metrics { local metrics_url="${1:-http://localhost:9464/metrics}" From 8a40ec4c5b999eac251822a3e95829fc7ec4df73 Mon Sep 17 00:00:00 2001 From: MeisterProper Date: Wed, 4 Mar 2026 16:15:37 +0100 Subject: [PATCH 42/44] Update Dockerfile --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index dd6a55d..26caf19 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,6 +72,7 @@ ENV JAVA_OPTS="" ENV OTEL_SERVICE_NAME=cibseven \ OTEL_JMX_CONFIG=/camunda/javaagent/jmx_config.yaml,/camunda/javaagent/jmx_custom_config.yaml \ OTEL_JAVAAGENT_LOGGING=application \ + OTEL_LOG_LEVEL=error \ OTEL_METRICS_EXPORTER=none \ OTEL_LOGS_EXPORTER=none \ OTEL_TRACES_EXPORTER=none \ From 563ecadf33a82f50317225110f60f197f8ee917b Mon Sep 17 00:00:00 2001 From: Fernando Mambrilla Garcia Date: Thu, 5 Mar 2026 17:34:35 +0100 Subject: [PATCH 43/44] fix(otel): add new dep via bootstrap classloader * wildfly-common now depends on smallrye --- cibseven-wildfly.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cibseven-wildfly.sh b/cibseven-wildfly.sh index 06e590c..d9fe9f4 100755 --- a/cibseven-wildfly.sh +++ b/cibseven-wildfly.sh @@ -72,7 +72,9 @@ fi # See https://github.com/prometheus/jmx_exporter/issues/344 LOG_MANAGER_PATH=$(find /camunda/modules -name "jboss-logmanager*.jar") COMMON_PATH=$(find /camunda/modules -name "wildfly-common*.jar") -export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -Dsun.util.logging.disableCallerCheck=true -Djava.util.logging.manager=org.jboss.logmanager.LogManager -Xbootclasspath/a:$LOG_MANAGER_PATH:$COMMON_PATH" +SMALLRYE_OS_PATH=$(find /camunda/modules -name "smallrye-common-os*.jar") +SMALLRYE_NET_PATH=$(find /camunda/modules -name "smallrye-common-net*.jar") +export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -Dsun.util.logging.disableCallerCheck=true -Djava.util.logging.manager=org.jboss.logmanager.LogManager -Xbootclasspath/a:$LOG_MANAGER_PATH:$COMMON_PATH:$SMALLRYE_OS_PATH:$SMALLRYE_NET_PATH" export PREPEND_JAVA_OPTS="${PREPEND_JAVA_OPTS} -javaagent:/camunda/javaagent/opentelemetry-javaagent-${OPENTELEMETRY_AGENT_VERSION}.jar" wait_for_it From f9ace127da690e5bba1907a26b6307047b0d999f Mon Sep 17 00:00:00 2001 From: MeisterProper Date: Mon, 23 Mar 2026 10:11:53 +0100 Subject: [PATCH 44/44] removed otel-log-option --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 26caf19..dd6a55d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -72,7 +72,6 @@ ENV JAVA_OPTS="" ENV OTEL_SERVICE_NAME=cibseven \ OTEL_JMX_CONFIG=/camunda/javaagent/jmx_config.yaml,/camunda/javaagent/jmx_custom_config.yaml \ OTEL_JAVAAGENT_LOGGING=application \ - OTEL_LOG_LEVEL=error \ OTEL_METRICS_EXPORTER=none \ OTEL_LOGS_EXPORTER=none \ OTEL_TRACES_EXPORTER=none \