diff --git a/cwms-data-api/build.gradle b/cwms-data-api/build.gradle index 2b896e1cf..32fb21049 100644 --- a/cwms-data-api/build.gradle +++ b/cwms-data-api/build.gradle @@ -147,6 +147,10 @@ dependencies { implementation project(":access-manager-api") implementation(libs.bundles.metrics) + implementation(libs.io.opentelemetry.api) + implementation(libs.io.opentelemetry.sdk) + implementation(libs.io.opentelemetry.exporter.logging) + implementation(libs.io.opentelemetry.instrumentation.java) implementation(libs.bundles.jackson) @@ -176,6 +180,7 @@ dependencies { } baseLibs(libs.ch.qos.logback) + baseLibs(libs.io.opentelemetry.instrumentation.logback.mdc) testImplementation(libs.bundles.testcontainers) @@ -327,6 +332,7 @@ task integrationTests(type: Test) { jvmArgs += "-Dcwms.dataapi.access.provider=MultipleAccessManager" jvmArgs += "-Dcwms.dataapi.access.providers=KeyAccessManager,CwmsAccessManager" jvmArgs += "-Dcatalina.base=$buildDir/tomcat" + //jvmArgs += "-Dflogger.backend_factory=com.google.common.flogger.backend.slf4j.Slf4jBackendFactory#getInstance" } task timeseriesReadBenchmark(type: JavaExec) { diff --git a/cwms-data-api/logback.xml b/cwms-data-api/logback.xml index ea197d5a4..3771148fb 100644 --- a/cwms-data-api/logback.xml +++ b/cwms-data-api/logback.xml @@ -14,6 +14,11 @@ + + + + + false @@ -21,6 +26,10 @@ build/cda.jsonl + + + + @@ -36,8 +45,8 @@ - - + + diff --git a/cwms-data-api/src/docker/logback-juli.xml b/cwms-data-api/src/docker/logback-juli.xml index fa337a1af..bcb9dfde6 100644 --- a/cwms-data-api/src/docker/logback-juli.xml +++ b/cwms-data-api/src/docker/logback-juli.xml @@ -21,9 +21,8 @@ - - + diff --git a/cwms-data-api/src/main/java/cwms/cda/ApiServlet.java b/cwms-data-api/src/main/java/cwms/cda/ApiServlet.java index ab5e22faa..e9e6a0915 100644 --- a/cwms-data-api/src/main/java/cwms/cda/ApiServlet.java +++ b/cwms-data-api/src/main/java/cwms/cda/ApiServlet.java @@ -218,6 +218,19 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.sql.DataSource; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.exporter.logging.LoggingSpanExporter; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; + import org.apache.http.entity.ContentType; import org.jetbrains.annotations.NotNull; import org.jooq.exception.DataAccessException; @@ -295,7 +308,7 @@ public class ApiServlet extends HttpServlet { public static final String DEFAULT_PROVIDER = "MultipleAccessManager"; - + private MetricRegistry metrics; private Meter totalRequests; diff --git a/cwms-data-api/src/main/java/cwms/cda/OpenTelemetrySetup.java b/cwms-data-api/src/main/java/cwms/cda/OpenTelemetrySetup.java new file mode 100644 index 000000000..000ec4373 --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/OpenTelemetrySetup.java @@ -0,0 +1,34 @@ +package cwms.cda; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.trace.SdkTracerProvider; + +public final class OpenTelemetrySetup { + private OpenTelemetrySetup() { + /* This utility class should not be instantiated */ + } + + /** + * Initializes the OpenTelemetry SDK with a logging span exporter and the W3C Trace Context + * propagator. + * + * @return A ready-to-use {@link OpenTelemetry} instance. + */ + @SuppressWarnings("null") // nothing here can be null without other exceptions getting thrown. + public static void initTelemetry() { + SdkTracerProvider sdkTracerProvider = + SdkTracerProvider.builder() + .build(); + + OpenTelemetrySdk.builder() + .setTracerProvider(sdkTracerProvider) + .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) + .buildAndRegisterGlobal(); + Runtime.getRuntime().addShutdownHook(new Thread(sdkTracerProvider::close)); + + } + +} diff --git a/cwms-data-api/src/main/java/cwms/cda/servlet/W3CTraceFilter.java b/cwms-data-api/src/main/java/cwms/cda/servlet/W3CTraceFilter.java new file mode 100644 index 000000000..1dc0b9c50 --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/servlet/W3CTraceFilter.java @@ -0,0 +1,80 @@ +package cwms.cda.servlet; + +import java.io.IOException; +import java.util.List; +import java.util.regex.Pattern; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.annotation.WebFilter; +import javax.servlet.http.HttpServletRequest; + +import cwms.cda.OpenTelemetrySetup; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.ContextKey; +import io.opentelemetry.context.propagation.TextMapGetter; + +/** + * + */ +@WebFilter(urlPatterns = {"*"}) +public final class W3CTraceFilter implements Filter { + + public static final ContextKey TRACE_PARENT = ContextKey.named("traceparent"); + public static final Pattern TRACE_PARENT_MATCHER = + Pattern.compile("[a-z0-9]{2}-[a-z0-9]{32}-[a-z0-9]{16}-[a-z0-9]{2}"); + + public W3CTraceFilter() { + OpenTelemetrySetup.initTelemetry(); + } + + @Override + public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) + throws IOException, ServletException { + var spanBuilder = GlobalOpenTelemetry.getTracer("cda") + .spanBuilder("Request") + .setSpanKind(SpanKind.SERVER); + var provided = ((HttpServletRequest)request).getHeader(TRACE_PARENT.toString()); + if (provided != null && !provided.isEmpty() && TRACE_PARENT_MATCHER.matcher(provided).matches()) { + var propagator = GlobalOpenTelemetry.getPropagators().getTextMapPropagator(); + var ctx = propagator.extract(Context.current(), provided, new TraceGetter()); + spanBuilder.setParent(ctx); + } + + var span = spanBuilder.startSpan(); + try (var scope = span.makeCurrent()) { + chain.doFilter(request, response); + } finally { + span.end(); + } + } + + /** + * A simple wrapper to just get the value in the required way. + */ + private static class TraceGetter implements TextMapGetter + { + @Override + public Iterable keys(@Nonnull String carrier) + { + return List.of(TRACE_PARENT.toString()); + } + + @Override + @Nullable + public String get(@Nullable String carrier, @Nonnull String key) { + if (TRACE_PARENT.toString().equalsIgnoreCase(key)) { + return carrier; + } else { + return null; + } + } + } +} diff --git a/cwms-data-api/src/test/java/cwms/cda/api/rating/RatingSpecControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/rating/RatingSpecControllerTestIT.java index 4e41a849e..6393fb4d6 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/rating/RatingSpecControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/rating/RatingSpecControllerTestIT.java @@ -39,6 +39,7 @@ import cwms.cda.data.dto.rating.RatingSpec; import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; +import cwms.cda.servlet.W3CTraceFilter; import javax.servlet.http.HttpServletResponse; import java.util.stream.IntStream; @@ -58,6 +59,7 @@ class RatingSpecControllerTestIT extends DataApiTestIT { @Test void test_empty_rating_spec() throws Exception { + final String TRACE_PARENT_VALUE = "00-f64a0407859e1a735c1a89c5c5b4f47f-09d07b8aaba94e49-01"; String locationId = "RatingSpecTestEmpty"; String officeId = "SPK"; createLocation(locationId, true, officeId); @@ -77,6 +79,7 @@ void test_empty_rating_spec() throws Exception { .contentType(Formats.XMLV2) .body(templateXml) .header("Authorization", user.toHeaderValue()) + .header(W3CTraceFilter.TRACE_PARENT.toString(), TRACE_PARENT_VALUE) .queryParam(OFFICE, officeId) .when() .redirects().follow(true) @@ -93,6 +96,7 @@ void test_empty_rating_spec() throws Exception { .contentType(Formats.XMLV2) .body(specXml) .header("Authorization", user.toHeaderValue()) + .header(W3CTraceFilter.TRACE_PARENT.toString(), TRACE_PARENT_VALUE) .queryParam(OFFICE, officeId) .when() .redirects().follow(true) @@ -110,6 +114,7 @@ void test_empty_rating_spec() throws Exception { .log().ifValidationFails(LogDetail.ALL,true) .accept(Formats.JSONV2) .queryParam(PAGE_SIZE, 500) + .header(W3CTraceFilter.TRACE_PARENT.toString(), TRACE_PARENT_VALUE) .when() .redirects().follow(true) .redirects().max(3) @@ -129,6 +134,7 @@ void test_empty_rating_spec() throws Exception { .contentType(Formats.JSONV2) .queryParam(OFFICE, officeId) .queryParam(RATING_ID_MASK, specContainer.specId) + .header(W3CTraceFilter.TRACE_PARENT.toString(), TRACE_PARENT_VALUE) .when() .redirects().follow(true) .redirects().max(3) @@ -148,6 +154,7 @@ void test_empty_rating_spec() throws Exception { .header("Authorization", user.toHeaderValue()) .queryParam(OFFICE, officeId) .queryParam(METHOD, JooqDao.DeleteMethod.DELETE_ALL) + .header(W3CTraceFilter.TRACE_PARENT.toString(), TRACE_PARENT_VALUE) .when() .redirects().follow(true) .redirects().max(3) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 310a50feb..2b0214623 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -39,6 +39,9 @@ openapi-validation = "2.44.9" javaparser = "3.26.2" togglz = "3.3.3" minio = "8.6.0" +opentelemetry = "1.63.0" +opentelemetry-java = "2.29.0" +opentelemetry-java-logging = "2.29.0-alpha" #Overrides classgraph = { strictly = '4.8.176' } @@ -63,6 +66,12 @@ google-flogger-api = { module = "com.google.flogger:flogger", version.ref = "flo google-flogger-system-backend = { module = "com.google.flogger:flogger-system-backend", version.ref = "flogger" } google-flogger-slf4j-backend = { module = "com.google.flogger:flogger-slf4j-backend", version.ref = "flogger" } ch-qos-logback = { module = "ch.qos.logback:logback-classic", version.ref = "logback" } +io-opentelemetry-instrumentation-logback-mdc = { module = "io.opentelemetry.instrumentation:opentelemetry-logback-mdc-1.0", version.ref = "opentelemetry-java-logging" } +io-opentelemetry-api = { module="io.opentelemetry:opentelemetry-api", version.ref = "opentelemetry" } +io-opentelemetry-sdk = { module="io.opentelemetry:opentelemetry-sdk", version.ref = "opentelemetry" } +io-opentelemetry-exporter-logging = { module="io.opentelemetry:opentelemetry-exporter-logging", version.ref="opentelemetry" } +io-opentelemetry-instrumentation-java = { module="io.opentelemetry.instrumentation:opentelemetry-instrumentation-api", version.ref = "opentelemetry-java" } + google-findbugs = { module = "com.google.code.findbugs:jsr305", version.ref = "google-findbugs" } google-errorProne = { module = "com.google.errorprone:error_prone_annotations", version.ref = "error_prone_annotations"} nucleus-data = { module = "mil.army.usace.hec:hec-nucleus-data", version.ref = "hec-nucleus" }