fix(telemetry): opt-in preservation of trace-context headers on subgraph requests

.changesets/fix_router_2060_preserve_subgraph_trace_context.md

@@ -0,0 +1,17 @@
+### Add an opt-in setting to preserve an existing trace-context header on outgoing subgraph requests ([PR #10014](https://github.com/apollographql/router/pull/10014))
+
+Under the OTLP exporter, the router's outbound HTTP client unconditionally re-injected its own span context into `traceparent`/`tracestate` (and the custom trace ID header, if `propagation.request.header_name` is configured) on every subgraph request, discarding any value already present on that header — whether set by a coprocessor, a Rhai script, or header propagation from the original client request. This silently broke trace correlation for setups that deliberately rewrite these headers before the subgraph call, for example a coprocessor inserting itself as a hop in the trace (standard, spec-compliant W3C Trace Context practice) or forwarding a client-generated trace ID for RUM correlation. Under the Datadog-native exporter this didn't happen, but only because Datadog's own propagator never touches those headers — an incidental side effect of that exporter, not a documented guarantee.
+
+A new setting, `telemetry.exporters.tracing.propagation.preserve_trace_context_on_subgraph_requests`, lets you opt into keeping whatever trace-context header is already present on a subgraph request instead of overwriting it:
+
+```yaml
+telemetry:
+  exporters:
+    tracing:
+      propagation:
+        preserve_trace_context_on_subgraph_requests: true
+```
+
+It defaults to `false`, so existing deployments are unaffected. When enabled, the router still performs its own trace-context injection exactly as before; it then restores whichever header was present beforehand, so every other propagator (baggage, Jaeger, Zipkin, Datadog, X-Ray) behaves identically to today regardless of this setting. Only one trace-ID-carrying header is preserved per call: the custom trace ID header takes priority when configured and non-empty, falling back to `traceparent` otherwise — the same precedence already used when extracting trace context from inbound requests. This setting only affects the router's calls to subgraphs; calls to coprocessors and connectors are unaffected.
+
+By [@OriginLeon](https://github.com/OriginLeon) in https://github.com/apollographql/router/pull/10014

apollo-router/src/configuration/metrics.rs

@@ -393,7 +393,12 @@ impl InstrumentData {
             opt.spans.supergraph,
             "$..spans.supergraph",
             opt.tracing.common.sampler,
-            "$..tracing.common.sampler"
+            "$..tracing.common.sampler",
+            // BEGIN/END ROUTER-2060
+            opt.tracing
+                .propagation
+                .preserve_trace_context_on_subgraph_requests,
+            "$..tracing.propagation[?(@.preserve_trace_context_on_subgraph_requests==true)]"
         );
 
         populate_config_instrument!(

apollo-router/src/configuration/snapshots/apollo_router__configuration__metrics__test__metrics@telemetry.router.yaml.snap

@@ -1,5 +1,6 @@
 ---
 source: apollo-router/src/configuration/metrics.rs
+assertion_line: 737
 expression: "& metrics.non_zero()"
 ---
 - name: apollo.router.config.telemetry
@@ -30,4 +31,5 @@ expression: "& metrics.non_zero()"
           opt.tracing.common.sampler: false
           opt.tracing.datadog: true
           opt.tracing.otlp: true
+          opt.tracing.propagation.preserve_trace_context_on_subgraph_requests: false
           opt.tracing.zipkin: true

apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap

@@ -1,5 +1,6 @@
 ---
 source: apollo-router/src/configuration/tests.rs
+assertion_line: 28
 expression: "&schema"
 ---
 {
@@ -8142,6 +8143,11 @@ expression: "&schema"
           "description": "Propagate Jaeger",
           "type": "boolean"
         },
+        "preserve_trace_context_on_subgraph_requests": {
+          "default": false,
+          "description": "If a trace-context header (traceparent/tracestate, or the custom trace ID header if\nconfigured) is already present on an outgoing subgraph request, keep it instead of\noverwriting it with the router's own span context.",
+          "type": "boolean"
+        },
         "request": {
           "allOf": [
             {

apollo-router/src/plugins/telemetry/config.rs

@@ -414,6 +414,12 @@ pub(crate) struct Propagation {
     pub(crate) zipkin: bool,
     /// Propagate AWS X-Ray
     pub(crate) aws_xray: bool,
+    // BEGIN ROUTER-2060
+    /// If a trace-context header (traceparent/tracestate, or the custom trace ID header if
+    /// configured) is already present on an outgoing subgraph request, keep it instead of
+    /// overwriting it with the router's own span context.
+    pub(crate) preserve_trace_context_on_subgraph_requests: bool,
+    // END ROUTER-2060
 }
 
 #[derive(Clone, Debug, Deserialize, JsonSchema, Default, PartialEq)]

apollo-router/src/router_factory.rs

@@ -29,6 +29,7 @@ use crate::plugin::Handler;
 use crate::plugin::PluginFactory;
 use crate::plugin::PluginInit;
 use crate::plugins::subscription::notification::Notify;
+use crate::plugins::telemetry::Telemetry; // BEGIN/END ROUTER-2060
 use crate::plugins::telemetry::reload::otel::apollo_opentelemetry_initialized;
 use crate::plugins::traffic_shaping::APOLLO_TRAFFIC_SHAPING;
 use crate::plugins::traffic_shaping::TrafficShaping;
@@ -41,6 +42,7 @@ use crate::services::SupergraphCreator;
 use crate::services::apollo_graph_reference;
 use crate::services::apollo_key;
 use crate::services::http::HttpClientServiceFactory;
+use crate::services::http::TraceContextPreservation; // BEGIN/END ROUTER-2060
 use crate::services::layers::persisted_queries::PersistedQueryLayer;
 use crate::services::layers::query_analysis::QueryAnalysisLayer;
 use crate::services::new_service::ServiceFactory;
@@ -446,6 +448,28 @@ pub(crate) async fn create_http_services(
         .and_then(|plugin| (*plugin.1).as_any().downcast_ref::<TrafficShaping>())
         .expect("traffic shaping should always be part of the plugin list");
 
+    // BEGIN ROUTER-2060
+    let telemetry_config = plugins
+        .iter()
+        .find(|i| i.0.as_str() == "apollo.telemetry") // BEGIN/END ROUTER-2060: registered as "{group}.{name}", see PluginFactory::new_private
+        .and_then(|plugin| (*plugin.1).as_any().downcast_ref::<Telemetry>())
+        .map(|t| t.config.clone());
+    let subgraph_trace_context_preservation = TraceContextPreservation {
+        enabled: telemetry_config
+            .as_ref()
+            .map(|c| {
+                c.exporters
+                    .tracing
+                    .propagation
+                    .preserve_trace_context_on_subgraph_requests
+            })
+            .unwrap_or(false),
+        custom_header_name: telemetry_config
+            .as_ref()
+            .and_then(|c| c.exporters.tracing.propagation.request.header_name.clone()),
+    };
+    // END ROUTER-2060
+
     let connector_subgraphs: HashSet<String> = schema
         .connectors
         .as_ref()
@@ -467,6 +491,7 @@ pub(crate) async fn create_http_services(
             configuration,
             &subgraph_tls_root_store,
             shaping.subgraph_client_config(name),
+            subgraph_trace_context_preservation.clone(), // BEGIN/END ROUTER-2060
         )?;
 
         let http_service_factory = HttpClientServiceFactory::new(http_service, plugins.clone());

apollo-router/src/services/http.rs

@@ -15,6 +15,7 @@ pub(crate) mod service;
 mod tests;
 
 pub(crate) use service::HttpClientService;
+pub(crate) use service::TraceContextPreservation; // BEGIN/END ROUTER-2060
 
 pub(crate) type BoxService = tower::util::BoxService<HttpRequest, HttpResponse, BoxError>;
 pub(crate) type BoxCloneService = tower::util::BoxCloneService<HttpRequest, HttpResponse, BoxError>;
@@ -56,6 +57,7 @@ impl HttpClientServiceFactory {
             configuration,
             &rustls::RootCertStore::empty(),
             client_config,
+            TraceContextPreservation::default(), // BEGIN/END ROUTER-2060
         )
         .unwrap();