diff --git a/docs/metrics.md b/docs/metrics.md index f87fa938..0242acb6 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -6,12 +6,16 @@ The Relay Proxy can export metrics via [OpenTelemetry Protocol (OTLP)](https://o ## Available metrics -| Metric | Type | Description | -|--------|------|-------------| -| `launchdarkly.relay.connections` | UpDownCounter | The number of currently active stream connections from SDKs to the Relay Proxy. | -| `launchdarkly.relay.requests` | Counter | The cumulative number of requests received by the Relay Proxy's [service endpoints](./endpoints.md) since startup. | -| `launchdarkly.relay.request.duration` | Histogram | The duration of requests to the Relay Proxy's service endpoints, in seconds. | -| `launchdarkly.relay.events.received.bytes` | Counter | The cumulative number of event bytes received by the Relay Proxy (measured after decompression). | +| Metric | Type | Unit | Description | +|--------|------|------|-------------| +| `http.server.active_requests` | UpDownCounter | `{request}` | The number of currently active stream connections from SDKs to the Relay Proxy. | +| `http.server.request.duration` | Histogram | `s` | The duration of requests to the Relay Proxy's service endpoints, in seconds. | +| `launchdarkly.relay.events.received.size` | Counter | `By` | The cumulative number of event bytes received by the Relay Proxy (measured after decompression). | +| `launchdarkly.relay.events.sent` | Counter | `{event}` | The cumulative number of events successfully sent to LaunchDarkly. | +| `launchdarkly.relay.events.sent.size` | Counter | `By` | The cumulative bytes of event payloads successfully sent to LaunchDarkly. | +| `launchdarkly.relay.events.send.errors` | Counter | `{event}` | The cumulative number of events that failed to send after all retries. | +| `launchdarkly.relay.events.dropped` | Counter | `{event}` | The cumulative number of events dropped due to capacity overflow. | +| `launchdarkly.relay.events.pending` | Gauge | `{event}` | The current number of events buffered in the queue. | ## Attributes @@ -19,16 +23,17 @@ All metrics include the following attributes: | Attribute | Description | |-----------|-------------| -| `relayId` | A unique identifier for this Relay Proxy instance, generated at startup. | -| `env` | The name of the LaunchDarkly environment as configured in the Relay Proxy. In automatic configuration or offline mode, this is the actual project and environment name from LaunchDarkly. Example: `MyApplication Staging` | -| `platformCategory` | The kind of SDK that generated the metric: `server`, `mobile`, or `browser`. | -| `userAgent` | The user agent of the SDK making the request. Example: `Node/3.4.0` | -| `sdkWrapper` | The SDK wrapper identifier, if provided. Example: `flutter-client/2.0.0` | -| `route` | The request URL path template. Variables appear as placeholders rather than actual values. Example: `/sdk/evalx/{envId}/contexts/{context}` | -| `method` | The HTTP method. Example: `GET` | +| `relay.id` | A unique identifier for this Relay Proxy instance, generated at startup. | +| `environment.name` | The name of the LaunchDarkly environment as configured in the Relay Proxy. In automatic configuration or offline mode, this is the actual project and environment name from LaunchDarkly. Example: `MyApplication Staging` | +| `platform.category` | The kind of SDK that generated the metric: `server`, `mobile`, or `browser`. | +| `user_agent` | The user agent of the SDK making the request. Example: `Node/3.4.0` | +| `sdk.wrapper` | The SDK wrapper identifier, if provided. Example: `flutter-client/2.0.0` | +| `http.route` | The request URL path template. Variables appear as placeholders rather than actual values. Example: `/sdk/evalx/{envId}/contexts/{context}` | +| `http.request.method` | The HTTP method. Example: `GET` | +| `url.scheme` | The URL scheme. Example: `https` | | `application.id` | The application identifier, extracted from the `application-id` field of the `X-LaunchDarkly-Tags` header. | | `application.version` | The application version, extracted from the `application-version` field of the `X-LaunchDarkly-Tags` header. | -| `instanceId` | The SDK instance identifier from the `X-LaunchDarkly-Instance-Id` header. | +| `instance.id` | The SDK instance identifier from the `X-LaunchDarkly-Instance-Id` header. | ## Backend-specific notes @@ -42,7 +47,7 @@ OTEL_EXPORTER_OTLP_ENDPOINT=http://:9090/api/v1/otlp/v1/metrics OTEL_EXPORTER_OTLP_PROTOCOL=http ``` -Prometheus converts OpenTelemetry metric names by replacing dots with underscores, so the metrics will appear as `launchdarkly_relay_connections`, `launchdarkly_relay_requests_total`, `launchdarkly_relay_request_duration_seconds`, and `launchdarkly_relay_events_received_bytes_total`. +Prometheus converts OpenTelemetry metric names by replacing dots with underscores, so the metrics will appear as `http_server_active_requests`, `http_server_request_duration_seconds`, `launchdarkly_relay_events_received_size_total`, etc. ### OpenTelemetry Collector diff --git a/internal/application/server.go b/internal/application/server.go index 65cb076a..02ac652f 100644 --- a/internal/application/server.go +++ b/internal/application/server.go @@ -39,7 +39,7 @@ func StartHTTPServer( } } - errCh := make(chan error) + errCh := make(chan error, 1) // Create a channel to listen for signals sigCh := make(chan os.Signal, 1) @@ -61,6 +61,7 @@ func StartHTTPServer( if err != nil && err != http.ErrServerClosed { errCh <- err } + close(errCh) }() // Handle graceful shutdown in a separate goroutine @@ -83,7 +84,6 @@ func StartHTTPServer( } else { loggers.Info("Server gracefully stopped") } - close(errCh) // Close the error channel after shutdown }() return srv, errCh diff --git a/internal/metrics/constants.go b/internal/metrics/constants.go index c8fa245f..9c62bd9c 100644 --- a/internal/metrics/constants.go +++ b/internal/metrics/constants.go @@ -5,19 +5,19 @@ import ( "time" "go.opentelemetry.io/otel/attribute" + semconv "go.opentelemetry.io/otel/semconv/v1.26.0" ) const ( // Metric instrument names. - connMeasureName = "launchdarkly.relay.connections" - requestMeasureName = "launchdarkly.relay.requests" - requestDurationMeasureName = "launchdarkly.relay.request.duration" - eventsReceivedBytesMeasureName = "launchdarkly.relay.events.received.bytes" - eventsSentCountMeasureName = "launchdarkly.relay.events.sent.count" - eventsSentBytesMeasureName = "launchdarkly.relay.events.sent.bytes" - eventsSentFailuresMeasureName = "launchdarkly.relay.events.sent.failures" - eventsSentDroppedMeasureName = "launchdarkly.relay.events.sent.dropped" - eventsSentPendingMeasureName = "launchdarkly.relay.events.sent.pending" + connMeasureName = "http.server.active_requests" + requestDurationMeasureName = "http.server.request.duration" + eventsReceivedMeasureName = "launchdarkly.relay.events.received.size" + eventsSentMeasureName = "launchdarkly.relay.events.sent" + eventsSentSizeMeasureName = "launchdarkly.relay.events.sent.size" + eventsSendErrorsMeasureName = "launchdarkly.relay.events.send.errors" + eventsDroppedMeasureName = "launchdarkly.relay.events.dropped" + eventsPendingMeasureName = "launchdarkly.relay.events.pending" defaultFlushInterval = time.Minute @@ -27,47 +27,69 @@ const ( ) var ( - relayIDAttrKey = attribute.Key("relayId") //nolint:gochecknoglobals - platformCategoryAttrKey = attribute.Key("platformCategory") //nolint:gochecknoglobals - userAgentAttrKey = attribute.Key("userAgent") //nolint:gochecknoglobals - sdkWrapperAttrKey = attribute.Key("sdkWrapper") //nolint:gochecknoglobals - routeAttrKey = attribute.Key("route") //nolint:gochecknoglobals - methodAttrKey = attribute.Key("method") //nolint:gochecknoglobals - envNameAttrKey = attribute.Key("env") //nolint:gochecknoglobals + relayIDAttrKey = attribute.Key("relay.id") //nolint:gochecknoglobals + platformCategoryAttrKey = attribute.Key("platform.category") //nolint:gochecknoglobals + userAgentAttrKey = attribute.Key("user_agent") //nolint:gochecknoglobals + sdkWrapperAttrKey = attribute.Key("sdk.wrapper") //nolint:gochecknoglobals + envNameAttrKey = attribute.Key("environment.name") //nolint:gochecknoglobals applicationIDAttrKey = attribute.Key("application.id") //nolint:gochecknoglobals applicationVersionAttrKey = attribute.Key("application.version") //nolint:gochecknoglobals - instanceIDAttrKey = attribute.Key("instanceId") //nolint:gochecknoglobals - statusCodeAttrKey = attribute.Key("statusCode") //nolint:gochecknoglobals + instanceIDAttrKey = attribute.Key("instance.id") //nolint:gochecknoglobals + + // OTEL HTTP semantic convention attribute keys (from semconv package) + httpRouteAttrKey = semconv.HTTPRouteKey //nolint:gochecknoglobals + httpRequestMethodAttrKey = semconv.HTTPRequestMethodKey //nolint:gochecknoglobals + httpResponseStatusAttrKey = semconv.HTTPResponseStatusCodeKey //nolint:gochecknoglobals + urlSchemeAttrKey = semconv.URLSchemeKey //nolint:gochecknoglobals + networkProtoVersionAttrKey = semconv.NetworkProtocolVersionKey //nolint:gochecknoglobals + errorTypeAttrKey = semconv.ErrorTypeKey //nolint:gochecknoglobals + statusCodeAttrKey = attribute.Key("status_code") //nolint:gochecknoglobals ) -// buildAttributes creates an OTel attribute set from base key-values plus per-request attributes. -// All string values should be pre-sanitized via sanitizeTagValue before calling this function. -func buildAttributes(baseKVs []attribute.KeyValue, platform, userAgent, sdkWrapper string) attribute.Set { - attrs := make([]attribute.KeyValue, len(baseKVs), len(baseKVs)+3) +// buildRequestAttributes creates an OTel attribute set for request metrics using semconv attribute names +// where applicable. All string values should be pre-sanitized via sanitizeTagValue before calling this function. +func buildRequestAttributes(baseKVs []attribute.KeyValue, platform, userAgent, sdkWrapper, route, method, urlScheme, applicationID, applicationVersion, instanceID string) attribute.Set { + attrs := make([]attribute.KeyValue, len(baseKVs), len(baseKVs)+9) copy(attrs, baseKVs) attrs = append(attrs, platformCategoryAttrKey.String(platform), userAgentAttrKey.String(userAgent), sdkWrapperAttrKey.String(sdkWrapper), + httpRouteAttrKey.String(route), + httpRequestMethodAttrKey.String(method), + urlSchemeAttrKey.String(urlScheme), + applicationIDAttrKey.String(applicationID), + applicationVersionAttrKey.String(applicationVersion), + instanceIDAttrKey.String(instanceID), ) return attribute.NewSet(attrs...) } -// buildRequestAttributes creates an OTel attribute set for request metrics (includes route and method). -// All string values should be pre-sanitized via sanitizeTagValue before calling this function. -func buildRequestAttributes(baseKVs []attribute.KeyValue, platform, userAgent, sdkWrapper, route, method, applicationID, applicationVersion, instanceID string) attribute.Set { - attrs := make([]attribute.KeyValue, len(baseKVs), len(baseKVs)+8) +// buildDurationAttributes creates an OTel attribute set for the http.server.request.duration histogram, +// including all semconv required/conditionally-required attributes. +func buildDurationAttributes(baseKVs []attribute.KeyValue, platform, userAgent, sdkWrapper, route, method, applicationID, applicationVersion, instanceID, urlScheme, protocolVersion, errorType string, statusCode int) attribute.Set { + attrs := make([]attribute.KeyValue, len(baseKVs), len(baseKVs)+14) copy(attrs, baseKVs) attrs = append(attrs, platformCategoryAttrKey.String(platform), userAgentAttrKey.String(userAgent), sdkWrapperAttrKey.String(sdkWrapper), - routeAttrKey.String(route), - methodAttrKey.String(method), + httpRouteAttrKey.String(route), + httpRequestMethodAttrKey.String(method), + urlSchemeAttrKey.String(urlScheme), applicationIDAttrKey.String(applicationID), applicationVersionAttrKey.String(applicationVersion), instanceIDAttrKey.String(instanceID), ) + if protocolVersion != "" { + attrs = append(attrs, networkProtoVersionAttrKey.String(protocolVersion)) + } + if statusCode > 0 { + attrs = append(attrs, httpResponseStatusAttrKey.Int(statusCode)) + } + if errorType != "" { + attrs = append(attrs, errorTypeAttrKey.String(errorType)) + } return attribute.NewSet(attrs...) } diff --git a/internal/metrics/measures.go b/internal/metrics/measures.go index 243e6507..ffcb03b0 100644 --- a/internal/metrics/measures.go +++ b/internal/metrics/measures.go @@ -2,7 +2,6 @@ package metrics import ( "context" - "strconv" "time" ldevents "github.com/launchdarkly/go-sdk-events/v3" @@ -13,7 +12,6 @@ import ( // Instruments holds the OTel metric instruments used for recording metrics. type Instruments struct { connections metric.Int64UpDownCounter // active connections (+1/-1) - requests metric.Int64Counter // cumulative HTTP requests requestDuration metric.Float64Histogram // request duration in seconds eventsReceivedBytes metric.Int64Counter // cumulative bytes of event data received eventsDropped metric.Int64Counter // cumulative count of events dropped due to capacity overflow @@ -27,7 +25,7 @@ type Instruments struct { // instruments should be incremented and what platform category to use. type Measure struct { recordConnections bool - recordRequests bool + recordDuration bool recordPolling bool platformCategory string } @@ -44,14 +42,14 @@ var ( // ServerConns is a Measure representing the current number of active stream connections from server-side SDKs. ServerConns = Measure{recordConnections: true, platformCategory: ServerPlatformCategory} - // BrowserRequests is a Measure representing the number of HTTP requests from browsers. - BrowserRequests = Measure{recordRequests: true, platformCategory: BrowserPlatformCategory} + // BrowserDuration is a Measure for recording request duration from browsers. + BrowserDuration = Measure{recordDuration: true, platformCategory: BrowserPlatformCategory} - // MobileRequests is a Measure representing the number of HTTP requests from mobile SDKs. - MobileRequests = Measure{recordRequests: true, platformCategory: MobilePlatformCategory} + // MobileDuration is a Measure for recording request duration from mobile SDKs. + MobileDuration = Measure{recordDuration: true, platformCategory: MobilePlatformCategory} - // ServerRequests is a Measure representing the number of HTTP requests from server-side SDKs. - ServerRequests = Measure{recordRequests: true, platformCategory: ServerPlatformCategory} + // ServerDuration is a Measure for recording request duration from server-side SDKs. + ServerDuration = Measure{recordDuration: true, platformCategory: ServerPlatformCategory} // ServerPollingRequests is a Measure representing the total number of polling style requests received from server-side SDKs. ServerPollingRequests = Measure{recordPolling: true, platformCategory: ServerPlatformCategory} @@ -70,41 +68,36 @@ func NewInstrumentsForTest(meter metric.Meter) (*Instruments, error) { if err != nil { return nil, err } - requests, err := meter.Int64Counter(requestMeasureName) - if err != nil { - return nil, err - } requestDuration, err := meter.Float64Histogram(requestDurationMeasureName) if err != nil { return nil, err } - eventsReceivedBytes, err := meter.Int64Counter(eventsReceivedBytesMeasureName) + eventsReceivedBytes, err := meter.Int64Counter(eventsReceivedMeasureName) if err != nil { return nil, err } - eventsDropped, err := meter.Int64Counter(eventsSentDroppedMeasureName) + eventsDropped, err := meter.Int64Counter(eventsDroppedMeasureName) if err != nil { return nil, err } - eventsSent, err := meter.Int64Counter(eventsSentCountMeasureName) + eventsSent, err := meter.Int64Counter(eventsSentMeasureName) if err != nil { return nil, err } - eventsFailedSend, err := meter.Int64Counter(eventsSentFailuresMeasureName) + eventsFailedSend, err := meter.Int64Counter(eventsSendErrorsMeasureName) if err != nil { return nil, err } - eventsBytesSent, err := meter.Int64Counter(eventsSentBytesMeasureName) + eventsBytesSent, err := meter.Int64Counter(eventsSentSizeMeasureName) if err != nil { return nil, err } - pendingEvents, err := meter.Int64Gauge(eventsSentPendingMeasureName) + pendingEvents, err := meter.Int64Gauge(eventsPendingMeasureName) if err != nil { return nil, err } return &Instruments{ connections: connections, - requests: requests, requestDuration: requestDuration, eventsReceivedBytes: eventsReceivedBytes, eventsDropped: eventsDropped, @@ -124,6 +117,11 @@ type RequestInfo struct { ApplicationID string ApplicationVersion string InstanceID string + // Semconv fields populated after handler execution + StatusCode int + URLScheme string + ProtocolVersion string + ErrorType string } func (ri RequestInfo) sanitized() (ua, wrapper, route, method, appID, appVersion, instanceID string) { @@ -145,7 +143,7 @@ func WithGauge(em *EnvironmentManager, instruments *Instruments, ri RequestInfo, } ua, wrapper, route, method, appID, appVersion, instanceID := ri.sanitized() - attrs := buildRequestAttributes(em.envKVs, measure.platformCategory, ua, wrapper, route, method, appID, appVersion, instanceID) + attrs := buildRequestAttributes(em.envKVs, measure.platformCategory, ua, wrapper, route, method, ri.URLScheme, appID, appVersion, instanceID) if instruments != nil { instruments.connections.Add(context.Background(), 1, metric.WithAttributeSet(attrs)) @@ -160,54 +158,34 @@ func WithGauge(em *EnvironmentManager, instruments *Instruments, ri RequestInfo, f() } -// WithCount runs a function and records a single-unit increment for the specified metric. -func WithCount(em *EnvironmentManager, instruments *Instruments, ri RequestInfo, f func(), measure Measure) { - if em == nil { - f() - return - } - - ua, wrapper, _, _, _, _, _ := ri.sanitized() - attrs := buildAttributes(em.envKVs, measure.platformCategory, ua, wrapper) - - if measure.recordRequests && instruments != nil { - instruments.requests.Add(context.Background(), 1, metric.WithAttributeSet(attrs)) - } - if measure.recordPolling && em.collector != nil { +// WithCount runs a function and records polling metrics if applicable. +func WithCount(em *EnvironmentManager, ri RequestInfo, f func(), measure Measure) { + if em != nil && measure.recordPolling && em.collector != nil { + ua, wrapper, _, _, _, _, _ := ri.sanitized() em.collector.RecordPollingRequest(measure.platformCategory, ua, wrapper) } f() } -// WithRouteCount records a route hit for the specified metric. -func WithRouteCount(ctx context.Context, em *EnvironmentManager, instruments *Instruments, ri RequestInfo, f func(), measure Measure) { - if em != nil && instruments != nil && measure.recordRequests { - ua, wrapper, route, method, appID, appVersion, instanceID := ri.sanitized() - attrs := buildRequestAttributes(em.envKVs, measure.platformCategory, ua, wrapper, route, method, appID, appVersion, instanceID) - instruments.requests.Add(ctx, 1, metric.WithAttributeSet(attrs)) - } - - f() -} - // RecordEventsReceivedBytes records the number of event bytes received. func RecordEventsReceivedBytes(ctx context.Context, instruments *Instruments, em *EnvironmentManager, platformCategory string, ri RequestInfo, bytes int64) { if em == nil || instruments == nil || bytes <= 0 { return } ua, wrapper, route, method, appID, appVersion, instanceID := ri.sanitized() - attrs := buildRequestAttributes(em.envKVs, platformCategory, ua, wrapper, route, method, appID, appVersion, instanceID) + attrs := buildRequestAttributes(em.envKVs, platformCategory, ua, wrapper, route, method, ri.URLScheme, appID, appVersion, instanceID) instruments.eventsReceivedBytes.Add(ctx, bytes, metric.WithAttributeSet(attrs)) } // RecordRequestDuration records a request duration measurement with the given attributes. +// Duration is recorded in seconds per OTEL HTTP semantic conventions. func RecordRequestDuration(ctx context.Context, instruments *Instruments, em *EnvironmentManager, ri RequestInfo, duration time.Duration, measure Measure) { - if em == nil || instruments == nil || !measure.recordRequests { + if em == nil || instruments == nil || !measure.recordDuration { return } ua, wrapper, route, method, appID, appVersion, instanceID := ri.sanitized() - attrs := buildRequestAttributes(em.envKVs, measure.platformCategory, ua, wrapper, route, method, appID, appVersion, instanceID) + attrs := buildDurationAttributes(em.envKVs, measure.platformCategory, ua, wrapper, route, method, appID, appVersion, instanceID, ri.URLScheme, ri.ProtocolVersion, ri.ErrorType, ri.StatusCode) instruments.requestDuration.Record(ctx, duration.Seconds(), metric.WithAttributeSet(attrs)) } @@ -261,7 +239,7 @@ func (r *EventMetricsRecorder) RecordEventsFailedSend(count int, metadata ldeven } kvs := make([]attribute.KeyValue, len(r.envKVs), len(r.envKVs)+1) copy(kvs, r.envKVs) - kvs = append(kvs, statusCodeAttrKey.String(strconv.Itoa(metadata.StatusCode))) + kvs = append(kvs, statusCodeAttrKey.Int(metadata.StatusCode)) attrs := attribute.NewSet(kvs...) r.instruments.eventsFailedSend.Add(context.Background(), int64(count), metric.WithAttributeSet(attrs)) } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 23d4c48b..07e42bf1 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -86,6 +86,10 @@ func NewManager( return nil, err } opts = append(opts, sdkmetric.WithResource(res)) + opts = append(opts, sdkmetric.WithView(sdkmetric.NewView( + sdkmetric.Instrument{Name: requestDurationMeasureName}, + sdkmetric.Stream{Aggregation: sdkmetric.AggregationBase2ExponentialHistogram{MaxSize: 160, MaxScale: 20}}, + ))) meterProvider = sdkmetric.NewMeterProvider(opts...) meter = meterProvider.Meter("ld-relay") if err := runtime.Start(runtime.WithMeterProvider(meterProvider)); err != nil { @@ -96,31 +100,33 @@ func NewManager( } connections, _ := meter.Int64UpDownCounter(connMeasureName, - otelmetric.WithDescription("current number of connections")) - requests, _ := meter.Int64Counter(requestMeasureName, - otelmetric.WithDescription("number of hits to a route")) + otelmetric.WithDescription("Number of active HTTP server requests"), + otelmetric.WithUnit("{request}")) requestDuration, _ := meter.Float64Histogram(requestDurationMeasureName, - otelmetric.WithDescription("request duration in seconds"), + otelmetric.WithDescription("Duration of HTTP server requests"), otelmetric.WithUnit("s")) - eventsReceivedBytes, _ := meter.Int64Counter(eventsReceivedBytesMeasureName, - otelmetric.WithDescription("cumulative bytes of event data received"), + eventsReceivedBytes, _ := meter.Int64Counter(eventsReceivedMeasureName, + otelmetric.WithDescription("Bytes of event data received"), otelmetric.WithUnit("By")) - eventsDropped, _ := meter.Int64Counter(eventsSentDroppedMeasureName, - otelmetric.WithDescription("cumulative count of events dropped due to capacity overflow")) - eventsSent, _ := meter.Int64Counter(eventsSentCountMeasureName, - otelmetric.WithDescription("cumulative count of events successfully sent")) - eventsFailedSend, _ := meter.Int64Counter(eventsSentFailuresMeasureName, - otelmetric.WithDescription("cumulative count of events that failed to send after all retries")) - eventsBytesSent, _ := meter.Int64Counter(eventsSentBytesMeasureName, - otelmetric.WithDescription("cumulative bytes of event payloads successfully sent"), + eventsDropped, _ := meter.Int64Counter(eventsDroppedMeasureName, + otelmetric.WithDescription("Events dropped due to capacity overflow"), + otelmetric.WithUnit("{event}")) + eventsSent, _ := meter.Int64Counter(eventsSentMeasureName, + otelmetric.WithDescription("Events successfully sent"), + otelmetric.WithUnit("{event}")) + eventsFailedSend, _ := meter.Int64Counter(eventsSendErrorsMeasureName, + otelmetric.WithDescription("Events that failed to send after all retries"), + otelmetric.WithUnit("{event}")) + eventsBytesSent, _ := meter.Int64Counter(eventsSentSizeMeasureName, + otelmetric.WithDescription("Bytes of event payloads successfully sent"), otelmetric.WithUnit("By")) - pendingEvents, _ := meter.Int64Gauge(eventsSentPendingMeasureName, - otelmetric.WithDescription("current number of events buffered in the queue")) + pendingEvents, _ := meter.Int64Gauge(eventsPendingMeasureName, + otelmetric.WithDescription("Events buffered in the queue"), + otelmetric.WithUnit("{event}")) instruments := &Instruments{ connections: connections, - requests: requests, requestDuration: requestDuration, eventsReceivedBytes: eventsReceivedBytes, eventsDropped: eventsDropped, diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 05c0e20b..18a7a9f6 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -7,6 +7,8 @@ import ( "github.com/launchdarkly/ld-relay/v9/config" + ldevents "github.com/launchdarkly/go-sdk-events/v3" + "github.com/launchdarkly/go-sdk-common/v3/ldlog" "github.com/stretchr/testify/assert" @@ -31,7 +33,6 @@ func TestNewManagerReturnsInstruments(t *testing.T) { instruments := manager.GetInstruments() assert.NotNil(t, instruments) assert.NotNil(t, instruments.connections) - assert.NotNil(t, instruments.requests) assert.NotNil(t, instruments.requestDuration) assert.NotNil(t, instruments.eventsReceivedBytes) } @@ -129,44 +130,21 @@ func TestConnectionMetrics(t *testing.T) { } } -func TestWithRouteCount(t *testing.T) { +func TestRecordRequestDuration(t *testing.T) { testWithOTel(t, func(p testWithOTelParams) { - WithRouteCount(context.Background(), p.env, p.instruments, RequestInfo{UserAgent: userAgentValue, Route: "someRoute", Method: "GET"}, func() {}, ServerRequests) + RecordRequestDuration(context.Background(), p.instruments, p.env, RequestInfo{UserAgent: userAgentValue, Route: "someRoute", Method: "GET"}, 50*time.Millisecond, ServerDuration) rm, err := p.collectMetrics() require.NoError(t, err) - m := findMetric(rm, requestMeasureName) - require.NotNil(t, m, "requests metric not found") - - // Verify the data has a data point with route and method attributes - sum, ok := m.Data.(metricdata.Sum[int64]) - require.True(t, ok, "expected Sum[int64] data") - require.NotEmpty(t, sum.DataPoints) - found := false - for _, dp := range sum.DataPoints { - routeVal, routeOK := dp.Attributes.Value(routeAttrKey) - methodVal, methodOK := dp.Attributes.Value(methodAttrKey) - if routeOK && methodOK && routeVal.AsString() == "someRoute" && methodVal.AsString() == "GET" { - assert.Equal(t, int64(1), dp.Value) - found = true - } - } - assert.True(t, found, "expected data point with route=someRoute, method=GET") - - // Verify RecordRequestDuration records to the histogram - RecordRequestDuration(context.Background(), p.instruments, p.env, RequestInfo{UserAgent: userAgentValue, Route: "someRoute", Method: "GET"}, 50*time.Millisecond, ServerRequests) - - rm, err = p.collectMetrics() - require.NoError(t, err) dm := findMetric(rm, requestDurationMeasureName) require.NotNil(t, dm, "request duration metric not found") hist, ok := dm.Data.(metricdata.Histogram[float64]) require.True(t, ok, "expected Histogram[float64] data") require.NotEmpty(t, hist.DataPoints) - found = false + found := false for _, dp := range hist.DataPoints { - routeVal, routeOK := dp.Attributes.Value(routeAttrKey) - methodVal, methodOK := dp.Attributes.Value(methodAttrKey) + routeVal, routeOK := dp.Attributes.Value(httpRouteAttrKey) + methodVal, methodOK := dp.Attributes.Value(httpRequestMethodAttrKey) if routeOK && methodOK && routeVal.AsString() == "someRoute" && methodVal.AsString() == "GET" { assert.Equal(t, uint64(1), dp.Count) assert.InDelta(t, 0.05, dp.Sum, 0.01, "expected ~50ms duration") @@ -183,7 +161,7 @@ func TestRecordEventsReceivedBytes(t *testing.T) { rm, err := p.collectMetrics() require.NoError(t, err) - m := findMetric(rm, eventsReceivedBytesMeasureName) + m := findMetric(rm, eventsReceivedMeasureName) require.NotNil(t, m, "events received bytes metric not found") sum, ok := m.Data.(metricdata.Sum[int64]) require.True(t, ok, "expected Sum[int64] data") @@ -213,20 +191,194 @@ func TestEventMetricsRecorderViaTestHelper(t *testing.T) { rm, err := p.collectMetrics() require.NoError(t, err) - droppedMetric := findMetric(rm, eventsSentDroppedMeasureName) + droppedMetric := findMetric(rm, eventsDroppedMeasureName) require.NotNil(t, droppedMetric, "events dropped metric not found") - sentMetric := findMetric(rm, eventsSentCountMeasureName) + sentMetric := findMetric(rm, eventsSentMeasureName) require.NotNil(t, sentMetric, "events sent metric not found") - bytesMetric := findMetric(rm, eventsSentBytesMeasureName) + bytesMetric := findMetric(rm, eventsSentSizeMeasureName) require.NotNil(t, bytesMetric, "events bytes sent metric not found") - pendingMetric := findMetric(rm, eventsSentPendingMeasureName) + pendingMetric := findMetric(rm, eventsPendingMeasureName) require.NotNil(t, pendingMetric, "events pending metric not found") }) } +func TestRecordRequestDurationWithAllAttributes(t *testing.T) { + testWithOTel(t, func(p testWithOTelParams) { + ri := RequestInfo{ + UserAgent: userAgentValue, + Route: "/sdk/eval", + Method: "GET", + URLScheme: "https", + ProtocolVersion: "1.1", + StatusCode: 200, + ErrorType: "", + } + RecordRequestDuration(context.Background(), p.instruments, p.env, ri, 100*time.Millisecond, ServerDuration) + + rm, err := p.collectMetrics() + require.NoError(t, err) + dm := findMetric(rm, requestDurationMeasureName) + require.NotNil(t, dm, "request duration metric not found") + hist, ok := dm.Data.(metricdata.Histogram[float64]) + require.True(t, ok, "expected Histogram[float64] data") + require.NotEmpty(t, hist.DataPoints) + + dp := hist.DataPoints[0] + assert.Equal(t, uint64(1), dp.Count) + + schemeVal, ok := dp.Attributes.Value(urlSchemeAttrKey) + assert.True(t, ok, "url.scheme attribute missing") + assert.Equal(t, "https", schemeVal.AsString()) + + protoVal, ok := dp.Attributes.Value(networkProtoVersionAttrKey) + assert.True(t, ok, "network.protocol.version attribute missing") + assert.Equal(t, "1.1", protoVal.AsString()) + + statusVal, ok := dp.Attributes.Value(httpResponseStatusAttrKey) + assert.True(t, ok, "http.response.status_code attribute missing") + assert.Equal(t, int64(200), statusVal.AsInt64()) + }) +} + +func TestRecordRequestDurationWithErrorType(t *testing.T) { + testWithOTel(t, func(p testWithOTelParams) { + ri := RequestInfo{ + UserAgent: userAgentValue, + Route: "/sdk/eval", + Method: "GET", + URLScheme: "http", + StatusCode: 500, + ErrorType: "500", + } + RecordRequestDuration(context.Background(), p.instruments, p.env, ri, 50*time.Millisecond, ServerDuration) + + rm, err := p.collectMetrics() + require.NoError(t, err) + dm := findMetric(rm, requestDurationMeasureName) + require.NotNil(t, dm) + hist, ok := dm.Data.(metricdata.Histogram[float64]) + require.True(t, ok) + require.NotEmpty(t, hist.DataPoints) + + dp := hist.DataPoints[0] + errVal, ok := dp.Attributes.Value(errorTypeAttrKey) + assert.True(t, ok, "error.type attribute missing") + assert.Equal(t, "500", errVal.AsString()) + + statusVal, ok := dp.Attributes.Value(httpResponseStatusAttrKey) + assert.True(t, ok, "http.response.status_code attribute missing") + assert.Equal(t, int64(500), statusVal.AsInt64()) + }) +} + +func TestRecordRequestDurationSkipsWhenMeasureDoesNotRecordDuration(t *testing.T) { + testWithOTel(t, func(p testWithOTelParams) { + // ServerConns has recordDuration: false + RecordRequestDuration(context.Background(), p.instruments, p.env, RequestInfo{UserAgent: userAgentValue}, 50*time.Millisecond, ServerConns) + + rm, err := p.collectMetrics() + require.NoError(t, err) + dm := findMetric(rm, requestDurationMeasureName) + if dm != nil { + hist, ok := dm.Data.(metricdata.Histogram[float64]) + if ok { + assert.Empty(t, hist.DataPoints, "expected no duration data points for non-duration measure") + } + } + }) +} + +func TestRecordEventsFailedSend(t *testing.T) { + testWithOTel(t, func(p testWithOTelParams) { + recorder := p.env.NewEventMetricsRecorder(p.instruments) + + recorder.RecordEventsFailedSend(3, ldevents.EventSendFailureMetadata{StatusCode: 429}) + + rm, err := p.collectMetrics() + require.NoError(t, err) + m := findMetric(rm, eventsSendErrorsMeasureName) + require.NotNil(t, m, "events send errors metric not found") + + sum, ok := m.Data.(metricdata.Sum[int64]) + require.True(t, ok, "expected Sum[int64] data") + require.NotEmpty(t, sum.DataPoints) + + dp := sum.DataPoints[0] + assert.Equal(t, int64(3), dp.Value) + + // Verify the status_code attribute is an int, not a string + statusVal, ok := dp.Attributes.Value(statusCodeAttrKey) + assert.True(t, ok, "status_code attribute missing") + assert.Equal(t, int64(429), statusVal.AsInt64()) + }) +} + +func TestRecordEventsFailedSendSkipsZeroCount(t *testing.T) { + testWithOTel(t, func(p testWithOTelParams) { + recorder := p.env.NewEventMetricsRecorder(p.instruments) + + recorder.RecordEventsFailedSend(0, ldevents.EventSendFailureMetadata{StatusCode: 500}) + + rm, err := p.collectMetrics() + require.NoError(t, err) + m := findMetric(rm, eventsSendErrorsMeasureName) + if m != nil { + sum, ok := m.Data.(metricdata.Sum[int64]) + if ok { + for _, dp := range sum.DataPoints { + assert.Equal(t, int64(0), dp.Value, "expected no data recorded for zero count") + } + } + } + }) +} + +func TestWithCountRecordsPolling(t *testing.T) { + publisher := newTestEventsPublisher() + + manager, err := NewManager(config.OpenTelemetryConfig{}, time.Millisecond*10, ldlog.NewDisabledLoggers()) + require.NoError(t, err) + defer manager.Close() + + env, err := manager.AddEnvironment("polling-test", publisher) + require.NoError(t, err) + + called := false + WithCount(env, RequestInfo{UserAgent: "test-agent"}, func() { + called = true + }, ServerPollingRequests) + + assert.True(t, called, "function should have been called") + + env.FlushEventsExporter() + metricsEvent := publisher.expectMetricsEvent(t, time.Second) + require.Len(t, metricsEvent.PollingCounts, 1) + assert.Equal(t, int64(1), metricsEvent.PollingCounts[0].Count) + assert.Equal(t, ServerPlatformCategory, metricsEvent.PollingCounts[0].PlatformCategory) +} + +func TestWithCountCallsFunctionWhenEnvNil(t *testing.T) { + called := false + WithCount(nil, RequestInfo{UserAgent: "test-agent"}, func() { + called = true + }, ServerPollingRequests) + assert.True(t, called, "function should have been called even with nil env") +} + +func TestWithCountCallsFunctionForNonPollingMeasure(t *testing.T) { + testWithOTel(t, func(p testWithOTelParams) { + called := false + // ServerDuration has recordPolling: false, so no polling metric should be recorded + WithCount(p.env, RequestInfo{UserAgent: userAgentValue}, func() { + called = true + }, ServerDuration) + assert.True(t, called, "function should have been called") + }) +} + func TestSanitizeTagValue(t *testing.T) { assert.Equal(t, "abc", sanitizeTagValue("abc")) assert.Equal(t, "not-provided", sanitizeTagValue("")) diff --git a/internal/middleware/metrics_middleware.go b/internal/middleware/metrics_middleware.go index 0cfdaf15..4480e853 100644 --- a/internal/middleware/metrics_middleware.go +++ b/internal/middleware/metrics_middleware.go @@ -1,8 +1,10 @@ package middleware import ( + "fmt" "io" "net/http" + "strings" "sync/atomic" "time" @@ -14,6 +16,35 @@ import ( "github.com/gorilla/mux" ) +// statusRecorder wraps http.ResponseWriter to capture the response status code. +type statusRecorder struct { + http.ResponseWriter + statusCode int + written bool +} + +func (sr *statusRecorder) WriteHeader(code int) { + if !sr.written { + sr.statusCode = code + sr.written = true + } + sr.ResponseWriter.WriteHeader(code) +} + +func (sr *statusRecorder) Write(b []byte) (int, error) { + if !sr.written { + sr.statusCode = 200 + sr.written = true + } + return sr.ResponseWriter.Write(b) +} + +func (sr *statusRecorder) Flush() { + if f, ok := sr.ResponseWriter.(http.Flusher); ok { + f.Flush() + } +} + // countingReader wraps an io.ReadCloser and counts the bytes read. type countingReader struct { reader io.ReadCloser @@ -51,6 +82,23 @@ func requestInfoFromHTTP(req *http.Request) metrics.RequestInfo { route, _ = r.GetPathTemplate() } appID, appVersion := parseApplicationTags(req) + + urlScheme := "http" + if req.TLS != nil { + urlScheme = "https" + } + + // Format per OTEL semconv: "1.0", "1.1", "2", "3" + // HTTP/2 and HTTP/3 use just the major version; HTTP/1.x includes the minor version. + protocolVersion := "" + if req.ProtoMajor > 0 { + if req.ProtoMajor == 1 { + protocolVersion = fmt.Sprintf("%d.%d", req.ProtoMajor, req.ProtoMinor) + } else { + protocolVersion = fmt.Sprintf("%d", req.ProtoMajor) + } + } + return metrics.RequestInfo{ UserAgent: getUserAgent(req), SDKWrapper: getSDKWrapper(req), @@ -59,6 +107,8 @@ func requestInfoFromHTTP(req *http.Request) metrics.RequestInfo { ApplicationID: appID, ApplicationVersion: appVersion, InstanceID: getInstanceID(req), + URLScheme: urlScheme, + ProtocolVersion: protocolVersion, } } @@ -66,7 +116,7 @@ func withCount(handler http.Handler, measure metrics.Measure) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { env := GetEnvContextInfo(req.Context()).Env ri := requestInfoFromHTTP(req) - metrics.WithCount(env.GetMetricsEnv(), getInstruments(env), ri, func() { + metrics.WithCount(env.GetMetricsEnv(), ri, func() { handler.ServeHTTP(w, req) }, measure) }) @@ -133,35 +183,38 @@ func CountClientConns(handler http.Handler) http.Handler { }) } -// DynamicRequestMetrics is a middleware function for FDv2 client-side endpoints that dynamically -// determines the request metrics based on the credential type. -func DynamicRequestMetrics() mux.MiddlewareFunc { +// DynamicDurationMetrics is a middleware function for FDv2 client-side endpoints that dynamically +// determines the duration metric platform based on the credential type. +func DynamicDurationMetrics() mux.MiddlewareFunc { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { cred := GetEnvContextInfo(req.Context()).Credential var measure metrics.Measure if _, ok := cred.(config.MobileKey); ok { - measure = metrics.MobileRequests + measure = metrics.MobileDuration } else { - measure = metrics.BrowserRequests + measure = metrics.BrowserDuration } - RequestMetrics(measure)(next).ServeHTTP(w, req) + DurationMetrics(measure)(next).ServeHTTP(w, req) }) } } -// RequestMetrics is a middleware function that increments the request counter -// and records the request duration for the specified metric. -func RequestMetrics(measure metrics.Measure) mux.MiddlewareFunc { +// DurationMetrics is a middleware function that records the request duration for the specified metric. +func DurationMetrics(measure metrics.Measure) mux.MiddlewareFunc { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { env := GetEnvContextInfo(req.Context()).Env ri := requestInfoFromHTTP(req) + recorder := &statusRecorder{ResponseWriter: w, statusCode: 200} start := time.Now() - metrics.WithRouteCount(req.Context(), env.GetMetricsEnv(), getInstruments(env), ri, func() { - next.ServeHTTP(w, req) - }, measure) - if w.Header().Get("X-Accel-Buffering") != "no" { + next.ServeHTTP(recorder, req) + // Don't record duration for streaming responses — their lifetime is unbounded + if !strings.HasPrefix(strings.ToLower(recorder.Header().Get("Content-Type")), "text/event-stream") { + ri.StatusCode = recorder.statusCode + if recorder.statusCode >= 500 { + ri.ErrorType = fmt.Sprintf("%d", recorder.statusCode) + } metrics.RecordRequestDuration(req.Context(), getInstruments(env), env.GetMetricsEnv(), ri, time.Since(start), measure) } }) diff --git a/internal/middleware/metrics_middleware_test.go b/internal/middleware/metrics_middleware_test.go index ce9a11cd..76443d77 100644 --- a/internal/middleware/metrics_middleware_test.go +++ b/internal/middleware/metrics_middleware_test.go @@ -106,63 +106,22 @@ func testCountConnections(t *testing.T, countFn func(http.Handler) http.Handler, countFn(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // While inside the handler, connections should be active rm := p.collectMetrics(t) - connMetric := st.FindMetricByName(rm, "launchdarkly.relay.connections") + connMetric := st.FindMetricByName(rm, "http.server.active_requests") require.NotNil(t, connMetric, "connections metric not found") assertMetricHasValue(t, connMetric, p.envName, category, 1) })).ServeHTTP(rr, req) // After handler returns, connection gauge should be 0 rm := p.collectMetrics(t) - connMetric := st.FindMetricByName(rm, "launchdarkly.relay.connections") + connMetric := st.FindMetricByName(rm, "http.server.active_requests") require.NotNil(t, connMetric, "connections metric not found") assertMetricHasValue(t, connMetric, p.envName, category, 0) }) } -func TestCountRequests(t *testing.T) { - t.Run("browser", func(t *testing.T) { - testCountRequests(t, metrics.BrowserRequests, "browser") - }) - t.Run("mobile", func(t *testing.T) { - testCountRequests(t, metrics.MobileRequests, "mobile") - }) - t.Run("server", func(t *testing.T) { - testCountRequests(t, metrics.ServerRequests, "server") - }) -} - -func testCountRequests(t *testing.T, measure metrics.Measure, category string) { - // We need to build a router here because RequestMetrics expects mux.CurrentRoute() to work. - router := mux.NewRouter() - router.Use(RequestMetrics(measure)) - router.Handle("/test-route", nullHandler()).Methods("GET") - - metricsMiddlewareTest(t, func(p metricsMiddlewareTestParams) { - makeRequest := func() *http.Request { - req, _ := http.NewRequest("GET", "/test-route", nil) - req.Header.Set("User-Agent", metricsTestUserAgent) - return req.WithContext(WithEnvContextInfo(req.Context(), EnvContextInfo{Env: p.env})) - } - - router.ServeHTTP(httptest.NewRecorder(), makeRequest()) - - rm := p.collectMetrics(t) - reqMetric := st.FindMetricByName(rm, "launchdarkly.relay.requests") - require.NotNil(t, reqMetric, "requests metric not found") - assertMetricHasValue(t, reqMetric, p.envName, category, 1) - - router.ServeHTTP(httptest.NewRecorder(), makeRequest()) - - rm = p.collectMetrics(t) - reqMetric = st.FindMetricByName(rm, "launchdarkly.relay.requests") - require.NotNil(t, reqMetric, "requests metric not found") - assertMetricHasValue(t, reqMetric, p.envName, category, 2) - }) -} - func TestRequestDuration(t *testing.T) { router := mux.NewRouter() - router.Use(RequestMetrics(metrics.ServerRequests)) + router.Use(DurationMetrics(metrics.ServerDuration)) router.Handle("/test-route", nullHandler()).Methods("GET") metricsMiddlewareTest(t, func(p metricsMiddlewareTestParams) { @@ -172,7 +131,7 @@ func TestRequestDuration(t *testing.T) { router.ServeHTTP(httptest.NewRecorder(), req) rm := p.collectMetrics(t) - durMetric := st.FindMetricByName(rm, "launchdarkly.relay.request.duration") + durMetric := st.FindMetricByName(rm, "http.server.request.duration") require.NotNil(t, durMetric, "request duration metric not found") hist, ok := durMetric.Data.(metricdata.Histogram[float64]) require.True(t, ok, "expected Histogram[float64] data") @@ -198,7 +157,7 @@ func TestEventBytesMetrics(t *testing.T) { router.ServeHTTP(httptest.NewRecorder(), req) rm := p.collectMetrics(t) - bytesMetric := st.FindMetricByName(rm, "launchdarkly.relay.events.received.bytes") + bytesMetric := st.FindMetricByName(rm, "launchdarkly.relay.events.received.size") require.NotNil(t, bytesMetric, "events received bytes metric not found") assertMetricHasValue(t, bytesMetric, p.envName, "server", 35) }) @@ -238,8 +197,8 @@ func assertMetricHasValue(t *testing.T, m *metricdata.Metrics, envName, platform require.True(t, ok, "expected Sum[int64] data for %s", m.Name) found := false for _, dp := range sum.DataPoints { - platVal, platOK := dp.Attributes.Value(attribute.Key("platformCategory")) - envVal, envOK := dp.Attributes.Value(attribute.Key("env")) + platVal, platOK := dp.Attributes.Value(attribute.Key("platform.category")) + envVal, envOK := dp.Attributes.Value(attribute.Key("environment.name")) if platOK && envOK && platVal.AsString() == platform && envVal.AsString() == envName { assert.Equal(t, expected, dp.Value, "unexpected value for %s (platform=%s, env=%s)", m.Name, platform, envName) found = true diff --git a/relay/relay_routes.go b/relay/relay_routes.go index 0323c665..bb44122f 100644 --- a/relay/relay_routes.go +++ b/relay/relay_routes.go @@ -61,7 +61,7 @@ func (r *Relay) makeRouter() *mux.Router { jsClientSelector, // selects an environment based on the client-side ID in the URL middleware.CORS, // must apply this after jsClientSelector because the CORS headers can be environment-specific middleware.TrackUsageActivity(metrics.BrowserPlatformCategory), - middleware.RequestMetrics(metrics.BrowserRequests), + middleware.DurationMetrics(metrics.BrowserDuration), ) } @@ -79,7 +79,7 @@ func (r *Relay) makeRouter() *mux.Router { serverSideMiddlewareStack := middleware.Chain( sdkKeySelector, middleware.TrackUsageActivity(metrics.ServerPlatformCategory), - middleware.RequestMetrics(metrics.ServerRequests), + middleware.DurationMetrics(metrics.ServerDuration), ) sdkRouter := router.PathPrefix("/sdk/").Subrouter() @@ -102,7 +102,7 @@ func (r *Relay) makeRouter() *mux.Router { clientSideFDv2EnvAuth, middleware.CORS, middleware.DynamicTrackUsageActivity(), - middleware.DynamicRequestMetrics(), + middleware.DynamicDurationMetrics(), ) clientSideFDv2StreamRouter.Use(clientSideFDv2StreamMiddleware, middleware.Streaming) clientSideFDv2PingHandler := pingStreamHandlerWithContextV2(r.mobileStreamProvider, r.jsClientStreamProvider) @@ -115,7 +115,7 @@ func (r *Relay) makeRouter() *mux.Router { clientSideFDv2EnvAuth, middleware.CORS, middleware.DynamicTrackUsageActivity(), - middleware.DynamicRequestMetrics(), + middleware.DynamicDurationMetrics(), ) clientSideFDv2PollRouter.Use(clientSideFDv2PollMiddleware) clientSideFDv2PollRouter.Handle("/{context}", middleware.DynamicPollingRequestCount(http.HandlerFunc(pollEvalHandlerV2))).Methods("GET", "OPTIONS") @@ -138,7 +138,7 @@ func (r *Relay) makeRouter() *mux.Router { mobileMiddlewareStack := middleware.Chain( mobileKeySelector, middleware.TrackUsageActivity(metrics.MobilePlatformCategory), - middleware.RequestMetrics(metrics.MobileRequests)) + middleware.DurationMetrics(metrics.MobileDuration)) msdkRouter := router.PathPrefix("/msdk/").Subrouter() msdkRouter.Use(mobileMiddlewareStack)