NVIDIA · kristinapathak · Jul 21, 2026 · Jul 20, 2026
@@ -61,6 +61,6 @@ WORKDIR /app
 COPY --from=builder /app/bin/byoo-otel-collector .
 COPY --from=builder-otelcol /app/output/otelcol-contrib .
 
-EXPOSE 14357/tcp 14358/tcp 13133/tcp 18888/tcp 19090/tcp
+EXPOSE 14357/tcp 14358/tcp 13133/tcp 18888/tcp 19090/tcp 19091/tcp
 
 ENTRYPOINT ["/app/byoo-otel-collector"]
@@ -104,6 +104,7 @@ Exposed Ports:
 - 14358: OTLP HTTP receiver
 - 13133: `/health?verbose` endpoint to get detailed health status of collector (healthcheck v2 extension)
 - 19090: `/metrics` endpoint for the byoo-otel-collector metrics
+- 19091: `/metrics` endpoint for optional SRE user metrics
 
 ### nvcf-otel-collector Image
 
@@ -128,6 +129,9 @@ Chunking is disabled by default. Configure it with:
 - `BYOO_LOG_CHUNK_MAX_BODY_BYTES`: maximum log body size in bytes before chunking. `0` disables the processor. Enabled values must be at least `4` bytes so chunks can preserve UTF-8 rune boundaries. Use `983040` bytes for normal BYOO deployments to leave room for log attributes and exporter envelope overhead under a `1000000` byte backend entry limit.
 - `BYOO_LOG_CHUNK_DRY_RUN`: records oversized-log metrics and warnings without mutating log payloads. Dry-run metric datapoints use `mode=dry_run`.
 - `BYOO_LOG_EXPORTER_BATCH_MAX_SIZE_BYTES`: serialized log export request batch size used for exporterhelper byte splitting. `0` or unset uses the default `1000000` bytes.
+- `BYOO_SRE_METRICS_ENABLED`: enables an additional OTLP-only metrics pipeline that exposes filtered user metrics through a Prometheus exporter on port `19091`. Disabled by default.
+- `BYOO_SRE_METRICS_FILTER_CONFIG`: optional YAML filter processor config for the SRE metrics pipeline. If unset, the default drops every metric except `BpsInstrument`, `FpsInstrument`, `RtdInstrument`, and `StageOpenDuration`, and drops datapoints/resources explicitly labeled `sre_metrics_enabled=false` or `sre_enabled_metrics=false`.
+- `BYOO_CUSTOMER_METRICS_DROP_LABELS`: comma-separated resource attribute names removed from the customer metrics pipeline when SRE metrics are enabled. If unset, defaults to `sre_metrics_enabled,sre_enabled_metrics`.
 
 When chunking is enabled, each emitted chunk preserves the original log metadata and adds these attributes so chunks can be grouped in the backend:
 

@@ -0,0 +1,298 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: ${env:OTEL_POD_IP:-0.0.0.0}:14357
+      http:
+        endpoint: ${env:OTEL_POD_IP:-0.0.0.0}:14358
+  prometheus:
+    config:
+      scrape_configs:
+        - honor_labels: true
+          job_name: nvcf-worker
+          metric_relabel_configs:
+            - action: labelkeep
+              regex: (__name__|container|job|pod|error_code|le)
+          metrics_path: /federate
+          params:
+            match[]:
+              - '{job=~"nvcf-worker", namespace="nvcf-backend", pod=~".*fake-instance-id.*", container=~"utils", __name__=~"(nvcf_worker_service_request_total|nvcf_worker_service_response_total|nvcf_worker_service_stream_streaming_app_ready|nvcf_worker_service_stream_streaming_app_ready_timestamp|nvcf_worker_service_stream_session_duration_seconds_bucket|nvcf_worker_service_stream_session_duration_seconds_count|nvcf_worker_service_stream_session_duration_seconds_sum|nvcf_worker_service_inference_request_time_seconds_total|nvcf_worker_service_inference_uploads_total|nvcf_worker_service_inference_bytes_total|nvcf_worker_service_inference_failure_total|nvcf_worker_service_worker_thread_count_total|nvcf_worker_service_worker_thread_busy_seconds_total|nvcf_worker_service_bytes_total|nvcf_worker_service_request_latency_seconds_bucket|nvcf_worker_service_request_latency_seconds_sum|nvcf_worker_service_request_latency_seconds_count|nvcf_worker_service_stream_latency_seconds_bucket|nvcf_worker_service_stream_latency_seconds_sum|nvcf_worker_service_stream_latency_seconds_count|nvct_worker_service_result_total|nvcf_worker_service_stream_active)"}'
+          scrape_interval: 30s
+          scrape_timeout: 25s
+          static_configs:
+            - targets:
+                - nvcf-byoo-prometheus-kube-prometheus.monitoring.svc.cluster.local:9090
+        - honor_labels: true
+          job_name: nvca
+          metric_relabel_configs:
+            - action: labelkeep
+              regex: (__name__|instance_type|nvca_cluster_group|nvca_cluster_name|nvca_nca_id|nvca_version|job|pod)
+          metrics_path: /federate
+          params:
+            match[]:
+              - '{job=~"nvca", namespace="nvca-system", pod=~"nvca.*", container="agent", __name__=~"(nvca_instance_type_allocatable|nvca_instance_type_capacity)"}'
+          scrape_interval: 30s
+          scrape_timeout: 25s
+          static_configs:
+            - targets:
+                - nvcf-byoo-prometheus-kube-prometheus.monitoring.svc.cluster.local:9090
+        - honor_labels: true
+          honor_timestamps: false
+          job_name: kubernetes-cadvisor
+          metric_relabel_configs:
+            - action: keep
+              regex: (^$|/dev/root|/dev/shm|/run)
+              source_labels:
+                - device
+            - regex: kubelet
+              replacement: kubernetes-cadvisor
+              source_labels:
+                - job
+              target_label: job
+            - action: labelkeep
+              regex: (__name__|container|cpu|device|image|job|interface|pod)
+          metrics_path: /federate
+          params:
+            match[]:
+              - '{job=~"kubelet", namespace="nvcf-backend", pod=~".*fake-instance-id.*", container=~"inference|task|", __name__=~"(container_cpu_cfs_throttled_periods_total|container_cpu_cfs_throttled_seconds_total|container_cpu_usage_seconds_total|container_memory_cache|container_memory_rss|container_memory_swap|container_memory_usage_bytes|container_memory_working_set_bytes|container_fs_limit_bytes|container_fs_usage_bytes|container_fs_reads_bytes_total|container_fs_reads_total|container_fs_writes_bytes_total|container_fs_writes_total|container_network_receive_bytes_total|container_network_receive_errors_total|container_network_receive_packets_dropped_total|container_network_receive_packets_total|container_network_transmit_bytes_total|container_network_transmit_errors_total|container_network_transmit_packets_dropped_total|container_network_transmit_packets_total)"}'
+          scrape_interval: 30s
+          scrape_timeout: 25s
+          static_configs:
+            - targets:
+                - nvcf-byoo-prometheus-kube-prometheus.monitoring.svc.cluster.local:9090
+        - honor_labels: true
+          job_name: kube-state-metrics
+          metric_relabel_configs:
+            - action: labelkeep
+              regex: (__name__|container|image|job|pod|reason|resource|unit)
+          metrics_path: /federate
+          params:
+            match[]:
+              - '{job=~"kube-state-metrics", namespace="nvcf-backend", pod=~".*fake-instance-id.*", container=~"inference|task", __name__=~"(kube_pod_container_info|kube_pod_container_resource_limits|kube_pod_container_resource_requests|kube_pod_container_status_last_terminated_exitcode|kube_pod_container_status_last_terminated_reason|kube_pod_container_status_ready|kube_pod_container_status_restarts_total|kube_pod_container_status_running|kube_pod_container_status_terminated|kube_pod_container_status_terminated_reason|kube_pod_container_status_waiting|kube_pod_container_status_waiting_reason|kube_pod_container_state_started)"}'
+          scrape_interval: 30s
+          scrape_timeout: 25s
+          static_configs:
+            - targets:
+                - nvcf-byoo-prometheus-kube-prometheus.monitoring.svc.cluster.local:9090
+        - honor_labels: true
+          job_name: nvidia-dcgm-exporter
+          metric_relabel_configs:
+            - action: labelmap
+              regex: exported_(.+)
+              replacement: $1
+            - action: labelkeep
+              regex: (__name__|container|DCGM_FI_DRIVER_VERSION|device|job|modelName|pci_bus_id|pod)
+          metrics_path: /federate
+          params:
+            match[]:
+              - '{job=~"nvidia-dcgm-exporter", exported_namespace="nvcf-backend", exported_pod=~".*fake-instance-id.*", exported_container=~"inference|task", __name__=~"DCGM_FI_DEV_GPU_UTIL|DCGM_FI_PROF_PIPE_TENSOR_ACTIVE|DCGM_FI_PROF_DRAM_ACTIVE|DCGM_FI_PROF_SM_ACTIVE|DCGM_FI_PROF_SM_OCCUPANCY|DCGM_FI_PROF_PCIE_TX_BYTES|DCGM_FI_PROF_PCIE_RX_BYTES|DCGM_FI_PROF_NVLINK_TX_BYTES|DCGM_FI_PROF_NVLINK_RX_BYTES|DCGM_FI_DEV_POWER_USAGE|DCGM_FI_DEV_VGPU_MEMORY_USAGE"}'
+          scrape_interval: 30s
+          scrape_timeout: 25s
+          static_configs:
+            - targets:
+                - nvcf-byoo-prometheus-kube-prometheus.monitoring.svc.cluster.local:9090
+        - job_name: opentelemetry-collector
+          metric_relabel_configs:
+            - action: keep
+              regex: (otelcol_receiver_refused_metric_points_total|otelcol_receiver_refused_spans_total|otelcol_receiver_refused_log_records_total|otelcol_receiver_accepted_metric_points_total|otelcol_receiver_accepted_spans_total|otelcol_receiver_accepted_log_records_total|otelcol_exporter_sent_metric_points_total|otelcol_exporter_sent_spans_total|otelcol_exporter_sent_log_records_total|otelcol_exporter_send_failed_metric_points_total|otelcol_exporter_send_failed_spans_total|otelcol_exporter_send_failed_log_records_total|otelcol_processor_incoming_items_total|otelcol_processor_outgoing_items_total)
+              source_labels:
+                - __name__
+          scrape_interval: 30s
+          scrape_timeout: 25s
+          static_configs:
+            - targets:
+                - ${env:OTEL_POD_IP:-0.0.0.0}:18888
+exporters:
+  prometheus/user-metrics:
+    enable_open_metrics: true
+    endpoint: ${env:OTEL_POD_IP:-0.0.0.0}:19091
+    metric_expiration: 5m
+    resource_to_telemetry_conversion:
+      enabled: true
+    send_timestamps: true
+  prometheusremotewrite/PROMETHEUS-customer-metrics-metrics:
+    endpoint: https://customer-metrics.example.invalid/api/v1/write
+    tls:
+      cert_file: /etc/byoo-otel-collector/secrets/customer-metrics-clientCert
+      key_file: /etc/byoo-otel-collector/secrets/customer-metrics-clientKey
+processors:
+  attributes/add-metadata:
+    actions:
+      - action: insert
+        key: nca_id
+        value: ${env:NCA_ID:-unknown}
+      - action: insert
+        key: cloud_provider
+        value: ${env:NVCF_CLOUD_PROVIDER:-unknown}
+      - action: insert
+        key: instance_id
+        value: ${env:NVCF_INSTANCE_ID:-unknown}
+      - action: insert
+        key: host.id
+        value: ${env:NVCF_INSTANCE_ID:-unknown}
+      - action: insert
+        key: function_id
+        value: ${env:NVCF_FUNCTION_ID:-unknown}
+      - action: insert
+        key: function_version_id
+        value: ${env:NVCF_FUNCTION_VERSION_ID:-unknown}
+      - action: insert
+        key: cloud_region
+        value: fake-zone-name
+      - action: insert
+        key: origin
+        value: nvcf-byoo
+  batch:
+    send_batch_max_size: 8192
+    send_batch_size: 4096
+    timeout: 400ms
+  batch/sre_metrics:
+    send_batch_max_size: 8192
+    send_batch_size: 4096
+    timeout: 400ms
+  filter/metrics:
+    metrics:
+      exclude:
+        match_type: strict
+        metric_names:
+          - scrape_duration_seconds
+          - scrape_samples_post_metric_relabeling
+          - scrape_samples_scraped
+          - scrape_series_added
+  filter/sre_metrics:
+    error_mode: ignore
+    metric_conditions:
+      - metric.name != "BpsInstrument" and metric.name != "FpsInstrument" and metric.name != "RtdInstrument" and metric.name != "StageOpenDuration"
+      - resource.attributes["sre_metrics_enabled"] == "false"
+      - resource.attributes["sre_enabled_metrics"] == "false"
+      - datapoint.attributes["sre_metrics_enabled"] == "false"
+      - datapoint.attributes["sre_enabled_metrics"] == "false"
+  memory_limiter:
+    check_interval: 1s
+    limit_percentage: 80
+    spike_limit_percentage: 10
+  metrics_transform:
+    transforms:
+      - action: update
+        include: ^.*$
+        match_type: regexp
+        operations:
+          - action: add_label
+            new_label: nca_id
+            new_value: ${env:NCA_ID:-unknown}
+          - action: add_label
+            new_label: cloud_provider
+            new_value: ${env:NVCF_CLOUD_PROVIDER:-unknown}
+          - action: add_label
+            new_label: instance_id
+            new_value: ${env:NVCF_INSTANCE_ID:-unknown}
+          - action: add_label
+            new_label: host.id
+            new_value: ${env:NVCF_INSTANCE_ID:-unknown}
+          - action: add_label
+            new_label: function_id
+            new_value: ${env:NVCF_FUNCTION_ID:-unknown}
+          - action: add_label
+            new_label: function_version_id
+            new_value: ${env:NVCF_FUNCTION_VERSION_ID:-unknown}
+          - action: add_label
+            new_label: cloud_region
+            new_value: fake-zone-name
+          - action: add_label
+            new_label: origin
+            new_value: nvcf-byoo
+  resource:
+    attributes:
+      - action: delete
+        key: service.instance.id
+  resource/customer_metrics:
+    attributes:
+      - action: delete
+        key: sre_metrics_enabled
+      - action: delete
+        key: sre_enabled_metrics
+extensions:
+  cgroup_runtime:
+    gomaxprocs:
+      enabled: true
+    gomemlimit:
+      enabled: true
+      ratio: 0.8
+  healthcheckv2:
+    component_health:
+      include_permanent_errors: true
+      include_recoverable_errors: true
+      recovery_duration: 1m
+    http:
+      config:
+        enabled: false
+      endpoint: ${env:OTEL_POD_IP:-0.0.0.0}:13133
+      status:
+        enabled: true
+        path: /health
+    use_v2: true
+service:
+  telemetry:
+    logs:
+      initial_fields:
+        public: "true"
+      level: warn
+    metrics:
+      level: detailed
+      readers:
+        - pull:
+            exporter:
+              prometheus:
+                host: ${env:OTEL_POD_IP:-0.0.0.0}
+                port: 18888
+    resource:
+      attributes:
+        - name: service.namespace
+          value: sr-fake-namespace
+        - name: service.name
+          value: byoo-otel-collector
+        - name: function.id
+          value: fake-function-id
+        - name: function.version.id
+          value: fake-function-version-id
+    traces:
+      processors:
+        - batch:
+            exporter:
+              otlp:
+                endpoint: ${env:OTEL_EXPORTER_OTLP_ENDPOINT:-http://localhost:4317}
+                headers:
+                  - name: lightstep-access-token
+                    value: ${env:OTEL_TRACING_ACCESS_TOKEN}
+                protocol: grpc
+  pipelines:
+    metrics:
+      receivers:
+        - otlp
+        - prometheus
+      exporters:
+        - prometheusremotewrite/PROMETHEUS-customer-metrics-metrics
+      processors:
+        - memory_limiter
+        - filter/metrics
+        - resource
+        - resource/customer_metrics
+        - metrics_transform
+        - batch
+    metrics/sre:
+      receivers:
+        - otlp
+      exporters:
+        - prometheus/user-metrics
+      processors:
+        - memory_limiter
+        - filter/sre_metrics
+        - resource
+        - metrics_transform
+        - batch/sre_metrics
+  extensions:
+    - healthcheckv2
+    - cgroup_runtime
@@ -47,6 +47,7 @@ type TemplateConfig struct {
 	InstanceID        string
 	ZoneName          string
 	LogChunking       LogChunkingConfig
+	SREMetrics        SREMetricsConfig
 	// LogExporterBatchMaxSizeBytes configures exporterhelper byte batching for logs.
 	// Zero uses the default selected for BYOO.
 	LogExporterBatchMaxSizeBytes int
@@ -57,6 +58,12 @@ type LogChunkingConfig struct {
 	DryRun       bool
 }
 
+type SREMetricsConfig struct {
+	Enabled                   bool
+	FilterConfig              map[string]interface{}
+	CustomerMetricsDropLabels []string
+}
+
 func ExecuteTemplate(w io.Writer, tcfg TemplateConfig) error {
 	var templateName string
 	switch tcfg.BackendType {

@@ -42,6 +42,9 @@ type envConfig struct {
 	ByooLogChunkMaxBodyBytes         int    `split_words:"true"`
 	ByooLogChunkDryRun               bool   `split_words:"true"`
 	ByooLogExporterBatchMaxSizeBytes int    `split_words:"true"`
+	ByooSreMetricsEnabled            bool   `split_words:"true"`
+	ByooSreMetricsFilterConfig       string `split_words:"true"`
+	ByooCustomerMetricsDropLabels    string `split_words:"true"`
 }
 
 func processEnvConfig(env *envConfig) error {
@@ -76,6 +79,15 @@ func getTemplateConfig() (TemplateConfig, error) {
 		return TemplateConfig{}, fmt.Errorf("BYOO_LOG_EXPORTER_BATCH_MAX_SIZE_BYTES: %w", err)
 	}
 	tcgf.LogExporterBatchMaxSizeBytes = logExporterBatchMaxSizeBytes
+	sreMetricsFilterConfig, err := resolvedSREMetricsFilterConfig(env.ByooSreMetricsFilterConfig)
+	if err != nil {
+		return TemplateConfig{}, fmt.Errorf("BYOO_SRE_METRICS_FILTER_CONFIG: %w", err)
+	}
+	tcgf.SREMetrics = SREMetricsConfig{
+		Enabled:                   env.ByooSreMetricsEnabled,
+		FilterConfig:              sreMetricsFilterConfig,
+		CustomerMetricsDropLabels: resolvedCustomerMetricsDropLabels(env.ByooCustomerMetricsDropLabels),
+	}
 
 	functionID := env.NvcfFunctionID
 	functionVersionID := env.NvcfFunctionVersionID