From 874db04b5d9f3f8a75a8978dcd182fa6f0b94b3f Mon Sep 17 00:00:00 2001 From: BrewingCoder Date: Sat, 9 May 2026 09:51:29 -0400 Subject: [PATCH] =?UTF-8?q?infra:=20drop=20OTel=20collector=20container=20?= =?UTF-8?q?=E2=80=94=20backend=20hosts=20OTLP=20directly=20(HOL-21)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The OTel collector container was a SaaS-shaped multi-protocol relay (fluentforward, syslog, AWS Firehose CloudWatch, OTLP gRPC/HTTP/HTTPS, TCP log) that for self-hosted deployments only delivers OTLP HTTP value the backend already provides natively. The .NET backend has a complete OTLP HTTP receiver at /otel/v1/{logs, traces,metrics}: JSON + protobuf, gzip + snappy decompression, CORS- enabled, project-id resolution from resource attributes. Self- instrumentation already targets it directly. SDKs just needed their default OTLP endpoint pointed at it. Removed: - compose.yml + compose.hobby-dotnet.yml: collector service - infra/docker/{collector.Dockerfile, collector.yml, configure-collector.sh} - start-infra.sh: collector from SERVICES list - .env.example: OTEL_COLLECTOR_* image vars - env.sh: collector hostname rewrites Updated SDK + frontend defaults from :4318 (collector) to :8082/otel (backend): - sdk/highlight-run/src/index.tsx: otlpEndpoint default - compose.hobby-dotnet.yml + compose.dev-frontend.yml: REACT_APP_OTLP_ENDPOINT - env.sh: OTLP_DOGFOOD_ENDPOINT, OTLP_ENDPOINT Verified: docker compose ps now shows 7 HoldFast containers (was 9 before HOL-19+HOL-21). Smoke test passes end-to-end. Subtask of HOL-17. Closes HOL-21. Co-Authored-By: Claude Opus 4.7 --- infra/docker/.env.example | 6 +- infra/docker/collector.Dockerfile | 19 ---- infra/docker/collector.yml | 158 -------------------------- infra/docker/compose.hobby-dotnet.yml | 23 +--- infra/docker/compose.yml | 35 +----- infra/docker/configure-collector.sh | 27 ----- infra/docker/env.sh | 9 +- infra/docker/start-infra.sh | 2 +- sdk/highlight-run/src/index.tsx | 5 +- 9 files changed, 22 insertions(+), 262 deletions(-) delete mode 100644 infra/docker/collector.Dockerfile delete mode 100644 infra/docker/collector.yml delete mode 100644 infra/docker/configure-collector.sh diff --git a/infra/docker/.env.example b/infra/docker/.env.example index bd411a6d..db882e16 100644 --- a/infra/docker/.env.example +++ b/infra/docker/.env.example @@ -21,9 +21,9 @@ COMPOSE_PATH_SEPARATOR=: # Infra dependencies — pinned for reproducibility. CLICKHOUSE_IMAGE_NAME=clickhouse/clickhouse-server:24.3.15.72-alpine KAFKA_IMAGE_NAME=confluentinc/cp-kafka:7.7.0 -OTEL_COLLECTOR_BUILD_IMAGE_NAME=alpine:3.21.3 -OTEL_COLLECTOR_IMAGE_NAME=otel/opentelemetry-collector-contrib:0.128.0 POSTGRES_IMAGE_NAME=ankane/pgvector:v0.5.1 +# OTEL collector image vars removed — HOL-21 dropped the collector container. +# Backend hosts the OTLP receiver at /otel/v1/{logs,traces,metrics}. REDIS_IMAGE_NAME=redis:8.0.2 ZOOKEEPER_IMAGE_NAME=confluentinc/cp-zookeeper:7.7.0 @@ -52,7 +52,7 @@ REACT_APP_DISABLE_ANALYTICS=false REACT_APP_FRONTEND_ORG=1 REACT_APP_FRONTEND_URI=http://localhost:3000 REACT_APP_IN_DOCKER=true -REACT_APP_OTLP_ENDPOINT=http://localhost:4318 +REACT_APP_OTLP_ENDPOINT=http://localhost:8082/otel REACT_APP_PRIVATE_GRAPH_URI=http://localhost:8082/private REACT_APP_PUBLIC_GRAPH_URI=http://localhost:8082/public diff --git a/infra/docker/collector.Dockerfile b/infra/docker/collector.Dockerfile deleted file mode 100644 index b662fe8c..00000000 --- a/infra/docker/collector.Dockerfile +++ /dev/null @@ -1,19 +0,0 @@ -ARG OTEL_COLLECTOR_BUILD_IMAGE_NAME="alpine:latest" -ARG OTEL_COLLECTOR_IMAGE_NAME="otel/opentelemetry-collector-contrib:latest" - -FROM ${OTEL_COLLECTOR_BUILD_IMAGE_NAME} AS collector-build - -COPY ./infra/docker/collector.yml /collector.yml -COPY ./infra/docker/configure-collector.sh /configure-collector.sh - -ARG IN_DOCKER_GO -ARG SSL -RUN chmod +x /configure-collector.sh && /configure-collector.sh - -FROM ${OTEL_COLLECTOR_IMAGE_NAME} AS collector - -COPY ./src/backend/localhostssl/server.crt /server.crt -COPY ./src/backend/localhostssl/server.key /server.key - -COPY --from=collector-build /collector.yml /etc/otel-collector-config.yaml -CMD ["--config=/etc/otel-collector-config.yaml"] diff --git a/infra/docker/collector.yml b/infra/docker/collector.yml deleted file mode 100644 index 13aa873b..00000000 --- a/infra/docker/collector.yml +++ /dev/null @@ -1,158 +0,0 @@ -receivers: - fluentforward: - endpoint: '0.0.0.0:24224' - tcplog: - listen_address: '0.0.0.0:34302' - awsfirehose/cwmetrics: - endpoint: '0.0.0.0:4433' - record_type: cwmetrics - include_metadata: true - awsfirehose/cwlogs: - endpoint: '0.0.0.0:4434' - record_type: cwlogs - include_metadata: true - awsfirehose/otlp_v1: - endpoint: '0.0.0.0:4435' - record_type: otlp_v1 - include_metadata: true - syslog: - udp: - listen_address: '0.0.0.0:6513' - tcp: - listen_address: '0.0.0.0:6514' - protocol: 'rfc5424' - enable_octet_counting: true - otlp: - protocols: - grpc: - endpoint: '0.0.0.0:4317' - include_metadata: true - max_recv_msg_size_mib: 1000 - http: - endpoint: '0.0.0.0:4318' - max_request_body_size: 0 - include_metadata: true - cors: - allowed_origins: - # Can't use '*' with credentials because it fails CORS - # validation on the preflight request. - - 'https://*' - - 'http://*' - - 'capacitor://*' - allowed_headers: - - 'X-Highlight-Request' - - 'Origin' - - 'Content-Type' - - 'Accept' - - 'Access-Control-Request-Headers' - - 'Access-Control-Request-Method' - - 'Access-Control-Allow-Origin' - - 'Access-Control-Allow-Methods' - - 'Access-Control-Max-Age' - - 'Referer' - - 'Traceparent' - - 'Tracestate' - max_age: 7200 - otlp/https: - protocols: - http: - endpoint: '0.0.0.0:8318' - max_request_body_size: 0 - include_metadata: true - cors: - allowed_origins: - - 'https://*' - allowed_headers: - - 'X-Highlight-Request' - - 'Origin' - - 'Content-Type' - - 'Accept' - - 'Access-Control-Request-Headers' - - 'Access-Control-Request-Method' - - 'Access-Control-Allow-Origin' - - 'Access-Control-Allow-Methods' - - 'Access-Control-Max-Age' - - 'Referer' - - 'Traceparent' - - 'Tracestate' - max_age: 7200 - tls: - cert_file: '/server.crt' - key_file: '/server.key' -exporters: - debug: - sampling_initial: 1 - sampling_thereafter: 1000 - verbosity: basic # set to 'detailed' to log individual data points - otlphttp: - endpoint: 'https://host.docker.internal:8082/otel' - compression: snappy - tls: - insecure_skip_verify: true - timeout: 30s - http2_read_idle_timeout: 30s - http2_ping_timeout: 30s - idle_conn_timeout: 60s - max_idle_conns: 100 - max_idle_conns_per_host: 100 - max_conns_per_host: 1000 - read_buffer_size: 32768 - write_buffer_size: 131072 - sending_queue: - num_consumers: 100 - queue_size: 10000 - retry_on_failure: - enabled: true - initial_interval: 1s - max_interval: 30s - max_elapsed_time: 300s -processors: - memory_limiter: - limit_mib: 14336 - spike_limit_mib: 1024 - check_interval: 0.1s - batch: - metadata_keys: - - x-highlight-project - metadata_cardinality_limit: 1000 - timeout: 1s - send_batch_size: 1000 - send_batch_max_size: 10000 - attributes: - actions: - - key: highlight.project_id - from_context: metadata.x-highlight-project - action: insert -service: - telemetry: - logs: - level: 'info' - metrics: - level: detailed - extensions: [health_check] - pipelines: - traces: - receivers: [otlp, otlp/https] - processors: [memory_limiter, batch, attributes] - exporters: [otlphttp, debug] - metrics: - receivers: - [otlp, otlp/https, awsfirehose/cwmetrics, awsfirehose/otlp_v1] - processors: [memory_limiter, batch, attributes] - exporters: [otlphttp, debug] - logs: - receivers: - [ - otlp, - otlp/https, - fluentforward, - tcplog, - syslog, - awsfirehose/cwlogs, - ] - processors: [memory_limiter, batch, attributes] - exporters: [otlphttp, debug] -extensions: - health_check: - endpoint: '0.0.0.0:4319' - path: '/health/status' diff --git a/infra/docker/compose.hobby-dotnet.yml b/infra/docker/compose.hobby-dotnet.yml index dc5ad35c..ac4592fd 100644 --- a/infra/docker/compose.hobby-dotnet.yml +++ b/infra/docker/compose.hobby-dotnet.yml @@ -39,24 +39,9 @@ services: timeout: 5s retries: 10 - collector: - container_name: collector - build: - context: ../.. - dockerfile: infra/docker/collector.Dockerfile - args: - - IN_DOCKER_GO=true - - SSL=false - restart: on-failure - extra_hosts: - - 'host.docker.internal:host-gateway' - ports: - - '0.0.0.0:4317:4317' - - '0.0.0.0:4318:4318' - - '0.0.0.0:4319:4319' - depends_on: - backend: - condition: service_healthy + # Collector removed in HOL-21 — backend has its own OTLP receiver at + # /otel/v1/{logs,traces,metrics}. SDKs should target the backend's + # OTLP endpoint (default http://backend:8082/otel). frontend: container_name: frontend @@ -74,7 +59,7 @@ services: - REACT_APP_FRONTEND_URI=${REACT_APP_FRONTEND_URI:-http://localhost:3000} - REACT_APP_PRIVATE_GRAPH_URI=${REACT_APP_PRIVATE_GRAPH_URI:-http://localhost:8082/private} - REACT_APP_PUBLIC_GRAPH_URI=${REACT_APP_PUBLIC_GRAPH_URI:-http://localhost:8082/public} - - REACT_APP_OTLP_ENDPOINT=${REACT_APP_OTLP_ENDPOINT:-http://localhost:4318} + - REACT_APP_OTLP_ENDPOINT=${REACT_APP_OTLP_ENDPOINT:-http://localhost:8082/otel} volumes: highlight-data: diff --git a/infra/docker/compose.yml b/infra/docker/compose.yml index db75cf9d..9a460161 100644 --- a/infra/docker/compose.yml +++ b/infra/docker/compose.yml @@ -90,36 +90,11 @@ services: - clickhouse-data:/var/lib/clickhouse - clickhouse-logs:/var/log/clickhouse-server - collector: - logging: *local-logging - restart: on-failure - build: - dockerfile: ./infra/docker/collector.Dockerfile - pull: true - target: collector - context: ../.. - args: - - IN_DOCKER_GO - - SSL - container_name: collector - extra_hosts: - - 'host.docker.internal:host-gateway' - volumes: - - ../../src/backend/localhostssl/server.crt:/server.crt - - ../../src/backend/localhostssl/server.key:/server.key - ports: - - '0.0.0.0:24224:24224' - - '0.0.0.0:34302:34302' - - '0.0.0.0:4317:4317' - - '0.0.0.0:4318:4318' - - '0.0.0.0:4319:4319' - - '0.0.0.0:4433:4433' - - '0.0.0.0:4434:4434' - - '0.0.0.0:4435:4435' - - '0.0.0.0:6513:6513' - - '0.0.0.0:6514:6514' - - '0.0.0.0:8318:8318' - - '0.0.0.0:8889:8888' + # Collector removed in HOL-21. The backend has a full OTLP HTTP receiver + # at /otel/v1/{logs,traces,metrics} (JSON + protobuf, gzip + snappy + # decompression). Self-hosted deployments don't need the SaaS-shaped + # multi-protocol relay (fluentforward, syslog, AWS Firehose, etc). + # Point SDKs at http://:8082/otel instead of http://:4318. volumes: postgres-data: diff --git a/infra/docker/configure-collector.sh b/infra/docker/configure-collector.sh deleted file mode 100644 index cd217f1a..00000000 --- a/infra/docker/configure-collector.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -ex - -COLLECTOR_CONFIG="./collector.yml" - -echo "Updating using SSL=$SSL and IN_DOCKER_GO=$IN_DOCKER_GO" -if [[ "$IN_DOCKER_GO" == "true" ]]; then - if [[ "$SSL" != "true" ]]; then - if grep -q 'https://host.docker.internal' "$COLLECTOR_CONFIG"; then - sed -i'' -e 's/https:\/\/host\.docker\.internal:8082/http:\/\/backend:8082/g' $COLLECTOR_CONFIG - fi - if grep -q 'insecure_skip_verify' "$COLLECTOR_CONFIG"; then - sed -i'' -e '90d;91d' $COLLECTOR_CONFIG - fi - else - if grep -q 'https://host.docker.internal' "$COLLECTOR_CONFIG"; then - sed -i'' -e 's/https:\/\/host\.docker\.internal:8082/https:\/\/backend:8082/g' $COLLECTOR_CONFIG - fi - fi -elif [[ "$SSL" != "true" ]]; then - if grep -q 'https://host.docker.internal' "$COLLECTOR_CONFIG"; then - sed -i'' -e 's/https:\/\/host\.docker/http:\/\/host\.docker/g' $COLLECTOR_CONFIG - fi - if grep -q 'insecure_skip_verify' "$COLLECTOR_CONFIG"; then - sed -i'' -e '90d;91d' $COLLECTOR_CONFIG - fi -fi -cat $COLLECTOR_CONFIG diff --git a/infra/docker/env.sh b/infra/docker/env.sh index c3cdfb32..2cce548e 100644 --- a/infra/docker/env.sh +++ b/infra/docker/env.sh @@ -29,8 +29,9 @@ if [[ "$*" == *"--go-docker"* ]]; then export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://kafka:9092" export KAFKA_SERVERS=kafka:9092 export ON_PREM=true - export OTLP_DOGFOOD_ENDPOINT=http://collector:4318 - export OTLP_ENDPOINT=http://collector:4318 + # HOL-21: collector container removed; backend hosts the OTLP receiver. + export OTLP_DOGFOOD_ENDPOINT=http://backend:8082/otel + export OTLP_ENDPOINT=http://backend:8082/otel export PSQL_HOST=postgres export REDIS_EVENTS_STAGING_ENDPOINT=redis:6379 echo "Using docker-internal infra." @@ -38,8 +39,8 @@ else export CLICKHOUSE_ADDRESS=localhost:9000 export KAFKA_ADVERTISED_LISTENERS="PLAINTEXT://localhost:9092" export KAFKA_SERVERS=localhost:9092 - export OTLP_DOGFOOD_ENDPOINT=http://localhost:4318 - export OTLP_ENDPOINT=http://localhost:4318 + export OTLP_DOGFOOD_ENDPOINT=http://localhost:8082/otel + export OTLP_ENDPOINT=http://localhost:8082/otel export PSQL_HOST=localhost export REDIS_EVENTS_STAGING_ENDPOINT=localhost:6379 fi diff --git a/infra/docker/start-infra.sh b/infra/docker/start-infra.sh index d42b48c9..a437efe9 100644 --- a/infra/docker/start-infra.sh +++ b/infra/docker/start-infra.sh @@ -4,7 +4,7 @@ source env.sh # startup the infra -SERVICES="clickhouse kafka postgres redis zookeeper collector" +SERVICES="clickhouse kafka postgres redis zookeeper" BUILD_ARGS="--build-arg OTEL_COLLECTOR_ALPINE_IMAGE_NAME=${OTEL_COLLECTOR_ALPINE_IMAGE_NAME} \ --build-arg OTEL_COLLECTOR_IMAGE_NAME=${OTEL_COLLECTOR_IMAGE_NAME}" diff --git a/sdk/highlight-run/src/index.tsx b/sdk/highlight-run/src/index.tsx index f475c6dc..916cdca6 100644 --- a/sdk/highlight-run/src/index.tsx +++ b/sdk/highlight-run/src/index.tsx @@ -130,9 +130,12 @@ const H: HighlightPublicInterface = { setupBrowserTracing({ backendUrl: options?.backendUrl ?? 'http://localhost:8082/public', + // HOL-21: backend hosts the OTLP receiver at /otel. + // The previous :4318 default targeted the dedicated + // collector container, which was removed. otlpEndpoint: options?.otlpEndpoint ?? - 'http://localhost:4318', + 'http://localhost:8082/otel', projectId: projectID, sessionSecureId: sessionSecureID, environment: options?.environment ?? 'production',