From d41394aae84c3e02e26cd68755c7e47ef02afbd9 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Fri, 22 May 2026 18:48:23 +0530 Subject: [PATCH] feat: adds dockercompose agent casting --- .../collectionagent/docker/compose/README.md | 85 ++++++ .../docker/compose/casting.yaml | 14 + .../docker/compose/casting.yaml.lock | 257 ++++++++++++++++++ .../collector/agent/agent.yaml | 88 ++++++ .../pours/collectionagent/compose.yaml | 19 ++ .../dockercomposecasting/casting.go | 86 ++++++ .../dockercomposecasting/embed.go | 15 + .../dockercomposecasting/embed_test.go | 34 +++ .../dockercomposecasting/enricher.go | 49 ++++ .../templates/agent/receivers.yaml.gotmpl | 34 +++ .../templates/compose.yaml.gotmpl | 20 ++ internal/casting/collectionagent/registry.go | 13 +- 12 files changed, 713 insertions(+), 1 deletion(-) create mode 100644 docs/examples/collectionagent/docker/compose/README.md create mode 100644 docs/examples/collectionagent/docker/compose/casting.yaml create mode 100644 docs/examples/collectionagent/docker/compose/casting.yaml.lock create mode 100644 docs/examples/collectionagent/docker/compose/pours/collectionagent/collector/agent/agent.yaml create mode 100644 docs/examples/collectionagent/docker/compose/pours/collectionagent/compose.yaml create mode 100644 internal/casting/collectionagent/dockercomposecasting/casting.go create mode 100644 internal/casting/collectionagent/dockercomposecasting/embed.go create mode 100644 internal/casting/collectionagent/dockercomposecasting/embed_test.go create mode 100644 internal/casting/collectionagent/dockercomposecasting/enricher.go create mode 100644 internal/casting/collectionagent/dockercomposecasting/templates/agent/receivers.yaml.gotmpl create mode 100644 internal/casting/collectionagent/dockercomposecasting/templates/compose.yaml.gotmpl diff --git a/docs/examples/collectionagent/docker/compose/README.md b/docs/examples/collectionagent/docker/compose/README.md new file mode 100644 index 0000000..7a2eb8b --- /dev/null +++ b/docs/examples/collectionagent/docker/compose/README.md @@ -0,0 +1,85 @@ +# Docker Compose (Collection Agent) + +| Field | Value | +| --- | --- | +| **Kind** | `CollectionAgent` | +| **Mode** | `docker` | +| **Flavor** | `compose` | +| **Platform** | `-` | + +## Overview + +Runs an OpenTelemetry collector as a Docker Compose service that ships host metrics, container metrics, and container logs to a remote SigNoz endpoint. + +## Prerequisites + +- Docker Engine 20.10+ +- Docker Compose v2 +- A reachable SigNoz endpoint (SigNoz Cloud, or a self-hosted ingester) + +## Configuration + +```yaml +apiVersion: v1alpha1 +kind: CollectionAgent +metadata: + name: signoz +spec: + deployment: + flavor: compose + mode: docker + collector: + kind: agent + spec: + env: + SIGNOZ_INGESTION_ENDPOINT: https://ingest..signoz.cloud:443 + SIGNOZ_INGESTION_KEY: +``` + +Replace `` and `` with your SigNoz Cloud values. For a self-hosted ingester, point `SIGNOZ_INGESTION_ENDPOINT` at it and omit `SIGNOZ_INGESTION_KEY` — the rendered exporter skips the ingestion header when the key is unset. + +## Deploy + +```bash +foundryctl cast -f casting.yaml +``` + +Or step by step: + +```bash +# Validate prerequisites +foundryctl gauge -f casting.yaml + +# Generate compose files +foundryctl forge -f casting.yaml + +# Start the agent +cd pours/collectionagent && docker compose up -d +``` + +## Generated output + +```text +pours/collectionagent/ + compose.yaml + collector/ + agent/ + agent.yaml +``` + +## After deployment + +```bash +# Check the collector container +docker ps --filter name=signoz-collectionagent-collector + +# Tail collector logs +docker compose -f pours/collectionagent/compose.yaml logs -f + +# Stop the agent +cd pours/collectionagent && docker compose down +``` + +## Customization + +Override the collector image, replicas, or env vars in `spec.collector.spec`. For platform-level changes to the generated `compose.yaml` or `agent.yaml` (memory limits, batch tuning, extra receivers, etc.), use [patches](../../../../concepts/patches.md). diff --git a/docs/examples/collectionagent/docker/compose/casting.yaml b/docs/examples/collectionagent/docker/compose/casting.yaml new file mode 100644 index 0000000..f755ef6 --- /dev/null +++ b/docs/examples/collectionagent/docker/compose/casting.yaml @@ -0,0 +1,14 @@ +apiVersion: v1alpha1 +kind: CollectionAgent +metadata: + name: signoz +spec: + deployment: + flavor: compose + mode: docker + collector: + kind: agent + spec: + env: + SIGNOZ_INGESTION_ENDPOINT: https://ingest..signoz.cloud:443 + SIGNOZ_INGESTION_KEY: diff --git a/docs/examples/collectionagent/docker/compose/casting.yaml.lock b/docs/examples/collectionagent/docker/compose/casting.yaml.lock new file mode 100644 index 0000000..dc7f064 --- /dev/null +++ b/docs/examples/collectionagent/docker/compose/casting.yaml.lock @@ -0,0 +1,257 @@ +apiVersion: v1alpha1 +kind: CollectionAgent +metadata: + name: signoz +spec: + collector: + kind: agent + spec: + cluster: + replicas: 1 + config: + data: + collector/agent/agent.yaml: | + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + docker_stats: + collection_interval: 10s + endpoint: unix:///var/run/docker.sock + timeout: 20s + filelog: + include: + - /var/lib/docker/containers/*/*-json.log + include_file_name: false + include_file_path: true + operators: + - add_metadata_from_filepath: false + format: docker + id: container-parser + type: container + start_at: end + hostmetrics: + collection_interval: 60s + root_path: /hostfs + scrapers: + cpu: {} + disk: {} + filesystem: {} + load: {} + memory: {} + network: {} + paging: {} + process: + mute_process_exe_error: true + mute_process_io_error: true + mute_process_name_error: true + mute_process_user_error: true + processes: {} + + processors: + memory_limiter: + check_interval: 5s + limit_mib: 4000 + spike_limit_mib: 800 + resourcedetection: + detectors: + - env + - system + - docker + timeout: 5s + batch: {} + + exporters: + otlphttp: + endpoint: ${env:SIGNOZ_INGESTION_ENDPOINT} + headers: + signoz-ingestion-key: ${env:SIGNOZ_INGESTION_KEY} + + service: + pipelines: + traces: + receivers: + - otlp + processors: + - memory_limiter + - resourcedetection + - batch + exporters: + - otlphttp + metrics: + receivers: + - otlp + - docker_stats + - hostmetrics + processors: + - memory_limiter + - resourcedetection + - batch + exporters: + - otlphttp + logs: + receivers: + - otlp + - filelog + processors: + - memory_limiter + - resourcedetection + - batch + exporters: + - otlphttp + enabled: true + env: + OTEL_COLLECTOR_ROLE: agent + SIGNOZ_INGESTION_ENDPOINT: https://ingest..signoz.cloud:443 + SIGNOZ_INGESTION_KEY: + image: otel/opentelemetry-collector-contrib:v0.139.0 + version: v0.139.0 + status: + config: + data: + collector/agent/agent.yaml: | + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + docker_stats: + collection_interval: 10s + endpoint: unix:///var/run/docker.sock + timeout: 20s + filelog: + include: + - /var/lib/docker/containers/*/*-json.log + include_file_name: false + include_file_path: true + operators: + - add_metadata_from_filepath: false + format: docker + id: container-parser + type: container + start_at: end + hostmetrics: + collection_interval: 60s + root_path: /hostfs + scrapers: + cpu: {} + disk: {} + filesystem: {} + load: {} + memory: {} + network: {} + paging: {} + process: + mute_process_exe_error: true + mute_process_io_error: true + mute_process_name_error: true + mute_process_user_error: true + processes: {} + + processors: + memory_limiter: + check_interval: 5s + limit_mib: 4000 + spike_limit_mib: 800 + resourcedetection: + detectors: + - env + - system + - docker + timeout: 5s + batch: {} + + exporters: + otlphttp: + endpoint: ${env:SIGNOZ_INGESTION_ENDPOINT} + headers: + signoz-ingestion-key: ${env:SIGNOZ_INGESTION_KEY} + + service: + pipelines: + traces: + receivers: + - otlp + processors: + - memory_limiter + - resourcedetection + - batch + exporters: + - otlphttp + metrics: + receivers: + - otlp + - docker_stats + - hostmetrics + processors: + - memory_limiter + - resourcedetection + - batch + exporters: + - otlphttp + logs: + receivers: + - otlp + - filelog + processors: + - memory_limiter + - resourcedetection + - batch + exporters: + - otlphttp + env: + OTEL_COLLECTOR_ROLE: agent + SIGNOZ_INGESTION_ENDPOINT: https://ingest..signoz.cloud:443 + SIGNOZ_INGESTION_KEY: + receivers: + docker_stats: + body: + collection_interval: 10s + endpoint: unix:///var/run/docker.sock + timeout: 20s + pipelines: + - metrics + filelog: + body: + include: + - /var/lib/docker/containers/*/*-json.log + include_file_name: false + include_file_path: true + operators: + - add_metadata_from_filepath: false + format: docker + id: container-parser + type: container + start_at: end + pipelines: + - logs + hostmetrics: + body: + collection_interval: 60s + root_path: /hostfs + scrapers: + cpu: {} + disk: {} + filesystem: {} + load: {} + memory: {} + network: {} + paging: {} + process: + mute_process_exe_error: true + mute_process_io_error: true + mute_process_name_error: true + mute_process_user_error: true + processes: {} + pipelines: + - metrics + resourceDetectors: + - docker + deployment: + flavor: compose + mode: docker diff --git a/docs/examples/collectionagent/docker/compose/pours/collectionagent/collector/agent/agent.yaml b/docs/examples/collectionagent/docker/compose/pours/collectionagent/collector/agent/agent.yaml new file mode 100644 index 0000000..a700607 --- /dev/null +++ b/docs/examples/collectionagent/docker/compose/pours/collectionagent/collector/agent/agent.yaml @@ -0,0 +1,88 @@ +exporters: + otlphttp: + endpoint: ${env:SIGNOZ_INGESTION_ENDPOINT} + headers: + signoz-ingestion-key: ${env:SIGNOZ_INGESTION_KEY} +processors: + batch: {} + memory_limiter: + check_interval: 5s + limit_mib: 4000 + spike_limit_mib: 800 + resourcedetection: + detectors: + - env + - system + - docker + timeout: 5s +receivers: + docker_stats: + collection_interval: 10s + endpoint: unix:///var/run/docker.sock + timeout: 20s + filelog: + include: + - /var/lib/docker/containers/*/*-json.log + include_file_name: false + include_file_path: true + operators: + - add_metadata_from_filepath: false + format: docker + id: container-parser + type: container + start_at: end + hostmetrics: + collection_interval: 60s + root_path: /hostfs + scrapers: + cpu: {} + disk: {} + filesystem: {} + load: {} + memory: {} + network: {} + paging: {} + process: + mute_process_exe_error: true + mute_process_io_error: true + mute_process_name_error: true + mute_process_user_error: true + processes: {} + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 +service: + pipelines: + logs: + exporters: + - otlphttp + processors: + - memory_limiter + - resourcedetection + - batch + receivers: + - otlp + - filelog + metrics: + exporters: + - otlphttp + processors: + - memory_limiter + - resourcedetection + - batch + receivers: + - otlp + - docker_stats + - hostmetrics + traces: + exporters: + - otlphttp + processors: + - memory_limiter + - resourcedetection + - batch + receivers: + - otlp diff --git a/docs/examples/collectionagent/docker/compose/pours/collectionagent/compose.yaml b/docs/examples/collectionagent/docker/compose/pours/collectionagent/compose.yaml new file mode 100644 index 0000000..ab0a794 --- /dev/null +++ b/docs/examples/collectionagent/docker/compose/pours/collectionagent/compose.yaml @@ -0,0 +1,19 @@ +name: signoz-collectionagent +services: + collector: + command: + - --config=/etc/otelcol-contrib/config.yaml + container_name: signoz-collectionagent-collector + environment: + OTEL_COLLECTOR_ROLE: agent + SIGNOZ_INGESTION_ENDPOINT: https://ingest..signoz.cloud:443 + SIGNOZ_INGESTION_KEY: + image: otel/opentelemetry-collector-contrib:v0.139.0 + network_mode: host + restart: unless-stopped + user: "0:0" + volumes: + - ./collector/agent/agent.yaml:/etc/otelcol-contrib/config.yaml:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /:/hostfs:ro diff --git a/internal/casting/collectionagent/dockercomposecasting/casting.go b/internal/casting/collectionagent/dockercomposecasting/casting.go new file mode 100644 index 0000000..87fd157 --- /dev/null +++ b/internal/casting/collectionagent/dockercomposecasting/casting.go @@ -0,0 +1,86 @@ +package dockercomposecasting + +import ( + "bytes" + "context" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + "github.com/signoz/foundry/api/v1alpha1/collectionagent" + foundryerrors "github.com/signoz/foundry/internal/errors" + collectionagentmolding "github.com/signoz/foundry/internal/molding/collectionagent" + "github.com/signoz/foundry/internal/molding/collectionagent/collectormolding" + "github.com/signoz/foundry/internal/pourer" +) + +type dockerComposeCasting struct { + logger *slog.Logger +} + +func New(logger *slog.Logger) *dockerComposeCasting { + return &dockerComposeCasting{logger: logger} +} + +func (c *dockerComposeCasting) Enricher(ctx context.Context, config *collectionagent.Casting) (collectionagentmolding.MoldingEnricher, error) { + return newEnricher(), nil +} + +func (c *dockerComposeCasting) Forge(ctx context.Context, config collectionagent.Casting, p *pourer.Pourer) error { + composeBuf := bytes.NewBuffer(nil) + if err := composeYAMLTemplate.Execute(composeBuf, config); err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to execute compose template") + } + p.AddYAML(composeBuf.Bytes(), "compose.yaml") + + agentConfigs := collectormolding.AgentConfigsOf(&config) + if len(agentConfigs) == 0 { + return foundryerrors.Newf(foundryerrors.TypeInternal, "agent molding produced no configs") + } + for _, ac := range agentConfigs { + p.AddYAML(ac.Content, ac.Path) + } + + return nil +} + +func (c *dockerComposeCasting) Cast(ctx context.Context, config collectionagent.Casting, outputPath string, p *pourer.Pourer) error { + c.logger.InfoContext(ctx, "casting collectionagent via docker compose", slog.String("casting.metadata.name", config.Metadata.Name)) + + composeFile := filepath.Join(outputPath, p.Dir(), "compose.yaml") + if _, err := os.Stat(composeFile); os.IsNotExist(err) { + return foundryerrors.Newf(foundryerrors.TypeNotFound, "compose file does not exist at path: %s", composeFile) + } + + runctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + composeCmd, err := getComposeCommand(runctx) + if err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeNotFound, "docker compose not available") + } + + args := append(composeCmd[1:], "-f", composeFile, "up", "-d") + c.logger.DebugContext(runctx, "running command", slog.String("command", strings.Join(append([]string{composeCmd[0]}, args...), " "))) + + cmd := exec.CommandContext(runctx, composeCmd[0], args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func getComposeCommand(ctx context.Context) ([]string, error) { + if _, err := exec.LookPath("docker"); err == nil { + cmd := exec.CommandContext(ctx, "docker", "compose", "version") + if err := cmd.Run(); err == nil { + return []string{"docker", "compose"}, nil + } + } + if _, err := exec.LookPath("docker-compose"); err == nil { + return []string{"docker-compose"}, nil + } + return nil, foundryerrors.Newf(foundryerrors.TypeNotFound, "neither 'docker compose' nor 'docker-compose' is available") +} diff --git a/internal/casting/collectionagent/dockercomposecasting/embed.go b/internal/casting/collectionagent/dockercomposecasting/embed.go new file mode 100644 index 0000000..39bca94 --- /dev/null +++ b/internal/casting/collectionagent/dockercomposecasting/embed.go @@ -0,0 +1,15 @@ +package dockercomposecasting + +import ( + "embed" + + "github.com/signoz/foundry/internal/domain" +) + +//go:embed templates/*.gotmpl templates/agent/*.gotmpl +var templates embed.FS + +var ( + composeYAMLTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/compose.yaml.gotmpl", domain.FormatYAML) + receiversTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/agent/receivers.yaml.gotmpl", domain.FormatYAML) +) diff --git a/internal/casting/collectionagent/dockercomposecasting/embed_test.go b/internal/casting/collectionagent/dockercomposecasting/embed_test.go new file mode 100644 index 0000000..759cde1 --- /dev/null +++ b/internal/casting/collectionagent/dockercomposecasting/embed_test.go @@ -0,0 +1,34 @@ +package dockercomposecasting + +import ( + "bytes" + "testing" + + "github.com/signoz/foundry/api/v1alpha1/collectionagent" + "github.com/stretchr/testify/assert" +) + +func TestTemplatesParse(t *testing.T) { + t.Parallel() + assert.NotEmpty(t, composeYAMLTemplate) + assert.NotEmpty(t, receiversTemplate) +} + +func TestComposeTemplateExecutes(t *testing.T) { + t.Parallel() + buf := bytes.NewBuffer(nil) + err := composeYAMLTemplate.Execute(buf, collectionagent.Default()) + assert.NoError(t, err) + assert.NotEmpty(t, buf.String()) + assert.Contains(t, buf.String(), "signoz-collectionagent") +} + +func TestReceiversTemplateExecutes(t *testing.T) { + t.Parallel() + var out map[string]map[string]any + err := receiversTemplate.RenderInto(nil, &out) + assert.NoError(t, err) + assert.Contains(t, out, "hostmetrics") + assert.Contains(t, out, "docker_stats") + assert.Contains(t, out, "filelog") +} diff --git a/internal/casting/collectionagent/dockercomposecasting/enricher.go b/internal/casting/collectionagent/dockercomposecasting/enricher.go new file mode 100644 index 0000000..1ad8922 --- /dev/null +++ b/internal/casting/collectionagent/dockercomposecasting/enricher.go @@ -0,0 +1,49 @@ +package dockercomposecasting + +import ( + "context" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/collectionagent" + collectionagentmolding "github.com/signoz/foundry/internal/molding/collectionagent" +) + +var _ collectionagentmolding.MoldingEnricher = (*enricher)(nil) + +type enricher struct{} + +func newEnricher() *enricher { + return &enricher{} +} + +func (e *enricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *collectionagent.Casting) error { + switch kind { + case v1alpha1.MoldingKindCollector: + return e.enrichCollector(config) + } + return nil +} + +func (e *enricher) enrichCollector(config *collectionagent.Casting) error { + status := &config.Spec.Collector.Status + + var bodies map[string]map[string]any + receiversTemplate.MustRenderInto(nil, &bodies) + + pipelines := map[string][]string{ + "hostmetrics": {"metrics"}, + "docker_stats": {"metrics"}, + "filelog": {"logs"}, + } + + status.Receivers = make(map[string]collectionagent.Component, len(bodies)) + for name, body := range bodies { + status.Receivers[name] = collectionagent.Component{ + Body: body, + Pipelines: pipelines[name], + } + } + + status.ResourceDetectors = []string{"docker"} + return nil +} diff --git a/internal/casting/collectionagent/dockercomposecasting/templates/agent/receivers.yaml.gotmpl b/internal/casting/collectionagent/dockercomposecasting/templates/agent/receivers.yaml.gotmpl new file mode 100644 index 0000000..72d8f2c --- /dev/null +++ b/internal/casting/collectionagent/dockercomposecasting/templates/agent/receivers.yaml.gotmpl @@ -0,0 +1,34 @@ +hostmetrics: + collection_interval: 60s + root_path: /hostfs + scrapers: + cpu: {} + disk: {} + load: {} + filesystem: {} + memory: {} + network: {} + paging: {} + process: + mute_process_name_error: true + mute_process_exe_error: true + mute_process_io_error: true + mute_process_user_error: true + processes: {} + +docker_stats: + endpoint: unix:///var/run/docker.sock + collection_interval: 10s + timeout: 20s + +filelog: + include: + - /var/lib/docker/containers/*/*-json.log + start_at: end + include_file_name: false + include_file_path: true + operators: + - id: container-parser + type: container + format: docker + add_metadata_from_filepath: false diff --git a/internal/casting/collectionagent/dockercomposecasting/templates/compose.yaml.gotmpl b/internal/casting/collectionagent/dockercomposecasting/templates/compose.yaml.gotmpl new file mode 100644 index 0000000..e9d186e --- /dev/null +++ b/internal/casting/collectionagent/dockercomposecasting/templates/compose.yaml.gotmpl @@ -0,0 +1,20 @@ +name: signoz-collectionagent +services: + collector: + image: {{ .Spec.Collector.Spec.Image }} + container_name: signoz-collectionagent-collector + restart: unless-stopped + user: "0:0" + network_mode: host + command: ["--config=/etc/otelcol-contrib/config.yaml"] +{{- if .Spec.Collector.Spec.Env }} + environment: +{{- range $key, $value := .Spec.Collector.Spec.Env }} + {{ $key }}: {{ $value }} +{{- end }} +{{- end }} + volumes: + - ./collector/agent/agent.yaml:/etc/otelcol-contrib/config.yaml:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock:ro + - /:/hostfs:ro diff --git a/internal/casting/collectionagent/registry.go b/internal/casting/collectionagent/registry.go index 0756fdb..3e1dbbd 100644 --- a/internal/casting/collectionagent/registry.go +++ b/internal/casting/collectionagent/registry.go @@ -4,8 +4,11 @@ import ( "log/slog" "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/internal/casting/collectionagent/dockercomposecasting" foundryerrors "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/tooler" + "github.com/signoz/foundry/internal/tooler/dockercomposetooler" + "github.com/signoz/foundry/internal/tooler/dockertooler" ) type CastingItem struct { @@ -19,7 +22,15 @@ type Registry struct { func NewRegistry(logger *slog.Logger) *Registry { return &Registry{ - castings: map[v1alpha1.TypeDeployment]CastingItem{}, + castings: map[v1alpha1.TypeDeployment]CastingItem{ + { + Mode: v1alpha1.ModeDocker, + Flavor: v1alpha1.FlavorCompose, + }: { + Casting: dockercomposecasting.New(logger), + Toolers: []tooler.Tooler{dockertooler.New(), dockercomposetooler.New()}, + }, + }, } }