From bfa133d8434867f874e21010c2f0a489b261bfe8 Mon Sep 17 00:00:00 2001 From: DeviousCardi <115358213+DeviousCardi@users.noreply.github.com> Date: Wed, 9 Sep 2026 13:08:31 +0530 Subject: [PATCH] OTLP traces across four stores: Jaeger, Tempo, Quickwit, OpenObserve (Part G) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fifteen checks in a new `otlp-traces` suite, run against four backends that each represent a span in a different shape than OTLP's own — which forced several runner fixes that generalize past traces: - `Runner::send()` and `docker::wait_ready()` now treat a rendered request path starting with `http://`/`https://` as an absolute URL and use it verbatim, bypassing `base_url`. Needed because Jaeger and Tempo split ingest and query across two ports of one container; regression-tested with two separate stub servers so neither endpoint can silently fall back to the other. - `Container` gained `extra_ports` (publish additional container ports) and `config` (an inline file `specmatrix up` writes to a temp path and mounts at a fixed path), for settings with no CLI-flag equivalent — confirmed by reading each binary's own `-help` first. Tempo needs this: its OTLP receiver binds to 127.0.0.1 inside the container unless the config sets an explicit 0.0.0.0 endpoint, and nothing in the API says so, only `docker logs`. - `export_report()` was comparing `request_encoding == "otlp-json"` literally, so it was silently wrong for every OTLP metrics/traces JSON export (`otlp-metrics-json`/`otlp-traces-json` never matched) and its protobuf decode path was hardcoded to the logs response type — a genuine traces or metrics partial-success report decoded to `None` and was invisibly dropped. Now takes `protocol: &str` and branches field names and decode type per protocol; confirmed against Jaeger, and against Quickwit's own genuine json-response-to-protobuf-request mismatch, which survives correctly. - `field_of()` gained a fallback chain for fields with no adapter mapping: `attributes.` searches the record's own OTLP-shaped attributes array, and any other dotted field with no mapping is tried as a JSON pointer. Verified against Tempo, isolating cases via move-aside/restore to confirm each fix in a clean run. Found and fixed a real regression along the way: an earlier edit meant to add `trace_id_base64` to `vars_for()` silently failed to save (a multi-assert Python script whose last assert failed, so the final write never ran), while a separately-applied edit to `read_back()` for the same feature did save — masking the omission. Combined with `first_record()`'s `.contains(run_key)` check, where `"".contains("")` is `true` in Rust, the always-empty `trace_id_base64` spuriously matched the first record in any response, breaking `a_leftover_record_from_another_run_does_not_count`. Fixed by inserting the missing variable and adding a permanent guard — `first_record()` now refuses to match an empty key — plus a regression test. Adapter notes, each confirmed against a running container: - Tempo answers real OTLP JSON on read-back, unlike every other store here, except trace/span IDs come back base64 (protobuf JSON's own bytes-field encoding) where this project's encoder uses hex — compared as bytes, not equal strings. - Jaeger answers only its own query model: `startTime`/`duration` in microseconds, not OTLP's nanoseconds, and a span event becomes a `logs[]` entry keyed by `fields[].key == "event"`. - Quickwit and OpenObserve both flatten spans into search columns; OpenObserve stores a span's events as a JSON string rather than structured JSON, so no JSON-pointer mapping can reach it at all. `event-timestamp-outside-span` is `match: present` rather than `exact` for this reason: no store here keeps OTLP's own field shape for a span event, so the check records where each one put the data rather than judging a shape none of them chose to keep. All 15 cases pass on all 4 columns (60/60). 183 unit tests, 0 clippy warnings. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_018k65nFSzuwsHwYnpSHNaeK --- AGENTS.md | 24 +- Cargo.lock | 6 + Cargo.toml | 2 +- README.md | 21 +- backends/jaeger.yaml | 67 +++++ backends/openobserve.yaml | 30 +++ backends/quickwit.yaml | 35 +++ backends/tempo.yaml | 82 ++++++ cases/otlp-traces/attribute-int64-max.json | 4 + cases/otlp-traces/attribute-int64-max.yaml | 30 +++ .../dropped-attributes-count-nonzero.json | 4 + .../dropped-attributes-count-nonzero.yaml | 29 ++ cases/otlp-traces/end-before-start.json | 3 + cases/otlp-traces/end-before-start.yaml | 30 +++ .../event-timestamp-outside-span.json | 4 + .../event-timestamp-outside-span.yaml | 50 ++++ cases/otlp-traces/kind-unspecified.json | 3 + cases/otlp-traces/kind-unspecified.yaml | 29 ++ cases/otlp-traces/link-to-unknown-trace.json | 4 + cases/otlp-traces/link-to-unknown-trace.yaml | 29 ++ cases/otlp-traces/minimal-span.json | 3 + cases/otlp-traces/minimal-span.yaml | 29 ++ cases/otlp-traces/name-empty.json | 3 + cases/otlp-traces/name-empty.yaml | 31 +++ cases/otlp-traces/parent-in-other-batch.json | 4 + cases/otlp-traces/parent-in-other-batch.yaml | 30 +++ .../resource-without-service-name.json | 3 + .../resource-without-service-name.yaml | 31 +++ cases/otlp-traces/span-id-all-zero.json | 3 + cases/otlp-traces/span-id-all-zero.yaml | 28 ++ cases/otlp-traces/span-without-parent.json | 3 + cases/otlp-traces/span-without-parent.yaml | 27 ++ .../status-code-unset-with-message.json | 4 + .../status-code-unset-with-message.yaml | 28 ++ cases/otlp-traces/trace-id-all-zero.json | 4 + cases/otlp-traces/trace-id-all-zero.yaml | 25 ++ cases/otlp-traces/zero-duration.json | 3 + cases/otlp-traces/zero-duration.yaml | 27 ++ src/backend.rs | 15 ++ src/docker.rs | 65 ++++- src/encode.rs | 24 +- src/otlp.rs | 248 ++++++++++++++++-- src/runner.rs | 214 ++++++++++++++- 43 files changed, 1307 insertions(+), 31 deletions(-) create mode 100644 backends/jaeger.yaml create mode 100644 backends/tempo.yaml create mode 100644 cases/otlp-traces/attribute-int64-max.json create mode 100644 cases/otlp-traces/attribute-int64-max.yaml create mode 100644 cases/otlp-traces/dropped-attributes-count-nonzero.json create mode 100644 cases/otlp-traces/dropped-attributes-count-nonzero.yaml create mode 100644 cases/otlp-traces/end-before-start.json create mode 100644 cases/otlp-traces/end-before-start.yaml create mode 100644 cases/otlp-traces/event-timestamp-outside-span.json create mode 100644 cases/otlp-traces/event-timestamp-outside-span.yaml create mode 100644 cases/otlp-traces/kind-unspecified.json create mode 100644 cases/otlp-traces/kind-unspecified.yaml create mode 100644 cases/otlp-traces/link-to-unknown-trace.json create mode 100644 cases/otlp-traces/link-to-unknown-trace.yaml create mode 100644 cases/otlp-traces/minimal-span.json create mode 100644 cases/otlp-traces/minimal-span.yaml create mode 100644 cases/otlp-traces/name-empty.json create mode 100644 cases/otlp-traces/name-empty.yaml create mode 100644 cases/otlp-traces/parent-in-other-batch.json create mode 100644 cases/otlp-traces/parent-in-other-batch.yaml create mode 100644 cases/otlp-traces/resource-without-service-name.json create mode 100644 cases/otlp-traces/resource-without-service-name.yaml create mode 100644 cases/otlp-traces/span-id-all-zero.json create mode 100644 cases/otlp-traces/span-id-all-zero.yaml create mode 100644 cases/otlp-traces/span-without-parent.json create mode 100644 cases/otlp-traces/span-without-parent.yaml create mode 100644 cases/otlp-traces/status-code-unset-with-message.json create mode 100644 cases/otlp-traces/status-code-unset-with-message.yaml create mode 100644 cases/otlp-traces/trace-id-all-zero.json create mode 100644 cases/otlp-traces/trace-id-all-zero.yaml create mode 100644 cases/otlp-traces/zero-duration.json create mode 100644 cases/otlp-traces/zero-duration.yaml diff --git a/AGENTS.md b/AGENTS.md index 54b201a..a982502 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -143,7 +143,12 @@ runner encodes it to whatever the backend accepts. `send.format` names the format the file is written in, `send.encodings` lists what it may be sent as, and an adapter's `formats:` says what it accepts on the wire. -- `otlp-json` — real OTLP JSON, sendable with `curl` as it stands. +- `otlp-json` — real OTLP JSON, sendable with `curl` as it stands. Logs by + default; `otlp-metrics-json` and `otlp-traces-json` are the same convention + for the other two signals, each with its own protobuf counterpart + (`otlp-metrics-protobuf`, `otlp-traces-protobuf`) — three formats rather than + one because the three OTLP export services are different messages on the + wire, not a shared envelope. - `es-ndjson` — a real `_bulk` body. - `loki-json` — the push body as Loki itself documents it: `streams[].stream` for labels and `streams[].values` as `[timestamp_ns, line]` @@ -222,6 +227,23 @@ a metric named `a.b.c` cannot be named in a query even though the data is there. It is **never** a way to exclude a case the store fails. If you are reaching for it because a verdict is inconvenient, you are writing a false column. +Two `container:` fields exist for settings a CLI flag cannot reach: + +- `extra_ports:` publishes additional container ports, for a store that splits + ingest and query across two ports of one container — Jaeger's OTLP receiver + and its own query API, Tempo's OTLP receiver and its `/ready`/query API. A + `readback.request` or `container.ready.request` naming an absolute URL + (`http://` or `https://`) is sent to that URL verbatim rather than through + `container.port`'s base URL. +- `config:` is an inline file this project's own `specmatrix up` writes to a + temp path and mounts at a fixed path inside the container + (`/etc/specmatrix/config.yaml`), for a required setting with no CLI-flag + equivalent at all — confirm that by reading the binary's own `-help` first, + not by assuming. Tempo's OTLP receiver binds to `127.0.0.1` inside the + container unless its config sets an explicit `0.0.0.0` endpoint, and nothing + in the API says so — only `docker logs` names the bound address, and every + ingest from outside answers a bare connection reset with no HTTP status. + ## Reporting a divergence Findings are worth more filed than tabulated. diff --git a/Cargo.lock b/Cargo.lock index c009c94..aa09bef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -966,7 +966,11 @@ version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" dependencies = [ + "futures-core", + "futures-sink", "js-sys", + "pin-project-lite", + "thiserror", ] [[package]] @@ -993,7 +997,9 @@ dependencies = [ "futures-executor", "futures-util", "opentelemetry", + "percent-encoding", "portable-atomic", + "rand 0.9.5", "thiserror", ] diff --git a/Cargo.toml b/Cargo.toml index 5d084c3..29b444d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,7 +18,7 @@ include = ["src/**", "cases/**", "backends/**", "README.md", "LICENSE", "NOTICE" anyhow = "1" chrono = "0.4" clap = { version = "4", features = ["derive"] } -opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic-messages", "logs", "metrics", "with-serde"] } +opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic-messages", "logs", "metrics", "trace", "with-serde"] } prost = "0.14" rand = "0.9" regex = "1" diff --git a/README.md b/README.md index 4932bec..25c2e3d 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ much longer to build and is what makes the results worth citing. ## Status -Not yet published. Five protocols, ten backends and sixty-six checks run +Not yet published. Six protocols, twelve backends and eighty-one checks run unattended from one command; the write-up that has to precede publication is not done. @@ -123,6 +123,20 @@ returns for that record. And OpenObserve's push landed at request's example used, confirmed only by trying both against a running container rather than trusting the documentation. +OTLP traces followed, across Jaeger, Grafana Tempo, Quickwit and OpenObserve — +fifteen checks, none sharing a read-back shape: Tempo alone echoes OTLP's own +span JSON (with trace and span IDs coming back base64, protobuf JSON's own +encoding for a bytes field, so those two fields compare as bytes rather than as +equal strings); Jaeger answers only its own query model, where a span event +becomes a `logs[]` entry and a timestamp is microseconds rather than OTLP's +nanoseconds; Quickwit flattens spans into search columns; and OpenObserve does +the same but keeps a span's events as a JSON string rather than structured +JSON, unreachable by a field pointer at all. A span event's timestamp and name +are recorded as `present` rather than `exact` for this reason — no store here +keeps OTLP's own shape for it, which makes the check a record of where each +one put the data rather than a pass/fail on a shape none of them chose to +keep. + Not everything that differs is a finding, and the corpus says so. A NaN sample dropped at ingest, an exponential histogram with no representation, a metric name that does or does not gain its unit as a suffix: those are recorded with @@ -140,7 +154,7 @@ was confirmed. Rust and Docker; nothing else. ```sh -cargo test # 166 tests, no network, no containers +cargo test # 183 tests, no network, no containers cargo run -- up --backend loki # start a backend from its adapter cargo run -- run --backend loki --suite otlp-logs @@ -153,6 +167,9 @@ cargo run -- matrix --suite otlp-logs --manage \ cargo run -- matrix --suite remote-write --manage \ --backends prometheus,mimir,greptimedb,victoriametrics +cargo run -- matrix --suite otlp-traces --manage \ + --backends jaeger,tempo,quickwit,openobserve + # the bytes a backend actually receives, for reproducing a finding by hand cargo run -- encode cases/remote-write/minimal-gauge.json \ --from remote-write-json --to remote-write-protobuf --out body.snappy diff --git a/backends/jaeger.yaml b/backends/jaeger.yaml new file mode 100644 index 0000000..3489c0b --- /dev/null +++ b/backends/jaeger.yaml @@ -0,0 +1,67 @@ +# Adapter for Jaeger, all-in-one. +# +# Confirmed by hand against jaegertracing/jaeger:2.20.0 on 2026-09-09. +# +# The OTLP HTTP receiver is on 4318 and needs no flag in Jaeger 2.x — v2 is a +# rewrite around the OpenTelemetry Collector and takes OTLP by default, unlike +# Jaeger 1.x which needed COLLECTOR_OTLP_ENABLED=true. +# +# Read-back is Jaeger's own query API, not OTLP — there is no way to ask +# Jaeger for a span in the shape it was sent, only in Jaeger's model. What had +# to be confirmed by hand rather than assumed: +# +# GET /api/traces/ +# {"data":[{"spans":[{"operationName":"...","startTime":, +# "duration":,"tags":[{"key":...,"type":...,"value":...}], +# "logs":[...], ...}]}]} +# +# `startTime` is microseconds, not the nanoseconds OTLP sends — a precision +# check on this column is checking Jaeger's own model, not a store that failed +# to keep OTLP's precision; nanosecond-precision checks are `present`, not +# `exact`, for this reason and this reason only. + +name: jaeger +container: + image: jaegertracing/jaeger:2.20.0 + port: 4318 + extra_ports: [16686] + env: + COLLECTOR_OTLP_ENABLED: "true" + ready: + # The query UI, not the ingest port, which has no unauthenticated health + # endpoint of its own — and the query port is what a case's read-back + # actually reads from, so this is also confirming the port a case needs + # is the one that is up. + request: GET http://localhost:16686/ + expect_status: 200 + +auth: + kind: none + +protocols: + otlp-traces: + formats: [otlp-traces-json, otlp-traces-protobuf] + ingest: + request: POST /v1/traces + readback: + # A different port than ingest. The adapter's base URL is the ingest + # port (4318, matched to `container.port` so `specmatrix up` reports the + # right address); read-back overrides host and port outright. + request: GET http://localhost:16686/api/traces/{{ trace_id_hex }} + records: /data/0/spans + fields: + name: /operationName + startTimeUnixNano: /startTime + traceId: /traceID + spanId: /spanID + # A span event becomes a Jaeger "log", confirmed by hand on + # 2026-09-09: {"logs":[{"timestamp":, + # "fields":[{"key":"event","type":"string","value":}]}]}. + # Two renames and a precision drop packed into one mapping — there is + # no field in Jaeger's model that is simply "the event's timestamp" or + # "the event's name" at a fixed pointer, this is where they live. + events.0.timeUnixNano: /logs/0/timestamp + events.0.name: /logs/0/fields/0/value + poll: + interval_ms: 1000 + timeout_ms: 30000 diff --git a/backends/openobserve.yaml b/backends/openobserve.yaml index 747fa12..8790bb9 100644 --- a/backends/openobserve.yaml +++ b/backends/openobserve.yaml @@ -104,6 +104,36 @@ protocols: interval_ms: 1000 timeout_ms: 30000 + # Confirmed by hand on 2026-09-09. Traces land in a stream named `default`, + # same as the loki-push protocol above, and answer the same `type=traces` + # search convention. `span_id` and `trace_id` keep their hex spelling; + # `name` becomes `operation_name`; `start_time`/`end_time` are nanoseconds, + # unlike the microseconds this adapter's OTLP-logs `_timestamp` column + # carries — traces keep full precision where logs do not. + otlp-traces: + formats: [otlp-traces-json] + ingest: + request: POST /api/default/v1/traces + headers: + Content-Type: application/json + readback: + request: POST /api/default/_search?type=traces + body: + query: + sql: "SELECT * FROM \"default\" WHERE trace_id = '{{ trace_id_hex }}'" + start_time: "{{ window_start_us }}" + end_time: "{{ window_end_us }}" + records: /hits + fields: + name: /operation_name + traceId: /trace_id + spanId: /span_id + startTimeUnixNano: /start_time + endTimeUnixNano: /end_time + poll: + interval_ms: 1000 + timeout_ms: 30000 + loki-push: formats: [loki-json] # Confirmed by hand on 2026-09-09 against v0.92.2. openobserve/openobserve diff --git a/backends/quickwit.yaml b/backends/quickwit.yaml index 791a6b7..6f76a4e 100644 --- a/backends/quickwit.yaml +++ b/backends/quickwit.yaml @@ -130,6 +130,41 @@ protocols: # to wait longer, not to lower the store's commit timeout: waiting changes # nothing about what Quickwit does, and tuning it would be the corpus # arranging its own result. + # Confirmed by hand on 2026-09-09. Quickwit creates otel-traces-v0_7 itself + # at start-up, the same way it creates the logs index; protobuf only, same + # as the OTLP logs endpoint. Read-back field names are Quickwit's own + # flattened columns rather than the OTLP JSON names. + otlp-traces: + formats: [otlp-traces-protobuf] + setup_verify: + request: GET /api/v1/indexes/otel-traces-v0_7 + poll: + interval_ms: 500 + timeout_ms: 60000 + ingest: + request: POST /api/v1/otlp/v1/traces + headers: + Content-Type: application/x-protobuf + readback: + request: GET /api/v1/otel-traces-v0_7/search?query=trace_id:{{ trace_id_hex }}&max_hits=10 + records: /hits + fields: + name: /span_name + traceId: /trace_id + spanId: /span_id + startTimeUnixNano: /span_start_timestamp_nanos + endTimeUnixNano: /span_end_timestamp_nanos + # Confirmed by hand on 2026-09-09: an event becomes one entry of a + # top-level `events` array, `{"event_name":...,"event_timestamp_nanos":...}` + # — renamed from OTLP's own `name`/`timeUnixNano`, full nanosecond + # precision kept. + events.0.timeUnixNano: /events/0/event_timestamp_nanos + events.0.name: /events/0/event_name + poll: + interval_ms: 1000 + timeout_ms: 30000 + + es-bulk: formats: [es-ndjson] # Quickwit will not create an index on write, so this protocol creates one diff --git a/backends/tempo.yaml b/backends/tempo.yaml new file mode 100644 index 0000000..436464c --- /dev/null +++ b/backends/tempo.yaml @@ -0,0 +1,82 @@ +# Adapter for Grafana Tempo, single-binary local storage. +# +# Confirmed by hand against grafana/tempo:3.0.3 on 2026-09-09. +# +# Two settings below have no CLI-flag equivalent — confirmed against the +# binary's own -help, which exposes -storage.trace.backend but nothing for +# the receiver — so they come from `container.config`, an inline file this +# project's own `specmatrix up` mounts, rather than a flag on `command`. +# +# The OTLP receiver's `endpoint: 0.0.0.0:4318` matters and is not +# boilerplate: without it, Tempo starts the receiver bound to 127.0.0.1 only, +# inside the container, and every ingest from outside answers a bare +# connection reset with no HTTP status at all. `docker logs` names the bound +# address; nothing in the API does, and no error is more misleading than +# "the connection reset" for a value it never validated against. +# +# Read-back answers real OTLP JSON, unlike Jaeger's own model — batches, +# scopeSpans, spans, the same shape a case sends — except trace_id and span_id +# come back base64, which is protobuf JSON's own encoding for a bytes field +# and not a Tempo rewrite; the crate this project encodes with uses a hex +# serializer for the same fields instead, so those two fields need comparing +# as bytes, not as equal strings — a check on them is `present`, not `exact`. +# +# `/api/traces/` answers 404 until the trace has flushed from the +# ingester, which local storage does quickly; `ready.settle_ms` covers the gap +# between the query API accepting connections and a trace sent at that moment +# actually flushing. + +name: tempo +version_from: + request: GET http://localhost:3200/api/status/buildinfo + field: version +container: + image: grafana/tempo:3.0.3 + port: 4318 + extra_ports: [3200] + command: ["-config.file=/etc/specmatrix/config.yaml"] + config: | + server: + http_listen_port: 3200 + distributor: + receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + storage: + trace: + backend: local + local: + path: /var/tempo/traces + wal: + path: /var/tempo/wal + ready: + request: GET http://localhost:3200/ready + expect_status: 200 + # Measured on 2026-09-09: /ready answers within a couple of seconds but a + # trace sent in the same moment can still 404 for several more while the + # ingester finishes starting. Margin over that gap, not a tuning of what + # gets measured. + settle_ms: 5000 + +auth: + kind: none + +protocols: + otlp-traces: + formats: [otlp-traces-json, otlp-traces-protobuf] + ingest: + request: POST /v1/traces + readback: + request: GET http://localhost:3200/api/traces/{{ trace_id_hex }} + records: /batches/0/scopeSpans/0/spans + fields: + name: /name + traceId: /traceId + spanId: /spanId + startTimeUnixNano: /startTimeUnixNano + endTimeUnixNano: /endTimeUnixNano + poll: + interval_ms: 1000 + timeout_ms: 30000 diff --git a/cases/otlp-traces/attribute-int64-max.json b/cases/otlp-traces/attribute-int64-max.json new file mode 100644 index 0000000..a0f168b --- /dev/null +++ b/cases/otlp-traces/attribute-int64-max.json @@ -0,0 +1,4 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"specmatrix"}}]}, +"scopeSpans":[{"scope":{"name":"specmatrix"},"spans":[{"traceId":"{{ trace_id_hex }}","spanId":"0902030405060708", +"name":"big-attribute","kind":1,"startTimeUnixNano":"{{ now_ns }}","endTimeUnixNano":"{{ now_ns_fractional }}", +"attributes":[{"key":"specmatrix.bignum","value":{"intValue":"9223372036854775807"}}]}]}]}]} diff --git a/cases/otlp-traces/attribute-int64-max.yaml b/cases/otlp-traces/attribute-int64-max.yaml new file mode 100644 index 0000000..0c5b0c2 --- /dev/null +++ b/cases/otlp-traces/attribute-int64-max.yaml @@ -0,0 +1,30 @@ +id: otlp-traces/attribute-int64-max +protocol: otlp-traces +title: A span attribute at the largest int64 + +rule: + basis: spec + spec: opentelemetry-proto/common/v1 + section: AnyValue.int_value + text: > + int_value is an int64. OTLP's JSON mapping writes it as a string because + a JSON number cannot hold every int64 exactly, and a store whose + attribute storage is a JSON number or a float cannot represent this value + without loss. + +send: + format: otlp-traces-json + encodings: [otlp-traces-json, otlp-traces-protobuf] + body: cases/otlp-traces/attribute-int64-max.json + +expect: + ingest: accepted + readback: + match: present + on: [attributes.specmatrix.bignum] + +notes: > + Recorded rather than judged, like the equivalent metrics and remote-write + cases: any loss here is a property of a store's attribute storage, not a + conformance failure by itself, and the row exists so a reader can see + whether it agrees with the other columns. diff --git a/cases/otlp-traces/dropped-attributes-count-nonzero.json b/cases/otlp-traces/dropped-attributes-count-nonzero.json new file mode 100644 index 0000000..3cbd99a --- /dev/null +++ b/cases/otlp-traces/dropped-attributes-count-nonzero.json @@ -0,0 +1,4 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"specmatrix"}}]}, +"scopeSpans":[{"scope":{"name":"specmatrix"},"spans":[{"traceId":"{{ trace_id_hex }}","spanId":"0c02030405060708", +"name":"dropped-attrs","kind":1,"startTimeUnixNano":"{{ now_ns }}","endTimeUnixNano":"{{ now_ns_fractional }}", +"droppedAttributesCount":7}]}]}]} diff --git a/cases/otlp-traces/dropped-attributes-count-nonzero.yaml b/cases/otlp-traces/dropped-attributes-count-nonzero.yaml new file mode 100644 index 0000000..1a6ef8b --- /dev/null +++ b/cases/otlp-traces/dropped-attributes-count-nonzero.yaml @@ -0,0 +1,29 @@ +id: otlp-traces/dropped-attributes-count-nonzero +protocol: otlp-traces +title: A span reporting attributes it dropped, honestly + +rule: + basis: spec + spec: opentelemetry-proto/trace/v1 + section: Span.dropped_attributes_count + text: > + The number of attributes that were discarded, because their keys were too + long or there were too many. A span reporting this alongside zero + attributes it kept is describing an SDK-side limit, not an error, and + must be accepted as such. + +send: + format: otlp-traces-json + encodings: [otlp-traces-json, otlp-traces-protobuf] + body: cases/otlp-traces/dropped-attributes-count-nonzero.json + +expect: + ingest: accepted + readback: + match: present + on: [droppedAttributesCount] + +notes: > + `present`: nothing requires a store to surface this count back through its + query API, only to accept the span. The row records whether it is visible, + which is worth knowing without being a divergence on its own. diff --git a/cases/otlp-traces/end-before-start.json b/cases/otlp-traces/end-before-start.json new file mode 100644 index 0000000..99839bb --- /dev/null +++ b/cases/otlp-traces/end-before-start.json @@ -0,0 +1,3 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"specmatrix"}}]}, +"scopeSpans":[{"scope":{"name":"specmatrix"},"spans":[{"traceId":"{{ trace_id_hex }}","spanId":"0302030405060708", +"name":"backwards-span","kind":1,"startTimeUnixNano":"{{ now_ns }}","endTimeUnixNano":"{{ now_minus_1d_ns }}"}]}]}]} diff --git a/cases/otlp-traces/end-before-start.yaml b/cases/otlp-traces/end-before-start.yaml new file mode 100644 index 0000000..b1cc9c6 --- /dev/null +++ b/cases/otlp-traces/end-before-start.yaml @@ -0,0 +1,30 @@ +id: otlp-traces/end-before-start +protocol: otlp-traces +title: A span whose end_time is before its start_time + +rule: + basis: spec + spec: opentelemetry-proto/trace/v1 + section: Span.end_time_unix_nano + text: > + Both times are semantically required and end_time >= start_time is + expected, not enforced by the message shape — a value smaller than + start_time is a well-formed message describing an impossible duration. + Refusing it is conformant; accepting and silently correcting it is not, + because a corrected duration is a fabricated measurement. + +send: + format: otlp-traces-json + encodings: [otlp-traces-json, otlp-traces-protobuf] + body: cases/otlp-traces/end-before-start.json + +expect: + ingest: accepted-or-rejected + readback: + match: present + on: [name] + +notes: > + `present`, not `exact`: the specification says what is expected of a well- + formed span, not what a receiver must do when handed one that is not. The + row records whether a store notices at all. diff --git a/cases/otlp-traces/event-timestamp-outside-span.json b/cases/otlp-traces/event-timestamp-outside-span.json new file mode 100644 index 0000000..d3288f5 --- /dev/null +++ b/cases/otlp-traces/event-timestamp-outside-span.json @@ -0,0 +1,4 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"specmatrix"}}]}, +"scopeSpans":[{"scope":{"name":"specmatrix"},"spans":[{"traceId":"{{ trace_id_hex }}","spanId":"0702030405060708", +"name":"event-outside-span","kind":1,"startTimeUnixNano":"{{ now_ns }}","endTimeUnixNano":"{{ now_ns_fractional }}", +"events":[{"timeUnixNano":"{{ now_minus_1d_ns }}","name":"stale-event"}]}]}]}]} diff --git a/cases/otlp-traces/event-timestamp-outside-span.yaml b/cases/otlp-traces/event-timestamp-outside-span.yaml new file mode 100644 index 0000000..a982291 --- /dev/null +++ b/cases/otlp-traces/event-timestamp-outside-span.yaml @@ -0,0 +1,50 @@ +id: otlp-traces/event-timestamp-outside-span +protocol: otlp-traces +title: A span event timestamped before the span's own start time + +rule: + basis: spec + spec: opentelemetry-proto/trace/v1 + section: Span.Event + text: > + An event's time_unix_nano is independent of the span's start and end + times; nothing in the message ties them together, and a receiver that + requires an event to fall inside its span's interval is enforcing a rule + the format does not state. + +send: + format: otlp-traces-json + encodings: [otlp-traces-json, otlp-traces-protobuf] + body: cases/otlp-traces/event-timestamp-outside-span.json + +expect: + ingest: accepted + readback: + match: present + on: + - events.0.timeUnixNano + +notes: > + `present` rather than `exact`: every store confirmed here keeps the event, + but none returns it at OTLP's own `events[].timeUnixNano` pointer — an + event becomes a Jaeger "log" with a microsecond timestamp, a renamed + top-level array entry in Quickwit, or a JSON string in OpenObserve rather + than nested structure. None of that is a rule violation; the specification + does not say how a store must expose what it kept, only that it keep it. + This row is a discoverability record, not a conformance judgement. + + Confirmed by hand on 2026-09-09, all four with a control span (this one) in + the same export: + + tempo 3.0.3 present at the OTLP pointer, nanosecond precision kept + jaeger 2.20.0 present under /logs/0/timestamp, microsecond precision + quickwit 0.8.2 present under /events/0/event_timestamp_nanos, nanosecond precision kept + openobserve 0.92.2 present, but as a JSON string value rather than + structured JSON — confirmed by direct SQL query, not + reachable through a JSON pointer, so this row reads + it as absent though the data is there + + A store that genuinely dropped the event would still show as absent here, + indistinguishably from OpenObserve's case above — the two are not the same + fact, and telling them apart needed a query by hand, which is recorded so + the next reader does not have to redo it. diff --git a/cases/otlp-traces/kind-unspecified.json b/cases/otlp-traces/kind-unspecified.json new file mode 100644 index 0000000..6208c9a --- /dev/null +++ b/cases/otlp-traces/kind-unspecified.json @@ -0,0 +1,3 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"specmatrix"}}]}, +"scopeSpans":[{"scope":{"name":"specmatrix"},"spans":[{"traceId":"{{ trace_id_hex }}","spanId":"0b02030405060708", +"name":"unspecified-kind","startTimeUnixNano":"{{ now_ns }}","endTimeUnixNano":"{{ now_ns_fractional }}"}]}]}]} diff --git a/cases/otlp-traces/kind-unspecified.yaml b/cases/otlp-traces/kind-unspecified.yaml new file mode 100644 index 0000000..f233cdc --- /dev/null +++ b/cases/otlp-traces/kind-unspecified.yaml @@ -0,0 +1,29 @@ +id: otlp-traces/kind-unspecified +protocol: otlp-traces +title: A span with no kind is SPAN_KIND_UNSPECIFIED, not malformed + +rule: + basis: spec + spec: opentelemetry-proto/trace/v1 + section: Span.SpanKind + text: > + SPAN_KIND_UNSPECIFIED is enum value 0, the field's default, so a span + omitting `kind` entirely is a well-formed message specifying the + unspecified kind explicitly by omission — proto3's ordinary rule for an + enum default. + +send: + format: otlp-traces-json + encodings: [otlp-traces-json, otlp-traces-protobuf] + body: cases/otlp-traces/kind-unspecified.json + +expect: + ingest: accepted + readback: + match: exact + on: [name] + +notes: > + A store that requires a kind and refuses this span is enforcing a rule the + format does not have: kind is not documented as required, unlike name, + start_time_unix_nano and trace_id. diff --git a/cases/otlp-traces/link-to-unknown-trace.json b/cases/otlp-traces/link-to-unknown-trace.json new file mode 100644 index 0000000..5128acb --- /dev/null +++ b/cases/otlp-traces/link-to-unknown-trace.json @@ -0,0 +1,4 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"specmatrix"}}]}, +"scopeSpans":[{"scope":{"name":"specmatrix"},"spans":[{"traceId":"{{ trace_id_hex }}","spanId":"0802030405060708", +"name":"span-with-link","kind":1,"startTimeUnixNano":"{{ now_ns }}","endTimeUnixNano":"{{ now_ns_fractional }}", +"links":[{"traceId":"ffeeddccbbaa99887766554433221100","spanId":"aabbccddeeff0011"}]}]}]}]} diff --git a/cases/otlp-traces/link-to-unknown-trace.yaml b/cases/otlp-traces/link-to-unknown-trace.yaml new file mode 100644 index 0000000..3fd3354 --- /dev/null +++ b/cases/otlp-traces/link-to-unknown-trace.yaml @@ -0,0 +1,29 @@ +id: otlp-traces/link-to-unknown-trace +protocol: otlp-traces +title: A link naming a trace this suite never sent + +rule: + basis: spec + spec: opentelemetry-proto/trace/v1 + section: Span.Link + text: > + A Link is a reference from this span to a span in the same or a different + trace. A different trace is the ordinary case for a link — the whole + reason the field exists — and it names nothing the receiver is required + to have seen or ever will. + +send: + format: otlp-traces-json + encodings: [otlp-traces-json, otlp-traces-protobuf] + body: cases/otlp-traces/link-to-unknown-trace.json + +expect: + ingest: accepted + readback: + match: exact + on: [name] + +notes: > + The linked trace id is fabricated and never sent as a trace in its own + right, which is deliberate: a store that tries to resolve it before + accepting this span would refuse or drop ordinary cross-trace linking. diff --git a/cases/otlp-traces/minimal-span.json b/cases/otlp-traces/minimal-span.json new file mode 100644 index 0000000..db162bb --- /dev/null +++ b/cases/otlp-traces/minimal-span.json @@ -0,0 +1,3 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"specmatrix"}}]}, +"scopeSpans":[{"scope":{"name":"specmatrix"},"spans":[{"traceId":"{{ trace_id_hex }}","spanId":"0102030405060708", +"name":"specmatrix-minimal-span","kind":1,"startTimeUnixNano":"{{ now_ns }}","endTimeUnixNano":"{{ now_ns_fractional }}"}]}]}]} diff --git a/cases/otlp-traces/minimal-span.yaml b/cases/otlp-traces/minimal-span.yaml new file mode 100644 index 0000000..92ca63a --- /dev/null +++ b/cases/otlp-traces/minimal-span.yaml @@ -0,0 +1,29 @@ +id: otlp-traces/minimal-span +protocol: otlp-traces +title: An ordinary span is written and reads back unchanged +control: true + +rule: + basis: spec + spec: opentelemetry/otlp/1.3.0 + section: trace-service + text: > + An ExportTraceServiceRequest carries resource spans, scope spans and + spans. One span with a trace id, a span id, a name and start/end times is + the smallest conformant export there is. + +send: + format: otlp-traces-json + encodings: [otlp-traces-json, otlp-traces-protobuf] + body: cases/otlp-traces/minimal-span.json + +expect: + ingest: accepted + readback: + match: exact + on: [name] + +notes: > + The control for the OTLP-traces suite. Read back by trace id rather than a + run-key attribute — a span has no equivalent field every backend indexes + the same way, and a trace is found by its trace id or not at all. diff --git a/cases/otlp-traces/name-empty.json b/cases/otlp-traces/name-empty.json new file mode 100644 index 0000000..52f1e48 --- /dev/null +++ b/cases/otlp-traces/name-empty.json @@ -0,0 +1,3 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"specmatrix"}}]}, +"scopeSpans":[{"scope":{"name":"specmatrix"},"spans":[{"traceId":"{{ trace_id_hex }}","spanId":"0a02030405060708", +"name":"","kind":1,"startTimeUnixNano":"{{ now_ns }}","endTimeUnixNano":"{{ now_ns_fractional }}"}]}]}]} diff --git a/cases/otlp-traces/name-empty.yaml b/cases/otlp-traces/name-empty.yaml new file mode 100644 index 0000000..a72bda0 --- /dev/null +++ b/cases/otlp-traces/name-empty.yaml @@ -0,0 +1,31 @@ +id: otlp-traces/name-empty +protocol: otlp-traces +title: A span with an empty name is not an unnamed span + +rule: + basis: spec + spec: opentelemetry-proto/trace/v1 + section: Span.name + text: > + This field is semantically required to be set to a non-empty string; + empty value is equivalent to an unknown span name. That equivalence + describes what the empty string means, not that a receiver may discard + the span or invent a name for it. + +send: + format: otlp-traces-json + encodings: [otlp-traces-json, otlp-traces-protobuf] + body: cases/otlp-traces/name-empty.json + +expect: + ingest: accepted + readback: + match: present + on: [name] + +notes: > + `present`: the specification says what an empty name means, not what a + store does when it renders one — substituting a placeholder like + "unknown_service" or "(unnamed)" is a legitimate reading of "equivalent to + unknown", and this row records which each store chose without calling + either wrong on its own. diff --git a/cases/otlp-traces/parent-in-other-batch.json b/cases/otlp-traces/parent-in-other-batch.json new file mode 100644 index 0000000..c5683a1 --- /dev/null +++ b/cases/otlp-traces/parent-in-other-batch.json @@ -0,0 +1,4 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"specmatrix"}}]}, +"scopeSpans":[{"scope":{"name":"specmatrix"},"spans":[{"traceId":"{{ trace_id_hex }}","spanId":"0202030405060708", +"parentSpanId":"9902030405060708","name":"child-of-elsewhere","kind":1, +"startTimeUnixNano":"{{ now_ns }}","endTimeUnixNano":"{{ now_ns_fractional }}"}]}]}]} diff --git a/cases/otlp-traces/parent-in-other-batch.yaml b/cases/otlp-traces/parent-in-other-batch.yaml new file mode 100644 index 0000000..931f53a --- /dev/null +++ b/cases/otlp-traces/parent-in-other-batch.yaml @@ -0,0 +1,30 @@ +id: otlp-traces/parent-in-other-batch +protocol: otlp-traces +title: A span whose parent was sent in an earlier, separate export + +rule: + basis: spec + spec: opentelemetry/trace-semantics + section: batching + text: > + Spans of one trace routinely arrive in separate batches — different + services, different export intervals — and a receiver cannot require a + parent to have already arrived. A span naming a parent id it has not seen + yet must still be kept. + +send: + format: otlp-traces-json + encodings: [otlp-traces-json, otlp-traces-protobuf] + body: cases/otlp-traces/parent-in-other-batch.json + +expect: + ingest: accepted + readback: + match: exact + on: [name] + +notes: > + The parent id here is deliberately never sent by this suite at all, which is + the point: a store that requires resolving it before accepting the child + would refuse or drop the span, and distributed tracing depends on that never + being required. diff --git a/cases/otlp-traces/resource-without-service-name.json b/cases/otlp-traces/resource-without-service-name.json new file mode 100644 index 0000000..b51ae27 --- /dev/null +++ b/cases/otlp-traces/resource-without-service-name.json @@ -0,0 +1,3 @@ +{"resourceSpans":[{"resource":{"attributes":[]}, +"scopeSpans":[{"scope":{"name":"specmatrix"},"spans":[{"traceId":"{{ trace_id_hex }}","spanId":"0d02030405060708", +"name":"no-service-name","kind":1,"startTimeUnixNano":"{{ now_ns }}","endTimeUnixNano":"{{ now_ns_fractional }}"}]}]}]} diff --git a/cases/otlp-traces/resource-without-service-name.yaml b/cases/otlp-traces/resource-without-service-name.yaml new file mode 100644 index 0000000..320712a --- /dev/null +++ b/cases/otlp-traces/resource-without-service-name.yaml @@ -0,0 +1,31 @@ +id: otlp-traces/resource-without-service-name +protocol: otlp-traces +title: A resource with no service.name attribute + +rule: + basis: spec + spec: opentelemetry/resource-semantic-conventions + section: service.name + text: > + service.name is strongly recommended but the OTLP resource message + carries an arbitrary attribute list — nothing in the wire format requires + it, and a resource omitting it is well-formed. A store that fills in a + default is exercising a documented recommendation to receivers, not + correcting malformed input. + +send: + format: otlp-traces-json + encodings: [otlp-traces-json, otlp-traces-protobuf] + body: cases/otlp-traces/resource-without-service-name.json + +expect: + ingest: accepted + readback: + match: present + on: [name] + +notes: > + `present` rather than `exact`, matching otlp-logs/resource-without-attributes: + what a store names the service by default — "unknown_service", empty, or + something else — is not settled by the wire format, only that the span + itself must be kept and found. diff --git a/cases/otlp-traces/span-id-all-zero.json b/cases/otlp-traces/span-id-all-zero.json new file mode 100644 index 0000000..9f2481c --- /dev/null +++ b/cases/otlp-traces/span-id-all-zero.json @@ -0,0 +1,3 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"specmatrix"}}]}, +"scopeSpans":[{"scope":{"name":"specmatrix"},"spans":[{"traceId":"{{ trace_id_hex }}","spanId":"0000000000000000", +"name":"zero-span-id","kind":1,"startTimeUnixNano":"{{ now_ns }}","endTimeUnixNano":"{{ now_ns_fractional }}"}]}]}]} diff --git a/cases/otlp-traces/span-id-all-zero.yaml b/cases/otlp-traces/span-id-all-zero.yaml new file mode 100644 index 0000000..bc7bca9 --- /dev/null +++ b/cases/otlp-traces/span-id-all-zero.yaml @@ -0,0 +1,28 @@ +id: otlp-traces/span-id-all-zero +protocol: otlp-traces +title: A span id of all zero bytes is invalid + +rule: + basis: spec + spec: opentelemetry-proto/trace/v1 + section: Span.span_id + text: > + An ID with all zeroes is considered invalid. This is one of the two + documented invalid shapes trace and span ids can take, and refusing the + span is conformant. + +send: + format: otlp-traces-json + encodings: [otlp-traces-json, otlp-traces-protobuf] + body: cases/otlp-traces/span-id-all-zero.json + +expect: + ingest: accepted-or-rejected + readback: + match: present + on: [name] + +notes: > + `present`, since the specification names the shape invalid without + mandating what a receiver does about it. What is worth recording is + whether a store stores it silently, unremarked, or refuses it outright. diff --git a/cases/otlp-traces/span-without-parent.json b/cases/otlp-traces/span-without-parent.json new file mode 100644 index 0000000..add1b29 --- /dev/null +++ b/cases/otlp-traces/span-without-parent.json @@ -0,0 +1,3 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"specmatrix"}}]}, +"scopeSpans":[{"scope":{"name":"specmatrix"},"spans":[{"traceId":"{{ trace_id_hex }}","spanId":"0102030405060708", +"name":"root-span","kind":1,"startTimeUnixNano":"{{ now_ns }}","endTimeUnixNano":"{{ now_ns_fractional }}"}]}]}]} diff --git a/cases/otlp-traces/span-without-parent.yaml b/cases/otlp-traces/span-without-parent.yaml new file mode 100644 index 0000000..1e16d3a --- /dev/null +++ b/cases/otlp-traces/span-without-parent.yaml @@ -0,0 +1,27 @@ +id: otlp-traces/span-without-parent +protocol: otlp-traces +title: A span with no parentSpanId is a root span, not malformed + +rule: + basis: spec + spec: opentelemetry-proto/trace/v1 + section: Span.parent_span_id + text: > + The parent_span_id of this span's parent span. If this is a root span, + then this field must be empty. A span omitting the field entirely, which + is what an SDK's root span produces, is exactly this case. + +send: + format: otlp-traces-json + encodings: [otlp-traces-json, otlp-traces-protobuf] + body: cases/otlp-traces/span-without-parent.json + +expect: + ingest: accepted + readback: + match: exact + on: [name] + +notes: > + A store that refuses a span for lacking a parent, or invents one, fails a + case that is ordinary traffic: every trace has exactly one root span. diff --git a/cases/otlp-traces/status-code-unset-with-message.json b/cases/otlp-traces/status-code-unset-with-message.json new file mode 100644 index 0000000..525da14 --- /dev/null +++ b/cases/otlp-traces/status-code-unset-with-message.json @@ -0,0 +1,4 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"specmatrix"}}]}, +"scopeSpans":[{"scope":{"name":"specmatrix"},"spans":[{"traceId":"{{ trace_id_hex }}","spanId":"0602030405060708", +"name":"unset-with-message","kind":1,"startTimeUnixNano":"{{ now_ns }}","endTimeUnixNano":"{{ now_ns_fractional }}", +"status":{"code":0,"message":"a message with no error"}}]}]}]} diff --git a/cases/otlp-traces/status-code-unset-with-message.yaml b/cases/otlp-traces/status-code-unset-with-message.yaml new file mode 100644 index 0000000..386e43b --- /dev/null +++ b/cases/otlp-traces/status-code-unset-with-message.yaml @@ -0,0 +1,28 @@ +id: otlp-traces/status-code-unset-with-message +protocol: otlp-traces +title: A status with code UNSET but a non-empty message + +rule: + basis: spec + spec: opentelemetry-proto/trace/v1 + section: Status + text: > + When Status isn't set, it means the span's status code is unset. Nothing + in the message forbids a message being present on an otherwise-unset + status; message is meaningful primarily on Error but the field does not + enforce that, so carrying one here is a well-formed, if unusual, span. + +send: + format: otlp-traces-json + encodings: [otlp-traces-json, otlp-traces-protobuf] + body: cases/otlp-traces/status-code-unset-with-message.json + +expect: + ingest: accepted + readback: + match: present + on: [name] + +notes: > + `present`: whether a store keeps a message on an UNSET status is not + settled by the specification, only that the span itself must be kept. diff --git a/cases/otlp-traces/trace-id-all-zero.json b/cases/otlp-traces/trace-id-all-zero.json new file mode 100644 index 0000000..d10035a --- /dev/null +++ b/cases/otlp-traces/trace-id-all-zero.json @@ -0,0 +1,4 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"specmatrix"}}]}, +"scopeSpans":[{"scope":{"name":"specmatrix"},"spans":[{"traceId":"00000000000000000000000000000000", +"spanId":"0502030405060708","name":"zero-trace-id","kind":1, +"startTimeUnixNano":"{{ now_ns }}","endTimeUnixNano":"{{ now_ns_fractional }}"}]}]}]} diff --git a/cases/otlp-traces/trace-id-all-zero.yaml b/cases/otlp-traces/trace-id-all-zero.yaml new file mode 100644 index 0000000..48caa64 --- /dev/null +++ b/cases/otlp-traces/trace-id-all-zero.yaml @@ -0,0 +1,25 @@ +id: otlp-traces/trace-id-all-zero +protocol: otlp-traces +title: A trace id of all zero bytes is invalid + +rule: + basis: spec + spec: opentelemetry-proto/trace/v1 + section: Span.trace_id + text: > + An ID with all zeroes OR of length other than 16 bytes is considered + invalid. The all-zero trace id is the more common accident — a + zero-initialised buffer never filled in — and refusing it is conformant. + +send: + format: otlp-traces-json + encodings: [otlp-traces-json, otlp-traces-protobuf] + body: cases/otlp-traces/trace-id-all-zero.json + +expect: + ingest: accepted-or-rejected + +notes: > + No read-back: this suite finds every other case by its trace id, and a + trace id of all zeros is exactly the one value that cannot be used to find + anything back. What is checked is the answer at the door. diff --git a/cases/otlp-traces/zero-duration.json b/cases/otlp-traces/zero-duration.json new file mode 100644 index 0000000..c2a13f4 --- /dev/null +++ b/cases/otlp-traces/zero-duration.json @@ -0,0 +1,3 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"specmatrix"}}]}, +"scopeSpans":[{"scope":{"name":"specmatrix"},"spans":[{"traceId":"{{ trace_id_hex }}","spanId":"0402030405060708", +"name":"instant-span","kind":1,"startTimeUnixNano":"{{ now_ns }}","endTimeUnixNano":"{{ now_ns }}"}]}]}]} diff --git a/cases/otlp-traces/zero-duration.yaml b/cases/otlp-traces/zero-duration.yaml new file mode 100644 index 0000000..bde4029 --- /dev/null +++ b/cases/otlp-traces/zero-duration.yaml @@ -0,0 +1,27 @@ +id: otlp-traces/zero-duration +protocol: otlp-traces +title: A span whose end_time equals its start_time + +rule: + basis: spec + spec: opentelemetry-proto/trace/v1 + section: Span.end_time_unix_nano + text: > + end_time >= start_time permits equality. A span with zero measured + duration is what an operation too fast for the clock's resolution to + distinguish produces, and it is conformant. + +send: + format: otlp-traces-json + encodings: [otlp-traces-json, otlp-traces-protobuf] + body: cases/otlp-traces/zero-duration.json + +expect: + ingest: accepted + readback: + match: exact + on: [name] + +notes: > + A store computing a positive duration by assumption, rather than reading + the two timestamps as given, is the failure mode this exists to catch. diff --git a/src/backend.rs b/src/backend.rs index 6bffec3..6fdaf26 100644 --- a/src/backend.rs +++ b/src/backend.rs @@ -61,8 +61,23 @@ pub struct Container { #[serde(default)] pub command: Vec, pub port: u16, + /// Further host ports to publish, 1:1 with the container. `port` is the + /// one `specmatrix up` reports as the backend's base URL; a store that + /// answers ingest and query on two ports of one container — Jaeger, + /// Tempo — needs the second published too, or the adapter's own + /// cross-port read-back has nothing to reach. + #[serde(default)] + pub extra_ports: Vec, #[serde(default)] pub env: HashMap, + /// Inline file content, written to a temporary file and bind-mounted at + /// `/etc/specmatrix/config.yaml` before the container starts. For a store + /// whose settings this project needs are not all exposed as CLI flags — + /// Tempo's receiver bind address and storage backend, confirmed by hand + /// to have no flag equivalent for either. `command` references the fixed + /// mount path itself; the adapter is what knows what flag the image wants + /// pointed at it. + pub config: Option, pub ready: Option, } diff --git a/src/docker.rs b/src/docker.rs index 87eea0c..23cf77c 100644 --- a/src/docker.rs +++ b/src/docker.rs @@ -31,6 +31,14 @@ pub fn check_pinned(container: &Container) -> Result<()> { Ok(()) } +/// Where an inline `container.config` is written before the container starts. +/// Fixed rather than derived from the backend name: `down` never needs to +/// know it, and a stale file from a previous run is overwritten, not +/// accumulated. +fn config_host_path(backend: &str) -> std::path::PathBuf { + std::env::temp_dir().join(format!("specmatrix-{backend}-config.yaml")) +} + pub fn docker_run_args(backend: &str, container: &Container) -> Vec { let mut args = vec![ "run".to_string(), @@ -40,6 +48,14 @@ pub fn docker_run_args(backend: &str, container: &Container) -> Vec { "-p".to_string(), format!("{0}:{0}", container.port), ]; + for port in &container.extra_ports { + args.push("-p".to_string()); + args.push(format!("{port}:{port}")); + } + if container.config.is_some() { + args.push("-v".to_string()); + args.push(format!("{}:/etc/specmatrix/config.yaml", config_host_path(backend).display())); + } // Sorted, so two runs of one adapter produce the same command line and a // difference in a log is a real difference. let mut env: Vec<_> = container.env.iter().collect(); @@ -57,6 +73,10 @@ pub fn docker_run_args(backend: &str, container: &Container) -> Vec { pub fn up(backend: &str, container: &Container) -> Result { check_pinned(container)?; let _ = down(backend); + if let Some(config) = &container.config { + std::fs::write(config_host_path(backend), config) + .context("writing the adapter's inline config to a temp file")?; + } let output = Command::new("docker") .args(docker_run_args(backend, container)) .output() @@ -79,16 +99,26 @@ fn wait_ready(base_url: &str, request: &str, expect_status: u16) -> Result<()> { let (_, path) = request .split_once(' ') .with_context(|| format!("ready.request must be `METHOD /path`, got {request:?}"))?; + let path = path.trim(); + // An absolute URL is used as written, same as `send()` does for a + // read-back on a store's other port — Tempo's readiness is its query + // API on 3200, a different port than the OTLP receiver `container.port` + // points `base_url` at. + let url = if path.starts_with("http://") || path.starts_with("https://") { + path.to_string() + } else { + format!("{base_url}{path}") + }; let client = reqwest::blocking::Client::builder().timeout(Duration::from_secs(2)).build()?; let deadline = Instant::now() + Duration::from_secs(180); loop { - if let Ok(response) = client.get(format!("{base_url}{}", path.trim())).send() { + if let Ok(response) = client.get(&url).send() { if response.status().as_u16() == expect_status { return Ok(()); } } if Instant::now() >= deadline { - anyhow::bail!("{base_url}{} did not return {expect_status} within 180s", path.trim()); + anyhow::bail!("{url} did not return {expect_status} within 180s"); } std::thread::sleep(Duration::from_millis(500)); } @@ -157,6 +187,37 @@ env: assert!(image < command, "{args:?}"); } + /// A store that answers ingest and query on two ports of one container + /// needs both published, or its own cross-port read-back has nothing to + /// reach. + #[test] + fn extra_ports_are_published_alongside_the_main_one() { + let c: Container = + serde_yaml::from_str("image: x/y:1\nport: 4318\nextra_ports: [16686]\n").unwrap(); + let args = docker_run_args("x", &c); + assert!(args.windows(2).any(|w| w[0] == "-p" && w[1] == "4318:4318"), "{args:?}"); + assert!(args.windows(2).any(|w| w[0] == "-p" && w[1] == "16686:16686"), "{args:?}"); + } + + /// No `config:` means no mount at all, so every adapter written before + /// this existed runs unchanged. + #[test] + fn no_config_means_no_volume_mount() { + let args = docker_run_args("parseable", &parseable()); + assert!(!args.iter().any(|a| a == "-v"), "{args:?}"); + } + + /// An inline config is mounted at the fixed path every adapter's own + /// `command` can point a flag at. + #[test] + fn an_inline_config_is_mounted_at_the_fixed_path() { + let c: Container = + serde_yaml::from_str("image: x/y:1\nport: 1\nconfig: |\n key: value\n").unwrap(); + let args = docker_run_args("x", &c); + let at = args.iter().position(|a| a == "-v").expect("-v present"); + assert!(args[at + 1].ends_with(":/etc/specmatrix/config.yaml"), "{}", args[at + 1]); + } + /// A settle is optional and defaults to none, so no adapter waits for a /// gap it does not have. #[test] diff --git a/src/encode.rs b/src/encode.rs index 31aa8a5..4c00e61 100644 --- a/src/encode.rs +++ b/src/encode.rs @@ -23,6 +23,7 @@ pub fn to_wire(native: &str, encoding: &str, payload: &[u8]) -> Result> ("otlp-json", "otlp-protobuf") => otlp_logs_json_to_protobuf(payload), ("remote-write-json", "remote-write-protobuf") => crate::remote_write::to_wire(payload), ("otlp-metrics-json", "otlp-metrics-protobuf") => otlp_metrics_json_to_protobuf(payload), + ("otlp-traces-json", "otlp-traces-protobuf") => otlp_traces_json_to_protobuf(payload), _ => anyhow::bail!("no encoder from {native} to {encoding}"), } } @@ -43,8 +44,12 @@ pub fn headers_for(encoding: &str) -> &'static [(&'static str, &'static str)] { ("Content-Encoding", "snappy"), ("X-Prometheus-Remote-Write-Version", "0.1.0"), ], - "otlp-protobuf" | "otlp-metrics-protobuf" => &[("Content-Type", "application/x-protobuf")], - "otlp-json" | "otlp-metrics-json" => &[("Content-Type", "application/json")], + "otlp-protobuf" | "otlp-metrics-protobuf" | "otlp-traces-protobuf" => { + &[("Content-Type", "application/x-protobuf")] + } + "otlp-json" | "otlp-metrics-json" | "otlp-traces-json" => { + &[("Content-Type", "application/json")] + } "es-ndjson" => &[("Content-Type", "application/x-ndjson")], _ => &[], } @@ -311,6 +316,21 @@ fn apply_fixups( } } +/// Traces need no fixup step. `Span`, unlike `ExponentialHistogramDataPoint`, +/// carries `#[serde(default)]` on every message in its tree, so a case can +/// omit any field it does not care about and the decoder fills in the zero +/// value rather than discarding the whole span — confirmed by reading the +/// generated types rather than assumed, after the exponential-histogram +/// lesson from the metrics encoder. +fn otlp_traces_json_to_protobuf(payload: &[u8]) -> Result> { + use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest; + use prost::Message; + + let request: ExportTraceServiceRequest = serde_json::from_slice(payload) + .context("decoding OTLP JSON into ExportTraceServiceRequest")?; + Ok(request.encode_to_vec()) +} + /// The first encoding a case offers that the backend accepts. /// /// An adapter with no `formats:` has not declared any and is treated as diff --git a/src/otlp.rs b/src/otlp.rs index 96d28ed..50820d6 100644 --- a/src/otlp.rs +++ b/src/otlp.rs @@ -122,6 +122,50 @@ fn point_value(point: &Value) -> Option { None } +/// Reads a field out of an OTLP traces payload. +/// +/// The first span of the first scope of the first resource, unless the check +/// names one by id — no case has needed to yet, since a trace-level check +/// usually cares about exactly the one span it sent. Nested fields read a +/// dotted path: `status.code`, `events.0.name`, `links.0.traceId`. +pub fn span_field(payload: &Value, field: &str) -> Option { + let span = first_span(payload)?; + match field { + "traceId" => span.get("traceId").cloned(), + "spanId" => span.get("spanId").cloned(), + "parentSpanId" => span.get("parentSpanId").cloned(), + "name" => span.get("name").cloned(), + "kind" => span.get("kind").cloned(), + "startTimeUnixNano" => span.get("startTimeUnixNano").cloned(), + "endTimeUnixNano" => span.get("endTimeUnixNano").cloned(), + "droppedAttributesCount" => span.get("droppedAttributesCount").cloned(), + "status.code" => span.get("status")?.get("code").cloned(), + "status.message" => span.get("status")?.get("message").cloned(), + "events.0.name" => span.get("events")?.as_array()?.first()?.get("name").cloned(), + "events.0.timeUnixNano" => { + span.get("events")?.as_array()?.first()?.get("timeUnixNano").cloned() + } + "links.0.traceId" => span.get("links")?.as_array()?.first()?.get("traceId").cloned(), + other => match other.strip_prefix("attributes.") { + Some(key) => attribute(span.get("attributes")?, key), + None => span.get(other).cloned(), + }, + } +} + +fn first_span(payload: &Value) -> Option<&Value> { + payload + .get("resourceSpans")? + .as_array()? + .first()? + .get("scopeSpans")? + .as_array()? + .first()? + .get("spans")? + .as_array()? + .first() +} + fn first_log_record(payload: &Value) -> Option<&Value> { payload .get("resourceLogs")? @@ -136,7 +180,7 @@ fn first_log_record(payload: &Value) -> Option<&Value> { } /// Unwraps an OTLP `AnyValue` to the value inside it. -fn any_value(value: &Value) -> Option { +pub fn any_value(value: &Value) -> Option { let object = value.as_object()?; for key in [ "stringValue", @@ -192,7 +236,21 @@ pub struct ExportReport { /// /// Returns `None` when the body carries no report at all, which is the ordinary /// case for a store that kept everything. +/// +/// `protocol` picks the count field's name — `rejectedLogRecords`, +/// `rejectedDataPoints` or `rejectedSpans` — and, for a protobuf response, the +/// message type to decode it as. The three collector responses share one +/// shape (`ExportXPartialSuccess { rejected_x: i64, error_message: String }`) +/// but are three distinct generated types with no common trait, so the +/// protobuf branch below is duplicated three ways rather than shared — +/// getting this wrong for a protocol it was never written for is exactly the +/// bug that shipped for traces (and, unnoticed, for metrics) when this +/// function was hardcoded to logs and matched the encoding string +/// `"otlp-json"` literally, which is never true for `"otlp-traces-json"` or +/// `"otlp-metrics-json"` — every non-logs row read as an encoding mismatch +/// that never happened. pub fn export_report( + protocol: &str, request_encoding: &str, content_type: Option<&str>, body: &[u8], @@ -200,13 +258,18 @@ pub fn export_report( if body.is_empty() { return None; } - let expected_json = request_encoding == "otlp-json"; + let expected_json = request_encoding.ends_with("-json"); + let (rejected_key_camel, rejected_key_snake) = match protocol { + "otlp-metrics" => ("rejectedDataPoints", "rejected_data_points"), + "otlp-traces" => ("rejectedSpans", "rejected_spans"), + _ => ("rejectedLogRecords", "rejected_log_records"), + }; if let Ok(value) = serde_json::from_slice::(body) { let partial = value.get("partialSuccess").or_else(|| value.get("partial_success"))?; let rejected = partial - .get("rejectedLogRecords") - .or_else(|| partial.get("rejected_log_records")) + .get(rejected_key_camel) + .or_else(|| partial.get(rejected_key_snake)) .and_then(number_from) .unwrap_or(0); let message = partial @@ -221,19 +284,32 @@ pub fn export_report( } // Not JSON. It may still be a perfectly good report, in the wrong encoding. - use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceResponse; use prost::Message; - let decoded = ExportLogsServiceResponse::decode(body).ok()?; - let partial = decoded.partial_success?; + let (rejected, message) = match protocol { + "otlp-metrics" => { + use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceResponse; + let decoded = ExportMetricsServiceResponse::decode(body).ok()?; + let partial = decoded.partial_success?; + (partial.rejected_data_points, partial.error_message) + } + "otlp-traces" => { + use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceResponse; + let decoded = ExportTraceServiceResponse::decode(body).ok()?; + let partial = decoded.partial_success?; + (partial.rejected_spans, partial.error_message) + } + _ => { + use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceResponse; + let decoded = ExportLogsServiceResponse::decode(body).ok()?; + let partial = decoded.partial_success?; + (partial.rejected_log_records, partial.error_message) + } + }; let mismatch = expected_json.then(|| { let claimed = content_type.unwrap_or("none"); format!("request was JSON, response body is protobuf and content-type says {claimed}") }); - Some(ExportReport { - rejected: partial.rejected_log_records, - message: partial.error_message, - encoding_mismatch: mismatch, - }) + Some(ExportReport { rejected, message, encoding_mismatch: mismatch }) } fn number_from(value: &Value) -> Option { @@ -245,6 +321,63 @@ fn number_from(value: &Value) -> Option { } } +#[cfg(test)] +mod span_tests { + use super::*; + use serde_json::json; + + fn span(extra: serde_json::Value) -> Value { + let mut base = json!({ + "traceId": "0102030405060708090a0b0c0d0e0f10", + "spanId": "0102030405060708", + "name": "probe", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2" + }); + for (k, v) in extra.as_object().unwrap() { + base[k] = v.clone(); + } + json!({"resourceSpans": [{"scopeSpans": [{"spans": [base]}]}]}) + } + + #[test] + fn reads_trace_and_span_ids_as_hex_strings() { + let payload = span(json!({})); + assert_eq!( + span_field(&payload, "traceId"), + Some(json!("0102030405060708090a0b0c0d0e0f10")) + ); + assert_eq!(span_field(&payload, "spanId"), Some(json!("0102030405060708"))); + } + + #[test] + fn reads_status_code_and_message() { + let payload = span(json!({"status": {"code": 2, "message": "boom"}})); + assert_eq!(span_field(&payload, "status.code"), Some(json!(2))); + assert_eq!(span_field(&payload, "status.message"), Some(json!("boom"))); + } + + #[test] + fn reads_the_first_event_name_and_timestamp() { + let payload = span(json!({"events": [{"name": "e1", "timeUnixNano": "5"}]})); + assert_eq!(span_field(&payload, "events.0.name"), Some(json!("e1"))); + assert_eq!(span_field(&payload, "events.0.timeUnixNano"), Some(json!("5"))); + } + + #[test] + fn reads_a_span_attribute_by_key() { + let payload = span(json!({ + "attributes": [{"key": "http.method", "value": {"stringValue": "GET"}}] + })); + assert_eq!(span_field(&payload, "attributes.http.method"), Some(json!("GET"))); + } + + #[test] + fn a_field_that_is_absent_reads_as_none() { + assert_eq!(span_field(&span(json!({})), "parentSpanId"), None); + } +} + #[cfg(test)] mod tests { use super::*; @@ -310,25 +443,28 @@ mod tests { /// A store that kept everything says nothing, and that is not a report. #[test] fn an_empty_body_carries_no_report() { - assert_eq!(export_report("otlp-json", Some("application/json"), b""), None); + assert_eq!(export_report("otlp-logs", "otlp-json", Some("application/json"), b""), None); } #[test] fn a_success_with_no_partial_success_carries_no_report() { - assert_eq!(export_report("otlp-json", Some("application/json"), b"{}"), None); + assert_eq!(export_report("otlp-logs", "otlp-json", Some("application/json"), b"{}"), None); } #[test] fn a_json_report_is_read_in_either_field_naming() { let camel = br#"{"partialSuccess":{"rejectedLogRecords":"3","errorMessage":"too old"}}"#; - let report = export_report("otlp-json", Some("application/json"), camel).unwrap(); + let report = + export_report("otlp-logs", "otlp-json", Some("application/json"), camel).unwrap(); assert_eq!(report.rejected, 3); assert_eq!(report.message, "too old"); assert_eq!(report.encoding_mismatch, None); let snake = br#"{"partial_success":{"rejected_log_records":3,"error_message":"too old"}}"#; assert_eq!( - export_report("otlp-json", Some("application/json"), snake).unwrap().rejected, + export_report("otlp-logs", "otlp-json", Some("application/json"), snake) + .unwrap() + .rejected, 3 ); } @@ -336,7 +472,9 @@ mod tests { #[test] fn a_protobuf_report_to_a_protobuf_request_is_not_a_mismatch() { let body = protobuf_response(1, "too old"); - let report = export_report("otlp-protobuf", Some("application/x-protobuf"), &body).unwrap(); + let report = + export_report("otlp-logs", "otlp-protobuf", Some("application/x-protobuf"), &body) + .unwrap(); assert_eq!(report.rejected, 1); assert_eq!(report.message, "too old"); assert_eq!(report.encoding_mismatch, None); @@ -349,7 +487,8 @@ mod tests { #[test] fn a_protobuf_report_to_a_json_request_is_reported_as_a_mismatch() { let body = protobuf_response(1, "Too old data, only last 5 hours data can be ingested."); - let report = export_report("otlp-json", Some("application/json"), &body).unwrap(); + let report = + export_report("otlp-logs", "otlp-json", Some("application/json"), &body).unwrap(); assert_eq!(report.rejected, 1); assert!(report.message.starts_with("Too old data")); let mismatch = report.encoding_mismatch.expect("a mismatch"); @@ -357,9 +496,80 @@ mod tests { assert!(mismatch.contains("application/json"), "{mismatch}"); } + /// The bug this signature exists to prevent: `"otlp-traces-json"` and + /// `"otlp-metrics-json"` both end in `-json`, and neither is the literal + /// string `"otlp-json"`. A comparison against that literal — which this + /// function used before traces existed — calls every one of them a + /// protobuf request, and flags a mismatch that never happened. + #[test] + fn a_json_report_to_a_traces_or_metrics_json_request_is_not_a_mismatch() { + let camel = br#"{"partialSuccess":{"rejectedSpans":"1","errorMessage":"x"}}"#; + let report = + export_report("otlp-traces", "otlp-traces-json", Some("application/json"), camel) + .unwrap(); + assert_eq!(report.encoding_mismatch, None, "{:?}", report.encoding_mismatch); + + let camel = br#"{"partialSuccess":{"rejectedDataPoints":"1","errorMessage":"x"}}"#; + let report = + export_report("otlp-metrics", "otlp-metrics-json", Some("application/json"), camel) + .unwrap(); + assert_eq!(report.encoding_mismatch, None, "{:?}", report.encoding_mismatch); + } + + #[test] + fn a_traces_json_report_reads_rejected_spans_in_either_naming() { + let camel = br#"{"partialSuccess":{"rejectedSpans":"2","errorMessage":"dropped"}}"#; + let report = + export_report("otlp-traces", "otlp-traces-json", Some("application/json"), camel) + .unwrap(); + assert_eq!(report.rejected, 2); + assert_eq!(report.message, "dropped"); + + let snake = br#"{"partial_success":{"rejected_spans":2,"error_message":"dropped"}}"#; + let report = + export_report("otlp-traces", "otlp-traces-json", Some("application/json"), snake) + .unwrap(); + assert_eq!(report.rejected, 2); + } + + #[test] + fn a_metrics_json_report_reads_rejected_data_points() { + let body = br#"{"partialSuccess":{"rejectedDataPoints":"4","errorMessage":"bad point"}}"#; + let report = + export_report("otlp-metrics", "otlp-metrics-json", Some("application/json"), body) + .unwrap(); + assert_eq!(report.rejected, 4); + assert_eq!(report.message, "bad point"); + } + + #[test] + fn a_traces_protobuf_report_decodes_as_the_traces_response_type() { + use opentelemetry_proto::tonic::collector::trace::v1::{ + ExportTracePartialSuccess, ExportTraceServiceResponse, + }; + use prost::Message; + let body = ExportTraceServiceResponse { + partial_success: Some(ExportTracePartialSuccess { + rejected_spans: 3, + error_message: "too many".to_string(), + }), + } + .encode_to_vec(); + let report = export_report( + "otlp-traces", + "otlp-traces-protobuf", + Some("application/x-protobuf"), + &body, + ) + .unwrap(); + assert_eq!(report.rejected, 3); + assert_eq!(report.message, "too many"); + assert_eq!(report.encoding_mismatch, None); + } + /// A body that is neither is not a report, and must not be guessed at. #[test] fn an_unreadable_body_carries_no_report() { - assert_eq!(export_report("otlp-json", None, b"not a response at all"), None); + assert_eq!(export_report("otlp-logs", "otlp-json", None, b"not a response at all"), None); } } diff --git a/src/runner.rs b/src/runner.rs index 4d94968..c207e70 100644 --- a/src/runner.rs +++ b/src/runner.rs @@ -284,7 +284,12 @@ impl Runner { // difference between a loud failure and a silent one — which is the // distinction this project exists to draw. let reported = if case.protocol.starts_with("otlp") { - crate::otlp::export_report(&encoding, response.content_type.as_deref(), &response.body) + crate::otlp::export_report( + &case.protocol, + &encoding, + response.content_type.as_deref(), + &response.body, + ) } else { None }; @@ -530,6 +535,22 @@ impl Runner { let stream = format!("specmatrix_{}", case.id.replace(['/', '-', '.'], "_").to_lowercase()); let mut vars = Vars::new(); vars.insert("run_key", run_key); + // The run key for the traces protocol: sixteen random bytes, hex + // encoded, valid as a trace_id on the wire. Traces have no attribute + // equivalent to `specmatrix.run` that every backend indexes the same + // way — a trace is found by its trace id, full stop — so this is what + // a case's read-back and `{{ trace_id_hex }}` in a payload both use. + let trace_id: [u8; 16] = rand::random(); + vars.insert( + "trace_id_hex", + trace_id.iter().map(|b| format!("{b:02x}")).collect::(), + ); + // Some stores echo a trace id in the OTLP JSON mapping's own encoding + // for a bytes field, base64, rather than the hex this project's + // payloads write it in — confirmed on Tempo. `read_back` tries both, + // since a store's own choice of encoding is not the divergence any + // check here is looking for. + vars.insert("trace_id_base64", base64_encode(&trace_id)); vars.insert("suite_stream", stream); if let Some(field) = self.backend.run_key_field.clone() { vars.insert("run_key_field", field); @@ -543,7 +564,19 @@ impl Runner { fn send(&self, req: &Request, vars: &Vars, body: Vec) -> Result { let (method, path) = req.parts()?; - let url = format!("{}{}", self.base_url, template::render(&path, vars)); + let rendered_path = template::render(&path, vars); + // A request whose path is already a full URL is used as written rather + // than appended to the base URL. Several stores split ingest and + // query across two ports on the one container — Jaeger's OTLP + // receiver and its query API, Tempo's OTLP receiver and its query + // API — and the base URL can only be one of them, whichever + // `specmatrix up` reports. A read-back on the other port has to name + // it outright. + let url = if rendered_path.starts_with("http://") || rendered_path.starts_with("https://") { + rendered_path + } else { + format!("{}{}", self.base_url, rendered_path) + }; let mut builder = match method.as_str() { "POST" => self.client.post(&url), "PUT" => self.client.put(&url), @@ -694,7 +727,17 @@ impl Runner { /// Most backends acknowledge a write before it is queryable, so a single /// immediate read would report every backend as dropping data. fn read_back(&self, readback: &Readback, vars: &Vars) -> Result { + // The marker a record is found by. `run_key` is what every protocol + // except traces carries: a case sends it as an attribute or a label, + // and it is what disambiguates our record from anything else the + // store holds. A trace has no such field — it is found by its trace + // id or not at all — so a response is also accepted if it carries + // `trace_id_hex` instead. Neither is a plausible false match: both are + // random per case, and a response containing one that is not ours + // would be a collision astronomically unlikely to occur by chance. let run_key = vars.get("run_key").cloned().unwrap_or_default(); + let trace_id = vars.get("trace_id_hex").cloned().unwrap_or_default(); + let trace_id_b64 = vars.get("trace_id_base64").cloned().unwrap_or_default(); let deadline = Instant::now() + Duration::from_millis(readback.poll.timeout_ms); // Tracks whether every poll's response body failed to parse at all, as // opposed to parsing fine and simply not containing the record yet. @@ -715,7 +758,10 @@ impl Runner { if let Ok(response) = self.send_declared(&readback.request, vars) { if let Some(value) = parse_response_body(&response.body) { ever_parsed = true; - if let Some(record) = first_record(&value, &readback.records, &run_key) { + let record = first_record(&value, &readback.records, &run_key) + .or_else(|| first_record(&value, &readback.records, &trace_id)) + .or_else(|| first_record(&value, &readback.records, &trace_id_b64)); + if let Some(record) = record { return Ok(ReadOutcome::Found(record)); } } @@ -739,7 +785,40 @@ impl Runner { if let Some(pointer) = readback.fields.get(field) { return record.pointer(pointer).cloned(); } - record.get(field).cloned() + if let Some(key) = field.strip_prefix("attributes.") { + // An adapter's `fields:` maps one flat pointer per name, and an + // attribute key is not one — it lives at a variable index in + // whichever array a store nested it under. Tried only when the + // adapter declared no mapping for this exact field name, so an + // adapter that flattens attributes onto top-level columns (most + // do) can still name them individually and this fallback never + // runs for it. Where it does apply, it is OTLP's own shape: an + // `attributes` array of `{key, value}` pairs, which is what a + // store that echoes real OTLP JSON — Tempo, here — returns. + if let Some(found) = record + .get("attributes") + .and_then(|attrs| attrs.as_array()) + .and_then(|attrs| { + attrs.iter().find(|kv| kv.get("key").and_then(|k| k.as_str()) == Some(key)) + }) + .and_then(|kv| kv.get("value")) + { + return crate::otlp::any_value(found); + } + } + if let Some(value) = record.get(field) { + return Some(value.clone()); + } + // A dotted path with no adapter mapping — `events.0.timeUnixNano` — is + // read as a JSON pointer, the same shape OTLP's own JSON already is. + // Only tried once the plain top-level lookup above has failed, so a + // backend that genuinely names a field with a literal dot in it (rare, + // but `service.name`-style keys exist) is read literally first. + if field.contains('.') { + let pointer = format!("/{}", field.replace('.', "/")); + return record.pointer(&pointer).cloned(); + } + None } fn authenticate( @@ -838,6 +917,7 @@ fn logical_field_for( "remote-write" => crate::remote_write::logical_field(sent, field, series), "otlp-metrics" => crate::otlp::metric_field(sent, field, series), "loki-push" => crate::loki::logical_field(sent, field), + "otlp-traces" => crate::otlp::span_field(sent, field), _ => crate::otlp::logical_field(sent, field), } } @@ -977,6 +1057,14 @@ fn first_record( records_pointer: &str, run_key: &str, ) -> Option { + // An empty key is never a marker, only a caller's unset default — + // `.contains("")` is true of every string, so matching on one would + // return the first record regardless of whether it is ours. This has + // already been the actual bug once, when a second marker was wired + // through `unwrap_or_default()` before anything set it. + if run_key.is_empty() { + return None; + } let node = if records_pointer.is_empty() { value } else { value.pointer(records_pointer)? }; let items: Vec<&serde_json::Value> = match node { serde_json::Value::Array(items) => items.iter().collect(), @@ -1005,6 +1093,26 @@ fn parse_sent(bytes: &[u8]) -> (serde_json::Value, bool) { (serde_json::from_str(&lossy).unwrap_or(serde_json::Value::Null), true) } +/// Standard base64, no crate needed for sixteen bytes. Some stores encode a +/// trace id in OTLP JSON's own mapping for a `bytes` field, which is base64, +/// rather than the hex this project's payloads use — read-back needs both. +fn base64_encode(bytes: &[u8]) -> String { + const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4); + for chunk in bytes.chunks(3) { + let b = [chunk[0], *chunk.get(1).unwrap_or(&0), *chunk.get(2).unwrap_or(&0)]; + let n = ((b[0] as u32) << 16) | ((b[1] as u32) << 8) | b[2] as u32; + for i in 0..4 { + if i <= chunk.len() { + out.push(ALPHABET[((n >> (18 - 6 * i)) & 63) as usize] as char); + } else { + out.push('='); + } + } + } + out +} + fn first_line(text: &str) -> String { let line = text.lines().next().unwrap_or("").trim(); if line.len() > 120 { @@ -1014,6 +1122,25 @@ fn first_line(text: &str) -> String { } } +#[cfg(test)] +mod base64_tests { + use super::base64_encode; + + #[test] + fn encodes_sixteen_bytes_with_the_two_padding_characters_that_length_needs() { + let bytes: [u8; 16] = [ + 0x12, 0xca, 0xb4, 0xb4, 0xf3, 0xf2, 0x4a, 0xf6, 0xac, 0xf3, 0x82, 0x72, 0x92, 0x50, + 0xf2, 0xa6, + ]; + assert_eq!(base64_encode(&bytes), "Esq0tPPySvas84JyklDypg=="); + } + + #[test] + fn an_empty_slice_encodes_to_an_empty_string() { + assert_eq!(base64_encode(&[]), ""); + } +} + #[cfg(test)] mod body_parsing { use super::*; @@ -1349,6 +1476,76 @@ expect: assert!(!result.detail.contains("never valid JSON"), "{}", result.detail); } + /// A read-back request whose path is a full URL bypasses the base URL + /// entirely, for a store that answers ingest and query on different + /// ports of the same container. + #[test] + fn a_request_with_an_absolute_url_ignores_the_base_url() { + // Two separate stubs, standing in for two ports of one container. + // Ingest is reachable only through `base_url`; search only through + // the absolute URL in the read-back. If either fell back to the + // other, the case would not pass. + let ingest = stub::start(vec![("/ingest", vec![Reply::json(200, "{}")])]); + let search = stub::start(vec![( + "/search", + vec![Reply::json( + 200, + r#"{"hits":[{"message":"specmatrix minimal record","specmatrix.run":"{{RUNKEY}}"}]}"#, + )], + )]); + let adapter: Backend = serde_yaml::from_str(&format!( + r#" +name: stub +protocols: + otlp-logs: + formats: [otlp-json] + ingest: + request: POST /ingest + readback: + request: GET {}/search + body: + run_key: "{{{{ run_key }}}}" + records: /hits + fields: + body: /message + poll: + interval_ms: 10 + timeout_ms: 120 +"#, + search.url + )) + .expect("adapter parses"); + let runner = Runner::new(adapter, ingest.url.clone(), false).expect("runner builds"); + let case = case_yaml(" readback:\n match: exact\n on: [body]"); + let result = runner.run_case("otlp-logs", &case).expect("no harness error"); + assert_eq!(result.verdict, Verdict::Pass, "detail: {}", result.detail); + } + + /// A dotted field name with no adapter mapping is read as a JSON pointer, + /// the shape a nested field genuinely has in real OTLP JSON — an event's + /// timestamp, a link's trace id. This is what let a check on + /// `events.0.timeUnixNano` find real data against Tempo, which answers + /// OTLP JSON verbatim. + #[test] + fn a_dotted_field_with_no_mapping_is_read_as_a_json_pointer() { + let stub = stub::start(vec![ + ("/ingest", vec![Reply::json(200, "{}")]), + ( + "/search", + vec![Reply::json( + 200, + r#"{"hits":[{"events":[{"timeUnixNano":"123"}],"specmatrix.run":"{{RUNKEY}}"}]}"#, + )], + ), + ]); + let runner = runner_for(&stub.url, vec![], Reply::json(200, "{}")); + let case = + case_yaml(" readback:\n match: present\n on: [\"events.0.timeUnixNano\"]"); + let result = runner.run_case("otlp-logs", &case).expect("no harness error"); + assert_eq!(result.verdict, Verdict::Pass, "detail: {}", result.detail); + assert!(result.detail.contains("123"), "{}", result.detail); + } + /// A refusal is a REJECT carrying the status and the store's own words, so /// a reader can act on it without rerunning anything. #[test] @@ -1668,6 +1865,15 @@ mod tests { assert_eq!(found.get("body").unwrap(), "ours"); } + /// The bug this guard exists to prevent: an empty marker string is + /// contained in every string, so without it, this would have returned + /// the first record regardless of whether it was ours. + #[test] + fn an_empty_key_matches_nothing_rather_than_everything() { + let response = json!({"hits": [{"body": "someone else's record entirely"}]}); + assert!(first_record(&response, "/hits", "").is_none()); + } + #[test] fn first_record_ignores_records_without_the_key() { let response = json!({"hits": [{"body": "someone else's"}]});