From c2b1e8d9367faed76803513953610aa48ffeb006 Mon Sep 17 00:00:00 2001 From: jbailes Date: Fri, 30 Oct 2026 21:30:46 +0100 Subject: [PATCH 1/7] D16: tap telemetry leaves the host the moment an event is stamped When the host stamps an event's seq it now writes one 128-byte, content-free record (the stamped frame header, a realtime stamp, nothing of the payload) into a single-producer ring in its own region, before the tap runs and before routing. One telemetry process, granted by the embedding from its attach hook, holds the consumer end (OURO_ATTACH_CAP_TELEMETRY, two more descriptors) and exports the records off the host. The host never blocks on it: a full ring counts the loss and the next record that fits follows a gap record carrying the count. The consumer is woken by doorbell once per pump, only when asleep, and a later consumer resumes at the same tail. Cost, from c/bench/dispatch.c: 87 ns per event without telemetry, 100 ns with it. The host caches the consumer index, reads the clock once per pump and writes the record with a little-endian fast path that matches the portable encoder byte for byte. The perf budget now gates the telemetry-on path too. --- c/CMakeLists.txt | 9 +- c/bench/baseline.json | 6 + c/bench/dispatch.c | 50 ++++ c/include/ouroboros/attach.h | 9 +- c/include/ouroboros/host.h | 30 +++ c/include/ouroboros/telemetry.h | 128 +++++++++++ c/src/client.c | 58 ++++- c/src/host.c | 28 +++ c/src/route.c | 10 + c/src/telemetry.c | 176 +++++++++++++++ c/src/telemetry_host.c | 195 ++++++++++++++++ c/src/telemetry_internal.h | 22 ++ c/tests/test_telemetry.c | 389 ++++++++++++++++++++++++++++++++ scripts/check_perf_budget.sh | 27 ++- spec/decisions.md | 98 +++++++- 15 files changed, 1212 insertions(+), 23 deletions(-) create mode 100644 c/include/ouroboros/telemetry.h create mode 100644 c/src/telemetry.c create mode 100644 c/src/telemetry_host.c create mode 100644 c/src/telemetry_internal.h create mode 100644 c/tests/test_telemetry.c diff --git a/c/CMakeLists.txt b/c/CMakeLists.txt index 85c8fa7..aa7f089 100644 --- a/c/CMakeLists.txt +++ b/c/CMakeLists.txt @@ -39,6 +39,7 @@ set(OURO_CLIENT_SOURCES src/module_runtime.c src/region.c src/ring.c + src/telemetry.c src/wire.c) set(OURO_HOST_SOURCES src/arena.c @@ -47,7 +48,8 @@ set(OURO_HOST_SOURCES src/local_attach.c src/region_host.c src/route.c - src/runtime.c) + src/runtime.c + src/telemetry_host.c) foreach(lib ouroboros-ring-client ouroboros-ring) if(lib STREQUAL "ouroboros-ring") @@ -95,11 +97,12 @@ if(BUILD_TESTING) set(OURO_TEST_region src/region.c src/region_host.c src/ring.c) set(OURO_TEST_arena src/arena.c src/region.c src/ring.c) foreach(name host route flow client capture local_attach doorbell runtime - module_embedded_runtime module_client_pool) + module_embedded_runtime module_client_pool telemetry) set(OURO_TEST_${name} ${OURO_ALL_SOURCES}) endforeach() foreach(name ring wire endpoint module_protocol layout region arena host route flow client - capture local_attach doorbell runtime module_embedded_runtime module_client_pool) + capture local_attach doorbell runtime module_embedded_runtime module_client_pool + telemetry) add_executable(test_${name} tests/test_${name}.c ${OURO_TEST_${name}}) target_compile_features(test_${name} PRIVATE c_std_11) target_compile_options(test_${name} PRIVATE ${OURO_WARNINGS}) diff --git a/c/bench/baseline.json b/c/bench/baseline.json index af861da..63a6aef 100644 --- a/c/bench/baseline.json +++ b/c/bench/baseline.json @@ -6,6 +6,12 @@ "ceiling_ns": 1000, "observed_ns": 134, "observed_note": "measured when the benchmark lived in aimee (reference dev box). The ceiling carries ~7x headroom so CI variance does not flake the check while a per-event syscall still trips it." + }, + "dispatch_overhead_telemetry_ns": { + "description": "The same dispatch with D16 telemetry on: the host writes one record per event and a consumer thread drains them as an exporter would (OURO_BENCH_TELEMETRY=1).", + "ceiling_ns": 1000, + "observed_ns": 100, + "observed_note": "measured 2026-10-30 on the reference dev box: 87 ns off, 100 ns on. Same headroom rule as dispatch_overhead_ns." } } } diff --git a/c/bench/dispatch.c b/c/bench/dispatch.c index 96c5ae2..e8ea9c4 100644 --- a/c/bench/dispatch.c +++ b/c/bench/dispatch.c @@ -8,15 +8,43 @@ * It prints the lines scripts/check_perf_budget.sh parses: * dispatch_overhead_ns= * dispatch_overhead_p99_ns= + * + * With OURO_BENCH_TELEMETRY=1 the host also writes a D16 record per event, and + * a consumer thread drains them as an exporter would, so the same number shows + * what the hand-off adds. */ #define _GNU_SOURCE +#include +#include #include #include #include #include #include +#include #include +#include + +static atomic_int telemetry_stop; +static atomic_ullong telemetry_drained; + +static void *telemetry_drain(void *p) +{ + ouro_telemetry_reader_t *r = p; + while (!atomic_load_explicit(&telemetry_stop, memory_order_relaxed)) + { + uint64_t n = ouro_telemetry_available(r); + if (n == 0) + { + (void)ouro_telemetry_wait(r, -1, 10); + continue; + } + ouro_telemetry_release(r, n); + atomic_fetch_add_explicit(&telemetry_drained, n, memory_order_relaxed); + } + return NULL; +} static uint64_t now_ns(void) { @@ -59,6 +87,19 @@ int main(int argc, char **argv) attach_client(&h, &sub); ouro_host_subscribe(&h, sub.reply.handle_id, KIND); + const char *tenv = getenv("OURO_BENCH_TELEMETRY"); + int telemetry = tenv && strcmp(tenv, "1") == 0; + ouro_telemetry_reader_t reader; + pthread_t drainer; + if (telemetry) + { + if (ouro_host_telemetry_enable(&h, 0) != OURO_HOST_OK || + ouro_telemetry_reader_open(dup(h.telemetry_region.fd), dup(h.telemetry_doorbell_fd), + &reader) != OURO_TELEMETRY_OK || + pthread_create(&drainer, NULL, telemetry_drain, &reader) != 0) + abort(); + } + /* One event's dispatch = publish (producer enqueue) + host pump (route) + * poll (consumer dequeue). Batched to the ring's data capacity so timer * granularity does not dominate; each batch's time is divided by its events. @@ -115,6 +156,15 @@ int main(int argc, char **argv) fprintf(stderr, "bench: %llu batches of %u events (median %llu ns/event, p99 %llu ns)\n", (unsigned long long)batches, batch, (unsigned long long)median, (unsigned long long)p99); + if (telemetry) + { + atomic_store(&telemetry_stop, 1); + pthread_join(drainer, NULL); + fprintf(stderr, "bench: telemetry on, %llu records drained, %llu lost\n", + (unsigned long long)atomic_load(&telemetry_drained), + (unsigned long long)ouro_telemetry_dropped(&reader)); + ouro_telemetry_reader_close(&reader); + } free(samples); ouro_client_detach(&pub); ouro_client_detach(&sub); diff --git a/c/include/ouroboros/attach.h b/c/include/ouroboros/attach.h index 3e08cbb..4aa75be 100644 --- a/c/include/ouroboros/attach.h +++ b/c/include/ouroboros/attach.h @@ -13,18 +13,25 @@ * so it carries OURO_ATTACH_FDS_DOORBELL. */ #define OURO_ATTACH_FDS 3 #define OURO_ATTACH_FDS_DOORBELL 5 -#define OURO_ATTACH_FDS_MAX OURO_ATTACH_FDS_DOORBELL +/* D16: a telemetry grant adds the telemetry region and its doorbell. */ +#define OURO_ATTACH_FDS_TELEMETRY 7 +#define OURO_ATTACH_FDS_MAX OURO_ATTACH_FDS_TELEMETRY #define OURO_ATTACH_FD_CONTROL 0 #define OURO_ATTACH_FD_ARENA 1 #define OURO_ATTACH_FD_QPAIR 2 #define OURO_ATTACH_FD_HOST_DOORBELL 3 #define OURO_ATTACH_FD_CLIENT_DOORBELL 4 +#define OURO_ATTACH_FD_TELEMETRY 5 +#define OURO_ATTACH_FD_TELEMETRY_BELL 6 /* Attach capabilities (D13): requested in the request, granted in the reply. */ #define OURO_ATTACH_CAP_DOORBELL 0x00000001u /* D15: the host holds an OS handle on the client's process and releases its slot * when the process exits, so the client need not heartbeat on a timer. */ #define OURO_ATTACH_CAP_PROCESS_LIVENESS 0x00000002u +/* D16: the consumer end of the host's telemetry ring. Granted only with the + * doorbell, to the one slot the embedding granted it. */ +#define OURO_ATTACH_CAP_TELEMETRY 0x00000004u /* The v0 request was the first 16 bytes of ouro_attach_request_t. A host still * accepts it, and an older host reads only those 16 bytes of a longer one. */ diff --git a/c/include/ouroboros/host.h b/c/include/ouroboros/host.h index 92a1fdc..1da8ad0 100644 --- a/c/include/ouroboros/host.h +++ b/c/include/ouroboros/host.h @@ -27,6 +27,7 @@ #include #include #include +#include #include /* A directory slot — host-private. A client never sees this or any other slot. */ @@ -97,6 +98,9 @@ typedef struct /* D15: the runtime holds a process handle for this client and releases the * slot when the process exits, so the heartbeat reap passes it by. */ int process_bound; + + /* D16: the embedding granted this slot the telemetry consumer end. */ + int telemetry_granted; } ouro_slot_t; /* Per-kind overflow policy (D5). Block is the safe default: a full destination @@ -207,6 +211,21 @@ typedef struct ouro_host /* D13: the host doorbell every doorbell client writes; -1 if it could not be * created, in which case no client is granted one and the caller polls. */ int doorbell_fd; + + /* D16: the telemetry ring, written as each event is stamped. telemetry is + * NULL until ouro_host_telemetry_enable. telemetry_slot is the slot holding + * the consumer end, UINT32_MAX for none. */ + ouro_region_t telemetry_region; + ouro_telemetry_region_t *telemetry; + ouro_ring_t telemetry_ring; + int telemetry_doorbell_fd; + uint32_t telemetry_slot; + uint64_t telemetry_head; /* the producer index, kept locally */ + uint64_t telemetry_tail_cache; /* the consumer index as last read */ + uint64_t telemetry_now; /* this pump's realtime, read at its first stamp */ + uint64_t telemetry_lost; /* records lost since the last gap record */ + int telemetry_unsignalled; /* written since the consumer was last signalled */ + int in_pump; /* signal once at the end of the pump, not per event */ } ouro_host_t; typedef enum @@ -298,6 +317,17 @@ const char *ouro_attach_status_name(ouro_attach_status_t s); /* Set the governance/audit tap. Pass NULL to clear. */ void ouro_host_set_tap(ouro_host_t *h, ouro_tap_fn fn, void *ctx); +/* D16: create the telemetry ring of `capacity` records (0: the default). From + * then on every seq-stamped event writes one record before the tap is called. + * Call before any client may ask for the grant. */ +ouro_host_result_t ouro_host_telemetry_enable(ouro_host_t *h, uint32_t capacity); + +/* D16: grant `slot` the telemetry consumer end. The embedding calls it from its + * attach hook, for the one principal it trusts with metadata about every event; + * the host then hands that client the region if it asked and no other live slot + * holds it. */ +ouro_host_result_t ouro_host_grant_telemetry(ouro_host_t *h, uint32_t slot); + /* Set the rare loss observer. Pass NULL to clear. */ void ouro_host_set_loss_sink(ouro_host_t *h, ouro_loss_fn fn, void *ctx); diff --git a/c/include/ouroboros/telemetry.h b/c/include/ouroboros/telemetry.h new file mode 100644 index 0000000..1cd2e0f --- /dev/null +++ b/c/include/ouroboros/telemetry.h @@ -0,0 +1,128 @@ +/* ouro_telemetry.h: tap telemetry handed off the host as each event is stamped (D16). + * + * When the host stamps an event's seq it writes one 128-byte, content-free record + * into a single-producer, single-consumer ring in its own region, before the tap + * callback and before routing. One telemetry process, granted by the embedding, + * holds the consumer end and exports the records off the host. The host never + * blocks on it: a full ring counts the record lost and the next record that fits + * is preceded by a gap record carrying the count. + * + * This header is the record and region contract plus the consumer API. The host + * side is in host.h (ouro_host_telemetry_enable, ouro_host_grant_telemetry). + */ +#ifndef OUROBOROS_TELEMETRY_H +#define OUROBOROS_TELEMETRY_H 1 + +#include +#include +#include + +#include +#include +#include +#include + +#define OURO_TELEMETRY_MAGIC 0x594d4c54u /* "TLMY" */ +#define OURO_TELEMETRY_VERSION 1 + +/* The record (D16). All integers little-endian. */ +#define OURO_TELEMETRY_RECORD_LEN 128 +#define OURO_TELEMETRY_OFF_TYPE 0 +#define OURO_TELEMETRY_OFF_FLAGS 4 +#define OURO_TELEMETRY_OFF_REALTIME_NS 8 +#define OURO_TELEMETRY_OFF_LOST 16 +#define OURO_TELEMETRY_OFF_FRAME 32 /* OURO_WIRE_HDR_LEN bytes, wire encoding */ +#define OURO_TELEMETRY_RECORD_EVENT 0u +#define OURO_TELEMETRY_RECORD_GAP 1u + +/* Ring capacity a host uses when the embedding passes 0: 8 MiB of records. */ +#define OURO_TELEMETRY_CAPACITY_DEFAULT 65536u + +/* Region offsets. The ring starts on its own cache line after the header, and + * consumer_waiting sits alone on the line between, so neither end's writes + * invalidate the line the other end writes. */ +#define OURO_TELEMETRY_REGION_OFF_WAITING 64 +#define OURO_TELEMETRY_REGION_OFF_RING 128 + +typedef struct +{ + _Atomic uint32_t magic; /* release-stored last; publishes the header */ + _Atomic uint32_t version; + _Atomic uint32_t record_size; + _Atomic uint32_t flags; + _Atomic uint64_t host_epoch; + _Atomic uint64_t dropped; /* host-written: every record ever lost */ + _Atomic uint64_t reserved[4]; + _Alignas(64) _Atomic uint32_t consumer_waiting; /* consumer-written (D13) */ +} ouro_telemetry_region_t; + +_Static_assert(offsetof(ouro_telemetry_region_t, dropped) == 24, "dropped offset is in D16"); +_Static_assert(offsetof(ouro_telemetry_region_t, consumer_waiting) == + OURO_TELEMETRY_REGION_OFF_WAITING, + "consumer_waiting offset is in D16"); +_Static_assert(sizeof(ouro_telemetry_region_t) <= OURO_TELEMETRY_REGION_OFF_RING, + "the ring follows the header"); + +/* Bytes of a telemetry region holding `capacity` records; 0 if invalid. */ +size_t ouro_telemetry_region_bytes(uint32_t capacity); + +/* A decoded record. For a gap, frame is all zero and lost is the count. */ +typedef struct +{ + uint32_t type; + uint64_t realtime_ns; + uint64_t lost; + ouro_frame_t frame; +} ouro_telemetry_record_t; + +typedef enum +{ + OURO_TELEMETRY_OK = 0, + OURO_TELEMETRY_ERR_ARG, + OURO_TELEMETRY_ERR_LAYOUT, /* region or record disagrees with D16 */ + OURO_TELEMETRY_ERR_OS +} ouro_telemetry_result_t; + +/* Encode a record into exactly OURO_TELEMETRY_RECORD_LEN bytes. */ +void ouro_telemetry_encode(uint8_t *out, uint32_t type, uint64_t realtime_ns, uint64_t lost, + const ouro_frame_t *frame); +ouro_telemetry_result_t ouro_telemetry_decode(const uint8_t *in, ouro_telemetry_record_t *out); + +/* The consumer end, held by the one telemetry process. */ +typedef struct +{ + ouro_region_t region; + ouro_telemetry_region_t *hdr; + ouro_ring_t ring; + int doorbell_fd; +} ouro_telemetry_reader_t; + +/* Attach as a client that also asks for the telemetry grant (D16). On success + * the client is attached as usual, and *granted says whether `reader` holds the + * consumer end; a host that did not grant it leaves the client attached and + * reader unmapped. */ +ouro_client_result_t ouro_client_attach_telemetry(int sock, ouro_client_t *c, + uint32_t principal_class, + uint32_t principal_ref, + ouro_telemetry_reader_t *reader, int *granted); + +/* Adopt a region from its descriptors (both are taken over). */ +ouro_telemetry_result_t ouro_telemetry_reader_open(int region_fd, int doorbell_fd, + ouro_telemetry_reader_t *r); +void ouro_telemetry_reader_close(ouro_telemetry_reader_t *r); + +/* Records written and not yet released. */ +uint64_t ouro_telemetry_available(const ouro_telemetry_reader_t *r); +/* The i-th unreleased record, i < ouro_telemetry_available. */ +const uint8_t *ouro_telemetry_peek(const ouro_telemetry_reader_t *r, uint64_t i); +/* Release the oldest n records once they have been exported. */ +void ouro_telemetry_release(ouro_telemetry_reader_t *r, uint64_t n); +/* Total records the host has lost since the region was created. */ +uint64_t ouro_telemetry_dropped(const ouro_telemetry_reader_t *r); + +/* Sleep until a record is available, `extra_fd` (may be -1, e.g. the client's + * host pidfd) is readable, or timeout_ms passes (negative: no timeout). Returns + * 1 when records are available, 0 on timeout or extra_fd, -1 on error. */ +int ouro_telemetry_wait(ouro_telemetry_reader_t *r, int extra_fd, int timeout_ms); + +#endif /* OUROBOROS_TELEMETRY_H */ diff --git a/c/src/client.c b/c/src/client.c index 259e8d6..712a845 100644 --- a/c/src/client.c +++ b/c/src/client.c @@ -10,6 +10,7 @@ #include #include +#include #include /* D15: a process handle on the peer at the other end of the attach socket. */ @@ -33,9 +34,13 @@ ouro_client_result_t ouro_client_attach(int sock, ouro_client_t *c) return ouro_client_attach_as(sock, c, 0, 0); } -ouro_client_result_t ouro_client_attach_as(int sock, ouro_client_t *c, uint32_t principal_class, - uint32_t principal_ref) +/* telemetry_fds is non-NULL when the caller asks for the D16 grant; on a grant + * it receives the region and doorbell descriptors, otherwise -1 each. */ +static ouro_client_result_t attach(int sock, ouro_client_t *c, uint32_t principal_class, + uint32_t principal_ref, int *telemetry_fds) { + if (telemetry_fds) + telemetry_fds[0] = telemetry_fds[1] = -1; if (!c || sock < 0) return OURO_CLIENT_ERR_ARG; memset(c, 0, sizeof *c); @@ -50,6 +55,8 @@ ouro_client_result_t ouro_client_attach_as(int sock, ouro_client_t *c, uint32_t req.principal_ref = principal_ref; /* D13/D15: an older host reads only the first 16 bytes and grants neither. */ req.capabilities = OURO_ATTACH_CAP_DOORBELL | OURO_ATTACH_CAP_PROCESS_LIVENESS; + if (telemetry_fds) + req.capabilities |= OURO_ATTACH_CAP_TELEMETRY; if (ouro_fd_send(sock, &req, sizeof req, NULL, 0) != 0) return OURO_CLIENT_ERR_OS; @@ -73,9 +80,12 @@ ouro_client_result_t ouro_client_attach_as(int sock, ouro_client_t *c, uint32_t /* The grant's capabilities and its descriptor count must agree. A host * grants only what was asked for, so any other bit is a protocol error. */ int doorbell = (c->reply.capabilities & OURO_ATTACH_CAP_DOORBELL) != 0; - if ((c->reply.capabilities & ~(OURO_ATTACH_CAP_DOORBELL | OURO_ATTACH_CAP_PROCESS_LIVENESS)) != - 0 || - nfd != (doorbell ? OURO_ATTACH_FDS_DOORBELL : OURO_ATTACH_FDS)) + int telemetry = (c->reply.capabilities & OURO_ATTACH_CAP_TELEMETRY) != 0; + uint32_t asked = req.capabilities; + if ((c->reply.capabilities & ~asked) != 0 || (telemetry && !doorbell) || + nfd != (telemetry ? OURO_ATTACH_FDS_TELEMETRY + : doorbell ? OURO_ATTACH_FDS_DOORBELL + : OURO_ATTACH_FDS)) { for (int i = 0; i < nfd; i++) close(fds[i]); @@ -128,15 +138,49 @@ ouro_client_result_t ouro_client_attach_as(int sock, ouro_client_t *c, uint32_t done: /* The mappings hold their own references; the region descriptors can be - * closed. The doorbells stay open while the client is attached. */ + * closed. The doorbells stay open while the client is attached, and the + * telemetry pair goes to the caller. */ for (int i = 0; i < nfd; i++) - if (!(rc == OURO_CLIENT_OK && doorbell && i >= OURO_ATTACH_FD_HOST_DOORBELL)) + { + if (rc == OURO_CLIENT_OK && telemetry && i >= OURO_ATTACH_FD_TELEMETRY) + telemetry_fds[i - OURO_ATTACH_FD_TELEMETRY] = fds[i]; + else if (!(rc == OURO_CLIENT_OK && doorbell && i >= OURO_ATTACH_FD_HOST_DOORBELL)) close(fds[i]); + } if (rc != OURO_CLIENT_OK) ouro_client_detach(c); return rc; } +ouro_client_result_t ouro_client_attach_as(int sock, ouro_client_t *c, uint32_t principal_class, + uint32_t principal_ref) +{ + return attach(sock, c, principal_class, principal_ref, NULL); +} + +ouro_client_result_t ouro_client_attach_telemetry(int sock, ouro_client_t *c, + uint32_t principal_class, + uint32_t principal_ref, + ouro_telemetry_reader_t *reader, int *granted) +{ + if (!reader || !granted) + return OURO_CLIENT_ERR_ARG; + *granted = 0; + memset(reader, 0, sizeof *reader); + reader->region.fd = reader->doorbell_fd = -1; + int fds[2]; + ouro_client_result_t rc = attach(sock, c, principal_class, principal_ref, fds); + if (rc != OURO_CLIENT_OK || fds[0] < 0) + return rc; + if (ouro_telemetry_reader_open(fds[0], fds[1], reader) != OURO_TELEMETRY_OK) + { + ouro_client_detach(c); + return OURO_CLIENT_ERR_PROTOCOL; + } + *granted = 1; + return OURO_CLIENT_OK; +} + static void doorbell_ring(int fd); void ouro_client_detach(ouro_client_t *c) diff --git a/c/src/host.c b/c/src/host.c index 465322c..a46c8a6 100644 --- a/c/src/host.c +++ b/c/src/host.c @@ -22,6 +22,8 @@ #include #include +#include "telemetry_internal.h" + /* Attach negotiation and frame validation must advertise the same version. */ #define OURO_HOST_WIRE_VERSION OURO_WIRE_VERSION @@ -131,6 +133,9 @@ static void slot_release(ouro_host_t *h, uint32_t idx) if (!s->in_use) return; ouro_route_forget_slot(h, idx); + /* D16: the region outlives its consumer; the next granted one resumes. */ + if (h->telemetry_slot == idx) + h->telemetry_slot = UINT32_MAX; /* A departing client's arena leases must not strand the arena: drop the refs * it held as a producer and as a consumer. */ ouro_arena_reap_producer(&h->arena, idx); @@ -185,6 +190,8 @@ ouro_host_result_t ouro_host_create(ouro_host_t *h, const ouro_host_config_t *cf h->control_fd = -1; h->arena_fd = -1; h->doorbell_fd = -1; + h->telemetry_doorbell_fd = -1; + h->telemetry_slot = UINT32_MAX; /* Control region. */ if (ouro_region_create("ring-control", ouro_control_bytes(), &h->control_region) != OURO_REGION_OK) @@ -276,6 +283,7 @@ void ouro_host_destroy(ouro_host_t *h) if (h->doorbell_fd >= 0) close(h->doorbell_fd); h->arena_fd = h->control_fd = h->doorbell_fd = -1; + ouro_telemetry_host_destroy(h); } /* ------------------------------------------------------------------ */ @@ -459,14 +467,34 @@ static ouro_host_result_t serve_attach(ouro_host_t *h, int conn_fd, uint32_t *sl } } + /* D16: the consumer end, to the one granted slot that asked, with a + * doorbell, while no other live slot holds it. */ + int telemetry_bell = -1; + if ((req.capabilities & OURO_ATTACH_CAP_TELEMETRY) && h->telemetry && s->telemetry_granted && + nfds == OURO_ATTACH_FDS_DOORBELL && h->telemetry_slot == UINT32_MAX) + { + telemetry_bell = dup(h->telemetry_doorbell_fd); + if (telemetry_bell >= 0) + { + reply.capabilities |= OURO_ATTACH_CAP_TELEMETRY; + nfds = OURO_ATTACH_FDS_TELEMETRY; + } + } + int gfds[OURO_ATTACH_FDS_MAX]; gfds[OURO_ATTACH_FD_CONTROL] = control_ro; gfds[OURO_ATTACH_FD_ARENA] = h->arena_fd; gfds[OURO_ATTACH_FD_QPAIR] = s->qpair_fd; gfds[OURO_ATTACH_FD_HOST_DOORBELL] = h->doorbell_fd; gfds[OURO_ATTACH_FD_CLIENT_DOORBELL] = s->doorbell_fd; + gfds[OURO_ATTACH_FD_TELEMETRY] = h->telemetry_region.fd; + gfds[OURO_ATTACH_FD_TELEMETRY_BELL] = telemetry_bell; int rc = ouro_fd_send(conn_fd, &reply, sizeof reply, gfds, nfds); close(control_ro); /* the client holds its own dup now */ + if (telemetry_bell >= 0) + close(telemetry_bell); + if (rc == 0 && (reply.capabilities & OURO_ATTACH_CAP_TELEMETRY)) + h->telemetry_slot = idx; if (rc != 0) { /* The client never received its grant; undo the admission. */ diff --git a/c/src/route.c b/c/src/route.c index bea246f..5decad2 100644 --- a/c/src/route.c +++ b/c/src/route.c @@ -40,6 +40,8 @@ #include #include +#include "telemetry_internal.h" + #define KIND_SERVER_NONE (-1) /* ---- slot bitmaps ---- */ @@ -244,6 +246,7 @@ static void emit_control(ouro_host_t *h, int dst, uint32_t kind, uint64_t corr, } f.seq = ++h->seq; f.dst_handle = dst < 0 ? 0 : (uint32_t)dst; + ouro_telemetry_stamped(h, &f); if (h->tap) h->tap(h->tap_ctx, &f, (const uint8_t *)payload, plen); if (h->loss && (kind == OURO_KIND_OVERFLOW || kind == OURO_KIND_PRODUCER_REAPED)) @@ -790,6 +793,7 @@ static void lease_release(ouro_host_t *h, uint32_t src, ouro_frame_t *f, const u f->payload_len == OURO_LEASE_RELEASE_LEN) p = ring_slot + f->payload_ref; f->seq = ++h->seq; + ouro_telemetry_stamped(h, f); if (h->tap) h->tap(h->tap_ctx, f, p, p ? f->payload_len : 0); if (!p || ouro_arena_release(&h->arena, le32_at(p + OURO_LEASE_RELEASE_OFF_LEASE), @@ -802,6 +806,9 @@ uint32_t ouro_host_pump(ouro_host_t *h) if (!h || !h->slots) return 0; uint32_t routed = 0; + /* D16: records are written per event, the consumer is signalled once. */ + h->in_pump = 1; + h->telemetry_now = 0; for (uint32_t s = 0; s < h->cfg.max_slots; s++) { @@ -863,6 +870,7 @@ uint32_t ouro_host_pump(ouro_host_t *h) else { f.seq = ++h->seq; + ouro_telemetry_stamped(h, &f); /* D16: off the host first */ if (h->tap) { const uint8_t *tap_payload = inl; @@ -918,5 +926,7 @@ uint32_t ouro_host_pump(ouro_host_t *h) OURO_HEARTBEAT_DETACHED) (void)ouro_host_release_slot(h, s); } + h->in_pump = 0; + ouro_telemetry_host_signal(h); return routed; } diff --git a/c/src/telemetry.c b/c/src/telemetry.c new file mode 100644 index 0000000..197f56c --- /dev/null +++ b/c/src/telemetry.c @@ -0,0 +1,176 @@ +/* telemetry.c: the D16 record encoding and the telemetry consumer end. */ +#include + +#include +#include +#include +#include +#include + +static void put32(uint8_t *p, uint32_t v) +{ + for (int i = 0; i < 4; i++) + p[i] = (uint8_t)(v >> (8 * i)); +} +static void put64(uint8_t *p, uint64_t v) +{ + for (int i = 0; i < 8; i++) + p[i] = (uint8_t)(v >> (8 * i)); +} +static uint32_t get32(const uint8_t *p) +{ + return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); +} +static uint64_t get64(const uint8_t *p) +{ + return (uint64_t)get32(p) | ((uint64_t)get32(p + 4) << 32); +} + +size_t ouro_telemetry_region_bytes(uint32_t capacity) +{ + size_t ring = ouro_ring_bytes(OURO_TELEMETRY_RECORD_LEN, capacity); + return ring ? OURO_TELEMETRY_REGION_OFF_RING + ring : 0; +} + +void ouro_telemetry_encode(uint8_t *out, uint32_t type, uint64_t realtime_ns, uint64_t lost, + const ouro_frame_t *frame) +{ + memset(out, 0, OURO_TELEMETRY_RECORD_LEN); + put32(out + OURO_TELEMETRY_OFF_TYPE, type); + put64(out + OURO_TELEMETRY_OFF_REALTIME_NS, realtime_ns); + put64(out + OURO_TELEMETRY_OFF_LOST, lost); + if (frame) + (void)ouro_wire_encode(frame, out + OURO_TELEMETRY_OFF_FRAME, OURO_WIRE_HDR_LEN); +} + +ouro_telemetry_result_t ouro_telemetry_decode(const uint8_t *in, ouro_telemetry_record_t *out) +{ + if (!in || !out) + return OURO_TELEMETRY_ERR_ARG; + memset(out, 0, sizeof *out); + out->type = get32(in + OURO_TELEMETRY_OFF_TYPE); + out->realtime_ns = get64(in + OURO_TELEMETRY_OFF_REALTIME_NS); + out->lost = get64(in + OURO_TELEMETRY_OFF_LOST); + if (get32(in + OURO_TELEMETRY_OFF_FLAGS) != 0) + return OURO_TELEMETRY_ERR_LAYOUT; + if (out->type == OURO_TELEMETRY_RECORD_GAP) + return out->lost ? OURO_TELEMETRY_OK : OURO_TELEMETRY_ERR_LAYOUT; + if (out->type != OURO_TELEMETRY_RECORD_EVENT || out->lost != 0) + return OURO_TELEMETRY_ERR_LAYOUT; + return ouro_wire_decode(in + OURO_TELEMETRY_OFF_FRAME, OURO_WIRE_HDR_LEN, &out->frame) == + OURO_WIRE_OK + ? OURO_TELEMETRY_OK + : OURO_TELEMETRY_ERR_LAYOUT; +} + +ouro_telemetry_result_t ouro_telemetry_reader_open(int region_fd, int doorbell_fd, + ouro_telemetry_reader_t *r) +{ + if (!r) + return OURO_TELEMETRY_ERR_ARG; + memset(r, 0, sizeof *r); + r->region.fd = -1; + r->doorbell_fd = doorbell_fd; + struct stat st; + if (region_fd < 0 || doorbell_fd < 0 || fstat(region_fd, &st) != 0 || + st.st_size <= OURO_TELEMETRY_REGION_OFF_RING) + goto fail; + if (ouro_region_map(region_fd, (size_t)st.st_size, 1, &r->region) != OURO_REGION_OK) + goto fail; + close(region_fd); /* the mapping holds its own reference */ + region_fd = r->region.fd = -1; + r->hdr = r->region.base; + /* Validate once, then use only the local copy (as every ring does). */ + if (atomic_load_explicit(&r->hdr->magic, memory_order_acquire) != OURO_TELEMETRY_MAGIC || + atomic_load_explicit(&r->hdr->version, memory_order_relaxed) != OURO_TELEMETRY_VERSION || + atomic_load_explicit(&r->hdr->record_size, memory_order_relaxed) != + OURO_TELEMETRY_RECORD_LEN || + ouro_ring_attach((uint8_t *)r->region.base + OURO_TELEMETRY_REGION_OFF_RING, + r->region.size - OURO_TELEMETRY_REGION_OFF_RING, + &r->ring) != OURO_RING_OK || + r->ring.slot_size != OURO_TELEMETRY_RECORD_LEN) + { + ouro_telemetry_reader_close(r); + return OURO_TELEMETRY_ERR_LAYOUT; + } + return OURO_TELEMETRY_OK; +fail: + if (region_fd >= 0) + close(region_fd); + ouro_telemetry_reader_close(r); + return OURO_TELEMETRY_ERR_OS; +} + +void ouro_telemetry_reader_close(ouro_telemetry_reader_t *r) +{ + if (!r) + return; + ouro_region_unmap(&r->region); + if (r->doorbell_fd >= 0) + close(r->doorbell_fd); + memset(r, 0, sizeof *r); + r->region.fd = -1; + r->doorbell_fd = -1; +} + +uint64_t ouro_telemetry_available(const ouro_telemetry_reader_t *r) +{ + if (!r || !r->ring.shared) + return 0; + uint64_t n = ouro_ring_count(&r->ring); + /* A head the host never wrote (a scribbled tail) is not a ring state. */ + return n > r->ring.capacity ? 0 : n; +} + +const uint8_t *ouro_telemetry_peek(const ouro_telemetry_reader_t *r, uint64_t i) +{ + if (!r || !r->ring.shared || i >= ouro_telemetry_available(r)) + return NULL; + uint64_t tail = atomic_load_explicit(&r->ring.shared->tail, memory_order_relaxed); + return r->ring.slots + (size_t)((tail + i) & r->ring.mask) * r->ring.slot_size; +} + +void ouro_telemetry_release(ouro_telemetry_reader_t *r, uint64_t n) +{ + if (!r || !r->ring.shared || n == 0) + return; + uint64_t avail = ouro_telemetry_available(r); + if (n > avail) + n = avail; + uint64_t tail = atomic_load_explicit(&r->ring.shared->tail, memory_order_relaxed); + /* Release: the reads of these slots happen-before the host reuses them. */ + atomic_store_explicit(&r->ring.shared->tail, tail + n, memory_order_release); +} + +uint64_t ouro_telemetry_dropped(const ouro_telemetry_reader_t *r) +{ + return r && r->hdr ? atomic_load_explicit(&r->hdr->dropped, memory_order_relaxed) : 0; +} + +int ouro_telemetry_wait(ouro_telemetry_reader_t *r, int extra_fd, int timeout_ms) +{ + if (!r || !r->hdr) + return -1; + /* D13: flag, fence, re-check, sleep. Either the host's fenced load sees the + * flag and rings, or this re-check sees its record. */ + atomic_store_explicit(&r->hdr->consumer_waiting, 1, memory_order_seq_cst); + atomic_thread_fence(memory_order_seq_cst); + if (ouro_telemetry_available(r) > 0) + { + atomic_store_explicit(&r->hdr->consumer_waiting, 0, memory_order_relaxed); + return 1; + } + struct pollfd p[2] = {{.fd = r->doorbell_fd, .events = POLLIN}, + {.fd = extra_fd, .events = POLLIN}}; + int rc = poll(p, extra_fd >= 0 ? 2 : 1, timeout_ms); + if (rc > 0 && (p[0].revents & POLLIN)) + { + uint64_t count; + while (read(r->doorbell_fd, &count, sizeof count) < 0 && errno == EINTR) + ; + } + atomic_store_explicit(&r->hdr->consumer_waiting, 0, memory_order_relaxed); + if (rc < 0 && errno != EINTR) + return -1; + return ouro_telemetry_available(r) > 0 ? 1 : 0; +} diff --git a/c/src/telemetry_host.c b/c/src/telemetry_host.c new file mode 100644 index 0000000..ac25951 --- /dev/null +++ b/c/src/telemetry_host.c @@ -0,0 +1,195 @@ +/* telemetry_host.c: the host writes one D16 record per seq-stamped event. */ +#include "telemetry_internal.h" + +#include +#include +#include +#include +#include +#include + +#include + +ouro_host_result_t ouro_host_telemetry_enable(ouro_host_t *h, uint32_t capacity) +{ + if (!h || !h->control) + return OURO_HOST_ERR_ARG; + if (h->telemetry) + return OURO_HOST_OK; + if (capacity == 0) + capacity = OURO_TELEMETRY_CAPACITY_DEFAULT; + size_t bytes = ouro_telemetry_region_bytes(capacity); + if (bytes == 0) + return OURO_HOST_ERR_ARG; + h->telemetry_doorbell_fd = -1; + h->telemetry_slot = UINT32_MAX; + if (ouro_region_create("ring-telemetry", bytes, &h->telemetry_region) != OURO_REGION_OK) + return OURO_HOST_ERR_REGION; + int fd = h->telemetry_region.fd; + if (ouro_region_map(fd, bytes, 1, &h->telemetry_region) != OURO_REGION_OK) + { + close(fd); + memset(&h->telemetry_region, 0, sizeof h->telemetry_region); + return OURO_HOST_ERR_REGION; + } + ouro_telemetry_region_t *t = h->telemetry_region.base; + uint8_t *ring_mem = (uint8_t *)h->telemetry_region.base + OURO_TELEMETRY_REGION_OFF_RING; + if (ouro_ring_init(ring_mem, bytes - OURO_TELEMETRY_REGION_OFF_RING, OURO_TELEMETRY_RECORD_LEN, + capacity, &h->telemetry_ring) != OURO_RING_OK) + goto fail; + h->telemetry_doorbell_fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC); + if (h->telemetry_doorbell_fd < 0) + goto fail; + atomic_store_explicit(&t->version, OURO_TELEMETRY_VERSION, memory_order_relaxed); + atomic_store_explicit(&t->record_size, OURO_TELEMETRY_RECORD_LEN, memory_order_relaxed); + atomic_store_explicit(&t->host_epoch, ouro_control_epoch(h->control), memory_order_relaxed); + atomic_store_explicit(&t->magic, OURO_TELEMETRY_MAGIC, memory_order_release); + h->telemetry_head = h->telemetry_tail_cache = h->telemetry_lost = 0; + h->telemetry = t; + return OURO_HOST_OK; +fail: + ouro_telemetry_host_destroy(h); + return OURO_HOST_ERR_OS; +} + +ouro_host_result_t ouro_host_grant_telemetry(ouro_host_t *h, uint32_t slot) +{ + if (!h || !h->slots || slot >= h->cfg.max_slots || !h->slots[slot].in_use) + return OURO_HOST_ERR_ARG; + h->slots[slot].telemetry_granted = 1; + return OURO_HOST_OK; +} + +void ouro_telemetry_host_destroy(ouro_host_t *h) +{ + h->telemetry = NULL; + if (h->telemetry_region.size) + { + int fd = h->telemetry_region.fd; + ouro_region_unmap(&h->telemetry_region); + if (fd >= 0) + close(fd); + } + memset(&h->telemetry_region, 0, sizeof h->telemetry_region); + memset(&h->telemetry_ring, 0, sizeof h->telemetry_ring); + if (h->telemetry_doorbell_fd >= 0) + close(h->telemetry_doorbell_fd); + h->telemetry_doorbell_fd = -1; + h->telemetry_slot = UINT32_MAX; +} + +static uint64_t realtime_ns(void) +{ + struct timespec ts; + if (clock_gettime(CLOCK_REALTIME, &ts) != 0) + return 0; + return (uint64_t)ts.tv_sec * 1000000000ULL + (uint64_t)ts.tv_nsec; +} + +/* The host's encoder. A slot's reserved bytes are zero from creation and the + * host never writes them, so only owned fields are stored. On a little-endian + * host the wire header after its magic is the frame's own bytes in two runs; + * the asserts pin that. The frame was validated at ingress (or built by the + * host), so it is not validated again. */ +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +_Static_assert(offsetof(ouro_frame_t, hdr_flags) == 0 && offsetof(ouro_frame_t, principal_ref) == 8, + "wire bytes 4..16 are frame bytes 0..12"); +_Static_assert(offsetof(ouro_frame_t, correlation_id) == 16 && + offsetof(ouro_frame_t, generation) == 60 && sizeof(ouro_frame_t) == 64, + "wire bytes 16..64 are frame bytes 16..64"); +static void put_record(uint8_t *slot, uint32_t type, uint64_t now, uint64_t lost, + const ouro_frame_t *f) +{ + memcpy(slot + OURO_TELEMETRY_OFF_TYPE, &type, 4); + memcpy(slot + OURO_TELEMETRY_OFF_REALTIME_NS, &now, 8); + memcpy(slot + OURO_TELEMETRY_OFF_LOST, &lost, 8); + uint8_t *hdr = slot + OURO_TELEMETRY_OFF_FRAME; + if (!f) + { + memset(hdr, 0, OURO_WIRE_HDR_LEN); + return; + } + uint32_t magic = OURO_WIRE_MAGIC; + memcpy(hdr, &magic, 4); + memcpy(hdr + 4, f, 12); + memcpy(hdr + 16, (const uint8_t *)f + 16, 48); +} +#else +static void put_record(uint8_t *slot, uint32_t type, uint64_t now, uint64_t lost, + const ouro_frame_t *f) +{ + ouro_telemetry_encode(slot, type, now, lost, f); +} +#endif + +/* The producer end, with the consumer's index cached: it is read across cores + * only when the cached view says the ring is full. */ +static uint8_t *claim(ouro_host_t *h, uint64_t need) +{ + uint64_t head = h->telemetry_head; + if (head - h->telemetry_tail_cache + need > h->telemetry_ring.capacity) + { + h->telemetry_tail_cache = + atomic_load_explicit(&h->telemetry_ring.shared->tail, memory_order_acquire); + /* A tail past head is a scribble, not a state: treat it as full. */ + if (h->telemetry_tail_cache > head || + head - h->telemetry_tail_cache + need > h->telemetry_ring.capacity) + return NULL; + } + return h->telemetry_ring.slots + (size_t)(head & h->telemetry_ring.mask) * OURO_TELEMETRY_RECORD_LEN; +} + +static void publish(ouro_host_t *h) +{ + /* Release: the record's bytes happen-before the consumer's acquire of head. */ + atomic_store_explicit(&h->telemetry_ring.shared->head, ++h->telemetry_head, + memory_order_release); +} + +void ouro_telemetry_host_write(ouro_host_t *h, const ouro_frame_t *f) +{ + /* One clock read per pump: records in a pump share its time, and seq orders + * them exactly. An event stamped outside a pump reads its own. */ + uint64_t now = h->telemetry_now; + if (!now || !h->in_pump) + now = h->telemetry_now = realtime_ns(); + if (h->telemetry_lost) + { + /* The gap record goes first, and only when the event fits after it, so the + * stream never says "n lost" and then loses the next one silently. */ + uint8_t *gap = claim(h, 2); + if (!gap) + goto lost; + put_record(gap, OURO_TELEMETRY_RECORD_GAP, now, h->telemetry_lost, NULL); + publish(h); + h->telemetry_lost = 0; + } + uint8_t *slot = claim(h, 1); + if (!slot) + goto lost; + put_record(slot, OURO_TELEMETRY_RECORD_EVENT, now, 0, f); + publish(h); + h->telemetry_unsignalled = 1; + if (!h->in_pump) + ouro_telemetry_host_signal(h); + return; +lost: + h->telemetry_lost++; + atomic_fetch_add_explicit(&h->telemetry->dropped, 1, memory_order_relaxed); +} + +void ouro_telemetry_host_signal(ouro_host_t *h) +{ + h->telemetry_now = 0; /* the next pump reads the clock again */ + if (!h->telemetry || !h->telemetry_unsignalled) + return; + h->telemetry_unsignalled = 0; + /* D13: pairs with the fence in ouro_telemetry_wait. */ + atomic_thread_fence(memory_order_seq_cst); + if (atomic_load_explicit(&h->telemetry->consumer_waiting, memory_order_relaxed) != 0) + { + uint64_t one = 1; + while (write(h->telemetry_doorbell_fd, &one, sizeof one) < 0 && errno == EINTR) + ; + } +} diff --git a/c/src/telemetry_internal.h b/c/src/telemetry_internal.h new file mode 100644 index 0000000..8b8d685 --- /dev/null +++ b/c/src/telemetry_internal.h @@ -0,0 +1,22 @@ +/* telemetry_internal.h: the host's D16 producer, called by route.c and host.c. */ +#ifndef OUROBOROS_TELEMETRY_INTERNAL_H +#define OUROBOROS_TELEMETRY_INTERNAL_H 1 + +#include + +/* Write the record for a just-stamped event. Never blocks; outside a pump it + * also signals the consumer. Inline so a host without telemetry pays a branch. */ +void ouro_telemetry_host_write(ouro_host_t *h, const ouro_frame_t *f); +static inline void ouro_telemetry_stamped(ouro_host_t *h, const ouro_frame_t *f) +{ + if (h->telemetry) + ouro_telemetry_host_write(h, f); +} + +/* Wake the consumer if it sleeps and something was written since it was last + * signalled. The pump calls it once on the way out. */ +void ouro_telemetry_host_signal(ouro_host_t *h); + +void ouro_telemetry_host_destroy(ouro_host_t *h); + +#endif diff --git a/c/tests/test_telemetry.c b/c/tests/test_telemetry.c new file mode 100644 index 0000000..5b10dbc --- /dev/null +++ b/c/tests/test_telemetry.c @@ -0,0 +1,389 @@ +/* test_telemetry.c: D16, tap telemetry leaves the host as each event is stamped. + * + * - A record round-trips, carries the stamped frame header, and holds no + * payload byte; a record with an unknown flag is refused. + * - Only the slot the embedding granted is handed the consumer end, only one + * at a time, and only to a client that asked. + * - Every stamped event is a record, in seq order, written before the tap + * runs. + * - A full ring never blocks the host: the loss is counted, and the next + * record that fits follows a gap record carrying the count. + * - A consumer asleep on the doorbell wakes for a pump; one that is awake is + * not signalled. + * - A later consumer resumes where the last one stopped. + * - Nothing leaks a descriptor. + */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +static void must(int cond, const char *what) +{ + if (!cond) + { + fprintf(stderr, "FAIL: %s\n", what); + abort(); + } +} + +static int open_fds(void) +{ + DIR *d = opendir("/proc/self/fd"); + if (!d) + return -1; + int n = 0; + while (readdir(d)) + n++; + closedir(d); + return n; +} + +#define KIND 0x2a01u + +static void host_config(ouro_host_config_t *cfg) +{ + memset(cfg, 0, sizeof *cfg); + cfg->max_slots = 4; + cfg->slot_size = 256; + cfg->inline_budget = 192; + cfg->queue_capacity = 64; + cfg->arena_size = 64 * 1024; +} + +/* The embedding's attach hook: principal_ref 7 is the telemetry exporter. */ +static ouro_attach_status_t grant_exporter(void *ctx, ouro_host_t *host, uint32_t slot, + const ouro_attach_request_t *req) +{ + (void)ctx; + if (req->principal_ref == 7) + must(ouro_host_grant_telemetry(host, slot) == OURO_HOST_OK, "grant"); + return OURO_ATTACH_OK; +} + +struct serve +{ + ouro_host_t *h; + int fd; +}; + +static void *serve_one(void *p) +{ + struct serve *s = p; + (void)ouro_host_serve_attach(s->h, s->fd); + return NULL; +} + +/* Attach over a real socketpair, asking for telemetry as `principal_ref`. */ +static void attach_consumer(ouro_host_t *h, uint32_t principal_ref, ouro_client_t *c, + ouro_telemetry_reader_t *r, int *granted) +{ + int sv[2]; + must(socketpair(AF_UNIX, SOCK_SEQPACKET | SOCK_CLOEXEC, 0, sv) == 0, "socketpair"); + struct serve s = {.h = h, .fd = sv[1]}; + pthread_t t; + must(pthread_create(&t, NULL, serve_one, &s) == 0, "serve thread"); + must(ouro_client_attach_telemetry(sv[0], c, 1, principal_ref, r, granted) == OURO_CLIENT_OK, + "consumer attached"); + pthread_join(t, NULL); + close(sv[0]); + close(sv[1]); +} + +static uint64_t tapped_seq[256]; +static int tapped; +static ouro_telemetry_reader_t *tap_reader; +static uint64_t tap_saw_records[256]; + +static void tap(void *ctx, const ouro_frame_t *f, const uint8_t *payload, uint32_t len) +{ + (void)ctx; + (void)payload; + (void)len; + if (tapped < 256) + { + tapped_seq[tapped] = f->seq; + tap_saw_records[tapped] = tap_reader ? ouro_telemetry_available(tap_reader) : 0; + tapped++; + } +} + +static void test_record_codec(void) +{ + ouro_frame_t f; + memset(&f, 0, sizeof f); + f.hdr_flags = OURO_F_NOTIFICATION | OURO_F_INLINE; + f.wire_version = OURO_WIRE_VERSION; + f.event_kind = KIND; + f.principal_ref = 9; + f.seq = 41; + f.payload_len = 5; + f.payload_ref = OURO_WIRE_HDR_LEN; + f.src_handle = 2; + uint8_t rec[OURO_TELEMETRY_RECORD_LEN]; + ouro_telemetry_encode(rec, OURO_TELEMETRY_RECORD_EVENT, 1234, 0, &f); + ouro_telemetry_record_t out; + must(ouro_telemetry_decode(rec, &out) == OURO_TELEMETRY_OK, "decodes"); + must(out.type == OURO_TELEMETRY_RECORD_EVENT && out.realtime_ns == 1234, "type and time"); + must(out.frame.seq == 41 && out.frame.event_kind == KIND && out.frame.principal_ref == 9 && + out.frame.payload_len == 5 && out.frame.src_handle == 2, + "the stamped header"); + for (int i = 96; i < OURO_TELEMETRY_RECORD_LEN; i++) + must(rec[i] == 0, "reserved bytes are zero"); + rec[OURO_TELEMETRY_OFF_FLAGS] = 1; + must(ouro_telemetry_decode(rec, &out) == OURO_TELEMETRY_ERR_LAYOUT, "unknown flag refused"); + + ouro_telemetry_encode(rec, OURO_TELEMETRY_RECORD_GAP, 5, 3, NULL); + must(ouro_telemetry_decode(rec, &out) == OURO_TELEMETRY_OK && out.type == OURO_TELEMETRY_RECORD_GAP && + out.lost == 3, + "gap"); + printf(" codec: a record is the stamped header and nothing of the payload\n"); +} + +static void test_grant(void) +{ + ouro_host_config_t cfg; + host_config(&cfg); + ouro_host_t h; + must(ouro_host_create(&h, &cfg, NULL, NULL) == OURO_HOST_OK, "host"); + ouro_host_set_attach_hook(&h, grant_exporter, NULL); + + /* Before enable, nobody is granted it. */ + ouro_client_t c; + ouro_telemetry_reader_t r; + int granted = 1; + attach_consumer(&h, 7, &c, &r, &granted); + must(!granted && !(c.reply.capabilities & OURO_ATTACH_CAP_TELEMETRY), "not enabled"); + ouro_client_detach(&c); + (void)ouro_host_pump(&h); + + must(ouro_host_telemetry_enable(&h, 64) == OURO_HOST_OK, "enable"); + /* A principal the embedding did not grant. */ + attach_consumer(&h, 8, &c, &r, &granted); + must(!granted, "ungranted principal refused"); + ouro_client_detach(&c); + (void)ouro_host_pump(&h); + + ouro_client_t a, b; + ouro_telemetry_reader_t ra, rb; + attach_consumer(&h, 7, &a, &ra, &granted); + must(granted && (a.reply.capabilities & OURO_ATTACH_CAP_TELEMETRY), "granted"); + attach_consumer(&h, 7, &b, &rb, &granted); + must(!granted, "only one consumer at a time"); + ouro_client_detach(&b); + ouro_telemetry_reader_close(&ra); + ouro_client_detach(&a); + (void)ouro_host_pump(&h); + attach_consumer(&h, 7, &b, &rb, &granted); + must(granted, "the next consumer is granted once the first is gone"); + ouro_telemetry_reader_close(&rb); + ouro_client_detach(&b); + ouro_host_destroy(&h); + printf(" grant: only the granted slot, one at a time, only when asked\n"); +} + +static void test_stream(void) +{ + ouro_host_config_t cfg; + host_config(&cfg); + ouro_host_t h; + must(ouro_host_create(&h, &cfg, NULL, NULL) == OURO_HOST_OK, "host"); + ouro_host_set_attach_hook(&h, grant_exporter, NULL); + must(ouro_host_telemetry_enable(&h, 64) == OURO_HOST_OK, "enable"); + ouro_host_set_tap(&h, tap, NULL); + + ouro_client_t cons; + ouro_telemetry_reader_t r; + int granted = 0; + attach_consumer(&h, 7, &cons, &r, &granted); + must(granted, "granted"); + tap_reader = &r; + tapped = 0; + + ouro_client_t prod, obs; + must(ouro_host_attach_local(&h, &prod, 1) == OURO_CLIENT_OK, "producer"); + must(ouro_host_attach_local(&h, &obs, 1) == OURO_CLIENT_OK, "observer"); + must(ouro_host_subscribe(&h, obs.reply.handle_id, KIND) == OURO_HOST_OK, "subscribe"); + + const char secret[] = "payload-never-exported"; + for (int i = 0; i < 10; i++) + must(ouro_client_publish(&prod, KIND, secret, sizeof secret) == OURO_CLIENT_OK, "publish"); + (void)ouro_host_pump(&h); + + must(tapped == 10, "tapped ten"); + must(ouro_telemetry_available(&r) == 10, "ten records"); + for (int i = 0; i < 10; i++) + { + must(tap_saw_records[i] == (uint64_t)i + 1, "each record is written before the tap runs"); + const uint8_t *raw = ouro_telemetry_peek(&r, (uint64_t)i); + ouro_telemetry_record_t rec; + must(ouro_telemetry_decode(raw, &rec) == OURO_TELEMETRY_OK, "decode"); + must(rec.type == OURO_TELEMETRY_RECORD_EVENT && rec.frame.seq == tapped_seq[i], "seq order"); + must(rec.frame.event_kind == KIND && rec.frame.payload_len == sizeof secret && + rec.frame.src_handle == prod.reply.handle_id, + "kind, length, source"); + must(memmem(raw, OURO_TELEMETRY_RECORD_LEN, "payload", 7) == NULL, "no payload byte"); + must(rec.realtime_ns > 0, "timestamped"); + uint8_t portable[OURO_TELEMETRY_RECORD_LEN]; + ouro_telemetry_encode(portable, rec.type, rec.realtime_ns, rec.lost, &rec.frame); + must(memcmp(portable, raw, sizeof portable) == 0, + "the host's fast encoder writes the portable encoding's bytes"); + } + ouro_telemetry_release(&r, 10); + must(ouro_telemetry_available(&r) == 0, "released"); + tap_reader = NULL; + + ouro_client_detach(&prod); + ouro_client_detach(&obs); + ouro_telemetry_reader_close(&r); + ouro_client_detach(&cons); + ouro_host_destroy(&h); + printf(" stream: every stamped event, in seq order, before the tap\n"); +} + +static void test_full_ring_gap(void) +{ + ouro_host_config_t cfg; + host_config(&cfg); + ouro_host_t h; + must(ouro_host_create(&h, &cfg, NULL, NULL) == OURO_HOST_OK, "host"); + ouro_host_set_attach_hook(&h, grant_exporter, NULL); + must(ouro_host_telemetry_enable(&h, 4) == OURO_HOST_OK, "enable, four records"); + ouro_client_t cons; + ouro_telemetry_reader_t r; + int granted = 0; + attach_consumer(&h, 7, &cons, &r, &granted); + must(granted, "granted"); + + ouro_client_t prod, obs; + must(ouro_host_attach_local(&h, &prod, 1) == OURO_CLIENT_OK, "producer"); + must(ouro_host_attach_local(&h, &obs, 1) == OURO_CLIENT_OK, "observer"); + must(ouro_host_subscribe(&h, obs.reply.handle_id, KIND) == OURO_HOST_OK, "subscribe"); + for (int i = 0; i < 10; i++) + must(ouro_client_publish(&prod, KIND, "x", 1) == OURO_CLIENT_OK, "publish"); + must(ouro_host_pump(&h) == 10, "every event routed: telemetry never blocks the host"); + must(ouro_telemetry_available(&r) == 4, "the ring is full"); + must(ouro_telemetry_dropped(&r) == 6, "six counted lost"); + + ouro_telemetry_record_t rec; + uint64_t last = 0; + for (int i = 0; i < 4; i++) + { + must(ouro_telemetry_decode(ouro_telemetry_peek(&r, (uint64_t)i), &rec) == OURO_TELEMETRY_OK, + "decode"); + must(rec.type == OURO_TELEMETRY_RECORD_EVENT, "events"); + last = rec.frame.seq; + } + ouro_telemetry_release(&r, 4); + /* Drain what the observer received so the next publish is not blocked. */ + ouro_event_t ev; + while (ouro_client_poll(&obs, &ev) == OURO_CLIENT_OK) + ; + must(ouro_client_publish(&prod, KIND, "y", 1) == OURO_CLIENT_OK, "publish"); + (void)ouro_host_pump(&h); + must(ouro_telemetry_available(&r) == 2, "a gap and the event"); + must(ouro_telemetry_decode(ouro_telemetry_peek(&r, 0), &rec) == OURO_TELEMETRY_OK && + rec.type == OURO_TELEMETRY_RECORD_GAP && rec.lost == 6, + "the gap carries the count"); + must(ouro_telemetry_decode(ouro_telemetry_peek(&r, 1), &rec) == OURO_TELEMETRY_OK && + rec.type == OURO_TELEMETRY_RECORD_EVENT && rec.frame.seq == last + 7, + "then the event, its seq showing what was lost"); + ouro_telemetry_release(&r, 2); + + ouro_client_detach(&prod); + ouro_client_detach(&obs); + ouro_telemetry_reader_close(&r); + ouro_client_detach(&cons); + ouro_host_destroy(&h); + printf(" full: the host never blocks; a gap record says how many were lost\n"); +} + +struct waiter +{ + ouro_telemetry_reader_t *r; + int result; +}; + +static void *wait_one(void *p) +{ + struct waiter *w = p; + w->result = ouro_telemetry_wait(w->r, -1, 5000); + return NULL; +} + +static void test_doorbell_and_resume(void) +{ + ouro_host_config_t cfg; + host_config(&cfg); + ouro_host_t h; + must(ouro_host_create(&h, &cfg, NULL, NULL) == OURO_HOST_OK, "host"); + ouro_host_set_attach_hook(&h, grant_exporter, NULL); + must(ouro_host_telemetry_enable(&h, 64) == OURO_HOST_OK, "enable"); + ouro_client_t cons; + ouro_telemetry_reader_t r; + int granted = 0; + attach_consumer(&h, 7, &cons, &r, &granted); + must(granted, "granted"); + ouro_client_t prod, obs; + must(ouro_host_attach_local(&h, &prod, 1) == OURO_CLIENT_OK, "producer"); + must(ouro_host_attach_local(&h, &obs, 1) == OURO_CLIENT_OK, "observer"); + must(ouro_host_subscribe(&h, obs.reply.handle_id, KIND) == OURO_HOST_OK, "subscribe"); + + /* Awake consumer: the host writes records and rings nothing. */ + must(ouro_client_publish(&prod, KIND, "a", 1) == OURO_CLIENT_OK, "publish"); + (void)ouro_host_pump(&h); + uint64_t count; + must(read(r.doorbell_fd, &count, sizeof count) < 0 && errno == EAGAIN, + "an awake consumer is not signalled"); + ouro_telemetry_release(&r, 1); + + /* Asleep: one pump of several events wakes it once. */ + struct waiter w = {.r = &r, .result = -2}; + pthread_t t; + must(pthread_create(&t, NULL, wait_one, &w) == 0, "waiter"); + while (!atomic_load(&r.hdr->consumer_waiting)) + sched_yield(); + for (int i = 0; i < 3; i++) + must(ouro_client_publish(&prod, KIND, "b", 1) == OURO_CLIENT_OK, "publish"); + (void)ouro_host_pump(&h); + pthread_join(t, NULL); + must(w.result == 1 && ouro_telemetry_available(&r) == 3, "woken for the pump"); + ouro_telemetry_release(&r, 1); /* exported one of three, then went away */ + + ouro_telemetry_reader_close(&r); + ouro_client_detach(&cons); + (void)ouro_host_pump(&h); + attach_consumer(&h, 7, &cons, &r, &granted); + must(granted && ouro_telemetry_available(&r) == 2, "the next consumer resumes at the tail"); + ouro_telemetry_release(&r, 2); + + ouro_client_detach(&prod); + ouro_client_detach(&obs); + ouro_telemetry_reader_close(&r); + ouro_client_detach(&cons); + ouro_host_destroy(&h); + printf(" doorbell: asleep wakes once per pump, awake is not signalled; resume at tail\n"); +} + +int main(void) +{ + int fds = open_fds(); + test_record_codec(); + test_grant(); + test_stream(); + test_full_ring_gap(); + test_doorbell_and_resume(); + must(open_fds() == fds, "no descriptor leaked"); + puts("telemetry: ok"); + return 0; +} diff --git a/scripts/check_perf_budget.sh b/scripts/check_perf_budget.sh index b0e88f1..474c881 100755 --- a/scripts/check_perf_budget.sh +++ b/scripts/check_perf_budget.sh @@ -9,14 +9,19 @@ baseline="$root/c/bench/baseline.json" bench="$build/ouroboros-ring-dispatch-bench" [ -x "$bench" ] || { echo "FAIL: $bench not built (cmake --build $build --target ouroboros-ring-dispatch-bench)" >&2; exit 1; } -ceiling=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["metrics"]["dispatch_overhead_ns"]["ceiling_ns"])' "$baseline") - -measured=$("$bench" 500000 | sed -n 's/^dispatch_overhead_ns=//p') -[ -n "$measured" ] || { echo "FAIL: the benchmark printed no dispatch_overhead_ns" >&2; exit 1; } - -echo "dispatch overhead: ${measured} ns/event (ceiling ${ceiling} ns)" -if [ "$measured" -gt "$ceiling" ]; then - echo "FAIL: ${measured} ns is over the ${ceiling} ns ceiling. Either a regression" >&2 - echo " landed or the ceiling needs a reviewed change in c/bench/baseline.json." >&2 - exit 1 -fi +check() { # metric env... + local metric=$1; shift + local ceiling measured + ceiling=$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["metrics"][sys.argv[2]]["ceiling_ns"])' "$baseline" "$metric") + measured=$(env "$@" "$bench" 500000 | sed -n 's/^dispatch_overhead_ns=//p') + [ -n "$measured" ] || { echo "FAIL: the benchmark printed no dispatch_overhead_ns" >&2; exit 1; } + echo "$metric: ${measured} ns/event (ceiling ${ceiling} ns)" + if [ "$measured" -gt "$ceiling" ]; then + echo "FAIL: ${measured} ns is over the ${ceiling} ns ceiling. Either a regression" >&2 + echo " landed or the ceiling needs a reviewed change in c/bench/baseline.json." >&2 + exit 1 + fi +} +check dispatch_overhead_ns OURO_BENCH_TELEMETRY=0 +# D16: the same dispatch with the telemetry hand-off on. +check dispatch_overhead_telemetry_ns OURO_BENCH_TELEMETRY=1 diff --git a/spec/decisions.md b/spec/decisions.md index b418664..4900232 100644 --- a/spec/decisions.md +++ b/spec/decisions.md @@ -1,4 +1,4 @@ -# Ring v0: rendered decisions (D1–D12) +# Ring v0: rendered decisions (D1–D16) - **State:** DECIDED. 2026-07-23, revision 7, after six rounds of roundtable review. - **Origin:** written while the ring was built inside @@ -814,6 +814,102 @@ already do by exiting. A client that stops heartbeating without the grant is rea --- +## D16: Tap telemetry leaves the host the moment an event is stamped + +*Added 2026-10-30, after v0. One attach capability bit, one new region, two more descriptors in a +telemetry grant; `layout_version` stays 1.* + +**Decision: when the host stamps an event's `seq`, it writes one fixed-size, content-free telemetry +record into a single-producer, single-consumer ring in its own region, before the tap callback runs +and before routing. A separate telemetry process holds the only consumer end and exports the records +off the host as soon as it is woken. The host never blocks, never allocates, and never makes a +system call per event for it. The ring owns the record and the hand-off; the exporter is a separate +process, so the ring library still never touches a network.** + +Before this, telemetry about ring traffic existed only as an embedding's in-process tap callback. In +the reference deployment that callback was the governance tap, and the Go `TapObserver` that would +have counted events could never be called, because only the C implementation has a host. So nothing +about ring traffic left the host at all. The guarantee this decision provides is that every +`seq`-stamped event is described off the host without waiting for the embedding, the tap, a module, +or a timer. The only window in which an event is known only to the host is the time between its +stamp and the telemetry process's next read. + +- **The record.** 128 bytes, little-endian, in `seq` order: + + | Offset | Field | | + |---|---|---| + | 0 | `type` u32 | `0` event, `1` gap | + | 4 | `flags` u32 | zero; a reader rejects a record with an unknown bit | + | 8 | `realtime_ns` u64 | `CLOCK_REALTIME` at the host's first stamp of the pump that stamped it | + | 16 | `lost` u64 | gap: records lost just before this one; event: zero | + | 24 | reserved u64 | zero | + | 32 | frame header, 64 bytes | the stamped header, encoded exactly as on the wire ([D4](#d4-slot-size-and-inline-budget-control-region-parameters-not-wire-fields)) | + | 96 | reserved, 32 bytes | zero | + + The clock is read once per pump, at its first stamp, and once for each event stamped outside a + pump: a pump takes microseconds, `seq` orders the records within it exactly, and a clock read per + event would cost more than the rest of the record. + + The frame header carries `seq`, kind, pattern and placement flags, `principal_ref`, + `src_handle`, `correlation_id`, `payload_len` and, for a host-generated control event, its + destination. **No payload byte is ever in a record.** A gap record's frame header is all zero. +- **The region.** One `memfd`, created by `ouro_host_telemetry_enable`, which outlives every + consumer: + + | Offset | Field | Written by | + |---|---|---| + | 0 | `magic` u32 `"TLMY"` (0x594d4c54) | host, last, with release order | + | 4 | `version` u32 = 1 | host | + | 8 | `record_size` u32 = 128 | host | + | 12 | `flags` u32 = 0 | host | + | 16 | `host_epoch` u64 | host | + | 24 | `dropped` u64 | host: every record ever lost | + | 64 | `consumer_waiting` u32 | consumer only ([D13](#d13-doorbells-idle-ends-sleep-in-the-kernel)) | + | 128 | an `ouro_ring` of 128-byte slots | head: host; tail: consumer | + + Geometry is validated once and copied, as for every ring. The host reads only `tail` and + `consumer_waiting` from shared memory after creation. +- **The hand-off.** The host writes the record, then publishes it with a release store of `head`. It + rings the telemetry doorbell under the D13 protocol (sequentially consistent fence, then a load of + `consumer_waiting`), but only once per `ouro_host_pump` and only if the consumer is asleep. The + host keeps its own copy of `head` and of the last `tail` it read, and reads `tail` across cores + only when that copy says the ring is full. Measured with `c/bench/dispatch.c`, the whole hand-off + adds about 13 ns to an event's 87 ns dispatch. An + event stamped outside a pump, such as the `producer_reaped` a reap emits, rings it at once. A busy + host whose exporter keeps up makes no system calls for telemetry at all. +- **Never blocks.** If the ring is full when an event is stamped, the host counts the record as + lost (`dropped` and a pending gap count) and goes on. The next record that fits is preceded by a gap + record carrying the count, so the exported stream says how much is missing and where, just as + `overflow` does for a consumer ([D6](#d6-the-tap-records-the-intended-stream-sheds-are-events)). + Telemetry loss never sheds, delays or reorders the events themselves. +- **Who consumes.** A client sets `OURO_ATTACH_CAP_TELEMETRY` (bit 2) with the doorbell bit. The host + grants it only when telemetry is enabled, the embedding granted that slot with + `ouro_host_grant_telemetry` from its attach hook, the doorbell was granted, and no other live slot + holds the grant. The grant adds two descriptors after the five of D13: the telemetry region, then + the telemetry doorbell. When the consumer's slot is released, the next granted consumer continues + from the same `tail`, so records written while no exporter was attached are not lost unless the ring + filled. +- **Export.** The ring ships the exporter as a separate process, not a library a host links: the + reference exporter reads every available record, sends them in one request, and advances `tail` + only when the collector has accepted them. It has no batching timer: a record waits only for the + request in flight ahead of it. Delivery is at least once, and `seq` identifies a duplicate. The + exporter's only timer is its retry backoff while the collector refuses or is unreachable, and + while it waits the ring fills and the host counts what it cannot hold. + +**What it does not change.** The tap callback ([D9](#d9-the-tap-in-this-tree-is-a-callback-plus-a-recording-test-double)) +is still the single full-stream observer of payloads, and capture ([D10](#d10-the-v0-capture-stream-format)) +is unchanged. The telemetry record is written before the tap is called, so a slow tap cannot delay +it. + +**Threat model ([D2](#d2-threat-model)).** The record is metadata, but metadata about every principal +on the ring, so the grant is the embedding's decision and is never implied by admission. The +consumer maps the region read-write, because it owns `tail` and `consumer_waiting`. Scribbling on +either can only make the host see the ring as full (records are counted lost) or skip a wakeup; both +are masked by the local geometry and cannot move where the host writes. A consumer cannot see +payloads, other clients' queues, or the arena through this grant. + +--- + ## Revision history - **r1** (2026-07-23), seven decisions presented as recommendations for the roundtable to converge. From a5f4208a13020eddbd15a28db9fff7a98f4c0bca Mon Sep 17 00:00:00 2001 From: jbailes Date: Fri, 30 Oct 2026 21:36:37 +0100 Subject: [PATCH 2/7] D16: the Go client consumes telemetry; every implementation holds the layout The layout table gains the D16 rows (the capability bit, the two descriptors, the region header and the record), and the C, Go and Rust layout tests hold them in both directions. The Go client asks for the grant through AttachTelemetry and reads the ring with a TelemetryReader under the D13 doorbell protocol. The conformance host turns telemetry on and grants one principal, and TestTelemetryConformance proves the Go reader against it: an ungranted principal gets no region, and the granted one reads its own notification and the C side's ack, in seq order, with no payload byte in either. Rust declares the layout and does not consume it; the reference exporter is Go. --- c/conformance/host.c | 15 ++- c/include/ouroboros/telemetry.h | 2 + c/tests/test_layout.c | 26 ++++ go/ring/client.go | 114 ++++++++++++----- go/ring/layout_test.go | 29 ++++- go/ring/telemetry.go | 213 ++++++++++++++++++++++++++++++++ go/ring/telemetry_test.go | 164 ++++++++++++++++++++++++ rust/src/layout.rs | 97 +++++++++++++++ vectors/gen_layout_vectors.py | 22 ++++ vectors/layout_vectors.tsv | 24 ++++ 10 files changed, 671 insertions(+), 35 deletions(-) create mode 100644 go/ring/telemetry.go create mode 100644 go/ring/telemetry_test.go diff --git a/c/conformance/host.c b/c/conformance/host.c index fb3f034..505f70e 100644 --- a/c/conformance/host.c +++ b/c/conformance/host.c @@ -32,6 +32,9 @@ * process liveness; the client's slot is never stale-reaped, so the next * client getting the freed slot proves the departing one announced its * detach. + * - D16: telemetry is on, and an external client attaching as principal_ref + * CONFORMANCE_TELEMETRY_PRINCIPAL is granted the consumer end, so it reads + * the record of every event it caused, its own and the C side's answers. * * Usage: ouroboros-ring-conformance-host [timeout-ms] * It listens, accepts exactly one external client, pumps and services until the @@ -166,12 +169,16 @@ static ouro_attach_status_t admit_peer(void *ctx, int fd, const ouro_attach_requ return OURO_ATTACH_OK; } +#define CONFORMANCE_TELEMETRY_PRINCIPAL 0x7E1Eu + static ouro_attach_status_t bind_peer(void *ctx, ouro_host_t *h, uint32_t slot, const ouro_attach_request_t *request) { - (void)ctx, (void)request; + (void)ctx; if (g_peer_known) (void)ouro_host_bind_process(h, slot); + if (request->principal_ref == CONFORMANCE_TELEMETRY_PRINCIPAL) + (void)ouro_host_grant_telemetry(h, slot); return OURO_ATTACH_OK; } @@ -208,6 +215,12 @@ int main(int argc, char **argv) return 1; } + if (ouro_host_telemetry_enable(&h, 256) != OURO_HOST_OK) + { + fprintf(stderr, "telemetry enable failed\n"); + return 1; + } + /* Internal C client: subscribes to KIND_NOTIFY and serves KIND_ECHO. */ ouro_client_t cc; if (attach_internal(&h, &cc) != 0) diff --git a/c/include/ouroboros/telemetry.h b/c/include/ouroboros/telemetry.h index 1cd2e0f..5f7de03 100644 --- a/c/include/ouroboros/telemetry.h +++ b/c/include/ouroboros/telemetry.h @@ -31,7 +31,9 @@ #define OURO_TELEMETRY_OFF_FLAGS 4 #define OURO_TELEMETRY_OFF_REALTIME_NS 8 #define OURO_TELEMETRY_OFF_LOST 16 +#define OURO_TELEMETRY_OFF_RESERVED 24 #define OURO_TELEMETRY_OFF_FRAME 32 /* OURO_WIRE_HDR_LEN bytes, wire encoding */ +#define OURO_TELEMETRY_OFF_RESERVED_TAIL 96 #define OURO_TELEMETRY_RECORD_EVENT 0u #define OURO_TELEMETRY_RECORD_GAP 1u diff --git a/c/tests/test_layout.c b/c/tests/test_layout.c index 851116b..f37610b 100644 --- a/c/tests/test_layout.c +++ b/c/tests/test_layout.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -212,6 +213,7 @@ int main(void) E("attach_request.v0_size", OURO_ATTACH_REQ_V0_BYTES), E("attach.cap.doorbell", OURO_ATTACH_CAP_DOORBELL), E("attach.cap.process_liveness", OURO_ATTACH_CAP_PROCESS_LIVENESS), + E("attach.cap.telemetry", OURO_ATTACH_CAP_TELEMETRY), E("attach_reply.magic", OURO_ATTACH_REPLY_MAGIC), E("attach_reply.size", sizeof(ouro_attach_reply_t)), E("attach_reply.off.magic", offsetof(ouro_attach_reply_t, magic)), @@ -236,6 +238,30 @@ int main(void) E("attach.descriptors_doorbell", OURO_ATTACH_FDS_DOORBELL), E("attach.fd.host_doorbell", OURO_ATTACH_FD_HOST_DOORBELL), E("attach.fd.client_doorbell", OURO_ATTACH_FD_CLIENT_DOORBELL), + E("attach.descriptors_telemetry", OURO_ATTACH_FDS_TELEMETRY), + E("attach.fd.telemetry", OURO_ATTACH_FD_TELEMETRY), + E("attach.fd.telemetry_doorbell", OURO_ATTACH_FD_TELEMETRY_BELL), + + E("telemetry.magic", OURO_TELEMETRY_MAGIC), + E("telemetry.version", OURO_TELEMETRY_VERSION), + E("telemetry.off.magic", offsetof(ouro_telemetry_region_t, magic)), + E("telemetry.off.version", offsetof(ouro_telemetry_region_t, version)), + E("telemetry.off.record_size", offsetof(ouro_telemetry_region_t, record_size)), + E("telemetry.off.flags", offsetof(ouro_telemetry_region_t, flags)), + E("telemetry.off.host_epoch", offsetof(ouro_telemetry_region_t, host_epoch)), + E("telemetry.off.dropped", offsetof(ouro_telemetry_region_t, dropped)), + E("telemetry.off.consumer_waiting", offsetof(ouro_telemetry_region_t, consumer_waiting)), + E("telemetry.off.ring", OURO_TELEMETRY_REGION_OFF_RING), + E("telemetry_record.off.type", OURO_TELEMETRY_OFF_TYPE), + E("telemetry_record.off.flags", OURO_TELEMETRY_OFF_FLAGS), + E("telemetry_record.off.realtime_ns", OURO_TELEMETRY_OFF_REALTIME_NS), + E("telemetry_record.off.lost", OURO_TELEMETRY_OFF_LOST), + E("telemetry_record.off.reserved", OURO_TELEMETRY_OFF_RESERVED), + E("telemetry_record.off.frame", OURO_TELEMETRY_OFF_FRAME), + E("telemetry_record.off.reserved_tail", OURO_TELEMETRY_OFF_RESERVED_TAIL), + E("telemetry_record.size", OURO_TELEMETRY_RECORD_LEN), + E("telemetry_record.type.event", OURO_TELEMETRY_RECORD_EVENT), + E("telemetry_record.type.gap", OURO_TELEMETRY_RECORD_GAP), E("lease_release.size", OURO_LEASE_RELEASE_LEN), E("lease_release.off.lease_id", OURO_LEASE_RELEASE_OFF_LEASE), diff --git a/go/ring/client.go b/go/ring/client.go index fb0e5c5..6ec0408 100644 --- a/go/ring/client.go +++ b/go/ring/client.go @@ -39,19 +39,25 @@ const ( // A successful attach carries exactly these descriptors, in this order. A // doorbell grant (D13) adds the host doorbell and the client's own. - attachDescriptors = 3 - attachDescriptorsDoorbell = 5 - attachFdControl = 0 - attachFdArena = 1 - attachFdQpair = 2 - attachFdHostDoorbell = 3 - attachFdClientDoorbell = 4 + attachDescriptors = 3 + attachDescriptorsDoorbell = 5 + attachDescriptorsTelemetry = 7 // D16: + the telemetry region and its doorbell + attachFdControl = 0 + attachFdArena = 1 + attachFdQpair = 2 + attachFdHostDoorbell = 3 + attachFdClientDoorbell = 4 + attachFdTelemetry = 5 + attachFdTelemetryBell = 6 // attachCapDoorbell asks for, and grants, a doorbell pair (D13). attachCapDoorbell uint32 = 0x1 // attachCapProcessLiveness (D15): the host holds a handle on this // process and releases the slot when it exits, so no timer heartbeat. attachCapProcessLiveness uint32 = 0x2 + // attachCapTelemetry (D16): the consumer end of the host's telemetry + // ring, granted with the doorbell to the one slot the embedding chose. + attachCapTelemetry uint32 = 0x4 // heartbeatDetached is the client_heartbeat a detaching client leaves // (D15): the host releases the slot at its next pump. @@ -187,6 +193,27 @@ func Attach(sock int) (*Client, error) { // ref as claims only: its admission policy also binds them to SO_PEERCRED and // the canonical executable path before granting event capabilities. func AttachAs(sock int, principalClass uint32, principalRef uint32) (*Client, error) { + c, _, err := attach(sock, principalClass, principalRef, false) + return c, err +} + +// AttachTelemetry attaches as AttachAs does and also asks for the D16 +// telemetry grant. The reader is nil when the host did not grant it; the client +// is attached either way. +func AttachTelemetry(sock int, principalClass uint32, principalRef uint32) (*Client, *TelemetryReader, error) { + c, tfds, err := attach(sock, principalClass, principalRef, true) + if err != nil || tfds == nil { + return c, nil, err + } + r, err := openTelemetryReader(tfds[0], tfds[1]) + if err != nil { + c.Detach() + return nil, nil, err + } + return c, r, nil +} + +func attach(sock int, principalClass uint32, principalRef uint32, telemetry bool) (*Client, []int, error) { var req [attachReqBytes]byte binary.LittleEndian.PutUint32(req[attachReqOffMagic:], attachReqMagic) binary.LittleEndian.PutUint16(req[attachReqOffWireMin:], WireVersion) @@ -194,17 +221,20 @@ func AttachAs(sock int, principalClass uint32, principalRef uint32) (*Client, er binary.LittleEndian.PutUint32(req[attachReqOffPrincipalClass:], principalClass) binary.LittleEndian.PutUint32(req[attachReqOffPrincipalRef:], principalRef) // D13/D15: an older host reads only the first 16 bytes and grants neither. - binary.LittleEndian.PutUint32(req[attachReqOffCapabilities:], - attachCapDoorbell|attachCapProcessLiveness) + asked := attachCapDoorbell | attachCapProcessLiveness + if telemetry { + asked |= attachCapTelemetry + } + binary.LittleEndian.PutUint32(req[attachReqOffCapabilities:], asked) if err := unix.Sendmsg(sock, req[:], nil, nil, 0); err != nil { - return nil, err + return nil, nil, err } replyBuf := make([]byte, attachReplyBytes) - oob := make([]byte, unix.CmsgSpace(attachDescriptorsDoorbell*4)) + oob := make([]byte, unix.CmsgSpace(attachDescriptorsTelemetry*4)) n, oobn, _, _, err := unix.Recvmsg(sock, replyBuf, oob, unix.MSG_CMSG_CLOEXEC) if err != nil { - return nil, err + return nil, nil, err } fds := parseRights(oob[:oobn]) closeAll := func() { @@ -214,24 +244,28 @@ func AttachAs(sock int, principalClass uint32, principalRef uint32) (*Client, er } if n != attachReplyBytes || binary.LittleEndian.Uint32(replyBuf[attachReplyOffMagic:]) != attachReplyMagic { closeAll() - return nil, ErrProtocol + return nil, nil, ErrProtocol } status := AttachStatus(binary.LittleEndian.Uint32(replyBuf[attachReplyOffStatus:])) if status != AttachOK { closeAll() - return &Client{Status: status}, fmt.Errorf("%w: %s", ErrDenied, status) + return &Client{Status: status}, nil, fmt.Errorf("%w: %s", ErrDenied, status) } // The grant's capabilities and descriptor count must agree. A host grants // only what was asked for, so any other bit is a protocol error. caps := binary.LittleEndian.Uint32(replyBuf[attachReplyOffCapabilities:]) doorbell := caps&attachCapDoorbell != 0 + granted := caps&attachCapTelemetry != 0 wantFds := attachDescriptors - if doorbell { + switch { + case granted: + wantFds = attachDescriptorsTelemetry + case doorbell: wantFds = attachDescriptorsDoorbell } - if caps&^(attachCapDoorbell|attachCapProcessLiveness) != 0 || len(fds) != wantFds { + if caps&^asked != 0 || (granted && !doorbell) || len(fds) != wantFds { closeAll() - return nil, ErrProtocol + return nil, nil, ErrProtocol } c := &Client{ @@ -249,7 +283,7 @@ func AttachAs(sock int, principalClass uint32, principalRef uint32) (*Client, er c.slotSize <= HdrLen || c.inlineBudget > c.slotSize || c.arenaSize == 0 || c.arenaSize > maxRegionBytes { closeAll() - return nil, ErrProtocol + return nil, nil, ErrProtocol } // fds[0] control (read-only), fds[1] arena (rw), fds[2] queue pair (rw). @@ -260,7 +294,7 @@ func AttachAs(sock int, principalClass uint32, principalRef uint32) (*Client, er ctlSize, err := fdSize(fds[attachFdControl]) if err != nil { closeAll() - return nil, err + return nil, nil, err } ctlLen := ctlBytes if ctlSize >= ctlBytesWithView { @@ -275,35 +309,45 @@ func AttachAs(sock int, principalClass uint32, principalRef uint32) (*Client, er have, err := fdSize(need.fd) if err != nil { closeAll() - return nil, err + return nil, nil, err } if have < int64(need.len) { closeAll() - return nil, ErrRegion + return nil, nil, ErrRegion } } ctlMem, err := unix.Mmap(fds[attachFdControl], 0, ctlLen, unix.PROT_READ, unix.MAP_SHARED) if err != nil { closeAll() - return nil, err + return nil, nil, err } arena, err := unix.Mmap(fds[attachFdArena], 0, arenaBytes, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED) if err != nil { unix.Munmap(ctlMem) closeAll() - return nil, err + return nil, nil, err } qpMem, err := unix.Mmap(fds[attachFdQpair], 0, qpBytesLen, unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED) if err != nil { unix.Munmap(ctlMem) unix.Munmap(arena) closeAll() - return nil, err + return nil, nil, err } - // The mappings hold their own references; the doorbells stay open. + // The mappings hold their own references; the doorbells stay open, and + // the telemetry pair goes to the caller. for _, fd := range fds[:attachDescriptors] { unix.Close(fd) } + var tfds []int + if granted { + tfds = fds[attachFdTelemetry:attachDescriptorsTelemetry] + } + closeTelemetry := func() { + for _, fd := range tfds { + unix.Close(fd) + } + } c.hostDoorbell, c.doorbell = -1, -1 if doorbell { c.haveDoorbell = true @@ -313,32 +357,38 @@ func AttachAs(sock int, principalClass uint32, principalRef uint32) (*Client, er c.ctlMem, c.arena, c.qpMem = ctlMem, arena, qpMem if c.control, err = AttachControl(ctlMem); err != nil { c.Detach() - return nil, err + closeTelemetry() + return nil, nil, err } if c.control.SlotSize() != c.slotSize || c.control.InlineBudget() != c.inlineBudget || c.control.QueueCapacity() != c.queueCapacity || c.control.ArenaSize() != c.arenaSize { c.Detach() - return nil, ErrRegion + closeTelemetry() + return nil, nil, ErrRegion } if c.arenaUsable, err = attachArena(arena, c.arenaSize); err != nil { c.Detach() - return nil, err + closeTelemetry() + return nil, nil, err } if c.qp, err = AttachQueuePair(qpMem); err != nil { c.Detach() - return nil, err + closeTelemetry() + return nil, nil, err } if c.qp.Inbound.SlotSize() != c.slotSize || c.qp.Inbound.Capacity() != c.queueCapacity || c.qp.Outbound.SlotSize() != c.slotSize || c.qp.Outbound.Capacity() != c.queueCapacity { c.Detach() - return nil, ErrRegion + closeTelemetry() + return nil, nil, ErrRegion } if c.haveDoorbell && c.control.flags()&ctlFlagDoorbell == 0 { c.Detach() - return nil, ErrProtocol + closeTelemetry() + return nil, nil, ErrProtocol } c.leaseView = c.control.leaseView() - return c, nil + return c, tfds, nil } // ArenaReadable reports whether this attachment can read arena-placed diff --git a/go/ring/layout_test.go b/go/ring/layout_test.go index 218a420..edd2354 100644 --- a/go/ring/layout_test.go +++ b/go/ring/layout_test.go @@ -106,6 +106,7 @@ func goLayout() map[string]uint64 { "attach_request.v0_size": attachReqV0Bytes, "attach.cap.doorbell": uint64(attachCapDoorbell), "attach.cap.process_liveness": uint64(attachCapProcessLiveness), + "attach.cap.telemetry": uint64(attachCapTelemetry), "attach_reply.magic": uint64(attachReplyMagic), "attach_reply.size": attachReplyBytes, "attach_reply.off.magic": attachReplyOffMagic, @@ -128,8 +129,32 @@ func goLayout() map[string]uint64 { "attach.fd.arena": attachFdArena, "attach.fd.qpair": attachFdQpair, "attach.descriptors_doorbell": attachDescriptorsDoorbell, - "attach.fd.host_doorbell": attachFdHostDoorbell, - "attach.fd.client_doorbell": attachFdClientDoorbell, + "attach.descriptors_telemetry": attachDescriptorsTelemetry, + "attach.fd.telemetry": attachFdTelemetry, + "attach.fd.telemetry_doorbell": attachFdTelemetryBell, + + "telemetry.magic": uint64(telemetryMagic), + "telemetry.version": uint64(telemetryVersion), + "telemetry.off.magic": telemetryRegionOffMagic, + "telemetry.off.version": telemetryRegionOffVersion, + "telemetry.off.record_size": telemetryRegionOffRecordSize, + "telemetry.off.flags": telemetryRegionOffFlags, + "telemetry.off.host_epoch": telemetryRegionOffHostEpoch, + "telemetry.off.dropped": telemetryRegionOffDropped, + "telemetry.off.consumer_waiting": telemetryRegionOffWaiting, + "telemetry.off.ring": telemetryRegionOffRing, + "telemetry_record.off.type": telemetryOffType, + "telemetry_record.off.flags": telemetryOffFlags, + "telemetry_record.off.realtime_ns": telemetryOffRealtimeNS, + "telemetry_record.off.lost": telemetryOffLost, + "telemetry_record.off.reserved": telemetryOffReserved, + "telemetry_record.off.frame": telemetryOffFrame, + "telemetry_record.off.reserved_tail": telemetryOffReservedTail, + "telemetry_record.size": TelemetryRecordLen, + "telemetry_record.type.event": uint64(TelemetryEvent), + "telemetry_record.type.gap": uint64(TelemetryGap), + "attach.fd.host_doorbell": attachFdHostDoorbell, + "attach.fd.client_doorbell": attachFdClientDoorbell, "lease_release.size": LeaseReleaseLen, "lease_release.off.lease_id": leaseReleaseOffLease, diff --git a/go/ring/telemetry.go b/go/ring/telemetry.go new file mode 100644 index 0000000..991d5a2 --- /dev/null +++ b/go/ring/telemetry.go @@ -0,0 +1,213 @@ +package ring + +import ( + "encoding/binary" + "errors" + "sync/atomic" + "time" + "unsafe" + + "golang.org/x/sys/unix" +) + +// D16: the host writes one content-free record per seq-stamped event into a +// single-producer ring in its own region, and one telemetry process holds the +// consumer end. These mirror c/include/ouroboros/telemetry.h. +const ( + telemetryMagic uint32 = 0x594d4c54 // "TLMY" + telemetryVersion uint32 = 1 + + // TelemetryRecordLen is the size of every record. + TelemetryRecordLen = 128 + + telemetryOffType = 0 + telemetryOffFlags = 4 + telemetryOffRealtimeNS = 8 + telemetryOffLost = 16 + telemetryOffReserved = 24 + telemetryOffFrame = 32 + telemetryOffReservedTail = 96 + + telemetryRegionOffMagic = 0 + telemetryRegionOffVersion = 4 + telemetryRegionOffRecordSize = 8 + telemetryRegionOffFlags = 12 + telemetryRegionOffHostEpoch = 16 + telemetryRegionOffDropped = 24 + telemetryRegionOffWaiting = 64 + telemetryRegionOffRing = 128 +) + +// TelemetryRecordType is a record's type field. +type TelemetryRecordType uint32 + +const ( + // TelemetryEvent describes one seq-stamped event. + TelemetryEvent TelemetryRecordType = 0 + // TelemetryGap says Lost records were dropped just before this one. + TelemetryGap TelemetryRecordType = 1 +) + +// ErrTelemetryRecord is returned for a record that disagrees with D16. +var ErrTelemetryRecord = errors.New("ring: malformed telemetry record") + +// TelemetryRecord is a decoded record. For a gap, Frame is zero and Lost is +// the count. +type TelemetryRecord struct { + Type TelemetryRecordType + RealtimeNS uint64 + Lost uint64 + Frame Frame +} + +// DecodeTelemetryRecord decodes one TelemetryRecordLen-byte record. +func DecodeTelemetryRecord(b []byte) (TelemetryRecord, error) { + var r TelemetryRecord + if len(b) < TelemetryRecordLen { + return r, ErrTelemetryRecord + } + r.Type = TelemetryRecordType(binary.LittleEndian.Uint32(b[telemetryOffType:])) + r.RealtimeNS = binary.LittleEndian.Uint64(b[telemetryOffRealtimeNS:]) + r.Lost = binary.LittleEndian.Uint64(b[telemetryOffLost:]) + if binary.LittleEndian.Uint32(b[telemetryOffFlags:]) != 0 { + return r, ErrTelemetryRecord + } + switch r.Type { + case TelemetryGap: + if r.Lost == 0 { + return r, ErrTelemetryRecord + } + return r, nil + case TelemetryEvent: + if r.Lost != 0 || Decode(b[telemetryOffFrame:telemetryOffFrame+HdrLen], &r.Frame) != OK { + return r, ErrTelemetryRecord + } + return r, nil + } + return r, ErrTelemetryRecord +} + +// TelemetryReader is the consumer end of the host's telemetry ring. One +// goroutine at a time, like Poll. +type TelemetryReader struct { + mem []byte + ring *Ring + waiting *uint32 + dropped *uint64 + bell int +} + +func openTelemetryReader(regionFd, bellFd int) (*TelemetryReader, error) { + fail := func(err error) (*TelemetryReader, error) { + unix.Close(regionFd) + unix.Close(bellFd) + return nil, err + } + size, err := fdSize(regionFd) + if err != nil { + return fail(err) + } + if size <= telemetryRegionOffRing || size > maxRegionBytes { + return fail(ErrRegion) + } + mem, err := unix.Mmap(regionFd, 0, int(size), unix.PROT_READ|unix.PROT_WRITE, unix.MAP_SHARED) + if err != nil { + return fail(err) + } + unix.Close(regionFd) // the mapping holds its own reference + // Validate once; afterwards only head, tail and the waiting flag are read. + if atomic.LoadUint32(u32ptr(mem, telemetryRegionOffMagic)) != telemetryMagic || + atomic.LoadUint32(u32ptr(mem, telemetryRegionOffVersion)) != telemetryVersion || + atomic.LoadUint32(u32ptr(mem, telemetryRegionOffRecordSize)) != TelemetryRecordLen { + unix.Munmap(mem) + unix.Close(bellFd) + return nil, ErrRegion + } + r, err := AttachRing(mem[telemetryRegionOffRing:]) + if err != nil || r.SlotSize() != TelemetryRecordLen { + unix.Munmap(mem) + unix.Close(bellFd) + if err == nil { + err = ErrRegion + } + return nil, err + } + return &TelemetryReader{ + mem: mem, ring: r, bell: bellFd, + waiting: u32ptr(mem, telemetryRegionOffWaiting), + dropped: u64ptr(mem, telemetryRegionOffDropped), + }, nil +} + +func u32ptr(b []byte, off int) *uint32 { return (*uint32)(unsafe.Pointer(&b[off])) } + +// Available reports the records written and not yet released. +func (t *TelemetryReader) Available() uint64 { + n := t.ring.Count() + if n > uint64(t.ring.Capacity()) { + return 0 // a tail the host never wrote is not a ring state + } + return n +} + +// Peek returns the i-th unreleased record's bytes, valid until Release. +func (t *TelemetryReader) Peek(i uint64) []byte { + if i >= t.Available() { + return nil + } + return t.ring.slot(atomic.LoadUint64(t.ring.tail) + i) +} + +// Release hands the oldest n records back to the host once they are exported. +func (t *TelemetryReader) Release(n uint64) { + if avail := t.Available(); n > avail { + n = avail + } + if n > 0 { + atomic.StoreUint64(t.ring.tail, atomic.LoadUint64(t.ring.tail)+n) + } +} + +// Dropped is every record the host has lost since the region was created. +func (t *TelemetryReader) Dropped() uint64 { return atomic.LoadUint64(t.dropped) } + +// Wait sleeps until a record is available or timeout passes (negative waits +// indefinitely), under the D13 protocol. It reports whether records are +// available. +func (t *TelemetryReader) Wait(timeout time.Duration) (bool, error) { + atomic.StoreUint32(t.waiting, 1) + defer atomic.StoreUint32(t.waiting, 0) + if t.Available() > 0 { + return true, nil + } + ms := -1 + if timeout >= 0 { + ms = int((timeout + time.Millisecond - 1) / time.Millisecond) + } + fds := []unix.PollFd{{Fd: int32(t.bell), Events: unix.POLLIN}} + n, err := unix.Poll(fds, ms) + if err != nil && err != unix.EINTR { + return false, err + } + if n > 0 { + var count [8]byte + _, _ = unix.Read(t.bell, count[:]) + } + return t.Available() > 0, nil +} + +// Interrupt wakes Wait from another goroutine. +func (t *TelemetryReader) Interrupt() { ringDoorbell(t.bell) } + +// Close unmaps the region. The host keeps it, and a later consumer resumes at +// the same tail. +func (t *TelemetryReader) Close() { + if t.mem != nil { + unix.Munmap(t.mem) + t.mem = nil + } + if t.bell >= 0 { + unix.Close(t.bell) + t.bell = -1 + } +} diff --git a/go/ring/telemetry_test.go b/go/ring/telemetry_test.go new file mode 100644 index 0000000..f54be07 --- /dev/null +++ b/go/ring/telemetry_test.go @@ -0,0 +1,164 @@ +package ring + +import ( + "bytes" + "encoding/binary" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + "golang.org/x/sys/unix" +) + +// A record built byte by byte from the layout table decodes, and one with an +// unknown flag, a zero-count gap or a bad frame is refused. +func TestTelemetryRecordDecode(t *testing.T) { + f := Frame{HdrFlags: FNotification | FInline, WireVersion: WireVersion, EventKind: 0x2a01, + PrincipalRef: 9, Seq: 41, PayloadLen: 5, PayloadRef: HdrLen, SrcHandle: 2} + rec := make([]byte, TelemetryRecordLen) + binary.LittleEndian.PutUint32(rec[telemetryOffType:], uint32(TelemetryEvent)) + binary.LittleEndian.PutUint64(rec[telemetryOffRealtimeNS:], 1234) + if f.Encode(rec[telemetryOffFrame:]) != HdrLen { + t.Fatal("encode") + } + got, err := DecodeTelemetryRecord(rec) + if err != nil || got.Type != TelemetryEvent || got.RealtimeNS != 1234 || got.Frame.Seq != 41 || + got.Frame.EventKind != 0x2a01 || got.Frame.SrcHandle != 2 || got.Frame.PayloadLen != 5 { + t.Fatalf("decoded %+v, %v", got, err) + } + bad := bytes.Clone(rec) + bad[telemetryOffFlags] = 1 + if _, err := DecodeTelemetryRecord(bad); err == nil { + t.Error("an unknown flag was accepted") + } + gap := make([]byte, TelemetryRecordLen) + binary.LittleEndian.PutUint32(gap[telemetryOffType:], uint32(TelemetryGap)) + if _, err := DecodeTelemetryRecord(gap); err == nil { + t.Error("a gap of zero was accepted") + } + binary.LittleEndian.PutUint64(gap[telemetryOffLost:], 3) + if g, err := DecodeTelemetryRecord(gap); err != nil || g.Lost != 3 { + t.Errorf("gap: %+v, %v", g, err) + } +} + +// The Go client, granted the consumer end by the C host, reads the record of +// every event it caused: its own notification, and the C side's ack, in seq +// order, with no payload byte in either. +func TestTelemetryConformance(t *testing.T) { + harness := os.Getenv("OURO_CONFORMANCE_HOST") + if harness == "" { + t.Skip("OURO_CONFORMANCE_HOST not set; see README.md, Building and testing") + } + sock := filepath.Join(t.TempDir(), "bus.sock") + cmd := exec.Command(harness, sock, "10000") + var out bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &out + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + if t.Failed() { + t.Logf("harness output:\n%s", out.String()) + } + }() + for deadline := time.Now().Add(5 * time.Second); ; time.Sleep(10 * time.Millisecond) { + if _, err := os.Stat(sock); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("harness socket did not appear") + } + } + + attachAs := func(ref uint32) (*Client, *TelemetryReader, int) { + fd, err := unix.Socket(unix.AF_UNIX, unix.SOCK_SEQPACKET, 0) + if err != nil { + t.Fatal(err) + } + if err := unix.Connect(fd, &unix.SockaddrUnix{Name: sock}); err != nil { + t.Fatal(err) + } + c, r, err := AttachTelemetry(fd, 1, ref) + if err != nil { + t.Fatalf("attach as %#x: %v", ref, err) + } + return c, r, fd + } + + // A principal the host did not grant is attached without the consumer end. + c, r, fd := attachAs(0x1234) + if r != nil { + t.Fatal("an ungranted principal was handed the telemetry region") + } + c.Detach() + unix.Close(fd) + time.Sleep(50 * time.Millisecond) // the host reaps it and accepts the next + + c, r, fd = attachAs(0x7E1E) + defer unix.Close(fd) + if r == nil { + t.Fatal("the granted principal was not handed the telemetry region") + } + defer r.Close() + defer c.Detach() + // Whatever the earlier attach caused is not this test's. + r.Release(r.Available()) + + payload := []byte("payload-never-exported") + if err := c.Publish(1000, payload); err != nil { // KIND_NOTIFY + t.Fatal(err) + } + var notify, ack *TelemetryRecord + deadline := time.Now().Add(5 * time.Second) + var lastSeq uint64 + for (notify == nil || ack == nil) && time.Now().Before(deadline) { + if ok, err := r.Wait(time.Second); err != nil { + t.Fatal(err) + } else if !ok { + continue + } + n := r.Available() + for i := uint64(0); i < n; i++ { + raw := r.Peek(i) + if bytes.Contains(raw, []byte("payload")) { + t.Fatal("a payload byte reached a telemetry record") + } + rec, err := DecodeTelemetryRecord(raw) + if err != nil { + t.Fatalf("record %d: %v", i, err) + } + if rec.Type != TelemetryEvent { + continue + } + if rec.Frame.Seq <= lastSeq { + t.Fatalf("seq %d after %d", rec.Frame.Seq, lastSeq) + } + lastSeq = rec.Frame.Seq + switch rec.Frame.EventKind { + case 1000: + notify = &rec + case 1001: + ack = &rec + } + } + r.Release(n) + } + if notify == nil || ack == nil { + t.Fatalf("records: notify %v ack %v", notify != nil, ack != nil) + } + if notify.Frame.SrcHandle != c.Handle || notify.Frame.PayloadLen != uint32(len(payload)) || + notify.RealtimeNS == 0 { + t.Errorf("notify record %+v", notify) + } + if ack.Frame.Seq <= notify.Frame.Seq { + t.Errorf("the ack (seq %d) is recorded after the notify (seq %d)", ack.Frame.Seq, notify.Frame.Seq) + } + if r.Dropped() != 0 { + t.Errorf("dropped %d", r.Dropped()) + } +} diff --git a/rust/src/layout.rs b/rust/src/layout.rs index c42b077..cf0418f 100644 --- a/rust/src/layout.rs +++ b/rust/src/layout.rs @@ -165,6 +165,40 @@ pub(crate) const ATTACH_CAP_PROCESS_LIVENESS: u32 = 0x2; /// D15: the heartbeat a detaching client leaves; the host releases the slot. pub(crate) const HEARTBEAT_DETACHED: u64 = u64::MAX; +/// D16: the telemetry consumer end. The Rust client does not consume it (the +/// reference exporter is Go); the layout is declared so the table is held in +/// full. +#[cfg_attr( + not(test), + expect(dead_code, reason = "D16: not a telemetry consumer; held to the table") +)] +pub(crate) mod telemetry { + pub(crate) const ATTACH_CAP_TELEMETRY: u32 = 0x4; + pub(crate) const ATTACH_FDS_TELEMETRY: usize = 7; + pub(crate) const ATTACH_FD_TELEMETRY: usize = 5; + pub(crate) const ATTACH_FD_TELEMETRY_DOORBELL: usize = 6; + pub(crate) const MAGIC: u32 = 0x594d_4c54; // "TLMY" + pub(crate) const VERSION: u32 = 1; + pub(crate) const OFF_MAGIC: usize = 0; + pub(crate) const OFF_VERSION: usize = 4; + pub(crate) const OFF_RECORD_SIZE: usize = 8; + pub(crate) const OFF_FLAGS: usize = 12; + pub(crate) const OFF_HOST_EPOCH: usize = 16; + pub(crate) const OFF_DROPPED: usize = 24; + pub(crate) const OFF_CONSUMER_WAITING: usize = 64; + pub(crate) const OFF_RING: usize = 128; + pub(crate) const RECORD_LEN: usize = 128; + pub(crate) const RECORD_OFF_TYPE: usize = 0; + pub(crate) const RECORD_OFF_FLAGS: usize = 4; + pub(crate) const RECORD_OFF_REALTIME_NS: usize = 8; + pub(crate) const RECORD_OFF_LOST: usize = 16; + pub(crate) const RECORD_OFF_RESERVED: usize = 24; + pub(crate) const RECORD_OFF_FRAME: usize = 32; + pub(crate) const RECORD_OFF_RESERVED_TAIL: usize = 96; + pub(crate) const RECORD_EVENT: u32 = 0; + pub(crate) const RECORD_GAP: u32 = 1; +} + // ---- lease-release payload (D12) --------------------------------------------- pub(crate) const LEASE_RELEASE_OFF_LEASE: usize = 0; pub(crate) const LEASE_RELEASE_OFF_GENERATION: usize = 4; @@ -357,6 +391,69 @@ mod tests { "attach.cap.process_liveness", ATTACH_CAP_PROCESS_LIVENESS.into(), ), + ( + "attach.cap.telemetry", + telemetry::ATTACH_CAP_TELEMETRY.into(), + ), + ( + "attach.descriptors_telemetry", + telemetry::ATTACH_FDS_TELEMETRY as u64, + ), + ("attach.fd.telemetry", telemetry::ATTACH_FD_TELEMETRY as u64), + ( + "attach.fd.telemetry_doorbell", + telemetry::ATTACH_FD_TELEMETRY_DOORBELL as u64, + ), + ("telemetry.magic", telemetry::MAGIC.into()), + ("telemetry.version", telemetry::VERSION.into()), + ("telemetry.off.magic", telemetry::OFF_MAGIC as u64), + ("telemetry.off.version", telemetry::OFF_VERSION as u64), + ( + "telemetry.off.record_size", + telemetry::OFF_RECORD_SIZE as u64, + ), + ("telemetry.off.flags", telemetry::OFF_FLAGS as u64), + ("telemetry.off.host_epoch", telemetry::OFF_HOST_EPOCH as u64), + ("telemetry.off.dropped", telemetry::OFF_DROPPED as u64), + ( + "telemetry.off.consumer_waiting", + telemetry::OFF_CONSUMER_WAITING as u64, + ), + ("telemetry.off.ring", telemetry::OFF_RING as u64), + ("telemetry_record.size", telemetry::RECORD_LEN as u64), + ( + "telemetry_record.off.type", + telemetry::RECORD_OFF_TYPE as u64, + ), + ( + "telemetry_record.off.flags", + telemetry::RECORD_OFF_FLAGS as u64, + ), + ( + "telemetry_record.off.realtime_ns", + telemetry::RECORD_OFF_REALTIME_NS as u64, + ), + ( + "telemetry_record.off.lost", + telemetry::RECORD_OFF_LOST as u64, + ), + ( + "telemetry_record.off.reserved", + telemetry::RECORD_OFF_RESERVED as u64, + ), + ( + "telemetry_record.off.frame", + telemetry::RECORD_OFF_FRAME as u64, + ), + ( + "telemetry_record.off.reserved_tail", + telemetry::RECORD_OFF_RESERVED_TAIL as u64, + ), + ( + "telemetry_record.type.event", + telemetry::RECORD_EVENT.into(), + ), + ("telemetry_record.type.gap", telemetry::RECORD_GAP.into()), ("lease_release.size", LEASE_RELEASE_LEN as u64), ("lease_release.off.lease_id", LEASE_RELEASE_OFF_LEASE as u64), ( diff --git a/vectors/gen_layout_vectors.py b/vectors/gen_layout_vectors.py index 2d91d1c..0f2863d 100755 --- a/vectors/gen_layout_vectors.py +++ b/vectors/gen_layout_vectors.py @@ -122,6 +122,7 @@ def struct(prefix, size, fields): row("attach_request.v0_size", 16, "D13: a host accepts the 16-byte v0 request too") row("attach.cap.doorbell", 0x1, "D13: request and grant a doorbell pair") row("attach.cap.process_liveness", 0x2, "D15: the host releases the slot when the client process exits") +row("attach.cap.telemetry", 0x4, "D16: the consumer end of the host's telemetry ring") row("attach_reply.magic", le_magic("ABRP")) struct("attach_reply", 48, [ ("magic", 4), ("status", 4), ("handle_id", 4), ("wire_version", 4), @@ -138,6 +139,27 @@ def struct(prefix, size, fields): row("attach.descriptors_doorbell", 5, "D13: then host doorbell, client doorbell") row("attach.fd.host_doorbell", 3) row("attach.fd.client_doorbell", 4) +row("attach.descriptors_telemetry", 7, "D16: then the telemetry region and its doorbell") +row("attach.fd.telemetry", 5) +row("attach.fd.telemetry_doorbell", 6) + +# ---- telemetry region and record (D16) --------------------------------------- +row("telemetry.magic", le_magic("TLMY")) +row("telemetry.version", 1) +row("telemetry.off.magic", 0) +row("telemetry.off.version", 4) +row("telemetry.off.record_size", 8) +row("telemetry.off.flags", 12) +row("telemetry.off.host_epoch", 16) +row("telemetry.off.dropped", 24, "host-written: every record ever lost") +row("telemetry.off.consumer_waiting", 64, "consumer-written, its own cache line") +row("telemetry.off.ring", 128, "a ring of telemetry_record.size slots") +struct("telemetry_record", 128, [ + ("type", 4), ("flags", 4), ("realtime_ns", 8), ("lost", 8), ("reserved", 8), + ("frame", 64), ("reserved_tail", 32), +]) +row("telemetry_record.type.event", 0) +row("telemetry_record.type.gap", 1) # ---- lease release payload (D12) --------------------------------------------- struct("lease_release", 8, [("lease_id", 4), ("generation", 4)]) diff --git a/vectors/layout_vectors.tsv b/vectors/layout_vectors.tsv index dcc11bf..e4fe2be 100644 --- a/vectors/layout_vectors.tsv +++ b/vectors/layout_vectors.tsv @@ -84,6 +84,7 @@ attach_request.size 20 attach_request.v0_size 16 attach.cap.doorbell 1 attach.cap.process_liveness 2 +attach.cap.telemetry 4 attach_reply.magic 0x50524241 attach_reply.off.magic 0x00000000 attach_reply.off.status 4 @@ -108,6 +109,29 @@ attach.fd.qpair 2 attach.descriptors_doorbell 5 attach.fd.host_doorbell 3 attach.fd.client_doorbell 4 +attach.descriptors_telemetry 7 +attach.fd.telemetry 5 +attach.fd.telemetry_doorbell 6 +telemetry.magic 0x594d4c54 +telemetry.version 1 +telemetry.off.magic 0x00000000 +telemetry.off.version 4 +telemetry.off.record_size 8 +telemetry.off.flags 12 +telemetry.off.host_epoch 16 +telemetry.off.dropped 24 +telemetry.off.consumer_waiting 64 +telemetry.off.ring 128 +telemetry_record.off.type 0 +telemetry_record.off.flags 4 +telemetry_record.off.realtime_ns 8 +telemetry_record.off.lost 16 +telemetry_record.off.reserved 24 +telemetry_record.off.frame 32 +telemetry_record.off.reserved_tail 96 +telemetry_record.size 128 +telemetry_record.type.event 0 +telemetry_record.type.gap 1 lease_release.off.lease_id 0 lease_release.off.generation 4 lease_release.size 8 From da06e233b2f7a8558555025b16c136dc41931ead Mon Sep 17 00:00:00 2001 From: jbailes Date: Fri, 30 Oct 2026 21:45:56 +0100 Subject: [PATCH 3/7] D16: the reference OTLP exporter, and telemetry documentation exporters/otlp is the reference telemetry process: it attaches with the D16 grant (ring.ConnectTelemetry), sleeps on the telemetry doorbell, and on each wakeup sends every available record as one OTLP/HTTP logs request, releasing the records only once the collector accepts them. It has no batching timer; its only timer is the retry backoff while the collector is down. It is its own Go module, so the client keeps no OpenTelemetry dependency, and CI tests it under the race detector and end to end against the C host. The Go reader also holds the host's pidfd (D15), so an exporter asleep on the doorbell wakes and exits when the host goes away. docs/telemetry.md explains the record, the hand-off, loss and the exporter, and the architecture, routing and conformance pages point at it. --- .github/workflows/ci.yml | 25 +++ README.md | 7 +- docs/README.md | 1 + docs/architecture.md | 10 +- docs/implementations-and-conformance.md | 1 + docs/routing-and-flow-control.md | 2 +- docs/telemetry.md | 140 ++++++++++++++ exporters/otlp/export.go | 199 ++++++++++++++++++++ exporters/otlp/export_test.go | 239 ++++++++++++++++++++++++ exporters/otlp/go.mod | 35 ++++ exporters/otlp/go.sum | 59 ++++++ exporters/otlp/main.go | 67 +++++++ go/ring/client.go | 1 + go/ring/client_connect.go | 47 +++++ go/ring/telemetry.go | 36 +++- go/ring/telemetry_test.go | 14 ++ 16 files changed, 874 insertions(+), 9 deletions(-) create mode 100644 docs/telemetry.md create mode 100644 exporters/otlp/export.go create mode 100644 exporters/otlp/export_test.go create mode 100644 exporters/otlp/go.mod create mode 100644 exporters/otlp/go.sum create mode 100644 exporters/otlp/main.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a418d97..c4f95d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,6 +101,31 @@ jobs: (cd go && CGO_ENABLED=0 go build -o "$RUNNER_TEMP/go-module-helper" ./ring/testdata/module_helper) timeout 60s build/ouroboros-ring-module-runtime-harness "$RUNNER_TEMP/go-module-helper" + # D16: the reference telemetry exporter, its own module so the client keeps + # no OpenTelemetry dependency. + exporter-otlp: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff + with: + go-version-file: exporters/otlp/go.mod + cache-dependency-path: exporters/otlp/go.sum + - name: Format + working-directory: exporters/otlp + run: test -z "$(gofmt -l .)" + - name: Vet and test with the race detector + working-directory: exporters/otlp + run: go vet ./... && go test -race -count=1 ./... + - name: Build the C conformance host + run: cmake -S c -B build -DBUILD_TESTING=ON && cmake --build build --target ouroboros-ring-conformance-host + - name: Exporter against the C host + working-directory: exporters/otlp + env: + OURO_CONFORMANCE_HOST: ${{ github.workspace }}/build/ouroboros-ring-conformance-host + run: go test -count=1 -run CHost -v ./... + rust: runs-on: ubuntu-24.04 timeout-minutes: 20 diff --git a/README.md b/README.md index 6822dbb..13ebbf9 100644 --- a/README.md +++ b/README.md @@ -16,15 +16,16 @@ and pins a release of it like any other. implementation would be. None links, imports or generates from another. - **The documentation** in [`docs/`](docs/README.md) explains how it works: the [architecture](docs/architecture.md), attach and admission, routing and - flow control, module calls, arena payloads, capture, conformance, and - versions. + flow control, module calls, arena payloads, capture, telemetry, conformance, + and versions. ## Implementations | Path | Language | What it is | Proven by | |---|---|---|---| -| [`c/`](c/) | C11, POSIX/Linux | The host and a client, the module runtime, and the conformance host harness. `libouroboros-ring` (all of it) and `libouroboros-ring-client` (attach, wire, region, ring, module protocol: what a module process links) | `c/tests/` and the module runtime harness (17 tests, also under ASan/UBSan; the arena under TSan); dispatch overhead within the budget in `c/bench/baseline.json` | +| [`c/`](c/) | C11, POSIX/Linux | The host and a client, the module runtime, and the conformance host harness. `libouroboros-ring` (all of it) and `libouroboros-ring-client` (attach, wire, region, ring, module protocol: what a module process links) | `c/tests/` and the module runtime harness (19 tests, also under ASan/UBSan; the arena under TSan); dispatch overhead, with and without telemetry, within the budget in `c/bench/baseline.json` | | [`go/`](go/) | Go, no cgo | Client, module caller and module runtime: `github.com/RakuenSoftware/ouroboros-ring/go/ring` | `go test ./...` with `CGO_ENABLED=0`; `scripts/check_go_no_cgo.sh` | +| [`exporters/otlp/`](exporters/otlp/) | Go | The reference telemetry exporter (D16): the one process a host grants its telemetry ring, sending every record to an OTLP collector. Its own module, so the client has no OpenTelemetry dependency | `go test -race`; `TestExportFromCHost` against the C host | | [`rust/`](rust/) | Rust 2024 | Client, module caller and module runtime: crate `ouroboros-ring`, depending only on `libc` | `cargo test`; `rust/tests/c_host_conformance.rs` against the C host | The host exists only in C. The Go and Rust implementations are clients diff --git a/docs/README.md b/docs/README.md index a6cc398..4ddf274 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,6 +13,7 @@ could disagree. | [Module calls](module-calls.md) | the AMOD envelope, the module runtime and the module caller, and how call outcomes are decided | | [Arena payloads](arena-payloads.md) | payloads too large for a slot: leases, producing next to the host, reading from any process | | [Capture](capture.md) | recording the tap stream, the capture format, and reading it back | +| [Telemetry](telemetry.md) | every stamped event described off the host at once: the record, the hand-off to one exporter process, loss, and the reference OTLP exporter | | [Implementations and conformance](implementations-and-conformance.md) | what C, Go and Rust each provide, the evidence that holds them to the contract, and adding a language | | [Versions and releases](versions-and-releases.md) | release tags, the contract's own version numbers, and how consumers pin a release | diff --git a/docs/architecture.md b/docs/architecture.md index fb01a58..f3fda4a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -121,7 +121,10 @@ sequenceDiagram 2. The host drains outbound rings. It overwrites `src_handle` from the slot the frame came from, checks the frame's placement against the live geometry, and stamps `seq`, the host-wide order that capture and replay use. -3. The host offers the event to the tap exactly once, before any routing +3. If telemetry is on, the host first writes the event's content-free record + into the telemetry ring, which one exporter process drains off the host + ([telemetry](telemetry.md), [D16](../spec/decisions.md#d16-tap-telemetry-leaves-the-host-the-moment-an-event-is-stamped)). + Then it offers the event to the tap exactly once, before any routing decision, so the tap sees the intended stream in total order ([D6](../spec/decisions.md#d6-the-tap-records-the-intended-stream-sheds-are-events)). 4. The host routes by pattern: a notification to the kind's subscribed @@ -180,6 +183,7 @@ flowchart TB mproto["module_protocol.c
AMOD envelope"] mrun["module_runtime.c
serving a module"] mcli["module_client.c
calling a module"] + tread["telemetry.c
D16 record codec, reader"] end subgraph hostlib["libouroboros-ring (adds the host)"] direction LR @@ -187,6 +191,7 @@ flowchart TB route["route.c
pump, routing, flow control"] arena["arena.c
leases, lease view"] capture["capture.c
tap recorder, reader"] + tlm["telemetry_host.c
D16 record writer"] rhost["region_host.c
memfd creation"] runtime["runtime.c
listener, grants, peer checks"] local["local_attach.c
in-process attach"] @@ -208,6 +213,7 @@ and Rust code creates no regions and accepts no attaches. | `c/tests/`, `c/bench/` | the C tests and the dispatch-overhead benchmark | | `go/ring/` | the Go client, module caller and module runtime (pure Go, no cgo) | | `rust/` | the Rust client, module caller and module runtime (depends only on `libc`) | +| `exporters/otlp/` | the reference telemetry exporter (D16), its own Go module | | `spec/`, `vectors/` | the contract: prose and the frozen numbers | ## Further reading @@ -220,6 +226,8 @@ and Rust code creates no regions and accepts no attaches. module caller. - [Arena payloads](arena-payloads.md): large payloads and leases. - [Capture](capture.md): the tap recorder and the capture format. +- [Telemetry](telemetry.md): every stamped event described off the host, and + the exporter that sends it. - [Implementations and conformance](implementations-and-conformance.md): the three implementations and how a new one is proven. - [Versions and releases](versions-and-releases.md): what each version number diff --git a/docs/implementations-and-conformance.md b/docs/implementations-and-conformance.md index a21efe7..80730ee 100644 --- a/docs/implementations-and-conformance.md +++ b/docs/implementations-and-conformance.md @@ -19,6 +19,7 @@ about the contract rather than one codebase agreeing with itself. | Sleeping on the doorbell (D13) | `ouro_host_wait`, `ouro_client_wait` | `Client.Wait` | `Client::wait` | | Notifications to modules and callers (D14) | `notify`, `overflow` | `Notify`, `Overflow`, `OnNotification`, `Publish` | `notify`, `overflow`, `started` | | Process liveness and announced detach (D15) | `ouro_host_bind_process`, `ouro_runtime_maintenance_due`, `ouro_client_liveness_watched` | `HasProcessLiveness` | `has_process_liveness`, `liveness_watched` | +| Telemetry consumer end (D16) | `ouro_host_telemetry_enable`, `ouro_host_grant_telemetry`, `ouro_client_attach_telemetry`, `ouro_telemetry_*` | `AttachTelemetry`, `ConnectTelemetry`, `TelemetryReader`; the OTLP exporter in `exporters/otlp` | layout only | | Runtime listener and grant files | `ouro_runtime_*` | no | no | | Dependencies | libc, pthreads | `golang.org/x/sys/unix`, no cgo | `libc` only, no build script | diff --git a/docs/routing-and-flow-control.md b/docs/routing-and-flow-control.md index 9626acf..a13c37d 100644 --- a/docs/routing-and-flow-control.md +++ b/docs/routing-and-flow-control.md @@ -21,7 +21,7 @@ flowchart TB decode --> valid{"well-formed and
granted?"} valid -- no --> drop["count as dropped
(capability_denied to the sender
for an ungranted kind)"] --> next valid -- yes --> seen{"already stamped
(a blocked retry)?"} - seen -- no --> stamp["stamp seq, offer to the tap"] --> route + seen -- no --> stamp["stamp seq, write the telemetry record,
offer to the tap"] --> route seen -- yes --> route["route by pattern"] route --> result{"every destination
resolved?"} result -- yes --> commit["commit: advance the outbound ring"] --> next diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 0000000..5cada86 --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,140 @@ +# Telemetry + +Every event the host stamps is described off the host the moment it is +stamped ([D16](../spec/decisions.md#d16-tap-telemetry-leaves-the-host-the-moment-an-event-is-stamped)). +The host writes one record per event into a ring that only one process reads, +and that process sends the records to a collector. The host never waits for it, +never talks to a network, and pays about 13 ns per event for the hand-off. + +```mermaid +flowchart LR + subgraph hostproc["Host process (the embedding)"] + P["pump: stamp seq"] + T["telemetry ring
(own memfd, 128-byte records)"] + TAP["tap callback
(capture, governance)"] + R["routing"] + P -- "1. record, before anything else" --> T + P -- "2." --> TAP + P -- "3." --> R + end + subgraph exporter["Telemetry process"] + X["reference exporter
exporters/otlp"] + end + T -- "doorbell, only when it sleeps" --> X + X -- "one OTLP request per wakeup" --> COL["collector
(off the host)"] + X -. "release tail once accepted" .-> T +``` + +## What a record holds + +A record is 128 bytes: a type (event or gap), a `CLOCK_REALTIME` timestamp, a +lost count for gaps, and the stamped 64-byte frame header encoded exactly as on +the wire. That header carries `seq`, kind, pattern and placement, the attested +`principal_ref`, the source handle, the correlation id and the payload length. +**No payload byte is ever in a record**, so nothing the collector receives can +contain content. The exact layout is in `vectors/layout_vectors.tsv` +(`telemetry.*`, `telemetry_record.*`). + +The clock is read once per pump, at its first stamp. Records in one pump share +that time, and `seq` orders them exactly. + +## The hand-off + +```mermaid +sequenceDiagram + participant H as host pump + participant T as telemetry ring + participant X as exporter + participant C as collector + X->>T: consumer_waiting = 1, fence, re-check: empty, sleep + H->>T: write record, release-store head (per event) + H->>H: tap, route + H->>T: end of pump: fence, consumer_waiting? ring doorbell + T-->>X: wake + X->>T: read every available record + X->>C: one export request + C-->>X: accepted + X->>T: release-store tail +``` + +- **Per event, the host** stores the record's fields directly into its slot and + publishes it with one release store. It reads the consumer's `tail` across + cores only when its cached copy says the ring is full. +- **Per pump, the host** rings the doorbell once, and only if the exporter is + asleep (the [D13](../spec/decisions.md#d13-doorbells-idle-ends-sleep-in-the-kernel) + protocol). A busy host whose exporter keeps up makes no system call for + telemetry. An event stamped outside a pump, such as the `producer_reaped` a + reap emits, rings it at once. +- **The exporter** has no batching timer. A wakeup sends everything available + in one request; what arrives while that request is in flight goes in the + next. It releases records only once the collector accepts them, so delivery + is at least once and `seq` identifies a duplicate. + +## When the exporter falls behind + +The host never blocks on telemetry. If the ring is full when an event is +stamped, the host counts the record lost (`dropped` in the region header, and a +pending gap count) and goes on routing. The next record that fits is preceded +by a gap record carrying the count, so the collector sees exactly how much is +missing and where. Telemetry loss never sheds, delays or reorders an event. + +While the collector refuses or cannot be reached, the reference exporter backs +off from 100 ms to 5 s and retries the same records. That backoff is its only +timer. + +If the exporter exits, the region stays with the host, and the next granted +exporter continues from the same `tail`. Nothing is lost unless the ring filled +in between. + +## Granting it + +The record is metadata about every principal on the ring, so who reads it is +the embedding's decision, never implied by admission: + +```c +/* At startup, before clients may attach. 0 = 65536 records (8 MiB). */ +ouro_host_telemetry_enable(&host, 0); + +/* In the attach hook, for the one principal trusted with it. */ +static ouro_attach_status_t hook(void *ctx, ouro_host_t *h, uint32_t slot, + const ouro_attach_request_t *req) +{ + if (req->principal_ref == MY_EXPORTER_PRINCIPAL) + ouro_host_grant_telemetry(h, slot); + return OURO_ATTACH_OK; +} +``` + +The host hands the consumer end (`OURO_ATTACH_CAP_TELEMETRY`, two more +descriptors after the doorbell pair) only to a granted slot that asked for it, +had a doorbell granted, and while no other live slot holds it. + +## The reference exporter + +`exporters/otlp` is a separate Go module, so the Go client keeps no +OpenTelemetry dependency. It sends OTLP/HTTP logs, one log record per ring +record: + +| OTLP field | From | +|---|---| +| event name | `ouro.ring.event`, `ouro.ring.gap`, or `ouro.ring.malformed` | +| timestamp | the record's `realtime_ns` | +| severity | info; warn for `overflow`, `producer_reaped`, `capability_denied` and gaps | +| `ouro.seq`, `ouro.event.kind`, `ouro.message.type`, `ouro.payload.placement`, `ouro.payload.bytes`, `ouro.principal_ref`, `ouro.source.handle`, `ouro.control` | the frame header | +| `ouro.correlation_id`, `ouro.destination.handle` | when set | +| `ouro.lost` | gap records | + +```sh +OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=https://collector:4318/v1/logs \ + ouroboros-ring-otlp -socket /run/my-host/ring.sock -principal-ref 0x7E1E +``` + +Endpoint, headers, TLS and timeouts come from the standard +`OTEL_EXPORTER_OTLP_*` variables. It exits 3 when the host goes away (it holds +the host's pidfd, [D15](../spec/decisions.md#d15-liveness-is-the-process-not-a-timer)), +so a supervisor restarts it against the new host. + +Any process in any language can be the exporter instead: the region, record and +grant are the contract, and the C (`ouro_client_attach_telemetry`, +`ouro_telemetry_*`) and Go (`ring.AttachTelemetry`, `ring.ConnectTelemetry`, +`ring.TelemetryReader`) clients both read them. diff --git a/exporters/otlp/export.go b/exporters/otlp/export.go new file mode 100644 index 0000000..449cd0c --- /dev/null +++ b/exporters/otlp/export.go @@ -0,0 +1,199 @@ +// Command ouroboros-ring-otlp is the reference D16 telemetry exporter: the one +// process a host grants the consumer end of its telemetry ring. It sends every +// record off the host as an OTLP log record as soon as it is woken, and hands +// records back to the host only once the collector has accepted them. +// +// It has no batching timer. A wakeup drains everything available into one +// request; records that arrive while that request is in flight go in the next. +// Its only timer is the retry backoff while the collector refuses or cannot be +// reached, and while it waits the ring fills and the host counts what it cannot +// hold, which reaches the collector as a gap record. +package main + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/RakuenSoftware/ouroboros-ring/go/ring" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/log" + sdklog "go.opentelemetry.io/otel/sdk/log" + "go.opentelemetry.io/otel/sdk/resource" +) + +const scope = "github.com/RakuenSoftware/ouroboros-ring/exporters/otlp" + +// maxBatch bounds one request. A larger backlog is sent as several requests, +// back to back. +const maxBatch = 2048 + +// source is the consumer end of the telemetry ring; *ring.TelemetryReader is one. +type source interface { + Available() uint64 + Peek(i uint64) []byte + Release(n uint64) +} + +// collect is a processor that holds what one drain emits, so the drain sends it +// as one request. It never exports on its own: no timer, no queue. +type collect struct{ records []sdklog.Record } + +func (c *collect) OnEmit(_ context.Context, r *sdklog.Record) error { + c.records = append(c.records, r.Clone()) + return nil +} +func (c *collect) Enabled(context.Context, sdklog.EnabledParameters) bool { return true } +func (c *collect) Shutdown(context.Context) error { return nil } +func (c *collect) ForceFlush(context.Context) error { return nil } + +// exporter turns telemetry records into OTLP log records. +type exporter struct { + buf *collect + logger log.Logger + out sdklog.Exporter + now func() time.Time +} + +func newExporter(out sdklog.Exporter, res *resource.Resource) *exporter { + buf := &collect{} + provider := sdklog.NewLoggerProvider(sdklog.WithProcessor(buf), sdklog.WithResource(res)) + return &exporter{buf: buf, logger: provider.Logger(scope), out: out, now: time.Now} +} + +// drain sends up to maxBatch available records in one request and releases +// them once it is accepted. It reports how many it released. +func (e *exporter) drain(ctx context.Context, src source) (uint64, error) { + n := src.Available() + if n == 0 { + return 0, nil + } + if n > maxBatch { + n = maxBatch + } + e.buf.records = e.buf.records[:0] + observed := e.now() + for i := uint64(0); i < n; i++ { + e.logger.Emit(ctx, toLog(src.Peek(i), observed)) + } + if err := e.out.Export(ctx, e.buf.records); err != nil { + return 0, err + } + src.Release(n) + return n, nil +} + +func pattern(flags uint16) string { + switch { + case flags&ring.FNotification != 0: + return "notification" + case flags&ring.FRequest != 0: + return "request" + case flags&ring.FReply != 0: + return "reply" + case flags&ring.FCancel != 0: + return "cancel" + } + return "invalid" +} + +func placement(flags uint16) string { + switch { + case flags&ring.FInline != 0: + return "inline" + case flags&ring.FArena != 0: + return "arena" + } + return "none" +} + +// toLog maps one record. Every attribute is a header field: the ring never +// puts a payload byte in a record, so none can reach the collector. +func toLog(raw []byte, observed time.Time) log.Record { + var out log.Record + out.SetObservedTimestamp(observed) + rec, err := ring.DecodeTelemetryRecord(raw) + if err != nil { + out.SetTimestamp(observed) + out.SetEventName("ouro.ring.malformed") + out.SetSeverity(log.SeverityError) + out.SetBody(attribute.StringValue("malformed ring telemetry record")) + return out + } + out.SetTimestamp(time.Unix(0, int64(rec.RealtimeNS))) + if rec.Type == ring.TelemetryGap { + out.SetEventName("ouro.ring.gap") + out.SetSeverity(log.SeverityWarn) + out.SetBody(attribute.StringValue("ring telemetry lost records")) + out.AddAttributes(attribute.Int64("ouro.lost", int64(rec.Lost))) + return out + } + f := rec.Frame + control := f.HdrFlags&ring.FControl != 0 + out.SetEventName("ouro.ring.event") + out.SetBody(attribute.StringValue("ring event")) + out.SetSeverity(log.SeverityInfo) + switch f.EventKind { + case ring.KindOverflow, ring.KindProducerReaped, ring.KindCapabilityDenied: + out.SetSeverity(log.SeverityWarn) + } + out.AddAttributes( + attribute.Int64("ouro.seq", int64(f.Seq)), + attribute.Int64("ouro.event.kind", int64(f.EventKind)), + attribute.String("ouro.message.type", pattern(f.HdrFlags)), + attribute.String("ouro.payload.placement", placement(f.HdrFlags)), + attribute.Int64("ouro.payload.bytes", int64(f.PayloadLen)), + attribute.Int64("ouro.principal_ref", int64(f.PrincipalRef)), + attribute.Int64("ouro.source.handle", int64(f.SrcHandle)), + attribute.Bool("ouro.control", control), + ) + if f.CorrelationID != 0 { + out.AddAttributes(attribute.Int64("ouro.correlation_id", int64(f.CorrelationID))) + } + if control { + out.AddAttributes(attribute.Int64("ouro.destination.handle", int64(f.DstHandle))) + } + return out +} + +// waiter is what run sleeps on; *ring.TelemetryReader is one. +type waiter interface { + source + Wait(time.Duration) (bool, error) + Interrupt() +} + +// run drains whenever records are available, sleeping on the doorbell between. +// It returns when ctx ends, or with ring.ErrEpoch when the host has gone. +func (e *exporter) run(ctx context.Context, r waiter, report func(error)) error { + stop := context.AfterFunc(ctx, r.Interrupt) + defer stop() + backoff := time.Duration(0) + for ctx.Err() == nil { + n, err := e.drain(ctx, r) + if err != nil { + if ctx.Err() != nil { + break + } + report(fmt.Errorf("export: %w", err)) + backoff = min(max(2*backoff, 100*time.Millisecond), 5*time.Second) + select { + case <-ctx.Done(): + case <-time.After(backoff): + } + continue + } + backoff = 0 + if n > 0 { + continue // drain the rest before sleeping + } + if _, err := r.Wait(-1); err != nil { + if errors.Is(err, ring.ErrEpoch) { + return err + } + report(fmt.Errorf("wait: %w", err)) + } + } + return nil +} diff --git a/exporters/otlp/export_test.go b/exporters/otlp/export_test.go new file mode 100644 index 0000000..a561ddd --- /dev/null +++ b/exporters/otlp/export_test.go @@ -0,0 +1,239 @@ +package main + +import ( + "bytes" + "context" + "encoding/binary" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/RakuenSoftware/ouroboros-ring/go/ring" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" + "go.opentelemetry.io/otel/sdk/resource" + collogs "go.opentelemetry.io/proto/otlp/collector/logs/v1" + logspb "go.opentelemetry.io/proto/otlp/logs/v1" + "google.golang.org/protobuf/proto" +) + +// collector is a fake OTLP/HTTP logs endpoint. status is what it answers. +type collector struct { + mu sync.Mutex + status int + requests int + records []*logspb.LogRecord + services []string +} + +func (c *collector) ServeHTTP(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + c.mu.Lock() + defer c.mu.Unlock() + c.requests++ + if c.status != http.StatusOK { + w.WriteHeader(c.status) + return + } + var req collogs.ExportLogsServiceRequest + if err := proto.Unmarshal(body, &req); err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + for _, rl := range req.ResourceLogs { + for _, a := range rl.GetResource().GetAttributes() { + if a.Key == "service.name" { + c.services = append(c.services, a.Value.GetStringValue()) + } + } + for _, sl := range rl.ScopeLogs { + c.records = append(c.records, sl.LogRecords...) + } + } + w.Header().Set("Content-Type", "application/x-protobuf") + out, _ := proto.Marshal(&collogs.ExportLogsServiceResponse{}) + _, _ = w.Write(out) +} + +func (c *collector) snapshot() (int, []*logspb.LogRecord) { + c.mu.Lock() + defer c.mu.Unlock() + return c.requests, append([]*logspb.LogRecord(nil), c.records...) +} + +func newCollector(t *testing.T) (*collector, *exporter) { + t.Helper() + col := &collector{status: http.StatusOK} + srv := httptest.NewServer(col) + t.Cleanup(srv.Close) + out, err := otlploghttp.New(context.Background(), otlploghttp.WithEndpointURL(srv.URL+"/v1/logs"), + otlploghttp.WithRetry(otlploghttp.RetryConfig{Enabled: false})) + if err != nil { + t.Fatal(err) + } + res := resource.NewSchemaless() + return col, newExporter(out, res) +} + +func attr(r *logspb.LogRecord, key string) (int64, string, bool) { + for _, a := range r.Attributes { + if a.Key == key { + return a.Value.GetIntValue(), a.Value.GetStringValue(), true + } + } + return 0, "", false +} + +// memSource is an in-memory ring of records. +type memSource struct { + records [][]byte + released uint64 +} + +func (m *memSource) Available() uint64 { return uint64(len(m.records)) - m.released } +func (m *memSource) Peek(i uint64) []byte { return m.records[m.released+i] } +func (m *memSource) Release(n uint64) { m.released += n } + +func eventRecord(seq uint64, kind uint32, plen uint32) []byte { + b := make([]byte, ring.TelemetryRecordLen) + binary.LittleEndian.PutUint64(b[8:], 1_700_000_000_000_000_000+seq) + f := ring.Frame{HdrFlags: ring.FNotification | ring.FInline, WireVersion: ring.WireVersion, + EventKind: kind, Seq: seq, PayloadLen: plen, PayloadRef: ring.HdrLen, SrcHandle: 3, PrincipalRef: 77} + f.Encode(b[32:]) + return b +} + +func gapRecord(lost uint64) []byte { + b := make([]byte, ring.TelemetryRecordLen) + binary.LittleEndian.PutUint32(b[0:], uint32(ring.TelemetryGap)) + binary.LittleEndian.PutUint64(b[8:], 1_700_000_000_000_000_000) + binary.LittleEndian.PutUint64(b[16:], lost) + return b +} + +func TestDrainIsOneRequestReleasedOnAccept(t *testing.T) { + col, e := newCollector(t) + src := &memSource{records: [][]byte{eventRecord(1, 900, 16), gapRecord(4), eventRecord(6, 901, 0)}} + + col.status = http.StatusServiceUnavailable + if _, err := e.drain(context.Background(), src); err == nil { + t.Fatal("a refused export reported success") + } + if src.released != 0 { + t.Fatalf("released %d records the collector refused", src.released) + } + + col.mu.Lock() + col.status = http.StatusOK + col.mu.Unlock() + n, err := e.drain(context.Background(), src) + if err != nil || n != 3 || src.released != 3 { + t.Fatalf("drain: n %d released %d err %v", n, src.released, err) + } + requests, recs := col.snapshot() + if requests != 2 || len(recs) != 3 { + t.Fatalf("requests %d records %d: one drain must be one request", requests, len(recs)) + } + if recs[0].EventName != "ouro.ring.event" || recs[1].EventName != "ouro.ring.gap" { + t.Errorf("event names %q %q", recs[0].EventName, recs[1].EventName) + } + if seq, _, _ := attr(recs[0], "ouro.seq"); seq != 1 { + t.Errorf("seq %d", seq) + } + if kind, _, _ := attr(recs[0], "ouro.event.kind"); kind != 900 { + t.Errorf("kind %d", kind) + } + if lost, _, _ := attr(recs[1], "ouro.lost"); lost != 4 { + t.Errorf("lost %d", lost) + } + if pr, _, _ := attr(recs[0], "ouro.principal_ref"); pr != 77 { + t.Errorf("principal %d", pr) + } + if _, p, _ := attr(recs[0], "ouro.message.type"); p != "notification" { + t.Errorf("pattern %q", p) + } + if recs[0].TimeUnixNano != 1_700_000_000_000_000_001 { + t.Errorf("timestamp %d is not the host's", recs[0].TimeUnixNano) + } + if n, _ := e.drain(context.Background(), src); n != 0 { + t.Errorf("an empty ring drained %d", n) + } +} + +// End to end against the real C host: the exporter, granted the consumer end, +// delivers the record of a published notification and of the C side's ack to +// the collector, and nothing of the payload. +func TestExportFromCHost(t *testing.T) { + harness := os.Getenv("OURO_CONFORMANCE_HOST") + if harness == "" { + t.Skip("OURO_CONFORMANCE_HOST not set") + } + sock := filepath.Join(t.TempDir(), "bus.sock") + cmd := exec.Command(harness, sock, "10000") + var hout bytes.Buffer + cmd.Stdout, cmd.Stderr = &hout, &hout + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + defer func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }() + for deadline := time.Now().Add(5 * time.Second); ; time.Sleep(10 * time.Millisecond) { + if _, err := os.Stat(sock); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("harness socket did not appear") + } + } + client, reader, err := ring.ConnectTelemetry(context.Background(), sock, 1, 0x7E1E) + if err != nil { + t.Fatal(err) + } + defer client.Detach() + defer reader.Close() + + col, e := newCollector(t) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- e.run(ctx, reader, func(err error) { t.Log(err) }) }() + + payload := []byte("payload-never-exported") + if err := client.Publish(1000, payload); err != nil { + t.Fatal(err) + } + var notify, ack bool + for deadline := time.Now().Add(5 * time.Second); !(notify && ack) && time.Now().Before(deadline); time.Sleep(10 * time.Millisecond) { + _, recs := col.snapshot() + for _, r := range recs { + wire, _ := proto.Marshal(r) + if bytes.Contains(wire, []byte("payload-never")) { + t.Fatal("a payload byte reached the collector") + } + switch kind, _, _ := attr(r, "ouro.event.kind"); kind { + case 1000: + notify = true + if h, _, _ := attr(r, "ouro.source.handle"); uint32(h) != client.Handle { + t.Errorf("source handle %d, client %d", h, client.Handle) + } + case 1001: + ack = true + } + } + } + if !notify || !ack { + t.Fatalf("collector saw notify %v ack %v\nharness:\n%s", notify, ack, hout.String()) + } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatalf("run: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("run did not stop when its context ended: the doorbell wait was not interrupted") + } +} diff --git a/exporters/otlp/go.mod b/exporters/otlp/go.mod new file mode 100644 index 0000000..c43c917 --- /dev/null +++ b/exporters/otlp/go.mod @@ -0,0 +1,35 @@ +module github.com/RakuenSoftware/ouroboros-ring/exporters/otlp + +go 1.26.10 + +require ( + github.com/RakuenSoftware/ouroboros-ring/go v0.0.0 + go.opentelemetry.io/otel v1.46.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 + go.opentelemetry.io/otel/log v0.22.0 + go.opentelemetry.io/otel/sdk v1.46.0 + go.opentelemetry.io/otel/sdk/log v0.22.0 + go.opentelemetry.io/proto/otlp v1.11.0 + google.golang.org/protobuf v1.36.12 +) + +require ( + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/metric v1.46.0 // indirect + go.opentelemetry.io/otel/trace v1.46.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/text v0.41.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect + google.golang.org/grpc v1.83.1 // indirect +) + +// The exporter is built from the same tree as the client it links. +replace github.com/RakuenSoftware/ouroboros-ring/go => ../../go diff --git a/exporters/otlp/go.sum b/exporters/otlp/go.sum new file mode 100644 index 0000000..6e37edc --- /dev/null +++ b/exporters/otlp/go.sum @@ -0,0 +1,59 @@ +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 h1:lYk7RmxdLK865qLwibroNGldHa1U7SWKYYvNjlK7PIo= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0/go.mod h1:6GvlND0H0xdUJanOtIAn0xfwLkauh1tmsYEEVSMDdqY= +go.opentelemetry.io/otel/log v0.22.0 h1:5DBNnfvaJ6CVdkJ+Jle8Tzs50aSSv49TXGj9XRsEYw0= +go.opentelemetry.io/otel/log v0.22.0/go.mod h1:gzOt/R67vF2GniAqWu8Qv0SXy89f71muHcrkz76PCdc= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= +go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= +go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= +go.opentelemetry.io/otel/sdk/log v0.22.0 h1:PRL+s6P63XT4E/bheEflopPUpVxuvANqZwtt89yhoGk= +go.opentelemetry.io/otel/sdk/log v0.22.0/go.mod h1:JNp0sBELrjCTcu5W3GzABVypeU6vDJjBS+X0JISuz+g= +go.opentelemetry.io/otel/sdk/log/logtest v0.22.0 h1:infPnfNrhCNgOUZRs3gWUg8vhoBUHihq02gwK05gzlg= +go.opentelemetry.io/otel/sdk/log/logtest v0.22.0/go.mod h1:gkQZA3z15Bv3KU9vigBTi8dFechSozRP7v94X4VZv+s= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 h1:ax2KzoSRIZU/M0cIxri3pKxy99vniH1PVxWC6si/eZI= +google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688/go.mod h1:1RJ9BQGyNdZwkGc1eTqkErfRZ6RJyYPHZo73BZ1vQqI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 h1:cYNAzI2sUwhmCcoj9TxvihSrqsxt6uIkj3rDRhSDmW4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= +google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= +google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/exporters/otlp/main.go b/exporters/otlp/main.go new file mode 100644 index 0000000..dd0e71b --- /dev/null +++ b/exporters/otlp/main.go @@ -0,0 +1,67 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + + "github.com/RakuenSoftware/ouroboros-ring/go/ring" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" + "go.opentelemetry.io/otel/sdk/resource" +) + +// Exit codes: 0 stopped by signal, 1 setup failed, 3 the host went away (a +// supervisor restarts the exporter, which attaches to the new host). +func main() { + socket := flag.String("socket", "", "the host's ring socket") + class := flag.Uint("principal-class", 0, "attach principal class") + ref := flag.Uint("principal-ref", 0, "attach principal ref the host grants telemetry to") + service := flag.String("service-name", "ouroboros-ring", "OTLP service.name, unless OTEL_SERVICE_NAME is set") + flag.Parse() + if *socket == "" { + fmt.Fprintln(os.Stderr, "ouroboros-ring-otlp: -socket is required") + os.Exit(1) + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + // Endpoint, headers, TLS and timeout come from the standard + // OTEL_EXPORTER_OTLP_* variables. Retries are this process's own: records + // stay in the ring until accepted, so the SDK must not also queue them. + out, err := otlploghttp.New(ctx, otlploghttp.WithRetry(otlploghttp.RetryConfig{Enabled: false})) + if err != nil { + fmt.Fprintf(os.Stderr, "ouroboros-ring-otlp: exporter: %v\n", err) + os.Exit(1) + } + defer out.Shutdown(context.Background()) + name := os.Getenv("OTEL_SERVICE_NAME") + if name == "" { + name = *service + } + res, err := resource.Merge(resource.Default(), + resource.NewSchemaless(attribute.String("service.name", name))) + if err != nil { + fmt.Fprintf(os.Stderr, "ouroboros-ring-otlp: resource: %v\n", err) + os.Exit(1) + } + + client, reader, err := ring.ConnectTelemetry(ctx, *socket, uint32(*class), uint32(*ref)) + if err != nil { + fmt.Fprintf(os.Stderr, "ouroboros-ring-otlp: attach %s: %v\n", *socket, err) + os.Exit(1) + } + defer client.Detach() + defer reader.Close() + + report := func(err error) { fmt.Fprintf(os.Stderr, "ouroboros-ring-otlp: %v\n", err) } + if err := newExporter(out, res).run(ctx, reader, report); errors.Is(err, ring.ErrEpoch) { + fmt.Fprintln(os.Stderr, "ouroboros-ring-otlp: the host went away") + os.Exit(3) + } +} diff --git a/go/ring/client.go b/go/ring/client.go index 6ec0408..6453dba 100644 --- a/go/ring/client.go +++ b/go/ring/client.go @@ -210,6 +210,7 @@ func AttachTelemetry(sock int, principalClass uint32, principalRef uint32) (*Cli c.Detach() return nil, nil, err } + r.host = hostProcess(sock) return c, r, nil } diff --git a/go/ring/client_connect.go b/go/ring/client_connect.go index 5a60616..3c3fd6c 100644 --- a/go/ring/client_connect.go +++ b/go/ring/client_connect.go @@ -81,3 +81,50 @@ func keepAlive(ctx context.Context, client heartbeater, interval time.Duration) } }() } + +// ErrTelemetryNotGranted is returned when the host attached the client but +// did not hand it the telemetry consumer end (D16): telemetry is off, the +// embedding did not grant this principal, or another consumer holds it. +var ErrTelemetryNotGranted = errors.New("ring: telemetry consumer end not granted") + +// ConnectTelemetry attaches to a daemon's ring as its telemetry consumer +// (D16). Refused connections are retried as ConnectClient retries them. +func ConnectTelemetry(ctx context.Context, socketPath string, principalClass, principalRef uint32) (*Client, *TelemetryReader, error) { + if socketPath == "" { + return nil, nil, ErrModuleConfig + } + if ctx == nil { + ctx = context.Background() + } + for { + fd, err := unix.Socket(unix.AF_UNIX, unix.SOCK_SEQPACKET|unix.SOCK_CLOEXEC, 0) + if err != nil { + return nil, nil, err + } + err = unix.Connect(fd, &unix.SockaddrUnix{Name: socketPath}) + if err == nil { + client, reader, attachErr := AttachTelemetry(fd, principalClass, principalRef) + unix.Close(fd) + if attachErr != nil { + return nil, nil, attachErr + } + if reader == nil { + client.Detach() + return nil, nil, ErrTelemetryNotGranted + } + if !client.HasProcessLiveness() { + keepAlive(ctx, client, clientHeartbeatInterval) + } + return client, reader, nil + } + unix.Close(fd) + if !errors.Is(err, unix.ECONNREFUSED) { + return nil, nil, err + } + select { + case <-ctx.Done(): + return nil, nil, ctx.Err() + case <-time.After(moduleConnectRetry): + } + } +} diff --git a/go/ring/telemetry.go b/go/ring/telemetry.go index 991d5a2..cd369ba 100644 --- a/go/ring/telemetry.go +++ b/go/ring/telemetry.go @@ -95,6 +95,21 @@ type TelemetryReader struct { waiting *uint32 dropped *uint64 bell int + host int // D15: a pidfd on the host, -1 where none could be opened +} + +// hostProcess opens a process handle on the peer of the attach socket (D15), +// so a consumer asleep on the doorbell wakes when the host exits. +func hostProcess(sock int) int { + cred, err := unix.GetsockoptUcred(sock, unix.SOL_SOCKET, unix.SO_PEERCRED) + if err != nil || cred.Pid <= 0 { + return -1 + } + fd, err := unix.PidfdOpen(int(cred.Pid), 0) + if err != nil { + return -1 + } + return fd } func openTelemetryReader(regionFd, bellFd int) (*TelemetryReader, error) { @@ -133,7 +148,7 @@ func openTelemetryReader(regionFd, bellFd int) (*TelemetryReader, error) { return nil, err } return &TelemetryReader{ - mem: mem, ring: r, bell: bellFd, + mem: mem, ring: r, bell: bellFd, host: -1, waiting: u32ptr(mem, telemetryRegionOffWaiting), dropped: u64ptr(mem, telemetryRegionOffDropped), }, nil @@ -173,7 +188,7 @@ func (t *TelemetryReader) Dropped() uint64 { return atomic.LoadUint64(t.dropped) // Wait sleeps until a record is available or timeout passes (negative waits // indefinitely), under the D13 protocol. It reports whether records are -// available. +// available, and returns ErrEpoch once the host has exited (D15). func (t *TelemetryReader) Wait(timeout time.Duration) (bool, error) { atomic.StoreUint32(t.waiting, 1) defer atomic.StoreUint32(t.waiting, 0) @@ -185,15 +200,24 @@ func (t *TelemetryReader) Wait(timeout time.Duration) (bool, error) { ms = int((timeout + time.Millisecond - 1) / time.Millisecond) } fds := []unix.PollFd{{Fd: int32(t.bell), Events: unix.POLLIN}} + if t.host >= 0 { + fds = append(fds, unix.PollFd{Fd: int32(t.host), Events: unix.POLLIN}) + } n, err := unix.Poll(fds, ms) if err != nil && err != unix.EINTR { return false, err } - if n > 0 { + if n > 0 && fds[0].Revents&unix.POLLIN != 0 { var count [8]byte _, _ = unix.Read(t.bell, count[:]) } - return t.Available() > 0, nil + if t.Available() > 0 { + return true, nil + } + if n > 0 && len(fds) > 1 && fds[1].Revents != 0 { + return false, ErrEpoch + } + return false, nil } // Interrupt wakes Wait from another goroutine. @@ -210,4 +234,8 @@ func (t *TelemetryReader) Close() { unix.Close(t.bell) t.bell = -1 } + if t.host >= 0 { + unix.Close(t.host) + t.host = -1 + } } diff --git a/go/ring/telemetry_test.go b/go/ring/telemetry_test.go index f54be07..7d57c4d 100644 --- a/go/ring/telemetry_test.go +++ b/go/ring/telemetry_test.go @@ -3,6 +3,7 @@ package ring import ( "bytes" "encoding/binary" + "errors" "os" "os/exec" "path/filepath" @@ -161,4 +162,17 @@ func TestTelemetryConformance(t *testing.T) { if r.Dropped() != 0 { t.Errorf("dropped %d", r.Dropped()) } + + // D15: a consumer asleep on the doorbell wakes when the host exits. + start := time.Now() + go func() { + time.Sleep(100 * time.Millisecond) + _ = cmd.Process.Kill() + }() + if _, err := r.Wait(10 * time.Second); !errors.Is(err, ErrEpoch) { + t.Fatalf("wait after the host exited: %v", err) + } + if waited := time.Since(start); waited > 5*time.Second { + t.Errorf("woke %v after the host exited, not at once", waited) + } } From f5ef366fe59008854e5367b7f0ed34e400193fde Mon Sep 17 00:00:00 2001 From: jbailes Date: Fri, 30 Oct 2026 21:50:23 +0100 Subject: [PATCH 4/7] D16: a runtime grant file can name the telemetry exporter A telemetry key set to yes in a principal's grant file has the runtime grant that slot the telemetry consumer end from its attach hook; any value other than yes or no is a malformed policy. test_runtime proves a granted principal is handed the region and another is not. --- c/include/ouroboros/runtime.h | 1 + c/src/runtime.c | 5 +++ c/tests/test_runtime.c | 74 +++++++++++++++++++++++++++++++++++ docs/attach-and-admission.md | 5 +++ docs/telemetry.md | 3 ++ spec/decisions.md | 4 +- 6 files changed, 90 insertions(+), 2 deletions(-) diff --git a/c/include/ouroboros/runtime.h b/c/include/ouroboros/runtime.h index ed04886..0a2f9f2 100644 --- a/c/include/ouroboros/runtime.h +++ b/c/include/ouroboros/runtime.h @@ -23,6 +23,7 @@ typedef struct size_t request_count; const uint32_t *serve; size_t serve_count; + int telemetry; /* D16: hand this principal the telemetry consumer end */ } ouro_runtime_grant_t; typedef struct ouro_runtime ouro_runtime_t; diff --git a/c/src/runtime.c b/c/src/runtime.c index b1fd86d..995060a 100644 --- a/c/src/runtime.c +++ b/c/src/runtime.c @@ -208,6 +208,8 @@ static ouro_attach_status_t runtime_bind(void *ctx, ouro_host_t *host, uint32_t for (size_t i = 0; i < grant->serve_count; i++) if (ouro_host_serve_kind(host, slot, grant->serve[i]) != OURO_HOST_OK) goto denied; + if (grant->telemetry && ouro_host_grant_telemetry(host, slot) != OURO_HOST_OK) + goto denied; process_handle_close(&runtime->slot_pidfd[slot]); runtime->slot_pidfd[slot] = peer_pidfd; /* D15: the accept loop watches this handle and releases the slot the moment @@ -518,6 +520,9 @@ static int parse_grant_file(const char *path, owned_grant_t *owned) ok = parse_kinds(value, owned->request, &owned->grant.request_count) == 0; else if (strcmp(key, "serve") == 0) ok = parse_kinds(value, owned->serve, &owned->grant.serve_count) == 0; + else if (strcmp(key, "telemetry") == 0) + ok = strcmp(value, "yes") == 0 ? (owned->grant.telemetry = 1, 1) + : strcmp(value, "no") == 0; else ok = 0; } diff --git a/c/tests/test_runtime.c b/c/tests/test_runtime.c index 00278af..159ed25 100644 --- a/c/tests/test_runtime.c +++ b/c/tests/test_runtime.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -30,6 +31,78 @@ static ouro_client_result_t attach(const char *path, uint32_t principal_ref, our return result; } +/* D16: a grant file says which principal is the telemetry exporter. Only that + * principal is handed the consumer end, and only it while it lives. */ +static void test_telemetry_grant(const char *directory, const char *executable) +{ + char policy[PATH_MAX], grant_path[PATH_MAX], bad_path[PATH_MAX], sock[PATH_MAX]; + assert(snprintf(policy, sizeof policy, "%s/tpolicy", directory) > 0); + assert(snprintf(grant_path, sizeof grant_path, "%s/exporter.grant", policy) > 0); + assert(snprintf(sock, sizeof sock, "%s/telemetry.sock", directory) > 0); + assert(mkdir(policy, 0700) == 0); + FILE *f = fopen(grant_path, "w"); + assert(f != NULL); + assert(fprintf(f, "version=1\nprincipal_class=1\nprincipal_ref=11\nuid=self\nexecutable=%s\n" + "telemetry=yes\n", + executable) > 0); + assert(fclose(f) == 0); + ouro_runtime_policy_t *parsed = NULL; + assert(ouro_runtime_policy_load_dir(policy, &parsed) == 0); + size_t count = 0; + const ouro_runtime_grant_t *loaded = ouro_runtime_policy_grants(parsed, &count); + assert(count == 1 && loaded[0].telemetry == 1); + ouro_runtime_policy_free(&parsed); + + /* Any other value is a malformed policy, not a quiet "no". */ + assert(snprintf(bad_path, sizeof bad_path, "%s/bad.grant", policy) > 0); + f = fopen(bad_path, "w"); + assert(f != NULL); + assert(fprintf(f, "version=1\nprincipal_class=1\nprincipal_ref=12\nuid=self\nexecutable=%s\n" + "telemetry=maybe\n", + executable) > 0); + assert(fclose(f) == 0); + assert(ouro_runtime_policy_load_dir(policy, &parsed) != 0); + assert(unlink(bad_path) == 0); + + ouro_runtime_grant_t grants[] = { + {.principal_class = 1, .principal_ref = 11, .uid = OURO_RUNTIME_SELF_UID, + .executable = executable, .telemetry = 1}, + {.principal_class = 1, .principal_ref = 12, .uid = OURO_RUNTIME_SELF_UID, + .executable = executable}}; + ouro_host_config_t host_config = {.max_slots = 8, .slot_size = 512, .inline_budget = 400, + .queue_capacity = 8, .arena_size = 16384}; + ouro_host_t host; + assert(ouro_host_create(&host, &host_config, NULL, NULL) == OURO_HOST_OK); + assert(ouro_host_telemetry_enable(&host, 64) == OURO_HOST_OK); + pthread_mutex_t host_lock = PTHREAD_MUTEX_INITIALIZER; + ouro_runtime_config_t config = {.socket_path = sock, .socket_mode = 0600, .backlog = 8, + .stale_after_ns = 60000000000ULL, .grants = grants, + .grant_count = 2}; + ouro_runtime_t *runtime = ouro_runtime_start(&host, &host_lock, &config); + assert(runtime != NULL); + + ouro_client_t other, exporter; + ouro_telemetry_reader_t reader; + int granted = 1, fd = -1; + assert(ouro_endpoint_connect(sock, &fd) == 0); + assert(ouro_client_attach_telemetry(fd, &other, 1, 12, &reader, &granted) == OURO_CLIENT_OK); + assert(ouro_endpoint_close(&fd) == 0); + assert(!granted); + assert(ouro_endpoint_connect(sock, &fd) == 0); + assert(ouro_client_attach_telemetry(fd, &exporter, 1, 11, &reader, &granted) == OURO_CLIENT_OK); + assert(ouro_endpoint_close(&fd) == 0); + assert(granted && reader.hdr != NULL); + + ouro_telemetry_reader_close(&reader); + ouro_client_detach(&exporter); + ouro_client_detach(&other); + ouro_runtime_stop(&runtime); + ouro_host_destroy(&host); + pthread_mutex_destroy(&host_lock); + assert(unlink(grant_path) == 0); + assert(rmdir(policy) == 0); +} + static int crash_client(const char *socket_path) { ouro_client_t client; @@ -367,6 +440,7 @@ int main(int argc, char **argv) pthread_mutex_destroy(&host_lock); test_role_admission(socket_path, executable); test_host_exit_wakes_client(directory, executable); + test_telemetry_grant(directory, executable); assert(unlink(grant_path) == 0); assert(rmdir(policy_path) == 0); assert(rmdir(directory) == 0); diff --git a/docs/attach-and-admission.md b/docs/attach-and-admission.md index 3c97ffc..c638e1d 100644 --- a/docs/attach-and-admission.md +++ b/docs/attach-and-admission.md @@ -128,6 +128,11 @@ serve=4353,4354 and must equal the peer's `/proc//exe`. The four kind lists say what the slot may publish, observe, request and serve. +An optional `telemetry=yes` (or `no`, the default) names the one principal the +host hands its telemetry ring +([telemetry](telemetry.md), [D16](../spec/decisions.md#d16-tap-telemetry-leaves-the-host-the-moment-an-event-is-stamped)). +The host still grants it only while no other live slot holds it. + The listener also handles the case heartbeats alone handle badly. A process that crashes cannot detach its slot, and its supervisor may restart it at once. When the kernel supports `pidfd`, the runtime keeps a handle on each admitted diff --git a/docs/telemetry.md b/docs/telemetry.md index 5cada86..6c30749 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -105,6 +105,9 @@ static ouro_attach_status_t hook(void *ctx, ouro_host_t *h, uint32_t slot, } ``` +A host that admits through the runtime listener says it in the exporter's +grant file instead: `telemetry=yes`. + The host hands the consumer end (`OURO_ATTACH_CAP_TELEMETRY`, two more descriptors after the doorbell pair) only to a granted slot that asked for it, had a doorbell granted, and while no other live slot holds it. diff --git a/spec/decisions.md b/spec/decisions.md index 4900232..05b8780 100644 --- a/spec/decisions.md +++ b/spec/decisions.md @@ -884,8 +884,8 @@ stamp and the telemetry process's next read. Telemetry loss never sheds, delays or reorders the events themselves. - **Who consumes.** A client sets `OURO_ATTACH_CAP_TELEMETRY` (bit 2) with the doorbell bit. The host grants it only when telemetry is enabled, the embedding granted that slot with - `ouro_host_grant_telemetry` from its attach hook, the doorbell was granted, and no other live slot - holds the grant. The grant adds two descriptors after the five of D13: the telemetry region, then + `ouro_host_grant_telemetry` from its attach hook (the runtime does so for a grant file that says + `telemetry=yes`), the doorbell was granted, and no other live slot holds the grant. The grant adds two descriptors after the five of D13: the telemetry region, then the telemetry doorbell. When the consumer's slot is released, the next granted consumer continues from the same `tail`, so records written while no exporter was attached are not lost unless the ring filled. From 56ac0ec14dec40067aa632647a7b96dfe49ff1f3 Mon Sep 17 00:00:00 2001 From: jbailes Date: Fri, 30 Oct 2026 21:52:43 +0100 Subject: [PATCH 5/7] Release 0.3.0: D16 telemetry --- VERSION | 2 +- docs/versions-and-releases.md | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/VERSION b/VERSION index 0ea3a94..0d91a54 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.2.0 +0.3.0 diff --git a/docs/versions-and-releases.md b/docs/versions-and-releases.md index 10c3713..9925760 100644 --- a/docs/versions-and-releases.md +++ b/docs/versions-and-releases.md @@ -60,7 +60,9 @@ Some changes need none of these: `OURO_CONTROL_F_DOORBELL`, the two waiting flags, and an attach request that grew by a `capabilities` word an older host never reads. Process liveness (D15) is one more bit in that word and one reserved `client_heartbeat` value, - `OURO_HEARTBEAT_DETACHED`, which no clock reading takes. + `OURO_HEARTBEAT_DETACHED`, which no clock reading takes. Telemetry (D16) is + a third bit, `OURO_ATTACH_CAP_TELEMETRY`, and a region of its own that only a + client granted it receives, so no existing layout changes. A host and client agree the wire version at attach ([attach and admission](attach-and-admission.md)). Capture files carry their From 6386656f01de312f00f0a6a5548edadb1d1f8b9a Mon Sep 17 00:00:00 2001 From: jbailes Date: Fri, 30 Oct 2026 22:00:16 +0100 Subject: [PATCH 6/7] D16: the exporter is a command in the Go module A separate exporters/otlp module could not be installed at a release without a second tag cut after the go/ tag, and every consumer would pin two things. As go/cmd/ouroboros-ring-otlp a consumer builds it at the ring release it already requires, and the OpenTelemetry versions match aimee's. The ring package imports none of them, and check_go_no_cgo.sh now fails if go/ring ever depends on more than the standard library and golang.org/x/sys. --- .github/workflows/ci.yml | 29 +-------- README.md | 2 +- docs/architecture.md | 2 +- docs/implementations-and-conformance.md | 2 +- docs/telemetry.md | 15 +++-- exporters/otlp/go.mod | 35 ----------- exporters/otlp/go.sum | 59 ------------------- .../cmd/ouroboros-ring-otlp}/export.go | 2 +- .../cmd/ouroboros-ring-otlp}/export_test.go | 0 .../cmd/ouroboros-ring-otlp}/main.go | 0 go/go.mod | 28 ++++++++- go/go.sum | 57 ++++++++++++++++++ scripts/check_go_no_cgo.sh | 13 +++- 13 files changed, 113 insertions(+), 131 deletions(-) delete mode 100644 exporters/otlp/go.mod delete mode 100644 exporters/otlp/go.sum rename {exporters/otlp => go/cmd/ouroboros-ring-otlp}/export.go (98%) rename {exporters/otlp => go/cmd/ouroboros-ring-otlp}/export_test.go (100%) rename {exporters/otlp => go/cmd/ouroboros-ring-otlp}/main.go (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4f95d6..d0526d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,7 +76,7 @@ jobs: cache-dependency-path: go/go.sum - name: Format working-directory: go - run: test -z "$(gofmt -l ring)" + run: test -z "$(gofmt -l .)" - name: Vet and test without cgo working-directory: go env: @@ -94,38 +94,13 @@ jobs: env: CGO_ENABLED: "0" OURO_CONFORMANCE_HOST: ${{ github.workspace }}/build/ouroboros-ring-conformance-host - run: go test -count=1 -run Conformance -v ./ring + run: go test -count=1 -run 'Conformance|CHost' -v ./ring ./cmd/... - name: C module caller against the Go module process run: | cmake --build build --target ouroboros-ring-module-runtime-harness (cd go && CGO_ENABLED=0 go build -o "$RUNNER_TEMP/go-module-helper" ./ring/testdata/module_helper) timeout 60s build/ouroboros-ring-module-runtime-harness "$RUNNER_TEMP/go-module-helper" - # D16: the reference telemetry exporter, its own module so the client keeps - # no OpenTelemetry dependency. - exporter-otlp: - runs-on: ubuntu-24.04 - timeout-minutes: 15 - steps: - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 - - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff - with: - go-version-file: exporters/otlp/go.mod - cache-dependency-path: exporters/otlp/go.sum - - name: Format - working-directory: exporters/otlp - run: test -z "$(gofmt -l .)" - - name: Vet and test with the race detector - working-directory: exporters/otlp - run: go vet ./... && go test -race -count=1 ./... - - name: Build the C conformance host - run: cmake -S c -B build -DBUILD_TESTING=ON && cmake --build build --target ouroboros-ring-conformance-host - - name: Exporter against the C host - working-directory: exporters/otlp - env: - OURO_CONFORMANCE_HOST: ${{ github.workspace }}/build/ouroboros-ring-conformance-host - run: go test -count=1 -run CHost -v ./... - rust: runs-on: ubuntu-24.04 timeout-minutes: 20 diff --git a/README.md b/README.md index 13ebbf9..3d7ce6b 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ and pins a release of it like any other. |---|---|---|---| | [`c/`](c/) | C11, POSIX/Linux | The host and a client, the module runtime, and the conformance host harness. `libouroboros-ring` (all of it) and `libouroboros-ring-client` (attach, wire, region, ring, module protocol: what a module process links) | `c/tests/` and the module runtime harness (19 tests, also under ASan/UBSan; the arena under TSan); dispatch overhead, with and without telemetry, within the budget in `c/bench/baseline.json` | | [`go/`](go/) | Go, no cgo | Client, module caller and module runtime: `github.com/RakuenSoftware/ouroboros-ring/go/ring` | `go test ./...` with `CGO_ENABLED=0`; `scripts/check_go_no_cgo.sh` | -| [`exporters/otlp/`](exporters/otlp/) | Go | The reference telemetry exporter (D16): the one process a host grants its telemetry ring, sending every record to an OTLP collector. Its own module, so the client has no OpenTelemetry dependency | `go test -race`; `TestExportFromCHost` against the C host | +| [`go/cmd/ouroboros-ring-otlp/`](go/cmd/ouroboros-ring-otlp/) | Go | The reference telemetry exporter (D16): the one process a host grants its telemetry ring, sending every record to an OTLP collector. In the Go module, so a consumer builds it at the ring release it already pins; the `ring` package imports none of its OpenTelemetry dependencies | `go test -race`; `TestExportFromCHost` against the C host; `scripts/check_go_no_cgo.sh` keeps OpenTelemetry out of `go/ring` | | [`rust/`](rust/) | Rust 2024 | Client, module caller and module runtime: crate `ouroboros-ring`, depending only on `libc` | `cargo test`; `rust/tests/c_host_conformance.rs` against the C host | The host exists only in C. The Go and Rust implementations are clients diff --git a/docs/architecture.md b/docs/architecture.md index f3fda4a..e0ce59e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -213,7 +213,7 @@ and Rust code creates no regions and accepts no attaches. | `c/tests/`, `c/bench/` | the C tests and the dispatch-overhead benchmark | | `go/ring/` | the Go client, module caller and module runtime (pure Go, no cgo) | | `rust/` | the Rust client, module caller and module runtime (depends only on `libc`) | -| `exporters/otlp/` | the reference telemetry exporter (D16), its own Go module | +| `go/cmd/ouroboros-ring-otlp/` | the reference telemetry exporter (D16) | | `spec/`, `vectors/` | the contract: prose and the frozen numbers | ## Further reading diff --git a/docs/implementations-and-conformance.md b/docs/implementations-and-conformance.md index 80730ee..f52eedc 100644 --- a/docs/implementations-and-conformance.md +++ b/docs/implementations-and-conformance.md @@ -19,7 +19,7 @@ about the contract rather than one codebase agreeing with itself. | Sleeping on the doorbell (D13) | `ouro_host_wait`, `ouro_client_wait` | `Client.Wait` | `Client::wait` | | Notifications to modules and callers (D14) | `notify`, `overflow` | `Notify`, `Overflow`, `OnNotification`, `Publish` | `notify`, `overflow`, `started` | | Process liveness and announced detach (D15) | `ouro_host_bind_process`, `ouro_runtime_maintenance_due`, `ouro_client_liveness_watched` | `HasProcessLiveness` | `has_process_liveness`, `liveness_watched` | -| Telemetry consumer end (D16) | `ouro_host_telemetry_enable`, `ouro_host_grant_telemetry`, `ouro_client_attach_telemetry`, `ouro_telemetry_*` | `AttachTelemetry`, `ConnectTelemetry`, `TelemetryReader`; the OTLP exporter in `exporters/otlp` | layout only | +| Telemetry consumer end (D16) | `ouro_host_telemetry_enable`, `ouro_host_grant_telemetry`, `ouro_client_attach_telemetry`, `ouro_telemetry_*` | `AttachTelemetry`, `ConnectTelemetry`, `TelemetryReader`; the OTLP exporter `cmd/ouroboros-ring-otlp` | layout only | | Runtime listener and grant files | `ouro_runtime_*` | no | no | | Dependencies | libc, pthreads | `golang.org/x/sys/unix`, no cgo | `libc` only, no build script | diff --git a/docs/telemetry.md b/docs/telemetry.md index 6c30749..674e054 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -18,7 +18,7 @@ flowchart LR P -- "3." --> R end subgraph exporter["Telemetry process"] - X["reference exporter
exporters/otlp"] + X["reference exporter
go/cmd/ouroboros-ring-otlp"] end T -- "doorbell, only when it sleeps" --> X X -- "one OTLP request per wakeup" --> COL["collector
(off the host)"] @@ -114,9 +114,16 @@ had a doorbell granted, and while no other live slot holds it. ## The reference exporter -`exporters/otlp` is a separate Go module, so the Go client keeps no -OpenTelemetry dependency. It sends OTLP/HTTP logs, one log record per ring -record: +`go/cmd/ouroboros-ring-otlp` is in the ring's Go module, so a consumer builds +it at the release it already pins: + +```sh +go build github.com/RakuenSoftware/ouroboros-ring/go/cmd/ouroboros-ring-otlp +``` + +The `ring` package imports none of its OpenTelemetry dependencies, and +`scripts/check_go_no_cgo.sh` fails the build if it ever does. It sends OTLP/HTTP +logs, one log record per ring record: | OTLP field | From | |---|---| diff --git a/exporters/otlp/go.mod b/exporters/otlp/go.mod deleted file mode 100644 index c43c917..0000000 --- a/exporters/otlp/go.mod +++ /dev/null @@ -1,35 +0,0 @@ -module github.com/RakuenSoftware/ouroboros-ring/exporters/otlp - -go 1.26.10 - -require ( - github.com/RakuenSoftware/ouroboros-ring/go v0.0.0 - go.opentelemetry.io/otel v1.46.0 - go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 - go.opentelemetry.io/otel/log v0.22.0 - go.opentelemetry.io/otel/sdk v1.46.0 - go.opentelemetry.io/otel/sdk/log v0.22.0 - go.opentelemetry.io/proto/otlp v1.11.0 - google.golang.org/protobuf v1.36.12 -) - -require ( - github.com/cenkalti/backoff/v5 v5.0.3 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/go-logr/logr v1.4.4 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect - go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel/metric v1.46.0 // indirect - go.opentelemetry.io/otel/trace v1.46.0 // indirect - golang.org/x/net v0.58.0 // indirect - golang.org/x/sys v0.48.0 // indirect - golang.org/x/text v0.41.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect - google.golang.org/grpc v1.83.1 // indirect -) - -// The exporter is built from the same tree as the client it links. -replace github.com/RakuenSoftware/ouroboros-ring/go => ../../go diff --git a/exporters/otlp/go.sum b/exporters/otlp/go.sum deleted file mode 100644 index 6e37edc..0000000 --- a/exporters/otlp/go.sum +++ /dev/null @@ -1,59 +0,0 @@ -github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= -github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= -github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= -github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= -github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= -go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= -go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= -go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 h1:lYk7RmxdLK865qLwibroNGldHa1U7SWKYYvNjlK7PIo= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0/go.mod h1:6GvlND0H0xdUJanOtIAn0xfwLkauh1tmsYEEVSMDdqY= -go.opentelemetry.io/otel/log v0.22.0 h1:5DBNnfvaJ6CVdkJ+Jle8Tzs50aSSv49TXGj9XRsEYw0= -go.opentelemetry.io/otel/log v0.22.0/go.mod h1:gzOt/R67vF2GniAqWu8Qv0SXy89f71muHcrkz76PCdc= -go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= -go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= -go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= -go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= -go.opentelemetry.io/otel/sdk/log v0.22.0 h1:PRL+s6P63XT4E/bheEflopPUpVxuvANqZwtt89yhoGk= -go.opentelemetry.io/otel/sdk/log v0.22.0/go.mod h1:JNp0sBELrjCTcu5W3GzABVypeU6vDJjBS+X0JISuz+g= -go.opentelemetry.io/otel/sdk/log/logtest v0.22.0 h1:infPnfNrhCNgOUZRs3gWUg8vhoBUHihq02gwK05gzlg= -go.opentelemetry.io/otel/sdk/log/logtest v0.22.0/go.mod h1:gkQZA3z15Bv3KU9vigBTi8dFechSozRP7v94X4VZv+s= -go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= -go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= -go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= -go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= -go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= -go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= -go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= -go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= -golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= -golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= -golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= -golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= -golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= -gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= -gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 h1:ax2KzoSRIZU/M0cIxri3pKxy99vniH1PVxWC6si/eZI= -google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688/go.mod h1:1RJ9BQGyNdZwkGc1eTqkErfRZ6RJyYPHZo73BZ1vQqI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 h1:cYNAzI2sUwhmCcoj9TxvihSrqsxt6uIkj3rDRhSDmW4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= -google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= -google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= -google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= -google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/exporters/otlp/export.go b/go/cmd/ouroboros-ring-otlp/export.go similarity index 98% rename from exporters/otlp/export.go rename to go/cmd/ouroboros-ring-otlp/export.go index 449cd0c..a35d0b6 100644 --- a/exporters/otlp/export.go +++ b/go/cmd/ouroboros-ring-otlp/export.go @@ -23,7 +23,7 @@ import ( "go.opentelemetry.io/otel/sdk/resource" ) -const scope = "github.com/RakuenSoftware/ouroboros-ring/exporters/otlp" +const scope = "github.com/RakuenSoftware/ouroboros-ring/go/cmd/ouroboros-ring-otlp" // maxBatch bounds one request. A larger backlog is sent as several requests, // back to back. diff --git a/exporters/otlp/export_test.go b/go/cmd/ouroboros-ring-otlp/export_test.go similarity index 100% rename from exporters/otlp/export_test.go rename to go/cmd/ouroboros-ring-otlp/export_test.go diff --git a/exporters/otlp/main.go b/go/cmd/ouroboros-ring-otlp/main.go similarity index 100% rename from exporters/otlp/main.go rename to go/cmd/ouroboros-ring-otlp/main.go diff --git a/go/go.mod b/go/go.mod index 175abea..d495e93 100644 --- a/go/go.mod +++ b/go/go.mod @@ -2,4 +2,30 @@ module github.com/RakuenSoftware/ouroboros-ring/go go 1.26.10 -require golang.org/x/sys v0.48.0 +require ( + go.opentelemetry.io/otel v1.46.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 + go.opentelemetry.io/otel/log v0.22.0 + go.opentelemetry.io/otel/sdk v1.46.0 + go.opentelemetry.io/otel/sdk/log v0.22.0 + go.opentelemetry.io/proto/otlp v1.11.0 + golang.org/x/sys v0.48.0 + google.golang.org/protobuf v1.36.12 +) + +require ( + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/metric v1.46.0 // indirect + go.opentelemetry.io/otel/trace v1.46.0 // indirect + golang.org/x/net v0.59.0 // indirect + golang.org/x/text v0.42.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect + google.golang.org/grpc v1.84.0 // indirect +) diff --git a/go/go.sum b/go/go.sum index e3bf0e4..e8b9b62 100644 --- a/go/go.sum +++ b/go/go.sum @@ -1,2 +1,59 @@ +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 h1:Bd7KaOxzULLxtZ/K5s1aLbWhR0+5RToO65TXHsf3bqQ= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0/go.mod h1:nN7ts3dFXKtCZWc//yfkpcQNKJABg16/uDVAZpLDalo= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0 h1:lYk7RmxdLK865qLwibroNGldHa1U7SWKYYvNjlK7PIo= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.22.0/go.mod h1:6GvlND0H0xdUJanOtIAn0xfwLkauh1tmsYEEVSMDdqY= +go.opentelemetry.io/otel/log v0.22.0 h1:5DBNnfvaJ6CVdkJ+Jle8Tzs50aSSv49TXGj9XRsEYw0= +go.opentelemetry.io/otel/log v0.22.0/go.mod h1:gzOt/R67vF2GniAqWu8Qv0SXy89f71muHcrkz76PCdc= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= +go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= +go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= +go.opentelemetry.io/otel/sdk/log v0.22.0 h1:PRL+s6P63XT4E/bheEflopPUpVxuvANqZwtt89yhoGk= +go.opentelemetry.io/otel/sdk/log v0.22.0/go.mod h1:JNp0sBELrjCTcu5W3GzABVypeU6vDJjBS+X0JISuz+g= +go.opentelemetry.io/otel/sdk/log/logtest v0.22.0 h1:infPnfNrhCNgOUZRs3gWUg8vhoBUHihq02gwK05gzlg= +go.opentelemetry.io/otel/sdk/log/logtest v0.22.0/go.mod h1:gkQZA3z15Bv3KU9vigBTi8dFechSozRP7v94X4VZv+s= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc h1:rZHRz0ogq4Pid1IKtA/ivHi8XONVF5tqUQXA2TtgpkE= +google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:K/n0XLazCJHjD73YuMyNJsN38BPPy1kfFnKxUXaThFU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc h1:4bNTbnb44EqGVy9HaQxSY2jnafSUdgV3qHtedvNpDKg= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc/go.mod h1:OaIUM3+LpYcK2GXM4FTmhWoIq371Owdr+Cc7/BsYHHc= +google.golang.org/grpc v1.84.0 h1:soMyaPJ8pAak5PIQ0DGBUir0XRo2fRoMqhNWMLlLxO0= +google.golang.org/grpc v1.84.0/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/scripts/check_go_no_cgo.sh b/scripts/check_go_no_cgo.sh index 7d8de4b..f89fe49 100755 --- a/scripts/check_go_no_cgo.sh +++ b/scripts/check_go_no_cgo.sh @@ -28,4 +28,15 @@ if ! CGO_ENABLED=0 go build "$pkg" >/dev/null 2>&1; then exit 1 fi -echo "check_go_no_cgo: ok — go/ring is pure Go" +# The client depends on the standard library and golang.org/x/sys alone. The +# telemetry exporter (cmd/ouroboros-ring-otlp, D16) shares the module and brings +# OpenTelemetry with it, so this keeps that from ever reaching the client. +extra=$(go list -deps -f '{{if not .Standard}}{{.ImportPath}}{{end}}' "$pkg" | + grep -v -e '^github.com/RakuenSoftware/ouroboros-ring/go/ring$' -e '^golang.org/x/sys/' || true) +if [ -n "$extra" ]; then + echo "FAIL: go/ring depends on more than the standard library and golang.org/x/sys:" >&2 + echo "$extra" >&2 + exit 1 +fi + +echo "check_go_no_cgo: ok — go/ring is pure Go and depends only on golang.org/x/sys" From 30ad784d994ce61dab4c945e162dee696a758d3f Mon Sep 17 00:00:00 2001 From: jbailes Date: Fri, 30 Oct 2026 22:03:32 +0100 Subject: [PATCH 7/7] D16: the exporter's core is the otlpexport package An embedding with its own collector policy and identity, such as aimee, runs otlpexport.New(...).Run in a process of its own; cmd/ouroboros-ring-otlp is now a thin command around it. The end-to-end test against the C host moved with the core. --- .github/workflows/ci.yml | 2 +- docs/telemetry.md | 5 ++- go/cmd/ouroboros-ring-otlp/main.go | 6 ++- .../export.go => otlpexport/otlpexport.go} | 43 +++++++++++-------- .../otlpexport_test.go} | 8 ++-- 5 files changed, 38 insertions(+), 26 deletions(-) rename go/{cmd/ouroboros-ring-otlp/export.go => otlpexport/otlpexport.go} (79%) rename go/{cmd/ouroboros-ring-otlp/export_test.go => otlpexport/otlpexport_test.go} (97%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0526d6..b4edd3d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,7 +94,7 @@ jobs: env: CGO_ENABLED: "0" OURO_CONFORMANCE_HOST: ${{ github.workspace }}/build/ouroboros-ring-conformance-host - run: go test -count=1 -run 'Conformance|CHost' -v ./ring ./cmd/... + run: go test -count=1 -run 'Conformance|CHost' -v ./ring ./otlpexport - name: C module caller against the Go module process run: | cmake --build build --target ouroboros-ring-module-runtime-harness diff --git a/docs/telemetry.md b/docs/telemetry.md index 674e054..ee89fc4 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -121,7 +121,10 @@ it at the release it already pins: go build github.com/RakuenSoftware/ouroboros-ring/go/cmd/ouroboros-ring-otlp ``` -The `ring` package imports none of its OpenTelemetry dependencies, and +Its core is the package `go/otlpexport`, for an embedding that applies its +own collector policy or identity and runs `otlpexport.New(...).Run` in a +process of its own. The `ring` package imports none of its OpenTelemetry +dependencies, and `scripts/check_go_no_cgo.sh` fails the build if it ever does. It sends OTLP/HTTP logs, one log record per ring record: diff --git a/go/cmd/ouroboros-ring-otlp/main.go b/go/cmd/ouroboros-ring-otlp/main.go index dd0e71b..1065c24 100644 --- a/go/cmd/ouroboros-ring-otlp/main.go +++ b/go/cmd/ouroboros-ring-otlp/main.go @@ -1,3 +1,6 @@ +// Command ouroboros-ring-otlp is the reference D16 telemetry exporter process: +// the one process a host grants the consumer end of its telemetry ring. See +// package otlpexport for what it does with the records. package main import ( @@ -9,6 +12,7 @@ import ( "os/signal" "syscall" + "github.com/RakuenSoftware/ouroboros-ring/go/otlpexport" "github.com/RakuenSoftware/ouroboros-ring/go/ring" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp" @@ -60,7 +64,7 @@ func main() { defer reader.Close() report := func(err error) { fmt.Fprintf(os.Stderr, "ouroboros-ring-otlp: %v\n", err) } - if err := newExporter(out, res).run(ctx, reader, report); errors.Is(err, ring.ErrEpoch) { + if err := otlpexport.New(out, res).Run(ctx, reader, report); errors.Is(err, ring.ErrEpoch) { fmt.Fprintln(os.Stderr, "ouroboros-ring-otlp: the host went away") os.Exit(3) } diff --git a/go/cmd/ouroboros-ring-otlp/export.go b/go/otlpexport/otlpexport.go similarity index 79% rename from go/cmd/ouroboros-ring-otlp/export.go rename to go/otlpexport/otlpexport.go index a35d0b6..86af9ff 100644 --- a/go/cmd/ouroboros-ring-otlp/export.go +++ b/go/otlpexport/otlpexport.go @@ -1,14 +1,17 @@ -// Command ouroboros-ring-otlp is the reference D16 telemetry exporter: the one -// process a host grants the consumer end of its telemetry ring. It sends every -// record off the host as an OTLP log record as soon as it is woken, and hands -// records back to the host only once the collector has accepted them. +// Package otlpexport is the reference D16 telemetry exporter: it sends every +// record a host writes to its telemetry ring off the host, as an OTLP log +// record, as soon as it is woken, and hands records back to the host only once +// the collector has accepted them. // // It has no batching timer. A wakeup drains everything available into one // request; records that arrive while that request is in flight go in the next. // Its only timer is the retry backoff while the collector refuses or cannot be // reached, and while it waits the ring fills and the host counts what it cannot // hold, which reaches the collector as a gap record. -package main +// +// cmd/ouroboros-ring-otlp is the command around it. An embedding that applies +// its own collector policy or identity runs Exporter.Run itself. +package otlpexport import ( "context" @@ -23,14 +26,14 @@ import ( "go.opentelemetry.io/otel/sdk/resource" ) -const scope = "github.com/RakuenSoftware/ouroboros-ring/go/cmd/ouroboros-ring-otlp" +const scope = "github.com/RakuenSoftware/ouroboros-ring/go/otlpexport" // maxBatch bounds one request. A larger backlog is sent as several requests, // back to back. const maxBatch = 2048 -// source is the consumer end of the telemetry ring; *ring.TelemetryReader is one. -type source interface { +// Source is the consumer end of the telemetry ring; *ring.TelemetryReader is one. +type Source interface { Available() uint64 Peek(i uint64) []byte Release(n uint64) @@ -48,23 +51,24 @@ func (c *collect) Enabled(context.Context, sdklog.EnabledParameters) bool { retu func (c *collect) Shutdown(context.Context) error { return nil } func (c *collect) ForceFlush(context.Context) error { return nil } -// exporter turns telemetry records into OTLP log records. -type exporter struct { +// Exporter turns telemetry records into OTLP log records. +type Exporter struct { buf *collect logger log.Logger out sdklog.Exporter now func() time.Time } -func newExporter(out sdklog.Exporter, res *resource.Resource) *exporter { +// New sends through out, under the resource res (service.name and the like). +func New(out sdklog.Exporter, res *resource.Resource) *Exporter { buf := &collect{} provider := sdklog.NewLoggerProvider(sdklog.WithProcessor(buf), sdklog.WithResource(res)) - return &exporter{buf: buf, logger: provider.Logger(scope), out: out, now: time.Now} + return &Exporter{buf: buf, logger: provider.Logger(scope), out: out, now: time.Now} } // drain sends up to maxBatch available records in one request and releases // them once it is accepted. It reports how many it released. -func (e *exporter) drain(ctx context.Context, src source) (uint64, error) { +func (e *Exporter) drain(ctx context.Context, src Source) (uint64, error) { n := src.Available() if n == 0 { return 0, nil @@ -157,16 +161,17 @@ func toLog(raw []byte, observed time.Time) log.Record { return out } -// waiter is what run sleeps on; *ring.TelemetryReader is one. -type waiter interface { - source +// Reader is what Run drains and sleeps on; *ring.TelemetryReader is one. +type Reader interface { + Source Wait(time.Duration) (bool, error) Interrupt() } -// run drains whenever records are available, sleeping on the doorbell between. -// It returns when ctx ends, or with ring.ErrEpoch when the host has gone. -func (e *exporter) run(ctx context.Context, r waiter, report func(error)) error { +// Run drains whenever records are available, sleeping on the doorbell between. +// report receives export and wait errors; Run retries after them. It returns +// nil when ctx ends, or ring.ErrEpoch when the host has gone. +func (e *Exporter) Run(ctx context.Context, r Reader, report func(error)) error { stop := context.AfterFunc(ctx, r.Interrupt) defer stop() backoff := time.Duration(0) diff --git a/go/cmd/ouroboros-ring-otlp/export_test.go b/go/otlpexport/otlpexport_test.go similarity index 97% rename from go/cmd/ouroboros-ring-otlp/export_test.go rename to go/otlpexport/otlpexport_test.go index a561ddd..6a0a74c 100644 --- a/go/cmd/ouroboros-ring-otlp/export_test.go +++ b/go/otlpexport/otlpexport_test.go @@ -1,4 +1,4 @@ -package main +package otlpexport import ( "bytes" @@ -66,7 +66,7 @@ func (c *collector) snapshot() (int, []*logspb.LogRecord) { return c.requests, append([]*logspb.LogRecord(nil), c.records...) } -func newCollector(t *testing.T) (*collector, *exporter) { +func newCollector(t *testing.T) (*collector, *Exporter) { t.Helper() col := &collector{status: http.StatusOK} srv := httptest.NewServer(col) @@ -77,7 +77,7 @@ func newCollector(t *testing.T) (*collector, *exporter) { t.Fatal(err) } res := resource.NewSchemaless() - return col, newExporter(out, res) + return col, New(out, res) } func attr(r *logspb.LogRecord, key string) (int64, string, bool) { @@ -199,7 +199,7 @@ func TestExportFromCHost(t *testing.T) { col, e := newCollector(t) ctx, cancel := context.WithCancel(context.Background()) done := make(chan error, 1) - go func() { done <- e.run(ctx, reader, func(err error) { t.Log(err) }) }() + go func() { done <- e.Run(ctx, reader, func(err error) { t.Log(err) }) }() payload := []byte("payload-never-exported") if err := client.Publish(1000, payload); err != nil {