From a6c4c9b7b345b7485d78a6e506a32f54563c6ca6 Mon Sep 17 00:00:00 2001
From: Yordis Prieto <yordis.prieto@gmail.com>
Date: Tue, 20 Oct 2026 21:37:15 -0400
Subject: [PATCH] fix(server): surface why process group metrics are not
 exported

Signed-off-by: Yordis Prieto <yordis.prieto@gmail.com>
---
 .../ProcessGroupMetrics.test.ts               | 65 ++++++++++++++++++-
 .../resourceTelemetry/ProcessGroupMetrics.ts  | 26 +++++++-
 2 files changed, 86 insertions(+), 5 deletions(-)

diff --git a/apps/server/src/resourceTelemetry/ProcessGroupMetrics.test.ts b/apps/server/src/resourceTelemetry/ProcessGroupMetrics.test.ts
index 694803b64842..f10937c4ebd4 100644
--- a/apps/server/src/resourceTelemetry/ProcessGroupMetrics.test.ts
+++ b/apps/server/src/resourceTelemetry/ProcessGroupMetrics.test.ts
@@ -1,7 +1,17 @@
 import { assert, describe, it } from "@effect/vitest";
-import type { ResourceTelemetryProcess } from "@t3tools/contracts";
+import type { ResourceTelemetryProcess, ResourceTelemetrySnapshot } from "@t3tools/contracts";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Logger from "effect/Logger";
+import * as References from "effect/References";
+import * as Ref from "effect/Ref";
+import * as TestClock from "effect/testing/TestClock";
 
-import { groupProcesses } from "./ProcessGroupMetrics.ts";
+import * as ServerConfig from "../config.ts";
+import * as ProcessGroupMetrics from "./ProcessGroupMetrics.ts";
+import * as ResourceTelemetry from "./ResourceTelemetry.ts";
+
+const { groupProcesses } = ProcessGroupMetrics;
 
 const process = (
   pid: number,
@@ -57,3 +67,54 @@ describe("groupProcesses", () => {
     });
   });
 });
+
+describe("ProcessGroupMetrics.layer", () => {
+  it.effect("logs the first failed sample and the recovery, not every repeat", () => {
+    const logs: Array<string> = [];
+    const logger = Logger.make(({ logLevel, message }) => {
+      logs.push(`${logLevel.toUpperCase()} ${String([message].flat()[0])}`);
+    });
+    return Effect.gen(function* () {
+      const outcomes = yield* Ref.make<ReadonlyArray<"fail" | "ok">>(["fail", "fail", "ok"]);
+      const refresh = Ref.modify(outcomes, ([next = "ok", ...rest]) => [next, rest] as const).pipe(
+        Effect.flatMap((next) =>
+          next === "fail"
+            ? Effect.fail(
+                new ResourceTelemetry.ResourceTelemetryRefreshFailed({
+                  operation: "refresh",
+                  cause: new Error("Resource monitor binary was not found for darwin/arm64."),
+                }),
+              )
+            : Effect.succeed({ processes: [] } as unknown as ResourceTelemetrySnapshot),
+        ),
+      );
+      const telemetry = { refresh } as unknown as ResourceTelemetry.ResourceTelemetry["Service"];
+
+      yield* Layer.build(
+        ProcessGroupMetrics.layer.pipe(
+          Layer.provide(Layer.succeed(ResourceTelemetry.ResourceTelemetry, telemetry)),
+          Layer.provide(
+            ServerConfig.layer({
+              otlpMetricsUrl: "http://localhost:4318/v1/metrics",
+            } as ServerConfig.ServerConfig["Service"]),
+          ),
+        ),
+      );
+      yield* TestClock.adjust("3 minutes");
+
+      assert.deepStrictEqual(logs, [
+        "WARN process group metrics are not exported: sample failed",
+        "DEBUG process group metrics sample failed",
+        "INFO process group metrics sample recovered",
+      ]);
+    }).pipe(
+      Effect.scoped,
+      Effect.provide(
+        Layer.mergeAll(
+          Logger.layer([logger], { mergeWithExisting: false }),
+          Layer.succeed(References.MinimumLogLevel, "Debug"),
+        ),
+      ),
+    );
+  });
+});
diff --git a/apps/server/src/resourceTelemetry/ProcessGroupMetrics.ts b/apps/server/src/resourceTelemetry/ProcessGroupMetrics.ts
index 6f8f29c37161..56a8c5b6cc03 100644
--- a/apps/server/src/resourceTelemetry/ProcessGroupMetrics.ts
+++ b/apps/server/src/resourceTelemetry/ProcessGroupMetrics.ts
@@ -2,6 +2,7 @@ import type { ResourceTelemetryProcess } from "@t3tools/contracts";
 import * as Effect from "effect/Effect";
 import * as Layer from "effect/Layer";
 import * as Metric from "effect/Metric";
+import * as Ref from "effect/Ref";
 
 import * as ServerConfig from "../config.ts";
 import { metricAttributes } from "../observability/Metrics.ts";
@@ -113,10 +114,29 @@ export const layer = Layer.effectDiscard(
         yield* Metric.update(Metric.withAttributes(processCount, attributes), total.count);
       }
     });
-    yield* sample().pipe(
-      Effect.catch((error) =>
-        Effect.logDebug("process group metrics sample failed", { error: error.message }),
+    // A failing collector would otherwise leave these series silently absent, so the first
+    // failure and the recovery are logged; the repeats stay at debug.
+    const failing = yield* Ref.make(false);
+    const recordSuccess = Ref.getAndSet(failing, false).pipe(
+      Effect.flatMap((wasFailing) =>
+        wasFailing ? Effect.logInfo("process group metrics sample recovered") : Effect.void,
       ),
+    );
+    const recordFailure = (error: ResourceTelemetry.ResourceTelemetryRefreshFailed) => {
+      const reason = error.cause instanceof Error ? error.cause.message : error.message;
+      return Ref.getAndSet(failing, true).pipe(
+        Effect.flatMap((wasFailing) =>
+          wasFailing
+            ? Effect.logDebug("process group metrics sample failed", { reason })
+            : Effect.logWarning("process group metrics are not exported: sample failed", {
+                reason,
+              }),
+        ),
+      );
+    };
+    yield* sample().pipe(
+      Effect.andThen(recordSuccess),
+      Effect.catch(recordFailure),
       Effect.andThen(Effect.sleep(SAMPLE_INTERVAL)),
       Effect.forever,
       Effect.forkScoped,