refactor: use sharded WAL

packages/utils/docs/profiler.md

@@ -275,6 +275,7 @@ const saved = profiler.measure('save-user', () => saveToDb(user), {
 - **Controllable over env vars**: Easily enable or disable profiling through environment variables.
 
 This profiler extends all options and API from Profiler with automatic process exit handling for buffered performance data.
+
 The NodeJSProfiler automatically subscribes to performance observation and installs exit handlers that flush buffered data on process termination (signals, fatal errors, or normal exit).
 
 ### Exit Handlers
@@ -283,6 +284,24 @@ The profiler automatically subscribes to process events (`exit`, `SIGINT`, `SIGT
 
 The `close()` method is idempotent and safe to call from exit handlers. It unsubscribes from exit handlers, closes the WAL sink, and unsubscribes from the performance observer, ensuring all buffered performance data is written before process termination.
 
+### Profiler Lifecycle States
+
+The NodeJSProfiler follows a state machine with three distinct states:
+
+**State Machine Flow**
+
+```
+active → finalized → cleaned
+  ↓         ↓
+  └─────────┘ (no transitions back)
+```
+
+- **active**: Profiler is running and collecting performance measurements
+- **finalized**: Profiler has been closed and all buffered data has been flushed to disk
+- **cleaned**: Profiler resources have been fully released
+
+Once a state transition occurs (e.g., `active` → `finalized`), there are no transitions back to previous states. This ensures data integrity and prevents resource leaks.
+
 ## Configuration
 
 ```ts
@@ -295,12 +314,16 @@ new NodejsProfiler<DomainEvents, Tracks>(options: NodejsProfilerOptions<DomainEv
 
 **Options:**
 
-| Property                 | Type                                    | Default    | Description                                                                     |
-| ------------------------ | --------------------------------------- | ---------- | ------------------------------------------------------------------------------- |
-| `encodePerfEntry`        | `PerformanceEntryEncoder<DomainEvents>` | _required_ | Function that encodes raw PerformanceEntry objects into domain-specific types   |
-| `captureBufferedEntries` | `boolean`                               | `true`     | Whether to capture performance entries that occurred before observation started |
-| `flushThreshold`         | `number`                                | `20`       | Threshold for triggering queue flushes based on queue length                    |
-| `maxQueueSize`           | `number`                                | `10_000`   | Maximum number of items allowed in the queue before new entries are dropped     |
+| Property                 | Type                                    | Default          | Description                                                                           |
+| ------------------------ | --------------------------------------- | ---------------- | ------------------------------------------------------------------------------------- |
+| `format`                 | `ProfilerFormat<DomainEvents>`          | _required_       | WAL format configuration for sharded write-ahead logging, including `encodePerfEntry` |
+| `measureName`            | `string`                                | _auto-generated_ | Optional folder name for sharding. If not provided, a new group ID will be generated  |
+| `outDir`                 | `string`                                | `'tmp/profiles'` | Output directory for WAL shards and final files                                       |
+| `outBaseName`            | `string`                                | _optional_       | Override the base name for WAL files (overrides format.baseName)                      |
+| `format.encodePerfEntry` | `PerformanceEntryEncoder<DomainEvents>` | _required_       | Function that encodes raw PerformanceEntry objects into domain-specific types         |
+| `captureBufferedEntries` | `boolean`                               | `true`           | Whether to capture performance entries that occurred before observation started       |
+| `flushThreshold`         | `number`                                | `20`             | Threshold for triggering queue flushes based on queue length                          |
+| `maxQueueSize`           | `number`                                | `10_000`         | Maximum number of items allowed in the queue before new entries are dropped           |
 
 ## API Methods
 

packages/utils/src/lib/create-runner-files.ts

@@ -1,8 +1,8 @@
 import { writeFile } from 'node:fs/promises';
 import path from 'node:path';
-import { threadId } from 'node:worker_threads';
 import type { RunnerFilesPaths } from '@code-pushup/models';
 import { ensureDirectoryExists, pluginWorkDir } from './file-system.js';
+import { getUniqueProcessThreadId } from './process-id.js';
 
 /**
  * Function to create timestamp nested plugin runner files for config and output.
@@ -14,9 +14,7 @@ export async function createRunnerFiles(
   pluginSlug: string,
   configJSON: string,
 ): Promise<RunnerFilesPaths> {
-  // Use timestamp + process ID + threadId
-  // This prevents race conditions when running the same plugin for multiple projects in parallel
-  const uniqueId = `${(performance.timeOrigin + performance.now()) * 10}-${process.pid}-${threadId}`;
+  const uniqueId = getUniqueProcessThreadId();
   const runnerWorkDir = path.join(pluginWorkDir(pluginSlug), uniqueId);
   const runnerConfigPath = path.join(runnerWorkDir, 'plugin-config.json');
   const runnerOutputPath = path.join(runnerWorkDir, 'runner-output.json');

packages/utils/src/lib/performance-observer.ts

@@ -122,14 +122,6 @@ export type PerformanceObserverOptions<T> = {
    * @default DEFAULT_MAX_QUEUE_SIZE (10000)
    */
   maxQueueSize?: number;
-
-  /**
-   * Name of the environment variable to check for debug mode.
-   * When the env var is set to 'true', encode failures create performance marks for debugging.
-   *
-   * @default 'CP_PROFILER_DEBUG'
-   */
-  debugEnvVar?: string;
 };
 
 /**
@@ -151,7 +143,7 @@ export type PerformanceObserverOptions<T> = {
  *   - Queue cleared after successful batch writes
  *
  * - Item Disposition Scenarios 💥
- *   - **Encode Failure**: ❌ Items lost when `encode()` throws. Creates perf mark if debug env var (specified by `debugEnvVar`) is set to 'true'.
+ *   - **Encode Failure**: ❌ Items lost when `encode()` throws. Creates perf mark if 'DEBUG' env var is set to 'true'.
  *   - **Sink Write Failure**: 💾 Items stay in queue when sink write fails during flush
  *   - **Sink Closed**: 💾 Items stay in queue when sink is closed during flush
  *   - **Proactive Flush Throws**: 💾 Items stay in queue when `flush()` throws during threshold check
@@ -210,22 +202,20 @@ export class PerformanceObserverSink<T> {
       captureBufferedEntries,
       flushThreshold = DEFAULT_FLUSH_THRESHOLD,
       maxQueueSize = DEFAULT_MAX_QUEUE_SIZE,
-      debugEnvVar = PROFILER_DEBUG_ENV_VAR,
     } = options;
     this.#encodePerfEntry = encodePerfEntry;
     this.#sink = sink;
     this.#buffered = captureBufferedEntries ?? true;
     this.#maxQueueSize = maxQueueSize;
     validateFlushThreshold(flushThreshold, this.#maxQueueSize);
     this.#flushThreshold = flushThreshold;
-    this.#debug = isEnvVarEnabled(debugEnvVar);
+    this.#debug = isEnvVarEnabled(PROFILER_DEBUG_ENV_VAR);
   }
 
   /**
    * Returns whether debug mode is enabled for encode failures.
    *
-   * Debug mode is determined by the environment variable specified by `debugEnvVar`
-   * (defaults to 'CP_PROFILER_DEBUG'). When enabled, encode failures create
+   * Debug mode is determined by the environment variable 'DEBUG'
    * performance marks for debugging.
    *
    * @returns true if debug mode is enabled, false otherwise

packages/utils/src/lib/performance-observer.unit.test.ts

@@ -373,27 +373,27 @@ describe('PerformanceObserverSink', () => {
       // Restore original env before each test
       if (originalEnv === undefined) {
         // eslint-disable-next-line functional/immutable-data
-        delete process.env.CP_PROFILER_DEBUG;
+        delete process.env.DEBUG;
       } else {
         // eslint-disable-next-line functional/immutable-data
-        process.env.CP_PROFILER_DEBUG = originalEnv;
+        process.env.DEBUG = originalEnv;
       }
     });
 
     afterEach(() => {
       // Restore original env after each test
       if (originalEnv === undefined) {
         // eslint-disable-next-line functional/immutable-data
-        delete process.env.CP_PROFILER_DEBUG;
+        delete process.env.DEBUG;
       } else {
         // eslint-disable-next-line functional/immutable-data
-        process.env.CP_PROFILER_DEBUG = originalEnv;
+        process.env.DEBUG = originalEnv;
       }
     });
 
     it('creates performance mark when encode fails and debug mode is enabled via env var', () => {
       // eslint-disable-next-line functional/immutable-data
-      process.env.CP_PROFILER_DEBUG = 'true';
+      process.env.DEBUG = 'true';
 
       const failingEncode = vi.fn(() => {
         throw new Error('EncodeError');
@@ -424,7 +424,7 @@ describe('PerformanceObserverSink', () => {
 
     it('does not create performance mark when encode fails and debug mode is disabled', () => {
       // eslint-disable-next-line functional/immutable-data
-      delete process.env.CP_PROFILER_DEBUG;
+      delete process.env.DEBUG;
 
       const failingEncode = vi.fn(() => {
         throw new Error('EncodeError');
@@ -455,7 +455,7 @@ describe('PerformanceObserverSink', () => {
 
     it('handles encode errors for unnamed entries correctly', () => {
       // eslint-disable-next-line functional/immutable-data
-      process.env.CP_PROFILER_DEBUG = 'true';
+      process.env.DEBUG = 'true';
 
       const failingEncode = vi.fn(() => {
         throw new Error('EncodeError');
@@ -483,7 +483,7 @@ describe('PerformanceObserverSink', () => {
 
     it('handles non-Error objects thrown from encode function', () => {
       // eslint-disable-next-line functional/immutable-data
-      process.env.CP_PROFILER_DEBUG = 'true';
+      process.env.DEBUG = 'true';
 
       const failingEncode = vi.fn(() => {
         throw 'String error';
@@ -739,16 +739,16 @@ describe('PerformanceObserverSink', () => {
 
     beforeEach(() => {
       // eslint-disable-next-line functional/immutable-data
-      delete process.env.CP_PROFILER_DEBUG;
+      delete process.env.DEBUG;
     });
 
     afterEach(() => {
       if (originalEnv === undefined) {
         // eslint-disable-next-line functional/immutable-data
-        delete process.env.CP_PROFILER_DEBUG;
+        delete process.env.DEBUG;
       } else {
         // eslint-disable-next-line functional/immutable-data
-        process.env.CP_PROFILER_DEBUG = originalEnv;
+        process.env.DEBUG = originalEnv;
       }
     });
 
@@ -760,7 +760,7 @@ describe('PerformanceObserverSink', () => {
 
     it('returns true when debug env var is set to "true"', () => {
       // eslint-disable-next-line functional/immutable-data
-      process.env.CP_PROFILER_DEBUG = 'true';
+      process.env.DEBUG = 'true';
 
       const observer = new PerformanceObserverSink(options);
 
@@ -769,7 +769,7 @@ describe('PerformanceObserverSink', () => {
 
     it('returns false when debug env var is set to a value other than "true"', () => {
       // eslint-disable-next-line functional/immutable-data
-      process.env.CP_PROFILER_DEBUG = 'false';
+      process.env.DEBUG = 'false';
 
       const observer = new PerformanceObserverSink(options);
 
@@ -778,35 +778,11 @@ describe('PerformanceObserverSink', () => {
 
     it('returns false when debug env var is set to empty string', () => {
       // eslint-disable-next-line functional/immutable-data
-      process.env.CP_PROFILER_DEBUG = '';
+      process.env.DEBUG = '';
 
       const observer = new PerformanceObserverSink(options);
 
       expect(observer.debug).toBeFalse();
     });
-
-    it('respects custom debugEnvVar option', () => {
-      // eslint-disable-next-line functional/immutable-data
-      process.env.CUSTOM_DEBUG_VAR = 'true';
-
-      const observer = new PerformanceObserverSink({
-        ...options,
-        debugEnvVar: 'CUSTOM_DEBUG_VAR',
-      });
-
-      expect(observer.debug).toBeTrue();
-
-      // eslint-disable-next-line functional/immutable-data
-      delete process.env.CUSTOM_DEBUG_VAR;
-    });
-
-    it('returns false when custom debugEnvVar is not set', () => {
-      const observer = new PerformanceObserverSink({
-        ...options,
-        debugEnvVar: 'CUSTOM_DEBUG_VAR',
-      });
-
-      expect(observer.debug).toBeFalse();
-    });
   });
 });