perf_hooks: implement qrde analysis support in Histogram

Implements quantile-respectful density estimate calcuation
on Histogram. Helps with tail-focused latency analysis
without retaining raw samples.

Baking this directly into Node.js, based on a 1 million
sample, 1k bin workload, this impl is roughly 150-325x
faster than performing the equivalent in Rscript.

Signed-off-by: James M Snell <jasnell@gmail.com>
Assisted-by: Opencode
PR-URL: #65806
Reviewed-By: Matteo Collina <matteo.collina@gmail.com>

Author: jasnell

benchmark/perf_hooks/histogram-qrde.js

@@ -0,0 +1,37 @@
+'use strict';
+
+const common = require('../common.js');
+const { createHistogram } = require('perf_hooks');
+
+const bench = common.createBenchmark(main, {
+  n: [5],
+  bins: [100, 1000],
+  samples: [1e6],
+  unique: [100, 1000, 10000],
+  dequantize: ['none', 'hdr', 'all'],
+}, {
+  test: {
+    n: 1,
+    bins: 10,
+    samples: 100,
+    unique: 10,
+  },
+});
+
+async function main({ n, bins, samples, unique, dequantize }) {
+  const histogram = createHistogram();
+  const maximum = 1e12;
+
+  for (let i = 0; i < samples; i++) {
+    const index = i % unique;
+    const rank = unique === 1 ? 0 : index / (unique - 1);
+    histogram.record(Math.max(1, Math.round(maximum ** rank)));
+  }
+
+  await histogram.qrde({ bins, dequantize });
+  bench.start();
+  for (let i = 0; i < n; i++) {
+    await histogram.qrde({ bins, dequantize });
+  }
+  bench.end(n);
+}

doc/api/perf_hooks.md

@@ -2427,6 +2427,82 @@ Returns the values at the specified percentiles, computed in a single
 efficient pass over the histogram data. More efficient than calling
 `histogram.percentile()` multiple times.
 
+### `histogram.qrde([options])`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* `options` {Object}
+  * `bins` {number} The number of equal-probability density bins to return.
+    Must be between 1 and 1000. Cannot be used with `probabilities`.
+    **Default:** `100`.
+  * `probabilities` {number\[]} Custom probability boundaries. The array must
+    contain between 2 and 1001 strictly increasing values, start with `0`, and
+    end with `1`. Cannot be used with `bins`.
+  * `dequantize` {string} Controls whether repeated bucket values are spread
+    deterministically over their equivalent-value ranges. May be `'none'`,
+    `'hdr'`, or `'all'`. **Default:** `'hdr'`.
+  * `cache` {boolean} When `true`, retains the expanded histogram snapshot for
+    reuse by subsequent calls with `cache: true`. The snapshot is invalidated
+    when the histogram is modified. **Default:** `false`.
+* Returns: {Promise} Fulfills with an {Object} containing:
+  * `probabilities` {Float64Array} The probability boundaries used by the
+    estimate.
+  * `quantiles` {Float64Array} The quantiles at the probability boundaries.
+  * `densities` {Float64Array} The density within each quantile interval.
+  * `count` {bigint} The number of values in the histogram snapshot.
+  * `bucketCount` {number} The number of occupied HDR buckets.
+  * `corrections` {number} The number of non-monotonic floating-point results
+    that were clamped to the preceding quantile.
+  * `dequantize` {string} The selected dequantization mode.
+
+Returns a quantile-respectful density estimate based on the Harrell-Davis
+quantile estimator. By default, `bins` generates equal probability boundaries.
+The `probabilities` option can instead focus the estimate on regions such as
+p90, p99, p99.9, and p99.99. The density for interval `i` contains probability
+mass `probabilities[i + 1] - probabilities[i]`. The histogram is snapshotted
+when the method is called. Snapshot expansion and the estimate are calculated
+in the libuv thread pool. Highly concentrated beta weights use a second-order
+asymptotic approximation to avoid numerical convergence loss at large sample
+counts.
+
+Setting `cache` to `true` avoids repeating snapshot capture and expansion when
+several estimates are requested from an unchanged histogram. The retained
+snapshot uses memory proportional to the number of occupied HDR buckets and is
+released when the histogram is next modified.
+
+QRDE temporarily uses approximately one additional HDR count array plus 32
+bytes per occupied bucket. With `cache: true`, the expanded 32-byte-per-bucket
+snapshot remains allocated. The following estimates use `lowest: 1` and
+`highest: Number.MAX_SAFE_INTEGER` and exclude allocator and JavaScript object
+overhead:
+
+| `figures` | Histogram | Maximum expanded snapshot | Peak cache-miss QRDE |
+| --------- | --------: | ------------------------: | -------------------: |
+| 1         |   6.3 KiB |                    25 KiB |               31 KiB |
+| 2         |    47 KiB |                   188 KiB |              235 KiB |
+| 3         |   352 KiB |                   1.4 MiB |              1.7 MiB |
+| 4         |   5.0 MiB |                    20 MiB |               25 MiB |
+| 5         |    37 MiB |                   148 MiB |              185 MiB |
+
+The maximum snapshot column assumes every representable bucket is occupied.
+Lower `highest` values reduce histogram and temporary copy sizes. Concurrent
+calls that miss the cache each require their own temporary copy and expanded
+snapshot.
+
+HDR histograms aggregate observations into equivalent-value buckets. The
+`'hdr'` dequantization mode models repeated values in buckets wider than one
+unit as a continuous uniform distribution over the bucket resolution. This
+reduces density artifacts introduced by HDR quantization while preserving
+repeated unit-resolution values as point masses. The `'all'` mode also
+dequantizes repeated unit-resolution values. Use `'none'` to calculate the
+grouped Harrell-Davis estimator using bucket midpoints directly.
+
+An empty histogram returns the requested `probabilities` but produces empty
+`quantiles` and `densities` arrays. A non-dequantized interval whose quantile
+boundaries are equal has an infinite density.
+
 ### `histogram.reset()`
 
 <!-- YAML

lib/internal/histogram.js

@@ -33,9 +33,11 @@ const {
 
 const {
   validateArray,
+  validateBoolean,
   validateInteger,
   validateNumber,
   validateObject,
+  validateOneOf,
 } = require('internal/validators');
 
 const {
@@ -45,6 +47,7 @@ const {
 const kDestroy = Symbol('kDestroy');
 const kHandle = Symbol('kHandle');
 const kRecordable = Symbol('kRecordable');
+const kQrdeDequantizationModes = ['none', 'hdr', 'all'];
 
 const {
   kClone,
@@ -594,6 +597,73 @@ class Histogram {
     return map;
   }
 
+  /**
+   * Builds a quantile-respectful density estimate using Harrell-Davis
+   * quantiles. Values can be spread deterministically within their HDR
+   * equivalent-value ranges to reduce quantization artifacts.
+   * @param {{ bins?: number, probabilities?: number[],
+   *   dequantize?: 'none'|'hdr'|'all', cache?: boolean }} [options]
+   * @returns {Promise<{
+   *   probabilities: Float64Array,
+   *   quantiles: Float64Array,
+   *   densities: Float64Array,
+   *   count: bigint,
+   *   bucketCount: number,
+   *   corrections: number,
+   *   dequantize: 'none'|'hdr'|'all',
+   * }>}
+   */
+  qrde(options = kEmptyObject) {
+    if (!isHistogram(this))
+      throw new ERR_INVALID_THIS('Histogram');
+    validateObject(options, 'options');
+    const { bins, probabilities, dequantize = 'hdr', cache = false } = options;
+    if (bins !== undefined && probabilities !== undefined) {
+      throw new ERR_INVALID_ARG_VALUE(
+        'options', options, '"bins" and "probabilities" are mutually exclusive');
+    }
+
+    let boundaries;
+    if (probabilities === undefined) {
+      const binCount = bins ?? 100;
+      validateInteger(binCount, 'options.bins', 1, 1000);
+      boundaries = new Float64Array(binCount + 1);
+      for (let i = 0; i <= binCount; i++) boundaries[i] = i / binCount;
+    } else {
+      validateArray(probabilities, 'options.probabilities', 2);
+      const length = probabilities.length;
+      if (length > 1001) {
+        throw new ERR_OUT_OF_RANGE(
+          'options.probabilities.length', '>= 2 && <= 1001', length);
+      }
+
+      boundaries = new Float64Array(length);
+      let previous = -1;
+      for (let i = 0; i < length; i++) {
+        const probability = probabilities[i];
+        validateNumber(probability, `options.probabilities[${i}]`, 0, 1);
+        if (probability <= previous) {
+          throw new ERR_INVALID_ARG_VALUE(
+            'options.probabilities', probabilities, 'must be strictly increasing');
+        }
+        boundaries[i] = probability;
+        previous = probability;
+      }
+      if (boundaries[0] !== 0 || boundaries[length - 1] !== 1) {
+        throw new ERR_INVALID_ARG_VALUE(
+          'options.probabilities', probabilities, 'must start with 0 and end with 1');
+      }
+    }
+
+    validateOneOf(dequantize,
+                  'options.dequantize', kQrdeDequantizationModes);
+    validateBoolean(cache, 'options.cache');
+    let mode = 0;
+    if (dequantize === 'hdr') mode = 1;
+    else if (dequantize === 'all') mode = 2;
+    return this[kHandle]?.qrde(boundaries, mode, cache);
+  }
+
   /**
    * @returns {void}
    */

src/histogram-inl.h

@@ -33,9 +33,15 @@ void Histogram::UpdateEwma(double value) {
   }
 }
 
+void Histogram::InvalidateRecordedSnapshot() {
+  mutation_generation_++;
+  recorded_snapshot_cache_.reset();
+}
+
 void Histogram::Reset() {
   RwLock::ScopedWriteLock lock(mutex_);
   hdr_reset(histogram_.get());
+  InvalidateRecordedSnapshot();
   exceeds_ = 0;
   prev_ = 0;
   ewma_mean_ = 0;
@@ -49,8 +55,10 @@ double Histogram::Add(const Histogram& other) {
     exceeds_ += other.exceeds_;
     if (other.prev_ > prev_) prev_ = other.prev_;
     // hdr_add merges all bucket counts and total_count internally.
-    return static_cast<double>(
-        hdr_add(histogram_.get(), other.histogram_.get()));
+    const double dropped =
+        static_cast<double>(hdr_add(histogram_.get(), other.histogram_.get()));
+    InvalidateRecordedSnapshot();
+    return dropped;
   };
 
   // When adding a histogram to itself, a single write lock suffices.
@@ -146,8 +154,10 @@ bool Histogram::RecordCorrected(int64_t value, int64_t expected_interval) {
       hdr_record_corrected_value(histogram_.get(), value, expected_interval);
   if (!recorded)
     exceeds_++;
-  else
+  else {
+    InvalidateRecordedSnapshot();
     UpdateEwma(static_cast<double>(value));
+  }
   return recorded;
 }
 
@@ -156,8 +166,10 @@ bool Histogram::Record(int64_t value) {
   bool recorded = hdr_record_value(histogram_.get(), value);
   if (!recorded)
     exceeds_++;
-  else
+  else {
+    InvalidateRecordedSnapshot();
     UpdateEwma(static_cast<double>(value));
+  }
   return recorded;
 }
 
@@ -170,8 +182,10 @@ uint64_t Histogram::RecordDelta() {
     delta = time - prev_;
     if (!hdr_record_value(histogram_.get(), delta))
       exceeds_++;
-    else
+    else {
+      InvalidateRecordedSnapshot();
       UpdateEwma(static_cast<double>(delta));
+    }
   }
   prev_ = time;
   return delta;