Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ext.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,8 @@ func (e *ExtEnv) X__table_base() *int32 { return &e.tableBase }
func (e *ExtEnv) CallWithContextValues(pCtx, nVal, pVal int32, fn func(Context, ...Value)) {
db := e.DB.(*Conn)
args := callbackArgs(db, nVal, ptr_t(pVal))
defer returnArgs(args)
fn(Context{db, ptr_t(pCtx)}, *args...)
returnArgs(args)
}

// ExtensionInit loads an SQLite extension library.
Expand Down
61 changes: 61 additions & 0 deletions ext/approx/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
# Approximate statistics SQLite extension

This package provides approximate aggregate functions for SQLite,
using mergeable data sketches:
- **distinct counts** (HyperLogLog), e.g. daily, weekly, monthly active users;
- **quantiles** (REQ), e.g. median and tail latencies;
- **top-K** (Frequent Items), e.g. most popular content, highest spenders.

Computing these exactly requires memory that grows with the input,
and exact results don't combine:
you can't sum seven daily counts of active users to get a weekly count,
or average daily medians to get a monthly median.

Sketches trade a small, bounded error for compact size
(about 1 KiB for distinct counts, tens of KiB for quantiles over millions of rows)
and mergeability: combining two sketches produces a sketch of the combined data.

### Roll-up tables

Sketches serialize to BLOBs, so you can store them in roll-up tables.
For example, you can accumulate daily sketches from raw events:

```sql
CREATE TABLE daily (day TEXT PRIMARY KEY, users BLOB, latency BLOB, pages BLOB);

INSERT INTO daily
SELECT date(time),
approx_count_distinct_accumulate(user_id),
approx_percentile_accumulate(latency),
approx_top_k_accumulate(path, 10)
FROM events
GROUP BY 1;
```

You can then combine them to answer coarser queries without rescanning `events`:

```sql
SELECT strftime('%Y-%m', day) AS month,
approx_count_distinct_estimate(approx_count_distinct_combine(users)) AS users,
approx_percentile_estimate(approx_percentile_combine(latency), 0.99) AS p99,
approx_top_k_estimate(approx_top_k_combine(pages), 3) AS top_pages
FROM daily
GROUP BY 1;
```

### Window functions

Every aggregate also works as a window function,
e.g. to compute a rolling count of weekly active users:

```sql
SELECT day,
approx_count_distinct_estimate(
approx_count_distinct_combine(users) OVER (ORDER BY day ROWS 6 PRECEDING)
) AS weekly_users
FROM daily;
```

### Storage format

The sketch BLOB storage format is specified in [`format.md`](format.md).
38 changes: 38 additions & 0 deletions ext/approx/approx.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@
// - approx_percentile_combine(B): merges REQ sketch BLOBs into a single sketch BLOB.
// - approx_percentile_estimate(B, P): estimates quantile(s) P in [0, 1] from a REQ sketch BLOB.
// - approx_percent_rank_estimate(B, V): estimates relative rank(s) in [0, 1] of V from a REQ sketch BLOB.
// - approx_top_k(X, [W,] K): approximate top K elements (and counts/sums) as JSON using a FI sketch.
// - approx_top_k_accumulate(X, [W,] K): accumulates elements into a FI sketch BLOB for up to K elements.
// - approx_top_k_combine(B): merges FI sketch BLOBs into a single sketch BLOB.
// - approx_top_k_estimate(B [, K]): estimates the top K elements as JSON from a FI sketch BLOB.
//
// [HyperLogLog] sketch BLOBs use the [Aggregate Knowledge HLL Storage Specification].
//
Expand All @@ -19,13 +23,18 @@
// Rank error matches typical KLL sketches at the median,
// and is at most 2x worse at the lower tail.
//
// [Frequent Items] (FI) sketches use weighted mergeable Misra-Gries summaries.
//
// [HyperLogLog]: https://github.com/segmentio/go-hll
// [Relative Error Quantiles]: https://arxiv.org/abs/2004.01668
// [Frequent Items]: https://arxiv.org/abs/1705.07001
// [Aggregate Knowledge HLL Storage Specification]: https://github.com/aggregateknowledge/hll-storage-spec
package approx

import (
"cmp"
"errors"
"math"

"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/util/sql3util"
Expand Down Expand Up @@ -58,5 +67,34 @@ func Register(db *sqlite3.Conn) error {
reqPercentileEstimate),
db.CreateFunction("approx_percent_rank_estimate", 2, json,
reqRankEstimate),
db.CreateWindowFunction("approx_top_k", 2, json,
sql3util.WindowFromCombiner[topSketch](1e4)),
db.CreateWindowFunction("approx_top_k", 3, json,
sql3util.WindowFromCombiner[topSketch](1e4)),
db.CreateWindowFunction("approx_top_k_accumulate", 2, flags,
sql3util.WindowFromCombiner[topAccumulate](1e4)),
db.CreateWindowFunction("approx_top_k_accumulate", 3, flags,
sql3util.WindowFromCombiner[topAccumulate](1e4)),
db.CreateWindowFunction("approx_top_k_combine", 1, flags,
sql3util.WindowFromCombiner[topCombine](1e4)),
db.CreateFunction("approx_top_k_estimate", 1, json,
topEstimate),
db.CreateFunction("approx_top_k_estimate", 2, json,
topEstimate),
)
}

func floatIsInt(f float64) (int64, bool) {
// math.MaxInt64 is not exactly representable as a float64 (it rounds to 2^63),
// but int64(2^63) may saturate back to math.MaxInt64.
i := int64(f)
return i, float64(i) == f && i != math.MaxInt64
}

func cmpIntFloat(i int64, f float64) int {
// Assumes both floatIsInt(f) and IsNaN(f) are false.
if c := cmp.Compare(float64(i), f); c != 0 {
return c
}
return -1 // float64(i) == f means f == 2^63
}
150 changes: 150 additions & 0 deletions ext/approx/format.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
# Sketch BLOB storage format

Serialization format for the HLL (cardinality), REQ (quantile), and FI (top-k)
sketch BLOBs produced and consumed by the `approx` SQLite extension in
[go-sqlite3](https://github.com/ncruces/go-sqlite3).

## Header byte

Every sketch BLOB begins with a single header byte using the
[Aggregate Knowledge HLL Storage Specification](https://github.com/aggregateknowledge/hll-storage-spec)
nibble layout: high nibble = schema version (`1`), low nibble = type code.

| First byte | Type | Format |
| ------------- | ---- | ------ |
| `0x11`–`0x14` | 1–4 | HLL (`EMPTY`, `EXPLICIT`, `SPARSE`, `FULL`) |
| `0x15` | 5 | REQ (quantiles) |
| `0x16` | 6 | FI (frequent items / top-k) |

Types `5` and `6` are private extensions in unassigned HLL v1 slots so sketch
types can be told apart by their first byte; everything after byte 0 is
format-specific.

## HLL sketch (cardinality)

HLL sketch BLOBs follow the
[Aggregate Knowledge HLL Storage Specification](https://github.com/aggregateknowledge/hll-storage-spec)
verbatim. `accumulate` emits `log2m = 11`, `regwidth = 5`, sparse
enabled, auto explicit threshold; `combine` requires inputs to share `log2m` and
`regwidth` and preserves them.

**Hashing.** Rows with no arguments or any `NULL` argument are ignored. Starting
from `seed = 0`, each argument updates `seed` to the first 64 bits (`h1`) of
`MurmurHash3_x64_128` initialized with `h1 = h2 = seed'`:

- **integer** (and **float** exactly representable as `int64`, incl. `±0.0`):
`seed' = seed`, 8-byte little-endian `i64`;
- **text**: `seed' = seed`, raw bytes;
- **blob**: `seed' = seed ^ 1`, raw bytes;
- **float** (other): `seed' = seed ^ 2`, 8-byte little-endian `f64`.

If any step yields `0`, the row is ignored; otherwise the final `seed` is added
to the HLL as a raw 64-bit hash. For single integer (`bigint`) or text values,
this matches [`postgresql-hll`](https://github.com/citusdata/postgresql-hll).

## REQ & FI conventions

The remaining sections specify the `go-sqlite3` custom formats (`0x15` and `0x16`):

- Fixed-width integers (`u16`, `i64`) and floats (`f32`, `f64`) are **little-endian**;
- `uvarint` is an unsigned LEB128 varint (Go `encoding/binary` `Uvarint`, `uint64`);
- `f32`/`f64` are IEEE-754 binary32/binary64 bit patterns; `NaN` is never valid.

## REQ sketch (quantiles)

A [relative-error quantiles sketch](https://arxiv.org/abs/2004.01668) in
high-rank-accuracy mode, storing `f32` values across weighted levels.
An item at level `h` represents `2^h` input items.

```
byte header = 0x15
byte k = 40 // initial section size; must be 40
level* // levels h = 0, 1, ... until EOF (at most 53)
```

An empty sketch is just the 2 header bytes. Levels repeat until the end of the
BLOB (the level count is implicit; a 54th level is invalid), contiguous from
`h = 0` with no empty levels:

```
uvarint state // compaction counter for level h
uvarint count // items in buffer, 1 <= count < 2*s*k (see below)
f32[] buf // count values; ±Inf allowed, no NaNs
```

Writers sort `buf` ascending; readers must not rely on the order.

The section parameters `(s, k)` are a function of `state`:
starting at `s = 3`, `k = 40`, whenever `state >= 2^(s-1)` the number of
sections doubles and `k` is divided by `sqrt(2)`, rounded to the nearest even
integer (minimum 4):

| `state` | `s` | `k` |
| --------- | --- | --- |
| `< 2^2` | 3 | 40 |
| `< 2^5` | 6 | 28 |
| `< 2^11` | 12 | 20 |
| `< 2^23` | 24 | 14 |
| `< 2^47` | 48 | 10 |
| otherwise | 96 | 8 |

**Compaction.** When a level reaches `2*s*k` items, a prefix of its smallest
items is compacted in pairs into level `h + 1` (retaining the exact minimum at
`h = 0`; see the paper and `req.go`), and `state` is incremented. Cascading
keeps every level below `2*s*k` items.

**Estimates.** Total weight `N = Σ 2^h * count_h`. This implementation estimates
`quantile(p)` by linear interpolation at weighted rank `p * (N - 1)` (matching
`percentile_cont`), and `percent_rank(v)` as its inverse (interpolating across
the weight of the largest item `< v`, divided by `N - 1`).

**Merge.** Concatenate buffers level by level (bitwise-ORing `state`), then
compact from `h = 0`.

## FI sketch (top-k)

A weighted, mergeable [Misra-Gries](https://arxiv.org/abs/1705.07001)
frequent-items summary. A single `0x16` byte (`len == 1`) is a valid empty
sketch; otherwise:

```
byte header = 0x16
u16 k // target top-K, 1 <= k <= 1000
u16 c // capacity, k <= c <= 4000 (writers use max(128, 4*k))
u16 count // number of entries, 0 <= count <= c
f64 offset // total weight purged by compaction, >= 0 (+Inf allowed)
entry[] entries // count entries; trailing bytes are invalid
```

Each entry:

```
byte type // 1 = integer, 2 = float, 3 = text
f64 weight // > 0 (+Inf allowed)
value // depends on type
```

- **integer (`1`)**: `i64`. Any numeric value exactly representable as `int64`
(including `0.0` and `-0.0`) uses this type;
- **float (`2`)**: `f64`; `±Inf` allowed, must not be `NaN` or exactly
representable as `int64`;
- **text (`3`)**: `uvarint` byte length, followed by raw bytes (UTF-8 is not
validated on read).

Entries are unique by `(type, value)`. Writers sort entries by descending
`weight` (breaking ties by ascending SQLite value order); readers must not rely
on the order.

**Estimates.** Each entry's true weight lies in `[weight, weight + offset]`, and
this implementation reports the upper bound `weight + offset` (untracked items
have true weight at most `offset`). The top-k are the `k` heaviest entries.

**Compaction.** Writers compact to at most `c` entries before serializing. To
compact when entry count exceeds `c`, let `δ` be the `(c+1)`-th largest weight:
drop entries with `weight <= δ`, subtract `δ` from every remaining weight, and
add `δ` to `offset`. *(In memory, compaction is deferred until inserting a new
key at `2*c` entries or merging to `>= 2*c` entries.)*

**Merge.** The 1-byte empty sketch is the identity. Otherwise take
`k = min(k1, k2)`, `c = min(c1, c2)`, sum `offset`s, sum weights of matching
entries, and compact.
15 changes: 4 additions & 11 deletions ext/approx/hll.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ func (b *hllCombine) Step(ctx sqlite3.Context, arg ...sqlite3.Value) {
other, err := hllFromBytes(arg[0].RawBlob())
if err != nil {
ctx.ResultError(fmt.Errorf("approx: %w", err))
return
return // notest
}
if b.sketch == nil {
b.sketch = &other
Expand Down Expand Up @@ -168,7 +168,7 @@ func hllEstimate(ctx sqlite3.Context, arg ...sqlite3.Value) {
sketch, err := hllFromBytes(arg[0].RawBlob())
if err != nil {
ctx.ResultError(fmt.Errorf("approx: %w", err))
return
return // notest
}
ctx.ResultInt64(int64(sketch.Cardinality()))
default:
Expand All @@ -177,7 +177,7 @@ func hllEstimate(ctx sqlite3.Context, arg ...sqlite3.Value) {
}

func hllFromBytes(data []byte) (hll.Hll, error) {
if len(data) > 0 && data[0] == reqHeader {
if len(data) > 0 && (data[0] == reqHeader || data[0] == topHeader) {
return hll.Hll{}, errutil.ErrorString("not an HLL sketch")
}
return hll.FromBytes(data)
Expand All @@ -197,7 +197,7 @@ func hashValue(seed uint64, val sqlite3.Value) uint64 {
case sqlite3.FLOAT:
var u uint64
f := val.Float()
if i, ok := floatIsInt64(f); ok {
if i, ok := floatIsInt(f); ok {
u = uint64(i)
} else {
seed ^= 2
Expand All @@ -212,10 +212,3 @@ func hashValue(seed uint64, val sqlite3.Value) uint64 {
}
return murmur3.SeedSum64(seed, data)
}

func floatIsInt64(f float64) (int64, bool) {
// math.MaxInt64 is not exactly representable as a float64 (it rounds to 2^63),
// but int64(2^63) may saturate back to math.MaxInt64.
i := int64(f)
return i, float64(i) == f && i != math.MaxInt64
}
17 changes: 17 additions & 0 deletions ext/approx/pool.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package approx

import "sync"

var float64sPool sync.Pool

func getFloat64s(count int) *[]float64 {
if p, ok := float64sPool.Get().(*[]float64); ok && cap(*p) >= count {
return p
}
w := make([]float64, 0, count)
return &w
}

func returnFloat64s(p *[]float64) {
float64sPool.Put(p)
}
6 changes: 3 additions & 3 deletions ext/approx/req.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ func (b *reqSketch) evalResult(ctx sqlite3.Context, isQuantile bool) {
for i, a := range *b.args {
if err := check(a); err != nil {
ctx.ResultError(err)
return
return // notest
}
many[i] = eval(a)
}
Expand Down Expand Up @@ -184,7 +184,7 @@ func (b *reqCombine) Step(ctx sqlite3.Context, arg ...sqlite3.Value) {
other, err := reqFromBytes(arg[0].RawBlob())
if err != nil {
ctx.ResultError(err)
return
return // notest
}
if b.sketch == nil {
b.sketch = other
Expand Down Expand Up @@ -223,7 +223,7 @@ func reqEstimate(ctx sqlite3.Context, arg []sqlite3.Value, isQuantile bool) {
sketch, err := reqFromBytes(arg[0].RawBlob())
if err != nil {
ctx.ResultError(err)
return
return // notest
}
var b reqSketch
b.sketch = sketch
Expand Down
Loading
Loading