From 60429a691f9b37b6cd77f107c073c3fbc73b42b2 Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Fri, 13 Nov 2026 16:27:41 +0000 Subject: [PATCH 1/8] Approximate Top K. --- ext/approx/approx.go | 21 + ext/approx/hll.go | 2 +- ext/approx/top.go | 872 +++++++++++++++++++++++++++++++++++++++++ ext/approx/top_test.go | 479 ++++++++++++++++++++++ 4 files changed, 1373 insertions(+), 1 deletion(-) create mode 100644 ext/approx/top.go create mode 100644 ext/approx/top_test.go diff --git a/ext/approx/approx.go b/ext/approx/approx.go index ee291f1a..0b4186bf 100644 --- a/ext/approx/approx.go +++ b/ext/approx/approx.go @@ -11,6 +11,10 @@ // - approx_percentile_combine(B): merges REQ sketch BLOBs into a single sketch BLOB. // - approx_percentile_estimate(B, P): estimates quantile(s) P in [0, 1] from a REQ sketch BLOB. // - approx_percent_rank_estimate(B, V): estimates relative rank(s) in [0, 1] of V from a REQ sketch BLOB. +// - approx_top_k(X, [W,] K): approximate top K elements (and counts/sums) as JSON using a FI sketch. +// - approx_top_k_accumulate(X, [W,] K): accumulates elements into a FI sketch BLOB for up to K elements. +// - approx_top_k_combine(B): merges FI sketch BLOBs into a single sketch BLOB. +// - approx_top_k_estimate(B [, K]): estimates the top K elements as JSON from a FI sketch BLOB. // // [HyperLogLog] sketch BLOBs use the [Aggregate Knowledge HLL Storage Specification]. // @@ -19,8 +23,11 @@ // Rank error matches typical KLL sketches at the median, // and is at most 2x worse at the lower tail. // +// [Frequent Items] (FI) sketches use weighted mergeable Misra-Gries summaries. +// // [HyperLogLog]: https://github.com/segmentio/go-hll // [Relative Error Quantiles]: https://arxiv.org/abs/2004.01668 +// [Frequent Items]: https://arxiv.org/abs/1705.07001 // [Aggregate Knowledge HLL Storage Specification]: https://github.com/aggregateknowledge/hll-storage-spec package approx @@ -58,5 +65,19 @@ func Register(db *sqlite3.Conn) error { reqPercentileEstimate), db.CreateFunction("approx_percent_rank_estimate", 2, json, reqRankEstimate), + db.CreateWindowFunction("approx_top_k", 2, json, + sql3util.WindowFromCombiner[topSketch](1e4)), + db.CreateWindowFunction("approx_top_k", 3, json, + sql3util.WindowFromCombiner[topSketch](1e4)), + db.CreateWindowFunction("approx_top_k_accumulate", 2, flags, + sql3util.WindowFromCombiner[topAccumulate](1e4)), + db.CreateWindowFunction("approx_top_k_accumulate", 3, flags, + sql3util.WindowFromCombiner[topAccumulate](1e4)), + db.CreateWindowFunction("approx_top_k_combine", 1, flags, + sql3util.WindowFromCombiner[topCombine](1e4)), + db.CreateFunction("approx_top_k_estimate", 1, json, + topEstimate), + db.CreateFunction("approx_top_k_estimate", 2, json, + topEstimate), ) } diff --git a/ext/approx/hll.go b/ext/approx/hll.go index 9b49d623..7134b443 100644 --- a/ext/approx/hll.go +++ b/ext/approx/hll.go @@ -177,7 +177,7 @@ func hllEstimate(ctx sqlite3.Context, arg ...sqlite3.Value) { } func hllFromBytes(data []byte) (hll.Hll, error) { - if len(data) > 0 && data[0] == reqHeader { + if len(data) > 0 && (data[0] == reqHeader || data[0] == topHeader) { return hll.Hll{}, errutil.ErrorString("not an HLL sketch") } return hll.FromBytes(data) diff --git a/ext/approx/top.go b/ext/approx/top.go new file mode 100644 index 00000000..b5b5c221 --- /dev/null +++ b/ext/approx/top.go @@ -0,0 +1,872 @@ +package approx + +import ( + "cmp" + "encoding/binary" + "encoding/json" + "fmt" + "maps" + "math" + "slices" + "strconv" + "unsafe" + + "github.com/ncruces/go-sqlite3" + "github.com/ncruces/go-sqlite3/internal/errutil" + "github.com/ncruces/go-sqlite3/internal/util" + "github.com/ncruces/sort/quick" +) + +const ( + topHeader = 1<<4 | 6 // HLL version 1, with an unassigned storage type + topMaxK = 1000 + topMinC = 128 + topMaxC = 4 * topMaxK + topSummaryTag = 0xff + errInvalidTopK = errutil.ErrorString("approx: invalid top-K sketch") +) + +func topCap(k int16) int16 { + return max(topMinC, 4*k) +} + +type topSketch struct { + ptr unsafe.Pointer // nil (empty/num), unsafe.StringData(str) (TEXT), or *topSummary + num uint64 // int64 bits (INTEGER), float64 bits (FLOAT), or len(str) (TEXT) + weight float64 // inline weight in unit state (> 0) + k int16 // target K in [1, 1000] (0 if Step never ran) + typ uint8 // 0 (empty), INTEGER, FLOAT, TEXT, or topSummaryTag +} + +func (b *topSketch) summary() *topSummary { + return (*topSummary)(b.ptr) +} + +func (b *topSketch) unitStr() string { + if b.typ != uint8(sqlite3.TEXT) { + return "" + } + return unsafe.String((*byte)(b.ptr), int(b.num)) +} + +func (b *topSketch) Value(ctx sqlite3.Context) { + switch b.typ { + case 0: + return + case topSummaryTag: + b.summary().evalResult(ctx, int(b.summary().k)) + default: + var buf [64]byte + out := append(buf[:0], '[') + out = appendEntryJSON(out, topEntry{ + str: b.unitStr(), + num: b.num, + weight: b.weight, + typ: b.typ, + }, 0) + out = append(out, ']') + ctx.ResultRawText(out) + ctx.ResultSubtype('J') + } +} + +func (b *topSketch) Step(ctx sqlite3.Context, arg ...sqlite3.Value) { + if b.k == 0 && b.typ != topSummaryTag { + k := arg[len(arg)-1].Int() + if k < 1 || k > topMaxK { + ctx.ResultError(fmt.Errorf("approx: invalid k: %d", k)) + return + } + b.k = int16(k) + } + + t := arg[0].Type() + if t == sqlite3.BLOB { + ctx.ResultError(errutil.ErrorString("approx: JSON cannot hold BLOB values")) + return + } + + w := 1.0 + if len(arg) == 3 { + if arg[1].NumericType() > sqlite3.FLOAT { + return + } + w = arg[1].Float() + if !(w > 0) || math.IsInf(w, 1) { + if w < 0 || math.IsInf(w, 1) { + ctx.ResultError(fmt.Errorf("approx: invalid weight: %g", w)) + } + return + } + } + + switch t { + case sqlite3.NULL: + return + case sqlite3.INTEGER: + b.insertNum(uint8(sqlite3.INTEGER), uint64(arg[0].Int64()), w) + case sqlite3.FLOAT: + f := arg[0].Float() + if i, ok := floatIsInt64(f); ok { + b.insertNum(uint8(sqlite3.INTEGER), uint64(i), w) + } else { + b.insertNum(uint8(sqlite3.FLOAT), math.Float64bits(f), w) + } + case sqlite3.TEXT: + b.insertRawText(arg[0].RawText(), w) + } +} + +func (b *topSketch) insertNum(typ uint8, num uint64, w float64) { + switch b.typ { + case 0: + b.typ = typ + b.num = num + b.weight = w + case topSummaryTag: + b.summary().addNum(typ, num, w) + default: + if b.typ == typ && b.num == num { + b.weight += w + return + } + s := b.promote() + s.addNum(typ, num, w) + } +} + +func (b *topSketch) insertRawText(raw []byte, w float64) { + if b.typ == uint8(sqlite3.TEXT) && string(raw) == b.unitStr() { + b.weight += w + return + } + b.insertText(string(raw), w) +} + +func (b *topSketch) insertText(str string, w float64) { + switch b.typ { + case 0: + b.typ = uint8(sqlite3.TEXT) + b.ptr = unsafe.Pointer(unsafe.StringData(str)) + b.num = uint64(len(str)) + b.weight = w + case topSummaryTag: + b.summary().addText(str, w) + default: + if b.typ == uint8(sqlite3.TEXT) && str == b.unitStr() { + b.weight += w + return + } + s := b.promote() + s.addText(str, w) + } +} + +func (b *topSketch) promote() *topSummary { + s := &topSummary{k: b.k, c: topCap(b.k)} + if b.typ == uint8(sqlite3.TEXT) { + s.addText(b.unitStr(), b.weight) + } else { + s.addNum(b.typ, b.num, b.weight) + } + *b = topSketch{ + ptr: unsafe.Pointer(s), + typ: topSummaryTag, + } + return s +} + +func (b *topSketch) foldK(k int16) { + if k == 0 { + return + } + if b.typ == topSummaryTag { + s := b.summary() + if k < s.k { + s.k = k + if c := topCap(k); c < s.c { + s.c = c + if s.len() >= 2*int(s.c) { + s.purge() + } + } + } + return + } + if b.k == 0 || k < b.k { + b.k = k + } +} + +func (b *topSketch) Combine(other topSketch) { + switch other.typ { + case 0: + b.foldK(other.k) + case uint8(sqlite3.INTEGER), uint8(sqlite3.FLOAT): + b.foldK(other.k) + b.insertNum(other.typ, other.num, other.weight) + case uint8(sqlite3.TEXT): + b.foldK(other.k) + b.insertText(other.unitStr(), other.weight) + case topSummaryTag: + os := other.summary() + switch b.typ { + case 0: + s := os.clone() + if b.k > 0 && b.k < s.k { + s.k = b.k + s.c = min(s.c, topCap(b.k)) + if s.len() >= 2*int(s.c) { + s.purge() + } + } + *b = topSketch{ + ptr: unsafe.Pointer(s), + typ: topSummaryTag, + } + case topSummaryTag: + b.summary().merge(os) + default: + s := os.clone() + if b.k < s.k { + s.k = b.k + s.c = min(s.c, topCap(b.k)) + } + if b.typ == uint8(sqlite3.TEXT) { + s.addText(b.unitStr(), b.weight) + } else { + s.addNum(b.typ, b.num, b.weight) + } + *b = topSketch{ + ptr: unsafe.Pointer(s), + typ: topSummaryTag, + } + } + } +} + +type topAccumulate struct{ topSketch } + +func (b *topAccumulate) Value(ctx sqlite3.Context) { + var buf [16]byte // fits empty and single-row numeric sketches + ctx.ResultBlob(b.appendBytes(buf[:0])) +} + +func (b *topAccumulate) Combine(other topAccumulate) { + b.topSketch.Combine(other.topSketch) +} + +type topCombine struct { + sketch *topSummary +} + +func (b *topCombine) Value(ctx sqlite3.Context) { + if b.sketch != nil { + var buf [16]byte + ctx.ResultBlob(b.sketch.appendBytes(buf[:0])) + } +} + +func (b *topCombine) Step(ctx sqlite3.Context, arg ...sqlite3.Value) { + switch t := arg[0].Type(); t { + case sqlite3.NULL: + return + case sqlite3.BLOB: + other, err := topFromBytes(arg[0].RawBlob()) + if err != nil { + ctx.ResultError(err) + return + } + if b.sketch == nil { + b.sketch = other + return + } + b.sketch.merge(other) + default: + ctx.ResultError(fmt.Errorf("approx: invalid type: %v", t)) + } +} + +func (b *topCombine) Combine(other topCombine) { + if other.sketch == nil { + return + } + if b.sketch == nil { + b.sketch = other.sketch.clone() + return + } + b.sketch.merge(other.sketch) +} + +func topEstimate(ctx sqlite3.Context, arg ...sqlite3.Value) { + switch t := arg[0].Type(); t { + case sqlite3.NULL: + return + case sqlite3.BLOB: + sketch, err := topFromBytes(arg[0].RawBlob()) + if err != nil { + ctx.ResultError(err) + return + } + var k int + if len(arg) == 1 { + if sketch.k == 0 { + return + } + k = int(sketch.k) + } else { + k = arg[1].Int() + maxK := topMaxC + if sketch.c > 0 { + maxK = int(sketch.c) + } + if k < 1 || k > maxK { + ctx.ResultError(fmt.Errorf("approx: invalid k: %d", k)) + return + } + if sketch.k == 0 { + return + } + } + sketch.evalResult(ctx, k) + default: + ctx.ResultError(fmt.Errorf("approx: invalid type: %v", t)) + } +} + +// topSummary implements a weighted mergeable Misra-Gries frequent items summary +// (Anderson et al., IMC 2017). +type topSummary struct { + ints map[int64]float64 + reals map[float64]float64 + texts map[string]float64 + scratch []float64 + offset float64 + k, c int16 +} + +func (s *topSummary) len() int { + return len(s.ints) + len(s.reals) + len(s.texts) +} + +func (s *topSummary) addNum(typ uint8, num uint64, w float64) { + if typ == uint8(sqlite3.INTEGER) { + k := int64(num) + if s.ints == nil { + s.ints = make(map[int64]float64) + } else if cur, ok := s.ints[k]; ok { + s.ints[k] = cur + w + return + } + if s.len() >= 2*int(s.c) { + s.purge() + } + s.ints[k] += w + return + } + + k := math.Float64frombits(num) + if s.reals == nil { + s.reals = make(map[float64]float64) + } else if cur, ok := s.reals[k]; ok { + s.reals[k] = cur + w + return + } + if s.len() >= 2*int(s.c) { + s.purge() + } + s.reals[k] += w +} + +func (s *topSummary) addText(k string, w float64) { + if s.texts == nil { + s.texts = make(map[string]float64) + } else if cur, ok := s.texts[k]; ok { + s.texts[k] = cur + w + return + } + if s.len() >= 2*int(s.c) { + s.purge() + } + s.texts[k] += w +} + +func (s *topSummary) purge() { + n := s.len() + if n <= int(s.c) { + return + } + w := slices.Grow(s.scratch[:0], n) + for _, v := range s.ints { + w = append(w, v) + } + for _, v := range s.reals { + w = append(w, v) + } + for _, v := range s.texts { + w = append(w, v) + } + + delta := quick.Select(w, len(w)-int(s.c)-1) + s.offset += delta + for k, v := range s.ints { + if v <= delta { + delete(s.ints, k) + } else { + s.ints[k] = v - delta + } + } + for k, v := range s.reals { + if v <= delta { + delete(s.reals, k) + } else { + s.reals[k] = v - delta + } + } + for k, v := range s.texts { + if v <= delta { + delete(s.texts, k) + } else { + s.texts[k] = v - delta + } + } + s.scratch = w[:0] +} + +func (s *topSummary) clone() *topSummary { + return &topSummary{ + ints: maps.Clone(s.ints), + reals: maps.Clone(s.reals), + texts: maps.Clone(s.texts), + offset: s.offset, + k: s.k, + c: s.c, + } +} + +func (s *topSummary) merge(other *topSummary) { + if other.k == 0 { + return + } + if s.k == 0 { + *s = *other.clone() + return + } + s.k = min(s.k, other.k) + s.c = min(s.c, other.c) + s.offset += other.offset + if len(other.ints) > 0 { + if s.ints == nil { + s.ints = maps.Clone(other.ints) + } else { + for k, v := range other.ints { + s.ints[k] += v + } + } + } + if len(other.reals) > 0 { + if s.reals == nil { + s.reals = maps.Clone(other.reals) + } else { + for k, v := range other.reals { + s.reals[k] += v + } + } + } + if len(other.texts) > 0 { + if s.texts == nil { + s.texts = maps.Clone(other.texts) + } else { + for k, v := range other.texts { + s.texts[k] += v + } + } + } + if s.len() >= 2*int(s.c) { + s.purge() + } +} + +type topEntry struct { + str string + num uint64 + weight float64 + typ uint8 +} + +func cmpEntry(a, b topEntry) int { + if c := cmp.Compare(b.weight, a.weight); c != 0 { + return c + } + aText := a.typ == uint8(sqlite3.TEXT) + bText := b.typ == uint8(sqlite3.TEXT) + if aText != bText { + if aText { + return +1 + } + return -1 + } + if aText { + return cmp.Compare(a.str, b.str) + } + if a.typ == b.typ { + if a.typ == uint8(sqlite3.INTEGER) { + return cmp.Compare(int64(a.num), int64(b.num)) + } + return cmp.Compare(math.Float64frombits(a.num), math.Float64frombits(b.num)) + } + // Mixed INTEGER and non-integral FLOAT. + if a.typ == uint8(sqlite3.INTEGER) { + fi := float64(int64(a.num)) + f := math.Float64frombits(b.num) + if fi != f { + return cmp.Compare(fi, f) + } + return -1 // f == 2^63 > MaxInt64 + } + fi := float64(int64(b.num)) + f := math.Float64frombits(a.num) + if fi != f { + return cmp.Compare(f, fi) + } + return +1 +} + +func (s *topSummary) entries() ([]topEntry, float64) { + n := s.len() + if n == 0 { + return nil, s.offset + } + out := make([]topEntry, 0, n) + for k, v := range s.ints { + out = append(out, topEntry{num: uint64(k), weight: v, typ: uint8(sqlite3.INTEGER)}) + } + for k, v := range s.reals { + out = append(out, topEntry{num: math.Float64bits(k), weight: v, typ: uint8(sqlite3.FLOAT)}) + } + for k, v := range s.texts { + out = append(out, topEntry{str: k, weight: v, typ: uint8(sqlite3.TEXT)}) + } + offset := s.offset + if len(out) > int(s.c) { + w := slices.Grow(s.scratch[:0], len(out)) + for i := range out { + w = append(w, out[i].weight) + } + delta := quick.Select(w, len(w)-int(s.c)-1) + s.scratch = w[:0] + offset += delta + kept := 0 + for _, e := range out { + if e.weight > delta { + e.weight -= delta + out[kept] = e + kept++ + } + } + out = out[:kept] + } + slices.SortFunc(out, cmpEntry) + return out, offset +} + +func (s *topSummary) evalResult(ctx sqlite3.Context, k int) { + entries, offset := s.entries() + if len(entries) == 0 && offset == 0 { + return + } + entries = entries[:min(k, len(entries))] + var buf [256]byte + out := append(buf[:0], '[') + for i, e := range entries { + if i > 0 { + out = append(out, ',') + } + out = appendEntryJSON(out, e, offset) + } + out = append(out, ']') + ctx.ResultRawText(out) + ctx.ResultSubtype('J') +} + +func appendEntryJSON(dst []byte, e topEntry, offset float64) []byte { + dst = append(dst, '[') + switch sqlite3.Datatype(e.typ) { + case sqlite3.INTEGER: + dst = strconv.AppendInt(dst, int64(e.num), 10) + case sqlite3.FLOAT: + dst = appendJSONFloat(dst, math.Float64frombits(e.num)) + default: + dst = appendJSONString(dst, e.str) + } + dst = append(dst, ',') + dst = appendJSONFloat(dst, e.weight+offset) + return append(dst, ']') +} + +func appendJSONFloat(dst []byte, f float64) []byte { + if i, ok := floatIsInt64(f); ok { + return strconv.AppendInt(dst, i, 10) + } + return util.AppendNumber(dst, f) +} + +func appendJSONString(dst []byte, s string) []byte { + for i := range len(s) { + if c := s[i]; c < 0x20 || c >= 0x7f || c == '"' || c == '\\' { + b, _ := json.Marshal(s) + return append(dst, b...) + } + } + dst = append(dst, '"') + dst = append(dst, s...) + return append(dst, '"') +} + +const ( + topKeyInt = 0 + topKeyFloat = 1 + topKeyText = 2 + topWeightF = 1 << 2 +) + +func (b *topSketch) appendBytes(out []byte) []byte { + switch b.typ { + case 0: + if b.k == 0 { + return append(out, topHeader, 0) + } + out = append(out, topHeader) + out = binary.AppendUvarint(out, uint64(b.k)) + out = binary.AppendUvarint(out, uint64(topCap(b.k))) + return binary.AppendUvarint(out, 0) + case topSummaryTag: + return b.summary().appendBytes(out) + default: + out = append(out, topHeader) + out = binary.AppendUvarint(out, uint64(b.k)) + out = binary.AppendUvarint(out, uint64(topCap(b.k))) + out = binary.AppendUvarint(out, 1<<1) // count=1, hasOffset=0 + return appendEntry(out, topEntry{ + str: b.unitStr(), + num: b.num, + weight: b.weight, + typ: b.typ, + }) + } +} + +func (s *topSummary) appendBytes(out []byte) []byte { + if s.k == 0 { + return append(out, topHeader, 0) + } + entries, offset := s.entries() + out = slices.Grow(out, 6+12*len(entries)) + out = append(out, topHeader) + out = binary.AppendUvarint(out, uint64(s.k)) + out = binary.AppendUvarint(out, uint64(s.c)) + hdr := uint64(len(entries)) << 1 + if offset > 0 { + hdr |= 1 + } + out = binary.AppendUvarint(out, hdr) + if offset > 0 { + out = binary.LittleEndian.AppendUint64(out, math.Float64bits(offset)) + } + for _, e := range entries { + out = appendEntry(out, e) + } + return out +} + +func appendEntry(out []byte, e topEntry) []byte { + var tag byte + switch sqlite3.Datatype(e.typ) { + case sqlite3.INTEGER: + tag = topKeyInt + case sqlite3.FLOAT: + tag = topKeyFloat + default: + tag = topKeyText + } + wi, intWeight := floatIsInt64(e.weight) + if !intWeight || wi <= 0 { + tag |= topWeightF + } + out = append(out, tag) + + switch sqlite3.Datatype(e.typ) { + case sqlite3.INTEGER: + out = binary.AppendVarint(out, int64(e.num)) + case sqlite3.FLOAT: + out = binary.LittleEndian.AppendUint64(out, e.num) + default: + out = binary.AppendUvarint(out, uint64(len(e.str))) + out = append(out, e.str...) + } + + if tag&topWeightF == 0 { + out = binary.AppendUvarint(out, uint64(wi)) + } else { + out = binary.LittleEndian.AppendUint64(out, math.Float64bits(e.weight)) + } + return out +} + +func topFromBytes(data []byte) (*topSummary, error) { + if len(data) < 2 { + return nil, errutil.ErrorString("approx: insufficient bytes to deserialize top-K sketch") + } + if data[0] != topHeader { + return nil, errutil.ErrorString("approx: not a top-K sketch") + } + rest := data[1:] + + k, n := binary.Uvarint(rest) + if n <= 0 { + return nil, errInvalidTopK + } + rest = rest[n:] + if k == 0 { + if len(rest) != 0 { + return nil, errInvalidTopK + } + return &topSummary{}, nil + } + if k > topMaxK { + return nil, errInvalidTopK + } + + c, n := binary.Uvarint(rest) + if n <= 0 || c < k || c > topMaxC { + return nil, errInvalidTopK + } + rest = rest[n:] + + hdr, n := binary.Uvarint(rest) + if n <= 0 { + return nil, errInvalidTopK + } + rest = rest[n:] + + count := hdr >> 1 + hasOffset := hdr&1 != 0 + if count > c { + return nil, errInvalidTopK + } + + s := &topSummary{k: int16(k), c: int16(c)} + if hasOffset { + if len(rest) < 8 { + return nil, errInvalidTopK + } + off := math.Float64frombits(binary.LittleEndian.Uint64(rest)) + if !(off > 0) || math.IsInf(off, 1) { + return nil, errInvalidTopK + } + s.offset = off + rest = rest[8:] + } + if uint64(len(rest))/3 < count { + return nil, errInvalidTopK + } + + for range count { + if len(rest) == 0 { + return nil, errInvalidTopK + } + tag := rest[0] + rest = rest[1:] + if tag & ^byte(0x07) != 0 { + return nil, errInvalidTopK + } + + var ( + iKey int64 + fKey float64 + sKey string + ) + keyType := tag & 0x03 + switch keyType { + case topKeyInt: + v, n := binary.Varint(rest) + if n <= 0 { + return nil, errInvalidTopK + } + iKey = v + rest = rest[n:] + case topKeyFloat: + if len(rest) < 8 { + return nil, errInvalidTopK + } + f := math.Float64frombits(binary.LittleEndian.Uint64(rest)) + if math.IsNaN(f) || math.IsInf(f, 0) { + return nil, errInvalidTopK + } + if _, ok := floatIsInt64(f); ok { + return nil, errInvalidTopK + } + fKey = f + rest = rest[8:] + case topKeyText: + strLen, n := binary.Uvarint(rest) + if n <= 0 || uint64(len(rest)-n) < strLen { + return nil, errInvalidTopK + } + rest = rest[n:] + sKey = string(rest[:strLen]) + rest = rest[strLen:] + default: + return nil, errInvalidTopK + } + + var weight float64 + if tag&topWeightF == 0 { + u, n := binary.Uvarint(rest) + if n <= 0 || u == 0 { + return nil, errInvalidTopK + } + weight = float64(u) + rest = rest[n:] + } else { + if len(rest) < 8 { + return nil, errInvalidTopK + } + w := math.Float64frombits(binary.LittleEndian.Uint64(rest)) + if !(w > 0) || math.IsInf(w, 1) { + return nil, errInvalidTopK + } + weight = w + rest = rest[8:] + } + + switch keyType { + case topKeyInt: + if s.ints == nil { + s.ints = make(map[int64]float64) + } else if _, dup := s.ints[iKey]; dup { + return nil, errInvalidTopK + } + s.ints[iKey] = weight + case topKeyFloat: + if s.reals == nil { + s.reals = make(map[float64]float64) + } else if _, dup := s.reals[fKey]; dup { + return nil, errInvalidTopK + } + s.reals[fKey] = weight + case topKeyText: + if s.texts == nil { + s.texts = make(map[string]float64) + } else if _, dup := s.texts[sKey]; dup { + return nil, errInvalidTopK + } + s.texts[sKey] = weight + } + } + + if len(rest) != 0 { + return nil, errInvalidTopK + } + return s, nil +} diff --git a/ext/approx/top_test.go b/ext/approx/top_test.go new file mode 100644 index 00000000..1262ff6c --- /dev/null +++ b/ext/approx/top_test.go @@ -0,0 +1,479 @@ +package approx_test + +import ( + "math" + "testing" + + "github.com/ncruces/go-sqlite3" + "github.com/ncruces/go-sqlite3/internal/testcfg" +) + +func TestRegister_top(t *testing.T) { + t.Parallel() + + db, err := sqlite3.OpenContext(testcfg.Context(t), ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + err = db.Exec(`CREATE TABLE data (x, w)`) + if err != nil { + t.Fatal(err) + } + + // Empty table. + stmt, _, err := db.Prepare(` + SELECT + approx_top_k(x, 5), + approx_top_k(x, w, 5), + approx_top_k_estimate(approx_top_k_accumulate(x, 5)), + approx_top_k_estimate(approx_top_k_accumulate(x, w, 5), 50), + approx_top_k_combine(x), + approx_top_k_estimate(NULL), + approx_top_k_estimate(NULL, 5) + FROM data`) + if err != nil { + t.Fatal(err) + } + if !stmt.Step() { + t.Fatal(stmt.Err()) + } + for i := range 7 { + if got := stmt.ColumnType(i); got != sqlite3.NULL { + t.Errorf("col %d: got %v, want NULL", i, got) + } + } + stmt.Close() + + // Mixed types, normalization (0 == -0.0, 1 == 1.0, large ints), weights, ignored NULL/0/non-numeric weights, tie-breaking. + err = db.Exec(` + INSERT INTO data (x, w) VALUES + (NULL, 10), + ('skip_null_w', NULL), + ('skip_zero_w', 0), + ('skip_text_w', 'not_num'), + (0, 1.5), + (-0.0, '2.5'), + (1, 2), + (1.0, 2), + ('1', 4), + ('', 4), + (1.5, 4), + (9007199254740992, 1), + (9007199254740993, 3), + (9007199254740992.0, 2) + `) + if err != nil { + t.Fatal(err) + } + + stmt, _, err = db.Prepare(` + SELECT + approx_top_k(x, 4), + approx_top_k(x, w, 6), + approx_top_k_estimate(approx_top_k_accumulate(x, w, 6)), + approx_top_k_estimate(approx_top_k_accumulate(x, w, 6), 3), + json_object('top', approx_top_k(x, w, 2)) + FROM data`) + if err != nil { + t.Fatal(err) + } + if !stmt.Step() { + t.Fatal(stmt.Err()) + } + // Counts (unweighted): 0 -> 2, 1 -> 2, 9007199254740992 -> 2; ties at 1 broken by 1.5 < 9007199254740993 < "" < "1" < ... + const wantCount4 = `[[0,2],[1,2],[9007199254740992,2],[1.5,1]]` + if got := stmt.ColumnText(0); got != wantCount4 { + t.Errorf("col 0: got %s, want %s", got, wantCount4) + } + // Weighted sums: 0 -> 4, 1 -> 4, 1.5 -> 4, "" -> 4, "1" -> 4 (sorted: 0 < 1 < 1.5 < "" < "1"), then 9007199254740992 -> 3, 9007199254740993 -> 3. + const wantSum6 = `[[0,4],[1,4],[1.5,4],["",4],["1",4],[9007199254740992,3]]` + if got := stmt.ColumnText(1); got != wantSum6 { + t.Errorf("col 1: got %s, want %s", got, wantSum6) + } + if got := stmt.ColumnText(2); got != wantSum6 { + t.Errorf("col 2: got %s, want %s", got, wantSum6) + } + const wantSum3 = `[[0,4],[1,4],[1.5,4]]` + if got := stmt.ColumnText(3); got != wantSum3 { + t.Errorf("col 3: got %s, want %s", got, wantSum3) + } + const wantNested = `{"top":[[0,4],[1,4]]}` + if got := stmt.ColumnText(4); got != wantNested { + t.Errorf("col 4: got %s, want %s", got, wantNested) + } + stmt.Close() + + // Single-key unit states (int, float, empty text, repeated text) + JSON subtype. + unitTests := []struct { + sql, want string + }{ + {`SELECT json_array(approx_top_k(42, 5))`, `[[[42,1]]]`}, + {`SELECT approx_top_k_estimate(approx_top_k_accumulate(42, 5))`, `[[42,1]]`}, + {`SELECT approx_top_k(2.5, 1.25, 5)`, `[[2.5,1.25]]`}, + {`SELECT approx_top_k_estimate(approx_top_k_accumulate(2.5, 1.25, 5))`, `[[2.5,1.25]]`}, + {`SELECT approx_top_k('', 5)`, `[["",1]]`}, + {`SELECT approx_top_k_estimate(approx_top_k_accumulate('', 5))`, `[["",1]]`}, + {`SELECT approx_top_k('a"b\c' || char(10) || 'π', 5)`, `[["a\"b\\c\nπ",1]]`}, + {`SELECT approx_top_k('a', 5) FROM generate_series(1, 3)`, `[["a",3]]`}, + {`SELECT approx_top_k(value, 5) FROM (SELECT 1.5 AS value UNION ALL SELECT 1.5)`, `[[1.5,2]]`}, + } + for _, tc := range unitTests { + stmt, _, err := db.Prepare(tc.sql) + if err != nil { + t.Fatal(err) + } + if !stmt.Step() { + t.Fatal(stmt.Err()) + } + if got := stmt.ColumnText(0); got != tc.want { + t.Errorf("%q: got %s, want %s", tc.sql, got, tc.want) + } + stmt.Close() + } + + // Sliding window over small sequence. + err = db.Exec(` + CREATE TABLE seq (x); + INSERT INTO seq VALUES ('a'), ('b'), ('a'), (NULL), ('b'), ('b'), ('c'); + `) + if err != nil { + t.Fatal(err) + } + stmt, _, err = db.Prepare(` + SELECT + approx_top_k(x, 2) OVER w, + approx_top_k_estimate(approx_top_k_accumulate(x, 2) OVER w) + FROM seq + WINDOW w AS (ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING)`) + if err != nil { + t.Fatal(err) + } + wantWin := [...]string{ + `[["a",1],["b",1]]`, + `[["a",2],["b",1]]`, + `[["a",1],["b",1]]`, + `[["a",1],["b",1]]`, + `[["b",2]]`, + `[["b",2],["c",1]]`, + `[["b",1],["c",1]]`, + } + for i := 0; stmt.Step(); i++ { + if got := stmt.ColumnText(0); got != wantWin[i] { + t.Errorf("row %d col 0: got %s, want %s", i, got, wantWin[i]) + } + if got := stmt.ColumnText(1); got != wantWin[i] { + t.Errorf("row %d col 1: got %s, want %s", i, got, wantWin[i]) + } + } + stmt.Close() +} + +func TestRegister_top_large(t *testing.T) { + t.Parallel() + + db, err := sqlite3.OpenContext(testcfg.Context(t), ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + // 50,000 rows: 25 heavy hitters ('k1'..'k25' with counts 2500, 2400, ..., 100 = 32,500 rows) + // plus 17,500 distinct singleton tail items. + err = db.Exec(` + CREATE TABLE stream (id INT, item TEXT, w REAL); + WITH RECURSIVE hh(i, lo, hi) AS ( + SELECT 1, 1, 2500 + UNION ALL + SELECT i + 1, hi + 1, hi + (26 - (i + 1)) * 100 + FROM hh WHERE i < 25 + ) + INSERT INTO stream + SELECT s.value, + COALESCE((SELECT 'k' || hh.i FROM hh WHERE s.value BETWEEN hh.lo AND hh.hi), 'tail_' || s.value), + 1.5 + FROM generate_series(1, 50000) AS s; + `) + if err != nil { + t.Fatal(err) + } + + stmt, _, err := db.Prepare(` + SELECT + approx_top_k(item, 5), + approx_top_k(item, w, 5) + FROM stream`) + if err != nil { + t.Fatal(err) + } + if !stmt.Step() { + t.Fatal(stmt.Err()) + } + var gotCounts, gotSums [][2]any + if err := stmt.ColumnJSON(0, &gotCounts); err != nil { + t.Fatal(err) + } + if err := stmt.ColumnJSON(1, &gotSums); err != nil { + t.Fatal(err) + } + stmt.Close() + + wantKeys := []string{"k1", "k2", "k3", "k4", "k5"} + wantCounts := []float64{2500, 2400, 2300, 2200, 2100} + // With K=5, C=128, max theoretical error <= 50000 / 129 ≈ 388. + const maxCountErr = 50000.0 / 129.0 + if len(gotCounts) != 5 || len(gotSums) != 5 { + t.Fatalf("got lengths %d/%d, want 5/5", len(gotCounts), len(gotSums)) + } + for i := range 5 { + if gotCounts[i][0] != wantKeys[i] { + t.Errorf("count[%d] key: got %v, want %v", i, gotCounts[i][0], wantKeys[i]) + } + estC := gotCounts[i][1].(float64) + if estC < wantCounts[i] || estC-wantCounts[i] > maxCountErr { + t.Errorf("count[%d]: got %v, want in [%v, %v]", i, estC, wantCounts[i], wantCounts[i]+maxCountErr) + } + + if gotSums[i][0] != wantKeys[i] { + t.Errorf("sum[%d] key: got %v, want %v", i, gotSums[i][0], wantKeys[i]) + } + wantW := wantCounts[i] * 1.5 + estW := gotSums[i][1].(float64) + if estW < wantW-1e-9 || estW-wantW > maxCountErr*1.5+1e-9 { + t.Errorf("sum[%d]: got %v, want in [%v, %v]", i, estW, wantW, wantW+maxCountErr*1.5) + } + } + + // Accumulate / Combine across shards with DIFFERENT K (and C), empty sketch, and NULL sketch. + err = db.Exec(` + CREATE TABLE sketches (grp INT, sketch BLOB); + INSERT INTO sketches VALUES (0, NULL); + INSERT INTO sketches SELECT 1, approx_top_k_accumulate(item, 10) FROM stream WHERE id < 0; + INSERT INTO sketches SELECT 2, approx_top_k_accumulate(NULL, 10); + INSERT INTO sketches SELECT 3, approx_top_k_accumulate(item, 5) FROM stream WHERE id % 2 = 0; + INSERT INTO sketches SELECT 4, approx_top_k_accumulate(item, 50) FROM stream WHERE id % 2 = 1; + `) + if err != nil { + t.Fatal(err) + } + + // Combined sketch has min K = 5, min C = 128; 2-arg estimate allows querying up to C = 128 (e.g. K = 20). + stmt, _, err = db.Prepare(` + SELECT + approx_top_k_estimate(approx_top_k_combine(sketch) OVER ()), + approx_top_k_estimate(approx_top_k_combine(sketch) OVER (), 20), + approx_top_k_estimate(approx_top_k_combine(sketch) OVER (ORDER BY grp ROWS 2 PRECEDING)) + FROM sketches + ORDER BY grp`) + if err != nil { + t.Fatal(err) + } + for i := 0; stmt.Step(); i++ { + var rowTop5, rowTop20 [][2]any + if err := stmt.ColumnJSON(0, &rowTop5); err != nil { + t.Fatal(err) + } + if len(rowTop5) != 5 || rowTop5[0][0] != "k1" { + t.Errorf("row %d top5: got %v", i, rowTop5) + } + if err := stmt.ColumnJSON(1, &rowTop20); err != nil { + t.Fatal(err) + } + if len(rowTop20) != 20 || rowTop20[0][0] != "k1" { + t.Errorf("row %d top20: got len %d", i, len(rowTop20)) + } + if i < 3 { + if got := stmt.ColumnType(2); got != sqlite3.NULL { + t.Errorf("row %d window: got %v, want NULL", i, got) + } + } else { + var winTop [][2]any + if err := stmt.ColumnJSON(2, &winTop); err != nil { + t.Fatal(err) + } + if len(winTop) != 5 || winTop[0][0] != "k1" { + t.Errorf("row %d window: got %v", i, winTop) + } + } + } + stmt.Close() + + // All-singleton boundary purge (256 distinct items, C = 128 -> count = 0, offset = 1 -> "[]"). + stmt, _, err = db.Prepare(` + SELECT + approx_top_k(value, 5), + approx_top_k_estimate(approx_top_k_accumulate(value, 5)) + FROM generate_series(1, 256)`) + if err != nil { + t.Fatal(err) + } + if !stmt.Step() { + t.Fatal(stmt.Err()) + } + if got := stmt.ColumnText(0); got != `[]` { + t.Errorf("got %q, want []", got) + } + if got := stmt.ColumnText(1); got != `[]` { + t.Errorf("got %q, want []", got) + } + stmt.Close() + + // MaxInt64 vs 2^63 float ordering check (both orders). + stmt, _, err = db.Prepare(` + SELECT + approx_top_k(x, 4), + approx_top_k_estimate(approx_top_k_combine(s)) + FROM ( + SELECT 9223372036854775807 AS x, approx_top_k_accumulate(value, 5) AS s FROM generate_series(1, 300) + UNION ALL + SELECT 9223372036854775808.0, approx_top_k_accumulate(value + 0.5, 1.25, 5) FROM generate_series(1, 300) + UNION ALL + SELECT 1.5, approx_top_k_accumulate(1, 5) + UNION ALL + SELECT 2, approx_top_k_accumulate(2.5, 1.25, 5) + )`) + if err != nil { + t.Fatal(err) + } + if !stmt.Step() { + t.Fatal(stmt.Err()) + } + var gotBig [][2]float64 + if err := stmt.ColumnJSON(0, &gotBig); err != nil { + t.Fatal(err) + } + if len(gotBig) != 4 || gotBig[0][0] != 1.5 || gotBig[1][0] != 2 || math.IsNaN(gotBig[2][0]) { + t.Errorf("got %v", gotBig) + } + if got := stmt.ColumnType(1); got != sqlite3.TEXT { + t.Errorf("got %v, want TEXT", got) + } + stmt.Close() + + // Sliding window over >2 distinct items (exercises unit + summary Combine in twoStack.Inverse). + stmt, _, err = db.Prepare(` + SELECT approx_top_k(CASE WHEN value % 3 = 0 THEN value + 0.5 ELSE value END, value % 5 + 3) + OVER (ROWS BETWEEN 5 PRECEDING AND 1 FOLLOWING) + FROM generate_series(1, 30)`) + if err != nil { + t.Fatal(err) + } + for stmt.Step() { + if got := stmt.ColumnType(0); got != sqlite3.TEXT { + t.Errorf("got %v, want TEXT", got) + } + } + if err := stmt.Err(); err != nil { + t.Fatal(err) + } + stmt.Close() +} + +func TestRegister_top_errors(t *testing.T) { + t.Parallel() + + db, err := sqlite3.OpenContext(testcfg.Context(t), ":memory:") + if err != nil { + t.Fatal(err) + } + defer db.Close() + + tests := []string{ + `SELECT approx_top_k(1, 0)`, + `SELECT approx_top_k(1, -1)`, + `SELECT approx_top_k(1, 1001)`, + `SELECT approx_top_k(1, NULL)`, + `SELECT approx_top_k(1, 'abc')`, + `SELECT approx_top_k(NULL, 0)`, + `SELECT approx_top_k(X'01', 5)`, + `SELECT approx_top_k(1, -0.5, 5)`, + `SELECT approx_top_k(1, 1e999, 5)`, + `SELECT approx_top_k_accumulate(1, 0)`, + `SELECT approx_top_k_accumulate(1, 1001)`, + `SELECT approx_top_k_accumulate(X'01', 5)`, + `SELECT approx_top_k_accumulate(1, -1, 5)`, + `SELECT approx_top_k_estimate('not a blob')`, + `SELECT approx_top_k_estimate('not a blob', 5)`, + `SELECT approx_top_k_estimate(X'00')`, + `SELECT approx_top_k_estimate(X'16')`, + `SELECT approx_top_k_estimate(X'160001')`, // trailing byte after K=0 + `SELECT approx_top_k_estimate(X'1600', 0)`, // K=0 on empty sketch + `SELECT approx_top_k_estimate(X'1600', 4001)`, // K>topMaxC on empty sketch + `SELECT approx_top_k_estimate(X'16EA07800100')`, // K=1002 > topMaxK + `SELECT approx_top_k_estimate(X'160A0500')`, // C=5 < K=10 + `SELECT approx_top_k_estimate(X'1601A11F00')`, // C=4001 > topMaxC + `SELECT approx_top_k_estimate(X'160180018202')`, // count=129 > C=128 + `SELECT approx_top_k_estimate(X'1601800101')`, // hasOffset=1, missing offset + `SELECT approx_top_k_estimate(X'16018001010000000000000000')`, // hasOffset=1, offset=0 + `SELECT approx_top_k_estimate(X'1601800102080001')`, // reserved tag bit set (0x08) + `SELECT approx_top_k_estimate(X'1601800102030001')`, // invalid keyType=3 + `SELECT approx_top_k_estimate(X'1601800102000200')`, // integer weight u=0 + `SELECT approx_top_k_estimate(X'160180010201000000000000F03F01')`, // integral float 1.0 in FLOAT key + `SELECT approx_top_k_estimate(X'1601800104000201000201')`, // duplicate int key 1 + `SELECT approx_top_k_estimate(approx_top_k_accumulate(1, 5), 0)`, + `SELECT approx_top_k_estimate(approx_top_k_accumulate(1, 5), 129)`, // > C=128 + `SELECT approx_top_k_estimate(approx_count_distinct_accumulate(1))`, + `SELECT approx_top_k_estimate(approx_percentile_accumulate(1))`, + `SELECT approx_count_distinct_estimate(approx_top_k_accumulate(1, 5))`, + `SELECT approx_percentile_estimate(approx_top_k_accumulate(1, 5), 0.5)`, + `SELECT approx_top_k_combine('not a blob')`, + `SELECT approx_top_k_combine(X'00')`, + `SELECT approx_top_k_combine(approx_count_distinct_accumulate(1))`, + `SELECT approx_top_k_combine(approx_percentile_accumulate(1))`, + } + for _, sql := range tests { + if err := db.Exec(sql); err == nil { + t.Errorf("expected error for %q", sql) + } + } +} + +func Benchmark_top(b *testing.B) { + db, err := sqlite3.OpenContext(b.Context(), ":memory:") + if err != nil { + b.Fatal(err) + } + defer db.Close() + + err = db.Exec(` + CREATE TABLE data (grp10 INT, grp2 INT, x INT); + INSERT INTO data SELECT value / 10, value / 2, value % 500 FROM generate_series(0, 49999); + CREATE INDEX idx_grp10 ON data (grp10, x); + CREATE INDEX idx_grp2 ON data (grp2, x); + `) + if err != nil { + b.Fatal(err) + } + + queries := []struct { + name, sql string + }{ + {"agg_5k", `SELECT approx_top_k(x, 10) FROM data WHERE rowid <= 5000`}, + {"agg_50k", `SELECT approx_top_k(x, 10) FROM data`}, + {"group_2k_x2", `SELECT approx_top_k(x, 10) FROM data WHERE grp2 < 2000 GROUP BY grp2`}, + {"group_1k_x10", `SELECT approx_top_k(x, 10) FROM data WHERE grp10 < 1000 GROUP BY grp10`}, + {"win_20", `SELECT approx_top_k(x, 10) OVER (ROWS 20 PRECEDING) FROM data WHERE rowid <= 2000`}, + {"win_250", `SELECT approx_top_k(x, 10) OVER (ROWS 250 PRECEDING) FROM data WHERE rowid <= 2000`}, + {"win_1000", `SELECT approx_top_k(x, 10) OVER (ROWS 1000 PRECEDING) FROM data WHERE rowid <= 2000`}, + } + + for _, q := range queries { + b.Run(q.name, func(b *testing.B) { + b.ReportAllocs() + stmt, _, err := db.Prepare(q.sql) + if err != nil { + b.Fatal(err) + } + defer stmt.Close() + + for b.Loop() { + for stmt.Step() { + } + if err := stmt.Reset(); err != nil { + b.Fatal(err) + } + } + }) + } +} From 4ed0a6738608c7050c9acf7616ee98623fe9ba07 Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Fri, 13 Nov 2026 17:45:48 +0000 Subject: [PATCH 2/8] Fixes. --- ext/approx/top.go | 208 +++++++++++++++++------------------------ ext/approx/top_test.go | 13 ++- 2 files changed, 99 insertions(+), 122 deletions(-) diff --git a/ext/approx/top.go b/ext/approx/top.go index b5b5c221..08bce0cf 100644 --- a/ext/approx/top.go +++ b/ext/approx/top.go @@ -3,12 +3,12 @@ package approx import ( "cmp" "encoding/binary" - "encoding/json" "fmt" "maps" "math" "slices" "strconv" + "unicode/utf8" "unsafe" "github.com/ncruces/go-sqlite3" @@ -92,7 +92,7 @@ func (b *topSketch) Step(ctx sqlite3.Context, arg ...sqlite3.Value) { return } w = arg[1].Float() - if !(w > 0) || math.IsInf(w, 1) { + if w <= 0 || math.IsInf(w, 1) { if w < 0 || math.IsInf(w, 1) { ctx.ResultError(fmt.Errorf("approx: invalid weight: %g", w)) } @@ -176,62 +176,33 @@ func (b *topSketch) promote() *topSummary { return s } -func (b *topSketch) foldK(k int16) { - if k == 0 { - return - } - if b.typ == topSummaryTag { - s := b.summary() - if k < s.k { - s.k = k - if c := topCap(k); c < s.c { - s.c = c - if s.len() >= 2*int(s.c) { - s.purge() - } - } +func (b *topSketch) Combine(other topSketch) { + if other.typ == 0 { + if b.k == 0 && b.typ != topSummaryTag { + b.k = other.k } return } - if b.k == 0 || k < b.k { - b.k = k + if b.k == 0 && b.typ != topSummaryTag { + b.k = other.k } -} - -func (b *topSketch) Combine(other topSketch) { switch other.typ { - case 0: - b.foldK(other.k) case uint8(sqlite3.INTEGER), uint8(sqlite3.FLOAT): - b.foldK(other.k) b.insertNum(other.typ, other.num, other.weight) case uint8(sqlite3.TEXT): - b.foldK(other.k) b.insertText(other.unitStr(), other.weight) case topSummaryTag: os := other.summary() switch b.typ { case 0: - s := os.clone() - if b.k > 0 && b.k < s.k { - s.k = b.k - s.c = min(s.c, topCap(b.k)) - if s.len() >= 2*int(s.c) { - s.purge() - } - } *b = topSketch{ - ptr: unsafe.Pointer(s), + ptr: unsafe.Pointer(os.clone()), typ: topSummaryTag, } case topSummaryTag: b.summary().merge(os) default: s := os.clone() - if b.k < s.k { - s.k = b.k - s.c = min(s.c, topCap(b.k)) - } if b.typ == uint8(sqlite3.TEXT) { s.addText(b.unitStr(), b.weight) } else { @@ -351,44 +322,27 @@ func (s *topSummary) len() int { func (s *topSummary) addNum(typ uint8, num uint64, w float64) { if typ == uint8(sqlite3.INTEGER) { - k := int64(num) - if s.ints == nil { - s.ints = make(map[int64]float64) - } else if cur, ok := s.ints[k]; ok { - s.ints[k] = cur + w - return - } - if s.len() >= 2*int(s.c) { - s.purge() - } - s.ints[k] += w - return - } - - k := math.Float64frombits(num) - if s.reals == nil { - s.reals = make(map[float64]float64) - } else if cur, ok := s.reals[k]; ok { - s.reals[k] = cur + w - return - } - if s.len() >= 2*int(s.c) { - s.purge() + addMap(s, &s.ints, int64(num), w) + } else { + addMap(s, &s.reals, math.Float64frombits(num), w) } - s.reals[k] += w } func (s *topSummary) addText(k string, w float64) { - if s.texts == nil { - s.texts = make(map[string]float64) - } else if cur, ok := s.texts[k]; ok { - s.texts[k] = cur + w + addMap(s, &s.texts, k, w) +} + +func addMap[K comparable](s *topSummary, m *map[K]float64, k K, w float64) { + if *m == nil { + *m = make(map[K]float64) + } else if cur, ok := (*m)[k]; ok { + (*m)[k] = cur + w return } if s.len() >= 2*int(s.c) { s.purge() } - s.texts[k] += w + (*m)[k] = w } func (s *topSummary) purge() { @@ -409,28 +363,20 @@ func (s *topSummary) purge() { delta := quick.Select(w, len(w)-int(s.c)-1) s.offset += delta - for k, v := range s.ints { - if v <= delta { - delete(s.ints, k) - } else { - s.ints[k] = v - delta - } - } - for k, v := range s.reals { - if v <= delta { - delete(s.reals, k) - } else { - s.reals[k] = v - delta - } - } - for k, v := range s.texts { + purgeMap(s.ints, delta) + purgeMap(s.reals, delta) + purgeMap(s.texts, delta) + s.scratch = w[:0] +} + +func purgeMap[K comparable](m map[K]float64, delta float64) { + for k, v := range m { if v <= delta { - delete(s.texts, k) + delete(m, k) } else { - s.texts[k] = v - delta + m[k] = v - delta } } - s.scratch = w[:0] } func (s *topSummary) clone() *topSummary { @@ -455,35 +401,24 @@ func (s *topSummary) merge(other *topSummary) { s.k = min(s.k, other.k) s.c = min(s.c, other.c) s.offset += other.offset - if len(other.ints) > 0 { - if s.ints == nil { - s.ints = maps.Clone(other.ints) - } else { - for k, v := range other.ints { - s.ints[k] += v - } - } + mergeMap(&s.ints, other.ints) + mergeMap(&s.reals, other.reals) + mergeMap(&s.texts, other.texts) + if s.len() >= 2*int(s.c) { + s.purge() } - if len(other.reals) > 0 { - if s.reals == nil { - s.reals = maps.Clone(other.reals) - } else { - for k, v := range other.reals { - s.reals[k] += v - } - } +} + +func mergeMap[K comparable](dst *map[K]float64, src map[K]float64) { + if len(src) == 0 { + return } - if len(other.texts) > 0 { - if s.texts == nil { - s.texts = maps.Clone(other.texts) - } else { - for k, v := range other.texts { - s.texts[k] += v - } - } + if *dst == nil { + *dst = maps.Clone(src) + return } - if s.len() >= 2*int(s.c) { - s.purge() + for k, v := range src { + (*dst)[k] += v } } @@ -612,14 +547,47 @@ func appendJSONFloat(dst []byte, f float64) []byte { } func appendJSONString(dst []byte, s string) []byte { - for i := range len(s) { - if c := s[i]; c < 0x20 || c >= 0x7f || c == '"' || c == '\\' { - b, _ := json.Marshal(s) - return append(dst, b...) + dst = append(dst, '"') + start := 0 + for i := 0; i < len(s); { + if c := s[i]; c < utf8.RuneSelf { + if c >= 0x20 && c != '"' && c != '\\' { + i++ + continue + } + dst = append(dst, s[start:i]...) + switch c { + case '"', '\\': + dst = append(dst, '\\', c) + case '\b': + dst = append(dst, `\b`...) + case '\f': + dst = append(dst, `\f`...) + case '\n': + dst = append(dst, `\n`...) + case '\r': + dst = append(dst, `\r`...) + case '\t': + dst = append(dst, `\t`...) + default: + const hex = "0123456789abcdef" + dst = append(dst, '\\', 'u', '0', '0', hex[c>>4], hex[c&0xf]) + } + i++ + start = i + continue } + r, width := utf8.DecodeRuneInString(s[i:]) + if r == utf8.RuneError && width == 1 { + dst = append(dst, s[start:i]...) + dst = append(dst, `\ufffd`...) + i++ + start = i + continue + } + i += width } - dst = append(dst, '"') - dst = append(dst, s...) + dst = append(dst, s[start:]...) return append(dst, '"') } @@ -761,7 +729,7 @@ func topFromBytes(data []byte) (*topSummary, error) { return nil, errInvalidTopK } off := math.Float64frombits(binary.LittleEndian.Uint64(rest)) - if !(off > 0) || math.IsInf(off, 1) { + if !(off > 0) { return nil, errInvalidTopK } s.offset = off @@ -800,7 +768,7 @@ func topFromBytes(data []byte) (*topSummary, error) { return nil, errInvalidTopK } f := math.Float64frombits(binary.LittleEndian.Uint64(rest)) - if math.IsNaN(f) || math.IsInf(f, 0) { + if math.IsNaN(f) { return nil, errInvalidTopK } if _, ok := floatIsInt64(f); ok { @@ -833,7 +801,7 @@ func topFromBytes(data []byte) (*topSummary, error) { return nil, errInvalidTopK } w := math.Float64frombits(binary.LittleEndian.Uint64(rest)) - if !(w > 0) || math.IsInf(w, 1) { + if !(w > 0) { return nil, errInvalidTopK } weight = w diff --git a/ext/approx/top_test.go b/ext/approx/top_test.go index 1262ff6c..fb610987 100644 --- a/ext/approx/top_test.go +++ b/ext/approx/top_test.go @@ -115,7 +115,16 @@ func TestRegister_top(t *testing.T) { {`SELECT approx_top_k_estimate(approx_top_k_accumulate(2.5, 1.25, 5))`, `[[2.5,1.25]]`}, {`SELECT approx_top_k('', 5)`, `[["",1]]`}, {`SELECT approx_top_k_estimate(approx_top_k_accumulate('', 5))`, `[["",1]]`}, - {`SELECT approx_top_k('a"b\c' || char(10) || 'π', 5)`, `[["a\"b\\c\nπ",1]]`}, + {`SELECT approx_top_k('a2 distinct items (exercises unit + summary Combine in twoStack.Inverse). stmt, _, err = db.Prepare(` - SELECT approx_top_k(CASE WHEN value % 3 = 0 THEN value + 0.5 ELSE value END, value % 5 + 3) + SELECT approx_top_k(CASE WHEN value % 3 = 0 THEN value + 0.5 ELSE value END, 5) OVER (ROWS BETWEEN 5 PRECEDING AND 1 FOLLOWING) FROM generate_series(1, 30)`) if err != nil { From 530d517f0e08892d10556ce9052e1664ffe2763b Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Fri, 13 Nov 2026 17:54:43 +0000 Subject: [PATCH 3/8] JSON. --- ext/approx/top.go | 67 +++++++++------------------------------- ext/approx/top_test.go | 24 ++++++++++++-- internal/util/json.go | 12 +++++++ internal/util/json_v2.go | 5 +++ 4 files changed, 53 insertions(+), 55 deletions(-) diff --git a/ext/approx/top.go b/ext/approx/top.go index 08bce0cf..7ff6ab6b 100644 --- a/ext/approx/top.go +++ b/ext/approx/top.go @@ -8,7 +8,6 @@ import ( "math" "slices" "strconv" - "unicode/utf8" "unsafe" "github.com/ncruces/go-sqlite3" @@ -57,13 +56,16 @@ func (b *topSketch) Value(ctx sqlite3.Context) { b.summary().evalResult(ctx, int(b.summary().k)) default: var buf [64]byte - out := append(buf[:0], '[') - out = appendEntryJSON(out, topEntry{ + out, err := appendEntryJSON(append(buf[:0], '['), topEntry{ str: b.unitStr(), num: b.num, weight: b.weight, typ: b.typ, }, 0) + if err != nil { + ctx.ResultError(err) + return + } out = append(out, ']') ctx.ResultRawText(out) ctx.ResultSubtype('J') @@ -517,14 +519,20 @@ func (s *topSummary) evalResult(ctx sqlite3.Context, k int) { if i > 0 { out = append(out, ',') } - out = appendEntryJSON(out, e, offset) + var err error + out, err = appendEntryJSON(out, e, offset) + if err != nil { + ctx.ResultError(err) + return + } } out = append(out, ']') ctx.ResultRawText(out) ctx.ResultSubtype('J') } -func appendEntryJSON(dst []byte, e topEntry, offset float64) []byte { +func appendEntryJSON(dst []byte, e topEntry, offset float64) ([]byte, error) { + var err error dst = append(dst, '[') switch sqlite3.Datatype(e.typ) { case sqlite3.INTEGER: @@ -532,11 +540,11 @@ func appendEntryJSON(dst []byte, e topEntry, offset float64) []byte { case sqlite3.FLOAT: dst = appendJSONFloat(dst, math.Float64frombits(e.num)) default: - dst = appendJSONString(dst, e.str) + dst, err = util.AppendQuote(dst, e.str) } dst = append(dst, ',') dst = appendJSONFloat(dst, e.weight+offset) - return append(dst, ']') + return append(dst, ']'), err } func appendJSONFloat(dst []byte, f float64) []byte { @@ -546,51 +554,6 @@ func appendJSONFloat(dst []byte, f float64) []byte { return util.AppendNumber(dst, f) } -func appendJSONString(dst []byte, s string) []byte { - dst = append(dst, '"') - start := 0 - for i := 0; i < len(s); { - if c := s[i]; c < utf8.RuneSelf { - if c >= 0x20 && c != '"' && c != '\\' { - i++ - continue - } - dst = append(dst, s[start:i]...) - switch c { - case '"', '\\': - dst = append(dst, '\\', c) - case '\b': - dst = append(dst, `\b`...) - case '\f': - dst = append(dst, `\f`...) - case '\n': - dst = append(dst, `\n`...) - case '\r': - dst = append(dst, `\r`...) - case '\t': - dst = append(dst, `\t`...) - default: - const hex = "0123456789abcdef" - dst = append(dst, '\\', 'u', '0', '0', hex[c>>4], hex[c&0xf]) - } - i++ - start = i - continue - } - r, width := utf8.DecodeRuneInString(s[i:]) - if r == utf8.RuneError && width == 1 { - dst = append(dst, s[start:i]...) - dst = append(dst, `\ufffd`...) - i++ - start = i - continue - } - i += width - } - dst = append(dst, s[start:]...) - return append(dst, '"') -} - const ( topKeyInt = 0 topKeyFloat = 1 diff --git a/ext/approx/top_test.go b/ext/approx/top_test.go index fb610987..97f4ac92 100644 --- a/ext/approx/top_test.go +++ b/ext/approx/top_test.go @@ -2,10 +2,12 @@ package approx_test import ( "math" + "strings" "testing" "github.com/ncruces/go-sqlite3" "github.com/ncruces/go-sqlite3/internal/testcfg" + "github.com/ncruces/go-sqlite3/internal/util" ) func TestRegister_top(t *testing.T) { @@ -106,6 +108,11 @@ func TestRegister_top(t *testing.T) { stmt.Close() // Single-key unit states (int, float, empty text, repeated text) + JSON subtype. + qASCII, _ := util.AppendQuote(nil, "a= 0x7f || c == '"' || c == '\\' || c == '<' || c == '>' || c == '&' { + b, err := json.Marshal(src) + return append(dst, b...), err + } + } + dst = append(dst, '"') + dst = append(dst, src...) + return append(dst, '"'), nil +} diff --git a/internal/util/json_v2.go b/internal/util/json_v2.go index 8ac7220f..b25609be 100644 --- a/internal/util/json_v2.go +++ b/internal/util/json_v2.go @@ -3,6 +3,7 @@ package util import ( + "encoding/json/jsontext" "encoding/json/v2" "math" "strconv" @@ -52,3 +53,7 @@ func AppendNumber(dst []byte, f float64) []byte { } return dst } + +func AppendQuote(dst []byte, src string) ([]byte, error) { + return jsontext.AppendQuote(dst, src) +} From 46360cb5f8f1c35b23c5f76aa319c8239949e2bd Mon Sep 17 00:00:00 2001 From: ncruces <5637550+ncruces@users.noreply.github.com> Date: Fri, 13 Nov 2026 18:56:56 +0000 Subject: [PATCH 4/8] JSON. --- ext/approx/top_test.go | 26 +++++++++++++------------- internal/util/json.go | 20 +++++++++++--------- json.go | 14 +++++--------- 3 files changed, 29 insertions(+), 31 deletions(-) diff --git a/ext/approx/top_test.go b/ext/approx/top_test.go index 97f4ac92..ff1f2cdf 100644 --- a/ext/approx/top_test.go +++ b/ext/approx/top_test.go @@ -424,20 +424,20 @@ func TestRegister_top_errors(t *testing.T) { `SELECT approx_top_k_estimate('not a blob', 5)`, `SELECT approx_top_k_estimate(X'00')`, `SELECT approx_top_k_estimate(X'16')`, - `SELECT approx_top_k_estimate(X'160001')`, // trailing byte after K=0 - `SELECT approx_top_k_estimate(X'1600', 0)`, // K=0 on empty sketch - `SELECT approx_top_k_estimate(X'1600', 4001)`, // K>topMaxC on empty sketch - `SELECT approx_top_k_estimate(X'16EA07800100')`, // K=1002 > topMaxK - `SELECT approx_top_k_estimate(X'160A0500')`, // C=5 < K=10 - `SELECT approx_top_k_estimate(X'1601A11F00')`, // C=4001 > topMaxC - `SELECT approx_top_k_estimate(X'160180018202')`, // count=129 > C=128 - `SELECT approx_top_k_estimate(X'1601800101')`, // hasOffset=1, missing offset - `SELECT approx_top_k_estimate(X'16018001010000000000000000')`, // hasOffset=1, offset=0 - `SELECT approx_top_k_estimate(X'1601800102080001')`, // reserved tag bit set (0x08) - `SELECT approx_top_k_estimate(X'1601800102030001')`, // invalid keyType=3 - `SELECT approx_top_k_estimate(X'1601800102000200')`, // integer weight u=0 + `SELECT approx_top_k_estimate(X'160001')`, // trailing byte after K=0 + `SELECT approx_top_k_estimate(X'1600', 0)`, // K=0 on empty sketch + `SELECT approx_top_k_estimate(X'1600', 4001)`, // K>topMaxC on empty sketch + `SELECT approx_top_k_estimate(X'16EA07800100')`, // K=1002 > topMaxK + `SELECT approx_top_k_estimate(X'160A0500')`, // C=5 < K=10 + `SELECT approx_top_k_estimate(X'1601A11F00')`, // C=4001 > topMaxC + `SELECT approx_top_k_estimate(X'160180018202')`, // count=129 > C=128 + `SELECT approx_top_k_estimate(X'1601800101')`, // hasOffset=1, missing offset + `SELECT approx_top_k_estimate(X'16018001010000000000000000')`, // hasOffset=1, offset=0 + `SELECT approx_top_k_estimate(X'1601800102080001')`, // reserved tag bit set (0x08) + `SELECT approx_top_k_estimate(X'1601800102030001')`, // invalid keyType=3 + `SELECT approx_top_k_estimate(X'1601800102000200')`, // integer weight u=0 `SELECT approx_top_k_estimate(X'160180010201000000000000F03F01')`, // integral float 1.0 in FLOAT key - `SELECT approx_top_k_estimate(X'1601800104000201000201')`, // duplicate int key 1 + `SELECT approx_top_k_estimate(X'1601800104000201000201')`, // duplicate int key 1 `SELECT approx_top_k_estimate(approx_top_k_accumulate(1, 5), 0)`, `SELECT approx_top_k_estimate(approx_top_k_accumulate(1, 5), 129)`, // > C=128 `SELECT approx_top_k_estimate(approx_count_distinct_accumulate(1))`, diff --git a/internal/util/json.go b/internal/util/json.go index 0115fe9d..8e27dac1 100644 --- a/internal/util/json.go +++ b/internal/util/json.go @@ -54,13 +54,15 @@ func AppendNumber(dst []byte, f float64) []byte { } func AppendQuote(dst []byte, src string) ([]byte, error) { - for i := range len(src) { - if c := src[i]; c < 0x20 || c >= 0x7f || c == '"' || c == '\\' || c == '<' || c == '>' || c == '&' { - b, err := json.Marshal(src) - return append(dst, b...), err - } - } - dst = append(dst, '"') - dst = append(dst, src...) - return append(dst, '"'), nil + err := json.NewEncoder(JSONWriter(func(p []byte) error { + dst = append(dst, p...) + return nil + })).Encode(src) + return dst, err +} + +type JSONWriter func(p []byte) error + +func (fn JSONWriter) Write(p []byte) (int, error) { + return len(p), fn(p[:len(p)-1]) // remove the newline } diff --git a/json.go b/json.go index e23148ec..1591b33b 100644 --- a/json.go +++ b/json.go @@ -27,9 +27,9 @@ func JSON(value any) any { // // https://sqlite.org/c3ref/result_blob.html func (ctx Context) ResultJSON(value any) { - err := json.NewEncoder(callbackWriter(func(p []byte) (int, error) { - ctx.ResultRawText(p[:len(p)-1]) // remove the newline - return 0, nil + err := json.NewEncoder(util.JSONWriter(func(p []byte) error { + ctx.ResultRawText(p) + return nil })).Encode(value) if err != nil { @@ -43,8 +43,8 @@ func (ctx Context) ResultJSON(value any) { // // https://sqlite.org/c3ref/bind_blob.html func (s *Stmt) BindJSON(param int, value any) error { - return json.NewEncoder(callbackWriter(func(p []byte) (int, error) { - return 0, s.BindRawText(param, p[:len(p)-1]) // remove the newline + return json.NewEncoder(util.JSONWriter(func(p []byte) error { + return s.BindRawText(param, p) })).Encode(value) } @@ -93,8 +93,4 @@ func (v Value) JSON(ptr any) error { return json.Unmarshal(data, ptr) } -type callbackWriter func(p []byte) (int, error) - -func (fn callbackWriter) Write(p []byte) (int, error) { return fn(p) } - func (c *Conn) uuid() error { return nil } From 1df5a664dea35c97b8ce376509b502f437852c05 Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Sat, 14 Nov 2026 09:21:56 +0000 Subject: [PATCH 5/8] Tweaks. --- ext/approx/approx.go | 17 ++ ext/approx/hll.go | 9 +- ext/approx/top.go | 538 ++++++++++++++--------------------------- ext/approx/top_test.go | 159 ++---------- 4 files changed, 223 insertions(+), 500 deletions(-) diff --git a/ext/approx/approx.go b/ext/approx/approx.go index 0b4186bf..306a5f45 100644 --- a/ext/approx/approx.go +++ b/ext/approx/approx.go @@ -32,7 +32,9 @@ package approx import ( + "cmp" "errors" + "math" "github.com/ncruces/go-sqlite3" "github.com/ncruces/go-sqlite3/util/sql3util" @@ -81,3 +83,18 @@ func Register(db *sqlite3.Conn) error { topEstimate), ) } + +func floatIsInt(f float64) (int64, bool) { + // math.MaxInt64 is not exactly representable as a float64 (it rounds to 2^63), + // but int64(2^63) may saturate back to math.MaxInt64. + i := int64(f) + return i, float64(i) == f && i != math.MaxInt64 +} + +func cmpIntFloat(i int64, f float64) int { + // Assumes both floatIsInt(f) and IsNan(f) are false. + if c := cmp.Compare(float64(i), f); c != 0 { + return c + } + return -1 // float64(i) == f means f == 2^63 +} diff --git a/ext/approx/hll.go b/ext/approx/hll.go index 7134b443..b4764f91 100644 --- a/ext/approx/hll.go +++ b/ext/approx/hll.go @@ -197,7 +197,7 @@ func hashValue(seed uint64, val sqlite3.Value) uint64 { case sqlite3.FLOAT: var u uint64 f := val.Float() - if i, ok := floatIsInt64(f); ok { + if i, ok := floatIsInt(f); ok { u = uint64(i) } else { seed ^= 2 @@ -212,10 +212,3 @@ func hashValue(seed uint64, val sqlite3.Value) uint64 { } return murmur3.SeedSum64(seed, data) } - -func floatIsInt64(f float64) (int64, bool) { - // math.MaxInt64 is not exactly representable as a float64 (it rounds to 2^63), - // but int64(2^63) may saturate back to math.MaxInt64. - i := int64(f) - return i, float64(i) == f && i != math.MaxInt64 -} diff --git a/ext/approx/top.go b/ext/approx/top.go index 7ff6ab6b..c9e0a295 100644 --- a/ext/approx/top.go +++ b/ext/approx/top.go @@ -16,12 +16,19 @@ import ( "github.com/ncruces/sort/quick" ) +type topTag uint8 + const ( - topHeader = 1<<4 | 6 // HLL version 1, with an unassigned storage type - topMaxK = 1000 - topMinC = 128 - topMaxC = 4 * topMaxK - topSummaryTag = 0xff + topHeader = 1<<4 | 6 // HLL version 1, with an unassigned storage type + topMaxK = 1000 + topMinC = 128 + topMaxC = 4 * topMaxK + + topIntTag topTag = 1 + topRealTag topTag = 2 + topTextTag topTag = 3 + topSummaryTag topTag = 0xff + errInvalidTopK = errutil.ErrorString("approx: invalid top-K sketch") ) @@ -30,22 +37,22 @@ func topCap(k int16) int16 { } type topSketch struct { - ptr unsafe.Pointer // nil (empty/num), unsafe.StringData(str) (TEXT), or *topSummary - num uint64 // int64 bits (INTEGER), float64 bits (FLOAT), or len(str) (TEXT) + ptr unsafe.Pointer // nil, unsafe.StringData(str), or *topSummary + bits uint64 // int64, float64, or len(str) weight float64 // inline weight in unit state (> 0) - k int16 // target K in [1, 1000] (0 if Step never ran) - typ uint8 // 0 (empty), INTEGER, FLOAT, TEXT, or topSummaryTag + k int16 // target K, or 0 if Step never ran + typ topTag } func (b *topSketch) summary() *topSummary { return (*topSummary)(b.ptr) } -func (b *topSketch) unitStr() string { - if b.typ != uint8(sqlite3.TEXT) { +func (b *topSketch) string() string { + if b.typ != topTextTag { return "" } - return unsafe.String((*byte)(b.ptr), int(b.num)) + return unsafe.String((*byte)(b.ptr), int(b.bits)) } func (b *topSketch) Value(ctx sqlite3.Context) { @@ -53,22 +60,16 @@ func (b *topSketch) Value(ctx sqlite3.Context) { case 0: return case topSummaryTag: - b.summary().evalResult(ctx, int(b.summary().k)) + s := b.summary() + entries, offset := s.entries() + evalEntries(ctx, entries, offset, int(s.k)) default: - var buf [64]byte - out, err := appendEntryJSON(append(buf[:0], '['), topEntry{ - str: b.unitStr(), - num: b.num, + evalEntries(ctx, []topEntry{{ + str: b.string(), + num: b.bits, weight: b.weight, typ: b.typ, - }, 0) - if err != nil { - ctx.ResultError(err) - return - } - out = append(out, ']') - ctx.ResultRawText(out) - ctx.ResultSubtype('J') + }}, 0, 1) } } @@ -106,123 +107,88 @@ func (b *topSketch) Step(ctx sqlite3.Context, arg ...sqlite3.Value) { case sqlite3.NULL: return case sqlite3.INTEGER: - b.insertNum(uint8(sqlite3.INTEGER), uint64(arg[0].Int64()), w) + b.insert(topIntTag, uint64(arg[0].Int64()), "", w) case sqlite3.FLOAT: f := arg[0].Float() - if i, ok := floatIsInt64(f); ok { - b.insertNum(uint8(sqlite3.INTEGER), uint64(i), w) + if i, ok := floatIsInt(f); ok { + b.insert(topIntTag, uint64(i), "", w) } else { - b.insertNum(uint8(sqlite3.FLOAT), math.Float64bits(f), w) + b.insert(topRealTag, math.Float64bits(f), "", w) } case sqlite3.TEXT: - b.insertRawText(arg[0].RawText(), w) - } -} - -func (b *topSketch) insertNum(typ uint8, num uint64, w float64) { - switch b.typ { - case 0: - b.typ = typ - b.num = num - b.weight = w - case topSummaryTag: - b.summary().addNum(typ, num, w) - default: - if b.typ == typ && b.num == num { + raw := arg[0].RawText() + if b.typ == topTextTag && string(raw) == b.string() { b.weight += w - return + } else { + b.insert(topTextTag, uint64(len(raw)), string(raw), w) } - s := b.promote() - s.addNum(typ, num, w) - } -} - -func (b *topSketch) insertRawText(raw []byte, w float64) { - if b.typ == uint8(sqlite3.TEXT) && string(raw) == b.unitStr() { - b.weight += w - return } - b.insertText(string(raw), w) } -func (b *topSketch) insertText(str string, w float64) { +func (b *topSketch) insert(typ topTag, num uint64, str string, w float64) { switch b.typ { case 0: - b.typ = uint8(sqlite3.TEXT) - b.ptr = unsafe.Pointer(unsafe.StringData(str)) - b.num = uint64(len(str)) + b.typ = typ + b.bits = num + if typ == topTextTag { + b.ptr = unsafe.Pointer(unsafe.StringData(str)) + } b.weight = w case topSummaryTag: - b.summary().addText(str, w) + b.summary().add(typ, num, str, w) default: - if b.typ == uint8(sqlite3.TEXT) && str == b.unitStr() { + if b.typ == typ && (typ == topTextTag && str == b.string() || typ != topTextTag && num == b.bits) { b.weight += w return } - s := b.promote() - s.addText(str, w) + s := &topSummary{k: b.k, c: topCap(b.k)} + s.add(b.typ, b.bits, b.string(), b.weight) + s.add(typ, num, str, w) + *b = topSketch{ptr: unsafe.Pointer(s), typ: topSummaryTag} } } -func (b *topSketch) promote() *topSummary { - s := &topSummary{k: b.k, c: topCap(b.k)} - if b.typ == uint8(sqlite3.TEXT) { - s.addText(b.unitStr(), b.weight) - } else { - s.addNum(b.typ, b.num, b.weight) - } - *b = topSketch{ - ptr: unsafe.Pointer(s), - typ: topSummaryTag, - } - return s -} - func (b *topSketch) Combine(other topSketch) { - if other.typ == 0 { - if b.k == 0 && b.typ != topSummaryTag { - b.k = other.k - } - return - } if b.k == 0 && b.typ != topSummaryTag { b.k = other.k } switch other.typ { - case uint8(sqlite3.INTEGER), uint8(sqlite3.FLOAT): - b.insertNum(other.typ, other.num, other.weight) - case uint8(sqlite3.TEXT): - b.insertText(other.unitStr(), other.weight) + case 0: + return case topSummaryTag: os := other.summary() switch b.typ { case 0: - *b = topSketch{ - ptr: unsafe.Pointer(os.clone()), - typ: topSummaryTag, - } + *b = topSketch{ptr: unsafe.Pointer(os.clone()), typ: topSummaryTag} case topSummaryTag: b.summary().merge(os) default: s := os.clone() - if b.typ == uint8(sqlite3.TEXT) { - s.addText(b.unitStr(), b.weight) - } else { - s.addNum(b.typ, b.num, b.weight) - } - *b = topSketch{ - ptr: unsafe.Pointer(s), - typ: topSummaryTag, - } + s.add(b.typ, b.bits, b.string(), b.weight) + *b = topSketch{ptr: unsafe.Pointer(s), typ: topSummaryTag} } + default: + b.insert(other.typ, other.bits, other.string(), other.weight) } } type topAccumulate struct{ topSketch } func (b *topAccumulate) Value(ctx sqlite3.Context) { - var buf [16]byte // fits empty and single-row numeric sketches - ctx.ResultBlob(b.appendBytes(buf[:0])) + var buf [32]byte // fits empty and single-row numeric sketches + switch b.typ { + case 0: + ctx.ResultBlob(appendSketchBytes(buf[:0], nil, 0, b.k, topCap(b.k))) + case topSummaryTag: + ctx.ResultBlob(b.summary().appendBytes(buf[:0])) + default: + ctx.ResultBlob(appendSketchBytes(buf[:0], []topEntry{{ + str: b.string(), + num: b.bits, + weight: b.weight, + typ: b.typ, + }}, 0, b.k, topCap(b.k))) + } } func (b *topAccumulate) Combine(other topAccumulate) { @@ -235,7 +201,7 @@ type topCombine struct { func (b *topCombine) Value(ctx sqlite3.Context) { if b.sketch != nil { - var buf [16]byte + var buf [32]byte ctx.ResultBlob(b.sketch.appendBytes(buf[:0])) } } @@ -301,7 +267,8 @@ func topEstimate(ctx sqlite3.Context, arg ...sqlite3.Value) { return } } - sketch.evalResult(ctx, k) + entries, offset := sketch.entries() + evalEntries(ctx, entries, offset, k) default: ctx.ResultError(fmt.Errorf("approx: invalid type: %v", t)) } @@ -322,18 +289,17 @@ func (s *topSummary) len() int { return len(s.ints) + len(s.reals) + len(s.texts) } -func (s *topSummary) addNum(typ uint8, num uint64, w float64) { - if typ == uint8(sqlite3.INTEGER) { +func (s *topSummary) add(typ topTag, num uint64, str string, w float64) { + switch typ { + case topIntTag: addMap(s, &s.ints, int64(num), w) - } else { + case topRealTag: addMap(s, &s.reals, math.Float64frombits(num), w) + default: + addMap(s, &s.texts, str, w) } } -func (s *topSummary) addText(k string, w float64) { - addMap(s, &s.texts, k, w) -} - func addMap[K comparable](s *topSummary, m *map[K]float64, k K, w float64) { if *m == nil { *m = make(map[K]float64) @@ -352,6 +318,7 @@ func (s *topSummary) purge() { if n <= int(s.c) { return } + w := slices.Grow(s.scratch[:0], n) for _, v := range s.ints { w = append(w, v) @@ -362,23 +329,13 @@ func (s *topSummary) purge() { for _, v := range s.texts { w = append(w, v) } + s.scratch = w[:0] delta := quick.Select(w, len(w)-int(s.c)-1) - s.offset += delta purgeMap(s.ints, delta) purgeMap(s.reals, delta) purgeMap(s.texts, delta) - s.scratch = w[:0] -} - -func purgeMap[K comparable](m map[K]float64, delta float64) { - for k, v := range m { - if v <= delta { - delete(m, k) - } else { - m[k] = v - delta - } - } + s.offset += delta } func (s *topSummary) clone() *topSummary { @@ -411,6 +368,16 @@ func (s *topSummary) merge(other *topSummary) { } } +func purgeMap[K comparable](m map[K]float64, delta float64) { + for k, v := range m { + if v <= delta { + delete(m, k) + } else { + m[k] = v - delta + } + } +} + func mergeMap[K comparable](dst *map[K]float64, src map[K]float64) { if len(src) == 0 { return @@ -428,45 +395,30 @@ type topEntry struct { str string num uint64 weight float64 - typ uint8 + typ topTag } func cmpEntry(a, b topEntry) int { if c := cmp.Compare(b.weight, a.weight); c != 0 { return c } - aText := a.typ == uint8(sqlite3.TEXT) - bText := b.typ == uint8(sqlite3.TEXT) - if aText != bText { - if aText { - return +1 - } - return -1 - } - if aText { - return cmp.Compare(a.str, b.str) - } if a.typ == b.typ { - if a.typ == uint8(sqlite3.INTEGER) { + switch a.typ { + case topIntTag: return cmp.Compare(int64(a.num), int64(b.num)) + case topRealTag: + return cmp.Compare(math.Float64frombits(a.num), math.Float64frombits(b.num)) + default: + return cmp.Compare(a.str, b.str) } - return cmp.Compare(math.Float64frombits(a.num), math.Float64frombits(b.num)) - } - // Mixed INTEGER and non-integral FLOAT. - if a.typ == uint8(sqlite3.INTEGER) { - fi := float64(int64(a.num)) - f := math.Float64frombits(b.num) - if fi != f { - return cmp.Compare(fi, f) - } - return -1 // f == 2^63 > MaxInt64 } - fi := float64(int64(b.num)) - f := math.Float64frombits(a.num) - if fi != f { - return cmp.Compare(f, fi) + if a.typ == topTextTag || b.typ == topTextTag { + return cmp.Compare(a.typ, b.typ) + } + if a.typ == topIntTag { + return cmpIntFloat(int64(a.num), math.Float64frombits(b.num)) } - return +1 + return -cmpIntFloat(int64(b.num), math.Float64frombits(a.num)) } func (s *topSummary) entries() ([]topEntry, float64) { @@ -476,13 +428,13 @@ func (s *topSummary) entries() ([]topEntry, float64) { } out := make([]topEntry, 0, n) for k, v := range s.ints { - out = append(out, topEntry{num: uint64(k), weight: v, typ: uint8(sqlite3.INTEGER)}) + out = append(out, topEntry{num: uint64(k), weight: v, typ: topIntTag}) } for k, v := range s.reals { - out = append(out, topEntry{num: math.Float64bits(k), weight: v, typ: uint8(sqlite3.FLOAT)}) + out = append(out, topEntry{num: math.Float64bits(k), weight: v, typ: topRealTag}) } for k, v := range s.texts { - out = append(out, topEntry{str: k, weight: v, typ: uint8(sqlite3.TEXT)}) + out = append(out, topEntry{str: k, weight: v, typ: topTextTag}) } offset := s.offset if len(out) > int(s.c) { @@ -507,8 +459,7 @@ func (s *topSummary) entries() ([]topEntry, float64) { return out, offset } -func (s *topSummary) evalResult(ctx sqlite3.Context, k int) { - entries, offset := s.entries() +func evalEntries(ctx sqlite3.Context, entries []topEntry, offset float64, k int) { if len(entries) == 0 && offset == 0 { return } @@ -520,213 +471,114 @@ func (s *topSummary) evalResult(ctx sqlite3.Context, k int) { out = append(out, ',') } var err error - out, err = appendEntryJSON(out, e, offset) - if err != nil { - ctx.ResultError(err) - return + out = append(out, '[') + switch e.typ { + case topIntTag: + out = strconv.AppendInt(out, int64(e.num), 10) + case topRealTag: + out = appendJSONFloat(out, math.Float64frombits(e.num)) + default: + out, err = util.AppendQuote(out, e.str) + if err != nil { + ctx.ResultError(err) + return + } } + out = append(out, ',') + out = appendJSONFloat(out, e.weight+offset) + out = append(out, ']') } out = append(out, ']') ctx.ResultRawText(out) ctx.ResultSubtype('J') } -func appendEntryJSON(dst []byte, e topEntry, offset float64) ([]byte, error) { - var err error - dst = append(dst, '[') - switch sqlite3.Datatype(e.typ) { - case sqlite3.INTEGER: - dst = strconv.AppendInt(dst, int64(e.num), 10) - case sqlite3.FLOAT: - dst = appendJSONFloat(dst, math.Float64frombits(e.num)) - default: - dst, err = util.AppendQuote(dst, e.str) - } - dst = append(dst, ',') - dst = appendJSONFloat(dst, e.weight+offset) - return append(dst, ']'), err -} - func appendJSONFloat(dst []byte, f float64) []byte { - if i, ok := floatIsInt64(f); ok { + if i, ok := floatIsInt(f); ok { return strconv.AppendInt(dst, i, 10) } return util.AppendNumber(dst, f) } -const ( - topKeyInt = 0 - topKeyFloat = 1 - topKeyText = 2 - topWeightF = 1 << 2 -) - -func (b *topSketch) appendBytes(out []byte) []byte { - switch b.typ { - case 0: - if b.k == 0 { - return append(out, topHeader, 0) - } - out = append(out, topHeader) - out = binary.AppendUvarint(out, uint64(b.k)) - out = binary.AppendUvarint(out, uint64(topCap(b.k))) - return binary.AppendUvarint(out, 0) - case topSummaryTag: - return b.summary().appendBytes(out) - default: - out = append(out, topHeader) - out = binary.AppendUvarint(out, uint64(b.k)) - out = binary.AppendUvarint(out, uint64(topCap(b.k))) - out = binary.AppendUvarint(out, 1<<1) // count=1, hasOffset=0 - return appendEntry(out, topEntry{ - str: b.unitStr(), - num: b.num, - weight: b.weight, - typ: b.typ, - }) - } -} - func (s *topSummary) appendBytes(out []byte) []byte { - if s.k == 0 { - return append(out, topHeader, 0) - } entries, offset := s.entries() - out = slices.Grow(out, 6+12*len(entries)) - out = append(out, topHeader) - out = binary.AppendUvarint(out, uint64(s.k)) - out = binary.AppendUvarint(out, uint64(s.c)) - hdr := uint64(len(entries)) << 1 - if offset > 0 { - hdr |= 1 - } - out = binary.AppendUvarint(out, hdr) - if offset > 0 { - out = binary.LittleEndian.AppendUint64(out, math.Float64bits(offset)) - } - for _, e := range entries { - out = appendEntry(out, e) - } - return out + return appendSketchBytes(out, entries, offset, s.k, s.c) } -func appendEntry(out []byte, e topEntry) []byte { - var tag byte - switch sqlite3.Datatype(e.typ) { - case sqlite3.INTEGER: - tag = topKeyInt - case sqlite3.FLOAT: - tag = topKeyFloat - default: - tag = topKeyText - } - wi, intWeight := floatIsInt64(e.weight) - if !intWeight || wi <= 0 { - tag |= topWeightF - } - out = append(out, tag) - - switch sqlite3.Datatype(e.typ) { - case sqlite3.INTEGER: - out = binary.AppendVarint(out, int64(e.num)) - case sqlite3.FLOAT: - out = binary.LittleEndian.AppendUint64(out, e.num) - default: - out = binary.AppendUvarint(out, uint64(len(e.str))) - out = append(out, e.str...) +func appendSketchBytes(out []byte, entries []topEntry, offset float64, k, c int16) []byte { + if k == 0 { + return append(out, topHeader) } - - if tag&topWeightF == 0 { - out = binary.AppendUvarint(out, uint64(wi)) - } else { + out = slices.Grow(out, 15+17*len(entries)) + out = append(out, topHeader) + out = binary.LittleEndian.AppendUint16(out, uint16(k)) + out = binary.LittleEndian.AppendUint16(out, uint16(c)) + out = binary.LittleEndian.AppendUint16(out, uint16(len(entries))) + out = binary.LittleEndian.AppendUint64(out, math.Float64bits(offset)) + for _, e := range entries { + out = append(out, byte(e.typ)) out = binary.LittleEndian.AppendUint64(out, math.Float64bits(e.weight)) + if e.typ == topTextTag { + out = binary.AppendUvarint(out, uint64(len(e.str))) + out = append(out, e.str...) + } else { + out = binary.LittleEndian.AppendUint64(out, e.num) + } } return out } func topFromBytes(data []byte) (*topSummary, error) { - if len(data) < 2 { + if len(data) == 0 { return nil, errutil.ErrorString("approx: insufficient bytes to deserialize top-K sketch") } if data[0] != topHeader { return nil, errutil.ErrorString("approx: not a top-K sketch") } - rest := data[1:] - - k, n := binary.Uvarint(rest) - if n <= 0 { - return nil, errInvalidTopK - } - rest = rest[n:] - if k == 0 { - if len(rest) != 0 { - return nil, errInvalidTopK - } + if len(data) == 1 { return &topSummary{}, nil } - if k > topMaxK { - return nil, errInvalidTopK - } - - c, n := binary.Uvarint(rest) - if n <= 0 || c < k || c > topMaxC { + if len(data) < 15 { return nil, errInvalidTopK } - rest = rest[n:] - hdr, n := binary.Uvarint(rest) - if n <= 0 { + k := binary.LittleEndian.Uint16(data[1:]) + c := binary.LittleEndian.Uint16(data[3:]) + count := binary.LittleEndian.Uint16(data[5:]) + off := math.Float64frombits(binary.LittleEndian.Uint64(data[7:])) + if k == 0 || k > topMaxK || c < k || c > topMaxC || count > c || !(off >= 0) { return nil, errInvalidTopK } - rest = rest[n:] - count := hdr >> 1 - hasOffset := hdr&1 != 0 - if count > c { + rest := data[15:] + if len(rest)/10 < int(count) { return nil, errInvalidTopK } s := &topSummary{k: int16(k), c: int16(c)} - if hasOffset { - if len(rest) < 8 { - return nil, errInvalidTopK - } - off := math.Float64frombits(binary.LittleEndian.Uint64(rest)) - if !(off > 0) { - return nil, errInvalidTopK - } + if off > 0 { s.offset = off - rest = rest[8:] - } - if uint64(len(rest))/3 < count { - return nil, errInvalidTopK } - for range count { - if len(rest) == 0 { + if len(rest) < 10 { return nil, errInvalidTopK } - tag := rest[0] - rest = rest[1:] - if tag & ^byte(0x07) != 0 { + typ := topTag(rest[0]) + w := math.Float64frombits(binary.LittleEndian.Uint64(rest[1:])) + if !(w > 0) { return nil, errInvalidTopK } + rest = rest[9:] - var ( - iKey int64 - fKey float64 - sKey string - ) - keyType := tag & 0x03 - switch keyType { - case topKeyInt: - v, n := binary.Varint(rest) - if n <= 0 { + var ok bool + switch typ { + case topIntTag: + if len(rest) < 8 { return nil, errInvalidTopK } - iKey = v - rest = rest[n:] - case topKeyFloat: + ok = putUnique(&s.ints, int64(binary.LittleEndian.Uint64(rest)), w) + rest = rest[8:] + case topRealTag: if len(rest) < 8 { return nil, errInvalidTopK } @@ -734,70 +586,38 @@ func topFromBytes(data []byte) (*topSummary, error) { if math.IsNaN(f) { return nil, errInvalidTopK } - if _, ok := floatIsInt64(f); ok { + if _, isInt := floatIsInt(f); isInt { return nil, errInvalidTopK } - fKey = f + ok = putUnique(&s.reals, f, w) rest = rest[8:] - case topKeyText: + case topTextTag: strLen, n := binary.Uvarint(rest) if n <= 0 || uint64(len(rest)-n) < strLen { return nil, errInvalidTopK } rest = rest[n:] - sKey = string(rest[:strLen]) + ok = putUnique(&s.texts, string(rest[:strLen]), w) rest = rest[strLen:] default: return nil, errInvalidTopK } - - var weight float64 - if tag&topWeightF == 0 { - u, n := binary.Uvarint(rest) - if n <= 0 || u == 0 { - return nil, errInvalidTopK - } - weight = float64(u) - rest = rest[n:] - } else { - if len(rest) < 8 { - return nil, errInvalidTopK - } - w := math.Float64frombits(binary.LittleEndian.Uint64(rest)) - if !(w > 0) { - return nil, errInvalidTopK - } - weight = w - rest = rest[8:] - } - - switch keyType { - case topKeyInt: - if s.ints == nil { - s.ints = make(map[int64]float64) - } else if _, dup := s.ints[iKey]; dup { - return nil, errInvalidTopK - } - s.ints[iKey] = weight - case topKeyFloat: - if s.reals == nil { - s.reals = make(map[float64]float64) - } else if _, dup := s.reals[fKey]; dup { - return nil, errInvalidTopK - } - s.reals[fKey] = weight - case topKeyText: - if s.texts == nil { - s.texts = make(map[string]float64) - } else if _, dup := s.texts[sKey]; dup { - return nil, errInvalidTopK - } - s.texts[sKey] = weight + if !ok { + return nil, errInvalidTopK } } - if len(rest) != 0 { return nil, errInvalidTopK } return s, nil } + +func putUnique[K comparable](m *map[K]float64, k K, w float64) bool { + if *m == nil { + *m = make(map[K]float64) + } else if _, dup := (*m)[k]; dup { + return false + } + (*m)[k] = w + return true +} diff --git a/ext/approx/top_test.go b/ext/approx/top_test.go index ff1f2cdf..ac15ff9c 100644 --- a/ext/approx/top_test.go +++ b/ext/approx/top_test.go @@ -1,13 +1,10 @@ package approx_test import ( - "math" - "strings" "testing" "github.com/ncruces/go-sqlite3" "github.com/ncruces/go-sqlite3/internal/testcfg" - "github.com/ncruces/go-sqlite3/internal/util" ) func TestRegister_top(t *testing.T) { @@ -84,12 +81,10 @@ func TestRegister_top(t *testing.T) { if !stmt.Step() { t.Fatal(stmt.Err()) } - // Counts (unweighted): 0 -> 2, 1 -> 2, 9007199254740992 -> 2; ties at 1 broken by 1.5 < 9007199254740993 < "" < "1" < ... const wantCount4 = `[[0,2],[1,2],[9007199254740992,2],[1.5,1]]` if got := stmt.ColumnText(0); got != wantCount4 { t.Errorf("col 0: got %s, want %s", got, wantCount4) } - // Weighted sums: 0 -> 4, 1 -> 4, 1.5 -> 4, "" -> 4, "1" -> 4 (sorted: 0 < 1 < 1.5 < "" < "1"), then 9007199254740992 -> 3, 9007199254740993 -> 3. const wantSum6 = `[[0,4],[1,4],[1.5,4],["",4],["1",4],[9007199254740992,3]]` if got := stmt.ColumnText(1); got != wantSum6 { t.Errorf("col 1: got %s, want %s", got, wantSum6) @@ -107,34 +102,21 @@ func TestRegister_top(t *testing.T) { } stmt.Close() - // Single-key unit states (int, float, empty text, repeated text) + JSON subtype. - qASCII, _ := util.AppendQuote(nil, "a count = 0, offset = 1 -> "[]"). - stmt, _, err = db.Prepare(` - SELECT - approx_top_k(value, 5), - approx_top_k_estimate(approx_top_k_accumulate(value, 5)) - FROM generate_series(1, 256)`) - if err != nil { - t.Fatal(err) - } - if !stmt.Step() { - t.Fatal(stmt.Err()) - } - if got := stmt.ColumnText(0); got != `[]` { - t.Errorf("got %q, want []", got) - } - if got := stmt.ColumnText(1); got != `[]` { - t.Errorf("got %q, want []", got) - } - stmt.Close() - - // MaxInt64 vs 2^63 float ordering check (both orders). - stmt, _, err = db.Prepare(` - SELECT - approx_top_k(x, 4), - approx_top_k_estimate(approx_top_k_combine(s)) - FROM ( - SELECT 9223372036854775807 AS x, approx_top_k_accumulate(value, 5) AS s FROM generate_series(1, 300) - UNION ALL - SELECT 9223372036854775808.0, approx_top_k_accumulate(value + 0.5, 1.25, 5) FROM generate_series(1, 300) - UNION ALL - SELECT 1.5, approx_top_k_accumulate(1, 5) - UNION ALL - SELECT 2, approx_top_k_accumulate(2.5, 1.25, 5) - )`) - if err != nil { - t.Fatal(err) - } - if !stmt.Step() { - t.Fatal(stmt.Err()) - } - var gotBig [][2]float64 - if err := stmt.ColumnJSON(0, &gotBig); err != nil { - t.Fatal(err) - } - if len(gotBig) != 4 || gotBig[0][0] != 1.5 || gotBig[1][0] != 2 || math.IsNaN(gotBig[2][0]) { - t.Errorf("got %v", gotBig) - } - if got := stmt.ColumnType(1); got != sqlite3.TEXT { - t.Errorf("got %v, want TEXT", got) - } - stmt.Close() - - // Sliding window over >2 distinct items (exercises unit + summary Combine in twoStack.Inverse). - stmt, _, err = db.Prepare(` - SELECT approx_top_k(CASE WHEN value % 3 = 0 THEN value + 0.5 ELSE value END, 5) - OVER (ROWS BETWEEN 5 PRECEDING AND 1 FOLLOWING) - FROM generate_series(1, 30)`) - if err != nil { - t.Fatal(err) - } - for stmt.Step() { - if got := stmt.ColumnType(0); got != sqlite3.TEXT { - t.Errorf("got %v, want TEXT", got) - } - } - if err := stmt.Err(); err != nil { - t.Fatal(err) - } - stmt.Close() } func TestRegister_top_errors(t *testing.T) { @@ -417,37 +315,32 @@ func TestRegister_top_errors(t *testing.T) { `SELECT approx_top_k(1, -0.5, 5)`, `SELECT approx_top_k(1, 1e999, 5)`, `SELECT approx_top_k_accumulate(1, 0)`, - `SELECT approx_top_k_accumulate(1, 1001)`, `SELECT approx_top_k_accumulate(X'01', 5)`, - `SELECT approx_top_k_accumulate(1, -1, 5)`, `SELECT approx_top_k_estimate('not a blob')`, - `SELECT approx_top_k_estimate('not a blob', 5)`, + `SELECT approx_top_k_estimate(X'')`, `SELECT approx_top_k_estimate(X'00')`, - `SELECT approx_top_k_estimate(X'16')`, - `SELECT approx_top_k_estimate(X'160001')`, // trailing byte after K=0 - `SELECT approx_top_k_estimate(X'1600', 0)`, // K=0 on empty sketch - `SELECT approx_top_k_estimate(X'1600', 4001)`, // K>topMaxC on empty sketch - `SELECT approx_top_k_estimate(X'16EA07800100')`, // K=1002 > topMaxK - `SELECT approx_top_k_estimate(X'160A0500')`, // C=5 < K=10 - `SELECT approx_top_k_estimate(X'1601A11F00')`, // C=4001 > topMaxC - `SELECT approx_top_k_estimate(X'160180018202')`, // count=129 > C=128 - `SELECT approx_top_k_estimate(X'1601800101')`, // hasOffset=1, missing offset - `SELECT approx_top_k_estimate(X'16018001010000000000000000')`, // hasOffset=1, offset=0 - `SELECT approx_top_k_estimate(X'1601800102080001')`, // reserved tag bit set (0x08) - `SELECT approx_top_k_estimate(X'1601800102030001')`, // invalid keyType=3 - `SELECT approx_top_k_estimate(X'1601800102000200')`, // integer weight u=0 - `SELECT approx_top_k_estimate(X'160180010201000000000000F03F01')`, // integral float 1.0 in FLOAT key - `SELECT approx_top_k_estimate(X'1601800104000201000201')`, // duplicate int key 1 + `SELECT approx_top_k_estimate(X'1601')`, + `SELECT approx_top_k_estimate(X'16', 0)`, + `SELECT approx_top_k_estimate(X'16', 4001)`, + `SELECT approx_top_k_estimate(X'160000800000000000000000000000')`, // K=0 + `SELECT approx_top_k_estimate(X'16E903A00F00000000000000000000')`, // K=1001 > topMaxK + `SELECT approx_top_k_estimate(X'160A00050000000000000000000000')`, // C=5 < K=10 + `SELECT approx_top_k_estimate(X'160100800081000000000000000000')`, // count=129 > C=128 + `SELECT approx_top_k_estimate(X'16010080000000000000000000F0BF')`, // offset=-1 + `SELECT approx_top_k_estimate(X'160100800001000000000000000000')`, // truncated entry + `SELECT approx_top_k_estimate(X'1601008000010000000000000000000100000000000000000100000000000000')`, // weight=0 + `SELECT approx_top_k_estimate(X'16010080000100000000000000000002000000000000F03F000000000000F87F')`, // NaN float key + `SELECT approx_top_k_estimate(X'16010080000100000000000000000002000000000000F03F000000000000F03F')`, // integral float key 1.0 + `SELECT approx_top_k_estimate(X'16010080000100000000000000000004000000000000F03F0000000000000000')`, // invalid typ=4 + `SELECT approx_top_k_estimate(X'16010080000200000000000000000001000000000000F03F010000000000000001000000000000F03F0100000000000000')`, // duplicate key `SELECT approx_top_k_estimate(approx_top_k_accumulate(1, 5), 0)`, - `SELECT approx_top_k_estimate(approx_top_k_accumulate(1, 5), 129)`, // > C=128 + `SELECT approx_top_k_estimate(approx_top_k_accumulate(1, 5), 129)`, `SELECT approx_top_k_estimate(approx_count_distinct_accumulate(1))`, `SELECT approx_top_k_estimate(approx_percentile_accumulate(1))`, `SELECT approx_count_distinct_estimate(approx_top_k_accumulate(1, 5))`, `SELECT approx_percentile_estimate(approx_top_k_accumulate(1, 5), 0.5)`, `SELECT approx_top_k_combine('not a blob')`, `SELECT approx_top_k_combine(X'00')`, - `SELECT approx_top_k_combine(approx_count_distinct_accumulate(1))`, - `SELECT approx_top_k_combine(approx_percentile_accumulate(1))`, } for _, sql := range tests { if err := db.Exec(sql); err == nil { From dff2fc278e94ce6bd7113ea78b335c62d474323e Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Sat, 14 Nov 2026 22:40:07 +0000 Subject: [PATCH 6/8] Document sketch BLOB storage format --- ext/approx/format.md | 150 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 ext/approx/format.md diff --git a/ext/approx/format.md b/ext/approx/format.md new file mode 100644 index 00000000..6a813c3f --- /dev/null +++ b/ext/approx/format.md @@ -0,0 +1,150 @@ +# Sketch BLOB storage format + +Serialization format for the HLL (cardinality), REQ (quantile), and FI (top-k) +sketch BLOBs produced and consumed by the `approx` SQLite extension in +[go-sqlite3](https://github.com/ncruces/go-sqlite3). + +## Header byte + +Every sketch BLOB begins with a single header byte using the +[Aggregate Knowledge HLL Storage Specification](https://github.com/aggregateknowledge/hll-storage-spec) +nibble layout: high nibble = schema version (`1`), low nibble = type code. + +| First byte | Type | Format | +| ------------- | ---- | ------ | +| `0x11`–`0x14` | 1–4 | HLL (`EMPTY`, `EXPLICIT`, `SPARSE`, `FULL`) | +| `0x15` | 5 | REQ (quantiles) | +| `0x16` | 6 | FI (frequent items / top-k) | + +Types `5` and `6` are private extensions in unassigned HLL v1 slots so sketch +types can be told apart by their first byte; everything after byte 0 is +format-specific. + +## HLL sketch (cardinality) + +HLL sketch BLOBs follow the +[Aggregate Knowledge HLL Storage Specification](https://github.com/aggregateknowledge/hll-storage-spec) +verbatim. `accumulate` emits `log2m = 11`, `regwidth = 5`, sparse +enabled, auto explicit threshold; `combine` requires inputs to share `log2m` and +`regwidth` and preserves them. + +**Hashing.** Rows with no arguments or any `NULL` argument are ignored. Starting +from `seed = 0`, each argument updates `seed` to the first 64 bits (`h1`) of +`MurmurHash3_x64_128` initialized with `h1 = h2 = seed'`: + +- **integer** (and **float** exactly representable as `int64`, incl. `±0.0`): + `seed' = seed`, 8-byte little-endian `i64`; +- **text**: `seed' = seed`, raw bytes; +- **blob**: `seed' = seed ^ 1`, raw bytes; +- **float** (other): `seed' = seed ^ 2`, 8-byte little-endian `f64`. + +If any step yields `0`, the row is ignored; otherwise the final `seed` is added +to the HLL as a raw 64-bit hash. For single integer (`bigint`) or text values, +this matches [`postgresql-hll`](https://github.com/citusdata/postgresql-hll). + +## REQ & FI conventions + +The remaining sections specify the `go-sqlite3` custom formats (`0x15` and `0x16`): + +- Fixed-width integers (`u16`, `i64`) and floats (`f32`, `f64`) are **little-endian**; +- `uvarint` is an unsigned LEB128 varint (Go `encoding/binary` `Uvarint`, `uint64`); +- `f32`/`f64` are IEEE-754 binary32/binary64 bit patterns; `NaN` is never valid. + +## REQ sketch (quantiles) + +A [relative-error quantiles sketch](https://arxiv.org/abs/2004.01668) in +high-rank-accuracy mode, storing `f32` values across weighted levels. +An item at level `h` represents `2^h` input items. + +``` +byte header = 0x15 +byte k = 40 // initial section size; must be 40 +level* // levels h = 0, 1, ... until EOF (at most 53) +``` + +An empty sketch is just the 2 header bytes. Levels repeat until the end of the +BLOB (the level count is implicit; a 54th level is invalid), contiguous from +`h = 0` with no empty levels: + +``` +uvarint state // compaction counter for level h +uvarint count // items in buffer, 1 <= count < 2*s*k (see below) +f32[] buf // count values; ±Inf allowed, no NaNs +``` + +Writers sort `buf` ascending; readers must not rely on the order. + +The section parameters `(s, k)` are a function of `state`: +starting at `s = 3`, `k = 40`, whenever `state >= 2^(s-1)` the number of +sections doubles and `k` is divided by `sqrt(2)`, rounded to the nearest even +integer (minimum 4): + +| `state` | `s` | `k` | +| --------- | --- | --- | +| `< 2^2` | 3 | 40 | +| `< 2^5` | 6 | 28 | +| `< 2^11` | 12 | 20 | +| `< 2^23` | 24 | 14 | +| `< 2^47` | 48 | 10 | +| otherwise | 96 | 8 | + +**Compaction.** When a level reaches `2*s*k` items, a prefix of its smallest +items is compacted in pairs into level `h + 1` (retaining the exact minimum at +`h = 0`; see the paper and `req.go`), and `state` is incremented. Cascading +keeps every level below `2*s*k` items. + +**Estimates.** Total weight `N = Σ 2^h * count_h`. This implementation estimates +`quantile(p)` by linear interpolation at weighted rank `p * (N - 1)` (matching +`percentile_cont`), and `percent_rank(v)` as its inverse (interpolating across +the weight of the largest item `< v`, divided by `N - 1`). + +**Merge.** Concatenate buffers level by level (bitwise-ORing `state`), then +compact from `h = 0`. + +## FI sketch (top-k) + +A weighted, mergeable [Misra-Gries](https://arxiv.org/abs/1705.07001) +frequent-items summary. A single `0x16` byte (`len == 1`) is a valid empty +sketch; otherwise: + +``` +byte header = 0x16 +u16 k // target top-K, 1 <= k <= 1000 +u16 c // capacity, k <= c <= 4000 (writers use max(128, 4*k)) +u16 count // number of entries, 0 <= count <= c +f64 offset // total weight purged by compaction, >= 0 (+Inf allowed) +entry[] entries // count entries; trailing bytes are invalid +``` + +Each entry: + +``` +byte type // 1 = integer, 2 = float, 3 = text +f64 weight // > 0 (+Inf allowed) +value // depends on type +``` + +- **integer (`1`)**: `i64`. Any numeric value exactly representable as `int64` + (including `0.0` and `-0.0`) uses this type; +- **float (`2`)**: `f64`; `±Inf` allowed, must not be `NaN` or exactly + representable as `int64`; +- **text (`3`)**: `uvarint` byte length, followed by raw bytes (UTF-8 is not + validated on read). + +Entries are unique by `(type, value)`. Writers sort entries by descending +`weight` (breaking ties by ascending SQLite value order); readers must not rely +on the order. + +**Estimates.** Each entry's true weight lies in `[weight, weight + offset]`, and +this implementation reports the upper bound `weight + offset` (untracked items +have true weight at most `offset`). The top-k are the `k` heaviest entries. + +**Compaction.** Writers compact to at most `c` entries before serializing. To +compact when entry count exceeds `c`, let `δ` be the `(c+1)`-th largest weight: +drop entries with `weight <= δ`, subtract `δ` from every remaining weight, and +add `δ` to `offset`. *(In memory, compaction is deferred until inserting a new +key at `2*c` entries or merging to `>= 2*c` entries.)* + +**Merge.** The 1-byte empty sketch is the identity. Otherwise take +`k = min(k1, k2)`, `c = min(c1, c2)`, sum `offset`s, sum weights of matching +entries, and compact. From ceb040dd200610ffbbab1f93353c8ef9aa2b6980 Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Mon, 16 Nov 2026 10:51:13 +0000 Subject: [PATCH 7/8] Readme. --- ext/approx/README.md | 61 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 ext/approx/README.md diff --git a/ext/approx/README.md b/ext/approx/README.md new file mode 100644 index 00000000..396d2a68 --- /dev/null +++ b/ext/approx/README.md @@ -0,0 +1,61 @@ +# Approximate statistics SQLite extension + +This package provides approximate aggregate functions for SQLite, +using mergeable data sketches: +- **distinct counts** (HyperLogLog), e.g. daily, weekly, monthly active users; +- **quantiles** (REQ), e.g. median and tail latencies; +- **top-K** (Frequent Items), e.g. most popular content, highest spenders. + +Computing these exactly requires memory that grows with the input, +and exact results don't combine: +you can't sum seven daily counts of active users to get a weekly count, +or average daily medians to get a monthly median. + +Sketches trade a small, bounded error for compact size +(about 1 KiB for distinct counts, tens of KiB for quantiles over millions of rows) +and mergeability: combining two sketches produces a sketch of the combined data. + +### Roll-up tables + +Sketches serialize to BLOBs, so you can store them in roll-up tables. +For example, you can accumulate daily sketches from raw events: + +```sql +CREATE TABLE daily (day TEXT PRIMARY KEY, users BLOB, latency BLOB, pages BLOB); + +INSERT INTO daily +SELECT date(time), + approx_count_distinct_accumulate(user_id), + approx_percentile_accumulate(latency), + approx_top_k_accumulate(path, 10) +FROM events +GROUP BY 1; +``` + +You can then combine them to answer coarser queries without rescanning `events`: + +```sql +SELECT strftime('%Y-%m', day) AS month, + approx_count_distinct_estimate(approx_count_distinct_combine(users)) AS users, + approx_percentile_estimate(approx_percentile_combine(latency), 0.99) AS p99, + approx_top_k_estimate(approx_top_k_combine(pages), 3) AS top_pages +FROM daily +GROUP BY 1; +``` + +### Window functions + +Every aggregate also works as a window function, +e.g. to compute a rolling count of weekly active users: + +```sql +SELECT day, + approx_count_distinct_estimate( + approx_count_distinct_combine(users) OVER (ORDER BY day ROWS 6 PRECEDING) + ) AS weekly_users +FROM daily; +``` + +### Storage format + +The sketch BLOB storage format is specified in [`format.md`](format.md). From cdfb2999c1fcc200213c07e54ca7d6411d9f7e4f Mon Sep 17 00:00:00 2001 From: Nuno Cruces Date: Mon, 16 Nov 2026 13:51:29 +0000 Subject: [PATCH 8/8] Tweaks. --- ext.go | 2 +- ext/approx/approx.go | 2 +- ext/approx/hll.go | 4 +- ext/approx/pool.go | 17 ++++++ ext/approx/req.go | 6 +- ext/approx/top.go | 123 ++++++++++++++++++++--------------------- ext/stats/moments.go | 20 ++++--- ext/stats/welford.go | 25 ++++++--- ext/unicode/unicode.go | 3 +- func.go | 8 +-- vtab.go | 5 +- 11 files changed, 122 insertions(+), 93 deletions(-) create mode 100644 ext/approx/pool.go diff --git a/ext.go b/ext.go index 78ff325a..481ac51e 100644 --- a/ext.go +++ b/ext.go @@ -104,8 +104,8 @@ func (e *ExtEnv) X__table_base() *int32 { return &e.tableBase } func (e *ExtEnv) CallWithContextValues(pCtx, nVal, pVal int32, fn func(Context, ...Value)) { db := e.DB.(*Conn) args := callbackArgs(db, nVal, ptr_t(pVal)) - defer returnArgs(args) fn(Context{db, ptr_t(pCtx)}, *args...) + returnArgs(args) } // ExtensionInit loads an SQLite extension library. diff --git a/ext/approx/approx.go b/ext/approx/approx.go index 306a5f45..d9c3c433 100644 --- a/ext/approx/approx.go +++ b/ext/approx/approx.go @@ -92,7 +92,7 @@ func floatIsInt(f float64) (int64, bool) { } func cmpIntFloat(i int64, f float64) int { - // Assumes both floatIsInt(f) and IsNan(f) are false. + // Assumes both floatIsInt(f) and IsNaN(f) are false. if c := cmp.Compare(float64(i), f); c != 0 { return c } diff --git a/ext/approx/hll.go b/ext/approx/hll.go index b4764f91..0f9f154c 100644 --- a/ext/approx/hll.go +++ b/ext/approx/hll.go @@ -123,7 +123,7 @@ func (b *hllCombine) Step(ctx sqlite3.Context, arg ...sqlite3.Value) { other, err := hllFromBytes(arg[0].RawBlob()) if err != nil { ctx.ResultError(fmt.Errorf("approx: %w", err)) - return + return // notest } if b.sketch == nil { b.sketch = &other @@ -168,7 +168,7 @@ func hllEstimate(ctx sqlite3.Context, arg ...sqlite3.Value) { sketch, err := hllFromBytes(arg[0].RawBlob()) if err != nil { ctx.ResultError(fmt.Errorf("approx: %w", err)) - return + return // notest } ctx.ResultInt64(int64(sketch.Cardinality())) default: diff --git a/ext/approx/pool.go b/ext/approx/pool.go new file mode 100644 index 00000000..9da36fdb --- /dev/null +++ b/ext/approx/pool.go @@ -0,0 +1,17 @@ +package approx + +import "sync" + +var float64sPool sync.Pool + +func getFloat64s(count int) *[]float64 { + if p, ok := float64sPool.Get().(*[]float64); ok && cap(*p) >= count { + return p + } + w := make([]float64, 0, count) + return &w +} + +func returnFloat64s(p *[]float64) { + float64sPool.Put(p) +} diff --git a/ext/approx/req.go b/ext/approx/req.go index 36ddcebd..e988e881 100644 --- a/ext/approx/req.go +++ b/ext/approx/req.go @@ -116,7 +116,7 @@ func (b *reqSketch) evalResult(ctx sqlite3.Context, isQuantile bool) { for i, a := range *b.args { if err := check(a); err != nil { ctx.ResultError(err) - return + return // notest } many[i] = eval(a) } @@ -184,7 +184,7 @@ func (b *reqCombine) Step(ctx sqlite3.Context, arg ...sqlite3.Value) { other, err := reqFromBytes(arg[0].RawBlob()) if err != nil { ctx.ResultError(err) - return + return // notest } if b.sketch == nil { b.sketch = other @@ -223,7 +223,7 @@ func reqEstimate(ctx sqlite3.Context, arg []sqlite3.Value, isQuantile bool) { sketch, err := reqFromBytes(arg[0].RawBlob()) if err != nil { ctx.ResultError(err) - return + return // notest } var b reqSketch b.sketch = sketch diff --git a/ext/approx/top.go b/ext/approx/top.go index c9e0a295..61d29187 100644 --- a/ext/approx/top.go +++ b/ext/approx/top.go @@ -24,6 +24,7 @@ const ( topMinC = 128 topMaxC = 4 * topMaxK + topEmptyTag topTag = 0 topIntTag topTag = 1 topRealTag topTag = 2 topTextTag topTag = 3 @@ -39,8 +40,8 @@ func topCap(k int16) int16 { type topSketch struct { ptr unsafe.Pointer // nil, unsafe.StringData(str), or *topSummary bits uint64 // int64, float64, or len(str) - weight float64 // inline weight in unit state (> 0) - k int16 // target K, or 0 if Step never ran + weight float64 // inline weight in unit state + k int16 // target K, 0 if Step never ran typ topTag } @@ -57,7 +58,7 @@ func (b *topSketch) string() string { func (b *topSketch) Value(ctx sqlite3.Context) { switch b.typ { - case 0: + case topEmptyTag: return case topSummaryTag: s := b.summary() @@ -78,34 +79,27 @@ func (b *topSketch) Step(ctx sqlite3.Context, arg ...sqlite3.Value) { k := arg[len(arg)-1].Int() if k < 1 || k > topMaxK { ctx.ResultError(fmt.Errorf("approx: invalid k: %d", k)) - return + return // notest } b.k = int16(k) } - t := arg[0].Type() - if t == sqlite3.BLOB { - ctx.ResultError(errutil.ErrorString("approx: JSON cannot hold BLOB values")) - return - } - w := 1.0 if len(arg) == 3 { if arg[1].NumericType() > sqlite3.FLOAT { return } w = arg[1].Float() - if w <= 0 || math.IsInf(w, 1) { - if w < 0 || math.IsInf(w, 1) { - ctx.ResultError(fmt.Errorf("approx: invalid weight: %g", w)) - } + if w == 0 { return } + if w < 0 || math.IsInf(w, 1) { + ctx.ResultError(fmt.Errorf("approx: invalid weight: %g", w)) + return // notest + } } - switch t { - case sqlite3.NULL: - return + switch arg[0].Type() { case sqlite3.INTEGER: b.insert(topIntTag, uint64(arg[0].Int64()), "", w) case sqlite3.FLOAT: @@ -122,53 +116,54 @@ func (b *topSketch) Step(ctx sqlite3.Context, arg ...sqlite3.Value) { } else { b.insert(topTextTag, uint64(len(raw)), string(raw), w) } + case sqlite3.BLOB: + ctx.ResultError(errutil.ErrorString("approx: JSON cannot hold BLOB values")) + } +} + +func (b *topSketch) Combine(other topSketch) { + if b.k == 0 && b.typ != topSummaryTag { + b.k = other.k + } + switch other.typ { + case topEmptyTag: + return + case topSummaryTag: + os := other.summary() + if b.typ == topSummaryTag { + b.summary().merge(os) + return + } + os = os.clone() + if b.typ != topEmptyTag { + os.add(b.typ, b.bits, b.string(), b.weight) + } + *b = topSketch{ptr: unsafe.Pointer(os), typ: topSummaryTag} + default: + b.insert(other.typ, other.bits, other.string(), other.weight) } } func (b *topSketch) insert(typ topTag, num uint64, str string, w float64) { switch b.typ { - case 0: + case topSummaryTag: + b.summary().add(typ, num, str, w) + case topEmptyTag: b.typ = typ b.bits = num + b.weight = w if typ == topTextTag { b.ptr = unsafe.Pointer(unsafe.StringData(str)) } - b.weight = w - case topSummaryTag: - b.summary().add(typ, num, str, w) default: if b.typ == typ && (typ == topTextTag && str == b.string() || typ != topTextTag && num == b.bits) { b.weight += w return } - s := &topSummary{k: b.k, c: topCap(b.k)} + s := topSummary{k: b.k, c: topCap(b.k)} s.add(b.typ, b.bits, b.string(), b.weight) s.add(typ, num, str, w) - *b = topSketch{ptr: unsafe.Pointer(s), typ: topSummaryTag} - } -} - -func (b *topSketch) Combine(other topSketch) { - if b.k == 0 && b.typ != topSummaryTag { - b.k = other.k - } - switch other.typ { - case 0: - return - case topSummaryTag: - os := other.summary() - switch b.typ { - case 0: - *b = topSketch{ptr: unsafe.Pointer(os.clone()), typ: topSummaryTag} - case topSummaryTag: - b.summary().merge(os) - default: - s := os.clone() - s.add(b.typ, b.bits, b.string(), b.weight) - *b = topSketch{ptr: unsafe.Pointer(s), typ: topSummaryTag} - } - default: - b.insert(other.typ, other.bits, other.string(), other.weight) + *b = topSketch{ptr: unsafe.Pointer(&s), typ: topSummaryTag} } } @@ -177,7 +172,7 @@ type topAccumulate struct{ topSketch } func (b *topAccumulate) Value(ctx sqlite3.Context) { var buf [32]byte // fits empty and single-row numeric sketches switch b.typ { - case 0: + case topEmptyTag: ctx.ResultBlob(appendSketchBytes(buf[:0], nil, 0, b.k, topCap(b.k))) case topSummaryTag: ctx.ResultBlob(b.summary().appendBytes(buf[:0])) @@ -214,7 +209,7 @@ func (b *topCombine) Step(ctx sqlite3.Context, arg ...sqlite3.Value) { other, err := topFromBytes(arg[0].RawBlob()) if err != nil { ctx.ResultError(err) - return + return // notest } if b.sketch == nil { b.sketch = other @@ -245,7 +240,7 @@ func topEstimate(ctx sqlite3.Context, arg ...sqlite3.Value) { sketch, err := topFromBytes(arg[0].RawBlob()) if err != nil { ctx.ResultError(err) - return + return // notest } var k int if len(arg) == 1 { @@ -261,7 +256,7 @@ func topEstimate(ctx sqlite3.Context, arg ...sqlite3.Value) { } if k < 1 || k > maxK { ctx.ResultError(fmt.Errorf("approx: invalid k: %d", k)) - return + return // notest } if sketch.k == 0 { return @@ -277,12 +272,11 @@ func topEstimate(ctx sqlite3.Context, arg ...sqlite3.Value) { // topSummary implements a weighted mergeable Misra-Gries frequent items summary // (Anderson et al., IMC 2017). type topSummary struct { - ints map[int64]float64 - reals map[float64]float64 - texts map[string]float64 - scratch []float64 - offset float64 - k, c int16 + ints map[int64]float64 + reals map[float64]float64 + texts map[string]float64 + offset float64 + k, c int16 } func (s *topSummary) len() int { @@ -319,7 +313,9 @@ func (s *topSummary) purge() { return } - w := slices.Grow(s.scratch[:0], n) + p := getFloat64s(n) + + w := (*p)[:0] for _, v := range s.ints { w = append(w, v) } @@ -329,13 +325,13 @@ func (s *topSummary) purge() { for _, v := range s.texts { w = append(w, v) } - s.scratch = w[:0] delta := quick.Select(w, len(w)-int(s.c)-1) purgeMap(s.ints, delta) purgeMap(s.reals, delta) purgeMap(s.texts, delta) s.offset += delta + returnFloat64s(p) } func (s *topSummary) clone() *topSummary { @@ -438,12 +434,13 @@ func (s *topSummary) entries() ([]topEntry, float64) { } offset := s.offset if len(out) > int(s.c) { - w := slices.Grow(s.scratch[:0], len(out)) + p := getFloat64s(len(out)) + w := (*p)[:0] for i := range out { w = append(w, out[i].weight) } delta := quick.Select(w, len(w)-int(s.c)-1) - s.scratch = w[:0] + returnFloat64s(p) offset += delta kept := 0 for _, e := range out { @@ -476,12 +473,12 @@ func evalEntries(ctx sqlite3.Context, entries []topEntry, offset float64, k int) case topIntTag: out = strconv.AppendInt(out, int64(e.num), 10) case topRealTag: - out = appendJSONFloat(out, math.Float64frombits(e.num)) + out = util.AppendNumber(out, math.Float64frombits(e.num)) default: out, err = util.AppendQuote(out, e.str) if err != nil { ctx.ResultError(err) - return + return // notest } } out = append(out, ',') diff --git a/ext/stats/moments.go b/ext/stats/moments.go index 09136196..121e4900 100644 --- a/ext/stats/moments.go +++ b/ext/stats/moments.go @@ -16,11 +16,11 @@ func (m moments) mean() float64 { } func (m moments) var_pop() float64 { - return m.m2.hi / float64(m.n) + return max(m.m2.hi, 0) / float64(m.n) } func (m moments) var_samp() float64 { - return m.m2.hi / float64(m.n-1) // Bessel's correction + return max(m.m2.hi, 0) / float64(m.n-1) // Bessel's correction } func (m moments) stddev_pop() float64 { @@ -33,7 +33,7 @@ func (m moments) stddev_samp() float64 { func (m moments) skewness_pop() float64 { m2 := m.m2.hi - if div := m2 * m2 * m2; div != 0 { + if div := m2 * m2 * m2; div > 0 { return m.m3.hi * math.Sqrt(float64(m.n)/div) } return math.NaN() @@ -42,7 +42,7 @@ func (m moments) skewness_pop() float64 { func (m moments) skewness_samp() float64 { n := m.n // https://mathworks.com/help/stats/skewness.html#f1132178 - return m.skewness_pop() * math.Sqrt(float64(n*(n-1))) / float64(n-2) + return m.skewness_pop() * math.Sqrt(float64(n)*float64(n-1)) / float64(n-2) } func (m moments) kurtosis_pop() float64 { @@ -51,7 +51,7 @@ func (m moments) kurtosis_pop() float64 { func (m moments) raw_kurtosis_pop() float64 { m2 := m.m2.hi - if div := m2 * m2; div != 0 { + if div := m2 * m2; m2 > 0 && div > 0 { return m.m4.hi * float64(m.n) / div } return math.NaN() @@ -60,14 +60,14 @@ func (m moments) raw_kurtosis_pop() float64 { func (m moments) kurtosis_samp() float64 { n := m.n k := math.FMA(m.raw_kurtosis_pop(), float64(n+1), float64(3-3*n)) - return k * float64(n-1) / float64((n-2)*(n-3)) + return k * float64(n-1) / (float64(n-2) * float64(n-3)) } func (m moments) raw_kurtosis_samp() float64 { n := m.n // https://mathworks.com/help/stats/kurtosis.html#f4975293 k := math.FMA(m.raw_kurtosis_pop(), float64(n+1), float64(3-3*n)) - return math.FMA(k, float64(n-1)/float64((n-2)*(n-3)), 3) + return math.FMA(k, float64(n-1)/(float64(n-2)*float64(n-3)), 3) } func (m *moments) enqueue(x float64) { @@ -77,7 +77,8 @@ func (m *moments) enqueue(x float64) { dn := d1 / float64(n) d2 := dn * dn t1 := d1 * dn * float64(n-1) - m.m4.add(t1*d2*float64(n*n-3*n+3) + 6*d2*m.m2.hi - 4*dn*m.m3.hi) + m.m4.add(t1*d2*math.FMA(float64(n), float64(n), float64(3-3*n)) + + 6*d2*m.m2.hi - 4*dn*m.m3.hi) m.m3.add(t1*dn*float64(n-2) - 3*dn*m.m2.hi) m.m2.add(t1) m.m1.add(dn) @@ -94,7 +95,8 @@ func (m *moments) dequeue(x float64) { dn := d1 / float64(n) d2 := dn * dn t1 := d1 * dn * float64(n+1) - m.m4.sub(t1*d2*float64(n*n+3*n+3) - 6*d2*m.m2.hi - 4*dn*m.m3.hi) + m.m4.sub(t1*d2*math.FMA(float64(n), float64(n), float64(3+3*n)) - + 6*d2*m.m2.hi - 4*dn*m.m3.hi) m.m3.sub(t1*dn*float64(n+2) - 3*dn*m.m2.hi) m.m2.sub(t1) m.m1.sub(dn) diff --git a/ext/stats/welford.go b/ext/stats/welford.go index 5b6115b4..9d6edb48 100644 --- a/ext/stats/welford.go +++ b/ext/stats/welford.go @@ -21,11 +21,11 @@ func (w welford) mean() float64 { } func (w welford) var_pop() float64 { - return w.m2.hi / float64(w.n) + return max(w.m2.hi, 0) / float64(w.n) } func (w welford) var_samp() float64 { - return w.m2.hi / float64(w.n-1) // Bessel's correction + return max(w.m2.hi, 0) / float64(w.n-1) // Bessel's correction } func (w welford) stddev_pop() float64 { @@ -74,7 +74,11 @@ func (w welford2) covar_samp() float64 { } func (w welford2) correlation() float64 { - return w.cov.hi / math.Sqrt(w.m2y.hi*w.m2x.hi) + m2y := w.m2y.hi + if div := m2y * w.m2x.hi; m2y > 0 && div > 0 { + return max(-1, min(+1, w.cov.hi/math.Sqrt(div))) + } + return math.NaN() } func (w welford2) regr_avgy() float64 { @@ -86,11 +90,11 @@ func (w welford2) regr_avgx() float64 { } func (w welford2) regr_syy() float64 { - return w.m2y.hi + return max(w.m2y.hi, 0) } func (w welford2) regr_sxx() float64 { - return w.m2x.hi + return max(w.m2x.hi, 0) } func (w welford2) regr_sxy() float64 { @@ -102,7 +106,10 @@ func (w welford2) regr_count() int64 { } func (w welford2) regr_slope() float64 { - return w.cov.hi / w.m2x.hi + if div := w.m2x.hi; div > 0 { + return w.cov.hi / div + } + return math.NaN() } func (w welford2) regr_intercept() float64 { @@ -113,7 +120,11 @@ func (w welford2) regr_intercept() float64 { } func (w welford2) regr_r2() float64 { - return w.cov.hi * w.cov.hi / (w.m2y.hi * w.m2x.hi) + m2y := w.m2y.hi + if div := m2y * w.m2x.hi; m2y > 0 && div > 0 { + return min(1, w.cov.hi*w.cov.hi/div) + } + return math.NaN() } func (w welford2) regr_json(dst []byte) []byte { diff --git a/ext/unicode/unicode.go b/ext/unicode/unicode.go index a0666781..73dc4a32 100644 --- a/ext/unicode/unicode.go +++ b/ext/unicode/unicode.go @@ -169,7 +169,6 @@ var unaccentPool = sync.Pool{ func unaccent(ctx sqlite3.Context, arg ...sqlite3.Value) { unaccent := unaccentPool.Get().(transform.Transformer) - defer unaccentPool.Put(unaccent) res, _, err := transform.Bytes(unaccent, arg[0].RawText()) if err != nil { @@ -177,6 +176,8 @@ func unaccent(ctx sqlite3.Context, arg ...sqlite3.Value) { } else { ctx.ResultRawText(res) } + + unaccentPool.Put(unaccent) } func normalize(ctx sqlite3.Context, arg ...sqlite3.Value) { diff --git a/func.go b/func.go index 2dfde038..34846d7f 100644 --- a/func.go +++ b/func.go @@ -186,17 +186,17 @@ func (e env) Xgo_compare(pApp, nKey1, pKey1, nKey2, pKey2 int32) int32 { func (e env) Xgo_func(pCtx, pApp, nArg, pArg int32) { db := e.DB.(*Conn) args := callbackArgs(db, nArg, ptr_t(pArg)) - defer returnArgs(args) fn := db.wrp.GetHandle(ptr_t(pApp)).(ScalarFunction) fn(Context{db, ptr_t(pCtx)}, *args...) + returnArgs(args) } func (e env) Xgo_step(pCtx, pAgg, pApp, nArg, pArg int32) { db := e.DB.(*Conn) args := callbackArgs(db, nArg, ptr_t(pArg)) - defer returnArgs(args) fn, _ := callbackAggregate(db, ptr_t(pAgg), ptr_t(pApp)) fn.Step(Context{db, ptr_t(pCtx)}, *args...) + returnArgs(args) } func (e env) Xgo_value(pCtx, pAgg, pApp, final int32) { @@ -221,15 +221,15 @@ func (e env) Xgo_value(pCtx, pAgg, pApp, final int32) { func (e env) Xgo_inverse(pCtx, pAgg, nArg, pArg int32) { db := e.DB.(*Conn) - args := callbackArgs(db, nArg, ptr_t(pArg)) - defer returnArgs(args) fn, ok := db.wrp.GetHandle(ptr_t(pAgg)).(WindowFunction) if !ok { Context{db, ptr_t(pCtx)}.ResultError( errutil.ErrorString("sqlite3: may not be used as a window function")) return // notest } + args := callbackArgs(db, nArg, ptr_t(pArg)) fn.Inverse(Context{db, ptr_t(pCtx)}, *args...) + returnArgs(args) } func callbackAggregate(db *Conn, pAgg, pApp ptr_t) (AggregateFunction, ptr_t) { diff --git a/vtab.go b/vtab.go index 54110fbc..9c1e75d5 100644 --- a/vtab.go +++ b/vtab.go @@ -486,7 +486,6 @@ func (e env) Xgo_vtab_best_index(pVTab, pIdxInfo int32) int32 { func (e env) Xgo_vtab_update(pVTab, nArg, pArg, pRowID int32) int32 { db := e.DB.(*Conn) args := callbackArgs(db, nArg, ptr_t(pArg)) - defer returnArgs(args) vtab := e.vtabGetHandle(pVTab).(VTabUpdater) rowID, err := vtab.Update(*args...) @@ -494,6 +493,7 @@ func (e env) Xgo_vtab_update(pVTab, nArg, pArg, pRowID int32) int32 { e.Memory.Write64(ptr_t(pRowID), uint64(rowID)) } + returnArgs(args) return e.vtabError(pVTab, _VTAB_ERROR, err, ERROR) } @@ -588,7 +588,6 @@ func (e env) Xgo_cur_close(pCur int32) int32 { func (e env) Xgo_cur_filter(pCur, idxNum, idxStr, nArg, pArg int32) int32 { db := e.DB.(*Conn) args := callbackArgs(db, nArg, ptr_t(pArg)) - defer returnArgs(args) var idxName string if idxStr != 0 { @@ -597,6 +596,8 @@ func (e env) Xgo_cur_filter(pCur, idxNum, idxStr, nArg, pArg int32) int32 { cursor := e.vtabGetHandle(pCur).(VTabCursor) err := cursor.Filter(int(idxNum), idxName, *args...) + + returnArgs(args) return e.vtabError(pCur, _CURSOR_ERROR, err, ERROR) }