Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ double? mean = await file.Scan<Reading>().AverageAsync(r => r.Celsius);

Spread spread = await file.Scan<Reading>()
.Where(r => r.Day >= 900)
.AggAsync<Spread>(a => (a.Min(r => r.Celsius), a.Max(r => r.Celsius), a.CountDistinct(r => r.City)));
.AggregateAsync<Spread>(a => (a.Min(r => r.Celsius), a.Max(r => r.Celsius), a.CountDistinct(r => r.City)));

[VortexRecord]
public partial record struct Spread(double? Min, double? Max, long Cities); // several answers go into a record
Expand Down
10 changes: 5 additions & 5 deletions bench/Vorticity.Benchmarks.Churn/Churn.cs
Original file line number Diff line number Diff line change
Expand Up @@ -338,8 +338,8 @@ private static (string Name, int Calls, Func<VortexDataset, Random, ValueTask<lo

return rows;
}),
("seek +10", 20, (dataset, random) => SeekAsync(dataset, 2 * random.NextInt64(keySpace / 2), SeekOp.AtOrAfter)),
("seek back -10", 20, (dataset, random) => SeekAsync(dataset, 2 * random.NextInt64(keySpace / 2), SeekOp.AtOrBefore)),
("seek +10", 20, (dataset, random) => SeekAsync(dataset, 2 * random.NextInt64(keySpace / 2), SeekMode.AtOrAfter)),
("seek back -10", 20, (dataset, random) => SeekAsync(dataset, 2 * random.NextInt64(keySpace / 2), SeekMode.AtOrBefore)),
("rows 100", 20, async (dataset, random) =>
{
long start = random.NextInt64(Math.Max(dataset.RowCount - 100, 1));
Expand Down Expand Up @@ -410,7 +410,7 @@ private static async ValueTask<long> PointAsync(VortexDataset dataset, long key)
return rows;
}

private static async ValueTask<long> SeekAsync(VortexDataset dataset, long key, SeekOp op)
private static async ValueTask<long> SeekAsync(VortexDataset dataset, long key, SeekMode op)
{
KeyCursor<long> cursor = await dataset.Scan<ChurnRow>().Keys(r => r.Key).OpenAsync().ConfigureAwait(false);
await using (cursor.ConfigureAwait(false))
Expand All @@ -423,8 +423,8 @@ private static async ValueTask<long> SeekAsync(VortexDataset dataset, long key,

for (int i = 0; i < 10; i++)
{
bool moved = op == SeekOp.AtOrBefore
? await cursor.PrevAsync().ConfigureAwait(false)
bool moved = op == SeekMode.AtOrBefore
? await cursor.PreviousAsync().ConfigureAwait(false)
: await cursor.NextAsync().ConfigureAwait(false);
if (!moved)
{
Expand Down
2 changes: 1 addition & 1 deletion bench/Vorticity.Benchmarks.Scenarios/ScenarioSet.cs
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ public static async Task<long> LookupSortedRuns(string path)
long found = 0;
foreach (FilterLiteral probe in prepared.Probes)
{
found += await cursor.SeekAsync(probe, SeekOp.Exact, CancellationToken.None) ? 1 : 0;
found += await cursor.SeekAsync(probe, SeekMode.Exact, CancellationToken.None) ? 1 : 0;
}

return found;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ internal sealed class DatasetKeyCursorBefore : IAsyncDisposable
private readonly List<int>[] _levels;
// The walks below start at the first key: no seek sets these.
private readonly FilterLiteral _target = default;
private readonly SeekOp _op = SeekOp.AtOrAfter;
private readonly SeekMode _op = SeekMode.AtOrAfter;
private bool _first;
private int _current = -1;
private bool _disposed;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@ public void Setup()
}

Refill();
long size = _cache.Size;
long size = _cache.HeldBytes;
Present();
if (_cache.Size != size || size != 64L * Entries)
if (_cache.HeldBytes != size || size != 64L * Entries)
{
throw new InvalidOperationException("The cache does not hold what it was given back.");
}
Expand All @@ -63,7 +63,7 @@ public void Setup()
public long Absent()
{
_cache.Evict(_absent);
return _cache.Size;
return _cache.HeldBytes;
}

/// <summary>A file of 16 segments evicted, and its segments kept again.</summary>
Expand All @@ -72,7 +72,7 @@ public long Present()
{
_cache.Evict(_closing);
Refill();
return _cache.Size;
return _cache.HeldBytes;
}

private void Refill()
Expand Down
16 changes: 8 additions & 8 deletions docs/design/07-dotnet-mapping.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,21 +15,21 @@ What a record member and a `Column<T>` may be, and what the column exposes:
| `Primitive` i8…i64, u8…u64 | `sbyte`…`long`, `byte`…`ulong` | `Values` as `ReadOnlySpan<T>`, an indexer | zero-copy |
| `Primitive` u16 | `char` too | the same | the UTF-16 code unit, lone surrogates included |
| `Primitive` i64, u64 | `nint`, `nuint` too, on a 64-bit host | the same | refused on a 32-bit host, whose native integer is narrower |
| `Primitive` i64 | `TimeSpan` too | an indexer, `CopyTo`, `Storage()` | 100 ns ticks, exact over the whole range |
| `Primitive` i64 | `TimeSpan` too | an indexer, `CopyTo`, `AsStorage()` | 100 ns ticks, exact over the whole range |
| `Primitive` f16, f32, f64 | `Half`, `float`, `double` | the same | exact, since `Half` is IEEE binary16 |
| `Decimal(p, s)` | `decimal` within [the bounds below](#2-decimal-why-systemdecimal-is-not-the-answer), `VortexDecimal` always | an indexer, `Storage<TStorage>()`, `Scale` | never lossy: a column too wide for `decimal` is refused when it binds |
| `Decimal(p, s)` | `decimal` within [the bounds below](#2-decimal-why-systemdecimal-is-not-the-answer), `VortexDecimal` always | an indexer, `AsStorage<TStorage>()`, `Scale` | never lossy: a column too wide for `decimal` is refused when it binds |
| `Decimal(p, 0)` | `Int128`, `UInt128` for `p ≤ 39`, `BigInteger` always | an indexer, `CopyTo`, and `Values` zero-copy over 128-bit storage | see [wider integers](#21-integers-wider-than-64-bits) |
| `Utf8` | `string` | a UTF-8 span per value, `GetString(i)` on request | see [strings](#4-strings-spans-first-string-on-request) |
| `Binary` | `ReadOnlyMemory<byte>`, `Memory<byte>`, `byte[]` | a span per value | |
| `Struct` | a nested `[VortexRecord]` type | `Columns<TNested>` | |
| `List`, `FixedSizeList` | `ReadOnlyMemory<T>`, `Memory<T>`, `T[]`, `List<T>`, `ImmutableArray<T>`, an interface an array implements | a `Range` per row, `Elements` as `Column<T>` | reading rows allocates one array, or one `List<T>`, per list |
| `List` of `Struct` | any of the above over a `[VortexRecord]` type | `ListOf<TNested>()`: a `Range` per row, `Elements` as `Columns<TNested>` | the elements are read in one pass, then cut per row |
| `Map` | `Dictionary<TKey, TValue>`, `IDictionary`, `IReadOnlyDictionary` | `MapOf<TKey, TValue>()`: a `Range` per row, `Keys` and `Values` | scalar keys and values, and a key is never null |
| `vortex.date` | `DateOnly` | an indexer, `Storage<int>()` or `Storage<long>()` | |
| `vortex.time` | `TimeOnly` | an indexer, `Storage<TStorage>()`, `Unit` | |
| `vortex.timestamp` | `DateTime` when naive or UTC, `DateTimeOffset` with a zone | an indexer, `Storage<long>()`, `Unit`, `TimeZone` | see [temporal extensions](#3-temporal-extensions-one-resolution-at-binding) |
| `vortex.date` | `DateOnly` | an indexer, `AsStorage<int>()` or `AsStorage<long>()` | |
| `vortex.time` | `TimeOnly` | an indexer, `AsStorage<TStorage>()`, `Unit` | |
| `vortex.timestamp` | `DateTime` when naive or UTC, `DateTimeOffset` with a zone | an indexer, `AsStorage<long>()`, `Unit`, `TimeZone` | see [temporal extensions](#3-temporal-extensions-one-resolution-at-binding) |
| `vortex.uuid` | `Guid` | an indexer | the byte order is converted explicitly, never implied |
| an extension registered on the session | the registered type | an indexer, `Storage<TStorage>()` | see [the session](14-public-api.md#2-the-session) |
| an extension registered on the session | the registered type | an indexer, `AsStorage<TStorage>()` | see [the session](14-public-api.md#2-the-session) |

`Union` and `Variant` columns appear in a schema but no `T` maps to them. An `enum` member maps to
its underlying integer.
Expand Down Expand Up @@ -72,7 +72,7 @@ The storage width is carried in `vortex.decimal`'s metadata as `values_type`: `i
declare any wider one, and often does. Upstream allows precision-2 values in an `i256` buffer, and a
decimal column from Arrow or Parquet arrives as `i128` or `i256` whatever its precision. The width is
therefore read from `values_type`, and only a width narrower than the precision requires is refused.
`Storage<TStorage>()` exposes what the file declared, for callers who do their own arithmetic without
`AsStorage<TStorage>()` exposes what the file declared, for callers who do their own arithmetic without
widening.

.NET has `Int128` but no `Int256`, so this library carries an internal `Int256` with what the decimal
Expand Down Expand Up @@ -114,7 +114,7 @@ So the zone is resolved once, when a `DateTimeOffset` member or column binds to
`TimeZoneInfo.FindSystemTimeZoneById`, and never per value. A naive or UTC timestamp binds as
`DateTime`, with `Kind` `Unspecified` or `Utc`, and needs no database. A zone the host cannot resolve
makes the `DateTimeOffset` binding fail with `VortexSchemaException`. The column still binds as
`long`, its storage, and `Storage()`, `Unit` and `TimeZone` let a caller convert with a zone of its
`long`, its storage, and `AsStorage()`, `Unit` and `TimeZone` let a caller convert with a zone of its
own choosing. The machine dependence is confined to one call, at one moment, with a refusal rather
than a guess.

Expand Down
4 changes: 2 additions & 2 deletions docs/design/09-contracts.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ that each one is not deduced differently at each call site.
| `KeyCursorBuilder<TKey>`, `KeyCursor<TKey>` | not thread-safe, like the scan |
| `ColumnsBuilder`, `ColumnBuilder<T>` | one thread. The builder a writer hands out belongs to the writer and is reused after every `WriteAsync` |
| `VortexFileWriter` (an append included), `VortexFileIndexer` | one writer per file. An append is not atomic, and `VortexFileRepair` truncates a torn one |
| `ScanPlan`, `CountPlan`, `OrderPlan`, `KeyPlan`, `ScanStatistics`, `WriteReport` | immutable records |
| `ScanPlan`, `CountPlan`, `OrderPlan`, `KeyPlan`, `ScanMetrics`, `WriteReport` | immutable records |
| decoders and kernels | pure functions over borrowed memory, with no shared mutable state |

## 2. Parallelism
Expand Down Expand Up @@ -127,7 +127,7 @@ bytes: `segments-requested`, `bytes-requested`, `zones-pruned`, `zones-total`, `
`cursor-seeks`, `cursor-steps`, `count-blocks-proven`, `count-blocks-decoded`, `key-order-windows`
and `key-order-window-splits`.

The third works per query: a scan's `ExplainAsync` is its plan before it runs, and its `Statistics`
The third works per query: a scan's `ExplainAsync` is its plan before it runs, and its `Metrics`
the same quantities measured afterwards ([observability.md](../guide/observability.md)).

## 6. Licensing and attribution
Expand Down
6 changes: 3 additions & 3 deletions docs/design/11-write-strategy.md
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ summarizes on one thread.

### 3.8 Appending to a file

`VortexSession.AppendAsync(path)` returns a writer that continues a file whose shape this writer can
`VortexSession.OpenWriterAsync(path)` returns a writer that continues a file whose shape this writer can
extend (a struct of columns, each a chunked layout of flat segments, all chunked alike), and refuses
any other file with the advice to rewrite it. The old segments stay where they are. New blocks are
numbered from the old row count, so their boundaries are the ones a single write would have made.
Expand Down Expand Up @@ -328,7 +328,7 @@ Two more promises complete the contract:

- A scan takes one lease per segment. The live splits register their segments in one request, each
distinct segment is read once in as few requests as coalescing allows, and every block in it
decodes from that lease. `ScanStatistics.Requests` is the plan's distinct segments, and
decodes from that lease. `ScanMetrics.Requests` is the plan's distinct segments, and
`BytesRequested` their bytes, within the coalescing gap.
- A selection can replace a compaction. A filtered batch is compacted to its surviving rows by
default. With `ScanOptions.Compact = false`, each live block is delivered whole with the
Expand All @@ -343,7 +343,7 @@ The index directory is read lazily, on the first filter that could use it, and u
the tail the open already read. The runs are read on the first query that needs them and cached on
the file, shared by every scan. `ExplainAsync` returns the plan without executing it (blocks, blocks
pruned by each structure with what consulting it cost, segments and bytes to read), and the scan's
`Statistics` report the same quantities once it has run, counted by the same sink.
`Metrics` report the same quantities once it has run, counted by the same sink.

## 7. Determinism, and its limit

Expand Down
8 changes: 4 additions & 4 deletions docs/design/12-index-reads.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,10 +96,10 @@ lands on its lowest row, and `AtOrBefore` on its highest. A range `[a, b)` is `A

### 3.2 Stepping and direction

`NextAsync` and `PrevAsync` move one entry in `(key, row)` order and return false past the ends.
`NextAsync` and `PreviousAsync` move one entry in `(key, row)` order and return false past the ends.
`NextKeyAsync` moves to the first entry of the next distinct key by seeking `After(Key)` rather than
stepping through the duplicates. That is a loose index scan, one seek per group, so a `GROUP BY` over
a key with a million rows costs one seek per value. `PrevKeyAsync` is `Before(Key)`.
a key with a million rows costs one seek per value. `PreviousKeyAsync` is `Before(Key)`.

A cursor has a direction. Stepping the other way seeks again at the current entry, which costs
`O(r log n)` for `r` runs, because a heap of run positions does not run backwards. Every merging
Expand Down Expand Up @@ -138,7 +138,7 @@ all.
### 3.5 Rank, select, and the count of a key

`RankAsync(k)` is the number of entries with a key `< k`, `SeekRankAsync(i)` positions on the entry
of rank `i`, and `KeyCountAsync()` is the number of entries that share the current key. All three
of rank `i`, and `CountAtKeyAsync()` is the number of entries that share the current key. All three
are exact on the two sources with rows, and refused on the others. A sorted array is its own
counter: rank is the sum over runs of each run's lower bound, in `O(r log n)`, and select is a binary
search over positions with a rank computed at each probe, in `O(r log² n)`.
Expand Down Expand Up @@ -239,7 +239,7 @@ process](05-benchmarks.md#3-in-one-process-after-warm-up), measured on [the benc
page](../guide/benchmarks.md#in-one-process-after-warm-up)). So it is the tool for a selective range,
a top-k or a merge, not for a whole uncorrelated column, where sorting the scan's output costs less.
The plan's `OrderPlan` states the source, the runs the range reaches and the entries it admits
before the read, and the scan's statistics count the windows and the splits they touched afterwards.
before the read, and the scan's metrics count the windows and the splits they touched afterwards.

The dataset's key-ordered reads and its compaction merge are built on this, one call per object (see
[what a compaction does](13-dataset.md#53-what-a-compaction-does) and [order beyond the
Expand Down
Loading
Loading