diff --git a/README.md b/README.md index 9eacd19d..b16a1ae9 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ double? mean = await file.Scan().AverageAsync(r => r.Celsius); Spread spread = await file.Scan() .Where(r => r.Day >= 900) - .AggAsync(a => (a.Min(r => r.Celsius), a.Max(r => r.Celsius), a.CountDistinct(r => r.City))); + .AggregateAsync(a => (a.Min(r => r.Celsius), a.Max(r => r.Celsius), a.CountDistinct(r => r.City))); [VortexRecord] public partial record struct Spread(double? Min, double? Max, long Cities); // several answers go into a record diff --git a/bench/Vorticity.Benchmarks.Churn/Churn.cs b/bench/Vorticity.Benchmarks.Churn/Churn.cs index 0405a655..c1705e70 100644 --- a/bench/Vorticity.Benchmarks.Churn/Churn.cs +++ b/bench/Vorticity.Benchmarks.Churn/Churn.cs @@ -338,8 +338,8 @@ private static (string Name, int Calls, Func SeekAsync(dataset, 2 * random.NextInt64(keySpace / 2), SeekOp.AtOrAfter)), - ("seek back -10", 20, (dataset, random) => SeekAsync(dataset, 2 * random.NextInt64(keySpace / 2), SeekOp.AtOrBefore)), + ("seek +10", 20, (dataset, random) => SeekAsync(dataset, 2 * random.NextInt64(keySpace / 2), SeekMode.AtOrAfter)), + ("seek back -10", 20, (dataset, random) => SeekAsync(dataset, 2 * random.NextInt64(keySpace / 2), SeekMode.AtOrBefore)), ("rows 100", 20, async (dataset, random) => { long start = random.NextInt64(Math.Max(dataset.RowCount - 100, 1)); @@ -410,7 +410,7 @@ private static async ValueTask PointAsync(VortexDataset dataset, long key) return rows; } - private static async ValueTask SeekAsync(VortexDataset dataset, long key, SeekOp op) + private static async ValueTask SeekAsync(VortexDataset dataset, long key, SeekMode op) { KeyCursor cursor = await dataset.Scan().Keys(r => r.Key).OpenAsync().ConfigureAwait(false); await using (cursor.ConfigureAwait(false)) @@ -423,8 +423,8 @@ private static async ValueTask SeekAsync(VortexDataset dataset, long key, for (int i = 0; i < 10; i++) { - bool moved = op == SeekOp.AtOrBefore - ? await cursor.PrevAsync().ConfigureAwait(false) + bool moved = op == SeekMode.AtOrBefore + ? await cursor.PreviousAsync().ConfigureAwait(false) : await cursor.NextAsync().ConfigureAwait(false); if (!moved) { diff --git a/bench/Vorticity.Benchmarks.Scenarios/ScenarioSet.cs b/bench/Vorticity.Benchmarks.Scenarios/ScenarioSet.cs index 172370a8..5e4bf457 100644 --- a/bench/Vorticity.Benchmarks.Scenarios/ScenarioSet.cs +++ b/bench/Vorticity.Benchmarks.Scenarios/ScenarioSet.cs @@ -274,7 +274,7 @@ public static async Task LookupSortedRuns(string path) long found = 0; foreach (FilterLiteral probe in prepared.Probes) { - found += await cursor.SeekAsync(probe, SeekOp.Exact, CancellationToken.None) ? 1 : 0; + found += await cursor.SeekAsync(probe, SeekMode.Exact, CancellationToken.None) ? 1 : 0; } return found; diff --git a/bench/Vorticity.Benchmarks/Complexity/DatasetKeyCursorBefore.cs b/bench/Vorticity.Benchmarks/Complexity/DatasetKeyCursorBefore.cs index d04abd27..9c46dc43 100644 --- a/bench/Vorticity.Benchmarks/Complexity/DatasetKeyCursorBefore.cs +++ b/bench/Vorticity.Benchmarks/Complexity/DatasetKeyCursorBefore.cs @@ -30,7 +30,7 @@ internal sealed class DatasetKeyCursorBefore : IAsyncDisposable private readonly List[] _levels; // The walks below start at the first key: no seek sets these. private readonly FilterLiteral _target = default; - private readonly SeekOp _op = SeekOp.AtOrAfter; + private readonly SeekMode _op = SeekMode.AtOrAfter; private bool _first; private int _current = -1; private bool _disposed; diff --git a/bench/Vorticity.Benchmarks/Complexity/SegmentCacheEvictBenchmarks.cs b/bench/Vorticity.Benchmarks/Complexity/SegmentCacheEvictBenchmarks.cs index 5a7aff8e..f3833617 100644 --- a/bench/Vorticity.Benchmarks/Complexity/SegmentCacheEvictBenchmarks.cs +++ b/bench/Vorticity.Benchmarks/Complexity/SegmentCacheEvictBenchmarks.cs @@ -46,9 +46,9 @@ public void Setup() } Refill(); - long size = _cache.Size; + long size = _cache.HeldBytes; Present(); - if (_cache.Size != size || size != 64L * Entries) + if (_cache.HeldBytes != size || size != 64L * Entries) { throw new InvalidOperationException("The cache does not hold what it was given back."); } @@ -63,7 +63,7 @@ public void Setup() public long Absent() { _cache.Evict(_absent); - return _cache.Size; + return _cache.HeldBytes; } /// A file of 16 segments evicted, and its segments kept again. @@ -72,7 +72,7 @@ public long Present() { _cache.Evict(_closing); Refill(); - return _cache.Size; + return _cache.HeldBytes; } private void Refill() diff --git a/docs/design/07-dotnet-mapping.md b/docs/design/07-dotnet-mapping.md index 91801859..1d1c2ee1 100644 --- a/docs/design/07-dotnet-mapping.md +++ b/docs/design/07-dotnet-mapping.md @@ -15,9 +15,9 @@ What a record member and a `Column` may be, and what the column exposes: | `Primitive` i8…i64, u8…u64 | `sbyte`…`long`, `byte`…`ulong` | `Values` as `ReadOnlySpan`, an indexer | zero-copy | | `Primitive` u16 | `char` too | the same | the UTF-16 code unit, lone surrogates included | | `Primitive` i64, u64 | `nint`, `nuint` too, on a 64-bit host | the same | refused on a 32-bit host, whose native integer is narrower | -| `Primitive` i64 | `TimeSpan` too | an indexer, `CopyTo`, `Storage()` | 100 ns ticks, exact over the whole range | +| `Primitive` i64 | `TimeSpan` too | an indexer, `CopyTo`, `AsStorage()` | 100 ns ticks, exact over the whole range | | `Primitive` f16, f32, f64 | `Half`, `float`, `double` | the same | exact, since `Half` is IEEE binary16 | -| `Decimal(p, s)` | `decimal` within [the bounds below](#2-decimal-why-systemdecimal-is-not-the-answer), `VortexDecimal` always | an indexer, `Storage()`, `Scale` | never lossy: a column too wide for `decimal` is refused when it binds | +| `Decimal(p, s)` | `decimal` within [the bounds below](#2-decimal-why-systemdecimal-is-not-the-answer), `VortexDecimal` always | an indexer, `AsStorage()`, `Scale` | never lossy: a column too wide for `decimal` is refused when it binds | | `Decimal(p, 0)` | `Int128`, `UInt128` for `p ≤ 39`, `BigInteger` always | an indexer, `CopyTo`, and `Values` zero-copy over 128-bit storage | see [wider integers](#21-integers-wider-than-64-bits) | | `Utf8` | `string` | a UTF-8 span per value, `GetString(i)` on request | see [strings](#4-strings-spans-first-string-on-request) | | `Binary` | `ReadOnlyMemory`, `Memory`, `byte[]` | a span per value | | @@ -25,11 +25,11 @@ What a record member and a `Column` may be, and what the column exposes: | `List`, `FixedSizeList` | `ReadOnlyMemory`, `Memory`, `T[]`, `List`, `ImmutableArray`, an interface an array implements | a `Range` per row, `Elements` as `Column` | reading rows allocates one array, or one `List`, per list | | `List` of `Struct` | any of the above over a `[VortexRecord]` type | `ListOf()`: a `Range` per row, `Elements` as `Columns` | the elements are read in one pass, then cut per row | | `Map` | `Dictionary`, `IDictionary`, `IReadOnlyDictionary` | `MapOf()`: a `Range` per row, `Keys` and `Values` | scalar keys and values, and a key is never null | -| `vortex.date` | `DateOnly` | an indexer, `Storage()` or `Storage()` | | -| `vortex.time` | `TimeOnly` | an indexer, `Storage()`, `Unit` | | -| `vortex.timestamp` | `DateTime` when naive or UTC, `DateTimeOffset` with a zone | an indexer, `Storage()`, `Unit`, `TimeZone` | see [temporal extensions](#3-temporal-extensions-one-resolution-at-binding) | +| `vortex.date` | `DateOnly` | an indexer, `AsStorage()` or `AsStorage()` | | +| `vortex.time` | `TimeOnly` | an indexer, `AsStorage()`, `Unit` | | +| `vortex.timestamp` | `DateTime` when naive or UTC, `DateTimeOffset` with a zone | an indexer, `AsStorage()`, `Unit`, `TimeZone` | see [temporal extensions](#3-temporal-extensions-one-resolution-at-binding) | | `vortex.uuid` | `Guid` | an indexer | the byte order is converted explicitly, never implied | -| an extension registered on the session | the registered type | an indexer, `Storage()` | see [the session](14-public-api.md#2-the-session) | +| an extension registered on the session | the registered type | an indexer, `AsStorage()` | see [the session](14-public-api.md#2-the-session) | `Union` and `Variant` columns appear in a schema but no `T` maps to them. An `enum` member maps to its underlying integer. @@ -72,7 +72,7 @@ The storage width is carried in `vortex.decimal`'s metadata as `values_type`: `i declare any wider one, and often does. Upstream allows precision-2 values in an `i256` buffer, and a decimal column from Arrow or Parquet arrives as `i128` or `i256` whatever its precision. The width is therefore read from `values_type`, and only a width narrower than the precision requires is refused. -`Storage()` exposes what the file declared, for callers who do their own arithmetic without +`AsStorage()` exposes what the file declared, for callers who do their own arithmetic without widening. .NET has `Int128` but no `Int256`, so this library carries an internal `Int256` with what the decimal @@ -114,7 +114,7 @@ So the zone is resolved once, when a `DateTimeOffset` member or column binds to `TimeZoneInfo.FindSystemTimeZoneById`, and never per value. A naive or UTC timestamp binds as `DateTime`, with `Kind` `Unspecified` or `Utc`, and needs no database. A zone the host cannot resolve makes the `DateTimeOffset` binding fail with `VortexSchemaException`. The column still binds as -`long`, its storage, and `Storage()`, `Unit` and `TimeZone` let a caller convert with a zone of its +`long`, its storage, and `AsStorage()`, `Unit` and `TimeZone` let a caller convert with a zone of its own choosing. The machine dependence is confined to one call, at one moment, with a refusal rather than a guess. diff --git a/docs/design/09-contracts.md b/docs/design/09-contracts.md index 8de44459..5ad1a87e 100644 --- a/docs/design/09-contracts.md +++ b/docs/design/09-contracts.md @@ -18,7 +18,7 @@ that each one is not deduced differently at each call site. | `KeyCursorBuilder`, `KeyCursor` | not thread-safe, like the scan | | `ColumnsBuilder`, `ColumnBuilder` | one thread. The builder a writer hands out belongs to the writer and is reused after every `WriteAsync` | | `VortexFileWriter` (an append included), `VortexFileIndexer` | one writer per file. An append is not atomic, and `VortexFileRepair` truncates a torn one | -| `ScanPlan`, `CountPlan`, `OrderPlan`, `KeyPlan`, `ScanStatistics`, `WriteReport` | immutable records | +| `ScanPlan`, `CountPlan`, `OrderPlan`, `KeyPlan`, `ScanMetrics`, `WriteReport` | immutable records | | decoders and kernels | pure functions over borrowed memory, with no shared mutable state | ## 2. Parallelism @@ -127,7 +127,7 @@ bytes: `segments-requested`, `bytes-requested`, `zones-pruned`, `zones-total`, ` `cursor-seeks`, `cursor-steps`, `count-blocks-proven`, `count-blocks-decoded`, `key-order-windows` and `key-order-window-splits`. -The third works per query: a scan's `ExplainAsync` is its plan before it runs, and its `Statistics` +The third works per query: a scan's `ExplainAsync` is its plan before it runs, and its `Metrics` the same quantities measured afterwards ([observability.md](../guide/observability.md)). ## 6. Licensing and attribution diff --git a/docs/design/11-write-strategy.md b/docs/design/11-write-strategy.md index 10d51ba3..ee71b14e 100644 --- a/docs/design/11-write-strategy.md +++ b/docs/design/11-write-strategy.md @@ -232,7 +232,7 @@ summarizes on one thread. ### 3.8 Appending to a file -`VortexSession.AppendAsync(path)` returns a writer that continues a file whose shape this writer can +`VortexSession.OpenWriterAsync(path)` returns a writer that continues a file whose shape this writer can extend (a struct of columns, each a chunked layout of flat segments, all chunked alike), and refuses any other file with the advice to rewrite it. The old segments stay where they are. New blocks are numbered from the old row count, so their boundaries are the ones a single write would have made. @@ -328,7 +328,7 @@ Two more promises complete the contract: - A scan takes one lease per segment. The live splits register their segments in one request, each distinct segment is read once in as few requests as coalescing allows, and every block in it - decodes from that lease. `ScanStatistics.Requests` is the plan's distinct segments, and + decodes from that lease. `ScanMetrics.Requests` is the plan's distinct segments, and `BytesRequested` their bytes, within the coalescing gap. - A selection can replace a compaction. A filtered batch is compacted to its surviving rows by default. With `ScanOptions.Compact = false`, each live block is delivered whole with the @@ -343,7 +343,7 @@ The index directory is read lazily, on the first filter that could use it, and u the tail the open already read. The runs are read on the first query that needs them and cached on the file, shared by every scan. `ExplainAsync` returns the plan without executing it (blocks, blocks pruned by each structure with what consulting it cost, segments and bytes to read), and the scan's -`Statistics` report the same quantities once it has run, counted by the same sink. +`Metrics` report the same quantities once it has run, counted by the same sink. ## 7. Determinism, and its limit diff --git a/docs/design/12-index-reads.md b/docs/design/12-index-reads.md index c1ada0c3..b4ee17b8 100644 --- a/docs/design/12-index-reads.md +++ b/docs/design/12-index-reads.md @@ -96,10 +96,10 @@ lands on its lowest row, and `AtOrBefore` on its highest. A range `[a, b)` is `A ### 3.2 Stepping and direction -`NextAsync` and `PrevAsync` move one entry in `(key, row)` order and return false past the ends. +`NextAsync` and `PreviousAsync` move one entry in `(key, row)` order and return false past the ends. `NextKeyAsync` moves to the first entry of the next distinct key by seeking `After(Key)` rather than stepping through the duplicates. That is a loose index scan, one seek per group, so a `GROUP BY` over -a key with a million rows costs one seek per value. `PrevKeyAsync` is `Before(Key)`. +a key with a million rows costs one seek per value. `PreviousKeyAsync` is `Before(Key)`. A cursor has a direction. Stepping the other way seeks again at the current entry, which costs `O(r log n)` for `r` runs, because a heap of run positions does not run backwards. Every merging @@ -138,7 +138,7 @@ all. ### 3.5 Rank, select, and the count of a key `RankAsync(k)` is the number of entries with a key `< k`, `SeekRankAsync(i)` positions on the entry -of rank `i`, and `KeyCountAsync()` is the number of entries that share the current key. All three +of rank `i`, and `CountAtKeyAsync()` is the number of entries that share the current key. All three are exact on the two sources with rows, and refused on the others. A sorted array is its own counter: rank is the sum over runs of each run's lower bound, in `O(r log n)`, and select is a binary search over positions with a rank computed at each probe, in `O(r log² n)`. @@ -239,7 +239,7 @@ process](05-benchmarks.md#3-in-one-process-after-warm-up), measured on [the benc page](../guide/benchmarks.md#in-one-process-after-warm-up)). So it is the tool for a selective range, a top-k or a merge, not for a whole uncorrelated column, where sorting the scan's output costs less. The plan's `OrderPlan` states the source, the runs the range reaches and the entries it admits -before the read, and the scan's statistics count the windows and the splits they touched afterwards. +before the read, and the scan's metrics count the windows and the splits they touched afterwards. The dataset's key-ordered reads and its compaction merge are built on this, one call per object (see [what a compaction does](13-dataset.md#53-what-a-compaction-does) and [order beyond the diff --git a/docs/design/14-public-api.md b/docs/design/14-public-api.md index ee6cabaf..39fd0f56 100644 --- a/docs/design/14-public-api.md +++ b/docs/design/14-public-api.md @@ -30,6 +30,7 @@ Seven rules decide every signature: query) ends in `Async`: `CountAsync`, `AverageAsync`, `ToRecordsAsync`. A method that only describes the query does not: `Where`, `GroupBy`, `Select`, `Take`, whose result is run by `await foreach`. A builder never reads, and what it checks is in the schema the open already read. + The other rules a name follows are in [names](#11-names). 7. Batches flow. Every source, operator and sink exchanges batches, pulled by its consumer and decoded ahead within a bounded window. A query answers as soon as its first batch is final, holds what is still open rather than what it read, stops reading when its consumer stops, and allocates nothing @@ -197,7 +198,7 @@ record reads, with as many values as the record has members, through `As` or a `Projection` of computed values the same way. `As()` turns any of them into a `Scan` over the result, filled by the result's columns, on which every operator and sink of a scan runs again: a filter, a group by of the groups, a write. -`AggAsync` answers a whole scan into a record the same way. +`AggregateAsync` answers a whole scan into a record the same way. A query runs block by block on the encoded form (a dictionary key by code, a run-end key by run, a sorted key by run detection, a constant one whole) and never hands a batch to the caller's code. It @@ -225,8 +226,8 @@ in no promised order without one. | `ToRecordsAsync()` | `IAsyncEnumerable` | a copy per row, and an allocation per row for text, lists and nested classes | the first split | | `Select(r => e)` | `Projection`, a value computed per row | the same as `ToRecordsAsync`, for the columns its element names | the first split | | `Select(r => (…)).As()` | a `Scan` of values computed per row | nothing per batch, for the columns its elements name | the first split | -| `CountAsync`, `AnyAsync`, `MinAsync`, `MaxAsync`, `SumAsync`, `AverageAsync`, `CountDistinctAsync`, `AggregateAsync`, `AggAsync(a => e)` | one value | the statistics when they settle it, blocks otherwise (see [answers without rows](12-index-reads.md#4-answers-without-rows)) | at once when the statistics settle it, at the end of the pass otherwise | -| `AggAsync(a => (…))` | several answers in one pass, into the record `TResult` | the same | the same | +| `CountAsync`, `AnyAsync`, `MinAsync`, `MaxAsync`, `SumAsync`, `AverageAsync`, `CountDistinctAsync`, `AggregateAsync(a => e)`, and `AggregateAsync` with an aggregator of one's own | one value | the statistics when they settle it, blocks otherwise (see [answers without rows](12-index-reads.md#4-answers-without-rows)) | at once when the statistics settle it, at the end of the pass otherwise | +| `AggregateAsync(a => (…))` | several answers in one pass, into the record `TResult` | the same | the same | | `GroupBy(…).Select(g => r)` | `Aggregation`, one value per group | one state per group, only the open groups on a key that streams, and `k` groups for a top-k | the first group closed on a key that streams, the end of the pass otherwise | | `GroupBy(…).Select(g => (…)).As()` | a `Scan` over the groups, its batches borrowed | the same, and nothing per batch or per group | the same | | `writer.WriteAsync(scan)` | the batches of any scan, written as they come | the writer's own work, and a rollup builds no record | | @@ -279,17 +280,17 @@ guide's writing pages show each form. | type | what it says | |---|---| | `VortexOpenOptions` | the initial tail read, a known length or schema, the decompression cap, statistics verification, a torn tail's policy, index fragments to attach | -| `ScanOptions` | batch cap, prefetch, degree, `Compact`, and the `Pruning` and `UseIndexes` switches, which exist to check the structures and never change a result | +| `ScanOptions` | batch cap, prefetch, degree, `Compact`, and the `UseStatistics` and `UseIndexes` switches, which exist to check the structures and never change a result | | `VortexWriteOptions` | blocks and chunk targets, the compression profile and per-column hints, the target edition, statistics, string bounds, the index policy, the identity, user metadata, the degree | | `IndexPolicy`, `EncodingHint`, `CompressionProfile` | see [the index policy](10-indexes.md#6-the-policy) and [choosing encodings](11-write-strategy.md#34-choose-exact-verdicts-then-bounded-trials) | -| `ScanPlan`, `PruningStep`, `CountPlan`, `OrderPlan`, `GroupPlan`, `GroupKeyPlan`, `GroupOrdering` | the plan before a read: blocks, live blocks, segments and bytes to read, what each structure pruned and what consulting it cost, and how a count, an order and a group by will be answered (see [plan and statistics](16-queries.md#10-plan-and-statistics)) | -| `ScanStatistics`, `GroupStatistics` | the same quantities, measured after the scan, and for a group by its groups, the most held at once and their bytes, what its core did, how each key block was grouped, and the time to its first batch | +| `ScanPlan`, `PruningStep`, `CountPlan`, `OrderPlan`, `GroupPlan`, `GroupKeyPlan`, `GroupOrdering` | the plan before a read: blocks, live blocks, segments and bytes to read, what each structure pruned and what consulting it cost, and how a count, an order and a group by will be answered (see [plan and metrics](16-queries.md#10-plan-and-metrics)) | +| `ScanMetrics`, `GroupMetrics` | the same quantities, measured after the scan, and for a group by its groups, the most held at once and their bytes, what its core did, how each key block was grouped, and the time to its first batch | | `WriteReport` | what the writer chose, per column and per chunk, and every index built or abandoned | | `VortexDiagnostics` | the names of the meter and the activity source (see [observability](09-contracts.md#5-observability)) | | `VortexException` and its four kinds | a malformed file, an unsupported component with its id and `ComponentKind`, a schema that does not fit, and a query that outgrows its memory or scratch budget (see [error handling](03-architecture.md#5-error-handling)) | | `VortexEditions` | the default, newest and floor editions, and the Rust version that reads each | -The plan and the statistics count in the same units. A plan's `Segments` and `BytesToRead` are what +The plan and the metrics count in the same units. A plan's `Segments` and `BytesToRead` are what the source will be asked for, each segment once, and `Requests` and `BytesRequested` are what it was asked for, so on a file with statistics the two agree. `BlocksDecoded` counts the blocks that reached the plain form, so a block answered from a dictionary's codes or a run's lengths is not counted as @@ -339,7 +340,7 @@ The analyzers ship in `Vorticity.Generators` ([diagnostics.md](../guide/diagnost | VX1007 | a type that cannot be a record: generic, `ref struct`, static, abstract, file-local, or without a usable constructor | | VX1008 | a member the reader cannot fill | | VX1009 | a component of a `GroupBy` key that is not a symbol of the scan, such as a literal or a captured value | -| VX1010 | an `As()` or an `AggAsync` whose record members do not take the selected values, position by position | +| VX1010 | an `As()` or an `AggregateAsync` whose record members do not take the selected values, position by position | | VX1011 | several selected values read without a record (enumerated, or awaited without `TResult`). It names `As()` and the record to declare | VX1001 to VX1004 are warnings. VX1005 to VX1008 are errors, and the generator emits nothing for a type @@ -365,3 +366,57 @@ VX1011 accompanies the compiler's own error, to name the fix. - `vxdump` is written against the tool path, the inspection members of `VortexFile` and the plans only, and published ahead of time. A section it could not print from the public API would be a gap in the API, not in the tool ([vxdump.md](../guide/vxdump.md)). + +## 11. Names + +A name says what a member hands back and what it costs to get, and a word means the same thing +wherever it appears. Rule 6 decides the `Async` suffix, and these rules decide the rest: + +1. A method's prefix says what it returns. `As` is the same data in another form, still borrowed + from the batch it comes from: `As()`, and the forms of a column, `AsCanonical()`, + `AsDictionary()`, `AsRunEnd()`, `AsConstant()` and `AsStorage()`. `To` is a copy the caller owns: + `ToOwned()`, `ToArray()`, `ToListAsync()`, `ToRecordsAsync()`. `Get` is a value computed or read: + `GetString(i)`, `GetLayoutAsync()`, `GetValidLengthAsync()`. A method without a prefix returns + what is already there, a part of the object or a value it holds: `Column(i)`, `Row(i)`, + `Builder()`, `ChunkRowsOf(i)`. +2. A method is a verb, so a method that runs names what it does: `ListObjectsAsync`, + `CountAtKeyAsync`, `OpenWriterAsync`. The operators and aggregates of a query keep the names LINQ + gives them (`Where`, `Select`, `Count`, `Average`), and a factory is named after what it builds + (`VortexType.List`, `IndexPolicy.Bloom`), as `Expression.Constant` is. +3. A word keeps one meaning. Statistics describe the data: a file's `Statistics`, its zone maps, + `UseStatistics`, `WriteStatistics`, `VerifyStatistics`. Metrics count what a query did: `Metrics`, + `ScanMetrics`, `GroupMetrics`. `Create` makes something new and `Open` takes something that + exists (`CreateWriter`, `OpenWriterAsync`, `OpenAsync`), and `Append` writes rows after those + already there (`VortexDataset.AppendAsync`, `AppendIndexesAsync`). A dataset deletes and updates + rows (`DeleteAsync`, `UpdateAsync`), and lists, removes and replaces objects (`ListObjectsAsync`, + `RemoveObjectsAsync`, `ReplaceObjectsAsync`). +4. Names are written out. The three exceptions are terms of the domain a reader meets on every + page: `Sym`, `Expr` and `DType`. +5. A quantity of bytes ends in `Bytes`: `MaxDecompressedBytes`, `IndexCacheBytes`, `CapacityBytes`. + The exceptions are the extent of a file or a range (`Length`, `Offset`), a limit the Vortex format + names (`MaxPostscriptSize`), and a member that comes from the BCL (`MaxBufferSize`). +6. A switch in an options type is a verb (`UseIndexes`, `UseStatistics`, `WriteStatistics`, + `MapFiles`, `UseLeases`), and a fact is an adjective or starts with `Is` or `Has` (`Durable`, + `IsClustered`, `HasRows`). +7. A unit that counts a quantity is plural, as in `TimeSpan.FromMilliseconds`: + `TimeUnit.Milliseconds`, `TimeUnit.Days`. A period of the calendar is singular, since + `Truncate(CalendarUnit.Day)` rounds down to one day. + +### Renamed + +The members these rules renamed, for a caller moving to the current names: + +| before | now | +|---|---| +| `scan.AggAsync(a => e)`, `scan.AggAsync(a => (…))` | `scan.AggregateAsync(a => e)`, `scan.AggregateAsync(a => (…))` | +| `Statistics` on a scan, a projection and an aggregation, `ScanStatistics`, `GroupStatistics` | `Metrics`, `ScanMetrics`, `GroupMetrics` | +| `session.AppendAsync(path)` | `session.OpenWriterAsync(path)` | +| `dataset.ObjectsAsync()`, `RemoveAsync(objects)`, `ReplaceAsync(removed, added)` | `ListObjectsAsync()`, `RemoveObjectsAsync(objects)`, `ReplaceObjectsAsync(removed, added)` | +| `cursor.KeyCountAsync()`, `PrevAsync()`, `PrevKeyAsync()`, `SeekOp` | `CountAtKeyAsync()`, `PreviousAsync()`, `PreviousKeyAsync()`, `SeekMode` | +| `schema.TryGetField(utf8, out int index)` | `schema.IndexOf(utf8)`, -1 when no field matches | +| `column.Canonical()`, `column.Storage()` | `column.AsCanonical()`, `column.AsStorage()` | +| `VortexFileRepair.ValidLengthAsync(path)` | `VortexFileRepair.GetValidLengthAsync(path)` | +| `MaxDecompressedSize`, `MaxBatchDecompressedSize` and `InitialReadSize` on `VortexOpenOptions`, `VortexLimits.DefaultMaxDecompressedSize` | `MaxDecompressedBytes`, `MaxBatchDecompressedBytes`, `InitialReadBytes`, `DefaultMaxDecompressedBytes` | +| `SegmentCache.Capacity`, `SegmentCache.Size` | `CapacityBytes`, `HeldBytes` | +| `ScanOptions.Pruning`, `VortexWriteOptions.Statistics`, `CompactionSchedule.Leases` | `UseStatistics`, `WriteStatistics`, `UseLeases` | +| `CompactionJob.Objects`, `GroupKeyPlan.Values`, `VortexDataset.ClusteringKeyPaths` | `ObjectKeys`, `ValueRange`, `ClusteringKey` | diff --git a/docs/design/15-compaction.md b/docs/design/15-compaction.md index 86782a9e..1c83e8a8 100644 --- a/docs/design/15-compaction.md +++ b/docs/design/15-compaction.md @@ -119,7 +119,7 @@ touches, and needs nothing else. the `Idle` sleep, and `Loops` and `Loop` for loops that know one another. The planner ranks the due jobs so that no two read or write the same level (a job on levels that share nothing never takes another's input), and a loop takes the one ranked at its index. A descent and a read of every leaf -rank the same jobs. `Leases` turns on leases instead, one per level a job touches, +rank the same jobs. `UseLeases` turns on leases instead, one per level a job touches, `leases//`, tried job by job in rank order. The key dates the lease, so vacuum deletes the ones that ended a window ago without a head request each. A job still running when its lease ends may collide with another loop's, which one commit then abandons, so a `LeaseSpan` of a diff --git a/docs/design/16-queries.md b/docs/design/16-queries.md index ca53fb9f..af2d9b8c 100644 --- a/docs/design/16-queries.md +++ b/docs/design/16-queries.md @@ -63,7 +63,7 @@ two columns of a file, and it buys names, batches by default, and no cap on the | `Aggregation`, `Aggregation`, `Projection`, `Projection` | `Skip`, `Take`, and `Distinct()` on a projection | the same kind, or an aggregation for `Distinct` | | `Aggregation`, `Aggregation`, `Projection`, `Projection` | `As()` | `Scan` over the result ([`As`](#71-astrecord)) | | `Aggregation`, `Projection` | `WithCancellation`, `ToListAsync`, `ToArrayAsync`, `ToValuesAsync` | the same kind, `ValueTask<…>` ([values and records](#72-values-and-records)), or `IAsyncEnumerable` | -| `Scan` | `AggAsync(a => e)`, `AggAsync(a => (e1, e2, …))`, `CountAsync`, `AverageAsync` and the other single answers | `ValueTask<…>` ([one row](#8-one-row-the-aggregates-of-the-whole-scan)) | +| `Scan` | `AggregateAsync(a => e)`, `AggregateAsync(a => (e1, e2, …))`, `CountAsync`, `AverageAsync` and the other single answers | `ValueTask<…>` ([one row](#8-one-row-the-aggregates-of-the-whole-scan)) | An operator that returns a query only describes it and runs nothing. `await foreach` runs it, `ExplainAsync` plans it without reading a data segment, and a method that returns a `ValueTask` runs @@ -95,12 +95,12 @@ What does not compile marks where a query over one table ends: | arithmetic on a column or a result | the operators are `[Obsolete(error)]`, since computing with results is C# after the sink | | a literal of a different type from the symbol's | as in a filter ([the symbolic algebra](14-public-api.md#52-the-symbolic-algebra)) | | `await` on a builder, `foreach` over a `GroupedScan` | a builder is not awaitable, and a group is not a value: a `Select` says what to deliver of it | -| several values read without a record: `await foreach` over a `select (a, b)`, `AggAsync(a => (x, y))` | several values have no .NET type until a record gives them one (CS8411, CS0411) | +| several values read without a record: `await foreach` over a `select (a, b)`, `AggregateAsync(a => (x, y))` | several values have no .NET type until a record gives them one (CS8411, CS0411) | What compiles but still cannot run throws when the operator runs ([errors](#11-errors-cancellation-disposal)), and an analyzer flags it at compile time where it can see it: VX1009 for a key component that is not -a symbol, and VX1010 for an `As` or an `AggAsync` whose record does not match the -values. VX1011 accompanies the compiler's error of the last row and names the fix, `As()` +a symbol, and VX1010 for an `As` or an `AggregateAsync` whose record does not match +the values. VX1011 accompanies the compiler's error of the last row and names the fix, `As()` and the record to declare. ## 2. The flow @@ -135,7 +135,7 @@ records](#72-values-and-records)). | `OrderBy` on a key that streams in that order | nothing more | its input's | | `OrderBy` then `Take(k)` | `k` groups | the end of its input | | `OrderBy`, any other | every group | the end of its input | -| `AggAsync` and the single answers | one state per aggregate | the end of its input, or at once when the statistics answer ([one row](#8-one-row-the-aggregates-of-the-whole-scan)) | +| `AggregateAsync` and the single answers | one state per aggregate | the end of its input, or at once when the statistics answer ([one row](#8-one-row-the-aggregates-of-the-whole-scan)) | This gives three promises, each backed by a gate ([how it is tested](#13-how-it-is-tested)): @@ -200,7 +200,7 @@ the order may depend on row order (a window, a first or last row, a tie, a custo backwards the rows of a group come last first and a window would take the groups from the other end. Without a window, a descending order uses the blocking pass, on every lane and with the blocks the zone maps settle (see [statistics and zone maps](#93-statistics-and-zone-maps)). The plan says which -component streams, or why none does ([plan and statistics](#10-plan-and-statistics)). +component streams, or why none does ([plan and metrics](#10-plan-and-metrics)). ### 2.4 Stopping early @@ -329,8 +329,8 @@ decimal by its unscaled value (since a column has one scale), and a timestamp by ## 5. Aggregates -The aggregates are the members of `Group`, and of `Aggregates` for `AggAsync` -([one row](#8-one-row-the-aggregates-of-the-whole-scan)), which has every one but `Key`. Each takes a +The aggregates are the members of `Group`, and of `Aggregates` for +`AggregateAsync` ([one row](#8-one-row-the-aggregates-of-the-whole-scan)), which has every one but `Key`. Each takes a lambda over the rows' probe, as LINQ's do, and the probe only exists inside it. | member | result | skips | with no value | @@ -349,8 +349,8 @@ lambda over the rows' probe, as LINQ's do, and the probe only exists inside it. when it is unknown (see [three-valued logic](08-semantics.md#3-three-valued-logic)). `All(x => p)` is `Count(x => p) == Count()`, and `Any(x => p)` is `Count(x => p) > 0`. "With no value" only happens in a group for an aggregate of a filtered group ([a filtered -group](#52-a-filtered-group)) or of a column whose every row is null, and in `AggAsync` for a scan -that keeps no row. +group](#52-a-filtered-group)) or of a column whose every row is null, and in `AggregateAsync` for a +scan that keeps no row. ### 5.1 Sums: exact, and the same bits whatever the cut @@ -412,7 +412,7 @@ select (g.Key, g.Count(), g.Where(x => x.Status >= 500).Count(), g.Where(x => x. `g.Count(x => p)` is `g.Where(x => p).Count()`, two `Where` calls join with `AND`, and `a.Where(x => p)` filters the scan's aggregates the same way. The predicate is evaluated once per batch however many aggregates read it, on the encoded form as a scan's filter is, and the columns it -reads join the pass, which the plan shows ([plan and statistics](#10-plan-and-statistics)). +reads join the pass, which the plan shows ([plan and metrics](#10-plan-and-metrics)). ### 5.3 A chosen row @@ -466,8 +466,8 @@ block](#91-keys-block-by-block)). A state is a result like any other, so its type must be one a column can hold: a number, a decimal, text, a date or a time, a uuid, a bool, or a `[VortexRecord]`, which a result holds as a struct column of its members and which a record of the selection declares as a member of that type. `Select` -refuses a state of any other type with `VortexSchemaException`, while `AggregateAsync`, one answer -over a whole scan, delivers any state as it is. +refuses a state of any other type with `VortexSchemaException`, while `AggregateAsync` of a single +answer over a whole scan delivers any state as it is. ### 5.5 One aggregate, wherever it is written @@ -588,12 +588,12 @@ await foreach (var (hour, city, readings, mean) in hourly) // the Scan …)` | the key cursor over the result's keys, sorted in memory unless the result arrives in their order, reserved under the session's memory budget until the cursor is disposed | | `Rows(range)`, `Rows(indices)` | positions in the order the result is delivered, read as a skip and a take of the stream | | `With(options)` | the options of what runs on the result, while the query keeps its own | -| `ExplainAsync`, `Statistics` | the query's plan with the result's operators after it, and what the whole pipeline did | +| `ExplainAsync`, `Metrics` | the query's plan with the result's operators after it, and what the whole pipeline did | A result has no zone maps and no indexes, so its filters evaluate every batch, and a filter on a key is better written before the `select`, where it prunes ([groups kept](#62-groups-kept-where)). A @@ -663,20 +663,21 @@ bits however the log was split ([sums](#51-sums-exact-and-the-same-bits-whatever ## 8. One row: the aggregates of the whole scan -`scan.AggAsync(a => e)` computes one answer, and `scan.AggAsync(a => (e1, e2, …))` computes -as many as are written, in one pass, into the record `TResult`, whose members take the answers in -order the way `As` has them take a `select`'s values, VX1010 included. The one type argument is all -the call names: +`scan.AggregateAsync(a => e)` computes one answer, and +`scan.AggregateAsync(a => (e1, e2, …))` computes as many as are written, in one pass, into +the record `TResult`, whose members take the answers in order the way `As` has them take a +`select`'s values, VX1010 included. The one type argument is all the call names: ```csharp -long readings = await scan.AggAsync(a => a.Count()); -Summary s = await scan.AggAsync(a => (a.Min(x => x.Celsius), a.Max(x => x.Celsius), a.Count())); +long readings = await scan.AggregateAsync(a => a.Count()); +Summary s = await scan.AggregateAsync(a => (a.Min(x => x.Celsius), a.Max(x => x.Celsius), a.Count())); ``` The answers come from the catalog of [aggregates](#5-aggregates), except the key, with a chosen row taken in the order of the whole scan, and `a.Where(x => p)` filters them. A single answer also has its own call: `CountAsync`, `AnyAsync`, `MinAsync`, `MaxAsync`, `SumAsync`, `AverageAsync`, -`CountDistinctAsync`, `AggregateAsync`. The file statistics answer a count, a minimum and a maximum +`CountDistinctAsync`, and `AggregateAsync` for [an aggregator of one's +own](#54-an-aggregator-of-ones-own). The file statistics answer a count, a minimum and a maximum before any read, and the sum and the mean of an integer or a decimal column when they carry a sum (this library's writer records none, but another writer may), and the `ValueTask` is then already complete (see [answers without rows](12-index-reads.md#4-answers-without-rows)). A float column's sum @@ -1040,7 +1041,7 @@ the merge is done, a million keys took 0.83 of the time and four aggregates 0.68 Not done yet: a key cursor over a result larger than its budget, which a merge of runs cannot seek in. -## 10. Plan and statistics +## 10. Plan and metrics `ExplainAsync` on an aggregation returns the scan's `ScanPlan` with `Grouping`, a `GroupPlan`, which is null without a group by. It states what costs, so the shape of LINQ does not hide it, from the @@ -1059,8 +1060,8 @@ query and the statistics alone: | `MostGroups` | the most groups the statistics allow, when they bound every component | A key's form is only known once its block is read, so the plan says what is possible and the -statistics say what happened. Once the result is read, its `ScanStatistics` carry `Grouping`, a -`GroupStatistics` with: +metrics say what happened. Once the result is read, its `ScanMetrics` carry `Grouping`, a +`GroupMetrics` with: * `Groups`, the groups the pass found before any operator on them, and `PeakGroups`, the most held at once, which a streaming group by keeps small, and which a core delivering part by part limits to @@ -1086,7 +1087,7 @@ counters, which every batch touches. | when | what | |---|---| | compile time | the table in [the shape](#1-the-shape), and VX1009, VX1010, VX1011 | -| the operator runs, and the lambda with it | a key component that is not a symbol (`ArgumentException`), a key or input type that does not group or aggregate (`VortexSchemaException`), a result that is not one or a chosen row's column used as an input (`InvalidOperationException`), an order on a result that has none (`ArgumentException`), a function's argument out of range (`ArgumentOutOfRangeException`), `As` or `AggAsync` on a record that does not match (`VortexSchemaException`) | +| the operator runs, and the lambda with it | a key component that is not a symbol (`ArgumentException`), a key or input type that does not group or aggregate (`VortexSchemaException`), a result that is not one or a chosen row's column used as an input (`InvalidOperationException`), an order on a result that has none (`ArgumentException`), a function's argument out of range (`ArgumentOutOfRangeException`), `As` or `AggregateAsync` on a record that does not match (`VortexSchemaException`) | | enumeration | a sum past its type (`OverflowException`), a query over its memory budget (`VortexMemoryException`), and the file's own errors (see [exceptions](14-public-api.md#7-options-plans-reports-diagnostics-exceptions)) | Cancellation is checked at every batch boundary, and, in the work a blocking operator does after its @@ -1152,7 +1153,9 @@ by, which is what the sum's overhead is set against. The query bench turns each |---|---| | `GroupBy(…).AggAsync(g => …)` | `GroupBy(…).Select(g => …)` | | `await foreach (var (city, n) in ….AggAsync(g => (g.Key, g.Count())))`, a tuple per group | `….Select(g => (g.Key, g.Count())).As()`, read as batches or with `ToRecordsAsync()` | -| `var (min, max) = await scan.AggAsync(a => (…, …))` | `MinMax m = await scan.AggAsync(a => (…, …))` | +| `var (min, max) = await scan.AggAsync(a => (…, …))` | `MinMax m = await scan.AggregateAsync(a => (…, …))` | +| `scan.AggAsync(a => e)` | `scan.AggregateAsync(a => e)` | +| `scan.Statistics`, `ScanStatistics`, `GroupStatistics` | `scan.Metrics`, `ScanMetrics`, `GroupMetrics` | | at most eight values in a selection | as many as the record has members | | `GroupBy(r => (r.City, r.Day))`, then `g.Key.Item1` | `g.Key.City` | | `Avg`, `AvgAsync` | `Average`, `AverageAsync` | @@ -1163,3 +1166,5 @@ by, which is what the sum's overhead is set against. The query bench turns each | `Sum` of a `float` as a `float` | as a `double` | | a float sum whose last bits depended on the degree | the same bits whatever the degree and the split, and closer to the exact sum | | `g.Aggregate, WelfordState>(r => r.Celsius)` | the same, or `g.Welford(x => x.Celsius)` through the aggregator's extension | + +The other members renamed since are listed in [the public API design](14-public-api.md#renamed). diff --git a/docs/guide/aggregates.md b/docs/guide/aggregates.md index 372fb57b..aca881b0 100644 --- a/docs/guide/aggregates.md +++ b/docs/guide/aggregates.md @@ -6,23 +6,23 @@ own, on as many cores as the session allows. ```csharp Scan recent = file.Scan().Where(r => r.Day >= 900); Summary s = await recent - .AggAsync(a => (a.Min(r => r.Celsius), a.Max(r => r.Celsius), a.Count(), a.CountDistinct(r => r.City))); + .AggregateAsync(a => (a.Min(r => r.Celsius), a.Max(r => r.Celsius), a.Count(), a.CountDistinct(r => r.City))); [VortexRecord] public partial record struct Summary(double? Min, double? Max, long Rows, long Cities); ``` ``` -AggAsync, four answers 1.9 ms min 10.1, max 49.9, 100000 rows, 8 cities; 1 blocks decoded +AggregateAsync, four answers 1.9 ms min 10.1, max 49.9, 100000 rows, 8 cities; 1 blocks decoded ``` ## Several answers in one pass -`AggAsync` takes a lambda over an `Aggregates` and computes all its answers in a single pass -over the rows the scan keeps. One answer comes back as itself, as in -`await scan.AggAsync(a => a.Count())`. Several answers, any number of them, go into the record named -by `AggAsync`, whose members take them in order. Each member has the type of its answer or -its nullable form. +`AggregateAsync` takes a lambda over an `Aggregates` and computes all its answers in a single +pass over the rows the scan keeps. One answer comes back as itself, as in +`await scan.AggregateAsync(a => a.Count())`. Several answers, any number of them, go into the record +named by `AggregateAsync`, whose members take them in order. Each member has the type of +its answer or its nullable form. The members of `a` are `Count()`, `CountDistinct`, `Sum`, `Min`, `Max`, `Average`, `Variance`, `StandardDeviation` and `Aggregate`, each over a column named as in a filter. There are also `Count`, @@ -36,7 +36,8 @@ Only one of them had a column brought to its plain form, and the others were rea dictionaries. For a single answer, the scan also has its own sinks: `CountAsync`, `AnyAsync`, `MinAsync`, -`MaxAsync`, `SumAsync`, `AverageAsync`, `CountDistinctAsync` and `AggregateAsync`. An aggregate runs +`MaxAsync`, `SumAsync`, `AverageAsync`, `CountDistinctAsync`, and `AggregateAsync` with an +[aggregator of your own](#an-aggregator-of-your-own). An aggregate runs block by block on the encoded form, never hands a batch to your code, keeps one state per chunk and merges the states at the end. [The query design](../design/16-queries.md#92-aggregates-block-by-block) lists what each encoding lets it skip. @@ -139,7 +140,7 @@ public partial record struct CityHeat(string City, long Rows, long Hot, double? for some row of the group, and `g.All(r => p)` whether it holds for every row. The predicate uses the language of a scan's `Where`, and a row counts where it is true, not where it is unknown. Every temperature present is above 10, but one in fifty is missing, so `All` is false. Two `Where` calls -keep the rows both keep. In `AggAsync`, `a.Where`, `a.Count(p)`, `a.Any` and `a.All` do the same +keep the rows both keep. In `AggregateAsync`, `a.Where`, `a.Count(p)`, `a.Any` and `a.All` do the same over the scan's rows. Each predicate is evaluated once per batch, however many aggregates read it, and its columns join diff --git a/docs/guide/append-and-repair.md b/docs/guide/append-and-repair.md index 0e20db94..e1e5a8e0 100644 --- a/docs/guide/append-and-repair.md +++ b/docs/guide/append-and-repair.md @@ -3,7 +3,7 @@ Add rows to a file you already wrote, give an append up, and recover a file whose tail was torn. ```csharp -await using (VortexFileWriter appender = await session.AppendAsync(path)) +await using (VortexFileWriter appender = await session.OpenWriterAsync(path)) { long resumeAt = appender.RowCount; // the file's row count, since it ended on a block await appender.WriteAsync(more.AsSpan(), ct); @@ -18,7 +18,7 @@ appended 5000 rows to 20000: resumed at 16384, RowCount 25000 after the write, t ## Where an append resumes -`AppendAsync` opens the file, keeps its chunks, and positions the writer after its rows. New rows +`OpenWriterAsync` opens the file, keeps its chunks, and positions the writer after its rows. New rows always follow the file's last row. What `appender.RowCount` says right after the open is where the rewrite starts: @@ -107,7 +107,7 @@ or that opens files it did not write, should set `Refuse`. ## Repairing ```csharp -long valid = await VortexFileRepair.ValidLengthAsync(path, ct); +long valid = await VortexFileRepair.GetValidLengthAsync(path, ct); VortexRepairResult repaired = await VortexFileRepair.RepairAsync(path, ct); // truncates to the last version that parses ``` @@ -116,18 +116,18 @@ valid length 29324; repaired: truncated True, 46919 -> 29324 the repaired file appends again: 24576 rows ``` -`ValidLengthAsync` finds the longest prefix that opens, without changing the file. `RepairAsync` +`GetValidLengthAsync` finds the longest prefix that opens, without changing the file. `RepairAsync` truncates the file to it, and leaves a valid file alone. After a repair, `TornTail` is null and the file accepts appends like any other. A file with no complete version has nothing to fall back to. A file cut short rather than appended to, such as an interrupted download or a full disk, throws `VortexFormatException` on open, and so do -`ValidLengthAsync` and `RepairAsync`. +`GetValidLengthAsync` and `RepairAsync`. ## Watch out * Nothing else may hold the file. An append opens it exclusively for writing, so a `VortexFile` still - open on the same path makes `AppendAsync` throw `IOException`. + open on the same path makes `OpenWriterAsync` throw `IOException`. * Each append draws a new identity, so every version of the bytes has its own. Pin it with `VortexWriteOptions.Identity` for a reproducible write ([writer-options.md](writer-options.md)). * An append takes the file's own settings when its options are null: the block size, the index diff --git a/docs/guide/blocks-and-chunks.md b/docs/guide/blocks-and-chunks.md index dd94493d..2c2a3f1f 100644 --- a/docs/guide/blocks-and-chunks.md +++ b/docs/guide/blocks-and-chunks.md @@ -106,7 +106,7 @@ are cut there however they arrive. * Flush when you need the bytes out, to bound memory or to feed a slow sink ([stream-to-an-object.md](stream-to-an-object.md)), not after every write. * An append keeps the file's block size, whatever its options say, and chunks its columns the same - way the file does. `AppendAsync` refuses a file whose columns were given chunks of their own + way the file does. `OpenWriterAsync` refuses a file whose columns were given chunks of their own (`ColumnChunkTargetBytes`) with `VortexUnsupportedException`. Rewrite such a file instead ([copy-a-file.md](copy-a-file.md)). * [11-write-strategy.md](../design/11-write-strategy.md) describes the design of the ingest, the diff --git a/docs/guide/cancel-work.md b/docs/guide/cancel-work.md index 5bb67379..77097995 100644 --- a/docs/guide/cancel-work.md +++ b/docs/guide/cancel-work.md @@ -83,7 +83,7 @@ a cancelled token does not guarantee nothing ran, so check the token yourself if ## A cursor A cancelled move throws and leaves the cursor without a position. `IsValid` returns false, and `Key`, -`Row` and `KeyCountAsync` throw `InvalidOperationException` until a move succeeds. Seek again: +`Row` and `CountAtKeyAsync` throw `InvalidOperationException` until a move succeeds. Seek again: ``` the cursor after it: valid False diff --git a/docs/guide/dataset-maintenance.md b/docs/guide/dataset-maintenance.md index 6147ba4f..6307b368 100644 --- a/docs/guide/dataset-maintenance.md +++ b/docs/guide/dataset-maintenance.md @@ -173,7 +173,7 @@ A loop runs the same jobs as `CompactAsync`, so it is safe against every writer loop. What several loops on one dataset can waste is a job that two of them run at once, which one commit then abandons. Loops that know about one another say so with `Loops` and `Loop`: each takes the due job ranked at its index, and no two ranked jobs read or write the same level. Loops that -cannot coordinate set `Leases` instead. Before a job, a loop creates `leases//` for each +cannot coordinate set `UseLeases` instead. Before a job, a loop creates `leases//` for each of its levels with put-if-absent, holds them until the end of the current `LeaseSpan` (a minute by default), and moves to the next job when another loop holds one. A loop remembers the leases it created, so it can run the next job on the same levels within the span without asking the store diff --git a/docs/guide/datasets.md b/docs/guide/datasets.md index fc1152fe..c242aa9d 100644 --- a/docs/guide/datasets.md +++ b/docs/guide/datasets.md @@ -76,7 +76,7 @@ imported: version 4, 120000 rows, 3 objects, lag 0 `ImportAsync` takes a key in the store, not a path on disk, so putting the bytes there is up to you. The file's columns must read as the dataset's, either the current ones or those of an earlier -schema of the dataset (see [Changing the schema](#changing-the-schema)). `ObjectsAsync()` lists the +schema of the dataset (see [Changing the schema](#changing-the-schema)). `ListObjectsAsync()` lists the objects of the version from the commit tree, without opening any object. ## Reading @@ -104,9 +104,9 @@ nothing, and `false` is a proof. Row positions are the dataset's. A batch's `StartRow`, `Rows(…)` and a key cursor's `Row` count the version's rows object after object, in the order a scan delivers them. A key cursor over the -clustering key merges the cursors of the objects, and its `KeyCountAsync` is the number of entries +clustering key merges the cursors of the objects, and its `CountAtKeyAsync` is the number of entries under the current key, here the thousand rows of day 75. It walks both ways like a file's cursor -(`SeekLastAsync`, `PrevAsync`, `PrevKeyAsync`, `AtOrBefore`, `Before`), and a step against the +(`SeekLastAsync`, `PreviousAsync`, `PreviousKeyAsync`, `AtOrBefore`, `Before`), and a step against the direction of the previous one seeks every object again at the current entry. Walking down, an object above level 0 opens only once the walk could reach its keys, while the level 0 objects all open at the seek since nothing bounds them from above. See [keys-in-order.md](keys-in-order.md). @@ -143,7 +143,7 @@ await using (ObjectDraft rewritten = dataset.StartObject()) .Rows(RowRange.FromLength(first.FirstRow, first.Rows)) .Where(r => r.Celsius.IsNotNull) .ToRecordsAsync()); - ReplaceResult replaced = await dataset.ReplaceAsync([first], [rewritten]); + ReplaceResult replaced = await dataset.ReplaceObjectsAsync([first], [rewritten]); } ``` @@ -153,9 +153,9 @@ removed the import: version 7, Applied, 109000 rows, 3 objects removing it again: Abandoned, version 7 ``` -`ReplaceAsync(removed, added)` swaps objects in one commit. A reader sees either the old ones or the -new ones, never both and never neither. `RemoveAsync(objects)` is the same with nothing added. Both -take `DataObject`s from `ObjectsAsync()`. The removed objects stay in the store, readable by the +`ReplaceObjectsAsync(removed, added)` swaps objects in one commit. A reader sees either the old ones +or the new ones, never both and never neither. `RemoveObjectsAsync(objects)` is the same with nothing +added. Both take `DataObject`s from `ListObjectsAsync()`. The removed objects stay in the store, readable by the versions that still name them, until vacuum deletes them. Removing an object the version no longer holds returns `Abandoned`: no version is created, the result names the latest one, and any objects a replace would have added are left for vacuum. diff --git a/docs/guide/diagnostics.md b/docs/guide/diagnostics.md index ac32ba9c..5f2bc1d9 100644 --- a/docs/guide/diagnostics.md +++ b/docs/guide/diagnostics.md @@ -351,10 +351,11 @@ every row of the result belongs in C# code after the query. A record does not match the values of a selection. This is an error. A selection of several values is read through a record: `As()` after a `Select`, and -`AggAsync` for the answers of a whole scan. The record's members, in declaration order, take -the elements in order, one each, and each member has its element's type or its nullable form. `As` and -`AggAsync` check this when they are built and throw `VortexSchemaException`, and the analyzer reads -the types of the tuple's elements in the lambda to flag the problem at compile time. +`AggregateAsync` for the answers of a whole scan. The record's members, in declaration +order, take the elements in order, one each, and each member has its element's type or its nullable +form. `As` and `AggregateAsync` check this when they are built and throw `VortexSchemaException`, +and the analyzer reads the types of the tuple's elements in the lambda to flag the problem at +compile time. ```csharp [VortexRecord] @@ -381,10 +382,11 @@ Declare one member per element, in the order of the tuple. Several values are read without a record. This is an error, reported beside the compiler's own. Several values have no .NET type until a record gives them one. So a `Select` of several values -returns an `Aggregation`, which is not enumerable, and `AggAsync` of several answers needs the record -they go into. The compiler refuses both cases, with CS8411 for an `await foreach`, and CS0411 for an -`AggAsync` whose type it cannot infer or for `System.Linq` operators applied to an `Aggregation`, but -its message does not say what to do. The analyzer reports this diagnostic beside it. +returns an `Aggregation`, which is not enumerable, and `AggregateAsync` of several answers needs the +record they go into. The compiler refuses both cases, with CS8411 for an `await foreach`, and CS0411 +for an `AggregateAsync` whose type it cannot infer or for `System.Linq` operators applied to an +`Aggregation`, but its message does not say what to do. The analyzer reports this diagnostic beside +it. ```csharp await foreach (var (city, count) in file.Scan() @@ -392,10 +394,10 @@ await foreach (var (city, count) in file.Scan() .Select(g => (g.Key, g.Count()))) // CS8411 and VX1011 var (min, max) = await file.Scan() - .AggAsync(a => (a.Min(r => r.Day), a.Max(r => r.Day))); // CS0411 and VX1011 + .AggregateAsync(a => (a.Min(r => r.Day), a.Max(r => r.Day))); // CS0411 and VX1011 ``` To fix it, declare a `[VortexRecord]` whose members take the values in order, and read the selection through it: `.As()`, enumerated as batches or with `ToRecordsAsync()`, or -`AggAsync(a => (…))`. A selection of one value, such as `Select(g => g.Count())`, comes back as -itself. +`AggregateAsync(a => (…))`. A selection of one value, such as `Select(g => g.Count())`, +comes back as itself. diff --git a/docs/guide/encoded-forms.md b/docs/guide/encoded-forms.md index 52654397..fd1dea89 100644 --- a/docs/guide/encoded-forms.md +++ b/docs/guide/encoded-forms.md @@ -68,7 +68,7 @@ writer chooses per column and per chunk, and its `WriteReport` lists what it cho | `AsDictionary()` | `DictionaryView`: `Codes`, one `uint` per row, `Values`, the distinct values as a `Column` in code order, and `Cardinality` | `InvalidOperationException` | | `AsRunEnd()` | `RunEndView`: `RunEnds`, the exclusive end of each run in the batch, `Values`, one per run, and `RunCount` | `InvalidOperationException` | | `AsConstant()` | the one value | `InvalidOperationException` | -| `Canonical()` | the column decoded once into contiguous memory | returns the column itself | +| `AsCanonical()` | the column decoded once into contiguous memory | returns the column itself | These views are described in [the public API design](../design/14-public-api.md#54-encoded-views-and-the-selection). Check diff --git a/docs/guide/errors.md b/docs/guide/errors.md index ffb8d4db..72a7c3ae 100644 --- a/docs/guide/errors.md +++ b/docs/guide/errors.md @@ -60,7 +60,7 @@ Everything else is plain .NET. An argument out of range is an `ArgumentException ## `VortexFormatException` This is a file that does not parse: bad offsets, a truncation, a field out of range, nesting deeper -than a cap allows, or a block that would decode to more than `MaxDecompressedSize`. It is half of +than a cap allows, or a block that would decode to more than `MaxDecompressedBytes`. It is half of the library's promise about hostile input: a malformed file produces this exception or a `VortexUnsupportedException`, never an out-of-bounds read, an unbounded allocation or a hang. [limits.md](limits.md) lists the caps and shows a thousand damaged files producing nothing else. diff --git a/docs/guide/filter-rows.md b/docs/guide/filter-rows.md index 61b55318..4a4ea22f 100644 --- a/docs/guide/filter-rows.md +++ b/docs/guide/filter-rows.md @@ -22,7 +22,7 @@ await foreach (var (day, _, _) in scan) earliest = Math.Min(earliest, day[0]); } -ScanStatistics stats = scan.Statistics; +ScanMetrics stats = scan.Metrics; ``` ``` @@ -133,7 +133,7 @@ predicate. Write the rows in the order of the column you filter on, or at least `ExplainAsync` reads the statistics and zone maps, plus at most the few segments of a sorted column it searches, and returns a `ScanPlan`: the live blocks, the segments and bytes they need, and one -`PruningStep` per structure consulted, with what it pruned and what consulting it cost. `Statistics`, +`PruningStep` per structure consulted, with what it pruned and what consulting it cost. `Metrics`, read after the sink, says what the scan actually did. Above, the plan counts 15 segments and 217 052 bytes, including the 8 172 bytes of zone maps it consulted. The scan that followed read the other 12 segments, 208 880 bytes. The file keeps the zone maps a plan or a scan has read, and the diff --git a/docs/guide/how-it-works.md b/docs/guide/how-it-works.md index d9caab82..e881b75d 100644 --- a/docs/guide/how-it-works.md +++ b/docs/guide/how-it-works.md @@ -93,7 +93,7 @@ Nothing is copied for you and nothing is left to dispose. Anything beyond borrowed columns is a sink you choose and pay for. `ToBatchesAsync` copies each batch once into buffers you own, `ToRecordsAsync` builds rows, and `CountAsync`, `SumAsync` and `GroupBy(…).Select(…)` run as operators of the scan, on the encoded blocks, answering from the -statistics when those are enough. `ExplainAsync` shows the plan before the run, and `Statistics` +statistics when those are enough. `ExplainAsync` shows the plan before the run, and `Metrics` shows what the scan did after it. See [scan-a-table.md](scan-a-table.md) and [statistics-and-pruning.md](statistics-and-pruning.md). diff --git a/docs/guide/keys-in-order.md b/docs/guide/keys-in-order.md index 42a093a0..87a1274f 100644 --- a/docs/guide/keys-in-order.md +++ b/docs/guide/keys-in-order.md @@ -12,13 +12,13 @@ await using (KeyCursor cursor = await file.Scan().Keys(r => r.Day) await cursor.NextKeyAsync(); Console.WriteLine($"next distinct key {cursor.Key} at row {cursor.Row}"); - if (await cursor.SeekAsync(900, SeekOp.AtOrAfter)) + if (await cursor.SeekAsync(900, SeekMode.AtOrAfter)) { Console.WriteLine($"seek to 900 landed on {cursor.Key} at row {cursor.Row}"); } Console.WriteLine($"rank of 900: {await cursor.RankAsync(900)}"); - Console.WriteLine($"{await cursor.KeyCountAsync()} entries share key {cursor.Key}"); + Console.WriteLine($"{await cursor.CountAtKeyAsync()} entries share key {cursor.Key}"); } ``` @@ -52,10 +52,10 @@ Every move is a `ValueTask`. `false` means there was nowhere to go, and `I | `SeekFirstAsync`, `SeekLastAsync` | go to either end | | `SeekAsync(key, op)` | `Exact`, `AtOrAfter`, `After`, `AtOrBefore`, `Before` | | `SeekRankAsync(n)` | go to the entry at position *n*, counting from zero | -| `NextAsync`, `PrevAsync` | move by one entry | -| `NextKeyAsync`, `PrevKeyAsync` | move to the first entry of the next or previous distinct key | +| `NextAsync`, `PreviousAsync` | move by one entry | +| `NextKeyAsync`, `PreviousKeyAsync` | move to the first entry of the next or previous distinct key | | `RankAsync(key)` | count the entries whose key is smaller | -| `KeyCountAsync()` | count the entries that share the current key, without moving | +| `CountAtKeyAsync()` | count the entries that share the current key, without moving | A key repeated over many rows is many entries, ordered by row. `Exact` lands on the first of them and `AtOrBefore` on the last, which is why the seek to 700 above stops at row 700 999. Two ranks count a diff --git a/docs/guide/limits.md b/docs/guide/limits.md index 90442792..0b2f7fbc 100644 --- a/docs/guide/limits.md +++ b/docs/guide/limits.md @@ -6,7 +6,7 @@ the caps in between. ```csharp await using VortexFile file = await VortexSession.Default.OpenAsync(path, new VortexOpenOptions { - MaxDecompressedSize = ceiling, + MaxDecompressedBytes = ceiling, VerifyStatistics = true, }); @@ -76,8 +76,8 @@ A file past any of them is malformed and reported with `VortexFormatException`. ## The ceiling you set -`VortexOpenOptions.MaxDecompressedSize` bounds what a single decode may produce, and a decode is at -most one chunk of one column. The default is `VortexLimits.DefaultMaxDecompressedSize`, 268 435 456 +`VortexOpenOptions.MaxDecompressedBytes` bounds what a single decode may produce, and a decode is at +most one chunk of one column. The default is `VortexLimits.DefaultMaxDecompressedBytes`, 268 435 456 bytes. It is a refusal, not a truncation, and it is the defence against a decompression bomb: a few bytes on disk that claim to expand to gigabytes. @@ -87,7 +87,7 @@ first decode in the sample produces 131 072 bytes, which is why 4 096 is refused plenty. Lower the ceiling for input you did not write, and raise it only for a trusted file whose chunks are genuinely large, such as long text values. -`VortexOpenOptions.MaxBatchDecompressedSize` bounds what the decodes of one batch produce together, +`VortexOpenOptions.MaxBatchDecompressedBytes` bounds what the decodes of one batch produce together, across all its columns. It is off by default, because a thousand columns each decoding near the ceiling make a batch a thousand times that, which is legitimate for a wide table you wrote and a bomb in a file you did not. For untrusted input, set it to a few times the widest batch you expect. A @@ -136,8 +136,8 @@ declares, each with `Supported`, so a caller can check before scanning. * The ceiling applies per decode, not per scan. A scan that decodes a thousand blocks of 1 MiB never comes near a 256 MiB ceiling, while a single block that claims 300 MiB trips it. The batch ceiling, when set, adds up the decodes of a batch's columns and starts again at the next batch. -* The open reads the file's tail in one request of `InitialReadSize` bytes, 64 KiB by default and - never less. A local file is the exception: it reads its last 8 KiB first, since a read costs it no +* The open reads the file's tail in one request of `InitialReadBytes`, 64 KiB by default and never + less. A local file is the exception: it reads its last 8 KiB first, since a read costs it no round trip. A footer larger than the first read costs a second read, not a refusal. * `IndexCacheBytes` on the session bounds how much of the decoded index runs each open file keeps. It is a memory ceiling rather than a safety one, and 0 keeps nothing. diff --git a/docs/guide/observability.md b/docs/guide/observability.md index 5365c75a..d4888ad5 100644 --- a/docs/guide/observability.md +++ b/docs/guide/observability.md @@ -94,14 +94,14 @@ scans it ran. `ExplainAsync` reads statistics and zone maps without starting one The `EventSource` named `Vorticity` publishes eleven incrementing counters about the engine's inner work: `segments-requested`, `bytes-requested`, `zones-pruned`, `zones-total`, `index-runs-read`, `cursor-seeks`, `cursor-steps`, `count-blocks-proven`, `count-blocks-decoded`, `key-order-windows` and -`key-order-window-splits`. They answer in production what a single scan's statistics answer in a +`key-order-window-splits`. They answer in production what a single scan's metrics answer in a test, above all whether an index earns the bytes it costs. Watch them with `dotnet-counters monitor --counters Vorticity`. Nothing is paid until a listener attaches: every hook is one flag test, and the counters are only created once a listener asks for them. ## The figures of one scan -The counters cover the whole process, while `Statistics` on a scan covers that scan alone and is +The counters cover the whole process, while `Metrics` on a scan covers that scan alone and is valid once its sink has run: ```csharp @@ -112,7 +112,7 @@ await foreach (Columns batch in scan) rows += batch.RowCount; } -ScanStatistics stats = scan.Statistics; +ScanMetrics stats = scan.Metrics; Console.WriteLine($"scan {run}: {stats.Rows} rows in {stats.Batches} batches, {stats.Requests} requests, " + $"{stats.BytesRequested} bytes, {stats.BlocksDecoded} blocks decoded, {stats.BlocksPruned} pruned, {stats.CacheHits} cache hits"); ``` @@ -122,16 +122,16 @@ scan 1: 100000 rows in 5 batches, 6 requests, 180368 bytes, 14 blocks decoded, 1 scan 2: 100000 rows in 5 batches, 6 requests, 180368 bytes, 14 blocks decoded, 109 pruned, 6 cache hits ``` -`ScanStatistics` is a `readonly record struct` with seven `long`s (`Rows`, `Batches`, `Requests`, +`ScanMetrics` is a `readonly record struct` with seven `long`s (`Rows`, `Batches`, `Requests`, `BytesRequested`, `BlocksDecoded`, `BlocksPruned` and `CacheHits`) plus `Grouping`, which a group by -fills with a `GroupStatistics`: the groups found and the most held at once, the peak memory, the +fills with a `GroupMetrics`: the groups found and the most held at once, the peak memory, the lanes, what was spilled and the time to the first batch. `Grouping` is null for a scan without a group by. `Requests` counts the segments the scan asked for, each once, including those the cache then served. The second scan finds all 6 of its requests in the cache, where the first found none. Neither asks for the zone maps it consults, nor for the other 6 segments its rows lie in, because those sit in the -tail the open already read and the file serves them itself. The same statistics exist on the tool +tail the open already read and the file serves them itself. The same metrics exist on the tool scan and on a grouped aggregation. What the scan will do before it runs is `ExplainAsync`, and [statistics-and-pruning.md](statistics-and-pruning.md) puts the two side by side. @@ -151,22 +151,22 @@ cache: 8 hits, 19 misses, 0 bytes held A path passed to `OpenAsync` is memory-mapped by its first scan, unless the session's `MapFiles` is false, and a mapping has nothing to cache: its reads never reach the cache or its counters. -`SegmentCache` exposes `Capacity`, `Size`, `Hits` and `Misses` for the whole session. A file's +`SegmentCache` exposes `CapacityBytes`, `HeldBytes`, `Hits` and `Misses` for the whole session. A file's entries leave the cache when the file is disposed, which is why nothing is held at the end. [threads.md](threads.md) explains when a cache is worth its memory. ## What it costs Without a listener it costs nothing. A counter with no listener costs a flag check, and an activity -is only created when a listener samples the source. A scan's own `Statistics` are counted either way, since +is only created when a listener samples the source. A scan's own `Metrics` are counted either way, since the counters are built from them. ## Watch out * The meter's counters carry no tags. To tell two files or two tenants apart, use one session each - and the scan's `Statistics`, or the activities, which nest under your own. + and the scan's `Metrics`, or the activities, which nest under your own. * The scan counters are added when the scan ends. -* A scan's `Statistics.CacheHits` counts the segments the cache served to that scan alone, even while +* A scan's `Metrics.CacheHits` counts the segments the cache served to that scan alone, even while other scans of the session run. The cache's own `Hits` counts the whole session's. ## Run it diff --git a/docs/guide/open-a-file.md b/docs/guide/open-a-file.md index 582f915b..593ca267 100644 --- a/docs/guide/open-a-file.md +++ b/docs/guide/open-a-file.md @@ -115,21 +115,21 @@ column the statistics cover asks for anything more. ```csharp CountingSource told = new CountingSource(new FileSegmentSource(path)); -VortexOpenOptions wide = new VortexOpenOptions { Length = bytes.Length, InitialReadSize = 256 * 1024 }; +VortexOpenOptions wide = new VortexOpenOptions { Length = bytes.Length, InitialReadBytes = 256 * 1024 }; await using (VortexFile file = await session.OpenAsync(told, wide)) ``` ``` -InitialReadSize 256 KiB: 1 request, 262172 bytes +InitialReadBytes 256 KiB: 1 request, 262172 bytes ``` | option | default | what it changes | |---|---|---| -| `InitialReadSize` | 65 536 | the size of the tail read. Raise it for a file with a large footer so the open stays one request, or to fetch a small file whole from a remote source | +| `InitialReadBytes` | 65 536 | the size of the tail read. Raise it for a file with a large footer so the open stays one request, or to fetch a small file whole from a remote source | | `Length` | probed | skips asking the source for its length | | `Schema` | the file's | a schema for a file written without one, or to skip reading the embedded one | | `TornTail` | `ReadPrevious` | `Refuse` throws on a torn tail instead of opening the version before it | -| `VerifyStatistics`, `MaxDecompressedSize` | off, 256 MiB | how far the open trusts the bytes ([limits.md](limits.md)) | +| `VerifyStatistics`, `MaxDecompressedBytes` | off, 256 MiB | how far the open trusts the bytes ([limits.md](limits.md)) | | `IndexFragments` | none | indexes built for this file elsewhere ([indexes.md](indexes.md)) | ## What a scan then reads diff --git a/docs/guide/project-columns.md b/docs/guide/project-columns.md index 9883bff3..06552aa8 100644 --- a/docs/guide/project-columns.md +++ b/docs/guide/project-columns.md @@ -35,7 +35,7 @@ leaves in all: `Id`, `StartedAt`, `DurationMs`, `Referrer`, `Pages`, `Origin.Cou ## What it saves The sample scans the file with four records and measures each one twice: with `ExplainAsync` before -the scan, from the layout alone, and with `Statistics` after it. +the scan, from the layout alone, and with `Metrics` after it. | record | columns | segments | bytes to read | requests | bytes read | |---|---|---|---|---|---| diff --git a/docs/guide/queries.md b/docs/guide/queries.md index add2c2c6..1ed0e9c4 100644 --- a/docs/guide/queries.md +++ b/docs/guide/queries.md @@ -132,6 +132,9 @@ explains when each one does. | `GroupBy(r => (r.City, r.Day))`, then `g.Key.Item1` | `g.Key.City` | | the groups at the end of the pass | as soon as they are final, on a key that streams | +On a whole scan, `AggAsync` became `AggregateAsync`, and +[the public API design](../design/14-public-api.md#renamed) lists every other renamed member. + ## Watch out * Without an order, which groups a `Take` keeps is not promised. On several lanes the groups come in diff --git a/docs/guide/scan-a-table.md b/docs/guide/scan-a-table.md index caf67972..a11143c6 100644 --- a/docs/guide/scan-a-table.md +++ b/docs/guide/scan-a-table.md @@ -77,7 +77,7 @@ beyond its iteration, copy it once with `ToOwned()`, or ask for owned batches wi ([owned-batches.md](owned-batches.md)). A scan is single-use. Each builder accepts one sink, and a second one throws -`InvalidOperationException`. You may call `ExplainAsync` before the sink and read `Statistics` after +`InvalidOperationException`. You may call `ExplainAsync` before the sink and read `Metrics` after it. The sinks and what each one costs are listed in [the public API design](../design/14-public-api.md#56-the-sinks). diff --git a/docs/guide/statistics-and-pruning.md b/docs/guide/statistics-and-pruning.md index d5657fda..b4a32862 100644 --- a/docs/guide/statistics-and-pruning.md +++ b/docs/guide/statistics-and-pruning.md @@ -12,7 +12,7 @@ await foreach (Columns _ in scan) { } -Print("Day >= 900", scan.Statistics); +Print("Day >= 900", scan.Metrics); ``` ``` @@ -49,10 +49,10 @@ by them, or left to decode. ## The numbers, after -`scan.Statistics` is valid once a sink has run. It is a `ScanStatistics`: the `Rows` and `Batches` +`scan.Metrics` is valid once a sink has run. It is a `ScanMetrics`: the `Rows` and `Batches` delivered, the `Requests` and `BytesRequested` made to the source, `BlocksDecoded`, `BlocksPruned`, and the `CacheHits` from the session's segment cache ([threads.md](threads.md)). For a group by, -`Grouping` adds a `GroupStatistics`: the groups found and the most held at once, the peak memory, the +`Grouping` adds a `GroupMetrics`: the groups found and the most held at once, the peak memory, the lanes, what was spilled, and the time to the first batch ([queries.md](queries.md)). The plan's bytes and the run's differ by the zone maps. The plan counts each segment once, 13 of them @@ -147,8 +147,8 @@ own, with what they cost to consult. the file. * `Blocks` and `LiveBlocks` follow `Rows(...)`: a range of ten rows plans 1 block of 1, and a take of two rows 2 of 2, with `Rows` equal to 2 ([read-rows-by-index.md](read-rows-by-index.md)). -* `Statistics` read before the sink has run are zeros, and a scan runs only one sink. -* `ScanOptions.Pruning = false` and `UseIndexes = false` turn the structures off, to check them, never +* `Metrics` read before the sink has run are zeros, and a scan runs only one sink. +* `ScanOptions.UseStatistics = false` and `UseIndexes = false` turn the structures off, to check them, never to change a result. What each statistic allows is defined in diff --git a/docs/guide/writer-options.md b/docs/guide/writer-options.md index 416873ac..4abba1df 100644 --- a/docs/guide/writer-options.md +++ b/docs/guide/writer-options.md @@ -43,7 +43,7 @@ that does not fit throws `ArgumentException` there, before a single byte is writ | `Hints` | empty | an encoding to try first, by column path | | `DegreeOfParallelism` | 0, the session's | the threads the writer summarizes its columns and compresses zstd frames on. The file is byte for byte the same whatever the degree ([threads.md](threads.md)) | | `TargetEdition` | `VortexEditions.Default` | the edition every component must belong to ([editions.md](editions.md)) | -| `Statistics` | `true` | the per-column statistics the open reads back: null count, order, and the minimum and maximum of a numeric column | +| `WriteStatistics` | `true` | the per-column statistics the open reads back: null count, order, and the minimum and maximum of a numeric column | | `StringBoundBytes` | 16 | the length of the bounds a text or binary column's zones carry, 0 for none | | `Indexes` | `IndexPolicy.None` | the indexes to build ([indexes.md](indexes.md)) | | `Identity` | drawn per write | the sixteen bytes that name this version of the file | @@ -53,7 +53,7 @@ that does not fit throws `ArgumentException` there, before a single byte is writ ## Compression The same million rows under each profile. The write time is the best of three, and the decode forces -every column of every batch to its plain form with `Canonical()`: +every column of every batch to its plain form with `AsCanonical()`: | profile | bytes | written in | decoded in | encodings | |---|---|---|---|---| @@ -142,7 +142,7 @@ block holds all of them and no bound can rule one out. ## Statistics, metadata and identity -* `Statistics = false` saved 224 bytes out of 1 508 316, and `file.Statistics.Count` is then 0. A +* `WriteStatistics = false` saved 224 bytes out of 1 508 316, and `file.Statistics.Count` is then 0. A question the statistics would have answered at open, such as a minimum or a count, then reads the zone maps or the data instead. The zone maps are kept, so `Day > 2000` still reads 0 of 123 blocks. * `Metadata` holds at most 14 entries, with keys of at most 64 UTF-8 bytes, besides the library's diff --git a/samples/Vorticity.Samples/Aggregates.cs b/samples/Vorticity.Samples/Aggregates.cs index 0321be85..d90d9c94 100644 --- a/samples/Vorticity.Samples/Aggregates.cs +++ b/samples/Vorticity.Samples/Aggregates.cs @@ -12,12 +12,12 @@ internal static async Task RunAsync() string path = await Demo.ReadingsAsync(); await using VortexFile file = await VortexFile.OpenAsync(path); - await BestOfThreeAsync("AggAsync, four answers", async () => + await BestOfThreeAsync("AggregateAsync, four answers", async () => { Scan recent = file.Scan().Where(r => r.Day >= 900); Summary s = await recent - .AggAsync(a => (a.Min(r => r.Celsius), a.Max(r => r.Celsius), a.Count(), a.CountDistinct(r => r.City))); - return $"min {s.Min}, max {s.Max}, {s.Rows} rows, {s.Cities} cities; {recent.Statistics.BlocksDecoded} blocks decoded"; + .AggregateAsync(a => (a.Min(r => r.Celsius), a.Max(r => r.Celsius), a.Count(), a.CountDistinct(r => r.City))); + return $"min {s.Min}, max {s.Max}, {s.Rows} rows, {s.Cities} cities; {recent.Metrics.BlocksDecoded} blocks decoded"; }); string[] lines = []; @@ -156,7 +156,7 @@ await BestOfThreeAsync("GroupBy(StartedAt, by hour)", async () => hours++; } - return $"{hours} hours; {day.Statistics.BlocksDecoded} blocks decoded"; + return $"{hours} hours; {day.Metrics.BlocksDecoded} blocks decoded"; }); foreach (string line in lines) @@ -192,7 +192,7 @@ await BestOfThreeAsync("GroupBy(DurationMs, buckets)", async () => } private static string Describe(WelfordState s, Scan scan) => - $"{s.Count} values, mean {s.Mean:F4}, variance {s.Variance:F4}, {scan.Statistics.BlocksDecoded} blocks decoded"; + $"{s.Count} values, mean {s.Mean:F4}, variance {s.Variance:F4}, {scan.Metrics.BlocksDecoded} blocks decoded"; private static async Task CompositeAsync(VortexFile file) { diff --git a/samples/Vorticity.Samples/AppendAndRepair.cs b/samples/Vorticity.Samples/AppendAndRepair.cs index a175cef7..4e4c7a74 100644 --- a/samples/Vorticity.Samples/AppendAndRepair.cs +++ b/samples/Vorticity.Samples/AppendAndRepair.cs @@ -18,7 +18,7 @@ internal static async Task RunAsync() Guid identity = await IdentityOf(path); Reading[] more = rows[16_384..24_576]; - await using (VortexFileWriter appender = await session.AppendAsync(path)) + await using (VortexFileWriter appender = await session.OpenWriterAsync(path)) { long resumeAt = appender.RowCount; // the file's row count, since it ended on a block await appender.WriteAsync(more.AsSpan(), ct); @@ -29,7 +29,7 @@ internal static async Task RunAsync() Console.WriteLine($"the identity is {(identity == await IdentityOf(path) ? "unchanged" : "drawn anew")} by an append"); await Create(session, path, rows.AsMemory(0, 20_000)); - await using (VortexFileWriter appender = await session.AppendAsync(path)) + await using (VortexFileWriter appender = await session.OpenWriterAsync(path)) { long resumeAt = appender.RowCount; await appender.WriteAsync(rows.AsSpan(20_000, 5_000), ct); @@ -57,7 +57,7 @@ private static async Task Rounds(VortexSession session, string path, Reading[] r long total = initial; for (int round = 0; round < rounds; round++) { - await using VortexFileWriter appender = await session.AppendAsync(path); + await using VortexFileWriter appender = await session.OpenWriterAsync(path); long resumed = appender.RowCount; await appender.WriteAsync(rows.AsSpan((int)total, each)); WriteReport report = await appender.CompleteAsync(); @@ -75,7 +75,7 @@ private static async Task Giving(VortexSession session, string path, Reading[] r { await Create(session, path, rows.AsMemory(0, 16_384)); long length = new FileInfo(path).Length; - await using (VortexFileWriter appender = await session.AppendAsync(path)) + await using (VortexFileWriter appender = await session.OpenWriterAsync(path)) { await appender.WriteAsync(rows.AsSpan(16_384, 50_000)); await appender.FlushAsync(); @@ -84,7 +84,7 @@ private static async Task Giving(VortexSession session, string path, Reading[] r Console.WriteLine($"an append abandoned after a flush: {grown} bytes on disk before Abandon, {new FileInfo(path).Length} after, {length} before the append"); } - await using (VortexFileWriter appender = await session.AppendAsync(path)) + await using (VortexFileWriter appender = await session.OpenWriterAsync(path)) { await appender.WriteAsync(rows.AsSpan(16_384, 50_000)); await appender.FlushAsync(); @@ -112,7 +112,7 @@ private static async Task Giving(VortexSession session, string path, Reading[] r try { await using VortexFile open = await VortexFile.OpenAsync(path); - await using VortexFileWriter refused = await session.AppendAsync(path); + await using VortexFileWriter refused = await session.OpenWriterAsync(path); Console.WriteLine("an append while the file is open for reading: accepted"); refused.Abandon(); } @@ -127,7 +127,7 @@ private static async Task Torn(VortexSession session, string path, Reading[] row { await Create(session, path, rows.AsMemory(0, 16_384)); long before = new FileInfo(path).Length; - await using (VortexFileWriter appender = await session.AppendAsync(path)) + await using (VortexFileWriter appender = await session.OpenWriterAsync(path)) { await appender.WriteAsync(rows.AsSpan(16_384, 8_192)); await appender.CompleteAsync(); @@ -161,18 +161,18 @@ private static async Task Torn(VortexSession session, string path, Reading[] row try { - await using VortexFileWriter refused = await session.AppendAsync(path); + await using VortexFileWriter refused = await session.OpenWriterAsync(path); } catch (VortexFormatException e) { Console.WriteLine($"an append to the torn file: {e.Message}"); } - long valid = await VortexFileRepair.ValidLengthAsync(path, ct); + long valid = await VortexFileRepair.GetValidLengthAsync(path, ct); VortexRepairResult repaired = await VortexFileRepair.RepairAsync(path, ct); // truncates to the last version that parses Console.WriteLine($"valid length {valid}; repaired: truncated {repaired.Truncated}, {repaired.OriginalLength} -> {repaired.Length}"); - await using (VortexFileWriter again = await session.AppendAsync(path)) + await using (VortexFileWriter again = await session.OpenWriterAsync(path)) { await again.WriteAsync(rows.AsSpan(16_384, 8_192), ct); WriteReport report = await again.CompleteAsync(ct); diff --git a/samples/Vorticity.Samples/BlocksAndChunks.cs b/samples/Vorticity.Samples/BlocksAndChunks.cs index ae75958b..0181ac0d 100644 --- a/samples/Vorticity.Samples/BlocksAndChunks.cs +++ b/samples/Vorticity.Samples/BlocksAndChunks.cs @@ -68,7 +68,7 @@ private static async Task Pattern(VortexSession session, string path, Reading[] long written = new FileInfo(path).Length; long resumesAt; - await using (VortexFileWriter appender = await session.AppendAsync(path)) + await using (VortexFileWriter appender = await session.OpenWriterAsync(path)) { resumesAt = appender.RowCount; appender.Abandon(); diff --git a/samples/Vorticity.Samples/CancelWork.cs b/samples/Vorticity.Samples/CancelWork.cs index 7c37e57b..92e6e89a 100644 --- a/samples/Vorticity.Samples/CancelWork.cs +++ b/samples/Vorticity.Samples/CancelWork.cs @@ -102,11 +102,11 @@ await ProbeAsync("ToRecordsAsync", async () => await using (KeyCursor cursor = await file.Scan().Keys(r => r.Day).OpenAsync()) { - await cursor.SeekAsync(500, SeekOp.Exact); - await ProbeAsync("KeyCursor.SeekAsync", async () => await cursor.SeekAsync(900, SeekOp.Exact, ct)); + await cursor.SeekAsync(500, SeekMode.Exact); + await ProbeAsync("KeyCursor.SeekAsync", async () => await cursor.SeekAsync(900, SeekMode.Exact, ct)); Console.WriteLine($" the cursor after it: valid {cursor.IsValid}"); await ProbeAsync(" its Key", () => Task.FromResult(cursor.Key)); - Console.WriteLine($" a new seek with no token: {await cursor.SeekAsync(900, SeekOp.Exact)}, key {cursor.Key}"); + Console.WriteLine($" a new seek with no token: {await cursor.SeekAsync(900, SeekMode.Exact)}, key {cursor.Key}"); } await WritesAsync(ct); @@ -141,7 +141,7 @@ private static async Task WritesAsync(CancellationToken cancelled) } long length = new FileInfo(appended).Length; - await using (VortexFileWriter appender = await VortexSession.Default.AppendAsync(appended)) + await using (VortexFileWriter appender = await VortexSession.Default.OpenWriterAsync(appended)) { await appender.WriteAsync(rows); await appender.FlushAsync(); diff --git a/samples/Vorticity.Samples/DatasetMaintenance.cs b/samples/Vorticity.Samples/DatasetMaintenance.cs index c9424e71..cfcd3fc7 100644 --- a/samples/Vorticity.Samples/DatasetMaintenance.cs +++ b/samples/Vorticity.Samples/DatasetMaintenance.cs @@ -32,7 +32,7 @@ internal static async Task RunAsync() $"objects by level [{string.Join(", ", plan.ObjectsByLevel)}], bytes by level [{string.Join(", ", plan.BytesByLevel)}]"); if (plan.Job is { } job) { - Console.WriteLine($" job: level {job.FromLevel} to {job.ToLevel}, {job.Trigger}, {job.Objects.Length} objects, {job.Rows} rows, " + + Console.WriteLine($" job: level {job.FromLevel} to {job.ToLevel}, {job.Trigger}, {job.ObjectKeys.Length} objects, {job.Rows} rows, " + $"{job.Bytes} bytes, target {job.TargetBytes}"); } diff --git a/samples/Vorticity.Samples/Datasets.cs b/samples/Vorticity.Samples/Datasets.cs index e9391ced..98ec7e02 100644 --- a/samples/Vorticity.Samples/Datasets.cs +++ b/samples/Vorticity.Samples/Datasets.cs @@ -22,7 +22,7 @@ internal static async Task RunAsync() { ClusteringKey = [Reading.ColumnNames.Day], }); - Console.WriteLine($"created: version {dataset.Version}, {dataset.RowCount} rows, clustered by {string.Join(", ", dataset.ClusteringKeyPaths)}"); + Console.WriteLine($"created: version {dataset.Version}, {dataset.RowCount} rows, clustered by {string.Join(", ", dataset.ClusteringKey)}"); await using (ObjectDraft draft = dataset.StartObject()) { @@ -52,7 +52,7 @@ internal static async Task RunAsync() ulong imported = await dataset.ImportAsync("imports/days-100-119.vortex"); Console.WriteLine($"imported: version {imported}, {dataset.RowCount} rows, {dataset.ObjectCount} objects, lag {dataset.Lag}"); - List objects = await dataset.ObjectsAsync().ToListAsync(); + List objects = await dataset.ListObjectsAsync().ToListAsync(); foreach (DataObject entry in objects) { Console.WriteLine($" {entry.Key}: level {entry.Level}, rows {entry.FirstRow} to {entry.FirstRow + entry.Rows}, {entry.Bytes} bytes"); @@ -74,12 +74,12 @@ internal static async Task RunAsync() await using (KeyCursor cursor = await dataset.Scan().Keys(r => r.Day).OpenAsync()) { - bool found = await cursor.SeekAsync(75, SeekOp.AtOrAfter); - Console.WriteLine($"key cursor: seek 75 found {found}, key {cursor.Key} at row {cursor.Row}; {await cursor.KeyCountAsync()} entries"); + bool found = await cursor.SeekAsync(75, SeekMode.AtOrAfter); + Console.WriteLine($"key cursor: seek 75 found {found}, key {cursor.Key} at row {cursor.Row}; {await cursor.CountAtKeyAsync()} entries"); await cursor.SeekLastAsync(); (int last, long lastRow) = (cursor.Key, cursor.Row); - await cursor.PrevKeyAsync(); + await cursor.PreviousKeyAsync(); Console.WriteLine($"walking down: last key {last} at row {lastRow}; the key before it {cursor.Key}, its last entry at row {cursor.Row}"); } @@ -102,15 +102,15 @@ await rewritten.Writer.WriteAsync(dataset.Scan() .Rows(RowRange.FromLength(first.FirstRow, first.Rows)) .Where(r => r.Celsius.IsNotNull) .ToRecordsAsync()); - ReplaceResult replaced = await dataset.ReplaceAsync([first], [rewritten]); + ReplaceResult replaced = await dataset.ReplaceObjectsAsync([first], [rewritten]); Console.WriteLine($"replaced {first.Key} by the rows with a temperature: version {replaced.Version}, {replaced.Outcome}, {dataset.RowCount} rows"); } - DataObject importedObject = (await dataset.ObjectsAsync().ToListAsync()).Single(o => o.Key.StartsWith("imports/", StringComparison.Ordinal)); - ReplaceResult removed = await dataset.RemoveAsync([importedObject]); + DataObject importedObject = (await dataset.ListObjectsAsync().ToListAsync()).Single(o => o.Key.StartsWith("imports/", StringComparison.Ordinal)); + ReplaceResult removed = await dataset.RemoveObjectsAsync([importedObject]); Console.WriteLine($"removed the import: version {removed.Version}, {removed.Outcome}, {dataset.RowCount} rows, {dataset.ObjectCount} objects"); - ReplaceResult again = await dataset.RemoveAsync([importedObject]); + ReplaceResult again = await dataset.RemoveObjectsAsync([importedObject]); Console.WriteLine($"removing it again: {again.Outcome}, version {again.Version}"); RowChangeResult deleted = await dataset.DeleteAsync(r => r.Day < 10); diff --git a/samples/Vorticity.Samples/EncodedForms.cs b/samples/Vorticity.Samples/EncodedForms.cs index 4861a34b..ca26861b 100644 --- a/samples/Vorticity.Samples/EncodedForms.cs +++ b/samples/Vorticity.Samples/EncodedForms.cs @@ -23,7 +23,7 @@ internal static async Task RunAsync() await BestOfThreeAsync("City by run, by hand", async () => (byRun = await CelsiusByCityAsync(file)).Length); Dictionary grouped = []; - ScanStatistics cityStats = default; + ScanMetrics cityStats = default; await BestOfThreeAsync("GroupBy(r => r.City)", async () => { grouped.Clear(); @@ -36,7 +36,7 @@ await BestOfThreeAsync("GroupBy(r => r.City)", async () => grouped[city] = total; } - cityStats = byCity.Statistics; + cityStats = byCity.Metrics; return grouped.Count; }); @@ -52,7 +52,7 @@ await BestOfThreeAsync("GroupBy(r => r.City)", async () => await BestOfThreeAsync("Referrer by code, by hand", async () => (byCode = await VisitsByReferrerAsync(visitFile)).Count); Dictionary byGroup = []; - ScanStatistics referrerStats = default; + ScanMetrics referrerStats = default; await BestOfThreeAsync("GroupBy(v => v.Referrer)", async () => { byGroup.Clear(); @@ -65,7 +65,7 @@ await BestOfThreeAsync("GroupBy(v => v.Referrer)", async () => byGroup[referrer ?? "(null)"] = (count, duration); } - referrerStats = byReferrer.Statistics; + referrerStats = byReferrer.Metrics; return byGroup.Count; }); @@ -203,8 +203,8 @@ private static async Task CensusAsync(VortexFile file, VortexFile visitFile) DictionaryView temperatures = celsius.AsDictionary(); RunEndView days = day.AsRunEnd(); RunEndView cities = city.AsRunEnd(); - Column decoded = city.Canonical(); - Console.WriteLine($"first batch: {day.Length} rows; Celsius {temperatures.Cardinality} distinct values, {temperatures.Values.NullCount} of them null; Day {days.RunCount} runs, City {cities.RunCount} runs; City.Canonical() is {decoded.Encoding}"); + Column decoded = city.AsCanonical(); + Console.WriteLine($"first batch: {day.Length} rows; Celsius {temperatures.Cardinality} distinct values, {temperatures.Values.NullCount} of them null; Day {days.RunCount} runs, City {cities.RunCount} runs; City.AsCanonical() is {decoded.Encoding}"); break; } } diff --git a/samples/Vorticity.Samples/FilterRows.cs b/samples/Vorticity.Samples/FilterRows.cs index 2840c2ad..97874dca 100644 --- a/samples/Vorticity.Samples/FilterRows.cs +++ b/samples/Vorticity.Samples/FilterRows.cs @@ -26,7 +26,7 @@ internal static async Task RunAsync() earliest = Math.Min(earliest, day[0]); } - ScanStatistics stats = scan.Statistics; + ScanMetrics stats = scan.Metrics; Console.WriteLine($"plan: {plan.LiveBlocks} of {plan.Blocks} blocks, {plan.Segments} segments, {plan.BytesToRead} bytes"); foreach (PruningStep step in plan.Pruning) @@ -94,7 +94,7 @@ private static async Task CountAsync(string text, Scan scan) { ScanPlan plan = await scan.ExplainAsync(); long count = await scan.CountAsync(); - ScanStatistics stats = scan.Statistics; + ScanMetrics stats = scan.Metrics; string how = plan.Count.Exact ? "exact from the structures" : $"{plan.Count.Pruned} pruned, {plan.Count.Proven} proven, {plan.Count.Decoded} to evaluate"; Console.WriteLine($"{text,-32} {count,7} rows {plan.LiveBlocks,3} of {plan.Blocks} blocks live {stats.Requests,3} requests {stats.BytesRequested,8} bytes ({how})"); } diff --git a/samples/Vorticity.Samples/HowItWorks.cs b/samples/Vorticity.Samples/HowItWorks.cs index 7d39d848..e760dd03 100644 --- a/samples/Vorticity.Samples/HowItWorks.cs +++ b/samples/Vorticity.Samples/HowItWorks.cs @@ -55,6 +55,6 @@ internal static async Task RunAsync() } Console.WriteLine($"plan: {plan.LiveBlocks} of {plan.Blocks} blocks live, {plan.Segments} segments, {plan.BytesToRead} bytes; " + - $"ran: {rows} rows, {scan.Statistics.BlocksDecoded} blocks decoded, {scan.Statistics.BlocksPruned} pruned; the lambda ran {calls} time"); + $"ran: {rows} rows, {scan.Metrics.BlocksDecoded} blocks decoded, {scan.Metrics.BlocksPruned} pruned; the lambda ran {calls} time"); } } diff --git a/samples/Vorticity.Samples/Indexes.cs b/samples/Vorticity.Samples/Indexes.cs index 41432166..4fa1afda 100644 --- a/samples/Vorticity.Samples/Indexes.cs +++ b/samples/Vorticity.Samples/Indexes.cs @@ -77,7 +77,7 @@ internal static async Task RunAsync() await Pruned(file, "Score == a value that exists", r => r.Score == score); await Pruned(file, "Score between 1000 and 1100", r => r.Score.Between(1_000, 1_100)); await using KeyCursor cursor = await file.Scan().Keys(r => r.Score).OpenAsync(ct); - await cursor.SeekAsync(1_000, SeekOp.AtOrAfter, ct); + await cursor.SeekAsync(1_000, SeekMode.AtOrAfter, ct); Console.WriteLine($" a key cursor on Score: the first key at or after 1000 is {cursor.Key}, at row {cursor.Row}; {await cursor.RankAsync(1_000, ct)} rows hold a smaller one"); } @@ -103,7 +103,7 @@ internal static async Task RunAsync() await using (VortexFile file = await VortexFile.OpenAsync(later)) { await using KeyCursor cursor = await file.Scan().Keys(r => r.Score).OpenAsync(ct); - await cursor.SeekAsync(1_000, SeekOp.AtOrAfter, ct); + await cursor.SeekAsync(1_000, SeekMode.AtOrAfter, ct); Console.WriteLine($" added afterwards: {added[0].Kind} {added[0].Outcome}, {added[0].Bytes} bytes; the file grew from {before} to {file.Length} bytes; " + $"the cursor finds {cursor.Key} at row {cursor.Row}"); } diff --git a/samples/Vorticity.Samples/KeysInOrder.cs b/samples/Vorticity.Samples/KeysInOrder.cs index ebe2eded..92abbf5a 100644 --- a/samples/Vorticity.Samples/KeysInOrder.cs +++ b/samples/Vorticity.Samples/KeysInOrder.cs @@ -18,25 +18,25 @@ internal static async Task RunAsync() await cursor.NextKeyAsync(); Console.WriteLine($"next distinct key {cursor.Key} at row {cursor.Row}"); - if (await cursor.SeekAsync(900, SeekOp.AtOrAfter)) + if (await cursor.SeekAsync(900, SeekMode.AtOrAfter)) { Console.WriteLine($"seek to 900 landed on {cursor.Key} at row {cursor.Row}"); } Console.WriteLine($"rank of 900: {await cursor.RankAsync(900)}"); - Console.WriteLine($"{await cursor.KeyCountAsync()} entries share key {cursor.Key}"); + Console.WriteLine($"{await cursor.CountAtKeyAsync()} entries share key {cursor.Key}"); await cursor.SeekRankAsync(123_456); Console.WriteLine($"entry of rank 123456: key {cursor.Key} at row {cursor.Row}"); await cursor.SeekLastAsync(); Console.WriteLine($"last key {cursor.Key} at row {cursor.Row}"); - bool found = await cursor.SeekAsync(1_234, SeekOp.Exact); + bool found = await cursor.SeekAsync(1_234, SeekMode.Exact); Console.WriteLine($"an exact seek to a key that is not there: {found}, cursor valid: {cursor.IsValid}"); long inRange = await cursor.RankAsync(102) - await cursor.RankAsync(100); Console.WriteLine($"rows with 100 <= day < 102, from two ranks: {inRange}"); - await cursor.SeekAsync(700, SeekOp.AtOrBefore); + await cursor.SeekAsync(700, SeekMode.AtOrBefore); long row = cursor.Row; Console.WriteLine($"seek at or before 700: key {cursor.Key} at row {row}, the last of its run"); await foreach (Columns cols in file.Scan().Rows(row)) @@ -50,7 +50,7 @@ internal static async Task RunAsync() Stopwatch watch = Stopwatch.StartNew(); for (int key = 0; key < 1_000; key++) { - await cursor.SeekAsync(key, SeekOp.AtOrAfter); + await cursor.SeekAsync(key, SeekMode.AtOrAfter); } Console.WriteLine($"1000 seeks in {watch.Elapsed.TotalMilliseconds:F1} ms"); diff --git a/samples/Vorticity.Samples/Limits.cs b/samples/Vorticity.Samples/Limits.cs index 5b325416..65355784 100644 --- a/samples/Vorticity.Samples/Limits.cs +++ b/samples/Vorticity.Samples/Limits.cs @@ -16,14 +16,14 @@ internal static async Task RunAsync() Console.WriteLine($"metadata segments {VortexLimits.MaxMetadataSegments}, key length {VortexLimits.MaxMetadataKeyLength}, " + $"compression specs {VortexLimits.MaxCompressionSpecs}, alignment {VortexLimits.MaxAlignment} (exponent {VortexLimits.MaxAlignmentExponent}), " + $"postscript {VortexLimits.MaxPostscriptSize}"); - Console.WriteLine($"decompression ceiling by default: {VortexLimits.DefaultMaxDecompressedSize} bytes per decode"); + Console.WriteLine($"decompression ceiling by default: {VortexLimits.DefaultMaxDecompressedBytes} bytes per decode"); string path = await Demo.ReadingsAsync(); - foreach (long ceiling in new long[] { 4_096, 1024 * 1024, VortexLimits.DefaultMaxDecompressedSize }) + foreach (long ceiling in new long[] { 4_096, 1024 * 1024, VortexLimits.DefaultMaxDecompressedBytes }) { await using VortexFile file = await VortexSession.Default.OpenAsync(path, new VortexOpenOptions { - MaxDecompressedSize = ceiling, + MaxDecompressedBytes = ceiling, VerifyStatistics = true, }); diff --git a/samples/Vorticity.Samples/NestedRecords.cs b/samples/Vorticity.Samples/NestedRecords.cs index 1a1665b2..40bb77bc 100644 --- a/samples/Vorticity.Samples/NestedRecords.cs +++ b/samples/Vorticity.Samples/NestedRecords.cs @@ -71,7 +71,7 @@ internal static async Task RunAsync() { } - Console.WriteLine($"every column of Visit: {whole.Statistics.BytesRequested} bytes; Origin.Country alone: {country.Statistics.BytesRequested} bytes"); + Console.WriteLine($"every column of Visit: {whole.Metrics.BytesRequested} bytes; Origin.Country alone: {country.Metrics.BytesRequested} bytes"); await ParcelsAsync(); } diff --git a/samples/Vorticity.Samples/Observability.cs b/samples/Vorticity.Samples/Observability.cs index d42d2fb0..9555c790 100644 --- a/samples/Vorticity.Samples/Observability.cs +++ b/samples/Vorticity.Samples/Observability.cs @@ -49,17 +49,17 @@ internal static async Task RunAsync() rows += batch.RowCount; } - ScanStatistics stats = scan.Statistics; + ScanMetrics stats = scan.Metrics; Console.WriteLine($"scan {run}: {stats.Rows} rows in {stats.Batches} batches, {stats.Requests} requests, " + $"{stats.BytesRequested} bytes, {stats.BlocksDecoded} blocks decoded, {stats.BlocksPruned} pruned, {stats.CacheHits} cache hits"); } Scan untyped = file.Scan("City").Where($"City = {"Paris"}"); long paris = await untyped.CountAsync(); - Console.WriteLine($"tool scan: {paris} rows, {untyped.Statistics.Requests} requests, {untyped.Statistics.BlocksDecoded} blocks decoded"); + Console.WriteLine($"tool scan: {paris} rows, {untyped.Metrics.Requests} requests, {untyped.Metrics.BlocksDecoded} blocks decoded"); } - Console.WriteLine($"cache: {session.Options.SegmentCache!.Hits} hits, {session.Options.SegmentCache.Misses} misses, {session.Options.SegmentCache.Size} bytes held"); + Console.WriteLine($"cache: {session.Options.SegmentCache!.Hits} hits, {session.Options.SegmentCache.Misses} misses, {session.Options.SegmentCache.HeldBytes} bytes held"); await using (VortexFileWriter writer = session.CreateWriter(Demo.Path("observed.vortex"))) { diff --git a/samples/Vorticity.Samples/OpenAFile.cs b/samples/Vorticity.Samples/OpenAFile.cs index 0ab47646..d0eb9eeb 100644 --- a/samples/Vorticity.Samples/OpenAFile.cs +++ b/samples/Vorticity.Samples/OpenAFile.cs @@ -42,10 +42,10 @@ internal static async Task RunAsync() } CountingSource told = new CountingSource(new FileSegmentSource(path)); - VortexOpenOptions wide = new VortexOpenOptions { Length = bytes.Length, InitialReadSize = 256 * 1024 }; + VortexOpenOptions wide = new VortexOpenOptions { Length = bytes.Length, InitialReadBytes = 256 * 1024 }; await using (VortexFile file = await session.OpenAsync(told, wide)) { - Console.WriteLine($"InitialReadSize 256 KiB: {told.Requests} request, {told.Bytes} bytes"); + Console.WriteLine($"InitialReadBytes 256 KiB: {told.Requests} request, {told.Bytes} bytes"); } await using VortexSession cached = VortexSession.Create(options => options.SegmentCache = new SegmentCache(64L * 1024 * 1024)); @@ -84,7 +84,7 @@ private static async Task TimeAsync(string how, VortexFile opened) _ = columns.RowCount; } - return ((source?.Requests ?? 0) - requests, (source?.Bytes ?? 0) - bytes, scan.Statistics.CacheHits); + return ((source?.Requests ?? 0) - requests, (source?.Bytes ?? 0) - bytes, scan.Metrics.CacheHits); } private sealed class CountingSource(ISegmentSource inner) : ISegmentSource diff --git a/samples/Vorticity.Samples/ProjectColumns.cs b/samples/Vorticity.Samples/ProjectColumns.cs index df8dd563..f89a3a32 100644 --- a/samples/Vorticity.Samples/ProjectColumns.cs +++ b/samples/Vorticity.Samples/ProjectColumns.cs @@ -52,7 +52,7 @@ private static async Task MeasureAsync(VortexFile file, string name) rows += columns.RowCount; } - ScanStatistics stats = scan.Statistics; + ScanMetrics stats = scan.Metrics; Console.WriteLine($"{name,-12} {Leaves(TRecord.Schema),8} {plan.Segments,9} {plan.BytesToRead,10} {stats.Requests,9} {stats.BytesRequested,10} ({rows} rows)"); } diff --git a/samples/Vorticity.Samples/Queries.cs b/samples/Vorticity.Samples/Queries.cs index 2e2bd722..e2d734e9 100644 --- a/samples/Vorticity.Samples/Queries.cs +++ b/samples/Vorticity.Samples/Queries.cs @@ -62,7 +62,7 @@ where g.Count() > 100 if (dayGroups++ == 0) { firstDay = Math.Min(firstDay, clock.Elapsed.TotalMilliseconds); - batchesAtFirst = days.Statistics.Batches; + batchesAtFirst = days.Metrics.Batches; } } @@ -78,7 +78,7 @@ where g.Count() > 100 Scan lastDays = readings.Scan(); List seven = await ToListAsync(lastDays.GroupBy(r => r.Day).OrderByDescending(g => g.Key).Take(7) .Select(g => (g.Key, g.Count())).As()); - Console.WriteLine($"the last seven days, {seven[0].Day} down to {seven[^1].Day}: {lastDays.Statistics.Rows} rows read of a million"); + Console.WriteLine($"the last seven days, {seven[0].Day} down to {seven[^1].Day}: {lastDays.Metrics.Rows} rows read of a million"); // An order on a key that does not stream, under a take: each lane keeps its best keys alone. List shortest = []; diff --git a/samples/Vorticity.Samples/ReadRows.cs b/samples/Vorticity.Samples/ReadRows.cs index bd9f52b3..cbcec0fd 100644 --- a/samples/Vorticity.Samples/ReadRows.cs +++ b/samples/Vorticity.Samples/ReadRows.cs @@ -53,7 +53,7 @@ internal static async Task RunAsync() (_, TimeSpan maxTime, long maxBytes) = await MeasureAsync(async () => (long)(await file.Scan().MaxAsync(r => r.Celsius) ?? 0)); Scan pushed = file.Scan(); double? max = await pushed.MaxAsync(r => r.Celsius); - Console.WriteLine($"MaxAsync, pushed: {max} in {maxTime.TotalMilliseconds:F3} ms, {maxBytes} bytes allocated, {pushed.Statistics.Requests} requests, {pushed.Statistics.BlocksDecoded} blocks decoded"); + Console.WriteLine($"MaxAsync, pushed: {max} in {maxTime.TotalMilliseconds:F3} ms, {maxBytes} bytes allocated, {pushed.Metrics.Requests} requests, {pushed.Metrics.BlocksDecoded} blocks decoded"); (long hot, TimeSpan hotTime, long hotBytes) = await MeasureAsync(() => HottestAsync(file)); Console.WriteLine($"the hottest rows, filtered before the sink: {hot} rows in {hotTime.TotalMilliseconds:F0} ms, {hotBytes / 1024} KiB allocated"); diff --git a/samples/Vorticity.Samples/ReadRowsByIndex.cs b/samples/Vorticity.Samples/ReadRowsByIndex.cs index fd4507c2..5f26cee0 100644 --- a/samples/Vorticity.Samples/ReadRowsByIndex.cs +++ b/samples/Vorticity.Samples/ReadRowsByIndex.cs @@ -20,13 +20,13 @@ internal static async Task RunAsync() } } - ScanStatistics took = take.Statistics; + ScanMetrics took = take.Metrics; Scan all = file.Scan(); await foreach (Columns _ in all) { } - ScanStatistics read = all.Statistics; + ScanMetrics read = all.Metrics; Console.WriteLine($"Rows(4, 900_000): {took.Requests} requests, {took.BytesRequested} bytes, {took.BlocksDecoded} blocks decoded"); Console.WriteLine($"the whole file: {read.Requests} requests, {read.BytesRequested} bytes, {read.BlocksDecoded} blocks decoded"); diff --git a/samples/Vorticity.Samples/ScanATable.cs b/samples/Vorticity.Samples/ScanATable.cs index 38cb3742..0b1dda18 100644 --- a/samples/Vorticity.Samples/ScanATable.cs +++ b/samples/Vorticity.Samples/ScanATable.cs @@ -13,7 +13,7 @@ internal static async Task RunAsync() double total = 0; long seen = 0; - ScanStatistics loop = default; + ScanMetrics loop = default; TimeSpan loopTime = default; for (int round = 0; round < 3; round++) { @@ -31,21 +31,21 @@ internal static async Task RunAsync() seen += celsius.Length; } - (loop, loopTime) = (scan.Statistics, clock.Elapsed); + (loop, loopTime) = (scan.Metrics, clock.Elapsed); } Console.WriteLine($"the loop: mean {total / seen:F4} over {seen} rows, {loopTime.TotalMilliseconds:F1} ms"); Console.WriteLine($" {loop.Batches} batches, {loop.BlocksDecoded} blocks decoded, {loop.Requests} requests, {loop.BytesRequested} bytes"); double? mean = null; - ScanStatistics avg = default; + ScanMetrics avg = default; TimeSpan avgTime = default; for (int round = 0; round < 3; round++) { Stopwatch clock = Stopwatch.StartNew(); Scan scan = file.Scan(); mean = await scan.AverageAsync(r => r.Celsius); - (avg, avgTime) = (scan.Statistics, clock.Elapsed); + (avg, avgTime) = (scan.Metrics, clock.Elapsed); } Console.WriteLine($"AverageAsync: mean {mean:F4}, {avgTime.TotalMilliseconds:F1} ms"); @@ -59,7 +59,7 @@ internal static async Task RunAsync() bool hasDaySum = file.Statistics[0].TryGetSum(out long daySum); Scan days = file.Scan(); double? meanDay = await days.AverageAsync(r => r.Day); - Console.WriteLine($"file statistics of Day: sum {(hasDaySum ? daySum.ToString() : "absent")}; AverageAsync(r => r.Day) {meanDay}, {days.Statistics.Requests} requests"); + Console.WriteLine($"file statistics of Day: sum {(hasDaySum ? daySum.ToString() : "absent")}; AverageAsync(r => r.Day) {meanDay}, {days.Metrics.Requests} requests"); long defaultBytes = await AllocatedAsync(file, new ScanOptions()); (long defaultBatches, _, _) = await BatchesAsync(file, new ScanOptions()); diff --git a/samples/Vorticity.Samples/StatisticsAndPruning.cs b/samples/Vorticity.Samples/StatisticsAndPruning.cs index 76bafe84..121f15f4 100644 --- a/samples/Vorticity.Samples/StatisticsAndPruning.cs +++ b/samples/Vorticity.Samples/StatisticsAndPruning.cs @@ -19,7 +19,7 @@ internal static async Task RunAsync() { } - Print("Day >= 900", scan.Statistics); + Print("Day >= 900", scan.Metrics); ScanPlan hot = await file.Scan().Where(r => r.Celsius > 45.0).ExplainAsync(); Print("Celsius > 45", hot); @@ -28,14 +28,14 @@ internal static async Task RunAsync() { } - Print("Celsius > 45", hotScan.Statistics); + Print("Celsius > 45", hotScan.Metrics); Scan everything = file.Scan(); await foreach (Columns _ in everything) { } - Print("no filter", everything.Statistics); + Print("no filter", everything.Metrics); ScanPlan none = await file.Scan().Where(r => r.Day >= 5_000).ExplainAsync(); Print("Day >= 5000", none); @@ -44,7 +44,7 @@ internal static async Task RunAsync() { } - Print("Day >= 5000", noneScan.Statistics); + Print("Day >= 5000", noneScan.Metrics); VortexFileStatistics statistics = file.Statistics; for (int i = 0; i < statistics.Count; i++) @@ -115,6 +115,6 @@ private static void Print(string what, ScanPlan plan) Console.WriteLine($" count: exact {count.Exact}, {count.Rows} rows, {count.Pruned} pruned, {count.Proven} proven, {count.Decoded} decoded"); } - private static void Print(string what, ScanStatistics stats) => + private static void Print(string what, ScanMetrics stats) => Console.WriteLine($"ran, {what}: {stats.Rows} rows in {stats.Batches} batches, {stats.Requests} requests, {stats.BytesRequested} bytes, {stats.BlocksDecoded} blocks decoded, {stats.BlocksPruned} pruned"); } diff --git a/samples/Vorticity.Samples/Threads.cs b/samples/Vorticity.Samples/Threads.cs index cf75e61d..59ed9fc0 100644 --- a/samples/Vorticity.Samples/Threads.cs +++ b/samples/Vorticity.Samples/Threads.cs @@ -41,8 +41,8 @@ internal static async Task RunAsync() SegmentCache cache = session.Options.SegmentCache!; Scan again = file.Scan(); await CountRowsAsync(again); - Console.WriteLine($"a fifth scan: {again.Statistics.Requests} requests, {again.Statistics.CacheHits} served by the cache"); - Console.WriteLine($"the cache: {cache.Size / 1024} KiB held, {cache.Hits} hits, {cache.Misses} misses"); + Console.WriteLine($"a fifth scan: {again.Metrics.Requests} requests, {again.Metrics.CacheHits} served by the cache"); + Console.WriteLine($"the cache: {cache.HeldBytes / 1024} KiB held, {cache.Hits} hits, {cache.Misses} misses"); Scan once = file.Scan(); await once.CountAsync(); diff --git a/samples/Vorticity.Samples/WriterOptions.cs b/samples/Vorticity.Samples/WriterOptions.cs index 0c823c36..8ca3073d 100644 --- a/samples/Vorticity.Samples/WriterOptions.cs +++ b/samples/Vorticity.Samples/WriterOptions.cs @@ -88,9 +88,9 @@ private static async Task Profiles(VortexSession session, string path, Reading[] Stopwatch clock = Stopwatch.StartNew(); await foreach (var (day, celsius, city) in file.Scan()) { - day.Canonical(); - celsius.Canonical(); - city.Canonical(); + day.AsCanonical(); + celsius.AsCanonical(); + city.AsCanonical(); } read = Math.Min(read, clock.ElapsedMilliseconds); @@ -163,7 +163,7 @@ private static async Task StringBounds(VortexSession session, string path) private static async Task Rest(VortexSession session, string path, Reading[] readings) { WriteReport with = await Write(session, path, readings, new VortexWriteOptions()); - WriteReport without = await Write(session, path, readings, new VortexWriteOptions { Statistics = false }); + WriteReport without = await Write(session, path, readings, new VortexWriteOptions { WriteStatistics = false }); await using (VortexFile file = await VortexFile.OpenAsync(path)) { ScanPlan plan = await file.Scan().Where(r => r.Day > 2_000).ExplainAsync(); diff --git a/src/Vorticity.Dataset/CompactionPlan.cs b/src/Vorticity.Dataset/CompactionPlan.cs index 04d5adf0..330dd134 100644 --- a/src/Vorticity.Dataset/CompactionPlan.cs +++ b/src/Vorticity.Dataset/CompactionPlan.cs @@ -301,7 +301,7 @@ internal CompactionJob( Bytes += input.Entry.Bytes; } - Objects = objects.MoveToImmutable(); + ObjectKeys = objects.MoveToImmutable(); } /// The level it empties. @@ -320,7 +320,7 @@ internal CompactionJob( public long TargetBytes { get; init; } /// The keys of the objects it reads, the source level's first. - public ImmutableArray Objects { get; init; } + public ImmutableArray ObjectKeys { get; init; } /// The rows it reads, which are the rows it writes. public long Rows { get; init; } diff --git a/src/Vorticity.Dataset/CompactionSchedule.cs b/src/Vorticity.Dataset/CompactionSchedule.cs index e4cbf15e..d0ba96f9 100644 --- a/src/Vorticity.Dataset/CompactionSchedule.cs +++ b/src/Vorticity.Dataset/CompactionSchedule.cs @@ -48,7 +48,7 @@ public sealed record CompactionSchedule /// same, until the span ends. Nothing releases a lease, so that a store that refuses deletes still /// works. Vacuum deletes the leases that ended before its window. /// - public bool Leases { get; init; } + public bool UseLeases { get; init; } /// /// How long a lease holds its levels, the same for every loop of a dataset; a minute by default. @@ -109,7 +109,7 @@ public static async Task RunAsync(VortexDataset dataset, CompactionSchedule sche ArgumentNullException.ThrowIfNull(schedule); schedule.Check(); TimeProvider clock = schedule.TimeProvider; - int ranked = schedule.Leases ? Math.Max(LeasedJobs, schedule.Loops) : schedule.Loop + 1; + int ranked = schedule.UseLeases ? Math.Max(LeasedJobs, schedule.Loops) : schedule.Loop + 1; LeaseBook leases = new LeaseBook(); // Off the caller's thread from the first turn: against a store that answers at once, the jobs @@ -147,7 +147,7 @@ public static async Task RunAsync(VortexDataset dataset, CompactionSchedule sche internal static async ValueTask ChooseAsync( IObjectStore store, CompactionSchedule schedule, CompactionPlan plan, LeaseBook leases, CancellationToken cancellationToken) { - if (!schedule.Leases) + if (!schedule.UseLeases) { return schedule.Loop < plan.Jobs.Length ? plan.Jobs[schedule.Loop] : null; } diff --git a/src/Vorticity.Dataset/DatasetKeyCursor.cs b/src/Vorticity.Dataset/DatasetKeyCursor.cs index 9205f097..04d03901 100644 --- a/src/Vorticity.Dataset/DatasetKeyCursor.cs +++ b/src/Vorticity.Dataset/DatasetKeyCursor.cs @@ -68,8 +68,8 @@ internal sealed partial class DatasetKeyCursor : IKeyWalker // the object a step against the heap's direction starts from; it steps rather than seeks. private Anchor _anchor; private FilterLiteral _target; - private SeekOp _beforePivot; - private SeekOp _fromPivot; + private SeekMode _beforePivot; + private SeekMode _fromPivot; private Slot? _pivot; private Slot? _current; private bool _disposed; @@ -252,14 +252,14 @@ public async ValueTask SeekLastAsync(CancellationToken cancellationToken = /// /// Positions relative to a key, given in the key's domain: the row encoding of the tuple for a - /// composite key. and walk down from + /// composite key. and walk down from /// where they land, and on a distinct walk land on that key's first entry. /// public async ValueTask SeekAsync( - FilterLiteral key, SeekOp op = SeekOp.AtOrAfter, CancellationToken cancellationToken = default) + FilterLiteral key, SeekMode op = SeekMode.AtOrAfter, CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(_disposed, this); - if (op is SeekOp.AtOrBefore or SeekOp.Before) + if (op is SeekMode.AtOrBefore or SeekMode.Before) { return await SeekDownAsync(key, op, cancellationToken).ConfigureAwait(false) && (!_distinct || await FirstOfKeyAsync(cancellationToken).ConfigureAwait(false)); @@ -268,12 +268,12 @@ public async ValueTask SeekAsync( // Every cursor goes to the lower bound even for an exact seek, exactness being decided on // the winner afterwards: seeking each one exactly would invalidate the cursors whose object // does not hold the key, and truncate their rows out of the rest of the merge. - if (!await SeekUpAsync(key, op == SeekOp.Exact ? SeekOp.AtOrAfter : op, cancellationToken).ConfigureAwait(false)) + if (!await SeekUpAsync(key, op == SeekMode.Exact ? SeekMode.AtOrAfter : op, cancellationToken).ConfigureAwait(false)) { return false; } - if (op == SeekOp.Exact && KeyCursor.Compare(Key, key) != 0) + if (op == SeekMode.Exact && KeyCursor.Compare(Key, key) != 0) { _current = null; return false; @@ -576,7 +576,7 @@ public ValueTask NextAsync(CancellationToken cancellationToken = default) /// Moves to the previous entry in key order; on a distinct walk, to the previous key's first /// entry. /// - public ValueTask PrevAsync(CancellationToken cancellationToken = default) + public ValueTask PreviousAsync(CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(_disposed, this); if (_current is null) @@ -605,14 +605,14 @@ public ValueTask NextKeyAsync(CancellationToken cancellationToken = defaul return new ValueTask(false); } - return _direction > 0 ? PastKeyAsync(cancellationToken) : SeekUpAsync(Key, SeekOp.After, cancellationToken); + return _direction > 0 ? PastKeyAsync(cancellationToken) : SeekUpAsync(Key, SeekMode.After, cancellationToken); } /// /// Moves to the last entry of the previous distinct key: walking down, every object positioned /// on the current key steps past it; walking up, the walk turns and seeks before it. /// - public ValueTask PrevKeyAsync(CancellationToken cancellationToken = default) + public ValueTask PreviousKeyAsync(CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(_disposed, this); if (_current is null) @@ -620,7 +620,7 @@ public ValueTask PrevKeyAsync(CancellationToken cancellationToken = defaul return new ValueTask(false); } - return _direction < 0 ? PastKeyAsync(cancellationToken) : SeekDownAsync(Key, SeekOp.Before, cancellationToken); + return _direction < 0 ? PastKeyAsync(cancellationToken) : SeekDownAsync(Key, SeekMode.Before, cancellationToken); } /// @@ -651,7 +651,7 @@ public async ValueTask RankAsync(FilterLiteral key, CancellationToken canc /// would have opened before the walk reached a key it could hold. /// /// The cursor is not positioned. - public async ValueTask KeyCountAsync(CancellationToken cancellationToken = default) + public async ValueTask CountAtKeyAsync(CancellationToken cancellationToken = default) { FilterLiteral key = Key; long count = 0; @@ -664,7 +664,7 @@ public async ValueTask KeyCountAsync(CancellationToken cancellationToken = if (slot.Live && KeyCursor.Compare(cursor.Key, key) == 0) { - count += await cursor.KeyCountAsync(cancellationToken).ConfigureAwait(false); + count += await cursor.CountAtKeyAsync(cancellationToken).ConfigureAwait(false); } else if (_direction < 0) { @@ -796,7 +796,7 @@ private async ValueTask StepAsync(CancellationToken cancellationToken) // The current entry is the top's: its cursor moves on and sinks to its place, or leaves. slot.Live = _direction > 0 ? await slot.Cursor!.NextAsync(cancellationToken).ConfigureAwait(false) - : await slot.Cursor!.PrevAsync(cancellationToken).ConfigureAwait(false); + : await slot.Cursor!.PreviousAsync(cancellationToken).ConfigureAwait(false); if (slot.Live) { SiftDown(0); @@ -824,14 +824,14 @@ private async ValueTask TurnAsync(CancellationToken cancellationToken) byte[]? sought = _bounded ? Encoded(key) : null; if (_direction > 0) { - _beforePivot = SeekOp.AtOrBefore; - _fromPivot = SeekOp.Before; + _beforePivot = SeekMode.AtOrBefore; + _fromPivot = SeekMode.Before; await BeginAsync(-1, Anchor.Target, sought, inclusive: true, cancellationToken).ConfigureAwait(false); } else { - _beforePivot = SeekOp.After; - _fromPivot = SeekOp.AtOrAfter; + _beforePivot = SeekMode.After; + _fromPivot = SeekMode.AtOrAfter; await BeginAsync(1, Anchor.Target, sought, inclusive: true, cancellationToken).ConfigureAwait(false); } @@ -840,7 +840,7 @@ private async ValueTask TurnAsync(CancellationToken cancellationToken) } /// Positions every object at or after by , walking up. - private async ValueTask SeekUpAsync(FilterLiteral key, SeekOp op, CancellationToken cancellationToken) + private async ValueTask SeekUpAsync(FilterLiteral key, SeekMode op, CancellationToken cancellationToken) { _target = key; _fromPivot = op; @@ -851,19 +851,19 @@ private async ValueTask SeekUpAsync(FilterLiteral key, SeekOp op, Cancella } /// Positions every object at or before by , walking down. - private async ValueTask SeekDownAsync(FilterLiteral key, SeekOp op, CancellationToken cancellationToken) + private async ValueTask SeekDownAsync(FilterLiteral key, SeekMode op, CancellationToken cancellationToken) { _target = key; _fromPivot = op; _pivot = null; - await BeginAsync(-1, Anchor.Target, _bounded ? Encoded(key) : null, inclusive: op != SeekOp.Before, cancellationToken).ConfigureAwait(false); + await BeginAsync(-1, Anchor.Target, _bounded ? Encoded(key) : null, inclusive: op != SeekMode.Before, cancellationToken).ConfigureAwait(false); await ResolveAsync(cancellationToken).ConfigureAwait(false); return Choose(); } /// A distinct walk's step down: the previous key's last entry, then that key's first. private async ValueTask PrevDistinctAsync(CancellationToken cancellationToken) => - await PrevKeyAsync(cancellationToken).ConfigureAwait(false) + await PreviousKeyAsync(cancellationToken).ConfigureAwait(false) && await FirstOfKeyAsync(cancellationToken).ConfigureAwait(false); /// @@ -871,7 +871,7 @@ await PrevKeyAsync(cancellationToken).ConfigureAwait(false) /// a key's last entry, and a distinct entry is the key's first. /// private ValueTask FirstOfKeyAsync(CancellationToken cancellationToken) => - SeekUpAsync(Key, SeekOp.AtOrAfter, cancellationToken); + SeekUpAsync(Key, SeekMode.AtOrAfter, cancellationToken); /// /// Moves every object on the current key past it in the heap's direction: each leaves the heap @@ -891,7 +891,7 @@ private async ValueTask PastKeyAsync(CancellationToken cancellationToken) Slot slot = _moved[i]; slot.Live = _direction > 0 ? await slot.Cursor!.NextKeyAsync(cancellationToken).ConfigureAwait(false) - : await slot.Cursor!.PrevKeyAsync(cancellationToken).ConfigureAwait(false); + : await slot.Cursor!.PreviousKeyAsync(cancellationToken).ConfigureAwait(false); if (slot.Live) { Push(slot); @@ -989,7 +989,7 @@ private async ValueTask ResolveAsync(CancellationToken cancellationToken) { Anchor.First => cursor.SeekFirstAsync(cancellationToken), Anchor.Last => cursor.SeekLastAsync(cancellationToken), - _ when slot == _pivot => _direction > 0 ? cursor.NextAsync(cancellationToken) : cursor.PrevAsync(cancellationToken), + _ when slot == _pivot => _direction > 0 ? cursor.NextAsync(cancellationToken) : cursor.PreviousAsync(cancellationToken), _ => cursor.SeekAsync(_target, _pivot is not null && Order(slot, _pivot) < 0 ? _beforePivot : _fromPivot, cancellationToken), }; diff --git a/src/Vorticity.Dataset/DatasetScanBuilder.cs b/src/Vorticity.Dataset/DatasetScanBuilder.cs index 9e9da8b2..341593c9 100644 --- a/src/Vorticity.Dataset/DatasetScanBuilder.cs +++ b/src/Vorticity.Dataset/DatasetScanBuilder.cs @@ -19,7 +19,7 @@ namespace Vorticity.Dataset; /// What a dataset scan skipped, and how: counters rather than timings, so that a caller can check /// how few objects were read instead of believing a speed. /// -internal sealed class DatasetScanMetrics +internal sealed class DatasetScanCounters { /// Leaf entries the walk reached. public long ObjectsConsidered { get; internal set; } @@ -67,8 +67,8 @@ internal sealed class DatasetScanBuilder private long[]? _take; private string[]? _orderPaths; private bool _descending; - private DatasetScanMetrics? _metrics; - private ScanMetrics? _counters; + private DatasetScanCounters? _metrics; + private ScanCounters? _counters; private ScanOptions _options = ScanOptions.Default; private bool _keepEncodings; private bool _sinkDecodes; @@ -216,14 +216,14 @@ public DatasetScanBuilder WithSummaries(bool summaries) } /// Records what the walk skipped. - public DatasetScanBuilder WithMetrics(DatasetScanMetrics metrics) + public DatasetScanBuilder WithMetrics(DatasetScanCounters metrics) { _metrics = metrics; return this; } /// Adds what every object's own scan requests, decodes and delivers to . - public DatasetScanBuilder WithCounters(ScanMetrics counters) + public DatasetScanBuilder WithCounters(ScanCounters counters) { _counters = counters; return this; @@ -497,7 +497,7 @@ public ValueTask MaxAsync(string path, CancellationToken cancella /// public async ValueTask ExplainAsync(CancellationToken cancellationToken = default) { - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); long objects = 0; long rows = 0; List keptByLevel = []; @@ -1570,7 +1570,7 @@ private ScanBuilder Configure(ScanBuilder scan, VortexExpr? filter, FieldMask? p int degree = _options.DegreeOfParallelism > 0 ? _options.DegreeOfParallelism : _dataset.Session.Options.MaxDegreeOfParallelism; - scan.WithPruning(_options.Pruning) + scan.WithPruning(_options.UseStatistics) .WithIndexes(_indexes) .WithDegreeOfParallelism(Math.Max(degree, 1)) .WithPrefetch(_options.Prefetch) diff --git a/src/Vorticity.Dataset/DatasetScanSource.cs b/src/Vorticity.Dataset/DatasetScanSource.cs index bd24e828..e6a14bca 100644 --- a/src/Vorticity.Dataset/DatasetScanSource.cs +++ b/src/Vorticity.Dataset/DatasetScanSource.cs @@ -35,16 +35,16 @@ internal DatasetScanSource(VortexDataset dataset, DatasetSnapshot version) /// The version every sink of the scan reads. internal ulong Version => _version.Version; - internal override IAsyncEnumerable BatchesAsync(ScanSpec spec, ScanMetrics metrics) => + internal override IAsyncEnumerable BatchesAsync(ScanSpec spec, ScanCounters metrics) => spec.MatchesNothing ? System.Linq.AsyncEnumerable.Empty() : Builder(spec, metrics).ExecuteAsync(); - internal override ValueTask CountAsync(ScanSpec spec, ScanMetrics metrics, CancellationToken cancellationToken) => + internal override ValueTask CountAsync(ScanSpec spec, ScanCounters metrics, CancellationToken cancellationToken) => spec.MatchesNothing ? ValueTask.FromResult(0L) : Builder(spec, metrics).CountAsync(cancellationToken); - internal override ValueTask AnyAsync(ScanSpec spec, ScanMetrics metrics, CancellationToken cancellationToken) => + internal override ValueTask AnyAsync(ScanSpec spec, ScanCounters metrics, CancellationToken cancellationToken) => spec.MatchesNothing ? ValueTask.FromResult(false) : Builder(spec, metrics).AnyAsync(cancellationToken); - internal override ValueTask ExtremeAsync(ScanSpec spec, FieldExpr column, bool min, ScanMetrics metrics, CancellationToken cancellationToken) + internal override ValueTask ExtremeAsync(ScanSpec spec, FieldExpr column, bool min, ScanCounters metrics, CancellationToken cancellationToken) { if (spec.MatchesNothing) { @@ -106,7 +106,7 @@ internal override bool OrdersOnAsking(FieldExpr column) => return null; } - SummaryPruner? pruner = spec.Options.Pruning && spec.Filter is { } filter ? new SummaryPruner(Compute.FunctionFieldExpr.Ranges(filter)) : null; + SummaryPruner? pruner = spec.Options.UseStatistics && spec.Filter is { } filter ? new SummaryPruner(Compute.FunctionFieldExpr.Ranges(filter)) : null; long from = spec.Rows?.Start ?? 0; long to = spec.Rows?.End ?? long.MaxValue; List<(PositionedObject Held, RowRange Rows)> objects = []; @@ -191,7 +191,7 @@ private async Task CutAsync(PositionedObject held, RowRange rows, lo } /// The engine's scan for , adding what the objects' scans count to . - internal DatasetScanBuilder Builder(ScanSpec spec, ScanMetrics? counters) + internal DatasetScanBuilder Builder(ScanSpec spec, ScanCounters? counters) { DatasetScanBuilder builder = new DatasetScanBuilder(_dataset, _version); if (spec.Projection is { } mask) @@ -220,7 +220,7 @@ internal DatasetScanBuilder Builder(ScanSpec spec, ScanMetrics? counters) } ScanOptions options = spec.Options; - builder.WithOptions(options, spec.KeepEncodings, spec.SinkDecodes, spec.PositionsUnread).WithIndexes(options.UseIndexes).WithSummaries(options.Pruning); + builder.WithOptions(options, spec.KeepEncodings, spec.SinkDecodes, spec.PositionsUnread).WithIndexes(options.UseIndexes).WithSummaries(options.UseStatistics); return counters is null ? builder : builder.WithCounters(counters); } } diff --git a/src/Vorticity.Dataset/DatasetSnapshot.cs b/src/Vorticity.Dataset/DatasetSnapshot.cs index 4841db9b..d1304c6e 100644 --- a/src/Vorticity.Dataset/DatasetSnapshot.cs +++ b/src/Vorticity.Dataset/DatasetSnapshot.cs @@ -85,7 +85,7 @@ internal async IAsyncEnumerable WalkAsync( SummaryPruner? pruner, long from, long to, - DatasetScanMetrics? metrics, + DatasetScanCounters? metrics, [EnumeratorCancellation] CancellationToken cancellationToken) { // Each node and each entry is asked about the pruner's columns alone, decoded out of its @@ -361,7 +361,7 @@ private bool MayMatch(PageReference reference, int depth, SummaryPruner pruner, long from, long to, Func? mayMatch, - DatasetScanMetrics? metrics, + DatasetScanCounters? metrics, [EnumeratorCancellation] CancellationToken cancellationToken) { List trees = []; diff --git a/src/Vorticity.Dataset/ObjectDraft.cs b/src/Vorticity.Dataset/ObjectDraft.cs index 4b638c5e..a0df72bb 100644 --- a/src/Vorticity.Dataset/ObjectDraft.cs +++ b/src/Vorticity.Dataset/ObjectDraft.cs @@ -7,7 +7,7 @@ namespace Vorticity.Dataset; /// A data object being written for a dataset: a core writer over a fresh key, which the dataset /// completes, puts in the store and commits when it is handed to /// or -/// . +/// . /// /// /// Write the rows through and leave its completion to the dataset. Disposing a diff --git a/src/Vorticity.Dataset/VortexDataset.cs b/src/Vorticity.Dataset/VortexDataset.cs index b1fb8833..5f38c16a 100644 --- a/src/Vorticity.Dataset/VortexDataset.cs +++ b/src/Vorticity.Dataset/VortexDataset.cs @@ -82,7 +82,7 @@ private VortexDataset(IObjectStore store, DatasetOptions options, ulong version, public long Lag => Snapshot.Levels.LagAtLevelZero(); /// The columns the dataset is ordered by, empty when objects are kept in the order they arrived. - public IReadOnlyList ClusteringKeyPaths => Snapshot.Header.ClusteringKey; + public IReadOnlyList ClusteringKey => Snapshot.Header.ClusteringKey; /// /// What the latest compaction run after a commit to fail on this handle raised @@ -259,7 +259,7 @@ public bool MayMatch(Func, Predicate> filter) /// Every data object of the version this handle holds, over every level, in the order a scan reads them. /// Cancels the reads of the tree. /// The objects; reading them reads tree pages only, never an object. - public async IAsyncEnumerable ObjectsAsync( + public async IAsyncEnumerable ListObjectsAsync( [EnumeratorCancellation] CancellationToken cancellationToken = default) { await foreach (PositionedObject held in @@ -422,18 +422,18 @@ public async ValueTask ImportAsync(string objectKey, CancellationToken ca } /// Removes data objects of the version this handle holds, in one commit. - /// Objects from . + /// Objects from . /// Cancels the commit. /// The version created, and whether the removal applied; the latest version, unchanged, when it did not. /// The objects stay in the store, readable by older versions, until vacuum deletes them. - public ValueTask RemoveAsync(IReadOnlyList objects, CancellationToken cancellationToken = default) => - ReplaceAsync(objects, [], cancellationToken); + public ValueTask RemoveObjectsAsync(IReadOnlyList objects, CancellationToken cancellationToken = default) => + ReplaceObjectsAsync(objects, [], cancellationToken); /// /// Removes data objects and adds others written through , in one commit: /// a reader sees either the old objects or the new ones, never both nor neither. /// - /// Objects from . + /// Objects from . /// Drafts whose rows are written; each is completed and put before the commit. /// Cancels the puts and the commit. /// @@ -447,7 +447,7 @@ public ValueTask RemoveAsync(IReadOnlyList objects, C /// /// An object does not come from this dataset's walk. /// A draft belongs to another handle, was already committed, or its writer was abandoned. - public async ValueTask ReplaceAsync( + public async ValueTask ReplaceObjectsAsync( IReadOnlyList removed, IReadOnlyList added, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(removed); @@ -459,7 +459,7 @@ public async ValueTask ReplaceAsync( ArgumentNullException.ThrowIfNull(gone, nameof(removed)); if (gone.TreeKey.Length == 0 || gone.Entry is not { } entry) { - throw new ArgumentException("An object to remove comes from ObjectsAsync, which knows where its entry is.", nameof(removed)); + throw new ArgumentException("An object to remove comes from ListObjectsAsync, which knows where its entry is.", nameof(removed)); } inputs.Add((gone.Level, Convert.FromHexString(gone.TreeKey))); @@ -714,7 +714,7 @@ public ValueTask PlanCompactionAsync( /// A job is the one runs, so a loop is safe against every writer and /// every other loop. Loops that know one another spread over the due jobs by /// ; loops that cannot count one another lease a job's - /// levels (). + /// levels (). /// /// The schedule's loop index, rate, sleep or lease span is out of range. public Task RunCompactionAsync(CompactionSchedule? schedule = null, CancellationToken cancellationToken = default) => @@ -764,7 +764,7 @@ internal static async ValueTask CreateAsync( $"This store already holds a dataset at version {existing}; open it rather than creating it."); } - _ = ClusteringKey.Declared(options.ClusteringKey, schema); + _ = global::Vorticity.Dataset.ClusteringKey.Declared(options.ClusteringKey, schema); CommitHeader template = new CommitHeader { @@ -908,7 +908,7 @@ internal ValueTask RankAsync(FilterLiteral key, CancellationToken cancella /// their ancestors', do not refute the pruner, in key order. A null pruner keeps every object. /// internal IAsyncEnumerable WalkAsync( - SummaryPruner? pruner, long from, long to, DatasetScanMetrics? metrics, CancellationToken cancellationToken) => + SummaryPruner? pruner, long from, long to, DatasetScanCounters? metrics, CancellationToken cancellationToken) => Snapshot.WalkAsync(pruner, from, to, metrics, cancellationToken); /// diff --git a/src/Vorticity.Generators/Analyzers/SelectionRecordAnalyzer.cs b/src/Vorticity.Generators/Analyzers/SelectionRecordAnalyzer.cs index 9063d5b8..944faefc 100644 --- a/src/Vorticity.Generators/Analyzers/SelectionRecordAnalyzer.cs +++ b/src/Vorticity.Generators/Analyzers/SelectionRecordAnalyzer.cs @@ -11,7 +11,7 @@ namespace Vorticity.Generators.Analyzers; /// /// The records a selection of several values is read through. VX1010: an As<TRecord>() -/// or an AggAsync<TResult> whose record does not take the elements of the selection, +/// or an AggregateAsync<TResult> whose record does not take the elements of the selection, /// position by position, which they refuse when they run. VX1011: several values read without a /// record, which the compiler refuses with an error that does not name the fix. /// @@ -50,7 +50,7 @@ public override void Initialize(AnalysisContext context) }); } - /// VX1010: the record of an As or an AggAsync against the elements the compiler sees. + /// VX1010: the record of an As or an AggregateAsync against the elements the compiler sees. private static void AnalyzeRecord(OperationAnalysisContext context, KnownSymbols known) { IInvocationOperation invocation = (IInvocationOperation)context.Operation; @@ -66,7 +66,7 @@ private static void AnalyzeRecord(OperationAnalysisContext context, KnownSymbols || KnownSymbols.Is(method.ContainingType.OriginalDefinition, known.ValueProjection) => [((INamedTypeSymbol)method.ContainingType).TypeArguments[0]], "As" when IsSeveral(method.ContainingType, known) => SelectedBy(invocation.Instance, known), - "AggAsync" when KnownSymbols.Is(method.ContainingType.OriginalDefinition, known.TypedScan) && invocation.Arguments.Length > 0 + "AggregateAsync" when KnownSymbols.Is(method.ContainingType.OriginalDefinition, known.TypedScan) && invocation.Arguments.Length > 0 && method.Parameters[0].Type is INamedTypeSymbol { TypeArguments.Length: 2 } lambda && lambda.TypeArguments[1].Name == "ITuple" => Elements(invocation.Arguments[0].Value, known), _ => null, @@ -134,11 +134,11 @@ private static void AnalyzeMember(SyntaxNodeAnalysisContext context, KnownSymbol $"A selection of several values has no {access.Name.Identifier.ValueText}: read it through a [VortexRecord] whose members take them in order, .As()")); } - /// VX1011: AggAsync(a => (…)) with no record named. + /// VX1011: AggregateAsync(a => (…)) with no record named. private static void AnalyzeAnswers(SyntaxNodeAnalysisContext context, KnownSymbols known) { InvocationExpressionSyntax invocation = (InvocationExpressionSyntax)context.Node; - if (invocation.Expression is not MemberAccessExpressionSyntax { Name: IdentifierNameSyntax { Identifier.ValueText: "AggAsync" } } access + if (invocation.Expression is not MemberAccessExpressionSyntax { Name: IdentifierNameSyntax { Identifier.ValueText: "AggregateAsync" } } access || invocation.ArgumentList.Arguments.Count == 0 || invocation.ArgumentList.Arguments[0].Expression is not LambdaExpressionSyntax { ExpressionBody: TupleExpressionSyntax } || context.SemanticModel.GetTypeInfo(access.Expression, context.CancellationToken).Type is not INamedTypeSymbol scan @@ -150,7 +150,7 @@ private static void AnalyzeAnswers(SyntaxNodeAnalysisContext context, KnownSymbo context.ReportDiagnostic(Diagnostic.Create( Descriptors.SeveralValuesWithoutRecord, access.Name.GetLocation(), - "Several answers go into a record: declare a [VortexRecord] whose members take them in order, and name it, AggAsync(…)")); + "Several answers go into a record: declare a [VortexRecord] whose members take them in order, and name it, AggregateAsync(…)")); } /// diff --git a/src/Vorticity.Generators/Descriptors.cs b/src/Vorticity.Generators/Descriptors.cs index e1612891..f9d536ce 100644 --- a/src/Vorticity.Generators/Descriptors.cs +++ b/src/Vorticity.Generators/Descriptors.cs @@ -100,7 +100,7 @@ internal static class Descriptors public static readonly DiagnosticDescriptor RecordDoesNotTakeSelection = new DiagnosticDescriptor( "VX1010", "A record does not take the values of a selection", - "{0}; As and AggAsync throw when they run", + "{0}; As and AggregateAsync throw when they run", "Usage", DiagnosticSeverity.Error, isEnabledByDefault: true, @@ -114,6 +114,6 @@ internal static class Descriptors "Usage", DiagnosticSeverity.Error, isEnabledByDefault: true, - "Several values have no .NET type until a record gives them one: declare a [VortexRecord] whose members take them in order, and read the selection with As(), or the answers of a scan with AggAsync.", + "Several values have no .NET type until a record gives them one: declare a [VortexRecord] whose members take them in order, and read the selection with As(), or the answers of a scan with AggregateAsync.", HelpLink + "vx1011"); } diff --git a/src/Vorticity/Advice/EncodingAdvisor.cs b/src/Vorticity/Advice/EncodingAdvisor.cs index 5faa78b0..dfb763a8 100644 --- a/src/Vorticity/Advice/EncodingAdvisor.cs +++ b/src/Vorticity/Advice/EncodingAdvisor.cs @@ -93,7 +93,7 @@ internal static class EncodingAdvisor /// private const double SameBytes = 0.01; - private static readonly VortexWriteOptions Plain = new() { Compression = CompressionProfile.None, Statistics = false }; + private static readonly VortexWriteOptions Plain = new() { Compression = CompressionProfile.None, WriteStatistics = false }; /// Advises on every column of the advice can measure. internal static async ValueTask AdviseAsync( diff --git a/src/Vorticity/Aggregation/Aggregates.cs b/src/Vorticity/Aggregation/Aggregates.cs index d0025d94..e82af681 100644 --- a/src/Vorticity/Aggregation/Aggregates.cs +++ b/src/Vorticity/Aggregation/Aggregates.cs @@ -5,7 +5,7 @@ namespace Vorticity; /// -/// The aggregates of a scan, for the lambda of AggAsync: each member is a symbol that stands +/// The aggregates of a scan, for the lambda of AggregateAsync: each member is a symbol that stands /// for one answer, and every answer the lambda returns is computed in one pass. /// /// The record the scan is typed by. @@ -244,7 +244,7 @@ private Sym Widened(Func, Sym> colu /// A probe of the row a choice keeps, of the rows the filter keeps. private Probe Chosen(AggregateKind kind, ColumnShape? by) => new Probe( - _binding ?? throw new InvalidOperationException("A chosen row is taken inside the lambda of AggAsync, which hands it its columns."), + _binding ?? throw new InvalidOperationException("A chosen row is taken inside the lambda of AggregateAsync, which hands it its columns."), null, Aggregators.Chosen(kind, by, _filter)); diff --git a/src/Vorticity/Aggregation/AggregationEngine.cs b/src/Vorticity/Aggregation/AggregationEngine.cs index d5d3e96a..55a9689e 100644 --- a/src/Vorticity/Aggregation/AggregationEngine.cs +++ b/src/Vorticity/Aggregation/AggregationEngine.cs @@ -139,8 +139,8 @@ internal AggregationPlan(ReadOnlySpan results, ColumnShape[] keys) /// The chunks the last run's top-k ranked on tasks of their own; one when it ranked its groups at once. internal int LastTopChunks { get; set; } - /// What the plan's last run did, as the query's statistics say it (docs/design/16-queries.md §10); null before one. - internal GroupStatistics? Statistics() + /// What the plan's last run did, as the query's metrics say it (docs/design/16-queries.md §10); null before one. + internal GroupMetrics? Metrics() { if (LastRun is not { } run) { @@ -148,7 +148,7 @@ internal AggregationPlan(ReadOnlySpan results, ColumnShape[] keys) } CoreRun? core = run.Core; - return new GroupStatistics( + return new GroupMetrics( LastGroups, PeakGroups, LastPeakBytes, @@ -3128,9 +3128,9 @@ internal abstract class AggregationHost { internal abstract ScanSource Source { get; } - internal abstract ScanMetrics Metrics { get; } + internal abstract ScanCounters Counters { get; } - internal abstract ScanStatistics Statistics { get; } + internal abstract ScanMetrics Metrics { get; } internal abstract ScanSpec Spec(); @@ -3145,7 +3145,7 @@ internal async ValueTask RunAsync(AggregationPlan plan, Canc try { ScanSpec spec = Spec(rows); - AggregationOutcome outcome = await AggregationEngine.RunAsync(Source, spec, Metrics, plan, cancellationToken).ConfigureAwait(false); + AggregationOutcome outcome = await AggregationEngine.RunAsync(Source, spec, Counters, plan, cancellationToken).ConfigureAwait(false); if (plan.Chosen.Length > 0) { // The columns of the chosen rows, read once every group's row is known. @@ -3157,7 +3157,7 @@ internal async ValueTask RunAsync(AggregationPlan plan, Canc try { - await ChosenFetch.FetchAsync(outcome, Source, spec, Metrics, groups, cancellationToken).ConfigureAwait(false); + await ChosenFetch.FetchAsync(outcome, Source, spec, Counters, groups, cancellationToken).ConfigureAwait(false); } catch { @@ -3196,7 +3196,7 @@ internal async ValueTask RunAsync(AggregationPlan plan, Canc // With no order nor window over the groups, a core delivers them part by part. bool parted = query.Plan.CoreParted && top is null && !Array.Exists(query.Operators, op => op is GroupOrder or GroupWindow); - AggregationOutcome outcome = await AggregationEngine.RunAsync(Source, spec, Metrics, query.Plan, cancellationToken, top, parted, builder).ConfigureAwait(false); + AggregationOutcome outcome = await AggregationEngine.RunAsync(Source, spec, Counters, query.Plan, cancellationToken, top, parted, builder).ConfigureAwait(false); query.PeakGroups = Math.Max(top?.Peak ?? 0, outcome.Keys?.Count ?? 1); if (outcome.Parts is not null && Array.Exists(query.Operators, op => op is GroupOrder)) { @@ -3245,7 +3245,7 @@ internal ScanSpec Spec(VortexExpr? rows) internal static class AggregationEngine { internal static async ValueTask RunAsync( - ScanSource source, ScanSpec spec, ScanMetrics metrics, AggregationPlan plan, CancellationToken cancellationToken, KeyTop? top = null, bool parted = false, + ScanSource source, ScanSpec spec, ScanCounters metrics, AggregationPlan plan, CancellationToken cancellationToken, KeyTop? top = null, bool parted = false, PartBuilder? builder = null) { IAggregateNode[] aggregates = plan.Aggregates; @@ -3306,7 +3306,7 @@ internal static async ValueTask RunAsync( // shares the live blocks, not the file's: a filter that keeps a few contiguous blocks keeps // every lane busy, rather than the one whose rows hold them. The settling read them already. int degree = Degree(source, pass); - if (settling is null && degree > 1 && pass.Take is null && pass.Filter is { } kept && pass.Options.Pruning && source is FileScanSource file) + if (settling is null && degree > 1 && pass.Take is null && pass.Filter is { } kept && pass.Options.UseStatistics && source is FileScanSource file) { BlockMask? live = await ZonePruningPlan .RefineAsync(file.File, file.File.LayoutTree, FunctionFieldExpr.Ranges(kept), cancellationToken, steps: null, metrics, pass.Options.UseIndexes) @@ -4698,7 +4698,7 @@ private static long Live(BlockMask? live, RowRange rows) return parts.Count > 1 ? [.. parts] : null; } - internal static async Task RunPartitionAsync(ScanSource source, ScanSpec spec, ScanMetrics metrics, AggregationPartition partition, CancellationToken cancellationToken) + internal static async Task RunPartitionAsync(ScanSource source, ScanSpec spec, ScanCounters metrics, AggregationPartition partition, CancellationToken cancellationToken) { long start = Stopwatch.GetTimestamp(); await foreach (RecordBatch batch in source.BatchesAsync(spec, metrics).WithCancellation(cancellationToken).ConfigureAwait(false)) @@ -4756,7 +4756,7 @@ private static AggregationRun Gathered(AggregationPartition[] partitions, long m /// all on its first rows, which a key whose cardinality changes as the rows go misleads together. /// private static async Task RunQueueAsync( - ScanSource source, ScanSpec spec, ScanMetrics metrics, AggregationPartition[] partitions, RowRange[] ranges, ZoneSettling? settling, + ScanSource source, ScanSpec spec, ScanCounters metrics, AggregationPartition[] partitions, RowRange[] ranges, ZoneSettling? settling, bool fan, int ahead, CancellationToken cancellationToken) { using CancellationTokenSource failed = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); diff --git a/src/Vorticity/Aggregation/Aggregators.cs b/src/Vorticity/Aggregation/Aggregators.cs index 5edf4542..62b89c27 100644 --- a/src/Vorticity/Aggregation/Aggregators.cs +++ b/src/Vorticity/Aggregation/Aggregators.cs @@ -18,7 +18,7 @@ internal static ColumnShape Input(RecordBinding? binding, Func(binding)).Column; @@ -73,7 +73,7 @@ internal static Predicate Rows(RecordBinding? binding, Func(binding)); @@ -727,7 +727,7 @@ private StatisticsView(VortexFile file) /// The view of for , or null when the statistics do not describe the scan's rows. internal static StatisticsView? For(ScanSource source, ScanSpec spec) => source is FileScanSource file && file.File.HasFileStatistics && spec.Filter is null && !spec.MatchesNothing - && spec.Rows is null && spec.Take is null && spec.Options.Pruning + && spec.Rows is null && spec.Take is null && spec.Options.UseStatistics ? new StatisticsView(file.File) : null; diff --git a/src/Vorticity/Aggregation/ChosenRows.cs b/src/Vorticity/Aggregation/ChosenRows.cs index f1b6ad21..7f995657 100644 --- a/src/Vorticity/Aggregation/ChosenRows.cs +++ b/src/Vorticity/Aggregation/ChosenRows.cs @@ -166,7 +166,7 @@ internal static class ChosenFetch /// The groups whose rows are read. /// Cancels the take. internal static async ValueTask FetchAsync( - AggregationOutcome outcome, ScanSource source, ScanSpec spec, ScanMetrics metrics, ReadOnlyMemory groups, CancellationToken cancellationToken) + AggregationOutcome outcome, ScanSource source, ScanSpec spec, ScanCounters metrics, ReadOnlyMemory groups, CancellationToken cancellationToken) { // What the fetch sorts and reads by, an entry a group and choice — its position, its owner, // its row and group, the take's position — reserved under the result's memory while it runs. @@ -184,7 +184,7 @@ internal static async ValueTask FetchAsync( } private static async ValueTask FetchRowsAsync( - AggregationOutcome outcome, ScanSource source, ScanSpec spec, ScanMetrics metrics, ReadOnlyMemory groups, QueryMemory? memory, CancellationToken cancellationToken) + AggregationOutcome outcome, ScanSource source, ScanSpec spec, ScanCounters metrics, ReadOnlyMemory groups, QueryMemory? memory, CancellationToken cancellationToken) { IChosenColumn[] chosen = outcome.Plan.Chosen; (IAggregateNode Row, int[] Columns)[] choices = outcome.Plan.ChosenRows; diff --git a/src/Vorticity/Aggregation/DistinctQuery.cs b/src/Vorticity/Aggregation/DistinctQuery.cs index 39a058c9..8cb40713 100644 --- a/src/Vorticity/Aggregation/DistinctQuery.cs +++ b/src/Vorticity/Aggregation/DistinctQuery.cs @@ -103,7 +103,7 @@ internal override VortexSchema Schema internal override VortexSession Session => Projection.Session; - internal override ScanMetrics Metrics => Projection.Metrics; + internal override ScanCounters Counters => Projection.Counters; internal override IAsyncEnumerator Batches(CancellationToken cancellationToken) => new DistinctBatches(this, cancellationToken); @@ -272,7 +272,7 @@ public async ValueTask MoveNextAsync() }; plan.Watch?.Invoke([_partition]); _outcome = new AggregationOutcome(plan, _partition.Slots, _partition.Keys, []); - _inner = projection.Host.Source.BatchesAsync(spec, projection.Metrics).GetAsyncEnumerator(_cancellationToken); + _inner = projection.Host.Source.BatchesAsync(spec, projection.Counters).GetAsyncEnumerator(_cancellationToken); } // Once the values asked for are met the scan is asked for nothing more. @@ -547,7 +547,7 @@ private async Task PassAsync(ProjectionQuery projection, ScanSpec spec, Aggregat Exception? failure = null; try { - AggregationOutcome outcome = await AggregationEngine.RunAsync(projection.Host.Source, spec, projection.Metrics, plan, cancellationToken).ConfigureAwait(false); + AggregationOutcome outcome = await AggregationEngine.RunAsync(projection.Host.Source, spec, projection.Counters, plan, cancellationToken).ConfigureAwait(false); try { if (outcome.Parts is CoreParts parts) diff --git a/src/Vorticity/Aggregation/ExternalSort.cs b/src/Vorticity/Aggregation/ExternalSort.cs index a3244159..4a87b7db 100644 --- a/src/Vorticity/Aggregation/ExternalSort.cs +++ b/src/Vorticity/Aggregation/ExternalSort.cs @@ -34,7 +34,7 @@ namespace Vorticity.Aggregating; /// internal sealed class ExternalSort : IAsyncDisposable { - private static readonly VortexWriteOptions RunOptions = new VortexWriteOptions { Compression = CompressionProfile.Fastest, Statistics = false }; + private static readonly VortexWriteOptions RunOptions = new VortexWriteOptions { Compression = CompressionProfile.Fastest, WriteStatistics = false }; private readonly VortexSession _session; private readonly VortexSchema _schema; @@ -562,7 +562,7 @@ private sealed class Cursor(ExternalSort sort, VortexFile file, int ordinal, Can internal async ValueTask NextBatchAsync() { _batches ??= file.ScanSource - .BatchesAsync(new ScanSpec { Options = ScanOptions.Default with { BatchRows = sort._batchRows } }, new ScanMetrics()) + .BatchesAsync(new ScanSpec { Options = ScanOptions.Default with { BatchRows = sort._batchRows } }, new ScanCounters()) .GetAsyncEnumerator(cancellationToken); while (await _batches.MoveNextAsync().ConfigureAwait(false)) { diff --git a/src/Vorticity/Aggregation/Group.cs b/src/Vorticity/Aggregation/Group.cs index 95539fa5..8ddaa727 100644 --- a/src/Vorticity/Aggregation/Group.cs +++ b/src/Vorticity/Aggregation/Group.cs @@ -255,7 +255,7 @@ private Sym Widened(Func, Sym> colu /// A probe of the row a choice keeps, of the rows the group's filter keeps. private Probe Chosen(AggregateKind kind, ColumnShape? by) => new Probe( - _binding ?? throw new InvalidOperationException("A chosen row is taken inside the lambda of Select or AggAsync, which hands it its columns."), + _binding ?? throw new InvalidOperationException("A chosen row is taken inside the lambda of Select or AggregateAsync, which hands it its columns."), null, Aggregators.Chosen(kind, by, _filter)); diff --git a/src/Vorticity/Aggregation/GroupBatches.cs b/src/Vorticity/Aggregation/GroupBatches.cs index e04c1f4c..4d892323 100644 --- a/src/Vorticity/Aggregation/GroupBatches.cs +++ b/src/Vorticity/Aggregation/GroupBatches.cs @@ -113,7 +113,7 @@ internal override VortexSchema Schema internal override VortexSession Session => Host.Source.Session; - internal override ScanMetrics Metrics => Host.Metrics; + internal override ScanCounters Counters => Host.Counters; internal override ResultQuery As(VortexSchema record, Type type) => Typed(record, type); @@ -170,7 +170,7 @@ internal IAsyncEnumerator Groups(CancellationToken cancellationToke : ZoneFinality.Candidate(this) ? new ZoneDecidedGroups(this, cancellationToken) : new GroupBatches(this, cancellationToken); - internal override GroupStatistics? Grouping => Plan.Grouped ? Plan.Statistics() : null; + internal override GroupMetrics? Grouping => Plan.Grouped ? Plan.Metrics() : null; internal override async ValueTask ExplainAsync(CancellationToken cancellationToken) { @@ -820,14 +820,14 @@ internal static int ChosenReader(GroupOperator[] operators) if (reader < operators) { - await ChosenFetch.FetchAsync(outcome, query.Host.Source, spec, query.Host.Metrics, groups.AsMemory(0, count), cancellationToken).ConfigureAwait(false); + await ChosenFetch.FetchAsync(outcome, query.Host.Source, spec, query.Host.Counters, groups.AsMemory(0, count), cancellationToken).ConfigureAwait(false); return await OperatorsAsync(query, outcome, reader, operators, groups, count, degree, cancellationToken).ConfigureAwait(false); } // No operator reads them: the result's window alone is read. int from = windowed ? (int)Math.Min(query.Skip, count) : 0; int to = windowed ? (int)Math.Min(count, Saturated(query.Skip, query.Take)) : count; - await ChosenFetch.FetchAsync(outcome, query.Host.Source, spec, query.Host.Metrics, groups.AsMemory(from, to - from), cancellationToken).ConfigureAwait(false); + await ChosenFetch.FetchAsync(outcome, query.Host.Source, spec, query.Host.Counters, groups.AsMemory(from, to - from), cancellationToken).ConfigureAwait(false); return (groups, count); } diff --git a/src/Vorticity/Aggregation/GroupPlan.cs b/src/Vorticity/Aggregation/GroupPlan.cs index 2a38aea7..50a224fa 100644 --- a/src/Vorticity/Aggregation/GroupPlan.cs +++ b/src/Vorticity/Aggregation/GroupPlan.cs @@ -35,8 +35,8 @@ public sealed record GroupPlan( /// A component of a group by's key, as the statistics describe it. /// The column it reads. /// Whether the statistics say the column is sorted, so that its rows come in runs of one value. -/// The values between its smallest and its largest, when the statistics hold them for an integer column: what a table of its groups indexes directly. -public sealed record GroupKeyPlan(string Column, bool Sorted, long? Values); +/// The values between its smallest and its largest, when the statistics hold them for an integer column: what a table of its groups indexes directly. +public sealed record GroupKeyPlan(string Column, bool Sorted, long? ValueRange); /// How the groups of a group by are ordered before they are delivered. public enum GroupOrdering @@ -74,7 +74,7 @@ public enum GroupOrdering /// Key blocks grouped by the codes of their dictionary, one lookup a distinct value. /// Key blocks grouped row by row, each value hashed. /// From the first move of the result to its first batch. -public sealed record GroupStatistics( +public sealed record GroupMetrics( long Groups, long PeakGroups, long PeakBytes, diff --git a/src/Vorticity/Aggregation/GroupedScan.cs b/src/Vorticity/Aggregation/GroupedScan.cs index d91ecc6f..afcf60a3 100644 --- a/src/Vorticity/Aggregation/GroupedScan.cs +++ b/src/Vorticity/Aggregation/GroupedScan.cs @@ -440,7 +440,7 @@ public sealed class Aggregation internal Aggregation(ResultQuery query) => _query = query; /// What the scan did, and its group by; valid once the aggregation has been enumerated. - public ScanStatistics Statistics => _query.Statistics(); + public ScanMetrics Metrics => _query.Metrics(); /// What the aggregation computes, for the tests that hold two spellings of a query to one plan. internal AggregationPlan Plan => ((AggregationQuery)_query).Plan; @@ -528,7 +528,7 @@ public sealed class Aggregation internal ResultQuery Query => _query; /// What the scan did, and its group by; valid once the result has been read. - public ScanStatistics Statistics => _query.Statistics(); + public ScanMetrics Metrics => _query.Metrics(); /// What the aggregation computes, for the tests that hold two spellings of a query to one plan. internal AggregationPlan Plan => ((AggregationQuery)_query).Plan; @@ -567,7 +567,7 @@ internal static Scan Scan(ResultQuery query) where TRecord : IVortexRecord { ResultQuery typed = query.As(TRecord.Schema, typeof(TRecord)); - return new Scan(new ResultScanSource(typed), typed.Metrics); + return new Scan(new ResultScanSource(typed), typed.Counters); } } diff --git a/src/Vorticity/Aggregation/StreamingGroups.cs b/src/Vorticity/Aggregation/StreamingGroups.cs index 59ee3459..cce41312 100644 --- a/src/Vorticity/Aggregation/StreamingGroups.cs +++ b/src/Vorticity/Aggregation/StreamingGroups.cs @@ -310,7 +310,7 @@ private async ValueTask NarrowAsync() _partition!.Settle(_settling, rest); } - _inner = _query.Host.Source.BatchesAsync(pass with { Rows = rest }, _query.Host.Metrics).GetAsyncEnumerator(_cancellationToken); + _inner = _query.Host.Source.BatchesAsync(pass with { Rows = rest }, _query.Host.Counters).GetAsyncEnumerator(_cancellationToken); } /// The end of the rows: every group closes, the null group last, and the plan keeps what the run did. @@ -392,7 +392,7 @@ private async ValueTask StartAsync() // close in: forward only, as the ranges grouped side by side are, and on a sorted key. ZoneSettling? settling = descending || _zones is not null ? null - : await ZoneSettling.PlanAsync(host.Source, pass, plan, host.Metrics, _cancellationToken).ConfigureAwait(false); + : await ZoneSettling.PlanAsync(host.Source, pass, plan, host.Counters, _cancellationToken).ConfigureAwait(false); if (settling is not null) { pass = settling.Pass(pass); @@ -414,16 +414,16 @@ private async ValueTask StartAsync() } _pass = pass; - _inner = host.Source.BatchesAsync(pass, host.Metrics).GetAsyncEnumerator(_cancellationToken); + _inner = host.Source.BatchesAsync(pass, host.Counters).GetAsyncEnumerator(_cancellationToken); return; } // On several lanes, each range of rows is grouped on its own, and the ranges follow one // another here in the order of the rows: the structures are read once for all of them. - if (settling is null && pass.Filter is { } filter && pass.Options.Pruning && host.Source is FileScanSource file) + if (settling is null && pass.Filter is { } filter && pass.Options.UseStatistics && host.Source is FileScanSource file) { BlockMask? live = await ZonePruningPlan - .RefineAsync(file.File, file.File.LayoutTree, FunctionFieldExpr.Ranges(filter), _cancellationToken, steps: null, host.Metrics, pass.Options.UseIndexes) + .RefineAsync(file.File, file.File.LayoutTree, FunctionFieldExpr.Ranges(filter), _cancellationToken, steps: null, host.Counters, pass.Options.UseIndexes) .ConfigureAwait(false); pass = pass with { Pruned = true, Live = live }; } @@ -448,7 +448,7 @@ private async ValueTask StartAsync() try { - await AggregationEngine.RunPartitionAsync(host.Source, lane with { Rows = rows }, host.Metrics, range, token).ConfigureAwait(false); + await AggregationEngine.RunPartitionAsync(host.Source, lane with { Rows = rows }, host.Counters, range, token).ConfigureAwait(false); } catch { @@ -531,7 +531,7 @@ private async ValueTask CloseAsync(bool all) int count = Through(0, reader, _closedCount); if (chosen && count > 0) { - await ChosenFetch.FetchAsync(_outcome!, _query.Host.Source, _pass!, _query.Host.Metrics, _closed.AsMemory(0, count), _cancellationToken) + await ChosenFetch.FetchAsync(_outcome!, _query.Host.Source, _pass!, _query.Host.Counters, _closed.AsMemory(0, count), _cancellationToken) .ConfigureAwait(false); } diff --git a/src/Vorticity/Aggregation/ZoneFinality.cs b/src/Vorticity/Aggregation/ZoneFinality.cs index 89633a93..c160e028 100644 --- a/src/Vorticity/Aggregation/ZoneFinality.cs +++ b/src/Vorticity/Aggregation/ZoneFinality.cs @@ -102,7 +102,7 @@ query.Host.Source.Session.Options.MemoryBudget is { } budget && query.Host.Sourc VortexFile file = ((FileScanSource)query.Host.Source).File; string path = query.Plan.Keys[0].Path; ZoneColumn? zones = (await ZonePruningPlan - .PlanAsync(file, file.LayoutTree, Expr.IsNotNull(Expr.Field(path)), cancellationToken, steps: null, query.Host.Metrics, indexes: false) + .PlanAsync(file, file.LayoutTree, Expr.IsNotNull(Expr.Field(path)), cancellationToken, steps: null, query.Host.Counters, indexes: false) .ConfigureAwait(false)).Zones?.Column(path); if (zones is not { HasStatistics: true } || zones.RowCount != file.RowCount) { diff --git a/src/Vorticity/Aggregation/ZoneSettling.cs b/src/Vorticity/Aggregation/ZoneSettling.cs index 3b1c39a5..1ccbe92c 100644 --- a/src/Vorticity/Aggregation/ZoneSettling.cs +++ b/src/Vorticity/Aggregation/ZoneSettling.cs @@ -59,7 +59,7 @@ private ZoneSettling(int[] blocks, Stored[] columns, long blockRows, BlockMask l /// The scan's sink, to which the zone maps read here are added. /// Cancels the reads. internal static async ValueTask PlanAsync( - ScanSource source, ScanSpec pass, AggregationPlan plan, ScanMetrics metrics, CancellationToken cancellationToken) + ScanSource source, ScanSpec pass, AggregationPlan plan, ScanCounters metrics, CancellationToken cancellationToken) { if (source is not FileScanSource { File: { } file } || !Applies(pass, plan) || !file.Schema.RootIsStruct) { @@ -211,7 +211,7 @@ internal RecordBatch Batch(int index, Scratch scratch) /// private static bool Applies(ScanSpec pass, AggregationPlan plan) { - if (!plan.Grouped || pass.Take is not null || pass.MatchesNothing || !pass.Options.Pruning + if (!plan.Grouped || pass.Take is not null || pass.MatchesNothing || !pass.Options.UseStatistics || plan.Filters.Filters.Length > 0 || plan.Chosen.Length > 0 || !Array.Exists(plan.Keys, key => key.Field is FunctionFieldExpr)) { diff --git a/src/Vorticity/Arrays/ArrayDecodeContext.cs b/src/Vorticity/Arrays/ArrayDecodeContext.cs index 38012a64..d2da8e47 100644 --- a/src/Vorticity/Arrays/ArrayDecodeContext.cs +++ b/src/Vorticity/Arrays/ArrayDecodeContext.cs @@ -821,7 +821,7 @@ public static int CheckedMultiply(int a, int b, string what) /// /// Counts bytes a decode of this batch produces, or stands for, - /// against . + /// against . /// /// What one decode materializes, already within its own ceiling. /// The batch would pass its ceiling. @@ -829,9 +829,9 @@ public static int CheckedMultiply(int a, int b, string what) internal void ChargeBatch(int byteLength) { long units = _scan.BatchDecoded + ((byteLength + 63L) >> 6); - if (units << 6 > Options.MaxBatchDecompressedSize) + if (units << 6 > Options.MaxBatchDecompressedBytes) { - ThrowOverBatch(Options.MaxBatchDecompressedSize); + ThrowOverBatch(Options.MaxBatchDecompressedBytes); } _scan.BatchDecoded = (int)Math.Min(units, int.MaxValue); @@ -985,7 +985,7 @@ private static void ThrowLength(ulong value, string encodingId, string what) => private static void ThrowOverBatch(long ceiling) => throw new VortexFormatException( $"The batch would materialize more than the {ceiling}-byte " + - "ceiling of VortexOpenOptions.MaxBatchDecompressedSize."); + "ceiling of VortexOpenOptions.MaxBatchDecompressedBytes."); [MethodImpl(MethodImplOptions.NoInlining)] [DoesNotReturn] diff --git a/src/Vorticity/Arrays/Decoders/Canonical/CanonicalSupport.cs b/src/Vorticity/Arrays/Decoders/Canonical/CanonicalSupport.cs index 8e3413de..612026cd 100644 --- a/src/Vorticity/Arrays/Decoders/Canonical/CanonicalSupport.cs +++ b/src/Vorticity/Arrays/Decoders/Canonical/CanonicalSupport.cs @@ -50,7 +50,7 @@ internal static VortexBuffer AllocateUninitialized( /// /// Materializes a buffer, refusing one larger than - /// . + /// . /// /// /// No allocation may be sized directly by a file-supplied value without a cap first. Several of @@ -94,9 +94,9 @@ internal static void RequireStandsForWithinBudget(ArrayDecodeContext context, in private static void RequireWithinBudget(ArrayDecodeContext context, int byteLength) { - if (byteLength > context.Options.MaxDecompressedSize) + if (byteLength > context.Options.MaxDecompressedBytes) { - ThrowOverBudget(byteLength, context.Options.MaxDecompressedSize); + ThrowOverBudget(byteLength, context.Options.MaxDecompressedBytes); } context.ChargeBatch(byteLength); @@ -514,7 +514,7 @@ private static void Gather(ref byte source, int size, out ulong first, out uint private static void ThrowOverBudget(int byteLength, long max) => throw new VortexFormatException( $"Decoding this array would materialize {byteLength} bytes in one buffer; " + - $"VortexReadOptions.MaxDecompressedSize is {max}."); + $"VortexOpenOptions.MaxDecompressedBytes is {max}."); [MethodImpl(MethodImplOptions.NoInlining)] [DoesNotReturn] diff --git a/src/Vorticity/Arrays/Decoders/Compressed/CompressedValues.cs b/src/Vorticity/Arrays/Decoders/Compressed/CompressedValues.cs index c427ff8a..6bf4e843 100644 --- a/src/Vorticity/Arrays/Decoders/Compressed/CompressedValues.cs +++ b/src/Vorticity/Arrays/Decoders/Compressed/CompressedValues.cs @@ -1037,7 +1037,7 @@ public static VortexBuffer AllocateUninitialized( { ArgumentNullException.ThrowIfNull(ctx); - long ceiling = ctx.Options.MaxDecompressedSize; + long ceiling = ctx.Options.MaxDecompressedBytes; if (byteLength > ceiling) { destination = default; @@ -1051,7 +1051,7 @@ public static VortexBuffer AllocateUninitialized( } /// /// Materializes a decoded buffer, refusing one larger than - /// . + /// . /// /// The decode context. /// Bytes to allocate; already checked for 32-bit overflow. @@ -1071,7 +1071,7 @@ public static VortexBuffer AllocateUninitialized( /// and here bounds the product. /// /// - /// The ceiling is , applied per decoded + /// The ceiling is , applied per decoded /// node's output buffer, never a constant invented at the call site. A caller with a genuinely /// larger column raises that option. /// @@ -1083,7 +1083,7 @@ public static VortexBuffer Allocate( { ArgumentNullException.ThrowIfNull(ctx); - long ceiling = ctx.Options.MaxDecompressedSize; + long ceiling = ctx.Options.MaxDecompressedBytes; if (byteLength > ceiling) { destination = default; diff --git a/src/Vorticity/Arrays/Decoders/Compressed/ZstdBuffersDecoder.cs b/src/Vorticity/Arrays/Decoders/Compressed/ZstdBuffersDecoder.cs index 6e7cea81..5ed2b8ae 100644 --- a/src/Vorticity/Arrays/Decoders/Compressed/ZstdBuffersDecoder.cs +++ b/src/Vorticity/Arrays/Decoders/Compressed/ZstdBuffersDecoder.cs @@ -109,11 +109,11 @@ public override int Decode(ArrayDecodeContext context, in ArrayNode node, DType for (int i = 0; i < buffers; i++) { long size = sizes[i]; - if (size < 0 || size > context.Options.MaxDecompressedSize) + if (size < 0 || size > context.Options.MaxDecompressedBytes) { CompressedThrow.Format( $"{Id}'s buffer {i} declares {size} bytes, above the " + - $"{context.Options.MaxDecompressedSize}-byte decompression ceiling."); + $"{context.Options.MaxDecompressedBytes}-byte decompression ceiling."); } int alignment = alignments[i] <= 0 ? 1 : alignments[i]; diff --git a/src/Vorticity/Arrays/Metadata/ZstdMetadata.cs b/src/Vorticity/Arrays/Metadata/ZstdMetadata.cs index 5255eefc..7c5ace4e 100644 --- a/src/Vorticity/Arrays/Metadata/ZstdMetadata.cs +++ b/src/Vorticity/Arrays/Metadata/ZstdMetadata.cs @@ -22,7 +22,7 @@ public ZstdFrameMetadata(ulong uncompressedSize, ulong valueCount) /// /// Uncompressed byte size of this frame (tag 1). The value comes from the file, so the decoder - /// must check it against before + /// must check it against before /// allocating. /// public ulong UncompressedSize { get; } diff --git a/src/Vorticity/Arrays/ScanContext.cs b/src/Vorticity/Arrays/ScanContext.cs index 0a8d754b..c8426b2c 100644 --- a/src/Vorticity/Arrays/ScanContext.cs +++ b/src/Vorticity/Arrays/ScanContext.cs @@ -488,7 +488,7 @@ private void WriteSelectionRange() /// scan like : set once per lane, never by , /// and the readers add to it what they materialize. /// - internal Scanning.ScanMetrics? Metrics { get; set; } + internal Scanning.ScanCounters? Metrics { get; set; } /// /// Whether a column's own node may stay a dictionary or run-end node rather than be decoded to diff --git a/src/Vorticity/Columns/Column.cs b/src/Vorticity/Columns/Column.cs index 8eb1523f..8db628bb 100644 --- a/src/Vorticity/Columns/Column.cs +++ b/src/Vorticity/Columns/Column.cs @@ -97,12 +97,12 @@ public T AsConstant() // One row of the constant, made canonical, rather than the whole column tiled to read its // first value: the value is the element, whatever the length. int one = Layouts.CanonicalSlice.SliceAcross(Arena, Arena, Node, 0, 1); - return ColumnReader.Read(new Column(Arena, one, Type, Extensions).Canonical(), 0); + return ColumnReader.Read(new Column(Arena, one, Type, Extensions).AsCanonical(), 0); } /// The column in canonical form: the values decoded, once, into contiguous memory. /// The canonical column; this one when it already is. - public Column Canonical() + public Column AsCanonical() { int canonical = EncodedForms.Canonical(Arena, Node); return canonical == Node ? this : new Column(Arena, canonical, Type, Extensions); diff --git a/src/Vorticity/Columns/ColumnExtensions.cs b/src/Vorticity/Columns/ColumnExtensions.cs index 4eda8c25..24556d96 100644 --- a/src/Vorticity/Columns/ColumnExtensions.cs +++ b/src/Vorticity/Columns/ColumnExtensions.cs @@ -103,7 +103,7 @@ public static class ColumnExtensions /// The unscaled values as stored, of the storage's width: sbyte to Int128. /// The storage's .NET type. /// The storage column. - public Column Storage() + public Column AsStorage() where TStorage : unmanaged => StorageOf(column); } @@ -122,7 +122,7 @@ public Column Storage() /// The unscaled values as stored. /// The storage's .NET type. /// The storage column. - public Column Storage() + public Column AsStorage() where TStorage : unmanaged => StorageOf(column); } @@ -138,7 +138,7 @@ public Column Storage() /// The unscaled values as stored. /// The storage's .NET type. /// The storage column. - public Column Storage() + public Column AsStorage() where TStorage : unmanaged => StorageOf(column); } @@ -164,7 +164,7 @@ public Column Storage() /// The stored values: days in an int, or milliseconds in a long. /// The storage's .NET type. /// The storage column. - public Column Storage() + public Column AsStorage() where TStorage : unmanaged => StorageOf(column); } @@ -193,7 +193,7 @@ public Column Storage() /// The stored values. /// The storage's .NET type: int for seconds and milliseconds, long otherwise. /// The storage column. - public Column Storage() + public Column AsStorage() where TStorage : unmanaged => StorageOf(column); } @@ -224,7 +224,7 @@ public Column Storage() /// The stored values, since the Unix epoch in Unit. /// The storage column. - public Column Storage() => StorageOf(column); + public Column AsStorage() => StorageOf(column); } extension(Column column) @@ -241,7 +241,7 @@ public Column Storage() /// The stored values, since the Unix epoch in Unit. /// The storage column. - public Column Storage() => StorageOf(column); + public Column AsStorage() => StorageOf(column); } extension(Column column) @@ -261,7 +261,7 @@ public Column Storage() /// The stored values, UTC since the Unix epoch in Unit. /// The storage column. - public Column Storage() => StorageOf(column); + public Column AsStorage() => StorageOf(column); } extension(Column column) @@ -316,7 +316,7 @@ public Column Storage() /// The unscaled values as stored, of the storage's width: sbyte to Int128, or 32 bytes. /// The storage's .NET type. /// The storage column. - public Column Storage() + public Column AsStorage() where TStorage : unmanaged => StorageOf(column); } @@ -374,7 +374,7 @@ public Column Storage() /// The unscaled values as stored, of the storage's width. /// The storage's .NET type. /// The storage column. - public Column Storage() + public Column AsStorage() where TStorage : unmanaged => StorageOf(column); } @@ -399,7 +399,7 @@ public Column Storage() /// The stored values: 100 ns ticks. /// The storage column. - public Column Storage() => StorageOf(column); + public Column AsStorage() => StorageOf(column); } extension(Column column) @@ -415,7 +415,7 @@ public Column Storage() /// The stored values: 100 ns ticks. /// The storage column. - public Column Storage() => StorageOf(column); + public Column AsStorage() => StorageOf(column); } extension(Column> column) @@ -489,7 +489,7 @@ public static class ExtensionColumnExtensions /// The stored values. /// The storage's .NET type. /// The storage column. - public Column Storage() + public Column AsStorage() where TStorage : unmanaged => ColumnExtensions.StorageOf(column); } } diff --git a/src/Vorticity/Compute/ZonePruningPlan.cs b/src/Vorticity/Compute/ZonePruningPlan.cs index 3a180b11..4c31284a 100644 --- a/src/Vorticity/Compute/ZonePruningPlan.cs +++ b/src/Vorticity/Compute/ZonePruningPlan.cs @@ -51,7 +51,7 @@ internal static class ZonePruningPlan /// internal static async ValueTask RefineAsync( VortexFile file, LayoutTree tree, VortexExpr filter, CancellationToken cancellationToken, - List? steps = null, Scanning.ScanMetrics? metrics = null, bool indexes = true) => + List? steps = null, Scanning.ScanCounters? metrics = null, bool indexes = true) => (await PlanAsync(file, tree, filter, cancellationToken, steps, metrics, indexes).ConfigureAwait(false)).Live; /// The mask, and the structures that refined it -- which a count asks again, per block. @@ -81,7 +81,7 @@ internal static class ZonePruningPlan /// internal static async ValueTask PlanAsync( VortexFile file, LayoutTree tree, VortexExpr filter, CancellationToken cancellationToken, - List? steps = null, Scanning.ScanMetrics? metrics = null, bool indexes = true, + List? steps = null, Scanning.ScanCounters? metrics = null, bool indexes = true, BlockMask? scope = null) { (ZonePruner? zones, int segments, long bytes) = @@ -89,7 +89,7 @@ internal static async ValueTask PlanAsync( // The price of consulting a structure belongs to the scan's cost: the zone maps read here // are bytes the scan asked its source for, and they go to the same sink the batches feed, // so that the sink and the source agree about what was requested. - Scanning.ScanMetrics.Note(metrics, segments, bytes); + Scanning.ScanCounters.Note(metrics, segments, bytes); long blockRows = Scanning.SplitPlan.NaturalBatchRows(tree); Indexes.BloomPruner? blooms = indexes && blockRows > 0 @@ -119,7 +119,7 @@ internal static async ValueTask PlanAsync( { before = Counted(live, scope); await blooms.RefineAsync(file, live, cancellationToken).ConfigureAwait(false); - Scanning.ScanMetrics.Note(metrics, blooms.Segments, blooms.Bytes); + Scanning.ScanCounters.Note(metrics, blooms.Segments, blooms.Bytes); Diagnostics.VortexEventSource.RunsRead(blooms.Segments); steps?.Add(new PruningStep("bloom filter", before - Counted(live, scope), blooms.Segments, blooms.Bytes)); } @@ -130,7 +130,7 @@ internal static async ValueTask PlanAsync( { before = Counted(live, scope); await locating.RefineAsync(file, live, cancellationToken).ConfigureAwait(false); - Scanning.ScanMetrics.Note(metrics, locating.Segments, locating.Bytes); + Scanning.ScanCounters.Note(metrics, locating.Segments, locating.Bytes); Diagnostics.VortexEventSource.RunsRead(locating.Segments); steps?.Add(new PruningStep("locating index", before - Counted(live, scope), locating.Segments, locating.Bytes)); located = locating.LocatesRows; @@ -150,7 +150,7 @@ private static int Counted(BlockMask live, BlockMask? scope) => /// group by whether its key lies scattered over its span or in the order of the rows. /// internal static async ValueTask ZonesAsync( - VortexFile file, FieldExpr field, Scanning.ScanMetrics? metrics, CancellationToken cancellationToken) + VortexFile file, FieldExpr field, Scanning.ScanCounters? metrics, CancellationToken cancellationToken) { if (!TryLocate(file.LayoutTree, field, out LayoutNode node, out ZoneMap map)) { @@ -164,7 +164,7 @@ private static int Counted(BlockMask live, BlockMask? scope) => ZoneColumn[] columns = new ZoneColumn[1]; (int segments, long bytes) = await DecodeAsync(file, [new Candidate(field, node, map)], columns, metrics, cancellationToken).ConfigureAwait(false); - Scanning.ScanMetrics.Note(metrics, segments, bytes); + Scanning.ScanCounters.Note(metrics, segments, bytes); return columns[0]; } @@ -189,7 +189,7 @@ private static int Counted(BlockMask live, BlockMask? scope) => /// consulting the zone maps cost this scan. /// private static async ValueTask<(ZonePruner? Pruner, int Segments, long Bytes)> BuildCountedAsync( - VortexFile file, LayoutTree tree, VortexExpr filter, Scanning.ScanMetrics? metrics, + VortexFile file, LayoutTree tree, VortexExpr filter, Scanning.ScanCounters? metrics, CancellationToken cancellationToken) { // The filter's own field references, whose paths are already split and encoded; a filter @@ -246,7 +246,7 @@ private static int Counted(BlockMask live, BlockMask? scope) => /// them in and keeps them on the file; returns the segments and bytes asked for. /// private static async ValueTask<(int Segments, long Bytes)> DecodeAsync( - VortexFile file, List candidates, ZoneColumn[] columns, Scanning.ScanMetrics? metrics, + VortexFile file, List candidates, ZoneColumn[] columns, Scanning.ScanCounters? metrics, CancellationToken cancellationToken) { // Sized for metadata rather than for a batch, and that choice is most of what pruning @@ -276,7 +276,7 @@ private static int Counted(BlockMask live, BlockMask? scope) => // Counted at the asking, like a batch's requests: the distinct zone-map segments and their // bytes, whatever the source then does about them. - int segments = Scanning.ScanMetrics.Unread(context.Segments, file.Segments, out long bytes); + int segments = Scanning.ScanCounters.Unread(context.Segments, file.Segments, out long bytes); await file.Segments.ReadManyAsync(context.Segments, cancellationToken).ConfigureAwait(false); diff --git a/src/Vorticity/Diagnostics/VortexDiagnostics.cs b/src/Vorticity/Diagnostics/VortexDiagnostics.cs index be1fa82d..1e375e16 100644 --- a/src/Vorticity/Diagnostics/VortexDiagnostics.cs +++ b/src/Vorticity/Diagnostics/VortexDiagnostics.cs @@ -63,7 +63,7 @@ internal static class VortexTelemetry } /// Adds what a finished scan did to the counters, and to its activity. - internal static void ScanEnded(ScanMetrics metrics, Activity? activity) + internal static void ScanEnded(ScanCounters metrics, Activity? activity) { // Each instrument on its own: a listener may enable some and not others, and an Add to an // instrument nobody listens to returns at once. diff --git a/src/Vorticity/Diagnostics/VortexEventSource.cs b/src/Vorticity/Diagnostics/VortexEventSource.cs index 0fc2701d..3d5b4668 100644 --- a/src/Vorticity/Diagnostics/VortexEventSource.cs +++ b/src/Vorticity/Diagnostics/VortexEventSource.cs @@ -6,7 +6,7 @@ namespace Vorticity.Diagnostics; /// /// The library's counters, published as EventCounters under the name Vorticity. -/// They answer in production what ScanMetrics answers for a single scan — above all, +/// They answer in production what ScanCounters answers for a single scan — above all, /// whether an index earns the bytes it costs. /// /// diff --git a/src/Vorticity/Diagnostics/VortexLimits.cs b/src/Vorticity/Diagnostics/VortexLimits.cs index 62ba269a..e76a52b8 100644 --- a/src/Vorticity/Diagnostics/VortexLimits.cs +++ b/src/Vorticity/Diagnostics/VortexLimits.cs @@ -44,7 +44,7 @@ public static class VortexLimits /// Default ceiling on the decompressed size of a single segment or buffer: a 1 KiB Zstd frame /// can claim to expand to 100 GiB. Configurable per open; this is the default. /// - public const long DefaultMaxDecompressedSize = 256L * 1024 * 1024; + public const long DefaultMaxDecompressedBytes = 256L * 1024 * 1024; /// /// Maximum depth of a FlatBuffers table traversal, independent of the semantic tree depths diff --git a/src/Vorticity/File/VortexFile.cs b/src/Vorticity/File/VortexFile.cs index cb67460d..4b3ea6e6 100644 --- a/src/Vorticity/File/VortexFile.cs +++ b/src/Vorticity/File/VortexFile.cs @@ -311,8 +311,8 @@ private static async ValueTask OpenTailAsync( // A file longer than the window is read first over what its source asks for, unless the // caller raised the window; the window holds any postscript a shorter read misses. - int window = InitialReadSize(options.InitialReadSize, fileLength); - int wanted = window < fileLength && options.InitialReadSize <= VortexOpenOptions.DefaultInitialReadSize + int window = InitialReadSize(options.InitialReadBytes, fileLength); + int wanted = window < fileLength && options.InitialReadBytes <= VortexOpenOptions.DefaultInitialReadBytes ? Math.Min(window, source.TailReadSize) : window; long tailOffset; diff --git a/src/Vorticity/File/VortexFileRepair.cs b/src/Vorticity/File/VortexFileRepair.cs index 9b2d0c2e..7d16fca0 100644 --- a/src/Vorticity/File/VortexFileRepair.cs +++ b/src/Vorticity/File/VortexFileRepair.cs @@ -78,7 +78,7 @@ public static async ValueTask RepairAsync(string path, Cance /// Cancels the scan. /// The prefix's length: the file's own when it is valid. /// No prefix of the file is a valid Vortex file. - public static async ValueTask ValidLengthAsync(string path, CancellationToken cancellationToken = default) + public static async ValueTask GetValidLengthAsync(string path, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(path); return await ValidEndAsync(path, new FileInfo(path).Length, cancellationToken).ConfigureAwait(false); diff --git a/src/Vorticity/File/VortexOpenOptions.cs b/src/Vorticity/File/VortexOpenOptions.cs index 62c7d419..ac906f59 100644 --- a/src/Vorticity/File/VortexOpenOptions.cs +++ b/src/Vorticity/File/VortexOpenOptions.cs @@ -27,24 +27,24 @@ public enum VortexTornTailPolicy : byte /// /// Every option here but exists to remove input/output or to bound work: a /// supplied drops the dtype segment from the tail read, a supplied -/// drops the length probe, and a raised can only +/// drops the length probe, and a raised can only /// reduce round trips. /// public sealed record VortexOpenOptions { - private readonly int _initialReadSize = DefaultInitialReadSize; + private readonly int _initialReadSize = DefaultInitialReadBytes; /// The file length, or -1 for unknown. private readonly long _length = -1; - private readonly long _maxDecompressedSize = VortexLimits.DefaultMaxDecompressedSize; + private readonly long _maxDecompressedSize = VortexLimits.DefaultMaxDecompressedBytes; /// The batch ceiling, or 0 for none, a value no ceiling takes: a nullable would widen the record. private readonly long _maxBatchDecompressedSize; private readonly VortexReadOptions? _read; private readonly Switches _switches; - /// unless a caller raises it. - internal const int DefaultInitialReadSize = 65_536; + /// unless a caller raises it. + internal const int DefaultInitialReadBytes = 65_536; /// The defaults. internal static VortexOpenOptions Default { get; } = new VortexOpenOptions(); @@ -57,7 +57,7 @@ public sealed record VortexOpenOptions /// 64 KiB do. /// /// The value is negative. - public int InitialReadSize + public int InitialReadBytes { get => _initialReadSize; init @@ -98,7 +98,7 @@ public bool VerifyStatistics /// The most bytes one decode may produce; a decode is a block. /// The value is not positive. - public long MaxDecompressedSize + public long MaxDecompressedBytes { get => _maxDecompressedSize; init @@ -110,13 +110,13 @@ public long MaxDecompressedSize /// The most bytes the decodes of one batch may produce together, across its columns; null, the default, sets no such ceiling. /// - /// bounds each decode, so a file of many columns, each declared + /// bounds each decode, so a file of many columns, each declared /// to decode near that ceiling, bounds a batch only at the ceiling times the columns. A service /// that reads files it does not trust sets this as well; a batch past it throws /// . /// /// The value is not positive. - public long? MaxBatchDecompressedSize + public long? MaxBatchDecompressedBytes { get => _maxBatchDecompressedSize == 0 ? null : _maxBatchDecompressedSize; init @@ -187,8 +187,8 @@ internal VortexReadOptions Read ? VortexReadOptions.Default : new VortexReadOptions { - MaxDecompressedSize = MaxDecompressedSize, - MaxBatchDecompressedSize = MaxBatchDecompressedSize ?? long.MaxValue, + MaxDecompressedBytes = MaxDecompressedBytes, + MaxBatchDecompressedBytes = MaxBatchDecompressedBytes ?? long.MaxValue, VerifyStatistics = VerifyStatistics, IndexCacheBytes = IndexCacheBytes, IndexFragments = Fragments(IndexFragments), @@ -202,8 +202,8 @@ internal VortexReadOptions Read /// Whether every value the read policy takes from these options is its default. private bool ReadsAsDefault => - MaxDecompressedSize == VortexLimits.DefaultMaxDecompressedSize - && MaxBatchDecompressedSize is null + MaxDecompressedBytes == VortexLimits.DefaultMaxDecompressedBytes + && MaxBatchDecompressedBytes is null && !VerifyStatistics && IndexCacheBytes == VortexReadOptions.DefaultIndexCacheBytes && IndexFragments.IsDefaultOrEmpty diff --git a/src/Vorticity/File/VortexReadOptions.cs b/src/Vorticity/File/VortexReadOptions.cs index 45a3520a..d0833470 100644 --- a/src/Vorticity/File/VortexReadOptions.cs +++ b/src/Vorticity/File/VortexReadOptions.cs @@ -13,7 +13,7 @@ internal sealed class VortexReadOptions /// The per-decode ceiling, kept as an : a decode produces buffers of an /// 's length at most, which a larger ceiling bounds no further. /// - private readonly int _maxDecompressedSize = (int)VortexLimits.DefaultMaxDecompressedSize; + private readonly int _maxDecompressedSize = (int)VortexLimits.DefaultMaxDecompressedBytes; private readonly long _maxBatchDecompressedSize = long.MaxValue; private readonly long _indexCacheBytes = DefaultIndexCacheBytes; @@ -25,10 +25,10 @@ internal sealed class VortexReadOptions /// /// Ceiling on the bytes one decompression step may produce. Defaults to - /// . + /// . /// /// The value is not positive. - public long MaxDecompressedSize + public long MaxDecompressedBytes { get => _maxDecompressedSize; init @@ -43,7 +43,7 @@ public long MaxDecompressedSize /// , none, by default. /// /// The value is not positive. - public long MaxBatchDecompressedSize + public long MaxBatchDecompressedBytes { get => _maxBatchDecompressedSize; init diff --git a/src/Vorticity/Indexes/BloomPruner.cs b/src/Vorticity/Indexes/BloomPruner.cs index 8d3355f1..06100272 100644 --- a/src/Vorticity/Indexes/BloomPruner.cs +++ b/src/Vorticity/Indexes/BloomPruner.cs @@ -333,7 +333,7 @@ private async ValueTask ReadAsync( VortexFile file, int origin, SegmentRequestSet requests, CancellationToken cancellationToken) { ISegmentReader reader = file.IndexSourceOf(origin); - Segments += Scanning.ScanMetrics.Unread(requests, reader, out long bytes); + Segments += Scanning.ScanCounters.Unread(requests, reader, out long bytes); Bytes += bytes; await reader.ReadManyAsync(requests, cancellationToken).ConfigureAwait(false); } diff --git a/src/Vorticity/Indexes/FenceTable.cs b/src/Vorticity/Indexes/FenceTable.cs index da529ea2..bfe56f85 100644 --- a/src/Vorticity/Indexes/FenceTable.cs +++ b/src/Vorticity/Indexes/FenceTable.cs @@ -612,7 +612,7 @@ private async ValueTask PageAsync( using (SegmentRequestSet requests = new SegmentRequestSet(1)) { int slot = requests.Add(new SegmentSpec(region.Offset, region.Length, region.AlignmentExponent, 0, 0)); - asked = Scanning.ScanMetrics.Unread(requests, source, out _) > 0; + asked = Scanning.ScanCounters.Unread(requests, source, out _) > 0; await source.ReadManyAsync(requests, cancellationToken).ConfigureAwait(false); VortexBuffer bytes = requests.GetBuffer(slot); if (!region.Holds(bytes.Span)) diff --git a/src/Vorticity/Indexes/KeyIndexPruner.cs b/src/Vorticity/Indexes/KeyIndexPruner.cs index 9ec26095..4d871963 100644 --- a/src/Vorticity/Indexes/KeyIndexPruner.cs +++ b/src/Vorticity/Indexes/KeyIndexPruner.cs @@ -298,7 +298,7 @@ private async ValueTask ClaimAsync(VortexFile file, BlockMask live, Cancellation } ISegmentReader reader = file.IndexSourceOf(origin); - Segments += Scanning.ScanMetrics.Unread(requests, reader, out long bytes); + Segments += Scanning.ScanCounters.Unread(requests, reader, out long bytes); Bytes += bytes; await reader.ReadManyAsync(requests, cancellationToken).ConfigureAwait(false); using ScanContext context = file.CreateIndexContext(origin); diff --git a/src/Vorticity/Keys/ExactCover.cs b/src/Vorticity/Keys/ExactCover.cs index fd12b94c..ecab4aec 100644 --- a/src/Vorticity/Keys/ExactCover.cs +++ b/src/Vorticity/Keys/ExactCover.cs @@ -86,7 +86,7 @@ internal static bool MayExist(VortexFile file, VortexExpr filter, bool indexes) /// The scan's sink, to which a sorted column adds what it reads; null when nobody asks. internal static async ValueTask TryCreateAsync( VortexFile file, VortexExpr filter, bool indexes, CancellationToken cancellationToken, - Compute.ZonePruner? zones = null, Scanning.ScanMetrics? metrics = null) + Compute.ZonePruner? zones = null, Scanning.ScanCounters? metrics = null) { string? path = null; if (!OneColumn(filter, ref path) || path is null) diff --git a/src/Vorticity/Keys/ExcludingKeySource.cs b/src/Vorticity/Keys/ExcludingKeySource.cs index 9eac0fec..0c520924 100644 --- a/src/Vorticity/Keys/ExcludingKeySource.cs +++ b/src/Vorticity/Keys/ExcludingKeySource.cs @@ -81,9 +81,9 @@ internal ExcludingKeySource(KeySource inner, IRowExclusion rows, ExcludedKey[] k internal override void Share(ScanSegments held, Arrays.RetainedChunks retained) => _inner.Share(held, retained); - internal override async ValueTask SeekAsync(FilterLiteral key, SeekOp op, CancellationToken cancellationToken) + internal override async ValueTask SeekAsync(FilterLiteral key, SeekMode op, CancellationToken cancellationToken) { - bool forward = op is not (SeekOp.AtOrBefore or SeekOp.Before); + bool forward = op is not (SeekMode.AtOrBefore or SeekMode.Before); if (!await _inner.SeekAsync(key, op, cancellationToken).ConfigureAwait(false) || !await KeptAsync(forward, cancellationToken).ConfigureAwait(false)) { @@ -92,7 +92,7 @@ internal override async ValueTask SeekAsync(FilterLiteral key, SeekOp op, // An exact seek landed on the key's first kept entry, or past the key when every one of // its entries is left out, in which case the key is not there. - if (op == SeekOp.Exact && KeyOrder.Total(_inner.Key, key) != 0) + if (op == SeekMode.Exact && KeyOrder.Total(_inner.Key, key) != 0) { _inner.Invalidate(); return false; @@ -113,16 +113,16 @@ internal override async ValueTask NextAsync(CancellationToken cancellation await _inner.NextAsync(cancellationToken).ConfigureAwait(false) && await KeptAsync(forward: true, cancellationToken).ConfigureAwait(false); - internal override async ValueTask PrevAsync(CancellationToken cancellationToken) => - await _inner.PrevAsync(cancellationToken).ConfigureAwait(false) + internal override async ValueTask PreviousAsync(CancellationToken cancellationToken) => + await _inner.PreviousAsync(cancellationToken).ConfigureAwait(false) && await KeptAsync(forward: false, cancellationToken).ConfigureAwait(false); internal override async ValueTask NextKeyAsync(CancellationToken cancellationToken) => await _inner.NextKeyAsync(cancellationToken).ConfigureAwait(false) && await KeptAsync(forward: true, cancellationToken).ConfigureAwait(false); - internal override async ValueTask PrevKeyAsync(CancellationToken cancellationToken) => - await _inner.PrevKeyAsync(cancellationToken).ConfigureAwait(false) + internal override async ValueTask PreviousKeyAsync(CancellationToken cancellationToken) => + await _inner.PreviousKeyAsync(cancellationToken).ConfigureAwait(false) && await KeptAsync(forward: false, cancellationToken).ConfigureAwait(false); internal override async ValueTask RankAsync(FilterLiteral key, CancellationToken cancellationToken) @@ -182,7 +182,7 @@ internal override async ValueTask SeekRankAsync(long rank, CancellationTok } } - internal override async ValueTask KeyCountAsync(CancellationToken cancellationToken) + internal override async ValueTask CountAtKeyAsync(CancellationToken cancellationToken) { FilterLiteral key = _inner.Key; return await UpperRankAsync(key, cancellationToken).ConfigureAwait(false) @@ -230,7 +230,7 @@ private async ValueTask KeptAsync(bool forward, CancellationToken cancella if (!(forward ? await _inner.NextAsync(cancellationToken).ConfigureAwait(false) - : await _inner.PrevAsync(cancellationToken).ConfigureAwait(false))) + : await _inner.PreviousAsync(cancellationToken).ConfigureAwait(false))) { return false; } diff --git a/src/Vorticity/Keys/KeyCursor.Typed.cs b/src/Vorticity/Keys/KeyCursor.Typed.cs index ab472e42..db8b8087 100644 --- a/src/Vorticity/Keys/KeyCursor.Typed.cs +++ b/src/Vorticity/Keys/KeyCursor.Typed.cs @@ -21,21 +21,21 @@ internal interface IKeyWalker : IAsyncDisposable ValueTask SeekLastAsync(CancellationToken cancellationToken = default); - ValueTask SeekAsync(FilterLiteral key, SeekOp op, CancellationToken cancellationToken = default); + ValueTask SeekAsync(FilterLiteral key, SeekMode op, CancellationToken cancellationToken = default); ValueTask SeekRankAsync(long rank, CancellationToken cancellationToken = default); ValueTask NextAsync(CancellationToken cancellationToken = default); - ValueTask PrevAsync(CancellationToken cancellationToken = default); + ValueTask PreviousAsync(CancellationToken cancellationToken = default); ValueTask NextKeyAsync(CancellationToken cancellationToken = default); - ValueTask PrevKeyAsync(CancellationToken cancellationToken = default); + ValueTask PreviousKeyAsync(CancellationToken cancellationToken = default); ValueTask RankAsync(FilterLiteral key, CancellationToken cancellationToken = default); - ValueTask KeyCountAsync(CancellationToken cancellationToken = default); + ValueTask CountAtKeyAsync(CancellationToken cancellationToken = default); } } @@ -132,25 +132,25 @@ internal KeyCursor(IKeyWalker walker, ColumnSym column) /// Positions relative to . /// The key, in the column's type. - /// Exact, at or after, after, at or before, before. + /// Exact, at or after, after, at or before, before. /// Cancels the reads. /// Whether an entry satisfies it. - public ValueTask SeekAsync(TKey key, SeekOp op, CancellationToken cancellationToken = default) + public ValueTask SeekAsync(TKey key, SeekMode mode, CancellationToken cancellationToken = default) { SymLowering.Placement at = Place(key); if (at.Exact && at.Beyond == 0) { - return MoveAsync(_walker.SeekAsync(at.Floor, op, cancellationToken)); + return MoveAsync(_walker.SeekAsync(at.Floor, mode, cancellationToken)); } // No entry holds the key: it lies between two stored keys, or beyond them all, so an // ordering seeks the neighbour on its side, and a seek that has none, an exact one // included, seeks past the largest key a column can store, which leaves the cursor // unpositioned as any seek that finds nothing does. - bool forward = op is SeekOp.AtOrAfter or SeekOp.After; - if (op == SeekOp.Exact || (at.Beyond > 0 && forward) || (at.Beyond < 0 && !forward)) + bool forward = mode is SeekMode.AtOrAfter or SeekMode.After; + if (mode == SeekMode.Exact || (at.Beyond > 0 && forward) || (at.Beyond < 0 && !forward)) { - return MoveAsync(_walker.SeekAsync(FilterLiteral.From(long.MaxValue), SeekOp.After, cancellationToken)); + return MoveAsync(_walker.SeekAsync(FilterLiteral.From(long.MaxValue), SeekMode.After, cancellationToken)); } if (at.Beyond != 0) @@ -158,7 +158,7 @@ public ValueTask SeekAsync(TKey key, SeekOp op, CancellationToken cancella return MoveAsync(at.Beyond > 0 ? _walker.SeekLastAsync(cancellationToken) : _walker.SeekFirstAsync(cancellationToken)); } - return MoveAsync(_walker.SeekAsync(at.Floor, forward ? SeekOp.After : SeekOp.AtOrBefore, cancellationToken)); + return MoveAsync(_walker.SeekAsync(at.Floor, forward ? SeekMode.After : SeekMode.AtOrBefore, cancellationToken)); } /// Positions on the entry of rank , counting from zero in key order. @@ -175,7 +175,7 @@ public ValueTask SeekAsync(TKey key, SeekOp op, CancellationToken cancella /// Moves to the previous entry. /// Cancels the reads. /// Whether there is one. - public ValueTask PrevAsync(CancellationToken cancellationToken = default) => MoveAsync(_walker.PrevAsync(cancellationToken)); + public ValueTask PreviousAsync(CancellationToken cancellationToken = default) => MoveAsync(_walker.PreviousAsync(cancellationToken)); /// Moves to the first entry of the next distinct key. /// Cancels the reads. @@ -185,7 +185,7 @@ public ValueTask SeekAsync(TKey key, SeekOp op, CancellationToken cancella /// Moves to the first entry of the previous distinct key. /// Cancels the reads. /// Whether there is one. - public ValueTask PrevKeyAsync(CancellationToken cancellationToken = default) => MoveAsync(_walker.PrevKeyAsync(cancellationToken)); + public ValueTask PreviousKeyAsync(CancellationToken cancellationToken = default) => MoveAsync(_walker.PreviousKeyAsync(cancellationToken)); /// The number of entries whose key is below . /// The key. @@ -209,7 +209,7 @@ public ValueTask RankAsync(TKey key, CancellationToken cancellationToken = /// Cancels the reads. /// The count, at least one. /// The cursor is not positioned, or it walks distinct keys without rows. - public ValueTask KeyCountAsync(CancellationToken cancellationToken = default) => Positioned().KeyCountAsync(cancellationToken); + public ValueTask CountAtKeyAsync(CancellationToken cancellationToken = default) => Positioned().CountAtKeyAsync(cancellationToken); private IKeyWalker Positioned() => _broken diff --git a/src/Vorticity/Keys/KeyCursor.cs b/src/Vorticity/Keys/KeyCursor.cs index 82549389..167c9f35 100644 --- a/src/Vorticity/Keys/KeyCursor.cs +++ b/src/Vorticity/Keys/KeyCursor.cs @@ -127,13 +127,13 @@ public long Row /// Whether an entry was found; the cursor is invalid when not. /// The key is of the wrong domain, or is null. public ValueTask SeekAsync( - FilterLiteral key, SeekOp op, CancellationToken cancellationToken = default) + FilterLiteral key, SeekMode op, CancellationToken cancellationToken = default) { RequireKey(key); ObjectDisposedException.ThrowIf(_disposed, this); Diagnostics.VortexEventSource.Seek(); ValueTask seek = _source.SeekAsync(key, op, cancellationToken); - return _distinct && op is SeekOp.AtOrBefore or SeekOp.Before + return _distinct && op is SeekMode.AtOrBefore or SeekMode.Before ? FirstOfKeyAsync(seek, cancellationToken) : seek; } @@ -144,7 +144,7 @@ public ValueTask SeekAsync( /// private async ValueTask FirstOfKeyAsync(ValueTask landing, CancellationToken cancellationToken) => await landing.ConfigureAwait(false) - && await _source.SeekAsync(_source.Key, SeekOp.AtOrAfter, cancellationToken).ConfigureAwait(false); + && await _source.SeekAsync(_source.Key, SeekMode.AtOrAfter, cancellationToken).ConfigureAwait(false); /// Positions on the smallest key's first entry. /// Cancels the read this makes. @@ -201,7 +201,7 @@ public ValueTask NextAsync(CancellationToken cancellationToken = default) /// Over sorted runs, stepping against the direction of the last step re-seeks at the current /// entry: a min-heap of run positions does not run backwards. /// - public ValueTask PrevAsync(CancellationToken cancellationToken = default) + public ValueTask PreviousAsync(CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(_disposed, this); if (!_source.IsValid) @@ -212,11 +212,11 @@ public ValueTask PrevAsync(CancellationToken cancellationToken = default) if (_distinct) { Diagnostics.VortexEventSource.Seek(); - return FirstOfKeyAsync(_source.PrevKeyAsync(cancellationToken), cancellationToken); + return FirstOfKeyAsync(_source.PreviousKeyAsync(cancellationToken), cancellationToken); } Diagnostics.VortexEventSource.Step(); - return _source.PrevAsync(cancellationToken); + return _source.PreviousAsync(cancellationToken); } /// @@ -235,11 +235,11 @@ public ValueTask NextKeyAsync(CancellationToken cancellationToken = defaul /// Steps to the last entry of the previous distinct key. /// Cancels the reads this makes. /// Whether there was a previous key. - public ValueTask PrevKeyAsync(CancellationToken cancellationToken = default) + public ValueTask PreviousKeyAsync(CancellationToken cancellationToken = default) { ObjectDisposedException.ThrowIf(_disposed, this); Diagnostics.VortexEventSource.Seek(); - return _source.IsValid ? _source.PrevKeyAsync(cancellationToken) : new ValueTask(false); + return _source.IsValid ? _source.PreviousKeyAsync(cancellationToken) : new ValueTask(false); } /// How many entries have a key below . @@ -321,11 +321,11 @@ public ValueTask SeekRankAsync(long rank, CancellationToken cancellationTo /// The count, at least one. /// The cursor's position does not move. /// The cursor is not positioned, or its source has no rows (). - public ValueTask KeyCountAsync(CancellationToken cancellationToken = default) + public ValueTask CountAtKeyAsync(CancellationToken cancellationToken = default) { RequireValid(); - RequireRows(nameof(KeyCountAsync)); - return _source.KeyCountAsync(cancellationToken); + RequireRows(nameof(CountAtKeyAsync)); + return _source.CountAtKeyAsync(cancellationToken); } /// diff --git a/src/Vorticity/Keys/KeyCursorBuilder.cs b/src/Vorticity/Keys/KeyCursorBuilder.cs index 9e87f198..2921cd56 100644 --- a/src/Vorticity/Keys/KeyCursorBuilder.cs +++ b/src/Vorticity/Keys/KeyCursorBuilder.cs @@ -219,7 +219,7 @@ private async ValueTask ExcludedAsync(KeySource source, Scanning.IRow /// internal static async ValueTask<(KeySource? Source, KeySourceKind Kind)> OpenSourceAsync( VortexFile file, string path, bool indexes, CancellationToken cancellationToken, - Compute.ZoneColumn? zones = null, Scanning.ScanMetrics? metrics = null) + Compute.ZoneColumn? zones = null, Scanning.ScanCounters? metrics = null) { bool runs = indexes && file.HasIndexDirectory; if (StatedSorted(file, path)) diff --git a/src/Vorticity/Keys/KeySource.cs b/src/Vorticity/Keys/KeySource.cs index 536be9d2..aaa1b50e 100644 --- a/src/Vorticity/Keys/KeySource.cs +++ b/src/Vorticity/Keys/KeySource.cs @@ -54,7 +54,7 @@ internal virtual void Share(Scanning.ScanSegments held, Arrays.RetainedChunks re internal abstract long Row { get; } /// Positions relative to a key, whose domain the caller has checked. - internal abstract ValueTask SeekAsync(FilterLiteral key, SeekOp op, CancellationToken cancellationToken); + internal abstract ValueTask SeekAsync(FilterLiteral key, SeekMode op, CancellationToken cancellationToken); /// Positions on the first entry. internal abstract ValueTask SeekFirstAsync(CancellationToken cancellationToken); @@ -66,13 +66,13 @@ internal virtual void Share(Scanning.ScanSegments held, Arrays.RetainedChunks re internal abstract ValueTask NextAsync(CancellationToken cancellationToken); /// Steps backward; the caller has checked the source is positioned. - internal abstract ValueTask PrevAsync(CancellationToken cancellationToken); + internal abstract ValueTask PreviousAsync(CancellationToken cancellationToken); /// The first entry of the next distinct key. internal abstract ValueTask NextKeyAsync(CancellationToken cancellationToken); /// The last entry of the previous distinct key. - internal abstract ValueTask PrevKeyAsync(CancellationToken cancellationToken); + internal abstract ValueTask PreviousKeyAsync(CancellationToken cancellationToken); /// The entries whose key is below ; the position does not move. internal abstract ValueTask RankAsync(FilterLiteral key, CancellationToken cancellationToken); @@ -91,7 +91,7 @@ internal virtual void Share(Scanning.ScanSegments held, Arrays.RetainedChunks re internal abstract ValueTask SeekRankAsync(long rank, CancellationToken cancellationToken); /// The entries sharing the current key; the position does not move. - internal abstract ValueTask KeyCountAsync(CancellationToken cancellationToken); + internal abstract ValueTask CountAtKeyAsync(CancellationToken cancellationToken); /// The runs holding an entry of ; the position is lost. internal virtual ValueTask RunsOverlappingAsync( diff --git a/src/Vorticity/Keys/MemoryKeySource.cs b/src/Vorticity/Keys/MemoryKeySource.cs index 16d1cee4..08efe8e3 100644 --- a/src/Vorticity/Keys/MemoryKeySource.cs +++ b/src/Vorticity/Keys/MemoryKeySource.cs @@ -77,9 +77,9 @@ internal static async ValueTask ReadAsync( return source; } - internal sealed override ValueTask SeekAsync(FilterLiteral key, SeekOp op, CancellationToken cancellationToken) + internal sealed override ValueTask SeekAsync(FilterLiteral key, SeekMode op, CancellationToken cancellationToken) { - if (op == SeekOp.Exact) + if (op == SeekMode.Exact) { int first = Bound(key, upper: false); _at = first < Bound(key, upper: true) ? first : -1; @@ -88,9 +88,9 @@ internal sealed override ValueTask SeekAsync(FilterLiteral key, SeekOp op, { _at = op switch { - SeekOp.AtOrAfter => Bound(key, upper: false), - SeekOp.After => Bound(key, upper: true), - SeekOp.AtOrBefore => Bound(key, upper: true) - 1, + SeekMode.AtOrAfter => Bound(key, upper: false), + SeekMode.After => Bound(key, upper: true), + SeekMode.AtOrBefore => Bound(key, upper: true) - 1, _ => Bound(key, upper: false) - 1, }; } @@ -116,7 +116,7 @@ internal sealed override ValueTask NextAsync(CancellationToken cancellatio return PositionedAsync(); } - internal sealed override ValueTask PrevAsync(CancellationToken cancellationToken) + internal sealed override ValueTask PreviousAsync(CancellationToken cancellationToken) { _at--; return PositionedAsync(); @@ -128,7 +128,7 @@ internal sealed override ValueTask NextKeyAsync(CancellationToken cancella return PositionedAsync(); } - internal sealed override ValueTask PrevKeyAsync(CancellationToken cancellationToken) + internal sealed override ValueTask PreviousKeyAsync(CancellationToken cancellationToken) { _at = KeyGallop.Past(new Run(this, _at), _at, 0, forward: false); return PositionedAsync(); @@ -149,7 +149,7 @@ internal sealed override ValueTask SeekRankAsync(long rank, CancellationTo return PositionedAsync(); } - internal sealed override ValueTask KeyCountAsync(CancellationToken cancellationToken) + internal sealed override ValueTask CountAtKeyAsync(CancellationToken cancellationToken) { int end = KeyGallop.Past(new Run(this, _at), _at, Count - 1, forward: true); int before = KeyGallop.Past(new Run(this, _at), _at, 0, forward: false); diff --git a/src/Vorticity/Keys/SeekOp.cs b/src/Vorticity/Keys/SeekMode.cs similarity index 97% rename from src/Vorticity/Keys/SeekOp.cs rename to src/Vorticity/Keys/SeekMode.cs index 70b5024f..4bc772e4 100644 --- a/src/Vorticity/Keys/SeekOp.cs +++ b/src/Vorticity/Keys/SeekMode.cs @@ -7,7 +7,7 @@ namespace Vorticity; /// highest. A range [a, b) is on a and then NextAsync /// while the key stays below b. /// -public enum SeekOp : byte +public enum SeekMode : byte { /// The first entry whose key equals the sought one; invalid when the key is absent. Exact, diff --git a/src/Vorticity/Keys/SortedColumnSource.cs b/src/Vorticity/Keys/SortedColumnSource.cs index d2706c3f..bb038bd7 100644 --- a/src/Vorticity/Keys/SortedColumnSource.cs +++ b/src/Vorticity/Keys/SortedColumnSource.cs @@ -76,7 +76,7 @@ private SortedColumnSource( /// The scan's sink, to which the zone map and the zones this source decodes are added; null when nobody asks. /// The source, or null with the reason it is not available. internal static async ValueTask<(SortedColumnSource? Source, string? Reason)> OpenAsync( - VortexFile file, string path, CancellationToken cancellationToken, ZoneColumn? known = null, ScanMetrics? metrics = null) + VortexFile file, string path, CancellationToken cancellationToken, ZoneColumn? known = null, ScanCounters? metrics = null) { DType schema = file.DType; if (schema.IsDefault || schema.Kind != DTypeKind.Struct) @@ -165,7 +165,7 @@ private SortedColumnSource( } /// The scan's sink, to which the zones this source decodes are added; null when nobody asks. - private ScanMetrics? Metrics { get; init; } + private ScanCounters? Metrics { get; init; } /// /// Makes the source read through what a scan holds, rather than through segments and chunks of @@ -517,7 +517,7 @@ private async ValueTask EnsureLoadedAsync(long row, CancellationToken cancellati held.Claim(_context.Segments, ticket, waiter: null); try { - if (ScanMetrics.Note(Metrics, _context.Segments)) + if (ScanCounters.Note(Metrics, _context.Segments)) { await _file.Segments.ReadManyAsync(_context.Segments, cancellationToken).ConfigureAwait(false); } @@ -534,7 +534,7 @@ private async ValueTask EnsureLoadedAsync(long row, CancellationToken cancellati throw; } - ScanMetrics.Served(Metrics, _context.Segments); + ScanCounters.Served(Metrics, _context.Segments); int root = SplitExecution.Execute(_context, _tree, in _mask, range, take: null); if (_ownsHeld) { diff --git a/src/Vorticity/Keys/SortedColumnWalker.cs b/src/Vorticity/Keys/SortedColumnWalker.cs index f2363dd7..25929afb 100644 --- a/src/Vorticity/Keys/SortedColumnWalker.cs +++ b/src/Vorticity/Keys/SortedColumnWalker.cs @@ -38,21 +38,21 @@ internal sealed class SortedColumnWalker : KeySource internal override void Share(Scanning.ScanSegments held, Arrays.RetainedChunks retained) => _source.Share(held, retained); internal override async ValueTask SeekAsync( - FilterLiteral key, SeekOp op, CancellationToken cancellationToken) + FilterLiteral key, SeekMode op, CancellationToken cancellationToken) { long at; switch (op) { - case SeekOp.AtOrAfter: + case SeekMode.AtOrAfter: at = await _source.LowerBoundAsync(key, cancellationToken).ConfigureAwait(false); break; - case SeekOp.After: + case SeekMode.After: at = await _source.UpperBoundAsync(key, cancellationToken).ConfigureAwait(false); break; - case SeekOp.AtOrBefore: + case SeekMode.AtOrBefore: at = await _source.UpperBoundAsync(key, cancellationToken).ConfigureAwait(false) - 1; break; - case SeekOp.Before: + case SeekMode.Before: at = await _source.LowerBoundAsync(key, cancellationToken).ConfigureAwait(false) - 1; break; default: @@ -87,13 +87,13 @@ internal override ValueTask SeekLastAsync(CancellationToken cancellationTo internal override ValueTask NextAsync(CancellationToken cancellationToken) => PositionAsync(_entry + 1, cancellationToken); - internal override ValueTask PrevAsync(CancellationToken cancellationToken) => + internal override ValueTask PreviousAsync(CancellationToken cancellationToken) => PositionAsync(_entry - 1, cancellationToken); internal override ValueTask NextKeyAsync(CancellationToken cancellationToken) => PastKeyAsync(forward: true, cancellationToken); - internal override ValueTask PrevKeyAsync(CancellationToken cancellationToken) => + internal override ValueTask PreviousKeyAsync(CancellationToken cancellationToken) => PastKeyAsync(forward: false, cancellationToken); /// @@ -154,7 +154,7 @@ private async ValueTask KeepingPositionAsync(ValueTask bound, Cancel internal override ValueTask SeekRankAsync(long rank, CancellationToken cancellationToken) => PositionAsync(rank, cancellationToken); - internal override async ValueTask KeyCountAsync(CancellationToken cancellationToken) + internal override async ValueTask CountAtKeyAsync(CancellationToken cancellationToken) { FilterLiteral key = Key; long low = await _source.LowerBoundAsync(key, cancellationToken).ConfigureAwait(false); diff --git a/src/Vorticity/Keys/SortedRunsSource.Dictionary.cs b/src/Vorticity/Keys/SortedRunsSource.Dictionary.cs index 97b9a953..22a37495 100644 --- a/src/Vorticity/Keys/SortedRunsSource.Dictionary.cs +++ b/src/Vorticity/Keys/SortedRunsSource.Dictionary.cs @@ -69,7 +69,7 @@ internal sealed partial class SortedRunsSource context.ResetBatch(); int slot = context.Segments.Add(SpecOf(file, flat)); - if (Scanning.ScanMetrics.Unread(context.Segments, file.Segments, out long bytes) > 0) + if (Scanning.ScanCounters.Unread(context.Segments, file.Segments, out long bytes) > 0) { asked++; read += bytes; diff --git a/src/Vorticity/Keys/SortedRunsSource.cs b/src/Vorticity/Keys/SortedRunsSource.cs index 1f20f87c..975938ff 100644 --- a/src/Vorticity/Keys/SortedRunsSource.cs +++ b/src/Vorticity/Keys/SortedRunsSource.cs @@ -384,17 +384,17 @@ private static (SortedRunsSource? Source, string? Reason) TryOpen( // ------------------------------------------------------------------------------ positioning internal override async ValueTask SeekAsync( - FilterLiteral key, SeekOp op, CancellationToken cancellationToken) + FilterLiteral key, SeekMode op, CancellationToken cancellationToken) { switch (op) { - case SeekOp.After: + case SeekMode.After: return await ForwardAsync(key, long.MaxValue, cancellationToken).ConfigureAwait(false); - case SeekOp.AtOrBefore: + case SeekMode.AtOrBefore: return await BackwardAsync(key, long.MaxValue, cancellationToken).ConfigureAwait(false); - case SeekOp.Before: + case SeekMode.Before: return await BackwardAsync(key, long.MinValue, cancellationToken).ConfigureAwait(false); - case SeekOp.AtOrAfter: + case SeekMode.AtOrAfter: return await ForwardAsync(key, long.MinValue, cancellationToken).ConfigureAwait(false); default: if (!await ForwardAsync(key, long.MinValue, cancellationToken).ConfigureAwait(false)) @@ -444,13 +444,13 @@ internal override async ValueTask SeekLastAsync(CancellationToken cancella internal override ValueTask NextAsync(CancellationToken cancellationToken) => _direction > 0 ? StepAsync(cancellationToken) : FlipAsync(forward: true, cancellationToken); - internal override ValueTask PrevAsync(CancellationToken cancellationToken) => + internal override ValueTask PreviousAsync(CancellationToken cancellationToken) => _direction < 0 ? StepAsync(cancellationToken) : FlipAsync(forward: false, cancellationToken); internal override ValueTask NextKeyAsync(CancellationToken cancellationToken) => _direction > 0 ? PastKeyAsync(forward: true, cancellationToken) : ForwardAsync(Key, long.MaxValue, cancellationToken); - internal override ValueTask PrevKeyAsync(CancellationToken cancellationToken) => + internal override ValueTask PreviousKeyAsync(CancellationToken cancellationToken) => _direction < 0 ? PastKeyAsync(forward: false, cancellationToken) : BackwardAsync(Key, long.MinValue, cancellationToken); /// @@ -767,7 +767,7 @@ private long CutInSegment(Run run, ReadOnlySpan key, long row) return run.CurrentStart + first; } - internal override async ValueTask KeyCountAsync(CancellationToken cancellationToken) + internal override async ValueTask CountAtKeyAsync(CancellationToken cancellationToken) { FilterLiteral key = Key; long count = 0; diff --git a/src/Vorticity/Scan/BatchAsyncEnumerable.cs b/src/Vorticity/Scan/BatchAsyncEnumerable.cs index 66ead49b..21e8e5c6 100644 --- a/src/Vorticity/Scan/BatchAsyncEnumerable.cs +++ b/src/Vorticity/Scan/BatchAsyncEnumerable.cs @@ -44,7 +44,7 @@ internal sealed class BatchAsyncEnumerable : IAsyncEnumerable private readonly bool _reverse; private readonly VortexExpr? _filter; private readonly RowSelection? _take; - private readonly ScanMetrics? _metrics; + private readonly ScanCounters? _metrics; private readonly DType _schema; /// The open file. @@ -69,7 +69,7 @@ internal BatchAsyncEnumerable( int degree, VortexExpr? filter, RowSelection? take, - ScanMetrics? metrics, + ScanCounters? metrics, bool reverse = false) { if (reverse && degree != 1) @@ -223,7 +223,7 @@ internal IAsyncEnumerator GetAsyncEnumerator( internal LayoutTree Tree => _tree; /// The caller's metrics sink, for the pruning pass to add its own reads to. - internal ScanMetrics? Metrics => _metrics; + internal ScanCounters? Metrics => _metrics; /// The split plan, which a key-ordered scan asks for the split of a row. internal SplitPlan Plan => _plan; @@ -273,7 +273,7 @@ internal sealed class BatchAsyncEnumerator : IAsyncEnumerator private readonly int _widenRows; private readonly RowSelection? _take; private readonly BlockMask? _live; - private readonly ScanMetrics? _metrics; + private readonly ScanCounters? _metrics; private readonly CancellationToken _token; private readonly Lane?[] _lanes; private readonly ScanSegments _segments; @@ -328,7 +328,7 @@ internal BatchAsyncEnumerator( VortexExpr? filter, RowSelection? take, BlockMask? live, - ScanMetrics? metrics, + ScanCounters? metrics, CancellationToken cancellationToken, bool filterProven = false, bool reverse = false, @@ -391,7 +391,7 @@ filter is not null && zones is not null } /// A lane on a context of its own, bound to what outlives every batch of the scan. - private Lane NewLane(VortexFile file, BlockMask? live, ScanMetrics? metrics, bool keepEncodings) + private Lane NewLane(VortexFile file, BlockMask? live, ScanCounters? metrics, bool keepEncodings) { // The mask, the metrics sink and the encoded delivery outlive every batch of the scan, so // they are set once here and never by `ResetBatch`; the readers read the mask in file @@ -552,7 +552,7 @@ private async ValueTask ReadAndCompleteAsync(Lane lane, bool read) throw; } - ScanMetrics.Served(_metrics, lane.Context.Segments); + ScanCounters.Served(_metrics, lane.Context.Segments); _segments.Release(lane.Sequence); bool produced = CompleteBatch(); @@ -586,7 +586,7 @@ private async ValueTask MoveNextAheadAsync(AheadReads ahead) await read.ConfigureAwait(false); _currentLane = lane; - ScanMetrics.Served(_metrics, lane.Context.Segments); + ScanCounters.Served(_metrics, lane.Context.Segments); _segments.Release(lane.Sequence); // The place this split leaves goes to the next one, whose read runs while this one decodes @@ -934,7 +934,7 @@ private void Register(ScanContext context, RowRange rows) /// cache may serve some without a read. /// /// Whether there is anything to ask. - private bool NoteRequests(ScanContext context) => ScanMetrics.Note(_metrics, context.Segments); + private bool NoteRequests(ScanContext context) => ScanCounters.Note(_metrics, context.Segments); /// Reads what the scan does not hold, or completes the set when it holds everything. private ValueTask ReadAsync(SegmentRequestSet segments, bool read) @@ -1711,7 +1711,7 @@ private async ValueTask RunLaneAsync(Lane lane) _segments.Publish(context.Segments, batch); } - ScanMetrics.Served(_metrics, context.Segments); + ScanCounters.Served(_metrics, context.Segments); if (!_compact && !_filterProven) { return ExecuteSelected(lane, rows); @@ -1826,8 +1826,8 @@ private void NoteDecoded(RecordBatch batch, Lane lane) } if (_sinkDecodes - && !ScanMetrics.Decoded(batch.Arena, batch.RootIndex) - && !ScanMetrics.Decoded(batch.Arena, lane.ReadRoot)) + && !ScanCounters.Decoded(batch.Arena, batch.RootIndex) + && !ScanCounters.Decoded(batch.Arena, lane.ReadRoot)) { return; } diff --git a/src/Vorticity/Scan/KeyOrderedBatches.cs b/src/Vorticity/Scan/KeyOrderedBatches.cs index c0d9422f..3ad88460 100644 --- a/src/Vorticity/Scan/KeyOrderedBatches.cs +++ b/src/Vorticity/Scan/KeyOrderedBatches.cs @@ -389,7 +389,7 @@ private async ValueTask CollectAsync() } _positioned = descending - ? await source.PrevAsync(_token).ConfigureAwait(false) + ? await source.PreviousAsync(_token).ConfigureAwait(false) : await source.NextAsync(_token).ConfigureAwait(false); if (!_positioned) { @@ -581,7 +581,7 @@ private async ValueTask DecodeGroupAsync(ScanContext context, GroupWaiter? await waiter!.ReadByOthersAsync().ConfigureAwait(false); } - if (ScanMetrics.Note(_scan.Metrics, context.Segments)) + if (ScanCounters.Note(_scan.Metrics, context.Segments)) { await _scan.File.Segments.ReadManyAsync(context.Segments, _token).ConfigureAwait(false); } @@ -598,7 +598,7 @@ private async ValueTask DecodeGroupAsync(ScanContext context, GroupWaiter? throw; } - ScanMetrics.Served(_scan.Metrics, context.Segments); + ScanCounters.Served(_scan.Metrics, context.Segments); _token.ThrowIfCancellationRequested(); int[] parts = ArrayPool.Shared.Rent(to - from); try diff --git a/src/Vorticity/Scan/ProjectionQuery.cs b/src/Vorticity/Scan/ProjectionQuery.cs index c2cc0035..81377c89 100644 --- a/src/Vorticity/Scan/ProjectionQuery.cs +++ b/src/Vorticity/Scan/ProjectionQuery.cs @@ -179,7 +179,7 @@ internal override VortexSchema Schema internal override VortexSession Session => Host.Source.Session; - internal override ScanMetrics Metrics => Host.Metrics; + internal override ScanCounters Counters => Host.Counters; internal override IAsyncEnumerator Batches(CancellationToken cancellationToken) => new ProjectedBatches(this, cancellationToken); @@ -294,7 +294,7 @@ public async ValueTask MoveNextAsync() { _query.Host.Begin(); _begun = true; - _inner = _query.Host.Source.BatchesAsync(_spec, _query.Metrics).GetAsyncEnumerator(_cancellationToken); + _inner = _query.Host.Source.BatchesAsync(_spec, _query.Counters).GetAsyncEnumerator(_cancellationToken); } // Once the window is served the scan is asked for nothing more: a Take stops the reads. diff --git a/src/Vorticity/Scan/Projections.cs b/src/Vorticity/Scan/Projections.cs index 754f61c2..5afe6bf1 100644 --- a/src/Vorticity/Scan/Projections.cs +++ b/src/Vorticity/Scan/Projections.cs @@ -24,7 +24,7 @@ public sealed class Projection internal Projection(ProjectionQuery query) => _query = query; /// What the scan did; valid once the projection has been enumerated. - public ScanStatistics Statistics => ScanStatistics.From(_query.Metrics); + public ScanMetrics Metrics => ScanMetrics.From(_query.Counters); /// Makes cancel the enumeration, as WithCancellation does a stream's. /// Cancels the scan at a batch boundary. @@ -105,7 +105,7 @@ public sealed class Projection internal Projection(ProjectionQuery query) => _query = query; /// What the scan did; valid once the projection has been read. - public ScanStatistics Statistics => ScanStatistics.From(_query.Metrics); + public ScanMetrics Metrics => ScanMetrics.From(_query.Counters); /// The rows past the first the scan keeps. /// The rows to pass over. diff --git a/src/Vorticity/Scan/ResultScanSource.cs b/src/Vorticity/Scan/ResultScanSource.cs index f7c55edf..2aa62e9c 100644 --- a/src/Vorticity/Scan/ResultScanSource.cs +++ b/src/Vorticity/Scan/ResultScanSource.cs @@ -28,7 +28,7 @@ internal abstract class ResultQuery internal abstract VortexSession Session { get; } /// What the query's own scan did, which a result's scan reports as its own. - internal abstract ScanMetrics Metrics { get; } + internal abstract ScanCounters Counters { get; } /// The result's batches; the query runs as they are asked for. internal abstract IAsyncEnumerator Batches(CancellationToken cancellationToken); @@ -49,10 +49,10 @@ internal abstract class ResultQuery internal virtual IVortexRecord? RecordOf(int column) => null; /// What the query's group by did, once it ran; null without a group by. - internal virtual GroupStatistics? Grouping => null; + internal virtual GroupMetrics? Grouping => null; - /// The scan's statistics, with what its group by did. - internal ScanStatistics Statistics() => ScanStatistics.From(Metrics) with { Grouping = Grouping }; + /// The scan's metrics, with what its group by did. + internal ScanMetrics Metrics() => ScanMetrics.From(Counters) with { Grouping = Grouping }; /// The window [, + ) of a window [, + ): the operators in the order written. internal static (long Skip, long Take) Within(long skipped, long taken, long skip, long take) @@ -80,9 +80,9 @@ internal sealed class ResultScanSource : ScanSource internal override VortexSession Session => _query.Session; - internal override IAsyncEnumerable BatchesAsync(ScanSpec spec, ScanMetrics metrics) => new Stream(this, spec); + internal override IAsyncEnumerable BatchesAsync(ScanSpec spec, ScanCounters metrics) => new Stream(this, spec); - internal override async ValueTask CountAsync(ScanSpec spec, ScanMetrics metrics, CancellationToken cancellationToken) + internal override async ValueTask CountAsync(ScanSpec spec, ScanCounters metrics, CancellationToken cancellationToken) { long count = 0; await foreach (RecordBatch batch in SelectedAsync(spec).WithCancellation(cancellationToken).ConfigureAwait(false)) @@ -93,7 +93,7 @@ internal override async ValueTask CountAsync(ScanSpec spec, ScanMetrics me return count; } - internal override async ValueTask AnyAsync(ScanSpec spec, ScanMetrics metrics, CancellationToken cancellationToken) + internal override async ValueTask AnyAsync(ScanSpec spec, ScanCounters metrics, CancellationToken cancellationToken) { await foreach (RecordBatch batch in SelectedAsync(spec).WithCancellation(cancellationToken).ConfigureAwait(false)) { @@ -106,7 +106,7 @@ internal override async ValueTask AnyAsync(ScanSpec spec, ScanMetrics metr return false; } - internal override async ValueTask ExtremeAsync(ScanSpec spec, FieldExpr column, bool min, ScanMetrics metrics, CancellationToken cancellationToken) + internal override async ValueTask ExtremeAsync(ScanSpec spec, FieldExpr column, bool min, ScanCounters metrics, CancellationToken cancellationToken) { FilterLiteral best = FilterLiteral.Null; int[] rows = []; diff --git a/src/Vorticity/Scan/Scan.Agg.cs b/src/Vorticity/Scan/Scan.Agg.cs index 0b52cc8c..e19c2833 100644 --- a/src/Vorticity/Scan/Scan.Agg.cs +++ b/src/Vorticity/Scan/Scan.Agg.cs @@ -9,12 +9,12 @@ namespace Vorticity; public sealed partial class Scan { - /// One answer over the rows the scan keeps: await scan.AggAsync(a => a.Count()). + /// One answer over the rows the scan keeps: await scan.AggregateAsync(a => a.Count()). /// The answer's type. /// A lambda over the scan's aggregates. /// Cancels the reads. /// The answer. - public async ValueTask AggAsync(Func, Sym> aggregate, CancellationToken cancellationToken = default) + public async ValueTask AggregateAsync(Func, Sym> aggregate, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(aggregate); ResultNode node = AggregationPlan.Result(aggregate(new Aggregates(Binding))); @@ -31,7 +31,7 @@ public async ValueTask AggAsync(Func, Sym> aggregat /// /// Several answers over the rows the scan keeps, computed in one pass, into a record whose - /// members take them in order: await scan.AggAsync<Summary>(a => (a.Min(x => x.Celsius), a.Count())). + /// members take them in order: await scan.AggregateAsync<Summary>(a => (a.Min(x => x.Celsius), a.Count())). /// /// The record, whose members, in declaration order, are of the answers' types or their nullable forms. /// A lambda over the scan's aggregates, returning a tuple of any length of them. @@ -39,7 +39,7 @@ public async ValueTask AggAsync(Func, Sym> aggregat /// The answers. /// An element of the tuple is not a symbol. /// The record has another number of members, or a member does not take its answer. - public async ValueTask AggAsync(Func, ITuple> aggregates, CancellationToken cancellationToken = default) + public async ValueTask AggregateAsync(Func, ITuple> aggregates, CancellationToken cancellationToken = default) where TResult : IVortexRecord { ArgumentNullException.ThrowIfNull(aggregates); diff --git a/src/Vorticity/Scan/Scan.Aggregates.cs b/src/Vorticity/Scan/Scan.Aggregates.cs index 20315a38..4da4ad43 100644 --- a/src/Vorticity/Scan/Scan.Aggregates.cs +++ b/src/Vorticity/Scan/Scan.Aggregates.cs @@ -170,9 +170,9 @@ private sealed class ScanHost : AggregationHost internal override ScanSource Source => _scan.Source; - internal override ScanMetrics Metrics => _scan.Metrics; + internal override ScanCounters Counters => _scan.Counters; - internal override ScanStatistics Statistics => _scan.Statistics; + internal override ScanMetrics Metrics => _scan.Metrics; internal override ScanSpec Spec() => _scan.Spec(); diff --git a/src/Vorticity/Scan/Scan.Tool.cs b/src/Vorticity/Scan/Scan.Tool.cs index 4af3f37d..2184733b 100644 --- a/src/Vorticity/Scan/Scan.Tool.cs +++ b/src/Vorticity/Scan/Scan.Tool.cs @@ -21,7 +21,7 @@ public sealed class Scan { private readonly ScanSource _source; private readonly FieldMask? _projection; - private readonly ScanMetrics _metrics = new ScanMetrics(); + private readonly ScanCounters _counters = new ScanCounters(); private VortexExpr? _filter; private RowRange? _rows; private long[]? _take; @@ -131,7 +131,7 @@ public Scan WithCancellation(CancellationToken cancellationToken) } /// What the sink did; valid once it has run. - public ScanStatistics Statistics => ScanStatistics.From(_metrics); + public ScanMetrics Metrics => ScanMetrics.From(_counters); /// Enumerates the batches as borrowed views. /// Cancels at a batch boundary. @@ -140,7 +140,7 @@ public AsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = { Begin(); CancellationToken token = cancellationToken.CanBeCanceled ? cancellationToken : _cancellation; - return new AsyncEnumerator(this, _source.BatchesAsync(Spec(), _metrics).GetAsyncEnumerator(token), Schema, _source.Session); + return new AsyncEnumerator(this, _source.BatchesAsync(Spec(), _counters).GetAsyncEnumerator(token), Schema, _source.Session); } /// The batches, each owned by the caller, who disposes it. @@ -151,7 +151,7 @@ public async IAsyncEnumerable ToBatchesAsync([EnumeratorCancellatio Begin(); try { - await foreach (RecordBatch batch in _source.BatchesAsync(Spec(), _metrics).WithCancellation(cancellationToken).ConfigureAwait(false)) + await foreach (RecordBatch batch in _source.BatchesAsync(Spec(), _counters).WithCancellation(cancellationToken).ConfigureAwait(false)) { yield return RecordBatch.Own(batch.Arena, batch.RootIndex, batch.StartRow, Schema, _source.Session, batch.SelectionWords, batch.SelectedRows); } @@ -170,7 +170,7 @@ public async ValueTask CountAsync(CancellationToken cancellationToken = de Begin(); try { - return await _source.CountAsync(Spec(), _metrics, cancellationToken).ConfigureAwait(false); + return await _source.CountAsync(Spec(), _counters, cancellationToken).ConfigureAwait(false); } finally { @@ -186,7 +186,7 @@ public async ValueTask AnyAsync(CancellationToken cancellationToken = defa Begin(); try { - return await _source.AnyAsync(Spec(), _metrics, cancellationToken).ConfigureAwait(false); + return await _source.AnyAsync(Spec(), _counters, cancellationToken).ConfigureAwait(false); } finally { @@ -222,7 +222,7 @@ private void End() { if (Interlocked.Exchange(ref _ended, 1) == 0) { - VortexTelemetry.ScanEnded(_metrics, _activity); + VortexTelemetry.ScanEnded(_counters, _activity); _activity = null; } } diff --git a/src/Vorticity/Scan/Scan.Typed.cs b/src/Vorticity/Scan/Scan.Typed.cs index b8de4936..a2dcd94f 100644 --- a/src/Vorticity/Scan/Scan.Typed.cs +++ b/src/Vorticity/Scan/Scan.Typed.cs @@ -23,7 +23,7 @@ public sealed partial class Scan where TRecord : IVortexRecord { private readonly ScanSource _source; - private readonly ScanMetrics _metrics; + private readonly ScanCounters _counters; private RecordBinding? _binding; private Predicate _filter = Predicate.All; private RowRange? _rows; @@ -36,15 +36,15 @@ public sealed partial class Scan private int _used; internal Scan(ScanSource source) - : this(source, new ScanMetrics()) + : this(source, new ScanCounters()) { } /// A scan that reports as its own: a result's, whose reads are its query's. - internal Scan(ScanSource source, ScanMetrics metrics) + internal Scan(ScanSource source, ScanCounters metrics) { _source = source; - _metrics = metrics; + _counters = metrics; } internal RecordBinding Binding => _binding ??= RecordBinding.For(_source.Schema, _source.Session.Options.Extensions); @@ -161,7 +161,7 @@ public Scan WithCancellation(CancellationToken cancellationToken) } /// What the last sink did; valid once it has run. - public ScanStatistics Statistics => ScanStatistics.From(_metrics); + public ScanMetrics Metrics => ScanMetrics.From(_counters); /// Enumerates the batches as borrowed columns: await foreach (var (day, celsius, city) in scan). /// Cancels the scan at a batch boundary. @@ -174,7 +174,7 @@ public AsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = // The one sink whose consumer can read a column encoded: Column.Encoding and its views. ScanSpec spec = Spec() with { KeepEncodings = true }; CancellationToken token = cancellationToken.CanBeCanceled ? cancellationToken : _cancellation; - return new AsyncEnumerator(this, _source.BatchesAsync(spec, _metrics).GetAsyncEnumerator(token), binding); + return new AsyncEnumerator(this, _source.BatchesAsync(spec, _counters).GetAsyncEnumerator(token), binding); } /// The batches, each owned by the caller, who disposes it. @@ -185,7 +185,7 @@ public async IAsyncEnumerable ToBatchesAsync([EnumeratorCancellatio Begin(); try { - await foreach (RecordBatch batch in _source.BatchesAsync(Spec(), _metrics).WithCancellation(cancellationToken).ConfigureAwait(false)) + await foreach (RecordBatch batch in _source.BatchesAsync(Spec(), _counters).WithCancellation(cancellationToken).ConfigureAwait(false)) { yield return Own(batch); } @@ -206,7 +206,7 @@ public async IAsyncEnumerable ToRecordsAsync([EnumeratorCancellation] C TRecord[] rows = []; try { - await foreach (RecordBatch batch in _source.BatchesAsync(Spec(), _metrics).WithCancellation(cancellationToken).ConfigureAwait(false)) + await foreach (RecordBatch batch in _source.BatchesAsync(Spec(), _counters).WithCancellation(cancellationToken).ConfigureAwait(false)) { int count = Fill(batch, binding, ref rows); for (int i = 0; i < count; i++) @@ -234,7 +234,7 @@ public async ValueTask CountAsync(CancellationToken cancellationToken = de Begin(); try { - return await _source.CountAsync(Spec(), _metrics, cancellationToken).ConfigureAwait(false); + return await _source.CountAsync(Spec(), _counters, cancellationToken).ConfigureAwait(false); } finally { @@ -250,7 +250,7 @@ public async ValueTask AnyAsync(CancellationToken cancellationToken = defa Begin(); try { - return await _source.AnyAsync(Spec(), _metrics, cancellationToken).ConfigureAwait(false); + return await _source.AnyAsync(Spec(), _counters, cancellationToken).ConfigureAwait(false); } finally { @@ -293,7 +293,7 @@ public ValueTask ExplainAsync(CancellationToken cancellationToken = de Options = _take is not null && _filter.IsAll ? _options with { Compact = false } : _options, }; - internal ScanMetrics Metrics => _metrics; + internal ScanCounters Counters => _counters; internal Predicate Filter => _filter; @@ -316,7 +316,7 @@ internal void End() return; } - VortexTelemetry.ScanEnded(_metrics, _activity); + VortexTelemetry.ScanEnded(_counters, _activity); _activity = null; } @@ -331,7 +331,7 @@ internal void End() FilterLiteral value; try { - value = await _source.ExtremeAsync(Spec(), target.Field, min, _metrics, cancellationToken).ConfigureAwait(false); + value = await _source.ExtremeAsync(Spec(), target.Field, min, _counters, cancellationToken).ConfigureAwait(false); } finally { diff --git a/src/Vorticity/Scan/ScanBuilder.cs b/src/Vorticity/Scan/ScanBuilder.cs index 1c6edbc4..c9885c55 100644 --- a/src/Vorticity/Scan/ScanBuilder.cs +++ b/src/Vorticity/Scan/ScanBuilder.cs @@ -42,7 +42,7 @@ internal sealed class ScanBuilder private int _maxBatchRows; private int _windowRows = FlatLayoutReader.WindowRows; private int _degree = Volatile.Read(ref s_defaultDegree); - private ScanMetrics? _metrics; + private ScanCounters? _metrics; private TerminalTiers _tiers = TerminalTiers.All; private string? _orderPath; private string[]? _orderComposite; @@ -358,7 +358,7 @@ public ScanBuilder Take(ReadOnlySpan rowIndices) /// /// What it costs is the key's correlation with file order: a window of a sorted column /// is one contiguous read, while a window of runs over an uncorrelated column can touch a - /// split per row. reports which. It is the tool for a + /// split per row. reports which. It is the tool for a /// selective range or a top-k, not for ordering a whole uncorrelated column. /// /// @@ -619,7 +619,7 @@ public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancel /// and bytes asked of the source, the values the flat reader materialized, the batches and /// rows produced -- the same quantities, once the scan has actually run. /// - public ScanBuilder WithMetrics(ScanMetrics metrics) + public ScanBuilder WithMetrics(ScanCounters metrics) { ArgumentNullException.ThrowIfNull(metrics); _metrics = metrics; @@ -708,7 +708,7 @@ public async System.Threading.Tasks.ValueTask ExplainAsync( // What the scan's first step does after the pruning: ask the filter's exact source, when // it has one, for the rows it proves, unless the zone maps already show they cannot fit a // batch. When they fit, the scan reads the splits holding them and nothing else. - (RowSelection? proven, string? structure, ScanMetrics? probe) = + (RowSelection? proven, string? structure, ScanCounters? probe) = await ProbeAsync(rows, natural, plan, pruning, cancellationToken).ConfigureAwait(false); // Counted over the splits the scan's rows touch, as the scan counts what it decodes and @@ -727,7 +727,7 @@ public async System.Threading.Tasks.ValueTask ExplainAsync( // they are registered with the scan's own; and opening it reads the column's zone map when // the filter's pruning did not read it already, which the scan counts as it does. Compute.ZoneColumn? keyZones = null; - ScanMetrics? orderCost = null; + ScanCounters? orderCost = null; FieldMask walk = read.RootMask; if (_orderPath is not null && _orderComposite is null && Keys.KeyCursorBuilder.StatedSorted(_file, _orderPath)) { @@ -735,7 +735,7 @@ public async System.Threading.Tasks.ValueTask ExplainAsync( keyZones = pruning.Zones?.Column(_orderPath); if (keyZones is null) { - orderCost = new ScanMetrics(); + orderCost = new ScanCounters(); keyZones = (await Compute.ZonePruningPlan .PlanAsync(_file, tree, Expr.IsNotNull(Expr.Field(_orderPath)), cancellationToken, steps: null, orderCost) .ConfigureAwait(false)).Zones?.Column(_orderPath); @@ -794,7 +794,7 @@ public async System.Threading.Tasks.ValueTask ExplainAsync( // The data segments of the live splits, plus what consulting each structure cost: // the same asking the metrics count, so that plan and outcome are one quantity. A // segment the scan holds is asked for once, so the distinct ones are what it asks for. - int toRead = ScanMetrics.Unread(segments, _file.Segments, out long bytes); + int toRead = ScanCounters.Unread(segments, _file.Segments, out long bytes); for (int i = 0; i < steps.Count; i++) { @@ -880,7 +880,7 @@ private bool Proves(Compute.ZonePruner zones, RowRange split) /// What the scan's first step asks the filter's exact source, as FilteredBatches asks it: /// the rows it proves when they fit a batch, the structure that answered, and what asking cost. /// - private async System.Threading.Tasks.ValueTask<(RowSelection? Proven, string? Structure, ScanMetrics? Cost)> ProbeAsync( + private async System.Threading.Tasks.ValueTask<(RowSelection? Proven, string? Structure, ScanCounters? Cost)> ProbeAsync( RowRange rows, long natural, SplitPlan plan, Compute.ZonePruningPlan.PruningPlan pruning, System.Threading.CancellationToken cancellationToken) { @@ -892,7 +892,7 @@ private bool Proves(Compute.ZonePruner zones, RowRange split) return (null, null, null); } - ScanMetrics cost = new ScanMetrics(); + ScanCounters cost = new ScanCounters(); Keys.ExactCover? cover = await Keys.ExactCover .TryCreateAsync(_file, _filter, _indexes, cancellationToken, zones, cost) .ConfigureAwait(false); diff --git a/src/Vorticity/Scan/ScanMetrics.cs b/src/Vorticity/Scan/ScanCounters.cs similarity index 97% rename from src/Vorticity/Scan/ScanMetrics.cs rename to src/Vorticity/Scan/ScanCounters.cs index 791d7018..462f82fa 100644 --- a/src/Vorticity/Scan/ScanMetrics.cs +++ b/src/Vorticity/Scan/ScanCounters.cs @@ -10,7 +10,7 @@ namespace Vorticity.Scanning; /// same instance, so running a scan twice sums, and a fresh count means a fresh object. The /// additions are interlocked because a scan above degree one runs its lanes on the thread pool. /// -internal sealed class ScanMetrics +internal sealed class ScanCounters { private long _segmentRequests; private long _bytesRequested; @@ -127,7 +127,7 @@ internal static int Unread(IO.SegmentRequestSet segments, IO.ISegmentReader? rea /// The scan's sink, or null. /// The request set, registered, filled with what the scan and the file's tail hold, and not yet read. /// Whether there is anything to ask: when not, the set is complete already and the source is not called. - internal static bool Note(ScanMetrics? metrics, IO.SegmentRequestSet segments) + internal static bool Note(ScanCounters? metrics, IO.SegmentRequestSet segments) { int count = Unread(segments, out long bytes); if (count == 0) @@ -143,7 +143,7 @@ internal static bool Note(ScanMetrics? metrics, IO.SegmentRequestSet segments) /// The scan's sink, or null. /// Segments asked for. /// Their bytes. - internal static void Note(ScanMetrics? metrics, long segments, long bytes) + internal static void Note(ScanCounters? metrics, long segments, long bytes) { metrics?.AddRequests(segments, bytes); Diagnostics.VortexEventSource.Requested(segments, bytes); @@ -155,7 +155,7 @@ internal static void Note(ScanMetrics? metrics, long segments, long bytes) /// /// The scan's sink, or null. /// The request set, read. - internal static void Served(ScanMetrics? metrics, IO.SegmentRequestSet segments) + internal static void Served(ScanCounters? metrics, IO.SegmentRequestSet segments) { int hits = segments.CacheHits; if (hits > 0 && metrics is not null) diff --git a/src/Vorticity/Scan/ScanOptions.cs b/src/Vorticity/Scan/ScanOptions.cs index a5395688..2dcb04ca 100644 --- a/src/Vorticity/Scan/ScanOptions.cs +++ b/src/Vorticity/Scan/ScanOptions.cs @@ -30,7 +30,7 @@ public int BatchRows } /// Whether statistics and zone maps may skip blocks; off only to check a file's statistics, never to change a result. - public bool Pruning { get; init; } = true; + public bool UseStatistics { get; init; } = true; /// Whether the file's indexes may skip blocks; off only to check an index, never to change a result. public bool UseIndexes { get; init; } = true; diff --git a/src/Vorticity/Scan/ScanPlan.cs b/src/Vorticity/Scan/ScanPlan.cs index 1340b437..ad3bb405 100644 --- a/src/Vorticity/Scan/ScanPlan.cs +++ b/src/Vorticity/Scan/ScanPlan.cs @@ -23,10 +23,10 @@ public sealed record PruningStep(string Structure, int BlocksPruned, int Segment /// Of those, the blocks no structure could prove empty: the ones the scan decodes. /// /// Segments the scan asks its source for: each data segment of the live blocks once, and what -/// consulting each structure reads. The execution's is the same count. +/// consulting each structure reads. The execution's is the same count. /// /// -/// Their bytes; the execution's is the same sum, and what +/// Their bytes; the execution's is the same sum, and what /// a source transfers exceeds it only by the gaps it bridges to coalesce neighbouring segments. /// /// False when the file's statistics or filters prove the scan empty. @@ -88,14 +88,14 @@ public sealed record OrderPlan(string Source, int Runs, long? Entries, bool Desc /// /// Blocks skipped by statistics, zone maps or indexes. /// Segments served by the session's cache. -public readonly record struct ScanStatistics( +public readonly record struct ScanMetrics( long Rows, long Batches, long Requests, long BytesRequested, long BlocksDecoded, long BlocksPruned, long CacheHits) { /// What a group by did with the rows; null without a group by, or before it ran. - public GroupStatistics? Grouping { get; init; } + public GroupMetrics? Grouping { get; init; } - internal static ScanStatistics From(ScanMetrics metrics) => new ScanStatistics( - metrics.Rows, metrics.Batches, metrics.SegmentRequests, metrics.BytesRequested, metrics.BlocksDecoded, metrics.BlocksPruned, metrics.CacheHits); + internal static ScanMetrics From(ScanCounters counters) => new ScanMetrics( + counters.Rows, counters.Batches, counters.SegmentRequests, counters.BytesRequested, counters.BlocksDecoded, counters.BlocksPruned, counters.CacheHits); } /// diff --git a/src/Vorticity/Scan/ScanSource.cs b/src/Vorticity/Scan/ScanSource.cs index 8b6a0869..8fbea56c 100644 --- a/src/Vorticity/Scan/ScanSource.cs +++ b/src/Vorticity/Scan/ScanSource.cs @@ -85,14 +85,14 @@ internal abstract class ScanSource internal abstract VortexSession Session { get; } /// The batches, borrowed: each is valid until the next is asked for. - internal abstract IAsyncEnumerable BatchesAsync(ScanSpec spec, ScanMetrics metrics); + internal abstract IAsyncEnumerable BatchesAsync(ScanSpec spec, ScanCounters metrics); - internal abstract ValueTask CountAsync(ScanSpec spec, ScanMetrics metrics, CancellationToken cancellationToken); + internal abstract ValueTask CountAsync(ScanSpec spec, ScanCounters metrics, CancellationToken cancellationToken); - internal abstract ValueTask AnyAsync(ScanSpec spec, ScanMetrics metrics, CancellationToken cancellationToken); + internal abstract ValueTask AnyAsync(ScanSpec spec, ScanCounters metrics, CancellationToken cancellationToken); /// The smallest or largest non-null value of a column among the rows the scan keeps, or a null literal. - internal abstract ValueTask ExtremeAsync(ScanSpec spec, FieldExpr column, bool min, ScanMetrics metrics, CancellationToken cancellationToken); + internal abstract ValueTask ExtremeAsync(ScanSpec spec, FieldExpr column, bool min, ScanCounters metrics, CancellationToken cancellationToken); internal abstract ValueTask ExplainAsync(ScanSpec spec, CancellationToken cancellationToken); @@ -173,12 +173,12 @@ internal sealed class FileScanSource : ScanSource internal override VortexSession Session => _file.Session; - internal override IAsyncEnumerable BatchesAsync(ScanSpec spec, ScanMetrics metrics) => + internal override IAsyncEnumerable BatchesAsync(ScanSpec spec, ScanCounters metrics) => spec.MatchesNothing ? System.Linq.AsyncEnumerable.Empty() : Refuted(spec) ? RefusedAsync(spec, metrics) : Builder(spec, metrics).ExecuteAsync(); - internal override async ValueTask CountAsync(ScanSpec spec, ScanMetrics metrics, CancellationToken cancellationToken) + internal override async ValueTask CountAsync(ScanSpec spec, ScanCounters metrics, CancellationToken cancellationToken) { if (spec.MatchesNothing || await RefuseAsync(spec, metrics, cancellationToken).ConfigureAwait(false)) { @@ -188,12 +188,12 @@ internal override async ValueTask CountAsync(ScanSpec spec, ScanMetrics me return await Builder(spec, metrics).CountAsync(cancellationToken).ConfigureAwait(false); } - internal override async ValueTask AnyAsync(ScanSpec spec, ScanMetrics metrics, CancellationToken cancellationToken) => + internal override async ValueTask AnyAsync(ScanSpec spec, ScanCounters metrics, CancellationToken cancellationToken) => !spec.MatchesNothing && !await RefuseAsync(spec, metrics, cancellationToken).ConfigureAwait(false) && await Builder(spec, metrics).AnyAsync(cancellationToken).ConfigureAwait(false); - internal override async ValueTask ExtremeAsync(ScanSpec spec, FieldExpr column, bool min, ScanMetrics metrics, CancellationToken cancellationToken) + internal override async ValueTask ExtremeAsync(ScanSpec spec, FieldExpr column, bool min, ScanCounters metrics, CancellationToken cancellationToken) { if (spec.MatchesNothing || await RefuseAsync(spec, metrics, cancellationToken).ConfigureAwait(false)) { @@ -208,7 +208,7 @@ internal override async ValueTask ExtremeAsync(ScanSpec spec, Fie /// The batches of a scan the file statistics refute: none, and its blocks counted pruned. private async IAsyncEnumerable RefusedAsync( - ScanSpec spec, ScanMetrics metrics, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) + ScanSpec spec, ScanCounters metrics, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken cancellationToken = default) { await RefuseAsync(spec, metrics, cancellationToken).ConfigureAwait(false); yield break; @@ -218,7 +218,7 @@ private async IAsyncEnumerable RefusedAsync( /// Whether the file statistics refute the scan; when they do, its blocks are counted pruned, as /// its plan counts them, and nothing is read. /// - private async ValueTask RefuseAsync(ScanSpec spec, ScanMetrics metrics, CancellationToken cancellationToken) + private async ValueTask RefuseAsync(ScanSpec spec, ScanCounters metrics, CancellationToken cancellationToken) { if (!Refuted(spec)) { @@ -234,7 +234,7 @@ private async ValueTask RefuseAsync(ScanSpec spec, ScanMetrics metrics, Ca /// gives. It consults no structure, and so reads nothing. /// private async ValueTask WholeAsync(ScanSpec spec, CancellationToken cancellationToken) => - ScanPlan.From(await Builder(spec with { Filter = null, OrderPath = null }, new ScanMetrics()) + ScanPlan.From(await Builder(spec with { Filter = null, OrderPath = null }, new ScanCounters()) .ExplainAsync(cancellationToken).ConfigureAwait(false)); internal override async ValueTask ExplainAsync(ScanSpec spec, CancellationToken cancellationToken) @@ -255,7 +255,7 @@ internal override async ValueTask ExplainAsync(ScanSpec spec, Cancella }; } - ScanExplanation plan = await Builder(spec, new ScanMetrics()).ExplainAsync(cancellationToken).ConfigureAwait(false); + ScanExplanation plan = await Builder(spec, new ScanCounters()).ExplainAsync(cancellationToken).ConfigureAwait(false); ScanPlan result = ScanPlan.From(plan); // A filter known to match nothing runs no scan at all: it reads nothing and counts zero. @@ -291,7 +291,7 @@ internal override bool TryBounds(int[] path, out FilterLiteral min, out FilterLi /// The segments the largest split of the scan asks for, over a source that copies them: the layout walked once, nothing read. internal override long ReadAheadBytes(ScanSpec spec) => - _file.Source.ReadsInPlace || spec.MatchesNothing ? 0 : Builder(spec, new ScanMetrics()).LargestSplitBytes(); + _file.Source.ReadsInPlace || spec.MatchesNothing ? 0 : Builder(spec, new ScanCounters()).LargestSplitBytes(); internal override async ValueTask OpenKeysAsync(string path, bool distinct, bool indexes, CancellationToken cancellationToken) => await KeysOf(path, distinct, indexes).OpenAsync(cancellationToken).ConfigureAwait(false); @@ -311,7 +311,7 @@ private KeyCursorBuilder KeysOf(string path, bool distinct, bool indexes) } /// The engine's builder for . - internal ScanBuilder Builder(ScanSpec spec, ScanMetrics metrics) + internal ScanBuilder Builder(ScanSpec spec, ScanCounters metrics) { ScanBuilder builder = _file.ScanBuilder().WithMetrics(metrics); if (spec.Projection is { } mask) @@ -349,7 +349,7 @@ internal ScanBuilder Builder(ScanSpec spec, ScanMetrics metrics) builder.WithMaxBatchRows(options.BatchRows); } - builder.WithPruning(options.Pruning).WithIndexes(options.UseIndexes); + builder.WithPruning(options.UseStatistics).WithIndexes(options.UseIndexes); int degree = options.DegreeOfParallelism > 0 ? options.DegreeOfParallelism : Session.Options.MaxDegreeOfParallelism; builder.WithDegreeOfParallelism(Math.Max(degree, 1)); builder.WithPrefetch(options.Prefetch).WithCompaction(options.Compact).WithEncodings(spec.KeepEncodings, spec.SinkDecodes); diff --git a/src/Vorticity/Scan/TerminalScan.cs b/src/Vorticity/Scan/TerminalScan.cs index 8849ad88..e50067bd 100644 --- a/src/Vorticity/Scan/TerminalScan.cs +++ b/src/Vorticity/Scan/TerminalScan.cs @@ -49,7 +49,7 @@ internal sealed class TerminalScan private readonly bool _prune; private readonly bool _indexes; private readonly TerminalTiers _tiers; - private readonly ScanMetrics? _metrics; + private readonly ScanCounters? _metrics; /// The rows a count leaves out whatever the filter says; null for none. Never with a take. private readonly IRowExclusion? _excluded; @@ -65,7 +65,7 @@ internal TerminalScan( RowSelection? take, bool prune, TerminalTiers tiers, - ScanMetrics? metrics, + ScanCounters? metrics, bool indexes = true, IRowExclusion? excluded = null) { @@ -310,7 +310,7 @@ private async ValueTask ReadAndExecuteAsync( decodes.Held.Claim(context.Segments, batch, waiter: null); try { - if (ScanMetrics.Note(_metrics, context.Segments)) + if (ScanCounters.Note(_metrics, context.Segments)) { await _file.Segments.ReadManyAsync(context.Segments, cancellationToken).ConfigureAwait(false); } @@ -327,9 +327,9 @@ private async ValueTask ReadAndExecuteAsync( decodes.Held.Publish(context.Segments, batch); decodes.Held.Release(batch); - ScanMetrics.Served(_metrics, context.Segments); + ScanCounters.Served(_metrics, context.Segments); int root = SplitExecution.Execute(context, _tree, in _mask, split, _take); - if (_metrics is not null && ScanMetrics.Decoded(context.Canonical, root)) + if (_metrics is not null && ScanCounters.Decoded(context.Canonical, root)) { _metrics.AddBlocksDecoded(decodes.Decoded.Add(split, _take, decodes.Live)); } diff --git a/src/Vorticity/Schema/VortexSchema.cs b/src/Vorticity/Schema/VortexSchema.cs index 32b89ebd..7f3587f9 100644 --- a/src/Vorticity/Schema/VortexSchema.cs +++ b/src/Vorticity/Schema/VortexSchema.cs @@ -82,21 +82,18 @@ public int IndexOf(string path) return Find(path); } - /// The UTF-8 form of , without allocating. + /// The UTF-8 form of , without allocating. /// The name or the .-separated path, as UTF-8. - /// The field's index within its own struct. - /// Whether a field matches. - public bool TryGetField(ReadOnlySpan pathUtf8, out int index) + /// The index, or -1 when no field matches or the bytes are not UTF-8. + public int IndexOf(ReadOnlySpan pathUtf8) { Span chars = pathUtf8.Length <= 256 ? stackalloc char[256] : new char[pathUtf8.Length]; if (Utf8.ToUtf16(pathUtf8, chars, out _, out int written) != System.Buffers.OperationStatus.Done) { - index = -1; - return false; + return -1; } - index = Find(chars[..written]); - return index >= 0; + return Find(chars[..written]); } /// The index of the first top-level field named exactly , or -1. diff --git a/src/Vorticity/Session/SegmentCache.cs b/src/Vorticity/Session/SegmentCache.cs index 9ac7f9da..e1bf3231 100644 --- a/src/Vorticity/Session/SegmentCache.cs +++ b/src/Vorticity/Session/SegmentCache.cs @@ -34,14 +34,14 @@ public sealed class SegmentCache public SegmentCache(long capacityBytes) { ArgumentOutOfRangeException.ThrowIfNegative(capacityBytes); - Capacity = capacityBytes; + CapacityBytes = capacityBytes; } /// The budget in bytes. - public long Capacity { get; } + public long CapacityBytes { get; } /// The bytes held now. - public long Size => Interlocked.Read(ref _size); + public long HeldBytes => Interlocked.Read(ref _size); /// Segments served from the cache since it was created. public long Hits => Interlocked.Read(ref _hits); @@ -107,7 +107,7 @@ internal bool TryGet(object source, long offset, int length, out SegmentOwner ow internal void Add(object source, long offset, SegmentOwner owner, VortexBuffer buffer) { int length = buffer.Length; - if (length > Capacity || length == 0) + if (length > CapacityBytes || length == 0) { return; } @@ -119,7 +119,7 @@ internal void Add(object source, long offset, SegmentOwner owner, VortexBuffer b Key key = new Key(source, offset, length); if (!_entries.ContainsKey(key)) { - while (_size + length > Capacity && _oldest is { } last) + while (_size + length > CapacityBytes && _oldest is { } last) { Unlink(last); UnlinkFromSource(last); diff --git a/src/Vorticity/Session/VortexSession.cs b/src/Vorticity/Session/VortexSession.cs index 32794a31..a3d639f5 100644 --- a/src/Vorticity/Session/VortexSession.cs +++ b/src/Vorticity/Session/VortexSession.cs @@ -392,7 +392,7 @@ public VortexFileWriter CreateWriter(PipeWriter sink, VortexSchema schema, Vorte /// Abandoned, or disposed without completing, it truncates the file back to what it was. /// /// The file's layout is not one this library can continue. - public ValueTask AppendAsync(string path, VortexWriteOptions? options = null, CancellationToken cancellationToken = default) + public ValueTask OpenWriterAsync(string path, VortexWriteOptions? options = null, CancellationToken cancellationToken = default) { ArgumentNullException.ThrowIfNull(path); ThrowIfDisposed(); diff --git a/src/Vorticity/Writing/VortexWriteOptions.cs b/src/Vorticity/Writing/VortexWriteOptions.cs index 6fb94b0f..2c33a3ec 100644 --- a/src/Vorticity/Writing/VortexWriteOptions.cs +++ b/src/Vorticity/Writing/VortexWriteOptions.cs @@ -154,7 +154,7 @@ public ImmutableDictionary Hints public VortexEdition TargetEdition { get; init; } = VortexEditions.Default; /// Whether the file carries its statistics: per column the null count and the order, and the exact minimum and maximum of a numeric column. - public bool Statistics { get; init; } = true; + public bool WriteStatistics { get; init; } = true; /// /// Whether a file this library opens, created or appended to, is on the device when @@ -270,11 +270,11 @@ internal bool Compress init => Compression = value ? (Compression == CompressionProfile.None ? CompressionProfile.Auto : Compression) : CompressionProfile.None; } - /// , as the engine names it. + /// , as the engine names it. internal bool FileStatistics { - get => Statistics; - init => Statistics = value; + get => WriteStatistics; + init => WriteStatistics = value; } /// Rows per chunk, as a multiple; null writes one chunk per call, which only the engine's own tests ask for. diff --git a/tests/MustNotCompile/Bodies/Queries.cs b/tests/MustNotCompile/Bodies/Queries.cs index dc3e73d6..6d4fe29c 100644 --- a/tests/MustNotCompile/Bodies/Queries.cs +++ b/tests/MustNotCompile/Bodies/Queries.cs @@ -74,11 +74,11 @@ internal static object RecordOfOtherLength(VortexFile file) => // The answers of a scan go into their record by the same rule. internal static async Task AnswersOfOtherTypes(VortexFile file) => - await file.Scan().AggAsync(a => (a.Max(r => r.City), a.Average(r => r.Celsius))); // expect: VX1010 "Element 2" + await file.Scan().AggregateAsync(a => (a.Max(r => r.City), a.Average(r => r.Celsius))); // expect: VX1010 "Element 2" - // The answers of a whole scan, several of them, go into the record AggAsync names. + // The answers of a whole scan, several of them, go into the record AggregateAsync names. internal static async Task SeveralAnswersWithoutARecord(VortexFile file) => - await file.Scan().AggAsync(a => (a.Count(), a.Max(r => r.Day))); // expect: CS0411 VX1011 + await file.Scan().AggregateAsync(a => (a.Count(), a.Max(r => r.Day))); // expect: CS0411 VX1011 // Computing with results is C# after the sink, not the plan's. internal static object ArithmeticOnAResult(VortexFile file) => diff --git a/tests/Vorticity.Tests/Aggregation/ChosenRowTests.cs b/tests/Vorticity.Tests/Aggregation/ChosenRowTests.cs index 9b284436..58c6d4cc 100644 --- a/tests/Vorticity.Tests/Aggregation/ChosenRowTests.cs +++ b/tests/Vorticity.Tests/Aggregation/ChosenRowTests.cs @@ -104,11 +104,11 @@ public async Task AWholeScanTakesItsChosenRowsInFileOrder() await using VortexFile file = await VortexFile.OpenAsync(path, Ct); Ends ends = await file.Scan() .Where(r => r.Minute >= 100) - .AggAsync(a => (a.First().Symbol, a.Last().Symbol, a.MaxBy(x => x.Size).Venue, a.MinBy(x => x.Price).Minute), Ct); + .AggregateAsync(a => (a.First().Symbol, a.Last().Symbol, a.MaxBy(x => x.Size).Venue, a.MinBy(x => x.Price).Minute), Ct); Trade[] kept = [.. rows.Where(r => r.Minute >= 100)]; Assert.Equal(new Ends(kept[0].Symbol, kept[^1].Symbol, Best(kept, r => r.Size, max: true)!.Value.Venue, Best(kept, r => r.Price, max: false)!.Value.Minute), ends); - Assert.Equal(kept[^1].Venue, await file.Scan().Where(r => r.Minute >= 100).AggAsync(a => a.Last().Venue, Ct)); + Assert.Equal(kept[^1].Venue, await file.Scan().Where(r => r.Minute >= 100).AggregateAsync(a => a.Last().Venue, Ct)); } finally { @@ -185,14 +185,14 @@ public async Task ATopKReadsTheRowsOfTheGroupsItDeliversAlone() .OrderByDescending(g => g.High.row.Price).ThenBy(g => g.Minute) .Take(10) .Select(g => g.High.position)]; - (List sizes, ScanStatistics reads) = await HighestAsync(file, 10, row: true); + (List sizes, ScanMetrics reads) = await HighestAsync(file, 10, row: true); Assert.Equal(top.Select(position => rows[position].Size), sizes); // What the rows cost: one chunk for one row; the chunks of the ten for ten; every // chunk for every group. The pass is the same with the rows or without them. - ScanStatistics one = Fetched((await HighestAsync(file, 1, row: true)).Reads, (await HighestAsync(file, 1, row: false)).Reads); - ScanStatistics ten = Fetched(reads, (await HighestAsync(file, 10, row: false)).Reads); - ScanStatistics all = Fetched((await HighestAsync(file, int.MaxValue, row: true)).Reads, (await HighestAsync(file, int.MaxValue, row: false)).Reads); + ScanMetrics one = Fetched((await HighestAsync(file, 1, row: true)).Reads, (await HighestAsync(file, 1, row: false)).Reads); + ScanMetrics ten = Fetched(reads, (await HighestAsync(file, 10, row: false)).Reads); + ScanMetrics all = Fetched((await HighestAsync(file, int.MaxValue, row: true)).Reads, (await HighestAsync(file, int.MaxValue, row: false)).Reads); int chunks = top.Select(position => position / ChunkRows).Distinct().Count(); Assert.True(one.Requests > 0 && one.BlocksDecoded > 0, $"{one}"); Assert.True(ten.Requests <= chunks * one.Requests, $"{ten.Requests} requests for the rows of {chunks} chunks, {one.Requests} for one"); @@ -231,8 +231,8 @@ public async Task AFirstAndALastShareOneTake() rows.GroupBy(r => r.Minute).Select(g => new MinuteEndPrices(g.Key, g.First().Price, g.Last().Price)), ends); Assert.Equal(ends.Select(e => e.Open).Order(), open.Order()); - ScanStatistics first = Fetched(opens.Statistics, none.Statistics); - ScanStatistics firstAndLast = Fetched(both.Statistics, none.Statistics); + ScanMetrics first = Fetched(opens.Metrics, none.Metrics); + ScanMetrics firstAndLast = Fetched(both.Metrics, none.Metrics); Assert.True(first.Requests > 0, $"{first}"); Assert.Equal(first.Requests, firstAndLast.Requests); Assert.Equal(first.BlocksDecoded, firstAndLast.BlocksDecoded); @@ -297,7 +297,7 @@ public async Task AFilterOnAChosenRowAfterATakeReadsTheGroupsTaken() .Take(10) .Select(g => g.Key); await unread.ToListAsync(Ct); - Assert.True(Fetched(filtered.Statistics, unread.Statistics).Requests <= 20, $"{Fetched(filtered.Statistics, unread.Statistics)}"); + Assert.True(Fetched(filtered.Metrics, unread.Metrics).Requests <= 20, $"{Fetched(filtered.Metrics, unread.Metrics)}"); } finally { @@ -311,21 +311,21 @@ public async Task AFilterOnAChosenRowAfterATakeReadsTheGroupsTaken() private static VortexWriteOptions Chunked => new VortexWriteOptions { RowBlockSize = ChunkRows, ChunkTargetBytes = 1 << 12 }; /// What the chosen rows' fetch read: a query's reads less those of the same query without them. - private static ScanStatistics Fetched(ScanStatistics with, ScanStatistics without) => - new ScanStatistics(0, 0, with.Requests - without.Requests, with.BytesRequested - without.BytesRequested, with.BlocksDecoded - without.BlocksDecoded, 0, 0); + private static ScanMetrics Fetched(ScanMetrics with, ScanMetrics without) => + new ScanMetrics(0, 0, with.Requests - without.Requests, with.BytesRequested - without.BytesRequested, with.BlocksDecoded - without.BlocksDecoded, 0, 0); /// The minutes of the highest prices: the size of the row of each, or the price alone; and what the query read. - private static async Task<(List Sizes, ScanStatistics Reads)> HighestAsync(VortexFile file, int take, bool row) + private static async Task<(List Sizes, ScanMetrics Reads)> HighestAsync(VortexFile file, int take, bool row) { if (row) { Aggregation sizes = file.Scan().GroupBy(r => r.Minute).OrderByDescending(g => g.Max(x => x.Price)).Take(take).Select(g => g.MaxBy(x => x.Price).Size); - return (await sizes.ToListAsync(Ct), sizes.Statistics); + return (await sizes.ToListAsync(Ct), sizes.Metrics); } Aggregation prices = file.Scan().GroupBy(r => r.Minute).OrderByDescending(g => g.Max(x => x.Price)).Take(take).Select(g => g.Max(x => x.Price)); await prices.ToListAsync(Ct); - return ([], prices.Statistics); + return ([], prices.Metrics); } /// The row of the largest or smallest value, the first on a tie, nulls and NaN never. diff --git a/tests/Vorticity.Tests/Aggregation/DatasetGroupTests.cs b/tests/Vorticity.Tests/Aggregation/DatasetGroupTests.cs index e5455f59..15ee799a 100644 --- a/tests/Vorticity.Tests/Aggregation/DatasetGroupTests.cs +++ b/tests/Vorticity.Tests/Aggregation/DatasetGroupTests.cs @@ -167,7 +167,7 @@ public async Task AnObjectIsCutBetweenItsChunksSoNoneIsReadTwice() // One object of many chunks, read on one lane and then on four: the four take it in // ranges, each a run of whole chunks, so every segment is asked for once either way. - (string Answers, ScanStatistics Statistics, AggregationRun Run)[] reads = new (string, ScanStatistics, AggregationRun)[2]; + (string Answers, ScanMetrics Metrics, AggregationRun Run)[] reads = new (string, ScanMetrics, AggregationRun)[2]; foreach ((int at, int degree) in ((int, int)[])[(0, 1), (1, 4)]) { Scan scan = dataset.Scan().With(new ScanOptions { DegreeOfParallelism = degree }); @@ -175,14 +175,14 @@ public async Task AnObjectIsCutBetweenItsChunksSoNoneIsReadTwice() List read = await ListAsync(desks.As()); reads[at] = ( string.Join(";", read.OrderBy(d => d.Desk, StringComparer.Ordinal).Select(d => $"{d.Desk}={d.Count}/{BitConverter.DoubleToInt64Bits(d.Sum):X}")), - scan.Statistics, + scan.Metrics, desks.Plan.LastRun!); } Assert.True(reads[1].Run.Lanes.Sum(lane => lane.Ranges) >= 4, $"{reads[1].Run.Lanes.Sum(lane => lane.Ranges)} ranges"); Assert.Equal(reads[0].Answers, reads[1].Answers); - Assert.Equal(reads[0].Statistics.Requests, reads[1].Statistics.Requests); - Assert.Equal(reads[0].Statistics.BytesRequested, reads[1].Statistics.BytesRequested); + Assert.Equal(reads[0].Metrics.Requests, reads[1].Metrics.Requests); + Assert.Equal(reads[0].Metrics.BytesRequested, reads[1].Metrics.BytesRequested); } finally { @@ -242,7 +242,7 @@ public async Task AGroupByOnTheClusteringKeyStreamsOverTheObjects() private static async Task MarkedAsync(VortexDataset dataset) { int marked = 0; - await foreach (DataObject held in dataset.ObjectsAsync(Ct)) + await foreach (DataObject held in dataset.ListObjectsAsync(Ct)) { marked += held.DeletedRows > 0 ? 1 : 0; } diff --git a/tests/Vorticity.Tests/Aggregation/FanTests.cs b/tests/Vorticity.Tests/Aggregation/FanTests.cs index 4c0d346c..cf97746d 100644 --- a/tests/Vorticity.Tests/Aggregation/FanTests.cs +++ b/tests/Vorticity.Tests/Aggregation/FanTests.cs @@ -50,7 +50,7 @@ public async Task TheFirstLaneThatMeetsManyValuesTurnsTheQuery(string shape, boo } Assert.Equal(rows.GroupBy(r => r.Key).ToDictionary(g => g.Key, g => new KeyTotal(g.Key, g.Count(), g.Sum(r => r.Value))), got); - GroupStatistics grouping = query.Statistics.Grouping!; + GroupMetrics grouping = query.Metrics.Grouping!; if (expected is null) { return; @@ -101,7 +101,7 @@ public async Task ALaneProjectsItsNewGroupsPastItsFirstRows(string shape, GroupC Assert.Equal(rows.Length, count); Assert.Equal(rows.Sum(r => r.Value), total); - GroupStatistics grouping = query.Statistics.Grouping!; + GroupMetrics grouping = query.Metrics.Grouping!; Assert.Equal(expected, grouping.CoreReason); Assert.Equal(rows.Select(r => r.Key).Distinct().Count(), grouping.Groups); if (expected == GroupCoreReason.Projection) diff --git a/tests/Vorticity.Tests/Aggregation/FilteredGroupTests.cs b/tests/Vorticity.Tests/Aggregation/FilteredGroupTests.cs index 65099d01..2c2983fd 100644 --- a/tests/Vorticity.Tests/Aggregation/FilteredGroupTests.cs +++ b/tests/Vorticity.Tests/Aggregation/FilteredGroupTests.cs @@ -178,7 +178,7 @@ public async Task TheAggregatesOfAScanAreFilteredToo(int degree) await using VortexFile file = await session.OpenAsync(path, cancellationToken: Ct); Totals totals = await file.Scan() .Where(r => r.Day >= 2) - .AggAsync(a => (a.Count(), a.Count(x => x.Status >= 500), a.Where(x => x.Region == "eu").Average(x => x.DurationMs), a.Any(x => x.Status == 503), a.All(x => x.Day < 10)), Ct); + .AggregateAsync(a => (a.Count(), a.Count(x => x.Status >= 500), a.Where(x => x.Region == "eu").Average(x => x.DurationMs), a.Any(x => x.Status == 503), a.All(x => x.Day < 10)), Ct); Request[] kept = [.. rows.Where(r => r.Day >= 2)]; Assert.Equal(kept.Length, totals.Rows); @@ -190,10 +190,10 @@ public async Task TheAggregatesOfAScanAreFilteredToo(int degree) // A count or an any of one filter alone is the scan's, its filter joined. Assert.Equal( new Answers(kept.Count(r => r.Status >= 500 && r.Region == "us"), kept.Any(r => r.Status >= 500 && r.Region == "us")), - await file.Scan().Where(r => r.Day >= 2).AggAsync(a => (a.Count(x => x.Status >= 500 & x.Region == "us"), a.Any(x => x.Status >= 500 & x.Region == "us")), Ct)); + await file.Scan().Where(r => r.Day >= 2).AggregateAsync(a => (a.Count(x => x.Status >= 500 & x.Region == "us"), a.Any(x => x.Status >= 500 & x.Region == "us")), Ct)); Assert.Equal( new Answers(0, false), - await file.Scan().AggAsync(a => (a.Count(x => Predicate.None), a.Where(x => x.Day > 5).Any(x => x.Day < 5)), Ct)); + await file.Scan().AggregateAsync(a => (a.Count(x => Predicate.None), a.Where(x => x.Day > 5).Any(x => x.Day < 5)), Ct)); } finally { diff --git a/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs b/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs index 05b0d4a2..0c78988a 100644 --- a/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs +++ b/tests/Vorticity.Tests/Aggregation/GroupAllocationTests.cs @@ -225,7 +225,7 @@ private static async Task MeasureAsync(string path, StringBuilder report) rows += Sum(batch.Column(counted).Values); } - return (rows, query.Statistics.Grouping!.CoreReason); + return (rows, query.Metrics.Grouping!.CoreReason); } private static long Sum(ReadOnlySpan values) diff --git a/tests/Vorticity.Tests/Aggregation/GroupCoreTests.cs b/tests/Vorticity.Tests/Aggregation/GroupCoreTests.cs index 0eefd94a..96ed50d4 100644 --- a/tests/Vorticity.Tests/Aggregation/GroupCoreTests.cs +++ b/tests/Vorticity.Tests/Aggregation/GroupCoreTests.cs @@ -58,8 +58,8 @@ public async Task EveryStateInARecordComesOutTheSame(int degree, double bypass, // Delivered part by part, every group once, a part let go before the next; and whole, as // an order over the groups asks it, the same again. - Assert.Equal(held.Count, core.Statistics.Grouping!.Groups); - Assert.InRange(core.Statistics.Grouping.PeakGroups, 1, held.Count); + Assert.Equal(held.Count, core.Metrics.Grouping!.Groups); + Assert.InRange(core.Metrics.Grouping.PeakGroups, 1, held.Count); Vorticity.Aggregation whole = Query(file, plan => { Tiny(plan); diff --git a/tests/Vorticity.Tests/Aggregation/GroupPlanTests.cs b/tests/Vorticity.Tests/Aggregation/GroupPlanTests.cs index 4347386e..29990962 100644 --- a/tests/Vorticity.Tests/Aggregation/GroupPlanTests.cs +++ b/tests/Vorticity.Tests/Aggregation/GroupPlanTests.cs @@ -30,7 +30,7 @@ public async Task APlanSaysTheKeyItsBoundsAndHowTheGroupsAreOrdered() GroupKeyPlan key = Assert.Single(plan.Keys); Assert.Equal("Key", key.Column); Assert.False(key.Sorted); - Assert.Equal(10_000, key.Values); + Assert.Equal(10_000, key.ValueRange); Assert.Equal(10_000, plan.MostGroups); Assert.Null(plan.Streaming); Assert.NotNull(plan.NotStreaming); @@ -88,9 +88,9 @@ public async Task TheStatisticsSayWhatTheGroupByDid() await using VortexFile file = await session.OpenAsync(path, cancellationToken: Ct); Aggregation counts = file.Scan().GroupBy(r => r.Key).Select(g => g.Count()); - Assert.Null(counts.Statistics.Grouping); + Assert.Null(counts.Metrics.Grouping); Assert.Equal(10_000, (await counts.ToListAsync(Ct)).Count); - GroupStatistics grouping = counts.Statistics.Grouping!; + GroupMetrics grouping = counts.Metrics.Grouping!; Assert.Equal(10_000, grouping.Groups); Assert.Equal(10_000, grouping.PeakGroups); Assert.True(grouping.PeakBytes > 0); @@ -111,7 +111,7 @@ public async Task TheStatisticsSayWhatTheGroupByDid() Assert.Equal(256, plan.CacheCapacity); Assert.True(plan.Alpha >= 1); Assert.Equal(10_000, (await held.ToListAsync(Ct)).Count); - GroupStatistics core = held.Statistics.Grouping!; + GroupMetrics core = held.Metrics.Grouping!; Assert.True(core.Core); Assert.True(core.CacheEvictions > 0); Assert.True(core.Tables > 0); diff --git a/tests/Vorticity.Tests/Aggregation/ParallelMergeTests.cs b/tests/Vorticity.Tests/Aggregation/ParallelMergeTests.cs index b29b46ab..167b0c9e 100644 --- a/tests/Vorticity.Tests/Aggregation/ParallelMergeTests.cs +++ b/tests/Vorticity.Tests/Aggregation/ParallelMergeTests.cs @@ -46,7 +46,7 @@ public async Task EveryStateMergesByParts() Assert.Contains(int.MinValue, read.Keys); // Each group counted once, as its part is read, whether the merge or the reader went ahead. - Assert.Equal(expected.Length, byKey.Statistics.Grouping!.Groups); + Assert.Equal(expected.Length, byKey.Metrics.Grouping!.Groups); foreach (IGrouping group in expected) { KeyStats got = read[group.Key ?? int.MinValue]; diff --git a/tests/Vorticity.Tests/Aggregation/ResultScanTests.cs b/tests/Vorticity.Tests/Aggregation/ResultScanTests.cs index 89d854fd..c296739b 100644 --- a/tests/Vorticity.Tests/Aggregation/ResultScanTests.cs +++ b/tests/Vorticity.Tests/Aggregation/ResultScanTests.cs @@ -58,7 +58,7 @@ public async Task ASelectionPastEightElementsIsReadThroughItsRecord() } [Fact] - public async Task TheAnswersOfAScanGoIntoTheRecordAggAsyncNames() + public async Task TheAnswersOfAScanGoIntoTheRecordAggregateAsyncNames() { (Visit[] rows, string path) = await WriteAsync(); try @@ -66,7 +66,7 @@ public async Task TheAnswersOfAScanGoIntoTheRecordAggAsyncNames() await using VortexFile file = await VortexFile.OpenAsync(path, Ct); Totals totals = await file.Scan() .Where(r => r.Day >= 3) - .AggAsync(a => (a.Count(), a.Sum(x => x.Pages), a.Max(x => x.City), a.Average(x => x.Seconds)), Ct); + .AggregateAsync(a => (a.Count(), a.Sum(x => x.Pages), a.Max(x => x.City), a.Average(x => x.Seconds)), Ct); Visit[] kept = [.. rows.Where(r => r.Day >= 3)]; Assert.Equal(kept.Length, totals.Count); @@ -105,7 +105,7 @@ public async Task ARecordThatDoesNotTakeTheSelectionIsRefusedBeforeAnyRead() Assert.Contains("may be null", notNullable.Message, StringComparison.Ordinal); Func, System.Runtime.CompilerServices.ITuple> twoCounts = a => (a.Count(), a.Count()); - await Assert.ThrowsAsync(async () => await file.Scan().AggAsync(twoCounts, Ct)); + await Assert.ThrowsAsync(async () => await file.Scan().AggregateAsync(twoCounts, Ct)); } finally { @@ -143,7 +143,7 @@ public async Task AResultsScanTakesTheOperatorsOfAScan() Assert.Equal(expected.Sum(g => g.Count), await Daily(file).SumAsync(g => g.Count, Ct)); Assert.Equal( new GroupsAndRows(expected.Length, expected.Sum(g => g.Count)), - await Daily(file).AggAsync(a => (a.Count(), a.Sum(x => x.Count)), Ct)); + await Daily(file).AggregateAsync(a => (a.Count(), a.Sum(x => x.Count)), Ct)); // Rows by their position in the order the result is delivered. List all = await ListAsync(Daily(file)); @@ -227,20 +227,20 @@ public async Task AResultsKeysAreWalkedInKeyOrderFromMemory() Assert.Equal(expected, walked); long middle = expected[expected.Length / 2].Key; - Assert.True(await counts.SeekAsync(middle, SeekOp.Exact, Ct)); + Assert.True(await counts.SeekAsync(middle, SeekMode.Exact, Ct)); Assert.Equal(expected.First(e => e.Key == middle).Row, counts.Row); - Assert.Equal(expected.Count(e => e.Key == middle), await counts.KeyCountAsync(Ct)); + Assert.Equal(expected.Count(e => e.Key == middle), await counts.CountAtKeyAsync(Ct)); Assert.Equal(expected.Count(e => e.Key < middle), await counts.RankAsync(middle, Ct)); - Assert.True(await counts.SeekAsync(middle, SeekOp.Before, Ct)); + Assert.True(await counts.SeekAsync(middle, SeekMode.Before, Ct)); Assert.Equal(expected.Last(e => e.Key < middle), (counts.Key, counts.Row)); - Assert.True(await counts.SeekAsync(middle, SeekOp.After, Ct)); + Assert.True(await counts.SeekAsync(middle, SeekMode.After, Ct)); Assert.Equal(expected.First(e => e.Key > middle), (counts.Key, counts.Row)); - Assert.False(await counts.SeekAsync(expected[^1].Key, SeekOp.After, Ct)); - Assert.False(await counts.SeekAsync(-1, SeekOp.Exact, Ct)); + Assert.False(await counts.SeekAsync(expected[^1].Key, SeekMode.After, Ct)); + Assert.False(await counts.SeekAsync(-1, SeekMode.Exact, Ct)); // Backwards a key at a time, each on its last entry. List<(long, long)> lasts = []; - for (bool on = await counts.SeekLastAsync(Ct); on; on = await counts.PrevKeyAsync(Ct)) + for (bool on = await counts.SeekLastAsync(Ct); on; on = await counts.PreviousKeyAsync(Ct)) { lasts.Add((counts.Key, counts.Row)); } @@ -260,8 +260,8 @@ public async Task AResultsKeysAreWalkedInKeyOrderFromMemory() } Assert.Equal(delivered.Select(g => g.City).Distinct().Order(StringComparer.Ordinal), walked); - Assert.True(await cities.SeekAsync("Caen", SeekOp.Exact, Ct)); - Assert.Equal(delivered.Count(g => g.City == "Caen"), await cities.KeyCountAsync(Ct)); + Assert.True(await cities.SeekAsync("Caen", SeekMode.Exact, Ct)); + Assert.Equal(delivered.Count(g => g.City == "Caen"), await cities.CountAtKeyAsync(Ct)); } // Floats, from a result that arrives in no order of theirs. diff --git a/tests/Vorticity.Tests/Aggregation/SpillTests.cs b/tests/Vorticity.Tests/Aggregation/SpillTests.cs index cf4340d2..7f98d79d 100644 --- a/tests/Vorticity.Tests/Aggregation/SpillTests.cs +++ b/tests/Vorticity.Tests/Aggregation/SpillTests.cs @@ -55,7 +55,7 @@ public async Task AGroupByItsBudgetCannotHoldSpillsAndEndsExact(int degree, int } Assert.Equal(Keys, read); - GroupStatistics grouping = sums.Statistics.Grouping!; + GroupMetrics grouping = sums.Metrics.Grouping!; Assert.True(grouping.SpilledParts > 0, $"no part spilled under {budget.CeilingBytes:N0} bytes of {peak:N0}"); Assert.True(grouping.SpilledBytes > 0); Assert.Equal(0, budget.ReservedBytes); @@ -172,7 +172,7 @@ async Task ReadAsync(QueryMemoryBudget budget) count++; } - Assert.True(sums.Statistics.Grouping!.SpilledParts > 0); + Assert.True(sums.Metrics.Grouping!.SpilledParts > 0); return count; } } @@ -371,7 +371,7 @@ public async Task AScratchBudgetTheSpillFitsChangesNothing() } Assert.Equal(Keys, read); - Assert.True(sums.Statistics.Grouping!.SpilledBytes > 0); + Assert.True(sums.Metrics.Grouping!.SpilledBytes > 0); Assert.Equal(0, room.ReservedBytes); Assert.Empty(Directory.EnumerateFileSystemEntries(scratch)); } @@ -408,7 +408,7 @@ public async Task AnOrderOverASpilledGroupBySortsItsGroupsInRuns(int degree) List bySum = [.. Enumerable.Range(0, Keys).OrderBy(k => expected[k]).ThenBy(k => k)]; Aggregation ordered = file.Scan().With(new ScanOptions { BatchRows = 1_024 }).GroupBy(r => r.Key).OrderBy(g => g.Sum(x => x.Value)).Select(g => g.Key); Assert.Equal(bySum, await ordered.ToListAsync(Ct)); - Assert.True(ordered.Statistics.Grouping!.SpilledParts > 0); + Assert.True(ordered.Metrics.Grouping!.SpilledParts > 0); List filtered = [.. bySum.Where(k => expected[k] > 300).Reverse().Skip(1_000).Take(50_000)]; Assert.Equal( diff --git a/tests/Vorticity.Tests/Aggregation/StreamingGroupTests.cs b/tests/Vorticity.Tests/Aggregation/StreamingGroupTests.cs index 88ff554a..6685fa07 100644 --- a/tests/Vorticity.Tests/Aggregation/StreamingGroupTests.cs +++ b/tests/Vorticity.Tests/Aggregation/StreamingGroupTests.cs @@ -145,19 +145,19 @@ public async Task ItAnswersBeforeTheEndAndATakeStopsTheRead() await using VortexFile file = await VortexFile.OpenAsync(path, Ct); Aggregation all = file.Scan().GroupBy(r => r.Hour).Select(g => g.Count()); await all.ToListAsync(Ct); - long batches = all.Statistics.Batches; + long batches = all.Metrics.Batches; // The first group closes with the first batch: the answer does not wait for the file. Aggregation first = file.Scan().GroupBy(r => r.Hour).Select(g => g.Count()); await using (IAsyncEnumerator counts = first.GetAsyncEnumerator(Ct)) { Assert.True(await counts.MoveNextAsync()); - Assert.True(batches >= 4 && first.Statistics.Batches <= 2, $"{first.Statistics.Batches} batches read for the first group of {batches}"); + Assert.True(batches >= 4 && first.Metrics.Batches <= 2, $"{first.Metrics.Batches} batches read for the first group of {batches}"); } Aggregation three = file.Scan().GroupBy(r => r.Hour).Take(3).Select(g => g.Count()); Assert.Equal(rows.Where(r => r.Hour is not null).GroupBy(r => r.Hour).OrderBy(g => g.Key).Take(3).Select(g => (long)g.Count()), await three.ToListAsync(Ct)); - Assert.True(three.Statistics.Batches <= 2, $"{three.Statistics.Batches} batches read for three groups of {batches}"); + Assert.True(three.Metrics.Batches <= 2, $"{three.Metrics.Batches} batches read for three groups of {batches}"); // A filter and an order on the key stream with it; the order the other way, under a take, // streams too, the rows read backwards. @@ -192,7 +192,7 @@ public async Task ADescendingOrderUnderATakeReadsTheRowsBackwardsAndStopsOnceSer List read = await ListAsync(last.As()); Assert.Equal(hours.Take(7).Select(g => (g.Key, (long)g.Count())), read.Select(h => (h.Hour, h.Count))); Assert.True(((AggregationQuery)last.Query).PeakGroups < hours.Length / 2, $"{((AggregationQuery)last.Query).PeakGroups} groups held of {hours.Length}"); - Assert.True(scan.Statistics.Rows < Rows / 2, $"{scan.Statistics.Rows} rows read for seven hours of {Rows}"); + Assert.True(scan.Metrics.Rows < Rows / 2, $"{scan.Metrics.Rows} rows read for seven hours of {Rows}"); Vorticity.Aggregation blocking = file.Scan().GroupBy(r => r.Hour).OrderByDescending(g => g.Key).Take(7).Select(g => (g.Key, g.Count(), g.Average(x => x.Price))); ((AggregationQuery)blocking.Query).Plan.Blocking = true; diff --git a/tests/Vorticity.Tests/Aggregation/VarianceTests.cs b/tests/Vorticity.Tests/Aggregation/VarianceTests.cs index 1c8ae02b..16c599f1 100644 --- a/tests/Vorticity.Tests/Aggregation/VarianceTests.cs +++ b/tests/Vorticity.Tests/Aggregation/VarianceTests.cs @@ -90,7 +90,7 @@ public async Task AVarianceIsTheSameBitsWhateverTheDegreeAndTheOrder() foreach (string file in new[] { path, other }) { await using VortexFile open = await session.OpenAsync(file, cancellationToken: Ct); - Whole whole = await open.Scan().AggAsync(a => (a.Variance(x => x.Celsius), a.StandardDeviation(x => x.Epoch)), Ct); + Whole whole = await open.Scan().AggregateAsync(a => (a.Variance(x => x.Celsius), a.StandardDeviation(x => x.Epoch)), Ct); List groups = await ListAsync(open.Scan() .GroupBy(r => r.Station) .OrderBy(g => g.Key) @@ -119,7 +119,7 @@ public async Task ValuesFarFromZeroKeepTheirSpread() try { await using VortexFile file = await VortexFile.OpenAsync(path, Ct); - Whole whole = await file.Scan().AggAsync(a => (a.Variance(x => x.Celsius), a.StandardDeviation(x => x.Epoch)), Ct); + Whole whole = await file.Scan().AggregateAsync(a => (a.Variance(x => x.Celsius), a.StandardDeviation(x => x.Epoch)), Ct); double expected = Deviation(rows.Select(r => r.Epoch))!.Value; Assert.True(Math.Abs(whole.EpochDeviation!.Value - expected) <= expected * 1e-12, $"{whole.EpochDeviation} against {expected}"); } diff --git a/tests/Vorticity.Tests/Aggregation/ZoneFinalityTests.cs b/tests/Vorticity.Tests/Aggregation/ZoneFinalityTests.cs index d058789e..4e7763a8 100644 --- a/tests/Vorticity.Tests/Aggregation/ZoneFinalityTests.cs +++ b/tests/Vorticity.Tests/Aggregation/ZoneFinalityTests.cs @@ -53,7 +53,7 @@ public async Task ANearlySortedKeyStreamsItsGroupsAsTheZonesProveThemFinal(int d // On one lane; on four, this file's few chunks are all in flight before the first range // is followed. - Assert.True(degree > 1 || scan.Statistics.Rows < Rows / 2, $"{scan.Statistics.Rows} rows read for five groups"); + Assert.True(degree > 1 || scan.Metrics.Rows < Rows / 2, $"{scan.Metrics.Rows} rows read for five groups"); // Down, the key's zones prove nothing: every group held, the same answers. List down = await ListAsync(file.Scan().GroupBy(r => r.At).OrderByDescending(g => g.Key).Take(3).Select(g => (g.Key, g.Count(), g.Sum(x => x.Value))).As()); diff --git a/tests/Vorticity.Tests/Aggregation/ZoneSettlingTests.cs b/tests/Vorticity.Tests/Aggregation/ZoneSettlingTests.cs index d6f2fbaf..74dca8b2 100644 --- a/tests/Vorticity.Tests/Aggregation/ZoneSettlingTests.cs +++ b/tests/Vorticity.Tests/Aggregation/ZoneSettlingTests.cs @@ -36,20 +36,20 @@ public async Task ADaysBlocksBetweenTwoMidnightsAreReadFromTheirZones() Scan streamed = file.Scan(); List days = await ListAsync(streamed.GroupBy(t => t.At.Truncate(CalendarUnit.Day)).Select(g => (g.Key, g.Count(), g.Min(t => t.Level), g.Max(t => t.At), g.Max(t => t.Price))).As()); Assert.Equal(expected, days.Select(Utc)); - Assert.Equal(straddling, streamed.Statistics.BlocksDecoded); + Assert.Equal(straddling, streamed.Metrics.BlocksDecoded); // Blocking, in the reverse order of the days, and on several lanes. Scan blocking = file.Scan(); days = await ListAsync(blocking.GroupBy(t => t.At.Truncate(CalendarUnit.Day)).OrderByDescending(g => g.Key).Select(g => (g.Key, g.Count(), g.Min(t => t.Level), g.Max(t => t.At), g.Max(t => t.Price))).As()); Assert.Equal(expected.Reverse(), days.Select(Utc)); - Assert.Equal(straddling, blocking.Statistics.BlocksDecoded); + Assert.Equal(straddling, blocking.Metrics.BlocksDecoded); await using VortexSession session = VortexSession.Create(options => options.MaxDegreeOfParallelism = 4); await using VortexFile shared = await session.OpenAsync(path, cancellationToken: Ct); Scan lanes = shared.Scan(); days = await ListAsync(lanes.GroupBy(t => t.At.Truncate(CalendarUnit.Day)).OrderByDescending(g => g.Key).Select(g => (g.Key, g.Count(), g.Min(t => t.Level), g.Max(t => t.At), g.Max(t => t.Price))).As()); Assert.Equal(expected.Reverse(), days.Select(Utc)); - Assert.Equal(straddling, lanes.Statistics.BlocksDecoded); + Assert.Equal(straddling, lanes.Metrics.BlocksDecoded); } finally { @@ -71,13 +71,13 @@ public async Task OnlyAFilterThatKeepsTheBlocksWholeLeavesThemToTheZones() Scan whole = file.Scan().Where(t => t.Level >= -500); List days = await ListAsync(whole.GroupBy(t => t.At.Truncate(CalendarUnit.Day)).Select(g => (g.Key, g.Count())).As()); Assert.Equal(rows.GroupBy(t => Day(t.At)).Select(g => (g.Key, (long)g.Count())), days.Select(d => (new DateTime(d.Day.Ticks, DateTimeKind.Utc), d.Count))); - Assert.Equal(straddling, whole.Statistics.BlocksDecoded); + Assert.Equal(straddling, whole.Metrics.BlocksDecoded); // A level above 0 is in every zone and not all of one: every block is read and filtered. Scan some = file.Scan().Where(t => t.Level > 0); days = await ListAsync(some.GroupBy(t => t.At.Truncate(CalendarUnit.Day)).Select(g => (g.Key, g.Count())).As()); Assert.Equal(rows.Where(t => t.Level > 0).GroupBy(t => Day(t.At)).Select(g => (g.Key, (long)g.Count())), days.Select(d => (new DateTime(d.Day.Ticks, DateTimeKind.Utc), d.Count))); - Assert.Equal((Rows + Block - 1) / Block, some.Statistics.BlocksDecoded); + Assert.Equal((Rows + Block - 1) / Block, some.Metrics.BlocksDecoded); // A range of days: the blocks of the days it keeps whole settle, the others are pruned. DateTime from = Day(Start).AddDays(3); @@ -85,7 +85,7 @@ public async Task OnlyAFilterThatKeepsTheBlocksWholeLeavesThemToTheZones() days = await ListAsync(range.GroupBy(t => t.At.Truncate(CalendarUnit.Day)).Select(g => (g.Key, g.Count())).As()); Tick[] kept = [.. rows.Where(t => Day(t.At) >= from && Day(t.At) < from.AddDays(2))]; Assert.Equal(kept.GroupBy(t => Day(t.At)).Select(g => (g.Key, (long)g.Count())), days.Select(d => (new DateTime(d.Day.Ticks, DateTimeKind.Utc), d.Count))); - Assert.True(range.Statistics.BlocksDecoded <= 3, $"{range.Statistics.BlocksDecoded} blocks decoded"); + Assert.True(range.Metrics.BlocksDecoded <= 3, $"{range.Metrics.BlocksDecoded} blocks decoded"); } finally { @@ -106,14 +106,14 @@ public async Task AnAggregateTheZonesDoNotHoldReadsEveryBlock() Scan summed = file.Scan(); List sums = await ListAsync(summed.GroupBy(t => t.At.Truncate(CalendarUnit.Day)).Select(g => (g.Key, g.Count(), g.Sum(t => t.Level))).As()); Assert.Equal(rows.GroupBy(t => Day(t.At)).Select(g => (g.Key, (long)g.Count(), (long)g.Sum(t => t.Level))), sums.Select(d => (new DateTime(d.Day.Ticks, DateTimeKind.Utc), d.Count, d.Levels))); - Assert.Equal((Rows + Block - 1) / Block, summed.Statistics.BlocksDecoded); + Assert.Equal((Rows + Block - 1) / Block, summed.Metrics.BlocksDecoded); Scan nullable = file.Scan(); List seen = await ListAsync(nullable.GroupBy(t => t.Seen.Truncate(CalendarUnit.Day)).Select(g => (g.Key, g.Count())).As()); Assert.Equal( rows.GroupBy(t => t.Seen is { } s ? Day(s) : (DateTime?)null).Select(g => (g.Key, (long)g.Count())).OrderBy(g => g.Key ?? DateTime.MaxValue), seen.Select(s => (s.Day is { } d ? new DateTime(d.Ticks, DateTimeKind.Utc) : (DateTime?)null, s.Count)).OrderBy(g => g.Item1 ?? DateTime.MaxValue)); - Assert.True(nullable.Statistics.BlocksDecoded > (Rows / Block) - 2, $"{nullable.Statistics.BlocksDecoded} blocks decoded"); + Assert.True(nullable.Metrics.BlocksDecoded > (Rows / Block) - 2, $"{nullable.Metrics.BlocksDecoded} blocks decoded"); } finally { diff --git a/tests/Vorticity.Tests/Api/AllocationContractTests.cs b/tests/Vorticity.Tests/Api/AllocationContractTests.cs index 6446f7f9..56437b7f 100644 --- a/tests/Vorticity.Tests/Api/AllocationContractTests.cs +++ b/tests/Vorticity.Tests/Api/AllocationContractTests.cs @@ -142,7 +142,7 @@ public async Task AggregatesAndCountsAllocateNothingBetweenTheirReads() [ ("SumAsync", all, async scan => await scan.SumAsync(r => r.Day)), ("AverageAsync", all, async scan => await scan.AverageAsync(r => r.Celsius)), - ("AggAsync", all, async scan => await scan.AggAsync(a => (a.Sum(r => r.Celsius), a.Count()))), + ("AggregateAsync", all, async scan => await scan.AggregateAsync(a => (a.Sum(r => r.Celsius), a.Count()))), ("GroupBy", all, async scan => { await foreach (CityCount group in scan.GroupBy(r => r.City).Select(g => (g.Key, g.Count())).As().ToRecordsAsync()) @@ -157,7 +157,7 @@ public async Task AggregatesAndCountsAllocateNothingBetweenTheirReads() GC.KeepAlive(group.City); } }), - ("AggAsync, filtered", all, async scan => await scan.AggAsync(a => (a.Where(r => r.Celsius > 30.0).Sum(r => r.Celsius), a.Count()))), + ("AggregateAsync, filtered", all, async scan => await scan.AggregateAsync(a => (a.Where(r => r.Celsius > 30.0).Sum(r => r.Celsius), a.Count()))), ("CountAsync", filtered, async scan => await scan.CountAsync()), ("SumAsync, filtered", filtered, async scan => await scan.SumAsync(r => r.Day)), ]; diff --git a/tests/Vorticity.Tests/Api/CorpusContractTests.cs b/tests/Vorticity.Tests/Api/CorpusContractTests.cs index 59f631b2..c733f51e 100644 --- a/tests/Vorticity.Tests/Api/CorpusContractTests.cs +++ b/tests/Vorticity.Tests/Api/CorpusContractTests.cs @@ -81,7 +81,7 @@ private static async Task Compare(string id, Vorticity.Scan scan, List f GC.KeepAlive(batch.RowCount); } - ScanStatistics statistics = scan.Statistics; + ScanMetrics statistics = scan.Metrics; if (plan.Segments != statistics.Requests || plan.BytesToRead != statistics.BytesRequested || plan.LiveBlocks != statistics.BlocksDecoded) { failures.Add(string.Create( diff --git a/tests/Vorticity.Tests/Api/PublicSurface.txt b/tests/Vorticity.Tests/Api/PublicSurface.txt index 50cf9478..d4135908 100644 --- a/tests/Vorticity.Tests/Api/PublicSurface.txt +++ b/tests/Vorticity.Tests/Api/PublicSurface.txt @@ -42,7 +42,7 @@ Vorticity: method Aggregation Skip(int) Vorticity: method Aggregation Take(int) Vorticity: method Scan As() Vorticity: method ValueTask ExplainAsync(CancellationToken = default) -Vorticity: property ScanStatistics Statistics { get; } +Vorticity: property ScanMetrics Metrics { get; } Vorticity: sealed class Vorticity.Aggregation Vorticity: method Aggregation Skip(int) Vorticity: method Aggregation Take(int) @@ -53,7 +53,7 @@ Vorticity: method Scan As() Vorticity: method ValueTask> ToListAsync(CancellationToken = default) Vorticity: method ValueTask ExplainAsync(CancellationToken = default) Vorticity: method ValueTask ToArrayAsync(CancellationToken = default) -Vorticity: property ScanStatistics Statistics { get; } +Vorticity: property ScanMetrics Metrics { get; } Vorticity: sealed class Vorticity.AlignedMemoryPool : MemoryPool Vorticity: ctor AlignedMemoryPool(int = default, long = default) Vorticity: method IMemoryOwner Rent(int = default) @@ -78,7 +78,7 @@ Vorticity: const CalendarUnit Week = 3 Vorticity: const CalendarUnit Year = 6 Vorticity: field byte value__ Vorticity: ref struct Vorticity.Column -Vorticity: method Column Canonical() +Vorticity: method Column AsCanonical() Vorticity: method DictionaryView AsDictionary() Vorticity: method RunEndView AsRunEnd() Vorticity: method T AsConstant() @@ -373,19 +373,19 @@ Vorticity: property string Path { get; init; } Vorticity: property string Reason { get; init; } Vorticity: static class Vorticity.ColumnExtensions Vorticity: method BigInteger get_Item(Column, int) -Vorticity: method Column> Storage(Column>) -Vorticity: method Column> Storage(Column>) +Vorticity: method Column> AsStorage(Column>) +Vorticity: method Column> AsStorage(Column>) Vorticity: method Column get_Elements(Column>) -Vorticity: method Column Storage(Column) -Vorticity: method Column Storage(Column) -Vorticity: method Column Storage(Column) -Vorticity: method Column Storage(Column>) -Vorticity: method Column Storage(Column) -Vorticity: method Column Storage(Column) -Vorticity: method Column Storage(Column) -Vorticity: method Column Storage(Column) -Vorticity: method Column Storage(Column) -Vorticity: method Column Storage(Column) +Vorticity: method Column AsStorage(Column) +Vorticity: method Column AsStorage(Column) +Vorticity: method Column AsStorage(Column) +Vorticity: method Column AsStorage(Column>) +Vorticity: method Column AsStorage(Column) +Vorticity: method Column AsStorage(Column) +Vorticity: method Column AsStorage(Column) +Vorticity: method Column AsStorage(Column) +Vorticity: method Column AsStorage(Column) +Vorticity: method Column AsStorage(Column) Vorticity: method DateOnly get_Item(Column, int) Vorticity: method DateTime get_Item(Column, int) Vorticity: method DateTimeOffset get_Item(Column, int) @@ -481,7 +481,7 @@ Vorticity: property Nullable Item[int] { get; } Vorticity: static class Vorticity.ColumnExtensions.$25275925CC94964FC987F5A3DDCCC144.$7F59E0AD859E1BC17E2EE4611F7CEB3D Vorticity: method void $(Column>) Vorticity: sealed class Vorticity.ColumnExtensions.$26B50BB7193039853C110A7B53B50037 -Vorticity: method Column Storage() +Vorticity: method Column AsStorage() Vorticity: method void CopyTo(Span) Vorticity: property DateTimeOffset Item[int] { get; } Vorticity: property TimeUnit Unit { get; } @@ -514,21 +514,21 @@ Vorticity: property ReadOnlySpan Bits { get; } Vorticity: static class Vorticity.ColumnExtensions.$54BEA188E0C86A3290A2D532424D9BF8.$E3EFEAEE4E24F41523F012496B295765 Vorticity: method void $(Column>) Vorticity: sealed class Vorticity.ColumnExtensions.$59DC5B9817BD0B185ED14AA61290FC18 -Vorticity: method Column Storage() +Vorticity: method Column AsStorage() Vorticity: method void CopyTo(Span) Vorticity: property TimeOnly Item[int] { get; } Vorticity: property TimeUnit Unit { get; } Vorticity: static class Vorticity.ColumnExtensions.$59DC5B9817BD0B185ED14AA61290FC18.$BDD5713B078ABE9C0CA15D729313818A Vorticity: method void $(Column) Vorticity: sealed class Vorticity.ColumnExtensions.$5DA42E352C660F31A814324CA3D6710E -Vorticity: method Column Storage() +Vorticity: method Column AsStorage() Vorticity: method void CopyTo(Span) Vorticity: property decimal Item[int] { get; } Vorticity: property int Scale { get; } Vorticity: static class Vorticity.ColumnExtensions.$5DA42E352C660F31A814324CA3D6710E.$FD7D0D75BE560C0BA5E8C0633258DD46 Vorticity: method void $(Column) Vorticity: sealed class Vorticity.ColumnExtensions.$6DA0AC3EDE4247869BB52ECFB4321B2A -Vorticity: method Column Storage() +Vorticity: method Column AsStorage() Vorticity: method void CopyTo(Span) Vorticity: property DateTime Item[int] { get; } Vorticity: property TimeUnit Unit { get; } @@ -540,40 +540,40 @@ Vorticity: property Nullable Item[int] { get; } Vorticity: static class Vorticity.ColumnExtensions.$739833C638BC2E1FDA8E7081F900ACDC.$D9A0C2862A95B879A345484410662F3C Vorticity: method void $(Column>) Vorticity: sealed class Vorticity.ColumnExtensions.$74D2991A37BBB0990C1D079414CF5377 -Vorticity: method Column Storage() +Vorticity: method Column AsStorage() Vorticity: method void CopyTo(Span>) Vorticity: property Nullable Item[int] { get; } Vorticity: property int Scale { get; } Vorticity: static class Vorticity.ColumnExtensions.$74D2991A37BBB0990C1D079414CF5377.$3853BBD2552A86D0075A37D9D5476488 Vorticity: method void $(Column>) Vorticity: sealed class Vorticity.ColumnExtensions.$7B643A86D38F2A5E728E286578F56E3A -Vorticity: method Column Storage() +Vorticity: method Column AsStorage() Vorticity: method void CopyTo(Span) Vorticity: property Int128 Item[int] { get; } Vorticity: property ReadOnlySpan Values { get; } Vorticity: static class Vorticity.ColumnExtensions.$7B643A86D38F2A5E728E286578F56E3A.$DC8C0C467D235D77257469297915BE5F Vorticity: method void $(Column) Vorticity: sealed class Vorticity.ColumnExtensions.$7FFF222DD38BFECFF8FBB10F27326BD4 -Vorticity: method Column> Storage() +Vorticity: method Column> AsStorage() Vorticity: method void CopyTo(Span>) Vorticity: property Nullable Item[int] { get; } Vorticity: property TimeUnit Unit { get; } Vorticity: static class Vorticity.ColumnExtensions.$7FFF222DD38BFECFF8FBB10F27326BD4.$23C3BB20829EA3B0410FDDB57A64B910 Vorticity: method void $(Column>) Vorticity: sealed class Vorticity.ColumnExtensions.$8F60AB3A3B8DB97391BCB982E98D1533 -Vorticity: method Column Storage() +Vorticity: method Column AsStorage() Vorticity: method void CopyTo(Span) Vorticity: property DateOnly Item[int] { get; } Vorticity: static class Vorticity.ColumnExtensions.$8F60AB3A3B8DB97391BCB982E98D1533.$8F7A59011FB521B2FE076679AB2061D0 Vorticity: method void $(Column) Vorticity: sealed class Vorticity.ColumnExtensions.$97B771CAE5245D13938FAD7A6A26F2B6 -Vorticity: method Column> Storage() +Vorticity: method Column> AsStorage() Vorticity: method void CopyTo(Span>) Vorticity: property Nullable Item[int] { get; } Vorticity: static class Vorticity.ColumnExtensions.$97B771CAE5245D13938FAD7A6A26F2B6.$DAC1F4EDDD9A0EF402004F0F2DB7D3F5 Vorticity: method void $(Column>) Vorticity: sealed class Vorticity.ColumnExtensions.$9D9F317AD1D9E24D853D8E25C38256FD -Vorticity: method Column Storage() +Vorticity: method Column AsStorage() Vorticity: method void CopyTo(Span) Vorticity: property TimeSpan Item[int] { get; } Vorticity: static class Vorticity.ColumnExtensions.$9D9F317AD1D9E24D853D8E25C38256FD.$41B8854DA8D375B9F0FC732133E4086F @@ -622,13 +622,13 @@ Vorticity: property UInt128 Item[int] { get; } Vorticity: static class Vorticity.ColumnExtensions.$E42392E2B31F9560973FC5BC31A2D718.$1D4FE02EA1564CBB61455E2EB667E5CE Vorticity: method void $(Column) Vorticity: sealed class Vorticity.ColumnExtensions.$FD0E6A78E6F80A864C64A1CE47A1CE69 -Vorticity: method Column Storage() +Vorticity: method Column AsStorage() Vorticity: method void CopyTo(Span) Vorticity: property BigInteger Item[int] { get; } Vorticity: static class Vorticity.ColumnExtensions.$FD0E6A78E6F80A864C64A1CE47A1CE69.$79B973E48551687E869571A021932A9C Vorticity: method void $(Column) Vorticity: sealed class Vorticity.ColumnExtensions.$FFA9D64B944DCB26CA7833C1AE6204D8 -Vorticity: method Column Storage() +Vorticity: method Column AsStorage() Vorticity: method void CopyTo(Span) Vorticity: property VortexDecimal Item[int] { get; } Vorticity: static class Vorticity.ColumnExtensions.$FFA9D64B944DCB26CA7833C1AE6204D8.$521280D8171F45A8C761FA8EE04F07D9 @@ -850,10 +850,10 @@ Vorticity: method void AppendNull() Vorticity: static class Vorticity.ExtensionColumnBuilderExtensions.$5C078CE680B13FBC6A28B66BA7C9B92E<$T0>.$563C57E0DB50454082D1ADD1217F0DC7 Vorticity: method void $(ColumnBuilder) Vorticity: static class Vorticity.ExtensionColumnExtensions -Vorticity: method Column Storage(Column) +Vorticity: method Column AsStorage(Column) Vorticity: method T get_Item(Column, int) Vorticity: sealed class Vorticity.ExtensionColumnExtensions.$E9C85CFA12EF808D3A0CF7D106ED4D24<$T0> -Vorticity: method Column Storage() +Vorticity: method Column AsStorage() Vorticity: property $T0 Item[int] { get; } Vorticity: static class Vorticity.ExtensionColumnExtensions.$E9C85CFA12EF808D3A0CF7D106ED4D24<$T0>.$98A11D5B73E3D7F5A7DF3816C44FD841 Vorticity: method void $(Column) @@ -926,9 +926,41 @@ Vorticity: method bool op_Inequality(GroupKeyPlan, GroupKeyPlan) Vorticity: method int GetHashCode() Vorticity: method string ToString() Vorticity: method void Deconstruct(out string, out bool, out Nullable) -Vorticity: property Nullable Values { get; init; } +Vorticity: property Nullable ValueRange { get; init; } Vorticity: property bool Sorted { get; init; } Vorticity: property string Column { get; init; } +Vorticity: sealed class Vorticity.GroupMetrics : IEquatable +Vorticity: ctor GroupMetrics(long, long, long, int, int, bool, long, long, long, long, long, int, int, int, long, long, long, long, TimeSpan) +Vorticity: method GroupMetrics $() +Vorticity: method bool Equals(GroupMetrics) +Vorticity: method bool Equals(object) +Vorticity: method bool op_Equality(GroupMetrics, GroupMetrics) +Vorticity: method bool op_Inequality(GroupMetrics, GroupMetrics) +Vorticity: method int GetHashCode() +Vorticity: method string ToString() +Vorticity: method void Deconstruct(out long, out long, out long, out int, out int, out bool, out long, out long, out long, out long, out long, out int, out int, out int, out long, out long, out long, out long, out TimeSpan) +Vorticity: property GroupCoreReason CoreReason { get; init; } +Vorticity: property TimeSpan TimeToFirstBatch { get; init; } +Vorticity: property bool Core { get; init; } +Vorticity: property int Lanes { get; init; } +Vorticity: property int MergeParts { get; init; } +Vorticity: property int SpilledParts { get; init; } +Vorticity: property int SpilledRuns { get; init; } +Vorticity: property int TableSplits { get; init; } +Vorticity: property int Tables { get; init; } +Vorticity: property long Bursts { get; init; } +Vorticity: property long BypassedRows { get; init; } +Vorticity: property long CacheEvictions { get; init; } +Vorticity: property long Groups { get; init; } +Vorticity: property long KeyBlocksByCode { get; init; } +Vorticity: property long KeyBlocksByRange { get; init; } +Vorticity: property long KeyBlocksHashed { get; init; } +Vorticity: property long PeakBytes { get; init; } +Vorticity: property long PeakGroups { get; init; } +Vorticity: property long PendingBytes { get; init; } +Vorticity: property long ReloadedBytes { get; init; } +Vorticity: property long SpilledBytes { get; init; } +Vorticity: property long TurnedAfterRows { get; init; } Vorticity: enum Vorticity.GroupOrdering Vorticity: const GroupOrdering None = 0 Vorticity: const GroupOrdering Sort = 3 @@ -958,38 +990,6 @@ Vorticity: property int ChosenRows { get; init; } Vorticity: property int Degree { get; init; } Vorticity: property string NotStreaming { get; init; } Vorticity: property string RowFilter { get; init; } -Vorticity: sealed class Vorticity.GroupStatistics : IEquatable -Vorticity: ctor GroupStatistics(long, long, long, int, int, bool, long, long, long, long, long, int, int, int, long, long, long, long, TimeSpan) -Vorticity: method GroupStatistics $() -Vorticity: method bool Equals(GroupStatistics) -Vorticity: method bool Equals(object) -Vorticity: method bool op_Equality(GroupStatistics, GroupStatistics) -Vorticity: method bool op_Inequality(GroupStatistics, GroupStatistics) -Vorticity: method int GetHashCode() -Vorticity: method string ToString() -Vorticity: method void Deconstruct(out long, out long, out long, out int, out int, out bool, out long, out long, out long, out long, out long, out int, out int, out int, out long, out long, out long, out long, out TimeSpan) -Vorticity: property GroupCoreReason CoreReason { get; init; } -Vorticity: property TimeSpan TimeToFirstBatch { get; init; } -Vorticity: property bool Core { get; init; } -Vorticity: property int Lanes { get; init; } -Vorticity: property int MergeParts { get; init; } -Vorticity: property int SpilledParts { get; init; } -Vorticity: property int SpilledRuns { get; init; } -Vorticity: property int TableSplits { get; init; } -Vorticity: property int Tables { get; init; } -Vorticity: property long Bursts { get; init; } -Vorticity: property long BypassedRows { get; init; } -Vorticity: property long CacheEvictions { get; init; } -Vorticity: property long Groups { get; init; } -Vorticity: property long KeyBlocksByCode { get; init; } -Vorticity: property long KeyBlocksByRange { get; init; } -Vorticity: property long KeyBlocksHashed { get; init; } -Vorticity: property long PeakBytes { get; init; } -Vorticity: property long PeakGroups { get; init; } -Vorticity: property long PendingBytes { get; init; } -Vorticity: property long ReloadedBytes { get; init; } -Vorticity: property long SpilledBytes { get; init; } -Vorticity: property long TurnedAfterRows { get; init; } Vorticity: sealed class Vorticity.GroupedScan Vorticity: method Aggregation Select(Func, ITuple>) Vorticity: method Aggregation Select(Func, Sym>) @@ -1118,13 +1118,13 @@ Vorticity: sealed class Vorticity.KeyCursor : IAsyncDisposable Vorticity: method ValueTask DisposeAsync() Vorticity: method ValueTask NextAsync(CancellationToken = default) Vorticity: method ValueTask NextKeyAsync(CancellationToken = default) -Vorticity: method ValueTask PrevAsync(CancellationToken = default) -Vorticity: method ValueTask PrevKeyAsync(CancellationToken = default) -Vorticity: method ValueTask SeekAsync(TKey, SeekOp, CancellationToken = default) +Vorticity: method ValueTask PreviousAsync(CancellationToken = default) +Vorticity: method ValueTask PreviousKeyAsync(CancellationToken = default) +Vorticity: method ValueTask SeekAsync(TKey, SeekMode, CancellationToken = default) Vorticity: method ValueTask SeekFirstAsync(CancellationToken = default) Vorticity: method ValueTask SeekLastAsync(CancellationToken = default) Vorticity: method ValueTask SeekRankAsync(long, CancellationToken = default) -Vorticity: method ValueTask KeyCountAsync(CancellationToken = default) +Vorticity: method ValueTask CountAtKeyAsync(CancellationToken = default) Vorticity: method ValueTask RankAsync(TKey, CancellationToken = default) Vorticity: property TKey Key { get; } Vorticity: property bool HasRows { get; } @@ -1243,7 +1243,7 @@ Vorticity: method Projection Skip(int) Vorticity: method Projection Take(int) Vorticity: method Scan As() Vorticity: method ValueTask ExplainAsync(CancellationToken = default) -Vorticity: property ScanStatistics Statistics { get; } +Vorticity: property ScanMetrics Metrics { get; } Vorticity: sealed class Vorticity.Projection Vorticity: method Aggregation Distinct() Vorticity: method IAsyncEnumerable ToValuesAsync(CancellationToken = default) @@ -1255,7 +1255,7 @@ Vorticity: method Scan As() Vorticity: method ValueTask> ToListAsync(CancellationToken = default) Vorticity: method ValueTask ExplainAsync(CancellationToken = default) Vorticity: method ValueTask ToArrayAsync(CancellationToken = default) -Vorticity: property ScanStatistics Statistics { get; } +Vorticity: property ScanMetrics Metrics { get; } Vorticity: sealed class Vorticity.PruningStep : IEquatable Vorticity: ctor PruningStep(string, int, int, long) Vorticity: method PruningStep $() @@ -1313,7 +1313,7 @@ Vorticity: method Scan.AsyncEnumerator GetAsyncEnumerator(CancellationToken Vorticity: method ValueTask ExplainAsync(CancellationToken = default) Vorticity: method ValueTask AnyAsync(CancellationToken = default) Vorticity: method ValueTask CountAsync(CancellationToken = default) -Vorticity: property ScanStatistics Statistics { get; } +Vorticity: property ScanMetrics Metrics { get; } Vorticity: sealed class Vorticity.Scan.AsyncEnumerator : IAsyncDisposable Vorticity: method ValueTask DisposeAsync() Vorticity: method ValueTask MoveNextAsync() @@ -1337,12 +1337,12 @@ Vorticity: method Scan.AsyncEnumerator GetAsyncEnumerator( Vorticity: method ValueTask> AverageAsync(Func, Sym>>, CancellationToken = default) Vorticity: method ValueTask> AverageAsync(Func, Sym>, CancellationToken = default) Vorticity: method ValueTask ExplainAsync(CancellationToken = default) -Vorticity: method ValueTask AggAsync(Func, Sym>, CancellationToken = default) +Vorticity: method ValueTask AggregateAsync(Func, Sym>, CancellationToken = default) Vorticity: method ValueTask MaxAsync(Func, Sym>, CancellationToken = default) Vorticity: method ValueTask MinAsync(Func, Sym>, CancellationToken = default) Vorticity: method ValueTask SumAsync(Func, Sym>>, CancellationToken = default) Vorticity: method ValueTask SumAsync(Func, Sym>, CancellationToken = default) -Vorticity: method ValueTask AggAsync(Func, ITuple>, CancellationToken = default) +Vorticity: method ValueTask AggregateAsync(Func, ITuple>, CancellationToken = default) Vorticity: method ValueTask AggregateAsync(Func, Sym>>, CancellationToken = default) Vorticity: method ValueTask AggregateAsync(Func, Sym>, CancellationToken = default) Vorticity: method ValueTask AnyAsync(CancellationToken = default) @@ -1364,11 +1364,28 @@ Vorticity: method ValueTask SumAsync(Func, Sym SumAsync(Func, Sym>, CancellationToken = default) Vorticity: method ValueTask SumAsync(Func, Sym>, CancellationToken = default) Vorticity: method ValueTask SumAsync(Func, Sym>, CancellationToken = default) -Vorticity: property ScanStatistics Statistics { get; } +Vorticity: property ScanMetrics Metrics { get; } Vorticity: sealed class Vorticity.Scan.AsyncEnumerator : IAsyncDisposable Vorticity: method ValueTask DisposeAsync() Vorticity: method ValueTask MoveNextAsync() Vorticity: property Columns Current { get; } +Vorticity: struct Vorticity.ScanMetrics : IEquatable +Vorticity: ctor ScanMetrics(long, long, long, long, long, long, long) +Vorticity: method bool Equals(ScanMetrics) +Vorticity: method bool Equals(object) +Vorticity: method bool op_Equality(ScanMetrics, ScanMetrics) +Vorticity: method bool op_Inequality(ScanMetrics, ScanMetrics) +Vorticity: method int GetHashCode() +Vorticity: method string ToString() +Vorticity: method void Deconstruct(out long, out long, out long, out long, out long, out long, out long) +Vorticity: property GroupMetrics Grouping { get; init; } +Vorticity: property long Batches { get; init; } +Vorticity: property long BlocksDecoded { get; init; } +Vorticity: property long BlocksPruned { get; init; } +Vorticity: property long BytesRequested { get; init; } +Vorticity: property long CacheHits { get; init; } +Vorticity: property long Requests { get; init; } +Vorticity: property long Rows { get; init; } Vorticity: sealed class Vorticity.ScanOptions : IEquatable Vorticity: ctor ScanOptions() Vorticity: method ScanOptions $() @@ -1379,8 +1396,8 @@ Vorticity: method bool op_Inequality(ScanOptions, ScanOptions) Vorticity: method int GetHashCode() Vorticity: method string ToString() Vorticity: property bool Compact { get; init; } -Vorticity: property bool Pruning { get; init; } Vorticity: property bool UseIndexes { get; init; } +Vorticity: property bool UseStatistics { get; init; } Vorticity: property int BatchRows { get; init; } Vorticity: property int DegreeOfParallelism { get; init; } Vorticity: property int Prefetch { get; init; } @@ -1404,41 +1421,24 @@ Vorticity: property int LiveBlocks { get; init; } Vorticity: property int Segments { get; init; } Vorticity: property long BytesToRead { get; init; } Vorticity: property long Rows { get; init; } -Vorticity: struct Vorticity.ScanStatistics : IEquatable -Vorticity: ctor ScanStatistics(long, long, long, long, long, long, long) -Vorticity: method bool Equals(ScanStatistics) -Vorticity: method bool Equals(object) -Vorticity: method bool op_Equality(ScanStatistics, ScanStatistics) -Vorticity: method bool op_Inequality(ScanStatistics, ScanStatistics) -Vorticity: method int GetHashCode() -Vorticity: method string ToString() -Vorticity: method void Deconstruct(out long, out long, out long, out long, out long, out long, out long) -Vorticity: property GroupStatistics Grouping { get; init; } -Vorticity: property long Batches { get; init; } -Vorticity: property long BlocksDecoded { get; init; } -Vorticity: property long BlocksPruned { get; init; } -Vorticity: property long BytesRequested { get; init; } -Vorticity: property long CacheHits { get; init; } -Vorticity: property long Requests { get; init; } -Vorticity: property long Rows { get; init; } Vorticity: sealed class Vorticity.ScratchBudget Vorticity: ctor ScratchBudget(long) Vorticity: property long CeilingBytes { get; } Vorticity: property long ReservedBytes { get; } -Vorticity: enum Vorticity.SeekOp -Vorticity: const SeekOp After = 2 -Vorticity: const SeekOp AtOrAfter = 1 -Vorticity: const SeekOp AtOrBefore = 3 -Vorticity: const SeekOp Before = 4 -Vorticity: const SeekOp Exact = 0 +Vorticity: enum Vorticity.SeekMode +Vorticity: const SeekMode After = 2 +Vorticity: const SeekMode AtOrAfter = 1 +Vorticity: const SeekMode AtOrBefore = 3 +Vorticity: const SeekMode Before = 4 +Vorticity: const SeekMode Exact = 0 Vorticity: field byte value__ Vorticity: sealed class Vorticity.SegmentCache Vorticity: ctor SegmentCache(long) Vorticity: method void Clear() -Vorticity: property long Capacity { get; } +Vorticity: property long CapacityBytes { get; } +Vorticity: property long HeldBytes { get; } Vorticity: property long Hits { get; } Vorticity: property long Misses { get; } -Vorticity: property long Size { get; } Vorticity: ref struct Vorticity.Selection Vorticity: method Selection.Enumerator GetEnumerator() Vorticity: method bool Contains(int) @@ -1709,7 +1709,7 @@ Vorticity: method ValueTask BuildFragmentAsync(VortexFile, In Vorticity: method ValueTask BuildFragmentAsync(string, IndexPolicy, Nullable = default, CancellationToken = default) Vorticity: static class Vorticity.VortexFileRepair Vorticity: method ValueTask RepairAsync(string, CancellationToken = default) -Vorticity: method ValueTask ValidLengthAsync(string, CancellationToken = default) +Vorticity: method ValueTask GetValidLengthAsync(string, CancellationToken = default) Vorticity: sealed class Vorticity.VortexFileStatistics Vorticity: property FieldStatistics Item[int] { get; } Vorticity: property int Count { get; } @@ -1807,7 +1807,7 @@ Vorticity: const int MaxLayoutDepth = 64 Vorticity: const int MaxMetadataKeyLength = 64 Vorticity: const int MaxMetadataSegments = 16 Vorticity: const int MaxPostscriptSize = 65527 -Vorticity: const long DefaultMaxDecompressedSize = 268435456 +Vorticity: const long DefaultMaxDecompressedBytes = 268435456 Vorticity: sealed class Vorticity.VortexMemoryException : VortexException Vorticity: ctor VortexMemoryException(string) Vorticity: ctor VortexMemoryException(string, Exception) @@ -1825,13 +1825,13 @@ Vorticity: method int GetHashCode() Vorticity: method string ToString() Vorticity: property ImmutableArray IndexFragments { get; init; } Vorticity: property Nullable Length { get; init; } -Vorticity: property Nullable MaxBatchDecompressedSize { get; init; } +Vorticity: property Nullable MaxBatchDecompressedBytes { get; init; } Vorticity: property VortexSchema Schema { get; init; } Vorticity: property VortexTornTailPolicy TornTail { get; init; } Vorticity: property bool IndexFragmentsNeedIdentity { get; init; } Vorticity: property bool VerifyStatistics { get; init; } -Vorticity: property int InitialReadSize { get; init; } -Vorticity: property long MaxDecompressedSize { get; init; } +Vorticity: property int InitialReadBytes { get; init; } +Vorticity: property long MaxDecompressedBytes { get; init; } Vorticity: sealed class Vorticity.VortexRecordAttribute : Attribute Vorticity: ctor VortexRecordAttribute() Vorticity: sealed class Vorticity.VortexRepairResult : IEquatable @@ -1854,10 +1854,10 @@ Vorticity: method bool Equals(VortexSchema) Vorticity: method bool Equals(object) Vorticity: method bool TryFormat(Span, out int, ReadOnlySpan, IFormatProvider) Vorticity: method bool TryFormat(Span, out int, ReadOnlySpan, IFormatProvider) -Vorticity: method bool TryGetField(ReadOnlySpan, out int) Vorticity: method bool op_Equality(VortexSchema, VortexSchema) Vorticity: method bool op_Inequality(VortexSchema, VortexSchema) Vorticity: method int GetHashCode() +Vorticity: method int IndexOf(ReadOnlySpan) Vorticity: method int IndexOf(string) Vorticity: method string ToString() Vorticity: method string ToString(string, IFormatProvider) @@ -1885,7 +1885,7 @@ Vorticity: method ValueTask AdviseAsync(VortexFile, Encoding Vorticity: method ValueTask AdviseAsync(ReadOnlyMemory, EncodingGoal = default, CancellationToken = default) Vorticity: method ValueTask OpenAsync(ISegmentSource, VortexOpenOptions = default, CancellationToken = default) Vorticity: method ValueTask OpenAsync(string, VortexOpenOptions = default, CancellationToken = default) -Vorticity: method ValueTask AppendAsync(string, VortexWriteOptions = default, CancellationToken = default) +Vorticity: method ValueTask OpenWriterAsync(string, VortexWriteOptions = default, CancellationToken = default) Vorticity: method VortexFileWriter CreateWriter(PipeWriter, VortexSchema, VortexWriteOptions = default) Vorticity: method VortexFileWriter CreateWriter(string, VortexSchema, VortexWriteOptions = default) Vorticity: method VortexFileWriter CreateWriter(string, VortexWriteOptions = default) @@ -2006,7 +2006,7 @@ Vorticity: property IndexPolicy Indexes { get; init; } Vorticity: property Nullable Identity { get; init; } Vorticity: property VortexEdition TargetEdition { get; init; } Vorticity: property bool Durable { get; init; } -Vorticity: property bool Statistics { get; init; } +Vorticity: property bool WriteStatistics { get; init; } Vorticity: property int BlockRows { get; init; } Vorticity: property int ChunkTargetBytes { get; init; } Vorticity: property int DegreeOfParallelism { get; init; } @@ -2053,7 +2053,7 @@ Vorticity.Dataset: method int GetHashCode() Vorticity.Dataset: method string ToString() Vorticity.Dataset: property CompactionStyle Style { get; init; } Vorticity.Dataset: property CompactionTrigger Trigger { get; init; } -Vorticity.Dataset: property ImmutableArray Objects { get; init; } +Vorticity.Dataset: property ImmutableArray ObjectKeys { get; init; } Vorticity.Dataset: property int FromLevel { get; init; } Vorticity.Dataset: property int ToLevel { get; init; } Vorticity.Dataset: property long Bytes { get; init; } @@ -2133,7 +2133,7 @@ Vorticity.Dataset: property IProgress Progress { get; init Vorticity.Dataset: property TimeProvider TimeProvider { get; init; } Vorticity.Dataset: property TimeSpan Idle { get; init; } Vorticity.Dataset: property TimeSpan LeaseSpan { get; init; } -Vorticity.Dataset: property bool Leases { get; init; } +Vorticity.Dataset: property bool UseLeases { get; init; } Vorticity.Dataset: property int Loop { get; init; } Vorticity.Dataset: property int Loops { get; init; } Vorticity.Dataset: property long BytesPerSecond { get; init; } @@ -2374,7 +2374,7 @@ Vorticity.Dataset: property TimeSpan Window { get; init; } Vorticity.Dataset: property long PagesRead { get; init; } Vorticity.Dataset: property ulong Latest { get; init; } Vorticity.Dataset: sealed class Vorticity.Dataset.VortexDataset : IAsyncDisposable -Vorticity.Dataset: method IAsyncEnumerable ObjectsAsync(CancellationToken = default) +Vorticity.Dataset: method IAsyncEnumerable ListObjectsAsync(CancellationToken = default) Vorticity.Dataset: method ObjectDraft StartObject() Vorticity.Dataset: method Scan Scan(ReadOnlySpan) Vorticity.Dataset: method Scan Scan() @@ -2384,8 +2384,8 @@ Vorticity.Dataset: method ValueTask PlanCompactionAsync(Comp Vorticity.Dataset: method ValueTask CompactAsync(CompactionOptions = default, CancellationToken = default) Vorticity.Dataset: method ValueTask VerifyAsync(Nullable = default, CancellationToken = default) Vorticity.Dataset: method ValueTask> RemoveTornCommitAsync(IObjectStore, CancellationToken = default) -Vorticity.Dataset: method ValueTask RemoveAsync(IReadOnlyList, CancellationToken = default) -Vorticity.Dataset: method ValueTask ReplaceAsync(IReadOnlyList, IReadOnlyList, CancellationToken = default) +Vorticity.Dataset: method ValueTask RemoveObjectsAsync(IReadOnlyList, CancellationToken = default) +Vorticity.Dataset: method ValueTask ReplaceObjectsAsync(IReadOnlyList, IReadOnlyList, CancellationToken = default) Vorticity.Dataset: method ValueTask DeleteAsync(VortexExpr, CancellationToken = default) Vorticity.Dataset: method ValueTask DeleteAsync(Func, Predicate>, CancellationToken = default) Vorticity.Dataset: method ValueTask UpdateAsync(Func, Predicate>, Func, CancellationToken = default) @@ -2399,7 +2399,7 @@ Vorticity.Dataset: method ValueTask ImportAsync(string, CancellationT Vorticity.Dataset: method ValueTask RefreshAsync(CancellationToken = default) Vorticity.Dataset: method bool MayMatch(Func, Predicate>) Vorticity.Dataset: property Exception LastInlineCompactionFailure { get; } -Vorticity.Dataset: property IReadOnlyList ClusteringKeyPaths { get; } +Vorticity.Dataset: property IReadOnlyList ClusteringKey { get; } Vorticity.Dataset: property VortexSchema Schema { get; } Vorticity.Dataset: property VortexSession Session { get; } Vorticity.Dataset: property long Lag { get; } diff --git a/tests/Vorticity.Tests/Api/PublicSurfaceTests.cs b/tests/Vorticity.Tests/Api/PublicSurfaceTests.cs index 508a630f..ce05d977 100644 --- a/tests/Vorticity.Tests/Api/PublicSurfaceTests.cs +++ b/tests/Vorticity.Tests/Api/PublicSurfaceTests.cs @@ -113,7 +113,7 @@ public void AsyncNamesWhatRunsAndNoBuilderCarriesIt() /// The types a member returns when it describes a query rather than running it. private static readonly string[] Builders = ["Scan", "GroupedScan", "OrderedGroupedScan", "Aggregation", "Projection", "KeyCursorBuilder"]; - /// The return type and the name of a rendered method line: method ValueTask<T1> AggAsync<T1>(…). + /// The return type and the name of a rendered method line: method ValueTask<T1> AggregateAsync<T1>(…). private static bool TryMethod(string line, out string returned, out string name) { returned = name = string.Empty; diff --git a/tests/Vorticity.Tests/Api/ReadContractTests.cs b/tests/Vorticity.Tests/Api/ReadContractTests.cs index 32b4c63d..25ed7ecf 100644 --- a/tests/Vorticity.Tests/Api/ReadContractTests.cs +++ b/tests/Vorticity.Tests/Api/ReadContractTests.cs @@ -50,7 +50,7 @@ public async Task EveryValueIsDecodedOnceWhateverTheLanes(int prefetch, int degr } Assert.Equal(ContractFile.Rows, rows); - Assert.Equal(3L * ContractFile.Rows, scan.Metrics.ValuesDecoded); + Assert.Equal(3L * ContractFile.Rows, scan.Counters.ValuesDecoded); // A parallel aggregation runs one scan per partition, cut at chunk boundaries. Scan grouped = file.Scan().With(options); @@ -61,7 +61,7 @@ public async Task EveryValueIsDecodedOnceWhateverTheLanes(int prefetch, int degr } Assert.Equal(ContractFile.Rows, counted); - Assert.Equal(ContractFile.Rows, grouped.Metrics.ValuesDecoded); + Assert.Equal(ContractFile.Rows, grouped.Counters.ValuesDecoded); } [Theory] @@ -77,7 +77,7 @@ public async Task EachSegmentIsRequestedOnceAndTheSourceSeesWhatTheStatisticsCou // decoded, so a scan after a plan on the same file would not ask for them again. That file // is opened as this one is, so that its tail holds what this one's holds. int planned = 0; - (ScanPlan plan, ScanStatistics statistics) = await RunAsync(file, planning, query, () => planned = source.Ranges.Length); + (ScanPlan plan, ScanMetrics statistics) = await RunAsync(file, planning, query, () => planned = source.Ranges.Length); SegmentRange[] asked = source.Ranges[planned..]; Assert.True(asked.Length > 0, $"{query}: the scan asked for nothing, so there is nothing to count"); @@ -124,10 +124,10 @@ public async Task AnEqualityOnSortedRunsRequestsEachSegmentOnce() "the plan consults no locating index: " + string.Join(", ", plan.Pruning) + "; indexes: " + string.Join(", ", (await file.GetIndexesAsync(TestContext.Current.CancellationToken)).Select(index => $"{index.Column} {index.Kind}"))); Assert.Equal(asked.Length, asked.Distinct().Count()); - Assert.Equal(plan.Segments, scan.Statistics.Requests); - Assert.Equal(scan.Statistics.Requests, asked.Length); - Assert.Equal(plan.BytesToRead, scan.Statistics.BytesRequested); - Assert.Equal(scan.Statistics.BytesRequested, asked.Sum(range => (long)range.Length)); + Assert.Equal(plan.Segments, scan.Metrics.Requests); + Assert.Equal(scan.Metrics.Requests, asked.Length); + Assert.Equal(plan.BytesToRead, scan.Metrics.BytesRequested); + Assert.Equal(scan.Metrics.BytesRequested, asked.Sum(range => (long)range.Length)); } [Theory] @@ -137,7 +137,7 @@ public async Task WhatAScanRequestsIsWhatItsPlanSaid(string query) await using VortexFile file = await VortexFile.OpenAsync(await ContractFile.PathAsync(), TestContext.Current.CancellationToken); await using VortexFile planning = await VortexFile.OpenAsync(await ContractFile.PathAsync(), TestContext.Current.CancellationToken); - (ScanPlan plan, ScanStatistics statistics) = await RunAsync(file, planning, query); + (ScanPlan plan, ScanMetrics statistics) = await RunAsync(file, planning, query); Assert.Equal(plan.Segments, statistics.Requests); Assert.Equal(plan.BytesToRead, statistics.BytesRequested); @@ -149,7 +149,7 @@ public async Task APrunedBlockIsNotDecoded(string query) { await using VortexFile file = await VortexFile.OpenAsync(await ContractFile.PathAsync(), TestContext.Current.CancellationToken); - (ScanPlan plan, ScanStatistics statistics) = await RunAsync(file, file, query); + (ScanPlan plan, ScanMetrics statistics) = await RunAsync(file, file, query); Assert.Equal(plan.LiveBlocks, statistics.BlocksDecoded); Assert.Equal(plan.Blocks, plan.LiveBlocks + plan.Pruning.Sum(step => step.BlocksPruned)); @@ -178,7 +178,7 @@ public async Task AQuestionTheStatisticsAnswerReadsNothing() Scan count = file.Scan(); Assert.Equal(ContractFile.Rows, await count.CountAsync(TestContext.Current.CancellationToken)); - Assert.Equal(0, count.Statistics.Requests); + Assert.Equal(0, count.Metrics.Requests); // A filter the file statistics refute: neither its plan nor its count reads a zone map. ScanPlan refuted = await file.Scan().Where(r => r.Day > 1_000_000).ExplainAsync(TestContext.Current.CancellationToken); @@ -189,23 +189,23 @@ public async Task AQuestionTheStatisticsAnswerReadsNothing() Scan none = file.Scan().Where(r => r.Day > 1_000_000); Assert.Equal(0, await none.CountAsync(TestContext.Current.CancellationToken)); - Assert.Equal(0, none.Statistics.Requests); + Assert.Equal(0, none.Metrics.Requests); Scan min = file.Scan(); Assert.Equal(0, await min.MinAsync(r => r.Day, TestContext.Current.CancellationToken)); - Assert.Equal(0, min.Statistics.Requests); + Assert.Equal(0, min.Metrics.Requests); Scan max = file.Scan(); Assert.Equal((ContractFile.Rows - 1) / 1_000, await max.MaxAsync(r => r.Day, TestContext.Current.CancellationToken)); - Assert.Equal(0, max.Statistics.Requests); + Assert.Equal(0, max.Metrics.Requests); Scan coldest = file.Scan(); Assert.Equal(10.1, await coldest.MinAsync(r => r.Celsius, TestContext.Current.CancellationToken)); - Assert.Equal(0, coldest.Statistics.Requests); + Assert.Equal(0, coldest.Metrics.Requests); Scan hottest = file.Scan(); Assert.Equal(49.9, await hottest.MaxAsync(r => r.Celsius, TestContext.Current.CancellationToken)); - Assert.Equal(0, hottest.Statistics.Requests); + Assert.Equal(0, hottest.Metrics.Requests); Assert.Equal(opened, source.Ranges.Length); } @@ -231,8 +231,8 @@ public async Task ADictionaryOrRunEndAggregateDoesNotDecode() } Assert.Equal(ContractFile.Rows, grouped); - Assert.True(groups.Statistics.Batches > 0, "the groups were answered without reading a batch"); - Assert.Equal(0, groups.Statistics.BlocksDecoded); + Assert.True(groups.Metrics.Batches > 0, "the groups were answered without reading a batch"); + Assert.Equal(0, groups.Metrics.BlocksDecoded); Scan sum = file.Scan(); long expected = 0; @@ -242,8 +242,8 @@ public async Task ADictionaryOrRunEndAggregateDoesNotDecode() } Assert.Equal(expected, await sum.SumAsync(r => r.Day, TestContext.Current.CancellationToken)); - Assert.True(sum.Statistics.Batches > 0, "the sum was answered without reading a batch"); - Assert.Equal(0, sum.Statistics.BlocksDecoded); + Assert.True(sum.Metrics.Batches > 0, "the sum was answered without reading a batch"); + Assert.Equal(0, sum.Metrics.BlocksDecoded); } /// One query over the file: its plan, then what running it did. @@ -251,7 +251,7 @@ public async Task ADictionaryOrRunEndAggregateDoesNotDecode() /// The query's name, one of . /// Called between the plan and the run, when given. /// The plan of over , then its scan over . - private static async Task<(ScanPlan Plan, ScanStatistics Statistics)> RunAsync( + private static async Task<(ScanPlan Plan, ScanMetrics Statistics)> RunAsync( VortexFile file, VortexFile planning, string query, Action? planned = null) { if (query == "projection") @@ -264,7 +264,7 @@ public async Task ADictionaryOrRunEndAggregateDoesNotDecode() Assert.Single(batch.Schema); } - return (toolPlan, tool.Statistics); + return (toolPlan, tool.Metrics); } ScanPlan plan = await Query(planning, query).ExplainAsync(TestContext.Current.CancellationToken); @@ -275,7 +275,7 @@ public async Task ADictionaryOrRunEndAggregateDoesNotDecode() Assert.True(columns.RowCount > 0); } - return (plan, scan.Statistics); + return (plan, scan.Metrics); } private static Scan Query(VortexFile file, string query) => query switch diff --git a/tests/Vorticity.Tests/Api/SinkContractTests.cs b/tests/Vorticity.Tests/Api/SinkContractTests.cs index 3fe25fe2..7a67c3dd 100644 --- a/tests/Vorticity.Tests/Api/SinkContractTests.cs +++ b/tests/Vorticity.Tests/Api/SinkContractTests.cs @@ -126,8 +126,8 @@ public async Task AScanCountsTheCacheHitsThatWereItsOwn() { } - Assert.Equal(0, cold.Statistics.CacheHits); - Assert.True(cold.Statistics.Requests > 0); + Assert.Equal(0, cold.Metrics.CacheHits); + Assert.True(cold.Metrics.Requests > 0); // A scan that runs while another scan of the session runs counts its own hits, not both. Scan outer = file.Scan(); @@ -141,7 +141,7 @@ public async Task AScanCountsTheCacheHitsThatWereItsOwn() { } - Assert.Equal(inner.Statistics.Requests, inner.Statistics.CacheHits); + Assert.Equal(inner.Metrics.Requests, inner.Metrics.CacheHits); while (await batches.MoveNextAsync()) { @@ -152,7 +152,7 @@ public async Task AScanCountsTheCacheHitsThatWereItsOwn() await batches.DisposeAsync(); } - Assert.Equal(outer.Statistics.Requests, outer.Statistics.CacheHits); - Assert.Equal(cold.Statistics.Requests, outer.Statistics.Requests); + Assert.Equal(outer.Metrics.Requests, outer.Metrics.CacheHits); + Assert.Equal(cold.Metrics.Requests, outer.Metrics.Requests); } } diff --git a/tests/Vorticity.Tests/Arrays/Decoders/Canonical/DecoderGuardTests.cs b/tests/Vorticity.Tests/Arrays/Decoders/Canonical/DecoderGuardTests.cs index 6471bf93..5640e3df 100644 --- a/tests/Vorticity.Tests/Arrays/Decoders/Canonical/DecoderGuardTests.cs +++ b/tests/Vorticity.Tests/Arrays/Decoders/Canonical/DecoderGuardTests.cs @@ -163,7 +163,7 @@ public void AConstantCannotConjureAnUnboundedBufferFromASmallNode() // fixed_size_list(i64)[100000] over 8192 rows is 819 200 000 elements. The elements child is // a single constant node of a few dozen bytes, so nothing in the file is large: the 6.5 GB // is conjured entirely by the row count times the dtype's list size. CheckedMultiply catches - // the products that overflow int; MaxDecompressedSize catches the rest. + // the products that overflow int; MaxDecompressedBytes catches the rest. using DecodeHarness h = new DecodeHarness(); DType dtype = h.Types.FixedSizeList( h.Types.Primitive(PType.I64, Nullability.NonNullable), 100000, Nullability.NonNullable); @@ -176,10 +176,10 @@ public void AConstantCannotConjureAnUnboundedBufferFromASmallNode() } [Fact] - public void MaxDecompressedSizeIsHonoured() + public void MaxDecompressedBytesIsHonoured() { using ScanContext scan = new ScanContext( - TestEncodings.Ids, new VortexReadOptions { MaxDecompressedSize = 1024 }); + TestEncodings.Ids, new VortexReadOptions { MaxDecompressedBytes = 1024 }); BlobBuilder b = new BlobBuilder(); BlobNode node = new BlobNode("vortex.constant") @@ -204,21 +204,21 @@ public void MaxDecompressedSizeIsHonoured() // Four columns of 128 i64 rows stand for 1024 bytes each: each at the ceiling of one decode, // together past a batch ceiling of three. [Fact] - public void MaxBatchDecompressedSizeBoundsTheColumnsOfABatchTogether() + public void MaxBatchDecompressedBytesBoundsTheColumnsOfABatchTogether() { BlobBuilder b = new BlobBuilder(); BlobNode node = new BlobNode("vortex.struct").WithChildren(Constant(b), Constant(b), Constant(b), Constant(b)); PinnedSegment segment = b.Build(node); using ScanContext narrow = new ScanContext( - TestEncodings.Ids, new VortexReadOptions { MaxDecompressedSize = 1024, MaxBatchDecompressedSize = 3 * 1024 }); + TestEncodings.Ids, new VortexReadOptions { MaxDecompressedBytes = 1024, MaxBatchDecompressedBytes = 3 * 1024 }); ArrayBlobReader.Load(narrow.Nodes, segment.Buffer, narrow.ArrayEncodings); VortexFormatException error = Assert.Throws( () => narrow.Decode.Decode(narrow.Nodes.Root, FourLongs(narrow), 128)); - Assert.Contains("MaxBatchDecompressedSize", error.Message, StringComparison.Ordinal); + Assert.Contains("MaxBatchDecompressedBytes", error.Message, StringComparison.Ordinal); using ScanContext wide = new ScanContext( - TestEncodings.Ids, new VortexReadOptions { MaxDecompressedSize = 1024, MaxBatchDecompressedSize = 4 * 1024 }); + TestEncodings.Ids, new VortexReadOptions { MaxDecompressedBytes = 1024, MaxBatchDecompressedBytes = 4 * 1024 }); for (int batch = 0; batch < 2; batch++) { // Each batch starts from nothing. diff --git a/tests/Vorticity.Tests/Arrays/Decoders/Compressed/AmplificationTests.cs b/tests/Vorticity.Tests/Arrays/Decoders/Compressed/AmplificationTests.cs index c060a28c..ebfd835f 100644 --- a/tests/Vorticity.Tests/Arrays/Decoders/Compressed/AmplificationTests.cs +++ b/tests/Vorticity.Tests/Arrays/Decoders/Compressed/AmplificationTests.cs @@ -7,7 +7,7 @@ // // The row count reaches a decoder from far away - the layout's row_count, or a chunked parent's // chunk_offsets - and nothing between there and the allocation bounds the product, so the guard -// has to live here. It is the read options' MaxDecompressedSize, not a number invented at the call +// has to live here. It is the read options' MaxDecompressedBytes, not a number invented at the call // site, and a caller with a genuinely larger column raises it. using System; using Vorticity.Arrays; @@ -109,7 +109,7 @@ public void RaisingTheCeilingLetsALargeColumnThrough() () => tight.Decode.DecodeRoot(tight.Nodes.Root, i64, rows)); } - VortexReadOptions relaxed = new() { MaxDecompressedSize = 1L << 31 }; + VortexReadOptions relaxed = new() { MaxDecompressedBytes = 1L << 31 }; using (ScanContext wide = new(specs, relaxed)) { Load(wide, segment); diff --git a/tests/Vorticity.Tests/Dataset/DatasetBudgetTests.cs b/tests/Vorticity.Tests/Dataset/DatasetBudgetTests.cs index dc6748bd..047a7d6e 100644 --- a/tests/Vorticity.Tests/Dataset/DatasetBudgetTests.cs +++ b/tests/Vorticity.Tests/Dataset/DatasetBudgetTests.cs @@ -375,7 +375,7 @@ public async Task AnEqualityOnANonClusteringColumnCostsAConstantPlusOneTermPerUn long sought = 2L * (interleaved ? (100L * objects) + (objects / 2) : ((long)(objects / 2) * rows) + 100); await using VortexDataset dataset = await VortexDataset.OpenAsync(store, options); store.Reset(); - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); long found = await dataset.ScanBuilder() .Where(Expr.Eq(Expr.Field("measure"), Expr.Literal(FilterLiteral.From(sought)))) .WithMetrics(metrics) diff --git a/tests/Vorticity.Tests/Dataset/DatasetChangeFuzzTests.cs b/tests/Vorticity.Tests/Dataset/DatasetChangeFuzzTests.cs index 957bc5da..9e44420c 100644 --- a/tests/Vorticity.Tests/Dataset/DatasetChangeFuzzTests.cs +++ b/tests/Vorticity.Tests/Dataset/DatasetChangeFuzzTests.cs @@ -145,7 +145,7 @@ public async Task AScheduleOfChangesAnswersAsTheModel(int seed, bool marked, boo } List down = []; - for (bool ok = await cursor.SeekLastAsync(ct); ok; ok = await cursor.PrevAsync(ct)) + for (bool ok = await cursor.SeekLastAsync(ct); ok; ok = await cursor.PreviousAsync(ct)) { down.Add(cursor.Key.SignedValue); } @@ -201,8 +201,8 @@ private static async Task AgreesAsync(VortexDataset reader, DatasetKeyCursor cur long pivot = keys[random.Next(keys.Count)]; Assert.Equal(keys.Count(key => key < pivot), await cursor.RankAsync(Vorticity.Expressions.FilterLiteral.From(pivot), ct)); - Assert.True(await cursor.SeekAsync(Vorticity.Expressions.FilterLiteral.From(pivot), SeekOp.AtOrAfter, ct)); - Assert.Equal(keys.Count(key => key == pivot), await cursor.KeyCountAsync(ct)); + Assert.True(await cursor.SeekAsync(Vorticity.Expressions.FilterLiteral.From(pivot), SeekMode.AtOrAfter, ct)); + Assert.Equal(keys.Count(key => key == pivot), await cursor.CountAtKeyAsync(ct)); for (int step = 0; step < 8; step++) { Assert.Equal(scanned[(int)cursor.Row].Key, cursor.Key.SignedValue); @@ -238,7 +238,7 @@ private static async Task AgreesAsync(VortexDataset reader, DatasetKeyCursor cur private static async Task MarkedAsync(VortexDataset reader, CancellationToken ct) { int marked = 0; - await foreach (DataObject held in reader.ObjectsAsync(ct)) + await foreach (DataObject held in reader.ListObjectsAsync(ct)) { marked += held.DeletedRows > 0 ? 1 : 0; } diff --git a/tests/Vorticity.Tests/Dataset/DatasetClusteringTests.cs b/tests/Vorticity.Tests/Dataset/DatasetClusteringTests.cs index 05d02ba7..9490b9a7 100644 --- a/tests/Vorticity.Tests/Dataset/DatasetClusteringTests.cs +++ b/tests/Vorticity.Tests/Dataset/DatasetClusteringTests.cs @@ -46,7 +46,7 @@ public async Task ObjectsAreHeldInClusteringKeyOrderWhateverOrderTheyArrivedIn() await using MemoryObjectStore store = new MemoryObjectStore(); await using VortexDataset dataset = await VortexDataset.CreateAsync(store, schema, Clustered(), ct); - Assert.Equal(["key"], dataset.ClusteringKeyPaths); + Assert.Equal(["key"], dataset.ClusteringKey); Assert.NotNull(dataset.Key); foreach (int residue in (int[])[3, 1, 0, 2]) @@ -87,7 +87,7 @@ public async Task EveryAppendedObjectCarriesTheMandatoryRun() // The column is not sorted, and yet a key cursor opens and seeks: that is the run. Assert.False(IsSorted(file, "key"), "the keys were shuffled; a sorted column would void this test"); await using KeyCursor cursor = await file.Keys("key").OpenAsync(ct); - Assert.True(await cursor.SeekAsync(FilterLiteral.From(4L * 7), SeekOp.Exact, ct)); + Assert.True(await cursor.SeekAsync(FilterLiteral.From(4L * 7), SeekMode.Exact, ct)); Assert.Equal(4L * 7, cursor.Key.SignedValue); } @@ -165,7 +165,7 @@ public async Task AnExactSeekLeavesTheMergeAbleToWalkOn() } await using DatasetKeyCursor cursor = await DatasetKeyCursor.OpenAsync(dataset, ct); - Assert.True(await cursor.SeekAsync(FilterLiteral.From(517L), SeekOp.Exact, ct)); + Assert.True(await cursor.SeekAsync(FilterLiteral.From(517L), SeekMode.Exact, ct)); List rest = [cursor.Key.SignedValue]; while (await cursor.NextAsync(ct)) @@ -183,7 +183,7 @@ public async Task AnExactSeekLeavesTheMergeAbleToWalkOn() Assert.Equal(expected, rest); // And a key no object holds is a refusal, not a position on the nearest one. - Assert.False(await cursor.SeekAsync(FilterLiteral.From(-1L), SeekOp.Exact, ct)); + Assert.False(await cursor.SeekAsync(FilterLiteral.From(-1L), SeekMode.Exact, ct)); Assert.False(cursor.IsValid); Assert.False(await cursor.NextAsync(ct)); } @@ -298,7 +298,7 @@ await file.ScanBuilder().Where(window).MaxAsync("key", ct), await dataset.ScanBuilder().Where(window).MaxAsync("key", ct)); // A value no object holds: `Any` refutes it from the summaries alone. - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); VortexExpr absent = Expr.Eq(Expr.Field("key"), Expr.Literal(FilterLiteral.From(-5L))); Assert.False(await dataset.ScanBuilder().Where(absent).WithMetrics(metrics).AnyAsync(ct)); Assert.Equal(0, metrics.ObjectsOpened); diff --git a/tests/Vorticity.Tests/Dataset/DatasetCompactionLoopTests.cs b/tests/Vorticity.Tests/Dataset/DatasetCompactionLoopTests.cs index d274ef4e..f4914d39 100644 --- a/tests/Vorticity.Tests/Dataset/DatasetCompactionLoopTests.cs +++ b/tests/Vorticity.Tests/Dataset/DatasetCompactionLoopTests.cs @@ -125,7 +125,7 @@ public async Task ALoopThatLeasesTakesTheNextJobWhenAnotherHoldsALevel() CancellationToken ct = TestContext.Current.CancellationToken; await using MemoryObjectStore store = new MemoryObjectStore(); ManualClock clock = new ManualClock(new DateTimeOffset(2026, 9, 18, 12, 0, 30, TimeSpan.Zero)); - CompactionSchedule leasing = new CompactionSchedule { Leases = true, TimeProvider = clock }; + CompactionSchedule leasing = new CompactionSchedule { UseLeases = true, TimeProvider = clock }; CompactionPlan plan = new CompactionPlan { Jobs = [Job(0, 1), Job(1, 2), Job(3, 3)] }; // The first loop leases the first job's levels; the second finds level 1 held, skips the job @@ -212,7 +212,7 @@ public async Task AScheduleOutOfRangeIsRefusedBeforeAnythingRuns() new CompactionSchedule { Loops = 0 }, new CompactionSchedule { Idle = TimeSpan.Zero }, new CompactionSchedule { BytesPerSecond = -1 }, - new CompactionSchedule { Leases = true, LeaseSpan = TimeSpan.FromMilliseconds(10) }, + new CompactionSchedule { UseLeases = true, LeaseSpan = TimeSpan.FromMilliseconds(10) }, ]) { await Assert.ThrowsAsync(() => dataset.RunCompactionAsync(wrong, ct)); diff --git a/tests/Vorticity.Tests/Dataset/DatasetCompactionTests.cs b/tests/Vorticity.Tests/Dataset/DatasetCompactionTests.cs index cc695d79..c89b0697 100644 --- a/tests/Vorticity.Tests/Dataset/DatasetCompactionTests.cs +++ b/tests/Vorticity.Tests/Dataset/DatasetCompactionTests.cs @@ -294,7 +294,7 @@ public async Task ACountOnTheKeyOpensOnlyTheObjectsTheRangeCuts() long counted = 0; foreach ((long low, long high) in ((long, long)[])[(100, 1_900), (0, 2_200), (777, 778), (1_500, 1_200)]) { - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); VortexExpr range = Expr.And( Expr.Ge(Expr.Field("key"), Expr.Literal(FilterLiteral.From(low))), Expr.Lt(Expr.Field("key"), Expr.Literal(FilterLiteral.From(high)))); @@ -313,7 +313,7 @@ public async Task ACountOnTheKeyOpensOnlyTheObjectsTheRangeCuts() } // A filter the key's summaries cannot count opens what the summaries keep, and answers the same. - DatasetScanMetrics other = new DatasetScanMetrics(); + DatasetScanCounters other = new DatasetScanCounters(); VortexExpr measure = Expr.Lt(Expr.Field("measure"), Expr.Literal(FilterLiteral.From(100.0))); Assert.Equal(appended.FindAll(key => key / 4.0 < 100.0).Count, await dataset.ScanBuilder().Where(measure).WithMetrics(other).CountAsync(ct)); Assert.Equal(0, other.ObjectsCounted); diff --git a/tests/Vorticity.Tests/Dataset/DatasetDeletionVectorTests.cs b/tests/Vorticity.Tests/Dataset/DatasetDeletionVectorTests.cs index 626f68d9..ac73d8b7 100644 --- a/tests/Vorticity.Tests/Dataset/DatasetDeletionVectorTests.cs +++ b/tests/Vorticity.Tests/Dataset/DatasetDeletionVectorTests.cs @@ -63,9 +63,9 @@ public async Task MarkedRowsReadAsARewriteWouldLeaveThem(bool clustered, bool co Assert.Equal(0, byRewrites.ObjectsMarked); } - Assert.True((await marked.ObjectsAsync(ct).ToListAsync(ct)).Exists(held => held.DeletedRows > 0)); + Assert.True((await marked.ListObjectsAsync(ct).ToListAsync(ct)).Exists(held => held.DeletedRows > 0)); Assert.All( - (await marked.ObjectsAsync(ct).ToListAsync(ct)).Where(held => held.DeletedRows > 0), + (await marked.ListObjectsAsync(ct).ToListAsync(ct)).Where(held => held.DeletedRows > 0), held => Assert.Equal(outOfLine || held.Entry!.VectorBytes > 256, held.Entry!.VectorAt.Exists)); await AgreeAsync(marked, rewritten, clustered, ct); Assert.True((await marked.VerifyAsync(cancellationToken: ct)).Holds); @@ -86,7 +86,7 @@ public async Task MarkedRowsReadAsARewriteWouldLeaveThem(bool clustered, bool co // Compaction writes the live rows only, and the marks end with the objects that held them. await DrainAsync(marked, ct, new CompactionOptions { LevelZeroCeiling = 0, TargetBytesAtLevelOne = 1 << 20 }); await DrainAsync(rewritten, ct, new CompactionOptions { LevelZeroCeiling = 0, TargetBytesAtLevelOne = 1 << 20 }); - Assert.All(await marked.ObjectsAsync(ct).ToListAsync(ct), held => Assert.Equal(0, held.DeletedRows)); + Assert.All(await marked.ListObjectsAsync(ct).ToListAsync(ct), held => Assert.Equal(0, held.DeletedRows)); Assert.Equal(Sorted(await RowsAsync(rewritten, ct)), Sorted(await RowsAsync(marked, ct))); Assert.True((await marked.VerifyAsync(cancellationToken: ct)).Holds); } @@ -115,9 +115,9 @@ public async Task ACursorOnAnotherColumnLeavesMarkedRowsOutOfItsKeysAndRanks() await using KeyCursor cursor = await dataset.Scan().Keys(r => r.City).OpenAsync(ct); Assert.Equal(model.Count(row => string.CompareOrdinal(row.City, "Lyon") < 0), await cursor.RankAsync("Lyon", ct)); - Assert.True(await cursor.SeekAsync("Lyon", SeekOp.Exact, ct)); - Assert.Equal(model.Count(row => row.City == "Lyon"), await cursor.KeyCountAsync(ct)); - Assert.False(await cursor.SeekAsync("Nice", SeekOp.Exact, ct)); + Assert.True(await cursor.SeekAsync("Lyon", SeekMode.Exact, ct)); + Assert.Equal(model.Count(row => row.City == "Lyon"), await cursor.CountAtKeyAsync(ct)); + Assert.False(await cursor.SeekAsync("Nice", SeekMode.Exact, ct)); List walked = []; for (bool ok = await cursor.SeekFirstAsync(ct); ok; ok = await cursor.NextAsync(ct)) @@ -221,7 +221,7 @@ public async Task MarksPastTheirShareOfAnObjectRewriteIt() Assert.Equal(0, second.ObjectsMarked); Assert.Equal(5, second.ObjectsIn); Assert.Equal(5, second.ObjectsOut); - Assert.All(await dataset.ObjectsAsync(ct).ToListAsync(ct), held => Assert.Equal(0, held.DeletedRows)); + Assert.All(await dataset.ListObjectsAsync(ct).ToListAsync(ct), held => Assert.Equal(0, held.DeletedRows)); Assert.Equal(5 * 170, dataset.RowCount); Assert.DoesNotContain(await RowsAsync(dataset, ct), row => row.Key % 1_000 < 30); } @@ -329,7 +329,7 @@ public async Task AnObjectWhoseMarksAreDueIsRewrittenAloneInItsLevel(string due) await using VortexDataset rewritten = await CreateAsync(rewrittenStore, Options(clustered: true, marking: false), ct); await DrainAsync(marked, ct); await DrainAsync(rewritten, ct); - DataObject before = Assert.Single(await marked.ObjectsAsync(ct).ToListAsync(ct)); + DataObject before = Assert.Single(await marked.ListObjectsAsync(ct).ToListAsync(ct)); // Half the rows in one run; or every other row of three hundred, in two deletes, each of which // a delete still marks, whose vector passes half a kilobyte. @@ -342,17 +342,17 @@ public async Task AnObjectWhoseMarksAreDueIsRewrittenAloneInItsLevel(string due) await rewritten.DeleteAsync(delete, ct); } - DataObject held = Assert.Single(await marked.ObjectsAsync(ct).ToListAsync(ct)); + DataObject held = Assert.Single(await marked.ListObjectsAsync(ct).ToListAsync(ct)); Assert.True(due == "share" ? held.DeletedRows * 2 >= held.Rows + held.DeletedRows : held.Entry!.VectorBytes * 2 >= 1 << 10); CompactionPlan plan = await marked.PlanCompactionAsync(null, ct); CompactionJob job = Assert.IsType(plan.Job); Assert.Equal((CompactionTrigger.Marks, before.Level, before.Level), (job.Trigger, job.FromLevel, job.ToLevel)); - Assert.Equal([held.Key], job.Objects); + Assert.Equal([held.Key], job.ObjectKeys); CompactionResult purged = Assert.IsType(await marked.CompactAsync(null, ct)); Assert.Equal((OperationOutcome.Applied, CompactionTrigger.Marks, 1L, 1L, held.Rows), (purged.Outcome, purged.Trigger, purged.ObjectsIn, purged.ObjectsOut, purged.Rows)); - DataObject after = Assert.Single(await marked.ObjectsAsync(ct).ToListAsync(ct)); + DataObject after = Assert.Single(await marked.ListObjectsAsync(ct).ToListAsync(ct)); Assert.Equal((before.Level, 0L, held.Rows), (after.Level, after.DeletedRows, after.Rows)); Assert.Null((await marked.PlanCompactionAsync(null, ct)).Job); @@ -374,7 +374,7 @@ public async Task AMarkedLeafRidesOnlyInTheCommitsThatWriteIt() await using CountingObjectStore store = new CountingObjectStore(inner); await using VortexDataset dataset = await CreateAsync(store, Options(clustered: true, marking: true), ct); await DrainAsync(dataset, ct); - int level = Assert.Single(await dataset.ObjectsAsync(ct).ToListAsync(ct)).Level; + int level = Assert.Single(await dataset.ListObjectsAsync(ct).ToListAsync(ct)).Level; // Unmarked, the leaf rides in the header of a commit that does not change its level. await AppendAsync(dataset, 9_000, ct); @@ -397,11 +397,11 @@ public async Task AMarkedLeafRidesOnlyInTheCommitsThatWriteIt() // request. A reader opening the version reads it in one, and proves as much once it has it. store.Reset(); Assert.False(dataset.MayMatch(r => r.Key == 50_000)); - int objects = (await dataset.ObjectsAsync(ct).ToListAsync(ct)).Count; + int objects = (await dataset.ListObjectsAsync(ct).ToListAsync(ct)).Count; Assert.Equal(0, store.Requests); await using VortexDataset cold = await VortexDataset.OpenAsync(store, Options(clustered: true, marking: true), ct); store.Reset(); - Assert.Equal(objects, (await cold.ObjectsAsync(ct).ToListAsync(ct)).Count); + Assert.Equal(objects, (await cold.ListObjectsAsync(ct).ToListAsync(ct)).Count); Assert.Equal(1, store.Requests); Assert.False(cold.MayMatch(r => r.Key == 50_000)); Assert.True((await dataset.VerifyAsync(cancellationToken: ct)).Holds); @@ -459,13 +459,13 @@ public async Task MarksRunUpToTheirBoundAndTheDeleteThatWouldPassItRewrites() break; } - marked = (await dataset.ObjectsAsync(ct).ToListAsync(ct)).Single(held => held.DeletedRows > 0).Entry!.Deletions.EncodedBytes; + marked = (await dataset.ListObjectsAsync(ct).ToListAsync(ct)).Single(held => held.DeletedRows > 0).Entry!.Deletions.EncodedBytes; } // The last marks fit the bound, and one run more, at two varints of the object's row // count, would not have: the bound counts the vector's own bytes, not a guess at them. Assert.InRange(marked, 256 - (2 * 2) + 1, 256); - Assert.All(await dataset.ObjectsAsync(ct).ToListAsync(ct), held => Assert.Equal(0, held.DeletedRows)); + Assert.All(await dataset.ListObjectsAsync(ct).ToListAsync(ct), held => Assert.Equal(0, held.DeletedRows)); Assert.True((await dataset.VerifyAsync(cancellationToken: ct)).Holds); } @@ -624,7 +624,7 @@ await Ordered(rewritten.Scan().Where(r => r.City != "Paris"), descend foreach (long key in (long[])[0, 1_040, 1_089, 2_001, 3_500, 4_199, 9_999]) { Assert.Equal(await byRewrites.RankAsync(key, ct), await byMarks.RankAsync(key, ct)); - foreach (SeekOp op in (SeekOp[])[SeekOp.Exact, SeekOp.AtOrAfter, SeekOp.After, SeekOp.AtOrBefore, SeekOp.Before]) + foreach (SeekMode op in (SeekMode[])[SeekMode.Exact, SeekMode.AtOrAfter, SeekMode.After, SeekMode.AtOrBefore, SeekMode.Before]) { bool found = await byRewrites.SeekAsync(key, op, ct); Assert.Equal(found, await byMarks.SeekAsync(key, op, ct)); @@ -641,7 +641,7 @@ await Ordered(rewritten.Scan().Where(r => r.City != "Paris"), descend Assert.True(await byMarks.SeekRankAsync(rank, ct)); Assert.True(await byRewrites.SeekRankAsync(rank, ct)); Assert.Equal(byRewrites.Key, byMarks.Key); - Assert.Equal(await byRewrites.KeyCountAsync(ct), await byMarks.KeyCountAsync(ct)); + Assert.Equal(await byRewrites.CountAtKeyAsync(ct), await byMarks.CountAtKeyAsync(ct)); } } @@ -697,7 +697,7 @@ private static async Task PositionsAsync(VortexDataset dataset, CancellationToke List<(long Key, long Row)> entries = []; for (bool ok = up ? await cursor.SeekFirstAsync(ct) : await cursor.SeekLastAsync(ct); ok; - ok = up ? await cursor.NextAsync(ct) : await cursor.PrevAsync(ct)) + ok = up ? await cursor.NextAsync(ct) : await cursor.PreviousAsync(ct)) { entries.Add((cursor.Key, cursor.Row)); } diff --git a/tests/Vorticity.Tests/Dataset/DatasetInlineCompactionTests.cs b/tests/Vorticity.Tests/Dataset/DatasetInlineCompactionTests.cs index c0b5c108..3b3fc492 100644 --- a/tests/Vorticity.Tests/Dataset/DatasetInlineCompactionTests.cs +++ b/tests/Vorticity.Tests/Dataset/DatasetInlineCompactionTests.cs @@ -89,7 +89,7 @@ public async Task AReplacementThatTakesLevelZeroPastItsCeilingCompactsItToo() ChangeRow[] rows = [.. Enumerable.Range(0, 50).Select(i => new ChangeRow(90_000L + i, 1.0, Cities[i % Cities.Length]))]; ObjectDraft draft = dataset.StartObject(); await draft.Writer.WriteAsync(rows, ct); - ReplaceResult replaced = await dataset.ReplaceAsync([], [draft], ct); + ReplaceResult replaced = await dataset.ReplaceObjectsAsync([], [draft], ct); Assert.Equal((OperationOutcome.Applied, replaced.Version + 1), (replaced.Outcome, dataset.Version)); Assert.Equal(0, dataset.Lag); } diff --git a/tests/Vorticity.Tests/Dataset/DatasetKeyCursorReadTests.cs b/tests/Vorticity.Tests/Dataset/DatasetKeyCursorReadTests.cs index 0de7f219..ff8551cb 100644 --- a/tests/Vorticity.Tests/Dataset/DatasetKeyCursorReadTests.cs +++ b/tests/Vorticity.Tests/Dataset/DatasetKeyCursorReadTests.cs @@ -53,7 +53,7 @@ public async Task ASeekOnAColdVersionReadsOnePathOfEachLevel(bool down) long opening = store.Requests; long key = (10L * Middle) + 2; - Assert.True(await cursor.SeekAsync(FilterLiteral.From(key), down ? SeekOp.AtOrBefore : SeekOp.AtOrAfter, ct)); + Assert.True(await cursor.SeekAsync(FilterLiteral.From(key), down ? SeekMode.AtOrBefore : SeekMode.AtOrAfter, ct)); Assert.Equal(key, cursor.Key.SignedValue); Assert.Equal((5L * Middle) + 2, cursor.Row); Assert.Equal(1, cursor.Cursors); @@ -75,7 +75,7 @@ public async Task ASeekOnAColdVersionReadsOnePathOfEachLevel(bool down) int at = written.IndexOf(key); for (int step = 1; step <= 5; step++) { - Assert.True(down ? await cursor.PrevAsync(ct) : await cursor.NextAsync(ct)); + Assert.True(down ? await cursor.PreviousAsync(ct) : await cursor.NextAsync(ct)); long expected = written[down ? at - step : at + step]; Assert.Equal(expected, cursor.Key.SignedValue); Assert.Equal((expected / 10 * 5) + (expected % 10), cursor.Row); @@ -171,7 +171,7 @@ private static async Task AssertWalksAsync(DatasetKeyCursor cursor, int count, A } Assert.Equal(count, index); - for (bool more = await cursor.SeekLastAsync(ct); more; more = await cursor.PrevAsync(ct)) + for (bool more = await cursor.SeekLastAsync(ct); more; more = await cursor.PreviousAsync(ct)) { at(--index); } @@ -186,7 +186,7 @@ private static async Task AssertWalksAsync(DatasetKeyCursor cursor, int count, A private static async Task AssertSeeksAsync( DatasetKeyCursor cursor, int count, FilterLiteral key, int below, int through, Action at, CancellationToken ct) { - foreach ((SeekOp op, int expected) in ((SeekOp, int)[])[(SeekOp.AtOrAfter, below), (SeekOp.After, through), (SeekOp.AtOrBefore, through - 1), (SeekOp.Before, below - 1)]) + foreach ((SeekMode op, int expected) in ((SeekMode, int)[])[(SeekMode.AtOrAfter, below), (SeekMode.After, through), (SeekMode.AtOrBefore, through - 1), (SeekMode.Before, below - 1)]) { bool inside = expected >= 0 && expected < count; Assert.Equal(inside, await cursor.SeekAsync(key, op, ct)); @@ -204,7 +204,7 @@ private static async Task AssertStepsAsync(DatasetKeyCursor cursor, int count, i foreach (int step in (int[])[1, 1, 1, -1, -1, -1, -1, -1, -1]) { index += step; - bool more = step > 0 ? await cursor.NextAsync(ct) : await cursor.PrevAsync(ct); + bool more = step > 0 ? await cursor.NextAsync(ct) : await cursor.PreviousAsync(ct); Assert.Equal(index >= 0 && index < count, more); if (!more) { diff --git a/tests/Vorticity.Tests/Dataset/DatasetKeyOrderTests.cs b/tests/Vorticity.Tests/Dataset/DatasetKeyOrderTests.cs index 267a0fa2..e12fac39 100644 --- a/tests/Vorticity.Tests/Dataset/DatasetKeyOrderTests.cs +++ b/tests/Vorticity.Tests/Dataset/DatasetKeyOrderTests.cs @@ -62,7 +62,7 @@ public async Task InKeyOrderAcrossInterleavedObjectsEqualsTheSortedRows() reversed.Reverse(); foreach (bool summaries in (bool[])[true, false]) { - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); Assert.Equal( sorted, await RowsAsync(dataset.ScanBuilder().WithSummaries(summaries).WithMetrics(metrics).InKeyOrder("key"))); @@ -158,16 +158,16 @@ public async Task OrderByKeyLimitKOpensOnlyTheObjectsThatCouldHoldTheFirstKRows( // Upward on the clustering key, by the tree's exact minima: one object for the first ten, // and two for the first three hundred, which cross one boundary and not a second. - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); Assert.Equal(sorted[..10], await FirstAsync(dataset.ScanBuilder().WithMetrics(metrics).InKeyOrder("key"), 10)); Assert.Equal(1, metrics.ObjectsOpened); - metrics = new DatasetScanMetrics(); + metrics = new DatasetScanCounters(); Assert.Equal(sorted[..300], await FirstAsync(dataset.ScanBuilder().WithMetrics(metrics).InKeyOrder("key"), 300)); Assert.Equal(2, metrics.ObjectsOpened); // Downward, by the summaries' maxima. - metrics = new DatasetScanMetrics(); + metrics = new DatasetScanCounters(); Assert.Equal( reversed[..10], await FirstAsync(dataset.ScanBuilder().WithMetrics(metrics).InKeyOrder("key", descending: true), 10)); @@ -176,7 +176,7 @@ public async Task OrderByKeyLimitKOpensOnlyTheObjectsThatCouldHoldTheFirstKRows( // Without the summaries, the same rows and every object opened: they are what bought the skip. foreach (bool descending in (bool[])[false, true]) { - metrics = new DatasetScanMetrics(); + metrics = new DatasetScanCounters(); Assert.Equal( descending ? reversed[..10] : sorted[..10], await FirstAsync( @@ -224,7 +224,7 @@ public async Task AFilterOnTheKeyNarrowsEveryObjectsOwnWalk() } // A key no object holds: the summaries refute every object, and nothing is opened. - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); VortexExpr absent = Expr.Lt(Expr.Field("key"), Expr.Literal(FilterLiteral.From(-1L))); Assert.Empty(await RowsAsync(dataset.ScanBuilder().Where(absent).WithMetrics(metrics).InKeyOrder("key"))); Assert.Equal(0, metrics.ObjectsOpened); @@ -321,7 +321,7 @@ public async Task AnotherColumnIsMergedToo_WithEveryKeptObjectAsABound() // Output-sensitive, and still pruned by the summaries: their minima say which quarter // holds the first ten rows, and the other three are not opened. - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); Assert.Equal(sorted[..10], await FirstAsync(dataset.ScanBuilder().WithMetrics(metrics).InKeyOrder("key"), 10)); Assert.Equal(1, metrics.ObjectsOpened); } @@ -485,11 +485,11 @@ private static async Task AssertOrderedAsync(VortexDataset dataset, List h Assert.Equal("key", plan.Order); Assert.Equal(cursors, plan.Cursors); - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); Assert.Equal(sorted, await RowsAsync(dataset.ScanBuilder().WithMetrics(metrics).InKeyOrder("key"))); Assert.Equal(cursors, metrics.Cursors); - metrics = new DatasetScanMetrics(); + metrics = new DatasetScanCounters(); Assert.Equal(reversed, await RowsAsync(dataset.ScanBuilder().WithMetrics(metrics).InKeyOrder("key", descending: true))); Assert.Equal(cursors, metrics.Cursors); diff --git a/tests/Vorticity.Tests/Dataset/DatasetPageCacheTests.cs b/tests/Vorticity.Tests/Dataset/DatasetPageCacheTests.cs index bc25fffe..e48ab06a 100644 --- a/tests/Vorticity.Tests/Dataset/DatasetPageCacheTests.cs +++ b/tests/Vorticity.Tests/Dataset/DatasetPageCacheTests.cs @@ -581,7 +581,7 @@ private static (PageReference Reference, byte[] Bytes) Page(int seed, int length private static async Task CountAsync(VortexDataset dataset, CancellationToken ct) { int count = 0; - await foreach (DataObject _ in dataset.ObjectsAsync(ct)) + await foreach (DataObject _ in dataset.ListObjectsAsync(ct)) { count++; } diff --git a/tests/Vorticity.Tests/Dataset/DatasetPlanTests.cs b/tests/Vorticity.Tests/Dataset/DatasetPlanTests.cs index 1210db57..22fd25fb 100644 --- a/tests/Vorticity.Tests/Dataset/DatasetPlanTests.cs +++ b/tests/Vorticity.Tests/Dataset/DatasetPlanTests.cs @@ -253,7 +253,7 @@ public async Task APlanAsksForTheSameFewPagesWhateverTheObjectsALevelHolds(int d CompactionJob job = Assert.IsType(plan.Job); Assert.Equal((CompactionTrigger.LevelSize, 2, 3), (job.Trigger, job.FromLevel, job.ToLevel)); Assert.Equal(objects.Where(o => o.Level == 2).Max(o => o.Entry.Bytes), job.Inputs[0].Entry.Bytes); - Assert.InRange(job.Objects.Length, 2, 6); + Assert.InRange(job.ObjectKeys.Length, 2, 6); Console.Out.Write(FormattableString.Invariant( $"COMPACTION PLAN: {objects.Count} objects over three levels: a descent asked the store for {asked} pages, a full read for {read}.\n")); @@ -635,7 +635,7 @@ private static void Same(CompactionPlan expected, CompactionPlan actual, int see CompactionJob other = Assert.IsType(actual.Job); Assert.Equal((job.Trigger, job.FromLevel, job.ToLevel, job.TargetBytes, job.Rows, job.Bytes), (other.Trigger, other.FromLevel, other.ToLevel, other.TargetBytes, other.Rows, other.Bytes)); Assert.Equal((job.Style, job.FirstRow, Convert.ToHexString(job.Stop.Span)), (other.Style, other.FirstRow, Convert.ToHexString(other.Stop.Span))); - Assert.True(job.Objects.SequenceEqual(other.Objects), $"{shape}: {string.Join(",", job.Objects)} against {string.Join(",", other.Objects)}"); + Assert.True(job.ObjectKeys.SequenceEqual(other.ObjectKeys), $"{shape}: {string.Join(",", job.ObjectKeys)} against {string.Join(",", other.ObjectKeys)}"); Assert.Equal( job.Inputs.Select(input => (input.Level, Convert.ToHexString(input.Key.Span))), other.Inputs.Select(input => (input.Level, Convert.ToHexString(input.Key.Span)))); diff --git a/tests/Vorticity.Tests/Dataset/DatasetRankSeekTests.cs b/tests/Vorticity.Tests/Dataset/DatasetRankSeekTests.cs index 5bea37d0..7fd926c0 100644 --- a/tests/Vorticity.Tests/Dataset/DatasetRankSeekTests.cs +++ b/tests/Vorticity.Tests/Dataset/DatasetRankSeekTests.cs @@ -98,7 +98,7 @@ public async Task EveryRankLandsWhereTheWalkDoesAndTheMergeGoesOnFromIt(Shape sh // Every object is left where the merge would have it, the ones past the key included, and // the walk from there crosses every boundary between objects, the ones still waiting too. FilterLiteral key = walked[rank].Key; - Assert.Equal(walked.FindAll(entry => entry.Key == key).Count, await cursor.KeyCountAsync(ct)); + Assert.Equal(walked.FindAll(entry => entry.Key == key).Count, await cursor.CountAtKeyAsync(ct)); for (int at = rank + 1; at < Math.Min(rank + 18, walked.Count); at++) { Assert.True(await cursor.NextAsync(ct)); diff --git a/tests/Vorticity.Tests/Dataset/DatasetReadAheadTests.cs b/tests/Vorticity.Tests/Dataset/DatasetReadAheadTests.cs index bc3e35a5..674775c6 100644 --- a/tests/Vorticity.Tests/Dataset/DatasetReadAheadTests.cs +++ b/tests/Vorticity.Tests/Dataset/DatasetReadAheadTests.cs @@ -25,7 +25,7 @@ public async Task AScanThatStopsAtItsFirstBatchOpensOneObject() { await using MemoryObjectStore store = new MemoryObjectStore(); await using VortexDataset dataset = await CreateAsync(store); - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); List keys = await KeysAsync(dataset.ScanBuilder().WithMetrics(metrics), batches: 1); Assert.Equal(Enumerable.Range(0, ObjectRows).Select(k => (long)k), keys); @@ -41,7 +41,7 @@ public async Task AScanThatReadsOnOpensTheNextObjectAheadUnderAPrefetch(int pref { await using MemoryObjectStore store = new MemoryObjectStore(); await using VortexDataset dataset = await CreateAsync(store); - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); List keys = await KeysAsync( dataset.ScanBuilder().WithOptions(new ScanOptions { Prefetch = prefetch }, keepEncodings: false).WithMetrics(metrics), int.MaxValue); @@ -56,7 +56,7 @@ public async Task AScanStoppedInsideTheDatasetGivesBackTheObjectOpenedAhead() { await using MemoryObjectStore store = new MemoryObjectStore(); await using VortexDataset dataset = await CreateAsync(store); - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); long read = 0; int after = -1; await foreach (RecordBatch batch in dataset.ScanBuilder().WithOptions(new ScanOptions { BatchRows = 100 }, keepEncodings: false).WithMetrics(metrics).ExecuteAsync(Ct)) @@ -94,7 +94,7 @@ public async Task AnObjectOpenedAheadThatCannotOpenFailsTheScanOnlyOnceItIsReach await using (VortexDataset written = await CreateAsync(store)) { List objects = []; - await foreach (DataObject held in written.ObjectsAsync(Ct)) + await foreach (DataObject held in written.ListObjectsAsync(Ct)) { objects.Add(held); } diff --git a/tests/Vorticity.Tests/Dataset/DatasetRowChangeTests.cs b/tests/Vorticity.Tests/Dataset/DatasetRowChangeTests.cs index 9eba29ed..bbcb6e8e 100644 --- a/tests/Vorticity.Tests/Dataset/DatasetRowChangeTests.cs +++ b/tests/Vorticity.Tests/Dataset/DatasetRowChangeTests.cs @@ -80,7 +80,7 @@ public async Task AnObjectIsRemovedWholeRewrittenOrLeftAsItsRowsSay() await using (dataset) { // Object 1 holds the keys 1000 to 1199: all of them go, and the object with them. - List objects = await dataset.ObjectsAsync(ct).ToListAsync(ct); + List objects = await dataset.ListObjectsAsync(ct).ToListAsync(ct); RowChangeResult removed = await dataset.DeleteAsync(r => r.Key >= 1_000 & r.Key < 2_000, ct); Assert.Equal(PerObject, removed.Rows); Assert.Equal(1, removed.ObjectsIn); @@ -91,14 +91,14 @@ public async Task AnObjectIsRemovedWholeRewrittenOrLeftAsItsRowsSay() // Fifty keys of object 2, which now starts at row 200: it is rewritten with the other // hundred and fifty, and the others, whose summaries refute the filter, are left as they are. - List untouched = (await dataset.ObjectsAsync(ct).ToListAsync(ct)) + List untouched = (await dataset.ListObjectsAsync(ct).ToListAsync(ct)) .Where(o => o.FirstRow != PerObject).Select(o => o.Key).ToList(); Assert.Equal(Objects - 2, untouched.Count); RowChangeResult rewritten = await dataset.DeleteAsync(r => r.Key >= 2_050 & r.Key < 2_100, ct); Assert.Equal(50, rewritten.Rows); Assert.Equal(1, rewritten.ObjectsIn); Assert.Equal(1, rewritten.ObjectsOut); - List now = (await dataset.ObjectsAsync(ct).ToListAsync(ct)).Select(o => o.Key).ToList(); + List now = (await dataset.ListObjectsAsync(ct).ToListAsync(ct)).Select(o => o.Key).ToList(); Assert.Equal(Objects - 1, now.Count); Assert.Equal(untouched, now.Intersect(untouched)); @@ -347,7 +347,7 @@ private static async Task AssertKeyOrderAsync(VortexDataset dataset, bool cluste } List down = []; - for (bool ok = await cursor.SeekLastAsync(ct); ok; ok = await cursor.PrevAsync(ct)) + for (bool ok = await cursor.SeekLastAsync(ct); ok; ok = await cursor.PreviousAsync(ct)) { down.Add(cursor.Key.SignedValue); } diff --git a/tests/Vorticity.Tests/Dataset/DatasetSchemaEvolutionTests.cs b/tests/Vorticity.Tests/Dataset/DatasetSchemaEvolutionTests.cs index 26c62ebc..5871c917 100644 --- a/tests/Vorticity.Tests/Dataset/DatasetSchemaEvolutionTests.cs +++ b/tests/Vorticity.Tests/Dataset/DatasetSchemaEvolutionTests.cs @@ -130,7 +130,7 @@ [.. keys.AsEnumerable().Reverse()], } List down = []; - for (bool ok = await cursor.SeekLastAsync(ct); ok; ok = await cursor.PrevAsync(ct)) + for (bool ok = await cursor.SeekLastAsync(ct); ok; ok = await cursor.PreviousAsync(ct)) { down.Add(cursor.Key.SignedValue); } diff --git a/tests/Vorticity.Tests/Dataset/DatasetSummaryTests.cs b/tests/Vorticity.Tests/Dataset/DatasetSummaryTests.cs index e9d2821f..c2580aa0 100644 --- a/tests/Vorticity.Tests/Dataset/DatasetSummaryTests.cs +++ b/tests/Vorticity.Tests/Dataset/DatasetSummaryTests.cs @@ -55,8 +55,8 @@ public async Task AFilterInsideOneObjectOpensOnlyThatObject() Expr.Ge(Expr.Field("key"), Expr.Literal(FilterLiteral.From(3_100L))), Expr.Lt(Expr.Field("key"), Expr.Literal(FilterLiteral.From(3_200L)))); - DatasetScanMetrics pruned = new DatasetScanMetrics(); - DatasetScanMetrics whole = new DatasetScanMetrics(); + DatasetScanCounters pruned = new DatasetScanCounters(); + DatasetScanCounters whole = new DatasetScanCounters(); List withSummaries = await KeysAsync(dataset.ScanBuilder().Where(filter).WithMetrics(pruned)); List without = await KeysAsync( dataset.ScanBuilder().Where(filter).WithSummaries(false).WithMetrics(whole)); @@ -95,7 +95,7 @@ public async Task AValueNoObjectHoldsOpensNothingAtAll() } VortexExpr absent = Expr.Eq(Expr.Field("key"), Expr.Literal(FilterLiteral.From(-1L))); - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); Assert.Empty(await KeysAsync(dataset.ScanBuilder().Where(absent).WithMetrics(metrics))); Assert.Equal(0, metrics.ObjectsOpened); Assert.Equal(4, metrics.ObjectsSkipped); @@ -126,14 +126,14 @@ public async Task ANodeThatRefutesThePredicateCostsItsSubtreeNoRead() Assert.True(dataset.Depth >= 3, $"sixteen objects in pages of 400 bytes should nest; depth is {dataset.Depth}"); VortexExpr absent = Expr.Gt(Expr.Field("key"), Expr.Literal(FilterLiteral.From(1_000_000L))); - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); Assert.Empty(await KeysAsync(dataset.ScanBuilder().Where(absent).WithMetrics(metrics))); Assert.True(metrics.SubtreesSkipped > 0, "a node's summaries should have refuted the predicate"); Assert.Equal(0, metrics.ObjectsConsidered); Assert.Equal(0, metrics.ObjectsOpened); // And the acceptance again: with the summaries off, the same empty answer the long way. - DatasetScanMetrics whole = new DatasetScanMetrics(); + DatasetScanCounters whole = new DatasetScanCounters(); Assert.Empty(await KeysAsync(dataset.ScanBuilder().Where(absent).WithSummaries(false).WithMetrics(whole))); Assert.Equal(objects, whole.ObjectsConsidered); Assert.Equal(0, whole.SubtreesSkipped); @@ -182,7 +182,7 @@ await KeysAsync(file.ScanBuilder().Rows(new RowRange(from, Math.Min(to, file.Row } // And it does not open what it does not need: rows 2 000..2 500 are object 2's alone. - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); Assert.Equal(500, (await KeysAsync(dataset.ScanBuilder().Rows(2_000, 2_500).WithMetrics(metrics))).Count); Assert.Equal(1, metrics.ObjectsOpened); Assert.Equal(1, metrics.ObjectsConsidered); @@ -218,7 +218,7 @@ await KeysAsync(file.ScanBuilder().Where(filter).Rows(new RowRange(from, to))), } // And the point: the subtrees below the filter were counted, not read. - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); Assert.Equal(80, (await KeysAsync(dataset.ScanBuilder().Where(filter).Rows(1_350, 1_430).WithMetrics(metrics))).Count); Assert.True(metrics.SubtreesSkipped > 0, "a node's summaries should have refuted the predicate"); } @@ -248,7 +248,7 @@ public async Task AFilteredRangeOverLevelsThatInterleaveReadsTheSubtreesItMustPl Assert.Equal(await KeysAsync(dataset.ScanBuilder().Where(filter).WithSummaries(false).Rows(from, to)), pruned); } - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); Assert.NotEmpty(await KeysAsync(dataset.ScanBuilder().Where(filter).Rows(1_200, 1_300).WithMetrics(metrics))); Assert.True(metrics.SubtreesSkipped > 0, "the subtrees no object of level 0 falls in should not be read"); } @@ -297,8 +297,8 @@ public async Task AnObjectOpenedOnceStaysOpen() await dataset.AppendAsync(Batches(types, schema, i * Rows, Rows), ct); } - DatasetScanMetrics first = new DatasetScanMetrics(); - DatasetScanMetrics second = new DatasetScanMetrics(); + DatasetScanCounters first = new DatasetScanCounters(); + DatasetScanCounters second = new DatasetScanCounters(); List once = await KeysAsync(dataset.ScanBuilder().WithMetrics(first)); List twice = await KeysAsync(dataset.ScanBuilder().WithMetrics(second)); diff --git a/tests/Vorticity.Tests/Dataset/DatasetWalkDownTests.cs b/tests/Vorticity.Tests/Dataset/DatasetWalkDownTests.cs index ff190e31..2c258460 100644 --- a/tests/Vorticity.Tests/Dataset/DatasetWalkDownTests.cs +++ b/tests/Vorticity.Tests/Dataset/DatasetWalkDownTests.cs @@ -34,7 +34,7 @@ public async Task TheWalkDownIsTheWalkUpReversed(Shape shape, bool text, bool so await using DatasetKeyCursor cursor = await built.OpenAsync(distinct: false, ct); List down = []; - for (bool ok = await cursor.SeekLastAsync(ct); ok; ok = await cursor.PrevAsync(ct)) + for (bool ok = await cursor.SeekLastAsync(ct); ok; ok = await cursor.PreviousAsync(ct)) { down.Add(Entry.Of(cursor)); } @@ -42,7 +42,7 @@ public async Task TheWalkDownIsTheWalkUpReversed(Shape shape, bool text, bool so down.Reverse(); Assert.Equal(built.Walked, down); Assert.False(cursor.IsValid); - Assert.False(await cursor.PrevAsync(ct)); + Assert.False(await cursor.PreviousAsync(ct)); Assert.False(await cursor.NextAsync(ct)); } @@ -69,7 +69,7 @@ public async Task AStepAgainstTheWalkLandsNextToWhereItWas(Shape shape, bool tex { bool up = random.Next(3) == 0 ? random.Next(2) == 0 : (step / 40) % 2 == 0; int expected = up ? at + 1 : at - 1; - bool moved = up ? await cursor.NextAsync(ct) : await cursor.PrevAsync(ct); + bool moved = up ? await cursor.NextAsync(ct) : await cursor.PreviousAsync(ct); Assert.Equal(expected >= 0 && expected < walked.Count, moved); if (!moved) { @@ -96,9 +96,9 @@ public async Task ASeekBelowAKeyLandsWhereASortedFileWould(Shape shape, bool tex await using DatasetKeyCursor cursor = await built.OpenAsync(distinct: false, ct); foreach (FilterLiteral probe in built.Probes()) { - foreach (SeekOp op in (SeekOp[])[SeekOp.AtOrBefore, SeekOp.Before]) + foreach (SeekMode op in (SeekMode[])[SeekMode.AtOrBefore, SeekMode.Before]) { - int expected = LastBelow(walked, probe, inclusive: op == SeekOp.AtOrBefore); + int expected = LastBelow(walked, probe, inclusive: op == SeekMode.AtOrBefore); bool found = await cursor.SeekAsync(probe, op, ct); Assert.Equal(expected >= 0, found); if (!found) @@ -113,7 +113,7 @@ public async Task ASeekBelowAKeyLandsWhereASortedFileWould(Shape shape, bool tex int at = expected; for (int step = 0; step < 3 && at > 0; step++) { - Assert.True(await cursor.PrevAsync(ct)); + Assert.True(await cursor.PreviousAsync(ct)); Assert.Equal(walked[--at], Entry.Of(cursor)); } @@ -141,9 +141,9 @@ public async Task TheKeyStepsDownLandOnThePreviousKeysLastEntry(Shape shape, boo while (true) { // Every entry of the key is counted wherever the walk stands among them. - Assert.Equal(CountOf(walked, walked[at].Key), await cursor.KeyCountAsync(ct)); + Assert.Equal(CountOf(walked, walked[at].Key), await cursor.CountAtKeyAsync(ct)); int previous = LastBelow(walked, walked[at].Key, inclusive: false); - bool moved = await cursor.PrevKeyAsync(ct); + bool moved = await cursor.PreviousKeyAsync(ct); Assert.Equal(previous >= 0, moved); if (!moved) { @@ -160,7 +160,7 @@ public async Task TheKeyStepsDownLandOnThePreviousKeysLastEntry(Shape shape, boo int next = at + 1; Assert.True(await cursor.NextKeyAsync(ct)); Assert.Equal(walked[next], Entry.Of(cursor)); - Assert.True(await cursor.PrevKeyAsync(ct)); + Assert.True(await cursor.PreviousKeyAsync(ct)); Assert.Equal(walked[at], Entry.Of(cursor)); } } @@ -191,7 +191,7 @@ public async Task ADistinctWalkDownVisitsEveryKeysFirstEntry(Shape shape, bool t } List down = []; - for (bool ok = await cursor.SeekLastAsync(ct); ok; ok = await cursor.PrevAsync(ct)) + for (bool ok = await cursor.SeekLastAsync(ct); ok; ok = await cursor.PreviousAsync(ct)) { down.Add(Entry.Of(cursor)); } @@ -202,9 +202,9 @@ public async Task ADistinctWalkDownVisitsEveryKeysFirstEntry(Shape shape, bool t // A distinct seek below a key lands on the first entry of the key it finds. Entry middle = firsts[firsts.Count / 2]; - Assert.True(await cursor.SeekAsync(middle.Key, SeekOp.AtOrBefore, ct)); + Assert.True(await cursor.SeekAsync(middle.Key, SeekMode.AtOrBefore, ct)); Assert.Equal(middle, Entry.Of(cursor)); - Assert.True(await cursor.SeekAsync(middle.Key, SeekOp.Before, ct)); + Assert.True(await cursor.SeekAsync(middle.Key, SeekMode.Before, ct)); Assert.Equal(firsts[(firsts.Count / 2) - 1], Entry.Of(cursor)); } @@ -232,13 +232,13 @@ public async Task AWalkDownOnTheClusteringKeyOpensAnObjectOnlyOnceItCouldHoldThe Assert.True(await cursor.SeekLastAsync(ct)); Assert.True(cursor.Cursors <= levelZero + above, $"{cursor.Cursors} cursors after the last key"); - Assert.True(await cursor.SeekAsync(built.Walked[0].Key, SeekOp.AtOrBefore, ct)); + Assert.True(await cursor.SeekAsync(built.Walked[0].Key, SeekMode.AtOrBefore, ct)); Assert.Equal(built.Walked[0].Key, cursor.Key); // The whole walk down reads every object, once. await using DatasetKeyCursor whole = await DatasetKeyCursor.OpenAsync(dataset, ct); long entries = 0; - for (bool ok = await whole.SeekLastAsync(ct); ok; ok = await whole.PrevAsync(ct)) + for (bool ok = await whole.SeekLastAsync(ct); ok; ok = await whole.PreviousAsync(ct)) { entries++; } diff --git a/tests/Vorticity.Tests/Dataset/MergeFanInTests.cs b/tests/Vorticity.Tests/Dataset/MergeFanInTests.cs index e86c24b3..e2153927 100644 --- a/tests/Vorticity.Tests/Dataset/MergeFanInTests.cs +++ b/tests/Vorticity.Tests/Dataset/MergeFanInTests.cs @@ -46,7 +46,7 @@ public async Task MergesSixtyFourObjectsInterleavedRowByRow() expected.Add(key); } - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); List walked = []; await foreach (RecordBatch batch in dataset.ScanBuilder().WithMetrics(metrics).InKeyOrder("key") .ExecuteAsync(ct).WithCancellation(CancellationToken.None)) @@ -179,7 +179,7 @@ public async Task AWalkAgainFindsTheObjectsTheCacheKept() } Assert.Equal(Objects * PerObject, await WalkAsync(dataset.ScanBuilder(), ct)); - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); Assert.Equal(Objects * PerObject, await WalkAsync(dataset.ScanBuilder().WithMetrics(metrics), ct)); Assert.Equal(Objects, metrics.ObjectsOpened); @@ -206,7 +206,7 @@ public async Task AWalkReadsTheStoreOnlyToOpenAnObject() Assert.Equal(Objects * PerObject, await WalkAsync(dataset.ScanBuilder(), ct)); Interlocked.Exchange(ref store.Count, 0); - DatasetScanMetrics metrics = new DatasetScanMetrics(); + DatasetScanCounters metrics = new DatasetScanCounters(); Assert.Equal(Objects * PerObject, await WalkAsync(dataset.ScanBuilder().WithMetrics(metrics), ct)); Assert.Equal(metrics.ObjectsOpened - metrics.CacheHits, Volatile.Read(ref store.Count)); diff --git a/tests/Vorticity.Tests/File/TailReadTests.cs b/tests/Vorticity.Tests/File/TailReadTests.cs index 37083a47..5fa99813 100644 --- a/tests/Vorticity.Tests/File/TailReadTests.cs +++ b/tests/Vorticity.Tests/File/TailReadTests.cs @@ -29,7 +29,7 @@ public async Task AnOpenReadsAgainWhatItsFirstReadMissed(int tailReadSize, int r Decoders.EnsureRegistered(); CancellationToken ct = TestContext.Current.CancellationToken; byte[] bytes = global::System.IO.File.ReadAllBytes(Corpus.Path(Entry)); - Assert.True(bytes.Length > VortexOpenOptions.DefaultInitialReadSize); + Assert.True(bytes.Length > VortexOpenOptions.DefaultInitialReadBytes); TailSized source = new TailSized(new MemorySegmentSource(bytes), tailReadSize); await using VortexFile file = await VortexFile.OpenAsync(source, VortexOpenOptions.Default, ct); @@ -53,7 +53,7 @@ public async Task ACallerThatRaisedTheWindowReadsItWhateverTheSourcePrefers() byte[] bytes = global::System.IO.File.ReadAllBytes(Corpus.Path(Entry)); TailSized source = new TailSized(new MemorySegmentSource(bytes), 16); - await using VortexFile file = await VortexFile.OpenAsync(source, VortexOpenOptions.Default with { InitialReadSize = 1 << 20 }, ct); + await using VortexFile file = await VortexFile.OpenAsync(source, VortexOpenOptions.Default with { InitialReadBytes = 1 << 20 }, ct); Assert.Equal(1, source.RangeReads); Assert.Equal(bytes.Length, source.RangeBytes); } diff --git a/tests/Vorticity.Tests/File/TornTailTests.cs b/tests/Vorticity.Tests/File/TornTailTests.cs index 44850b13..68bbc6a5 100644 --- a/tests/Vorticity.Tests/File/TornTailTests.cs +++ b/tests/Vorticity.Tests/File/TornTailTests.cs @@ -120,7 +120,7 @@ public async Task AnEndRecordAcrossTwoWindowsOfTheWalkIsFound(int below) try { await System.IO.File.WriteAllBytesAsync(path, torn, ct); - Assert.Equal(whole.LongLength, await VortexFileRepair.ValidLengthAsync(path, ct)); + Assert.Equal(whole.LongLength, await VortexFileRepair.GetValidLengthAsync(path, ct)); } finally { diff --git a/tests/Vorticity.Tests/File/VortexFileResourceTests.cs b/tests/Vorticity.Tests/File/VortexFileResourceTests.cs index 9aeb3d3c..42b48c48 100644 --- a/tests/Vorticity.Tests/File/VortexFileResourceTests.cs +++ b/tests/Vorticity.Tests/File/VortexFileResourceTests.cs @@ -170,9 +170,9 @@ public async Task ConcurrentReadersOfOneOpenFileSeeTheSameAnswers() [Fact] public void ReadOptionsRejectANonPositiveDecompressionCeiling() { - Assert.Throws(() => new VortexReadOptions { MaxDecompressedSize = 0 }); - Assert.Throws(() => new VortexReadOptions { MaxDecompressedSize = -1 }); - Assert.Equal(VortexLimits.DefaultMaxDecompressedSize, VortexReadOptions.Default.MaxDecompressedSize); + Assert.Throws(() => new VortexReadOptions { MaxDecompressedBytes = 0 }); + Assert.Throws(() => new VortexReadOptions { MaxDecompressedBytes = -1 }); + Assert.Equal(VortexLimits.DefaultMaxDecompressedBytes, VortexReadOptions.Default.MaxDecompressedBytes); Assert.False(VortexReadOptions.Default.VerifyStatistics); } @@ -180,15 +180,15 @@ public void ReadOptionsRejectANonPositiveDecompressionCeiling() public void OpenOptionsRejectNonsenseAndCarryTheirDefaults() { Assert.Throws(() => new VortexOpenOptions { Length = -2 }); - Assert.Throws(() => new VortexOpenOptions { InitialReadSize = -1 }); + Assert.Throws(() => new VortexOpenOptions { InitialReadBytes = -1 }); Assert.Null(VortexOpenOptions.Default.Length); Assert.Equal(-1, VortexOpenOptions.Default.FileLength); - Assert.Equal(65_536, VortexOpenOptions.Default.InitialReadSize); + Assert.Equal(65_536, VortexOpenOptions.Default.InitialReadBytes); // The read policy follows the public options unless one is given. - Assert.Equal(VortexReadOptions.Default.MaxDecompressedSize, VortexOpenOptions.Default.Read.MaxDecompressedSize); - Assert.Equal(1024, new VortexOpenOptions { MaxDecompressedSize = 1024 }.Read.MaxDecompressedSize); + Assert.Equal(VortexReadOptions.Default.MaxDecompressedBytes, VortexOpenOptions.Default.Read.MaxDecompressedBytes); + Assert.Equal(1024, new VortexOpenOptions { MaxDecompressedBytes = 1024 }.Read.MaxDecompressedBytes); Assert.True(VortexOpenOptions.Default.DType.IsDefault); Assert.False(VortexOpenOptions.Default.LeaveSourceOpen); } @@ -196,17 +196,17 @@ public void OpenOptionsRejectNonsenseAndCarryTheirDefaults() [Fact] public void ABatchHasNoDecompressionCeilingUnlessAsked() { - Assert.Null(VortexOpenOptions.Default.MaxBatchDecompressedSize); - Assert.Equal(long.MaxValue, VortexOpenOptions.Default.Read.MaxBatchDecompressedSize); - Assert.Equal(4096, new VortexOpenOptions { MaxBatchDecompressedSize = 4096 }.Read.MaxBatchDecompressedSize); - Assert.Throws(() => new VortexOpenOptions { MaxBatchDecompressedSize = 0 }); - Assert.Throws(() => new VortexReadOptions { MaxBatchDecompressedSize = -1 }); + Assert.Null(VortexOpenOptions.Default.MaxBatchDecompressedBytes); + Assert.Equal(long.MaxValue, VortexOpenOptions.Default.Read.MaxBatchDecompressedBytes); + Assert.Equal(4096, new VortexOpenOptions { MaxBatchDecompressedBytes = 4096 }.Read.MaxBatchDecompressedBytes); + Assert.Throws(() => new VortexOpenOptions { MaxBatchDecompressedBytes = 0 }); + Assert.Throws(() => new VortexReadOptions { MaxBatchDecompressedBytes = -1 }); } [Fact] public async Task ReadOptionsReachTheOpenFile() { - VortexReadOptions read = new VortexReadOptions { VerifyStatistics = true, MaxDecompressedSize = 1024 }; + VortexReadOptions read = new VortexReadOptions { VerifyStatistics = true, MaxDecompressedBytes = 1024 }; DTypeArena arena = new DTypeArena(); byte[] bytes = SyntheticVortexFile.Build(new SyntheticFileSpec { diff --git a/tests/Vorticity.Tests/File/VortexFileRoundTripTests.cs b/tests/Vorticity.Tests/File/VortexFileRoundTripTests.cs index 9c4ba6f5..1b3abc83 100644 --- a/tests/Vorticity.Tests/File/VortexFileRoundTripTests.cs +++ b/tests/Vorticity.Tests/File/VortexFileRoundTripTests.cs @@ -77,7 +77,7 @@ public async Task RaisingTheInitialReadSizeRemovesTheSecondRead() byte[] bytes = FileWithFooterOutsideTheTailWindow(arena, filler: 70_000); TestSegmentSource source = new TestSegmentSource(bytes); - VortexOpenOptions options = new VortexOpenOptions { InitialReadSize = bytes.Length }; + VortexOpenOptions options = new VortexOpenOptions { InitialReadBytes = bytes.Length }; await using VortexFile file = await VortexFile.OpenAsync(source, options, CancellationToken.None); Assert.Equal(8193, file.RowCount); @@ -92,7 +92,7 @@ public async Task LoweringTheInitialReadSizeIsIgnored() TestSegmentSource lowered = new TestSegmentSource(bytes); await using (await VortexFile.OpenAsync( - lowered, new VortexOpenOptions { InitialReadSize = 16 }, CancellationToken.None)) + lowered, new VortexOpenOptions { InitialReadBytes = 16 }, CancellationToken.None)) { } diff --git a/tests/Vorticity.Tests/IO/GroupRoundTripTests.cs b/tests/Vorticity.Tests/IO/GroupRoundTripTests.cs index f2fbccfa..a4b5139b 100644 --- a/tests/Vorticity.Tests/IO/GroupRoundTripTests.cs +++ b/tests/Vorticity.Tests/IO/GroupRoundTripTests.cs @@ -208,7 +208,7 @@ public async Task AGroupByOverACopyingSourceReservesItsReadsInFlight() Options = new ScanOptions { DegreeOfParallelism = 1, Prefetch = 0 }, ReadAhead = readAhead ? Vorticity.Scanning.BatchAsyncEnumerator.ReadAheadSplits : 0, }; - await foreach (RecordBatch batch in new FileScanSource(file).BatchesAsync(spec, new Vorticity.Scanning.ScanMetrics()).WithCancellation(Ct)) + await foreach (RecordBatch batch in new FileScanSource(file).BatchesAsync(spec, new Vorticity.Scanning.ScanCounters()).WithCancellation(Ct)) { rows += batch.RowCount; } diff --git a/tests/Vorticity.Tests/IO/LocalFileSourceTests.cs b/tests/Vorticity.Tests/IO/LocalFileSourceTests.cs index 9c5eb24f..44ceb17f 100644 --- a/tests/Vorticity.Tests/IO/LocalFileSourceTests.cs +++ b/tests/Vorticity.Tests/IO/LocalFileSourceTests.cs @@ -149,7 +149,7 @@ public async Task AFileTheOpenReadWholeIsScannedFromThatReadAndNeverMapped() { Decoders.EnsureRegistered(); string path = Corpus.Path("encodings/bool"); - Assert.True(new System.IO.FileInfo(path).Length <= VortexOpenOptions.DefaultInitialReadSize); + Assert.True(new System.IO.FileInfo(path).Length <= VortexOpenOptions.DefaultInitialReadBytes); // A session of its own, for the reason the test above gives. await using VortexSession session = VortexSession.Create(_ => { }); diff --git a/tests/Vorticity.Tests/IO/MappedFileCacheTests.cs b/tests/Vorticity.Tests/IO/MappedFileCacheTests.cs index 09fd1544..dda429f4 100644 --- a/tests/Vorticity.Tests/IO/MappedFileCacheTests.cs +++ b/tests/Vorticity.Tests/IO/MappedFileCacheTests.cs @@ -321,7 +321,7 @@ private static async Task WriteSpreadIdsAsync(string path, CancellationTok await writer.CompleteAsync(ct); } - Assert.True(new FileInfo(path).Length > VortexOpenOptions.DefaultInitialReadSize); + Assert.True(new FileInfo(path).Length > VortexOpenOptions.DefaultInitialReadBytes); return sum; } diff --git a/tests/Vorticity.Tests/IO/SegmentCacheTests.cs b/tests/Vorticity.Tests/IO/SegmentCacheTests.cs index 0809739a..cdc4bfdc 100644 --- a/tests/Vorticity.Tests/IO/SegmentCacheTests.cs +++ b/tests/Vorticity.Tests/IO/SegmentCacheTests.cs @@ -49,7 +49,7 @@ public async Task ASegmentReadAsAViewOfARunIsKeptInABlockOfItsOwn() kept.Release(); } - Assert.Equal(4_000, cache.Size); + Assert.Equal(4_000, cache.HeldBytes); } finally { @@ -81,7 +81,7 @@ public async Task ASlotFilledBeforeTheReadIsNotKept() await reader.ReadManyAsync(set, CancellationToken.None); Assert.False(cache.TryGet(source, 0, 1_000, out _, out _)); - Assert.Equal(1_000, cache.Size); + Assert.Equal(1_000, cache.HeldBytes); } finally { @@ -167,7 +167,7 @@ public void KeptSegmentsLeaveInTheirOrderOfUseAndWithTheirFile() model.RemoveAll(entry => ReferenceEquals(entry.File, file)); } - Assert.Equal(model.Count * 64L, cache.Size); + Assert.Equal(model.Count * 64L, cache.HeldBytes); } // Each segment the model holds is there; looked up from the oldest, they leave again in @@ -180,7 +180,7 @@ public void KeptSegmentsLeaveInTheirOrderOfUseAndWithTheirFile() } cache.Clear(); - Assert.Equal(0, cache.Size); + Assert.Equal(0, cache.HeldBytes); Assert.False(cache.TryGet(model[0].File, model[0].Offset, 64, out _, out _)); } } diff --git a/tests/Vorticity.Tests/Indexes/BloomIndexTests.cs b/tests/Vorticity.Tests/Indexes/BloomIndexTests.cs index 4a488ae1..68b92a8b 100644 --- a/tests/Vorticity.Tests/Indexes/BloomIndexTests.cs +++ b/tests/Vorticity.Tests/Indexes/BloomIndexTests.cs @@ -440,8 +440,8 @@ public async Task AnUnindexedScanReadsNoIndexSegment() await using Written written = await Written.CreateAsync(Policy()); int present = Key(40_000); - ScanMetrics on = new ScanMetrics(); - ScanMetrics off = new ScanMetrics(); + ScanCounters on = new ScanCounters(); + ScanCounters off = new ScanCounters(); Assert.Equal( await written.File.ScanBuilder().Where(Parse($"key = {present}")).WithMetrics(on).CountAsync(ct), await written.File.ScanBuilder().Where(Parse($"key = {present}")).WithMetrics(off).WithIndexes(false).CountAsync(ct)); diff --git a/tests/Vorticity.Tests/Indexes/FenceTreeTests.cs b/tests/Vorticity.Tests/Indexes/FenceTreeTests.cs index 930e4413..69cc541a 100644 --- a/tests/Vorticity.Tests/Indexes/FenceTreeTests.cs +++ b/tests/Vorticity.Tests/Indexes/FenceTreeTests.cs @@ -626,13 +626,13 @@ private static async Task AssertWalkAsync(VortexFile file, int rows) for (int at = 0; at < rows; at += rows / 7) { long key = expected[at].Key; - Assert.True(await cursor.SeekAsync(FilterLiteral.From(key), SeekOp.AtOrAfter)); + Assert.True(await cursor.SeekAsync(FilterLiteral.From(key), SeekMode.AtOrAfter)); Assert.Equal(expected[at].Row, cursor.Row); } Assert.True(await cursor.SeekLastAsync()); Assert.Equal(expected[^1].Row, cursor.Row); - Assert.True(await cursor.PrevAsync()); + Assert.True(await cursor.PreviousAsync()); Assert.Equal(expected[^2].Row, cursor.Row); Assert.Equal(rows, cursor.EntryCount); diff --git a/tests/Vorticity.Tests/Indexes/LyingIndexTests.cs b/tests/Vorticity.Tests/Indexes/LyingIndexTests.cs index 07e1904a..9a1d49d8 100644 --- a/tests/Vorticity.Tests/Indexes/LyingIndexTests.cs +++ b/tests/Vorticity.Tests/Indexes/LyingIndexTests.cs @@ -238,11 +238,11 @@ private static async Task ExerciseAsync(byte[] bytes, int policy, bool verify) steps++; } - _ = await cursor.SeekAsync(probe, SeekOp.AtOrAfter, timeout.Token); + _ = await cursor.SeekAsync(probe, SeekMode.AtOrAfter, timeout.Token); _ = await cursor.SeekLastAsync(timeout.Token); - _ = await cursor.PrevAsync(timeout.Token); + _ = await cursor.PreviousAsync(timeout.Token); _ = await cursor.RankAsync(probe, timeout.Token); - _ = await cursor.KeyCountAsync(timeout.Token); + _ = await cursor.CountAtKeyAsync(timeout.Token); } await using (KeyCursor distinct = await file.Keys(distinctColumn).Distinct().OpenAsync(timeout.Token)) diff --git a/tests/Vorticity.Tests/Indexes/ReadBudgetTests.cs b/tests/Vorticity.Tests/Indexes/ReadBudgetTests.cs index 570156a7..cceca721 100644 --- a/tests/Vorticity.Tests/Indexes/ReadBudgetTests.cs +++ b/tests/Vorticity.Tests/Indexes/ReadBudgetTests.cs @@ -191,7 +191,7 @@ public async Task APointLookupCostsTheSameOnASparseFileOfOneAndOfTenGibibytes() Assert.Equal(1, matches); await using (KeyCursor cursor = await file.Keys("k").WithSource(KeySourceKind.SortedRuns).OpenAsync()) { - Assert.True(await cursor.SeekAsync(FilterLiteral.From(K(row)), SeekOp.Exact)); + Assert.True(await cursor.SeekAsync(FilterLiteral.From(K(row)), SeekMode.Exact)); Assert.Equal(row, cursor.Row); } diff --git a/tests/Vorticity.Tests/Indexes/RunMergeTests.cs b/tests/Vorticity.Tests/Indexes/RunMergeTests.cs index b250aaa0..2ee76208 100644 --- a/tests/Vorticity.Tests/Indexes/RunMergeTests.cs +++ b/tests/Vorticity.Tests/Indexes/RunMergeTests.cs @@ -557,7 +557,7 @@ private static async Task AssertWalkAsync(VortexFile file, int rows) // Backward, and a seek into the middle. long middle = expected[rows / 2].Key; - Assert.True(await cursor.SeekAsync(FilterLiteral.From(middle), SeekOp.AtOrAfter)); + Assert.True(await cursor.SeekAsync(FilterLiteral.From(middle), SeekMode.AtOrAfter)); Assert.Equal(expected[expected.FindIndex(e => e.Key >= middle)].Row, cursor.Row); Assert.True(await cursor.SeekLastAsync()); Assert.Equal(expected[^1].Row, cursor.Row); diff --git a/tests/Vorticity.Tests/Keys/CompositeKeyTests.cs b/tests/Vorticity.Tests/Keys/CompositeKeyTests.cs index 12673f52..2ab736ef 100644 --- a/tests/Vorticity.Tests/Keys/CompositeKeyTests.cs +++ b/tests/Vorticity.Tests/Keys/CompositeKeyTests.cs @@ -92,7 +92,7 @@ public async Task APrefixOfTheLeadingColumnIsASeekAndAWalk(string country) byte[] prefix = RowEncoder.EncodeKey([FilterLiteral.From(country)], [Asc]); await using KeyCursor cursor = await written.File.Keys("country", "city").OpenAsync(ct); List rows = []; - for (bool ok = await cursor.SeekAsync(FilterLiteral.From(prefix), SeekOp.AtOrAfter, ct); + for (bool ok = await cursor.SeekAsync(FilterLiteral.From(prefix), SeekMode.AtOrAfter, ct); ok && cursor.KeyBytes.StartsWith(prefix); ok = await cursor.NextAsync(ct)) { @@ -127,7 +127,7 @@ public async Task AnIntegerLeadsAKeyAtItsOwnWidthAndSignsOrderIt() // A seek at -1 lands on the first row whose number is -1. byte[] minusOne = RowEncoder.EncodeKey([FilterLiteral.From(-1L)], [I32], [Asc]); - Assert.True(await cursor.SeekAsync(FilterLiteral.From(minusOne), SeekOp.AtOrAfter, ct)); + Assert.True(await cursor.SeekAsync(FilterLiteral.From(minusOne), SeekMode.AtOrAfter, ct)); Assert.Equal(-1, Number((int)cursor.Row)); Assert.True(cursor.KeyBytes.StartsWith(minusOne)); diff --git a/tests/Vorticity.Tests/Keys/DistinctWalkTests.cs b/tests/Vorticity.Tests/Keys/DistinctWalkTests.cs index 9859c4c0..e7b0e3b3 100644 --- a/tests/Vorticity.Tests/Keys/DistinctWalkTests.cs +++ b/tests/Vorticity.Tests/Keys/DistinctWalkTests.cs @@ -72,7 +72,7 @@ public async Task EveryKeyIsVisitedOnceAtItsFirstRowBothWays(KeySourceKind sourc } Assert.Equal(lengths.Length, key); - for (bool ok = await cursor.SeekLastAsync(ct); ok; ok = await cursor.PrevAsync(ct)) + for (bool ok = await cursor.SeekLastAsync(ct); ok; ok = await cursor.PreviousAsync(ct)) { key--; Assert.Equal(key, cursor.Key.SignedValue); @@ -123,12 +123,12 @@ public async Task NextKeyReachesEachKeysFirstEntryAndPrevKeyThePreviousKeysLast( Assert.True(await cursor.SeekLastAsync(ct)); for (int key = lengths.Length - 2; key >= 0; key--) { - Assert.True(await cursor.PrevKeyAsync(ct)); + Assert.True(await cursor.PreviousKeyAsync(ct)); Assert.Equal(key, cursor.Key.SignedValue); Assert.Equal(lastRow[key], cursor.Row); } - Assert.False(await cursor.PrevKeyAsync(ct)); + Assert.False(await cursor.PreviousKeyAsync(ct)); } finally { diff --git a/tests/Vorticity.Tests/Keys/KeyCursorTests.cs b/tests/Vorticity.Tests/Keys/KeyCursorTests.cs index 2cbf3fe3..e92a3ba2 100644 --- a/tests/Vorticity.Tests/Keys/KeyCursorTests.cs +++ b/tests/Vorticity.Tests/Keys/KeyCursorTests.cs @@ -64,11 +64,11 @@ public async Task TheFiveOperatorsLandWhereTheOracleSaysAndTheWalkIsExactlyTheEn long upper = UpperBound(oracle, key); bool present = lower < oracle.Count && Same(oracle[(int)lower], key); - await Check(cursor, key, SeekOp.Exact, present ? lower : -1, oracle); - await Check(cursor, key, SeekOp.AtOrAfter, lower < oracle.Count ? lower : -1, oracle); - await Check(cursor, key, SeekOp.After, upper < oracle.Count ? upper : -1, oracle); - await Check(cursor, key, SeekOp.AtOrBefore, upper - 1, oracle); - await Check(cursor, key, SeekOp.Before, lower - 1, oracle); + await Check(cursor, key, SeekMode.Exact, present ? lower : -1, oracle); + await Check(cursor, key, SeekMode.AtOrAfter, lower < oracle.Count ? lower : -1, oracle); + await Check(cursor, key, SeekMode.After, upper < oracle.Count ? upper : -1, oracle); + await Check(cursor, key, SeekMode.AtOrBefore, upper - 1, oracle); + await Check(cursor, key, SeekMode.Before, lower - 1, oracle); } } @@ -102,7 +102,7 @@ public async Task AFullWalkEqualsTheMaterializedColumnInOrderAndReversesExactly( // Backward is the exact reverse, and a direction flip after a step needs no re-seek here. at = oracle.Count; - for (bool ok = await cursor.SeekLastAsync(ct); ok; ok = await cursor.PrevAsync(ct)) + for (bool ok = await cursor.SeekLastAsync(ct); ok; ok = await cursor.PreviousAsync(ct)) { at--; Assert.True(Same(oracle[at], cursor.Key), $"{column} reversed at {at}"); @@ -112,7 +112,7 @@ public async Task AFullWalkEqualsTheMaterializedColumnInOrderAndReversesExactly( Assert.True(await cursor.SeekRankAsync(oracle.Count / 2, ct)); Assert.True(await cursor.NextAsync(ct)); - Assert.True(await cursor.PrevAsync(ct)); + Assert.True(await cursor.PreviousAsync(ct)); Assert.True(Same(oracle[oracle.Count / 2], cursor.Key)); } @@ -133,7 +133,7 @@ public async Task RankAndSelectAgreeWithEachOtherAndWithTheKeyCount(string colum Assert.True(await cursor.SeekRankAsync(i, ct), $"rank {i}"); FilterLiteral key = cursor.Key; long rank = await cursor.RankAsync(key, ct); - long count = await cursor.KeyCountAsync(ct); + long count = await cursor.CountAtKeyAsync(ct); // The invariant of rank and select: select lands inside its key's own slice. Assert.True(rank <= i, $"{column}: rank {rank} above {i}"); @@ -182,7 +182,7 @@ public async Task NextKeyIsOneSeekPerGroupAndPrevKeyIsItsReverse(string column, // just before the last key's first, the last group being short. Assert.True(await cursor.SeekLastAsync(ct)); long lastGroupStart = LowerBound(oracle, oracle[^1]); - Assert.True(await cursor.PrevKeyAsync(ct)); + Assert.True(await cursor.PreviousKeyAsync(ct)); Assert.Equal(lastGroupStart - 1, cursor.Row - FirstRow(column)); } @@ -201,9 +201,9 @@ public async Task NullsAreNeverVisitedAndAreNotEntries() Assert.Equal(0L, cursor.Key.SignedValue); // And a seek below every key lands on the first entry, never inside the null run. - Assert.True(await cursor.SeekAsync(FilterLiteral.From(-1L), SeekOp.AtOrAfter, ct)); + Assert.True(await cursor.SeekAsync(FilterLiteral.From(-1L), SeekMode.AtOrAfter, ct)); Assert.Equal(Nulls, cursor.Row); - Assert.False(await cursor.SeekAsync(FilterLiteral.From(-1L), SeekOp.Before, ct)); + Assert.False(await cursor.SeekAsync(FilterLiteral.From(-1L), SeekMode.Before, ct)); } [Fact] @@ -222,12 +222,12 @@ public async Task ASortedColumnPutsTheTwoZerosInOneKeyAndCompareSeparatesThem() // Row 0 is -0.0 and rows 1 and 2 are +0.0, so the key holds three entries and its first is // the negative zero: one key, and the walk starts where IEEE says it does. await using KeyCursor cursor = await written.File.Keys("floats_f64").OpenAsync(ct); - Assert.True(await cursor.SeekAsync(FilterLiteral.From(0.0), SeekOp.Exact, ct)); + Assert.True(await cursor.SeekAsync(FilterLiteral.From(0.0), SeekMode.Exact, ct)); Assert.Equal(0, cursor.Row); - Assert.Equal(3L, await cursor.KeyCountAsync(ct)); + Assert.Equal(3L, await cursor.CountAtKeyAsync(ct)); Assert.True(double.IsNegative(cursor.Key.FloatValue)); - Assert.True(await cursor.SeekAsync(FilterLiteral.From(-0.0), SeekOp.Exact, ct)); + Assert.True(await cursor.SeekAsync(FilterLiteral.From(-0.0), SeekMode.Exact, ct)); Assert.Equal(0, cursor.Row); } @@ -266,7 +266,7 @@ public async Task ADistinctWalkOfASortedColumnIsEachKeyAtItsFirstRow(string colu } Assert.Equal(firsts.Count, index); - for (bool ok = await cursor.SeekLastAsync(ct); ok; ok = await cursor.PrevAsync(ct)) + for (bool ok = await cursor.SeekLastAsync(ct); ok; ok = await cursor.PreviousAsync(ct)) { index--; Assert.True(SameIeee(firsts[index].Key, cursor.Key), $"key {index}: {Describe(cursor.Key)}"); @@ -322,9 +322,9 @@ public async Task ASeekOfTheWrongDomainIsRefusedAtTheCall() await using KeyCursor cursor = await written.File.Keys("strict_i64").OpenAsync(ct); await Assert.ThrowsAsync( - async () => await cursor.SeekAsync(FilterLiteral.From("nope"), SeekOp.Exact, ct)); + async () => await cursor.SeekAsync(FilterLiteral.From("nope"), SeekMode.Exact, ct)); await Assert.ThrowsAsync( - async () => await cursor.SeekAsync(FilterLiteral.Null, SeekOp.Exact, ct)); + async () => await cursor.SeekAsync(FilterLiteral.Null, SeekMode.Exact, ct)); Assert.Throws(() => cursor.Key); } @@ -339,7 +339,7 @@ public async Task AStepInsideALoadedZoneAllocatesNothing() await using Written written = await Written.CreateAsync(); await using KeyCursor cursor = await written.File.Keys("keys_utf8").OpenAsync(ct); - Assert.True(await cursor.SeekAsync(Utf8Key(200), SeekOp.AtOrAfter, ct)); + Assert.True(await cursor.SeekAsync(Utf8Key(200), SeekMode.AtOrAfter, ct)); // Warm the path, then measure steps that stay inside the zone the seek loaded. long bytes = 0; @@ -349,7 +349,7 @@ public async Task AStepInsideALoadedZoneAllocatesNothing() bytes += cursor.KeyBytes.Length; } - Assert.True(await cursor.SeekAsync(Utf8Key(200), SeekOp.AtOrAfter, ct)); + Assert.True(await cursor.SeekAsync(Utf8Key(200), SeekMode.AtOrAfter, ct)); long before = GC.GetAllocatedBytesForCurrentThread(); for (int step = 0; step < 64; step++) { @@ -374,7 +374,7 @@ public async Task ASortedColumnDeliversItsKeyOrderAsAContiguousRead(bool descend { Decoders.EnsureRegistered(); await using Written written = await Written.CreateAsync(); - ScanMetrics metrics = new ScanMetrics(); + ScanCounters metrics = new ScanCounters(); List values = []; await foreach (RecordBatch batch in written.File.ScanBuilder() .InKeyOrder("strict_i64", descending) @@ -527,7 +527,7 @@ private static bool Step(KeyCursor cursor) } private static async Task Check( - KeyCursor cursor, FilterLiteral key, SeekOp op, long expected, List oracle) + KeyCursor cursor, FilterLiteral key, SeekMode op, long expected, List oracle) { bool found = await cursor.SeekAsync(key, op); Assert.True( diff --git a/tests/Vorticity.Tests/Keys/RankSeekTests.cs b/tests/Vorticity.Tests/Keys/RankSeekTests.cs index 19a47e04..d7c185e9 100644 --- a/tests/Vorticity.Tests/Keys/RankSeekTests.cs +++ b/tests/Vorticity.Tests/Keys/RankSeekTests.cs @@ -74,7 +74,7 @@ public async Task EveryRankLandsOnItsEntryAndTheCursorStepsOnBothWays(int[] runs for (int step = 0; step < 5 && at > 0; step++) { - Assert.True(await cursor.PrevAsync(ct)); + Assert.True(await cursor.PreviousAsync(ct)); AssertAt(cursor, oracle, --at, strings); } } diff --git a/tests/Vorticity.Tests/Keys/SortedColumnLieTests.cs b/tests/Vorticity.Tests/Keys/SortedColumnLieTests.cs index eba27533..ff97c9fe 100644 --- a/tests/Vorticity.Tests/Keys/SortedColumnLieTests.cs +++ b/tests/Vorticity.Tests/Keys/SortedColumnLieTests.cs @@ -69,9 +69,9 @@ public async Task ASeekIntoTheLyingZoneIsRefusedUnderVerification() await using KeyCursor cursor = await file.Keys("sorted").WithSource(KeySourceKind.SortedColumn).OpenAsync(ct); // A zone far from the lie decodes and verifies; the lying zone does not. - Assert.True(await cursor.SeekAsync(FilterLiteral.From(Value(100)), SeekOp.Exact, ct)); + Assert.True(await cursor.SeekAsync(FilterLiteral.From(Value(100)), SeekMode.Exact, ct)); await Assert.ThrowsAsync( - async () => await cursor.SeekAsync(FilterLiteral.From(Value(Target - 10)), SeekOp.AtOrAfter, ct)); + async () => await cursor.SeekAsync(FilterLiteral.From(Value(Target - 10)), SeekMode.AtOrAfter, ct)); } [Fact] diff --git a/tests/Vorticity.Tests/Keys/SortedRunsCursorTests.cs b/tests/Vorticity.Tests/Keys/SortedRunsCursorTests.cs index ea563ebb..13b1bc51 100644 --- a/tests/Vorticity.Tests/Keys/SortedRunsCursorTests.cs +++ b/tests/Vorticity.Tests/Keys/SortedRunsCursorTests.cs @@ -106,7 +106,7 @@ public async Task AFullWalkIsTheOracleForwardAndBackward(string column) Assert.Equal(oracle.Count, index); Assert.False(cursor.IsValid); - for (bool ok = await cursor.SeekLastAsync(ct); ok; ok = await cursor.PrevAsync(ct)) + for (bool ok = await cursor.SeekLastAsync(ct); ok; ok = await cursor.PreviousAsync(ct)) { AssertAt(cursor, oracle, --index); } @@ -128,11 +128,11 @@ public async Task TheFiveOperatorsLandWhereTheOracleSays(string column) { int lower = Lower(oracle, key); int upper = Upper(oracle, key); - await Check(cursor, oracle, key, SeekOp.AtOrAfter, lower); - await Check(cursor, oracle, key, SeekOp.After, upper); - await Check(cursor, oracle, key, SeekOp.AtOrBefore, upper - 1); - await Check(cursor, oracle, key, SeekOp.Before, lower - 1); - await Check(cursor, oracle, key, SeekOp.Exact, lower < upper ? lower : -1); + await Check(cursor, oracle, key, SeekMode.AtOrAfter, lower); + await Check(cursor, oracle, key, SeekMode.After, upper); + await Check(cursor, oracle, key, SeekMode.AtOrBefore, upper - 1); + await Check(cursor, oracle, key, SeekMode.Before, lower - 1); + await Check(cursor, oracle, key, SeekMode.Exact, lower < upper ? lower : -1); Assert.Equal(lower, await cursor.RankAsync(key, ct)); } @@ -160,7 +160,7 @@ public async Task StepsAcrossAFlipStayOnTheOracle(string column) for (int s = 0; s < steps; s++) { int next = forward ? index + 1 : index - 1; - bool moved = forward ? await cursor.NextAsync(ct) : await cursor.PrevAsync(ct); + bool moved = forward ? await cursor.NextAsync(ct) : await cursor.PreviousAsync(ct); Assert.Equal(next >= 0 && next < oracle.Count, moved); if (!moved) { @@ -189,7 +189,7 @@ public async Task RankSelectAndKeyCountAgree(string column) Assert.True(await cursor.SeekRankAsync(i, ct)); AssertAt(cursor, oracle, i); FilterLiteral key = cursor.Key; - long count = await cursor.KeyCountAsync(ct); + long count = await cursor.CountAtKeyAsync(ct); Assert.Equal(Upper(oracle, key) - Lower(oracle, key), count); // The count leaves the position where it was. @@ -231,11 +231,11 @@ public async Task NextKeyVisitsEachKeyOnceAndPrevKeyReversesIt(string column) Assert.True(await cursor.SeekLastAsync(ct)); for (int g = firsts.Count - 1; g > 0; g--) { - Assert.True(await cursor.PrevKeyAsync(ct)); + Assert.True(await cursor.PreviousKeyAsync(ct)); AssertAt(cursor, oracle, firsts[g] - 1); } - Assert.False(await cursor.PrevKeyAsync(ct)); + Assert.False(await cursor.PreviousKeyAsync(ct)); } [Fact] @@ -333,21 +333,21 @@ public async Task ASeekOfTheWrongDomainIsRefusedAndTheZerosAreTwoKeys() await using Written written = await Written.CreateAsync(); await using KeyCursor cursor = await written.File.Keys("f32").OpenAsync(ct); - await Assert.ThrowsAsync(async () => await cursor.SeekAsync(FilterLiteral.From(1L), SeekOp.Exact, ct)); + await Assert.ThrowsAsync(async () => await cursor.SeekAsync(FilterLiteral.From(1L), SeekMode.Exact, ct)); - Assert.True(await cursor.SeekAsync(FilterLiteral.From(-0.0), SeekOp.Exact, ct)); + Assert.True(await cursor.SeekAsync(FilterLiteral.From(-0.0), SeekMode.Exact, ct)); Assert.True(double.IsNegative(cursor.Key.FloatValue)); - long negative = await cursor.KeyCountAsync(ct); - Assert.True(await cursor.SeekAsync(FilterLiteral.From(0.0), SeekOp.Exact, ct)); + long negative = await cursor.CountAtKeyAsync(ct); + Assert.True(await cursor.SeekAsync(FilterLiteral.From(0.0), SeekMode.Exact, ct)); Assert.False(double.IsNegative(cursor.Key.FloatValue)); Assert.Equal(Rows / Floats.Length, negative); - Assert.Equal(Rows / Floats.Length, await cursor.KeyCountAsync(ct)); + Assert.Equal(Rows / Floats.Length, await cursor.CountAtKeyAsync(ct)); // The NaNs are the ends: nothing after the positive one, nothing before the negative one, // and each is a key of its own. - Assert.False(await cursor.SeekAsync(FilterLiteral.From(PositiveNaN), SeekOp.After, ct)); - Assert.False(await cursor.SeekAsync(FilterLiteral.From(NegativeNaN), SeekOp.Before, ct)); - Assert.True(await cursor.SeekAsync(FilterLiteral.From(NegativeNaN), SeekOp.Exact, ct)); + Assert.False(await cursor.SeekAsync(FilterLiteral.From(PositiveNaN), SeekMode.After, ct)); + Assert.False(await cursor.SeekAsync(FilterLiteral.From(NegativeNaN), SeekMode.Before, ct)); + Assert.True(await cursor.SeekAsync(FilterLiteral.From(NegativeNaN), SeekMode.Exact, ct)); Assert.True(double.IsNaN(cursor.Key.FloatValue) && double.IsNegative(cursor.Key.FloatValue)); Assert.True(await cursor.SeekLastAsync(ct)); Assert.True(double.IsNaN(cursor.Key.FloatValue) && !double.IsNegative(cursor.Key.FloatValue)); @@ -377,7 +377,7 @@ public async Task AnExactCoverCountsReadsAndBoundsLikeTheDecode(string text) VortexFile file = written.File; // The count and the membership, from the runs alone: no data segment is read. - ScanMetrics metrics = new ScanMetrics(); + ScanCounters metrics = new ScanCounters(); Assert.Equal(oracle.Count, await file.ScanBuilder().Where(filter).WithMetrics(metrics).CountAsync(ct)); Assert.Equal(0, metrics.ValuesDecoded); Assert.Equal(oracle.Count > 0, await file.ScanBuilder().Where(filter).AnyAsync(ct)); @@ -401,8 +401,8 @@ public async Task AnExactCoverCountsReadsAndBoundsLikeTheDecode(string text) // The scan delivers exactly the proven rows, with the index and without; with it, a batch's // worth of rows or fewer is a take, which decodes no more than the pruned scan does. - ScanMetrics indexed = new ScanMetrics(); - ScanMetrics unindexed = new ScanMetrics(); + ScanCounters indexed = new ScanCounters(); + ScanCounters unindexed = new ScanCounters(); List on = await RowsOf(file.ScanBuilder().Where(filter).WithMetrics(indexed)); List off = await RowsOf(file.ScanBuilder().Where(filter).WithIndexes(false).WithMetrics(unindexed)); Assert.True( @@ -709,7 +709,7 @@ private static async Task AssertDistinctWalk(VortexFile file, string column, Key } Assert.Equal(firsts.Count, index); - for (bool ok = await cursor.SeekLastAsync(); ok; ok = await cursor.PrevAsync()) + for (bool ok = await cursor.SeekLastAsync(); ok; ok = await cursor.PreviousAsync()) { AssertDistinctAt(cursor, firsts, --index, rows); } @@ -718,16 +718,16 @@ private static async Task AssertDistinctWalk(VortexFile file, string column, Key // Backward seeks land on the key's first entry too. int middle = firsts.Count / 2; - Assert.True(await cursor.SeekAsync(firsts[middle].Key, SeekOp.AtOrBefore)); + Assert.True(await cursor.SeekAsync(firsts[middle].Key, SeekMode.AtOrBefore)); AssertDistinctAt(cursor, firsts, middle, rows); - Assert.True(await cursor.SeekAsync(firsts[middle].Key, SeekOp.Before)); + Assert.True(await cursor.SeekAsync(firsts[middle].Key, SeekMode.Before)); AssertDistinctAt(cursor, firsts, middle - 1, rows); if (!rows) { Assert.Null(cursor.EntryCount); Assert.Throws(() => cursor.Row); - await Assert.ThrowsAsync(async () => await cursor.KeyCountAsync()); + await Assert.ThrowsAsync(async () => await cursor.CountAtKeyAsync()); await Assert.ThrowsAsync(async () => await cursor.RankAsync(firsts[0].Key)); await Assert.ThrowsAsync(async () => await cursor.SeekRankAsync(0)); } @@ -780,7 +780,7 @@ public async Task AKeyOrderedScanIsTheOracleWindowByWindow(string column, string Decoders.EnsureRegistered(); await using Written written = await Written.CreateAsync(); List oracle = Oracle(column); - ScanMetrics metrics = new ScanMetrics(); + ScanCounters metrics = new ScanCounters(); Vorticity.Scanning.ScanBuilder scan = written.File.ScanBuilder() .InKeyOrder(column, descending) .WithDegreeOfParallelism(degree) @@ -881,14 +881,14 @@ public async Task AConsumerThatStopsNeverReadsThePastWindows() { Decoders.EnsureRegistered(); await using Written written = await Written.CreateAsync(); - ScanMetrics first = new ScanMetrics(); + ScanCounters first = new ScanCounters(); await foreach (RecordBatch batch in written.File.ScanBuilder().InKeyOrder("text").WithMaxBatchRows(10).WithMetrics(first).ExecuteAsync()) { Assert.Equal(10, batch.RowCount); break; } - ScanMetrics all = new ScanMetrics(); + ScanCounters all = new ScanCounters(); await foreach (RecordBatch batch in written.File.ScanBuilder().InKeyOrder("text").WithMaxBatchRows(10).WithMetrics(all).ExecuteAsync()) { Assert.True(batch.RowCount <= 10); @@ -1048,7 +1048,7 @@ private static void AssertAt(KeyCursor cursor, List oracle, int index) } } - private static async Task Check(KeyCursor cursor, List oracle, FilterLiteral key, SeekOp op, int expected) + private static async Task Check(KeyCursor cursor, List oracle, FilterLiteral key, SeekMode op, int expected) { bool found = await cursor.SeekAsync(key, op); bool exists = expected >= 0 && expected < oracle.Count; diff --git a/tests/Vorticity.Tests/Layouts/SharedChildTests.cs b/tests/Vorticity.Tests/Layouts/SharedChildTests.cs index c2f9329e..81760459 100644 --- a/tests/Vorticity.Tests/Layouts/SharedChildTests.cs +++ b/tests/Vorticity.Tests/Layouts/SharedChildTests.cs @@ -60,7 +60,7 @@ public async Task ADictionarysValuesAreDecodedOnceForEveryWindowOfTheChunk(int p } Assert.Equal(Rows, seen); - Assert.Equal(2L * Rows, scan.Metrics.ValuesDecoded); + Assert.Equal(2L * Rows, scan.Counters.ValuesDecoded); Assert.Equal(1, ArrayDecodeContext.SharedChildrenDecoded - before); } finally diff --git a/tests/Vorticity.Tests/Layouts/WindowedChunkTests.cs b/tests/Vorticity.Tests/Layouts/WindowedChunkTests.cs index 1b47200d..49666888 100644 --- a/tests/Vorticity.Tests/Layouts/WindowedChunkTests.cs +++ b/tests/Vorticity.Tests/Layouts/WindowedChunkTests.cs @@ -48,7 +48,7 @@ public async Task AChunkLargerThanAWindowReadsBackRowForRowAndDecodesEachValueOn } Assert.Equal(Rows, seen); - Assert.Equal(5L * Rows, scan.Metrics.ValuesDecoded); + Assert.Equal(5L * Rows, scan.Counters.ValuesDecoded); } finally { @@ -91,7 +91,7 @@ public async Task ARunEndChunkKeptEncodedReadsBackInWindowsRunForRun() } Assert.Equal(Rows, seen); - Assert.Equal(Rows, scan.Metrics.ValuesDecoded); + Assert.Equal(Rows, scan.Counters.ValuesDecoded); } finally { diff --git a/tests/Vorticity.Tests/Matrix/DecimalAggregateTests.cs b/tests/Vorticity.Tests/Matrix/DecimalAggregateTests.cs index a5960f82..195a54f9 100644 --- a/tests/Vorticity.Tests/Matrix/DecimalAggregateTests.cs +++ b/tests/Vorticity.Tests/Matrix/DecimalAggregateTests.cs @@ -55,7 +55,7 @@ public async Task EveryDecimalWidthAggregatesInOnePassAndPerGroup(Shape shape) // In one pass: the answers a single scan computes together are the ones computed apart. (VortexDecimal? min, VortexDecimal? max, long distinct, double? mean) = await file.Scan() - .AggAsync(a => (a.Min(r => r.Wide256), a.Max(r => r.Wide256), a.CountDistinct(r => r.Wide256), a.Average(r => r.Wide256)), ct); + .AggregateAsync(a => (a.Min(r => r.Wide256), a.Max(r => r.Wide256), a.CountDistinct(r => r.Wide256), a.Average(r => r.Wide256)), ct); List wide = rows.Wide256.Select(v => Unscaled(v!.Value, 10)).ToList(); Assert.Equal(wide.Min(), Unscaled(min!.Value, 10)); Assert.Equal(wide.Max(), Unscaled(max!.Value, 10)); diff --git a/tests/Vorticity.Tests/Scan/BlockPruningDecodeTests.cs b/tests/Vorticity.Tests/Scan/BlockPruningDecodeTests.cs index a74e18da..9ce3dcf1 100644 --- a/tests/Vorticity.Tests/Scan/BlockPruningDecodeTests.cs +++ b/tests/Vorticity.Tests/Scan/BlockPruningDecodeTests.cs @@ -7,7 +7,7 @@ // rows as a counted range -- and an encoding that selects without a full decode materializes the // live block and nothing else. // -// The quantity is the scan's `ScanMetrics.ValuesDecoded`, the count `FlatLayoutDecodeCountTests` +// The quantity is the scan's `ScanCounters.ValuesDecoded`, the count `FlatLayoutDecodeCountTests` // holds the scan and the take to, and it is exact here too: one block's rows, pruned; the whole chunk, // unpruned. Two shapes, because two readers have the branch: a file whose single chunk IS the root // (the flat reader sees the mask in file coordinates), and a file of many chunks (the chunked @@ -145,8 +145,8 @@ public async Task TheSplitsTheZoneMapsProveAreNotEvaluated() proven += least >= low && greatest < high ? 1 : 0; } - ScanMetrics pruned = new ScanMetrics(); - ScanMetrics unpruned = new ScanMetrics(); + ScanCounters pruned = new ScanCounters(); + ScanCounters unpruned = new ScanCounters(); List kept = await Collect(path, band, prune: true, pruned); List all = await Collect(path, band, prune: false, unpruned); @@ -165,7 +165,7 @@ public async Task TheSplitsTheZoneMapsProveAreNotEvaluated() } } - private static async Task> Collect(string path, VortexExpr filter, bool prune, ScanMetrics metrics) + private static async Task> Collect(string path, VortexExpr filter, bool prune, ScanCounters metrics) { List values = []; byte[] name = System.Text.Encoding.UTF8.GetBytes("v"); @@ -244,7 +244,7 @@ private static async Task FirstValue(string path) private static async Task<(List Values, long Decoded)> ReadField( string path, VortexExpr filter, bool prune, int cap) { - ScanMetrics metrics = new ScanMetrics(); + ScanCounters metrics = new ScanCounters(); List values = []; byte[] name = System.Text.Encoding.UTF8.GetBytes(string.Empty); await using VortexFile file = await VortexFile.OpenAsync(path, CancellationToken.None); @@ -290,7 +290,7 @@ private static string Describe(LayoutNode node, int depth) private static async Task<(List Values, long Decoded)> Read(string path, VortexExpr filter, bool prune) { - ScanMetrics metrics = new ScanMetrics(); + ScanCounters metrics = new ScanCounters(); List values = []; byte[] name = System.Text.Encoding.UTF8.GetBytes("v"); await using VortexFile file = await VortexFile.OpenAsync(path, CancellationToken.None); diff --git a/tests/Vorticity.Tests/Scan/FilterOwnPassTests.cs b/tests/Vorticity.Tests/Scan/FilterOwnPassTests.cs index 876717f4..863d2eff 100644 --- a/tests/Vorticity.Tests/Scan/FilterOwnPassTests.cs +++ b/tests/Vorticity.Tests/Scan/FilterOwnPassTests.cs @@ -112,7 +112,7 @@ public async Task ASplitTheFilterKeepsNothingOfDecodesOnlyTheFiltersColumn(strin string path = Write(); try { - ScanMetrics metrics = new ScanMetrics(); + ScanCounters metrics = new ScanCounters(); long rows = 0; await using (VortexFile file = await VortexFile.OpenAsync(path, CancellationToken.None)) { @@ -171,7 +171,7 @@ public async Task ARunKeptOfEveryZoneReadsEachChunkOnceAndSlicesIt() // The first 2 000 rows of every zone: too few of a split to read the projection whole, // but every row of what they span, in a chunk of twelve batches that no window smaller // than the chunk cuts. - ScanMetrics metrics = new ScanMetrics(); + ScanCounters metrics = new ScanCounters(); List got = []; await using (VortexFile file = await VortexFile.OpenAsync(path, CancellationToken.None)) { @@ -297,7 +297,7 @@ private static VortexExpr Filter(string name) /// Every column of every row the filter keeps, at a degree, and the values the scan decoded. private static async Task<(List Values, long Decoded)> ReadAsync(string path, string filter, int degree) { - ScanMetrics metrics = new ScanMetrics(); + ScanCounters metrics = new ScanCounters(); List values = []; await using (VortexFile file = await VortexFile.OpenAsync(path, CancellationToken.None)) { diff --git a/tests/Vorticity.Tests/Scan/FilterPushDownTests.cs b/tests/Vorticity.Tests/Scan/FilterPushDownTests.cs index ee461015..eefb85ef 100644 --- a/tests/Vorticity.Tests/Scan/FilterPushDownTests.cs +++ b/tests/Vorticity.Tests/Scan/FilterPushDownTests.cs @@ -1,7 +1,7 @@ // An equality a string encoding answers from what it stores, and the proof that the column is // never built to answer it. // -// The quantity is the scan's `ScanMetrics.ValuesDecoded`, the same count the block-pruning tests read: +// The quantity is the scan's `ScanCounters.ValuesDecoded`, the same count the block-pruning tests read: // it counts what a scan materialized, so a push that works shows as a count near the number of // rows the predicate keeps rather than near the number of rows the file holds. Correctness is // asserted against the same scan without a predicate, filtered in memory, so the two paths have to @@ -100,7 +100,7 @@ private static (EncodingHint Hint, Func Value, string Needle) Fixtu private static async Task<(List Values, long Decoded)> ReadAsync( string path, VortexExpr? filter) { - ScanMetrics metrics = new ScanMetrics(); + ScanCounters metrics = new ScanCounters(); List values = []; await using VortexFile file = await VortexFile.OpenAsync(path, CancellationToken.None); ScanBuilder scan = file.ScanBuilder().WithMetrics(metrics); diff --git a/tests/Vorticity.Tests/Scan/FilteredFirstBatchTests.cs b/tests/Vorticity.Tests/Scan/FilteredFirstBatchTests.cs index f2454f37..e83e2063 100644 --- a/tests/Vorticity.Tests/Scan/FilteredFirstBatchTests.cs +++ b/tests/Vorticity.Tests/Scan/FilteredFirstBatchTests.cs @@ -33,14 +33,14 @@ public async Task AFilteredTakeAsksForAsManySegmentsOnAFileSixteenTimesLarger() Projection fromSixteen = sixteen.Scan().Where(r => r.Noise > 5).Select(r => r.Key).Take(10); Assert.Equal(expected, await fromOne.ToListAsync(Ct)); Assert.Equal(expected, await fromSixteen.ToListAsync(Ct)); - Assert.Equal(fromOne.Statistics.Requests, fromSixteen.Statistics.Requests); - Assert.True(fromSixteen.Statistics.BytesRequested >= fromOne.Statistics.BytesRequested); + Assert.Equal(fromOne.Metrics.Requests, fromSixteen.Metrics.Requests); + Assert.True(fromSixteen.Metrics.BytesRequested >= fromOne.Metrics.BytesRequested); // The first batch of the scan alone, as a consumer that stops there reads it. Scan scanOne = one.Scan().Where(r => r.Noise > 5); Scan scanSixteen = sixteen.Scan().Where(r => r.Noise > 5); Assert.Equal(await FirstAsync(scanOne), await FirstAsync(scanSixteen)); - Assert.Equal(scanOne.Statistics.Requests, scanSixteen.Statistics.Requests); + Assert.Equal(scanOne.Metrics.Requests, scanSixteen.Metrics.Requests); } finally { diff --git a/tests/Vorticity.Tests/Scan/FlatLayoutDecodeCountTests.cs b/tests/Vorticity.Tests/Scan/FlatLayoutDecodeCountTests.cs index ccd39b9d..f3a6eb66 100644 --- a/tests/Vorticity.Tests/Scan/FlatLayoutDecodeCountTests.cs +++ b/tests/Vorticity.Tests/Scan/FlatLayoutDecodeCountTests.cs @@ -54,7 +54,7 @@ public async Task AScanMaterializesEveryValueOncePerBatch() try { - ScanMetrics metrics = new ScanMetrics(); + ScanCounters metrics = new ScanCounters(); long batches = 0; long rows = 0; await using (VortexFile opened = await VortexFile.OpenAsync(path, CancellationToken.None)) @@ -221,7 +221,7 @@ public async Task ATakeOnAnOversizedChunkMaterializesItOnce(bool booleans) expected.Add(all[(int)index]); } - ScanMetrics metrics = new ScanMetrics(); + ScanCounters metrics = new ScanCounters(); List taken = []; await using (VortexFile opened = await VortexFile.OpenAsync(path, CancellationToken.None)) { @@ -276,7 +276,7 @@ public async Task ANarrowFilterOnASortedColumnDecodesTheZonesItLandsOn() string path = await WriteSortedAsync(SortedRows); try { - ScanMetrics metrics = new ScanMetrics(); + ScanCounters metrics = new ScanCounters(); long rows = 0; VortexExpr band = Expr.And( Expr.Ge(Expr.Field("key"), Expr.Literal(FilterLiteral.From(Low))), diff --git a/tests/Vorticity.Tests/Scan/PrefixPushDownTests.cs b/tests/Vorticity.Tests/Scan/PrefixPushDownTests.cs index 44d692f3..36752db7 100644 --- a/tests/Vorticity.Tests/Scan/PrefixPushDownTests.cs +++ b/tests/Vorticity.Tests/Scan/PrefixPushDownTests.cs @@ -6,7 +6,7 @@ // ordinal StartsWith, for prefixes that end inside a multi-byte character, are longer than every // value, are empty, or match nothing: the answer has to agree row for row whichever path gave it, // and a null row is never selected. The quantity that shows the encoding answered is the scan's -// `ScanMetrics.ValuesDecoded`, as in FilterPushDownTests. +// `ScanCounters.ValuesDecoded`, as in FilterPushDownTests. using System; using System.Buffers.Binary; using System.Collections.Generic; @@ -147,7 +147,7 @@ private static string[] WordPrefixes() private static async Task<(List Values, long Decoded)> ReadAsync(string path, VortexExpr? filter) { - ScanMetrics metrics = new ScanMetrics(); + ScanCounters metrics = new ScanCounters(); List values = []; await using VortexFile file = await VortexFile.OpenAsync(path, CancellationToken.None); ScanBuilder scan = file.ScanBuilder().WithMetrics(metrics); diff --git a/tests/Vorticity.Tests/Scan/ScanContextPoolTests.cs b/tests/Vorticity.Tests/Scan/ScanContextPoolTests.cs index 5dba5b99..e1065f6c 100644 --- a/tests/Vorticity.Tests/Scan/ScanContextPoolTests.cs +++ b/tests/Vorticity.Tests/Scan/ScanContextPoolTests.cs @@ -42,7 +42,7 @@ public async Task AContextComesBackHoldingNothingOfTheScanItServed() DTypeArena types = context.Types; types.Primitive(PType.I32, Nullability.NonNullable); context.LiveBlocks = new BlockMask(first.RowCount, 1024); - context.Metrics = new ScanMetrics(); + context.Metrics = new ScanCounters(); context.KeepEncodings = true; context.ExchangePushedFields(FieldMask.All); context.FieldsHonoured = true; diff --git a/tests/Vorticity.Tests/Scan/ScanExclusionTests.cs b/tests/Vorticity.Tests/Scan/ScanExclusionTests.cs index 96685ee6..2ffc2a35 100644 --- a/tests/Vorticity.Tests/Scan/ScanExclusionTests.cs +++ b/tests/Vorticity.Tests/Scan/ScanExclusionTests.cs @@ -229,7 +229,7 @@ public async Task AKeyCursorStepsAndRanksOverTheRowsLeftOut(string column) Assert.Equal(kept, up); List<(long Key, long Row)> down = []; - for (bool at = await cursor.SeekLastAsync(ct); at; at = await cursor.PrevAsync(ct)) + for (bool at = await cursor.SeekLastAsync(ct); at; at = await cursor.PreviousAsync(ct)) { down.Add((cursor.Key.SignedValue, cursor.Row)); } @@ -246,7 +246,7 @@ public async Task AKeyCursorStepsAndRanksOverTheRowsLeftOut(string column) { Assert.True(await cursor.SeekRankAsync(rank, ct)); Assert.Equal(kept[rank], (cursor.Key.SignedValue, cursor.Row)); - Assert.Equal(kept.Count(entry => entry.Key == kept[rank].Key), await cursor.KeyCountAsync(ct)); + Assert.Equal(kept.Count(entry => entry.Key == kept[rank].Key), await cursor.CountAtKeyAsync(ct)); } } finally diff --git a/tests/Vorticity.Tests/Scan/ScanExplainTests.cs b/tests/Vorticity.Tests/Scan/ScanExplainTests.cs index 79d9d830..ea241d6f 100644 --- a/tests/Vorticity.Tests/Scan/ScanExplainTests.cs +++ b/tests/Vorticity.Tests/Scan/ScanExplainTests.cs @@ -1,4 +1,4 @@ -// Explain and ScanMetrics: the plan without executing, and the same quantities after execution, +// Explain and ScanCounters: the plan without executing, and the same quantities after execution, // without which nobody can tell whether an index earns its bytes. // // Three things are held. `ExplainAsync` reads nothing but the zone maps a filter can use (the @@ -72,7 +72,7 @@ public async Task ThePlanOfAPrunedScanReadsOnlyTheZoneMapsAndAgreesWithTheMask() Assert.True(plan.FileMayMatch); // The scan that follows, with its metrics set against what a caller counts by hand. - ScanMetrics metrics = new ScanMetrics(); + ScanCounters metrics = new ScanCounters(); counting.ResetCounters(); long rows = 0; long batches = 0; diff --git a/tests/Vorticity.Tests/Scan/ScanTerminalTests.cs b/tests/Vorticity.Tests/Scan/ScanTerminalTests.cs index 0fb08e9e..619d78ed 100644 --- a/tests/Vorticity.Tests/Scan/ScanTerminalTests.cs +++ b/tests/Vorticity.Tests/Scan/ScanTerminalTests.cs @@ -167,7 +167,7 @@ public async Task WithoutAFilterTheCountIsArithmeticAndReadsNothing() Decoders.EnsureRegistered(); await using VortexFile file = await VortexFile.OpenAsync(Corpus.Path(Zoned), CancellationToken.None); - ScanMetrics metrics = new ScanMetrics(); + ScanCounters metrics = new ScanCounters(); Assert.Equal(Rows, await file.ScanBuilder().WithMetrics(metrics).CountAsync(ct)); Assert.True(await file.ScanBuilder().WithMetrics(metrics).AnyAsync(ct)); Assert.Equal(5_000, await file.ScanBuilder().Rows(RowRange.FromLength(1_000, 5_000)).WithMetrics(metrics).CountAsync(ct)); @@ -195,8 +195,8 @@ public async Task TheFullBlockProofCountsWithoutReadingTheBlock() Assert.Equal(1, plan.LiveBlocks); long zoneMaps = plan.Pruning[0].SegmentsRead; - ScanMetrics proven = new ScanMetrics(); - ScanMetrics decoded = new ScanMetrics(); + ScanCounters proven = new ScanCounters(); + ScanCounters decoded = new ScanCounters(); Assert.Equal(Block, await file.ScanBuilder().Where(filter).WithMetrics(proven).CountAsync(ct)); Assert.Equal(Block, await file.ScanBuilder().Where(filter).WithMetrics(decoded) .WithTiers(TerminalTiers.All & ~TerminalTiers.FullBlock).CountAsync(ct)); @@ -209,7 +209,7 @@ public async Task TheFullBlockProofCountsWithoutReadingTheBlock() Assert.True(decoded.ValuesDecoded > proven.ValuesDecoded); // Under a take the same proof serves, whole: every taken row of a block proven whole. - ScanMetrics taken = new ScanMetrics(); + ScanCounters taken = new ScanCounters(); Assert.Equal(2, await file.ScanBuilder().Where(filter).Take([5_200L, 5_300L, 20_000L]).WithMetrics(taken).CountAsync(ct)); Assert.Equal(0, taken.SegmentRequests); } @@ -225,7 +225,7 @@ public async Task AnEmptyMaskAnswersWithoutADataRead() ScanExplanation plan = await file.ScanBuilder().Where(filter).ExplainAsync(ct); Assert.Equal(0, plan.LiveBlocks); - ScanMetrics metrics = new ScanMetrics(); + ScanCounters metrics = new ScanCounters(); Assert.False(await file.ScanBuilder().Where(filter).WithMetrics(metrics).AnyAsync(ct)); Assert.Equal(0, await file.ScanBuilder().Where(filter).WithMetrics(metrics).CountAsync(ct)); @@ -246,8 +246,8 @@ public async Task AnyStopsAtTheFirstSplitThatCounts() VortexExpr filter = Filter("nulls > 30000"); await using VortexFile file = await VortexFile.OpenAsync(Corpus.Path(Zoned), CancellationToken.None); - ScanMetrics any = new ScanMetrics(); - ScanMetrics count = new ScanMetrics(); + ScanCounters any = new ScanCounters(); + ScanCounters count = new ScanCounters(); TerminalTiers decodeOnly = TerminalTiers.All & ~TerminalTiers.FullBlock; Assert.True(await file.ScanBuilder().Where(filter).WithMetrics(any).WithTiers(decodeOnly).AnyAsync(ct)); Assert.True(await file.ScanBuilder().Where(filter).WithMetrics(count).WithTiers(decodeOnly).CountAsync(ct) > 0); @@ -267,7 +267,7 @@ public async Task TheFileStatisticAnswersTheWholeFileWithoutARead() await using VortexFile file = await VortexFile.OpenAsync(Corpus.Path(Zoned), CancellationToken.None); Assert.True(file.HasFileStatistics); - ScanMetrics metrics = new ScanMetrics(); + ScanCounters metrics = new ScanCounters(); FilterLiteral min = await file.ScanBuilder().WithMetrics(metrics).MinAsync("monotone", ct); FilterLiteral max = await file.ScanBuilder().WithMetrics(metrics).MaxAsync("monotone", ct); FilterLiteral last = await file.ScanBuilder().WithMetrics(metrics).MaxAsync("strs", ct); @@ -291,8 +291,8 @@ public async Task TheZoneBoundsAnswerWholeZonesWithoutADecode() await using VortexFile file = await VortexFile.OpenAsync(Corpus.Path(Zoned), CancellationToken.None); RowRange whole = new RowRange(0, Rows); - ScanMetrics bounds = new ScanMetrics(); - ScanMetrics decode = new ScanMetrics(); + ScanCounters bounds = new ScanCounters(); + ScanCounters decode = new ScanCounters(); FilterLiteral fromBounds = await file.ScanBuilder().Rows(whole).WithMetrics(bounds).MinAsync("monotone", ct); FilterLiteral fromDecode = await file.ScanBuilder().Rows(whole).WithMetrics(decode) .WithTiers(TerminalTiers.All & ~TerminalTiers.ZoneBounds).MinAsync("monotone", ct); @@ -315,8 +315,8 @@ public async Task AnInexactBoundIsACandidateDecodedOnlyWhenItCouldWin() await using VortexFile file = await VortexFile.OpenAsync(Corpus.Path(Zoned), CancellationToken.None); RowRange whole = new RowRange(0, Rows); - ScanMetrics bounded = new ScanMetrics(); - ScanMetrics decoded = new ScanMetrics(); + ScanCounters bounded = new ScanCounters(); + ScanCounters decoded = new ScanCounters(); FilterLiteral min = await file.ScanBuilder().Rows(whole).WithMetrics(bounded).MinAsync("strs", ct); FilterLiteral max = await file.ScanBuilder().Rows(whole).WithMetrics(bounded).MaxAsync("strs", ct); FilterLiteral minDecoded = await file.ScanBuilder().Rows(whole).WithMetrics(decoded).WithTiers(TerminalTiers.None).MinAsync("strs", ct); diff --git a/tests/Vorticity.Tests/Scan/SelectTests.cs b/tests/Vorticity.Tests/Scan/SelectTests.cs index 0a9d63eb..9e492b51 100644 --- a/tests/Vorticity.Tests/Scan/SelectTests.cs +++ b/tests/Vorticity.Tests/Scan/SelectTests.cs @@ -106,8 +106,8 @@ public async Task SkipAndTakeAreTheWindowOfTheRowsKept() Assert.Equal(10, (await ten.ToListAsync(Ct)).Count); Projection all = file.Scan().Select(r => r.Id); Assert.Equal(Rows, (await all.ToListAsync(Ct)).Count); - Assert.Equal(1, ten.Statistics.Batches); - Assert.True(ten.Statistics.Requests < all.Statistics.Requests, $"{ten.Statistics.Requests} requests against {all.Statistics.Requests}"); + Assert.Equal(1, ten.Metrics.Batches); + Assert.True(ten.Metrics.Requests < all.Metrics.Requests, $"{ten.Metrics.Requests} requests against {all.Metrics.Requests}"); } finally { @@ -133,7 +133,7 @@ public async Task DistinctValuesStreamAsTheyAreMet() // The first three cities met, without reading past the rows that hold them. Aggregation three = file.Scan().Select(r => r.City).Distinct().Take(3); Assert.Equal(rows.Select(r => r.City).Distinct().Take(3), await three.ToListAsync(Ct)); - Assert.Equal(1, three.Statistics.Batches); + Assert.Equal(1, three.Metrics.Batches); } finally { diff --git a/tests/Vorticity.Tests/Types/FieldNameLookupTests.cs b/tests/Vorticity.Tests/Types/FieldNameLookupTests.cs index 04704c25..90c716d9 100644 --- a/tests/Vorticity.Tests/Types/FieldNameLookupTests.cs +++ b/tests/Vorticity.Tests/Types/FieldNameLookupTests.cs @@ -32,9 +32,8 @@ public void AFieldIsFoundByNameAndByPathWhateverTheWidth(int width) Assert.Equal(-1, schema.IndexOf($"nested.leaf{width}")); Assert.Equal(-1, schema.IndexOf($"col{width + 5}")); Assert.Equal(-1, schema.IndexOf("col0.leaf1")); - Assert.True(schema.TryGetField("nested.leaf5"u8, out int leaf)); - Assert.Equal(5, leaf); - Assert.False(schema.TryGetField([0xC3, 0x28], out _)); + Assert.Equal(5, schema.IndexOf("nested.leaf5"u8)); + Assert.Equal(-1, schema.IndexOf([0xC3, 0x28])); // The tool scan's column names resolve the same way, a path to its indices. Assert.Equal([width, 5], ToolPaths.Resolve(schema, "nested.leaf5")); diff --git a/tests/Vorticity.Tests/Writing/DurableWriteTests.cs b/tests/Vorticity.Tests/Writing/DurableWriteTests.cs index 876a8464..f34c1d41 100644 --- a/tests/Vorticity.Tests/Writing/DurableWriteTests.cs +++ b/tests/Vorticity.Tests/Writing/DurableWriteTests.cs @@ -57,7 +57,7 @@ public async Task ADurableAppendReachesTheDeviceBeforeItsNewPostscript() } List flushes; - await using (VortexFileWriter append = await VortexSession.Default.AppendAsync(path, new VortexWriteOptions { Durable = true }, TestContext.Current.CancellationToken)) + await using (VortexFileWriter append = await VortexSession.Default.OpenWriterAsync(path, new VortexWriteOptions { Durable = true }, TestContext.Current.CancellationToken)) { flushes = FilePipeWriter.WatchDiskFlushes(append.FilePipe!); await WriteRowsAsync(append, Rows); diff --git a/tests/Vorticity.Tests/Writing/FileStatisticsWriteTests.cs b/tests/Vorticity.Tests/Writing/FileStatisticsWriteTests.cs index 0419fbb9..068d8fbd 100644 --- a/tests/Vorticity.Tests/Writing/FileStatisticsWriteTests.cs +++ b/tests/Vorticity.Tests/Writing/FileStatisticsWriteTests.cs @@ -133,7 +133,7 @@ public async Task TheReaderAnswersAWholeFileExtremeFromTheSegment() Decoders.EnsureRegistered(); await using Written written = await Written.CreateAsync(new VortexWriteOptions { RowBlockSize = Block }); - ScanMetrics metrics = new ScanMetrics(); + ScanCounters metrics = new ScanCounters(); FilterLiteral min = await written.File.ScanBuilder().WithMetrics(metrics).MinAsync("strict_i64", ct); FilterLiteral max = await written.File.ScanBuilder().WithMetrics(metrics).MaxAsync("dups_u32", ct); diff --git a/tests/Vorticity.Tests/Writing/RoundTripSweepTests.cs b/tests/Vorticity.Tests/Writing/RoundTripSweepTests.cs index 391d9198..5cd4c513 100644 --- a/tests/Vorticity.Tests/Writing/RoundTripSweepTests.cs +++ b/tests/Vorticity.Tests/Writing/RoundTripSweepTests.cs @@ -363,7 +363,7 @@ private static async Task CompactedAsync(VortexFile source, VortexWriteO Assert.Equal((3L, 1L, CompactionStyle.Tiered), (result.ObjectsIn, result.ObjectsOut, result.Style)); string key = string.Empty; - await foreach (DataObject held in dataset.ObjectsAsync()) + await foreach (DataObject held in dataset.ListObjectsAsync()) { key = held.Key; }