From a1cb6f66cfe127cf0adf7ba13cfd69715c177a33 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sat, 6 Jun 2026 20:19:44 +1000 Subject: [PATCH 01/43] fix (UI): Fix display of path in grid (incorrectly concatenating partial paths) --- src/cdeLib/Entities/EntryHelper.cs | 5 ++++- src/cdeLibTest/EntryHelperTest.cs | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/cdeLib/Entities/EntryHelper.cs b/src/cdeLib/Entities/EntryHelper.cs index 0a61bdc..a4e8c6e 100644 --- a/src/cdeLib/Entities/EntryHelper.cs +++ b/src/cdeLib/Entities/EntryHelper.cs @@ -31,10 +31,13 @@ public static IEnumerable GetPairDirEntries(IEnumerable /// public static string MakeFullPath(ICommonEntry parentEntry, ICommonEntry dirEntry) { + // Resolve parent path FIRST — this may recursively re-enter MakeFullPath and + // mutate the shared ThreadLocal StringBuilder, so do it before we clear/use sb. + var parentPath = parentEntry.FullPath; + var sb = PathBuilder.Value!; sb.Clear(); - var parentPath = parentEntry.FullPath; if (parentPath != null) { sb.Append(parentPath); diff --git a/src/cdeLibTest/EntryHelperTest.cs b/src/cdeLibTest/EntryHelperTest.cs index 5ef647f..4ec5421 100644 --- a/src/cdeLibTest/EntryHelperTest.cs +++ b/src/cdeLibTest/EntryHelperTest.cs @@ -113,6 +113,26 @@ public void MakeFullPath_MultipleCallsWithVaryingLengths_DoesNotThrow() shortResult2.ShouldBe(@"C:\short"); } + [Test] + public void FullPath_NestedDirEntries_DoesNotDuplicateParentPath() + { + // Regression: DirEntry.FullPath recurses into MakeFullPath using a shared + // ThreadLocal StringBuilder. Previously the buffer was cleared before the + // recursive parent lookup, so the parent path was left in the buffer and + // appended twice (e.g. "D:\ArchiveD:\Archive\Apps2021"). + var re = new RootEntry(_config) { Path = @"D:\" }; + var archive = new DirEntry(true) { Path = "Archive" }; + var apps = new DirEntry(true) { Path = "Apps2021" }; + re.AddChild(archive); + archive.AddChild(apps); + re.SetInMemoryFields(); + + // Depths 1-3 should all produce a single, correct full path. + re.FullPath.ShouldBe(@"D:\"); + archive.FullPath.ShouldBe(@"D:\Archive"); + apps.FullPath.ShouldBe(@"D:\Archive\Apps2021"); + } + [Test] public void MakeFullPathPooled_IsAliasForMakeFullPath() { From f02f1cbfdd59c23f383449d206fb7ccbd2384657 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sat, 6 Jun 2026 20:44:27 +1000 Subject: [PATCH 02/43] build: Migrate build automation from Nuke to Fallout Replace the Nuke build with Fallout (Fallout.Common), moving config from .nuke/ to .fallout/ and updating the bootstrap scripts and build project. Adopt the .slnx solution format and refresh project package references, including pinning Microsoft.CodeAnalysis.*.Workspaces to 4.14.0 in cdeBenchmarks to resolve an NU1608 transitive version conflict. Update CLAUDE.md to document the Fallout build system. --- {.nuke => .fallout}/build.schema.json | 4 ++-- {.nuke => .fallout}/parameters.json | 0 build.ps1 | 2 +- build.sh | 2 +- build/Build.cs | 18 +++++++++--------- build/Configuration.cs | 2 +- build/_build.csproj | 11 ++++++----- claude.md | 7 ++++--- src/cde.slnx | 20 ++++++++++++++++++++ src/cde/cde.csproj | 22 +++++++++++----------- src/cdeBenchmarks/cdeBenchmarks.csproj | 5 +++++ src/cdeLib/cdeLib.csproj | 20 ++++++++++---------- src/cdeLibTest/cdeLibTest.csproj | 8 ++++---- src/cdeWin/cdeWin.csproj | 4 ++-- src/cdeWinTest/cdeWinTest.csproj | 6 +++--- 15 files changed, 79 insertions(+), 52 deletions(-) rename {.nuke => .fallout}/build.schema.json (97%) rename {.nuke => .fallout}/parameters.json (100%) create mode 100644 src/cde.slnx diff --git a/.nuke/build.schema.json b/.fallout/build.schema.json similarity index 97% rename from .nuke/build.schema.json rename to .fallout/build.schema.json index a1458c4..ca39e4c 100644 --- a/.nuke/build.schema.json +++ b/.fallout/build.schema.json @@ -44,7 +44,7 @@ "Quiet" ] }, - "NukeBuild": { + "FalloutBuild": { "properties": { "Continue": { "type": "boolean", @@ -120,7 +120,7 @@ } }, { - "$ref": "#/definitions/NukeBuild" + "$ref": "#/definitions/FalloutBuild" } ] } diff --git a/.nuke/parameters.json b/.fallout/parameters.json similarity index 100% rename from .nuke/parameters.json rename to .fallout/parameters.json diff --git a/build.ps1 b/build.ps1 index bbaa118..6256a32 100644 --- a/build.ps1 +++ b/build.ps1 @@ -14,7 +14,7 @@ $PSScriptRoot = Split-Path $MyInvocation.MyCommand.Path -Parent ########################################################################### $BuildProjectFile = "$PSScriptRoot\build\_build.csproj" -$TempDirectory = "$PSScriptRoot\\.nuke\temp" +$TempDirectory = "$PSScriptRoot\\.fallout\temp" $DotNetGlobalFile = "$PSScriptRoot\\global.json" $DotNetInstallUrl = "https://dot.net/v1/dotnet-install.ps1" diff --git a/build.sh b/build.sh index e8961f9..2e66cb3 100755 --- a/build.sh +++ b/build.sh @@ -10,7 +10,7 @@ SCRIPT_DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd) ########################################################################### BUILD_PROJECT_FILE="$SCRIPT_DIR/build/_build.csproj" -TEMP_DIRECTORY="$SCRIPT_DIR//.nuke/temp" +TEMP_DIRECTORY="$SCRIPT_DIR//.fallout/temp" DOTNET_GLOBAL_FILE="$SCRIPT_DIR//global.json" DOTNET_INSTALL_URL="https://dot.net/v1/dotnet-install.sh" diff --git a/build/Build.cs b/build/Build.cs index 431fc50..f2709ec 100644 --- a/build/Build.cs +++ b/build/Build.cs @@ -2,18 +2,18 @@ using JetBrains.Annotations; using Microsoft.Build.Evaluation; using Microsoft.Build.Locator; -using Nuke.Common; -using Nuke.Common.CI; -using Nuke.Common.Git; -using Nuke.Common.IO; -using Nuke.Common.ProjectModel; -using Nuke.Common.Tools.DotNet; -using Nuke.Common.Utilities.Collections; -using static Nuke.Common.Tools.DotNet.DotNetTasks; +using Fallout.Common; +using Fallout.Common.CI; +using Fallout.Common.Git; +using Fallout.Common.IO; +using Fallout.Common.ProjectModel; +using Fallout.Common.Tools.DotNet; +using Fallout.Common.Utilities.Collections; +using static Fallout.Common.Tools.DotNet.DotNetTasks; // ReSharper disable AllUnderscoreLocalParameterName [ShutdownDotNetAfterServerBuild] -class Build : NukeBuild +class Build : FalloutBuild { public Build() { diff --git a/build/Configuration.cs b/build/Configuration.cs index 78049f7..fe274ae 100644 --- a/build/Configuration.cs +++ b/build/Configuration.cs @@ -1,5 +1,5 @@ using System.ComponentModel; -using Nuke.Common.Tooling; +using Fallout.Common.Tooling; [TypeConverter(typeof(TypeConverter))] public class Configuration : Enumeration diff --git a/build/_build.csproj b/build/_build.csproj index 318bb57..c57e7f4 100644 --- a/build/_build.csproj +++ b/build/_build.csproj @@ -5,19 +5,20 @@ net10.0 CS0649;CS0169 - .. - .. - 1 + .. + .. + 1 true default + - + compile; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/claude.md b/claude.md index a9ef7c2..fe3db4a 100644 --- a/claude.md +++ b/claude.md @@ -126,9 +126,10 @@ Entry (base class) ### Build System -- **Nuke Build** - Build automation - - `build.cmd` / `build.ps1` / `build.sh` - Build scripts - - `build/Build.cs` - Build definition +- **Fallout Build** - Build automation (replaced Nuke) + - `build.cmd` / `build.ps1` / `build.sh` - Build scripts (bootstrap `build/_build.csproj`) + - `build/Build.cs` - Build definition (uses `Fallout.Common`) + - `.fallout/` - Fallout config, parameters, and temp/log output - Command: `build.cmd publish` - Creates artifacts in `./artifacts` ### Key Command Handlers diff --git a/src/cde.slnx b/src/cde.slnx new file mode 100644 index 0000000..a4ff9d1 --- /dev/null +++ b/src/cde.slnx @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/src/cde/cde.csproj b/src/cde/cde.csproj index a874c1f..4a0d9fb 100644 --- a/src/cde/cde.csproj +++ b/src/cde/cde.csproj @@ -17,27 +17,27 @@ - + - + - + - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - - + + + + - + diff --git a/src/cdeBenchmarks/cdeBenchmarks.csproj b/src/cdeBenchmarks/cdeBenchmarks.csproj index 693f219..787c451 100644 --- a/src/cdeBenchmarks/cdeBenchmarks.csproj +++ b/src/cdeBenchmarks/cdeBenchmarks.csproj @@ -11,6 +11,11 @@ + + + diff --git a/src/cdeLib/cdeLib.csproj b/src/cdeLib/cdeLib.csproj index 6141f9e..d0cce39 100644 --- a/src/cdeLib/cdeLib.csproj +++ b/src/cdeLib/cdeLib.csproj @@ -5,29 +5,29 @@ - + - - - - - + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + - + diff --git a/src/cdeLibTest/cdeLibTest.csproj b/src/cdeLibTest/cdeLibTest.csproj index cba3f6e..410cce2 100644 --- a/src/cdeLibTest/cdeLibTest.csproj +++ b/src/cdeLibTest/cdeLibTest.csproj @@ -15,12 +15,12 @@ - - + + - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/cdeWin/cdeWin.csproj b/src/cdeWin/cdeWin.csproj index ac09834..c1de4f0 100644 --- a/src/cdeWin/cdeWin.csproj +++ b/src/cdeWin/cdeWin.csproj @@ -24,10 +24,10 @@ - + - + diff --git a/src/cdeWinTest/cdeWinTest.csproj b/src/cdeWinTest/cdeWinTest.csproj index cd030f7..b67c271 100644 --- a/src/cdeWinTest/cdeWinTest.csproj +++ b/src/cdeWinTest/cdeWinTest.csproj @@ -8,10 +8,10 @@ - + - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive From a10a84d4cc37dbc6531ff3bf37600a246e8e1a84 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sat, 6 Jun 2026 21:28:38 +1000 Subject: [PATCH 03/43] perf(catalog): add memory + search measurement harness and baseline (Phase 0) Adds a standalone retained-footprint probe (cdeMemProbe) with a deterministic synthetic-catalog generator, plus BenchmarkDotNet load and search suites in cdeBenchmarks. Commits the locked baseline every later phase is measured against. - cdeMemProbe: loads a .cde, settles the GC, reports retained managed heap / working set / bytes-per-entry (BDN MemoryDiagnoser cannot measure retained heap). - SyntheticCatalog: shared, seeded generator (~50:1 file:dir) used by probe + benches. - CatalogLoadBenchmarks / SearchBenchmarks: load wall-clock and find throughput across {substring,regex} x {name,path}. - baseline/: footprint (~212 B/entry; 2.0 GB heap @ 10M) and search timings. Baseline already confirms two plan hypotheses: hashed vs no-hash footprint differ by only ~5 B/entry (Hash16 is inline regardless), and path search allocates ~973 MB/query. --- .gitignore | 3 + src/cde.slnx | 1 + src/cdeBenchmarks/CatalogFixture.cs | 54 ++++++ src/cdeBenchmarks/CatalogLoadBenchmarks.cs | 46 +++++ src/cdeBenchmarks/SearchBenchmarks.cs | 95 +++++++++++ src/cdeBenchmarks/baseline/README.md | 64 +++++++ .../baseline/footprint-baseline.csv | 4 + src/cdeBenchmarks/baseline/search-baseline.md | 35 ++++ src/cdeBenchmarks/cdeBenchmarks.csproj | 3 + src/cdeMemProbe/Program.cs | 160 ++++++++++++++++++ src/cdeMemProbe/SyntheticCatalog.cs | 102 +++++++++++ src/cdeMemProbe/cdeMemProbe.csproj | 19 +++ 12 files changed, 586 insertions(+) create mode 100644 src/cdeBenchmarks/CatalogFixture.cs create mode 100644 src/cdeBenchmarks/CatalogLoadBenchmarks.cs create mode 100644 src/cdeBenchmarks/SearchBenchmarks.cs create mode 100644 src/cdeBenchmarks/baseline/README.md create mode 100644 src/cdeBenchmarks/baseline/footprint-baseline.csv create mode 100644 src/cdeBenchmarks/baseline/search-baseline.md create mode 100644 src/cdeMemProbe/Program.cs create mode 100644 src/cdeMemProbe/SyntheticCatalog.cs create mode 100644 src/cdeMemProbe/cdeMemProbe.csproj diff --git a/.gitignore b/.gitignore index 5f48a8e..4319a55 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,6 @@ src/.vs/config/applicationhost.config **/Properties/launchSettings.json /.claude/settings.local.json /src/.claude/settings.local.json + +# BenchmarkDotNet writes results here when running cdeBenchmarks +**/BenchmarkDotNet.Artifacts/ diff --git a/src/cde.slnx b/src/cde.slnx index a4ff9d1..b328ed5 100644 --- a/src/cde.slnx +++ b/src/cde.slnx @@ -13,6 +13,7 @@ + diff --git a/src/cdeBenchmarks/CatalogFixture.cs b/src/cdeBenchmarks/CatalogFixture.cs new file mode 100644 index 0000000..94f46ff --- /dev/null +++ b/src/cdeBenchmarks/CatalogFixture.cs @@ -0,0 +1,54 @@ +using System; +using System.IO; +using cdeLib.Catalog; +using cdeLib.Entities; +using cdeMemProbe; +using Serilog; + +namespace cdeBenchmarks; + +/// +/// Thin helper that turns the shared generator into the two shapes +/// the benchmarks need: an in-memory tree (for search) and an on-disk .cde file (for load). +/// Centralised so every benchmark and phase measures the identical fixture. +/// +internal static class CatalogFixture +{ + private static readonly ILogger Silent = new LoggerConfiguration().CreateLogger(); + + /// Build the synthetic tree in memory (no disk I/O). + public static RootEntry BuildInMemory(int entryCount, bool withHashes) + => SyntheticCatalog.Generate(entryCount, withHashes); + + /// Generate the synthetic tree and serialize it to a fresh temp .cde file; returns the path. + public static string WriteTemp(int entryCount, bool withHashes) + { + var root = SyntheticCatalog.Generate(entryCount, withHashes); + var path = Path.Combine( + Path.GetTempPath(), + $"cde-bench-{entryCount}{(withHashes ? "-hashed" : "")}-{Guid.NewGuid():N}.cde"); + root.ActualFileName = path; + using var repo = new CatalogRepository(Silent); + repo.Save(root).GetAwaiter().GetResult(); + return path; + } + + /// Load a catalog from disk through the real production load path. + public static RootEntry Load(string path) + { + using var repo = new CatalogRepository(Silent); + return repo.LoadDirCacheAsync(path).GetAwaiter().GetResult(); + } + + public static void TryDelete(string path) + { + try + { + if (path != null && File.Exists(path)) File.Delete(path); + } + catch (IOException) + { + // Best-effort temp cleanup; ignore. + } + } +} diff --git a/src/cdeBenchmarks/CatalogLoadBenchmarks.cs b/src/cdeBenchmarks/CatalogLoadBenchmarks.cs new file mode 100644 index 0000000..1ec58a7 --- /dev/null +++ b/src/cdeBenchmarks/CatalogLoadBenchmarks.cs @@ -0,0 +1,46 @@ +using BenchmarkDotNet.Attributes; +using cdeLib.Entities; + +namespace cdeBenchmarks; + +/// +/// End-to-end catalog LOAD benchmark: deserialize a real on-disk .cde through the production +/// load path (CatalogRepository.LoadDirCacheAsync → MessagePack → SetInMemoryFields). +/// +/// Measures load wall-clock and (via MemoryDiagnoser) allocations during load. Retained footprint +/// is measured separately by the standalone cdeMemProbe — BenchmarkDotNet cannot report +/// steady-state retained heap, only per-iteration allocations. +/// +/// dotnet run -c Release --filter *CatalogLoad* +/// +[MemoryDiagnoser] +[SimpleJob(warmupCount: 1, iterationCount: 3)] // Load is expensive; keep iteration count modest. +public class CatalogLoadBenchmarks +{ + private string _catalogPath = null!; + + // 10M takes real time/disk; start at 1M for routine runs and opt into 10M explicitly via filter. + [Params(1_000_000)] + public int EntryCount { get; set; } + + [Params(false, true)] + public bool WithHashes { get; set; } + + [GlobalSetup] + public void Setup() + { + _catalogPath = CatalogFixture.WriteTemp(EntryCount, WithHashes); + } + + [GlobalCleanup] + public void Cleanup() + { + CatalogFixture.TryDelete(_catalogPath); + } + + [Benchmark(Description = "Load catalog from disk")] + public RootEntry LoadCatalog() + { + return CatalogFixture.Load(_catalogPath); + } +} diff --git a/src/cdeBenchmarks/SearchBenchmarks.cs b/src/cdeBenchmarks/SearchBenchmarks.cs new file mode 100644 index 0000000..3c3373c --- /dev/null +++ b/src/cdeBenchmarks/SearchBenchmarks.cs @@ -0,0 +1,95 @@ +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using BenchmarkDotNet.Attributes; +using cdeLib; +using cdeLib.Entities; + +namespace cdeBenchmarks; + +/// +/// Search (find) throughput across the matrix {substring, regex} × {name, path}, against a fixed +/// in-memory synthetic catalog loaded once. The visitor only counts matches (no console I/O) so the +/// measurement reflects traversal + matching cost, not output. +/// +/// Patterns are chosen with known hit-rates: +/// *NoMatch – 0 hits → pure full-scan + match throughput (worst case, most representative of cost) +/// *Common – ~8% hits (".txt") → includes per-match visitor + full-path-build cost +/// +/// dotnet run -c Release --filter *Search* +/// +[MemoryDiagnoser] +[SimpleJob(warmupCount: 2, iterationCount: 5)] +public class SearchBenchmarks +{ + private IList _roots = null!; + + [Params(1_000_000)] + public int EntryCount { get; set; } + + [GlobalSetup] + public void Setup() + { + // Hashes are irrelevant to search cost; use the no-hash fixture. + _roots = new List { CatalogFixture.BuildInMemory(EntryCount, withHashes: false) }; + } + + private FindOptions MakeOptions(string pattern, bool regexMode, bool includePath, StrongBox counter) + => new() + { + Pattern = pattern, + RegexMode = regexMode, + IncludePath = includePath, + IncludeFiles = true, + IncludeFolders = true, + LimitResultCount = int.MaxValue, + VisitorFunc = (_, _) => + { + Interlocked.Increment(ref counter.Value); + return true; + }, + }; + + // Production path: synchronous Find (plain bool delegate, no per-entry async state machine). + // FindService routes the CLI through this as of Phase 1. + private int RunFind(string pattern, bool regexMode, bool includePath) + { + var counter = new StrongBox(); + MakeOptions(pattern, regexMode, includePath, counter).Find(_roots); + return counter.Value; + } + + // Legacy work-stealing async path, kept only to track the gap that motivated the Phase 1 switch. + private int RunFindAsyncLegacy(string pattern, bool regexMode, bool includePath) + { + var counter = new StrongBox(); + MakeOptions(pattern, regexMode, includePath, counter).FindAsync(_roots).GetAwaiter().GetResult(); + return counter.Value; + } + + [Benchmark(Description = "substring, name, no match (full scan)")] + public int SubstringNameNoMatch() => RunFind("zzzznomatchzzzz", regexMode: false, includePath: false); + + [Benchmark(Description = "substring, name, ~8% hits (.txt)")] + public int SubstringNameCommon() => RunFind(".txt", regexMode: false, includePath: false); + + [Benchmark(Description = "substring, path, no match (full scan + path build)")] + public int SubstringPathNoMatch() => RunFind("zzzznomatchzzzz", regexMode: false, includePath: true); + + [Benchmark(Description = "substring, path, ~8% hits (.txt)")] + public int SubstringPathCommon() => RunFind(".txt", regexMode: false, includePath: true); + + [Benchmark(Description = "regex, name, no match (full scan)")] + public int RegexNameNoMatch() => RunFind("zzzz[0-9]nomatch", regexMode: true, includePath: false); + + [Benchmark(Description = "regex, path, ~8% hits (\\.txt$)")] + public int RegexPathCommon() => RunFind(@"\.txt$", regexMode: true, includePath: true); + + // --- Legacy async path (deprecated for the CLI in Phase 1; kept to track the gap) --- + + [Benchmark(Description = "LEGACY-async substring, name, no match")] + public int LegacySubstringNameNoMatch() => RunFindAsyncLegacy("zzzznomatchzzzz", regexMode: false, includePath: false); + + [Benchmark(Description = "LEGACY-async substring, path, no match")] + public int LegacySubstringPathNoMatch() => RunFindAsyncLegacy("zzzznomatchzzzz", regexMode: false, includePath: true); +} diff --git a/src/cdeBenchmarks/baseline/README.md b/src/cdeBenchmarks/baseline/README.md new file mode 100644 index 0000000..efe743f --- /dev/null +++ b/src/cdeBenchmarks/baseline/README.md @@ -0,0 +1,64 @@ +# Memory & Search Baseline (Phase 0) + +This directory holds the committed baseline that every memory/search optimization phase is +measured against. The goal of the work: **reduce the in-memory footprint of a loaded catalog +(primary)** and **improve search performance (secondary)**. A catalog format-version bump is +permitted in later phases. + +All phases must re-run the *same* harness against the *same* synthetic fixture (same seed) so +deltas are attributable to the change under test, not to fixture drift. + +## Harness + +| Tool | Project | Measures | +|------|---------|----------| +| `cdeMemProbe` | `src/cdeMemProbe` | **Retained** managed heap + working set after load, bytes/entry, load wall-clock | +| `CatalogLoadBenchmarks` | `src/cdeBenchmarks` | Load wall-clock + allocations (BenchmarkDotNet) | +| `SearchBenchmarks` | `src/cdeBenchmarks` | Find throughput across {substring,regex} × {name,path} | +| `SyntheticCatalog` | `src/cdeMemProbe` | Shared deterministic fixture generator (seed 42, ~50:1 file:dir) | + +`cdeMemProbe` is a separate minimal process on purpose: BenchmarkDotNet's `MemoryDiagnoser` +reports *allocations during a run*, not the *steady-state retained heap* — which is the headline +metric here. The probe loads one catalog, settles the GC (`Collect` ×2 + `WaitForPendingFinalizers`), +then reports `GC.GetTotalMemory(true)` and process working/private set. + +## Reproduce + +```powershell +# Build +dotnet build src/cdeMemProbe/cdeMemProbe.csproj -c Release +dotnet build src/cdeBenchmarks/cdeBenchmarks.csproj -c Release + +# Footprint: generate a fixture, then measure it in a clean process +$probe = "src/cdeMemProbe/bin/Release/net10.0/cdeMemProbe.dll" +dotnet $probe --generate 1000000 --out $env:TEMP\fix-1m.cde +dotnet $probe $env:TEMP\fix-1m.cde # add --hashes to --generate for the hashed variant + +# Load timing + allocations +dotnet run --project src/cdeBenchmarks -c Release -- --filter *CatalogLoad* + +# Search timings +dotnet run --project src/cdeBenchmarks -c Release -- --filter *Search* +``` + +## Baseline results — 2026-06-06 + +Machine: Windows 11, .NET 10, Server GC. Synthetic fixture, seed 42, ~50 files per directory. +Raw footprint rows in [`footprint-baseline.csv`](./footprint-baseline.csv). + +### Footprint (retained managed heap after load) + +| Entries | Hashes | Managed heap | Working set | **Bytes / entry** | Load ms | +|--------:|:------:|-------------:|------------:|------------------:|--------:| +| 1,000,000 | no | 202.1 MB | 356.5 MB | **211.94** | 1385 | +| 1,000,000 | yes | 207.0 MB | 373.8 MB | **217.07** | 1290 | +| 10,000,000 | no | 2018.7 MB | 2887.0 MB | **211.67** | 8447 | + +Key observation: hashed vs no-hash footprint is nearly identical (+~5 B/entry), because the +16-byte `Hash16` struct is stored **inline on every entry whether or not a hash is set**. This is +the direct evidence behind the Phase 3 plan to move hashes off-entry into a side table +(~16 B/entry reclaimable in the common no-hash case). + +### Search timings (1,000,000-entry fixture) + +See [`search-baseline.md`](./search-baseline.md) for the full BenchmarkDotNet table. diff --git a/src/cdeBenchmarks/baseline/footprint-baseline.csv b/src/cdeBenchmarks/baseline/footprint-baseline.csv new file mode 100644 index 0000000..8cc9487 --- /dev/null +++ b/src/cdeBenchmarks/baseline/footprint-baseline.csv @@ -0,0 +1,4 @@ +file,entries,loadMs,managedBytes,peakWorkingSet,privateBytes,bytesPerEntry +cde-base-1000000.cde,1000000,1385,211940976,373788672,373096448,211.94 +cde-base-1000000-h.cde,1000000,1290,217069152,391979008,395849728,217.07 +cde-base-10000000.cde,10000000,8447,2116746320,3027185664,3121905664,211.67 diff --git a/src/cdeBenchmarks/baseline/search-baseline.md b/src/cdeBenchmarks/baseline/search-baseline.md new file mode 100644 index 0000000..1719032 --- /dev/null +++ b/src/cdeBenchmarks/baseline/search-baseline.md @@ -0,0 +1,35 @@ +# Search Baseline — 2026-06-06 + +BenchmarkDotNet, .NET 10, Server GC. Fixture: 1,000,000-entry synthetic catalog (seed 42), +in memory. Driven through the production path `FindOptions.FindAsync` (what `FindService` / +`cde find` uses). Visitor counts matches only — no console I/O. + +| Method | Mean | StdDev | Allocated | +|--------|-----:|-------:|----------:| +| substring, name, no match (full scan) | 1.746 s | 0.043 s | 58.7 MB | +| substring, name, ~8% hits (.txt) | 1.722 s | 0.007 s | 58.7 MB | +| substring, path, no match (full scan + path build) | 1.755 s | 0.026 s | 978.7 MB | +| substring, path, ~8% hits (.txt) | 1.774 s | 0.012 s | 978.7 MB | +| regex, name, no match (full scan) | 1.817 s | 0.079 s | 460.9 MB | +| regex, path, ~8% hits (`\.txt$`) | 1.828 s | 0.006 s | 1349.0 MB | + +## What this reveals (drives Phase 1) + +- **Path search allocates ~0.98–1.35 GB per single 1M-entry query.** `GetPatternMatcher` builds a + full-path string (`EntryHelper.MakeFullPathPooled`) for *every candidate* — and for a no-match + query that's every entry. **S1**: match over pooled `Span` (`MemoryExtensions.Contains`) + instead of allocating a string per entry; pass the already-built path to the visitor on a match + rather than rebuilding it in `FindService.FindAsync`. Target: path allocation → near-zero. + +- **Even name-only, no-match scan is ~1.7 s and allocates 58.7 MB** for what should be an + allocation-free linear scan. The cost is the per-entry `async Task` processor + (`CreateAsyncProcessor`) plus `Task.Yield()` / `Task.Delay(0)` every 500 / 2000 entries. The + synchronous `Find` path (`GetFindFunc`, a plain `bool` delegate) has none of this. **S3**: route + the CLI through the synchronous path (or make parallel/serial adaptive) and drop per-entry yields. + +- Mean time is dominated by async overhead, not matching: name-no-match (58 MB alloc) and + path-no-match (978 MB alloc) have nearly identical ~1.75 s means. Cutting the async overhead + should move the needle more than matching micro-opts. + +These numbers are the bar Phase 1 must beat. Re-run with +`dotnet run --project src/cdeBenchmarks -c Release -- --filter *Search*`. diff --git a/src/cdeBenchmarks/cdeBenchmarks.csproj b/src/cdeBenchmarks/cdeBenchmarks.csproj index 787c451..149b349 100644 --- a/src/cdeBenchmarks/cdeBenchmarks.csproj +++ b/src/cdeBenchmarks/cdeBenchmarks.csproj @@ -20,6 +20,9 @@ + + diff --git a/src/cdeMemProbe/Program.cs b/src/cdeMemProbe/Program.cs new file mode 100644 index 0000000..ff97c49 --- /dev/null +++ b/src/cdeMemProbe/Program.cs @@ -0,0 +1,160 @@ +using System; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Threading.Tasks; +using cdeLib.Catalog; +using cdeLib.Entities; +using Serilog; + +namespace cdeMemProbe; + +/// +/// Standalone footprint probe for cde catalogs. Measures the RETAINED managed heap and process +/// working set after a catalog is fully loaded — the headline metric for the memory-reduction +/// effort. BenchmarkDotNet's MemoryDiagnoser measures allocations during a run, not steady-state +/// retained size, so this lives in its own minimal process. +/// +/// Usage: +/// cdeMemProbe --generate <N> [--hashes] [--out <path.cde>] [--seed N] +/// Generate a synthetic catalog of ~N entries and save it. Prints the path. No measurement. +/// +/// cdeMemProbe <path.cde> [--no-header] +/// Load the catalog, settle the GC, and emit one CSV row of footprint metrics. +/// +/// Typical flow (clean measurement = generate and measure in separate processes): +/// cdeMemProbe --generate 1000000 --out fixture-1m.cde +/// cdeMemProbe fixture-1m.cde +/// +public static class Program +{ + private const string CsvHeader = + "file,entries,loadMs,managedBytes,peakWorkingSet,privateBytes,bytesPerEntry"; + + public static async Task Main(string[] args) + { + if (args.Length == 0) + { + Console.Error.WriteLine( + "usage: cdeMemProbe --generate [--hashes] [--out ] [--seed N]\n" + + " cdeMemProbe [--no-header]"); + return 1; + } + + // Silent Serilog logger (no sinks) so CatalogRepository stays quiet and out of the CSV. + var logger = new LoggerConfiguration().CreateLogger(); + + if (HasFlag(args, "--generate", out var genValue)) + { + return await GenerateAsync(args, genValue, logger); + } + + return await MeasureAsync(args[0], !HasFlag(args, "--no-header", out _), logger); + } + + private static async Task GenerateAsync(string[] args, string? countArg, ILogger logger) + { + if (!int.TryParse(countArg, NumberStyles.Integer, CultureInfo.InvariantCulture, out var count) || count <= 0) + { + Console.Error.WriteLine("--generate requires a positive entry count, e.g. --generate 1000000"); + return 1; + } + + var withHashes = HasFlag(args, "--hashes", out _); + var seed = HasFlag(args, "--seed", out var seedArg) && int.TryParse(seedArg, out var s) ? s : 42; + var outPath = HasFlag(args, "--out", out var outArg) && !string.IsNullOrWhiteSpace(outArg) + ? outArg! + : Path.Combine(Path.GetTempPath(), $"cde-synthetic-{count}{(withHashes ? "-hashed" : "")}.cde"); + + Console.Error.WriteLine($"Generating ~{count:N0} entries (hashes={withHashes}, seed={seed}) ..."); + var sw = Stopwatch.StartNew(); + var root = SyntheticCatalog.Generate(count, withHashes, seed); + root.ActualFileName = outPath; + using (var repo = new CatalogRepository(logger)) + { + await repo.Save(root); + } + sw.Stop(); + + var actual = root.FileEntryCount + root.DirEntryCount; + var fileInfo = new FileInfo(outPath); + Console.Error.WriteLine( + $"Wrote {actual:N0} entries to {outPath} ({fileInfo.Length:N0} bytes on disk) in {sw.ElapsedMilliseconds:N0} ms"); + Console.WriteLine(outPath); + return 0; + } + + private static async Task MeasureAsync(string file, bool printHeader, ILogger logger) + { + if (!File.Exists(file)) + { + Console.Error.WriteLine($"catalog not found: {file}"); + return 1; + } + + var sw = Stopwatch.StartNew(); + RootEntry root; + using (var repo = new CatalogRepository(logger)) + { + root = await repo.LoadDirCacheAsync(file); + } + sw.Stop(); + + if (root == null) + { + Console.Error.WriteLine($"failed to load catalog: {file}"); + return 1; + } + + var entries = root.FileEntryCount + root.DirEntryCount; + + // Settle the GC so GetTotalMemory reflects retained (live) objects, not transient load garbage. + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + var managedBytes = GC.GetTotalMemory(true); + using var proc = Process.GetCurrentProcess(); + var peakWorkingSet = proc.PeakWorkingSet64; + var privateBytes = proc.PrivateMemorySize64; + + // Keep the tree alive across the measurement so it counts toward the live heap. + GC.KeepAlive(root); + + var bytesPerEntry = entries > 0 ? (double)managedBytes / entries : 0; + + if (printHeader) + { + Console.WriteLine(CsvHeader); + } + + Console.WriteLine(string.Join(',', + Path.GetFileName(file), + entries.ToString(CultureInfo.InvariantCulture), + sw.ElapsedMilliseconds.ToString(CultureInfo.InvariantCulture), + managedBytes.ToString(CultureInfo.InvariantCulture), + peakWorkingSet.ToString(CultureInfo.InvariantCulture), + privateBytes.ToString(CultureInfo.InvariantCulture), + bytesPerEntry.ToString("F2", CultureInfo.InvariantCulture))); + return 0; + } + + /// + /// Returns true if is present. If the next token is not another flag it + /// is returned as (so both --generate 100 and bare flags work). + /// + private static bool HasFlag(string[] args, string name, out string? value) + { + value = null; + for (var i = 0; i < args.Length; i++) + { + if (!string.Equals(args[i], name, StringComparison.OrdinalIgnoreCase)) continue; + if (i + 1 < args.Length && !args[i + 1].StartsWith("--", StringComparison.Ordinal)) + { + value = args[i + 1]; + } + return true; + } + return false; + } +} diff --git a/src/cdeMemProbe/SyntheticCatalog.cs b/src/cdeMemProbe/SyntheticCatalog.cs new file mode 100644 index 0000000..80b75eb --- /dev/null +++ b/src/cdeMemProbe/SyntheticCatalog.cs @@ -0,0 +1,102 @@ +using System; +using System.Collections.Generic; +using cdeLib.Entities; + +namespace cdeMemProbe; + +/// +/// Deterministic synthetic .cde catalog generator used to measure memory footprint and +/// search performance reproducibly across optimization phases. Builds a +/// tree in memory with a realistic file:directory ratio, then the caller saves/loads it. +/// +/// The SAME fixture (same seed + parameters) must be used for every phase so per-entry-byte and +/// search-timing deltas are attributable to the change under test, not to fixture drift. +/// +public static class SyntheticCatalog +{ + // Representative extension/prefix spread; interning means duplicates are deduplicated in memory + // exactly as a real catalog would be. + private static readonly string[] Extensions = + [".txt", ".jpg", ".pdf", ".doc", ".mp4", ".zip", ".exe", ".dll", ".cs", ".json", ".png", ".log"]; + + private static readonly string[] Prefixes = + ["Document", "Image", "Video", "Archive", "Config", "Data", "Log", "Report", "Backup", "Cache"]; + + private static readonly DateTime BaseDate = new(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + /// + /// Build a synthetic tree of approximately entries + /// (files + directories combined), using a ~:1 file/dir ratio. + /// Generation is breadth-first so the tree is bushy and shallow like a real file system. + /// + /// Approximate total of files + directories to generate. + /// When true, every file gets an MD5-sized hash set (worst case for memory). + /// Deterministic RNG seed; keep fixed across phases. + /// Files added per directory (drives the file:dir ratio). + /// Sub-directories added per directory (drives breadth). + public static RootEntry Generate( + int targetEntryCount, + bool withHashes = false, + int seed = 42, + int filesPerDir = 50, + int subDirsPerDir = 5) + { + var random = new Random(seed); + var root = new RootEntry + { + Path = @"C:\synthetic", + DefaultFileName = "synthetic.cde", + DriveLetterHint = "C", + VolumeName = "SYNTHETIC", + Description = $"Synthetic catalog ~{targetEntryCount} entries (hashes={withHashes})", + Children = new List(), + }; + + var created = 0; + // Directories that can still receive children. Breadth-first never drains before the target + // is hit because each directory enqueues more directories than it dequeues. + var queue = new Queue(); + + Populate(root, null); + while (queue.Count > 0 && created < targetEntryCount) + { + Populate(null, queue.Dequeue()); + } + + root.SetInMemoryFields(); + return root; + + void Populate(RootEntry? rootParent, DirEntry? dirParent) + { + for (var i = 0; i < filesPerDir && created < targetEntryCount; i++) + { + var file = new DirEntry(false); + var ext = Extensions[random.Next(Extensions.Length)]; + var prefix = Prefixes[random.Next(Prefixes.Length)]; + file.SetPath($"{prefix}_{created:D7}{ext}"); + file.Size = random.Next(1, 50_000_000); + file.Modified = BaseDate.AddMinutes(random.Next(0, 5_000_000)); + if (withHashes) + { + // Mix of unique and duplicate hashes (every 17th repeats) to exercise dupe paths. + file.SetHash(created % 17 == 0 ? 17 : created); + } + + if (rootParent != null) rootParent.AddChild(file); + else dirParent!.AddChild(file); + created++; + } + + for (var i = 0; i < subDirsPerDir && created < targetEntryCount; i++) + { + var dir = new DirEntry(true); + dir.SetPath($"dir_{created:D7}"); + dir.Modified = BaseDate.AddMinutes(random.Next(0, 5_000_000)); + if (rootParent != null) rootParent.AddChild(dir); + else dirParent!.AddChild(dir); + created++; + queue.Enqueue(dir); + } + } + } +} diff --git a/src/cdeMemProbe/cdeMemProbe.csproj b/src/cdeMemProbe/cdeMemProbe.csproj new file mode 100644 index 0000000..ea8ef5b --- /dev/null +++ b/src/cdeMemProbe/cdeMemProbe.csproj @@ -0,0 +1,19 @@ + + + + Exe + net10.0 + enable + enable + latest + + true + true + + + + + + + From 8a9067c6c4ce765795b53f783ced0f4a4421ef9b Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sat, 6 Jun 2026 21:28:48 +1000 Subject: [PATCH 04/43] perf(catalog): 7-49x faster find + 8 bytes/entry (Phase 1, non-breaking) S3 - route the CLI find path (FindService) through the synchronous FindOptions.Find instead of the work-stealing FindAsync. The async path ran every entry through an async Task state machine plus Task.Yield/Task.Delay every 500/2000 entries. Measured on a 1M-entry catalog: name search 1746ms -> 36ms (48x), path search 1755ms -> 236ms (7.4x), regex name 1817ms -> 60ms (30x). cdeWin already used the sync path; only the CLI was on the slow one. R1a - FileEntryCount/DirEntryCount changed long -> uint on DirEntry/RootEntry/ ICommonEntry. They were already (uint)-truncated at assignment, so semantics are unchanged. In-memory only, no catalog format impact. Confirmed 211.94 -> 203.94 B/entry. Tests: cdeLibTest 126 passed. Format-affecting work (path-alloc S1, file/dir split, hash side-table) is deferred to Phases 2-3. --- src/cdeBenchmarks/baseline/phase-results.md | 52 +++++++++++++++++++++ src/cdeLib/Entities/DirEntry.cs | 4 +- src/cdeLib/Entities/ICommonEntry.cs | 4 +- src/cdeLib/Entities/RootEntry.cs | 4 +- src/cdeLib/FindService.cs | 35 ++++++++------ 5 files changed, 79 insertions(+), 20 deletions(-) create mode 100644 src/cdeBenchmarks/baseline/phase-results.md diff --git a/src/cdeBenchmarks/baseline/phase-results.md b/src/cdeBenchmarks/baseline/phase-results.md new file mode 100644 index 0000000..f1786b4 --- /dev/null +++ b/src/cdeBenchmarks/baseline/phase-results.md @@ -0,0 +1,52 @@ +# Phase Results vs Baseline + +Tracks the measured delta of each optimization phase against the committed Phase 0 baseline +(`footprint-baseline.csv`, `search-baseline.md`). Same fixture, same harness, every phase. + +--- + +## Phase 1 — free non-breaking wins (no format change) + +**Changes shipped** +- **S3** — `FindService` (the CLI `cde find` path) now runs the synchronous `FindOptions.Find` + instead of the work-stealing `FindAsync`. The async path ran every entry through an + `async Task` state machine plus `Task.Yield()`/`Task.Delay(0)` every 500/2000 entries. + (`src/cdeLib/FindService.cs`) +- **R1a** — `FileEntryCount` / `DirEntryCount` changed `long → uint` on `DirEntry`, `RootEntry`, + `ICommonEntry`. Values were already `(uint)`-truncated at assignment, so semantics are + unchanged; in-memory only, no catalog format impact. + +**Re-sequenced (not done in Phase 1)** +- **S1** (path-build allocation, the ~973 MB below) → moved to **Phase 2**, where carrying the + running path down the traversal stack makes it allocation-free without duplicated work. +- **S2** (`SearchValues` name prefilter) → dropped as low value: name search is already ~35 ms, + and its remaining 52 MB is the `DirEntry.Path` getter re-joining the split extension, which a + prefilter would not remove. + +### Search (1,000,000-entry fixture) + +| Query | Baseline (async) | Phase 1 (sync) | Speedup | +|-------|-----------------:|---------------:|--------:| +| substring, name, no match | 1746 ms | **36.3 ms** | 48× | +| substring, name, ~8% (.txt) | 1722 ms | **35.4 ms** | 49× | +| substring, path, no match | 1755 ms | **235.6 ms** | 7.4× | +| substring, path, ~8% (.txt) | 1774 ms | **243.0 ms** | 7.3× | +| regex, name, no match | 1817 ms | **59.6 ms** | 30× | +| regex, path, ~8% (`\.txt$`) | 1828 ms | **253.1 ms** | 7.2× | + +Allocation is unchanged by Phase 1 (name 52.7 MB, path 972.7 MB) — that is S1, targeted in Phase 2. +The retained `LEGACY-async*` benchmarks still measure ~1710–1764 ms, confirming the old path. + +### Footprint (retained managed heap) + +| Entries | Hashes | Baseline B/entry | Phase 1 B/entry | Saved | +|--------:|:------:|-----------------:|----------------:|------:| +| 1,000,000 | no | 211.94 | **203.94** | 8.00 B/entry | + +Exactly the predicted 8 bytes/entry (two `long`→`uint`). At 1M entries: 202.1 MB → 194.5 MB managed. + +### Correctness +- `cdeLibTest`: 126 passed, 0 failed (7 skipped). +- Full solution (`cde.slnx`) builds clean. +- Note: `cdeLibSpec` / `cdeLibSpec2` are legacy `net48` projects incompatible with the `net10` + library and do not restore — pre-existing, unrelated to this work, not in the solution. diff --git a/src/cdeLib/Entities/DirEntry.cs b/src/cdeLib/Entities/DirEntry.cs index 0af273e..43b80cd 100644 --- a/src/cdeLib/Entities/DirEntry.cs +++ b/src/cdeLib/Entities/DirEntry.cs @@ -160,13 +160,13 @@ public bool IsDefaultSort /// if this is a directory number of files contained in its hierarchy /// [IgnoreMember] - public long FileEntryCount { get; set; } + public uint FileEntryCount { get; set; } /// /// if this is a directory number of dirs contained in its hierarchy /// [IgnoreMember] - public long DirEntryCount { get; set; } + public uint DirEntryCount { get; set; } public void SetHash(byte[] hash) { diff --git a/src/cdeLib/Entities/ICommonEntry.cs b/src/cdeLib/Entities/ICommonEntry.cs index a6fbf43..ac39697 100644 --- a/src/cdeLib/Entities/ICommonEntry.cs +++ b/src/cdeLib/Entities/ICommonEntry.cs @@ -33,8 +33,8 @@ public interface ICommonEntry bool IsDirectory { get; set; } bool PathProblem { get; } - long FileEntryCount { get; set; } - long DirEntryCount { get; set; } + uint FileEntryCount { get; set; } + uint DirEntryCount { get; set; } DateTime Modified { get; set; } bool IsHashDone { get; set; } bool IsPartialHash { get; set; } diff --git a/src/cdeLib/Entities/RootEntry.cs b/src/cdeLib/Entities/RootEntry.cs index c425206..97b06fc 100644 --- a/src/cdeLib/Entities/RootEntry.cs +++ b/src/cdeLib/Entities/RootEntry.cs @@ -639,13 +639,13 @@ public bool IsDefaultSort /// if this is a directory number of files contained in its hierarchy /// [IgnoreMember] - public long FileEntryCount { get; set; } + public uint FileEntryCount { get; set; } /// /// if this is a directory number of dirs contained in its hierarchy /// [IgnoreMember] - public long DirEntryCount { get; set; } + public uint DirEntryCount { get; set; } public void SetHash(byte[] hash) { diff --git a/src/cdeLib/FindService.cs b/src/cdeLib/FindService.cs index 6ee59dc..ec355df 100644 --- a/src/cdeLib/FindService.cs +++ b/src/cdeLib/FindService.cs @@ -44,19 +44,10 @@ public void Find(string pattern, string param, IList rootEntries) public void Find(string pattern, bool regexMode, bool includePath, IList rootEntries) { - // Use async version for better performance - FindAsync(pattern, regexMode, includePath, rootEntries).GetAwaiter().GetResult(); - } - - public async Task FindAsync(string pattern, string param, IList rootEntries) - { - var regexMode = param is ParamGrep or ParamGrepPath; - var includePath = param is ParamGrepPath or ParamFindPath; - await FindAsync(pattern, regexMode, includePath, rootEntries); - } - - public async Task FindAsync(string pattern, bool regexMode, bool includePath, IList rootEntries) - { + // Use the synchronous traversal: it is dramatically faster than the work-stealing async + // path, which ran every entry through an async Task state machine plus per-entry + // Task.Yield/Task.Delay. Measured on a 1M-entry catalog: ~49x faster for name search and + // ~7x for path search (see src/cdeBenchmarks/baseline/search-baseline.md). var totalFound = 0L; var findOptions = new FindOptions { @@ -75,10 +66,26 @@ public async Task FindAsync(string pattern, bool regexMode, bool includePath, IL }; var timer = System.Diagnostics.Stopwatch.StartNew(); - await findOptions.FindAsync(rootEntries); + findOptions.Find(rootEntries); timer.Stop(); Log.Logger.Information( "Search Execution Time: {ExecutionTime}, Matching pattern {Pattern}, Total found {TotalFound}", timer.ElapsedMilliseconds, pattern, totalFound); } + + public Task FindAsync(string pattern, string param, IList rootEntries) + { + var regexMode = param is ParamGrep or ParamGrepPath; + var includePath = param is ParamGrepPath or ParamFindPath; + return FindAsync(pattern, regexMode, includePath, rootEntries); + } + + public Task FindAsync(string pattern, bool regexMode, bool includePath, IList rootEntries) + { + // Search is CPU-bound; the synchronous path is the fast one. Keep the async signature for + // API compatibility but run the fast core. Callers wanting off-thread execution should + // wrap this in Task.Run themselves. + Find(pattern, regexMode, includePath, rootEntries); + return Task.CompletedTask; + } } \ No newline at end of file From 15a30688b34a10d59ff6237df68a170bf1af3655 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sat, 6 Jun 2026 21:35:34 +1000 Subject: [PATCH 05/43] perf(find): allocation-free path search via single-pass path build (Phase 2) Reimplement EntryHelper.MakeFullPath to build the full path in one pass up the ParentCommonEntry chain instead of recursively calling parent.FullPath at every level (which allocated a fresh full-path string per ancestor). Add FullPathContains that matches a substring over a pooled Span with no result-string allocation, wired into the find substring path matcher. Measured on a 1M-entry catalog (cumulative with Phase 1): - substring path search: 1755ms/978MB -> 102ms/53MB (17x faster, 94.6% less alloc) - regex path search: 1828ms/1349MB -> 136ms/265MB All path-building callers (find, dupes, GUI, dump) get the cheaper single-pass build. cdeLibTest 126 passed, including EntryHelper tests that assert exact path strings. Note: dropping ParentCommonEntry (originally Phase 2) folds into Phase 3 - .FullPath is only called on directories or via PairDirEntry, so only files can shed the pointer, which needs the file/dir type split. --- src/cdeBenchmarks/baseline/phase-results.md | 37 +++++++ src/cdeLib/Entities/EntryHelper.cs | 101 +++++++++++++++----- src/cdeLib/FindOptions.cs | 6 +- 3 files changed, 119 insertions(+), 25 deletions(-) diff --git a/src/cdeBenchmarks/baseline/phase-results.md b/src/cdeBenchmarks/baseline/phase-results.md index f1786b4..832e4c5 100644 --- a/src/cdeBenchmarks/baseline/phase-results.md +++ b/src/cdeBenchmarks/baseline/phase-results.md @@ -50,3 +50,40 @@ Exactly the predicted 8 bytes/entry (two `long`→`uint`). At 1M entries: 202.1 - Full solution (`cde.slnx`) builds clean. - Note: `cdeLibSpec` / `cdeLibSpec2` are legacy `net48` projects incompatible with the `net10` library and do not restore — pre-existing, unrelated to this work, not in the solution. + +--- + +## Phase 2 — allocation-free path search (S1), no format change + +**Changes shipped** +- Reimplemented `EntryHelper.MakeFullPath` as a single pass that walks the `ParentCommonEntry` + chain into the shared `StringBuilder`, instead of the old recursion that allocated a fresh + full-path string at every ancestor level. All path-building callers (find, dupes, GUI, dump) + benefit. (`src/cdeLib/Entities/EntryHelper.cs`) +- Added `EntryHelper.FullPathContains` — builds the path into a pooled `char[]` and matches over a + `Span` (`MemoryExtensions.Contains`), allocating no result string. Wired into the find + substring path matcher. (`src/cdeLib/Entities/EntryHelper.cs`, `src/cdeLib/FindOptions.cs`) + +**Re-sequencing note:** the original Phase 2 also dropped `ParentCommonEntry` (~8 B/entry). Code +evidence shows `.FullPath` / `GetListFromRoot` are called only on **directories** or via +`PairDirEntry` (which carries an explicit parent) — files never need a standalone parent pointer. +The cdeWin GUI depends heavily on directory parent pointers, so removing the field from *all* +entries is a large, risky GUI refactor for the same 8 B that R1a already delivered safely. The +parent pointer can only be reclaimed from **files**, which requires the file/dir type split — so +`ParentCommonEntry` removal folds into **Phase 3**, validated by the split. + +### Search (1,000,000-entry fixture) — cumulative + +| Query | Baseline | Phase 1 | Phase 2 | Alloc (base → P2) | +|-------|---------:|--------:|--------:|------------------:| +| substring, path, no match | 1755 ms | 236 ms | **101.6 ms** | 978.7 MB → **52.7 MB** | +| substring, path, ~8% (.txt) | 1774 ms | 243 ms | **96.4 ms** | 978.7 MB → **52.7 MB** | +| regex, path, ~8% (`\.txt$`) | 1828 ms | 253 ms | **136 ms** | 1349 MB → **265 MB** | + +Path-search allocation cut ~94.6% (residual 52.7 MB is the per-file `DirEntry.Path` extension +rejoin, same as name search). Name-search numbers are unchanged from Phase 1. + +### Correctness +- `cdeLibTest`: 126 passed (incl. all `EntryHelper` / `GetListFromRoot` / `RootEntry` path tests + that assert exact path strings — confirms the single-pass build is identical). +- Full solution builds clean. diff --git a/src/cdeLib/Entities/EntryHelper.cs b/src/cdeLib/Entities/EntryHelper.cs index a4e8c6e..f4d3142 100644 --- a/src/cdeLib/Entities/EntryHelper.cs +++ b/src/cdeLib/Entities/EntryHelper.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading; @@ -25,41 +26,73 @@ public static IEnumerable GetPairDirEntries(IEnumerable return new PairDirEntryEnumerator(rootEntries); } + // Reused per-thread scratch buffer for allocation-free path matching (see FullPathContains). + private static readonly ThreadLocal PathMatchBuffer = + new(() => new char[512]); + /// - /// Creates a full path using a ThreadLocal StringBuilder to reduce allocations. - /// Still allocates the final string, but avoids intermediate allocations from Path.Combine. + /// Builds the full path of (whose directory parent is + /// ) into the shared ThreadLocal StringBuilder in a single pass, + /// walking the parent chain via ParentCommonEntry. Avoids the recursive per-level full-path + /// string allocations of the old approach (which called parent.FullPath at every level) — only + /// each segment's name is materialised. Returns the StringBuilder for the caller to consume. /// - public static string MakeFullPath(ICommonEntry parentEntry, ICommonEntry dirEntry) + private static StringBuilder BuildFullPath(ICommonEntry parentEntry, ICommonEntry dirEntry) { - // Resolve parent path FIRST — this may recursively re-enter MakeFullPath and - // mutate the shared ThreadLocal StringBuilder, so do it before we clear/use sb. - var parentPath = parentEntry.FullPath; - var sb = PathBuilder.Value!; sb.Clear(); - if (parentPath != null) - { - sb.Append(parentPath); - if (sb.Length > 0) - { - var lastChar = sb[^1]; - if (lastChar != '\\' && lastChar != '/') - sb.Append(System.IO.Path.DirectorySeparatorChar); - } - } - + AppendAncestorPath(sb, parentEntry); + AppendSeparatorIfNeeded(sb); sb.Append(dirEntry.Path ?? "dnull"); - var result = sb.ToString(); - // Prevent StringBuilder from growing unbounded in long-running processes - // Only shrink if the capacity is large AND current content fits in target size + // Prevent the pooled StringBuilder from growing unbounded in long-running processes. if (sb.Capacity > 1024 && sb.Length <= 512) { sb.Capacity = 512; } - return result; + return sb; + } + + /// + /// Append the full path of to root-first by + /// recursing up the ParentCommonEntry chain. The root (null parent) contributes its stored + /// FullPath; every descendant contributes its own Path segment. No intermediate path strings. + /// + private static void AppendAncestorPath(StringBuilder sb, ICommonEntry entry) + { + if (entry == null) return; + + var parent = entry.ParentCommonEntry; + if (parent == null) + { + // Root entry: FullPath is a stored field on RootEntry (no recursion/allocation). + sb.Append(entry.FullPath ?? entry.Path ?? string.Empty); + return; + } + + AppendAncestorPath(sb, parent); + AppendSeparatorIfNeeded(sb); + sb.Append(entry.Path ?? "dnull"); + } + + private static void AppendSeparatorIfNeeded(StringBuilder sb) + { + if (sb.Length == 0) return; + var last = sb[^1]; + if (last != '\\' && last != '/') + sb.Append(System.IO.Path.DirectorySeparatorChar); + } + + /// + /// Creates a full path using a ThreadLocal StringBuilder to reduce allocations. + /// Still allocates the final string, but avoids intermediate allocations from Path.Combine + /// and from recursive per-level full-path string building. + /// + public static string MakeFullPath(ICommonEntry parentEntry, ICommonEntry dirEntry) + { + return BuildFullPath(parentEntry, dirEntry).ToString(); } /// @@ -71,6 +104,28 @@ public static string MakeFullPathPooled(ICommonEntry parentEntry, ICommonEntry d return MakeFullPath(parentEntry, dirEntry); } + /// + /// Allocation-free full-path substring test: builds the full path into pooled buffers and + /// matches over a span, without allocating the result string. + /// Used by find's substring path search, the dominant allocator in path queries. + /// + public static bool FullPathContains(ICommonEntry parentEntry, ICommonEntry dirEntry, + string pattern, StringComparison comparison) + { + var sb = BuildFullPath(parentEntry, dirEntry); + var length = sb.Length; + + var buffer = PathMatchBuffer.Value!; + if (buffer.Length < length) + { + buffer = new char[Math.Max(length, buffer.Length * 2)]; + PathMatchBuffer.Value = buffer; + } + + sb.CopyTo(0, buffer, 0, length); + return buffer.AsSpan(0, length).Contains(pattern, comparison); + } + /// /// Recursive traversal /// diff --git a/src/cdeLib/FindOptions.cs b/src/cdeLib/FindOptions.cs index f94b1cc..81c74e0 100644 --- a/src/cdeLib/FindOptions.cs +++ b/src/cdeLib/FindOptions.cs @@ -286,9 +286,11 @@ private Func GetPatternMatcher() : (p, d) => regex.IsMatch(d.Path); } - // String matching with StringComparison for better performance + // String matching with StringComparison for better performance. + // Path mode uses the allocation-free span matcher (avoids building a full-path string per + // candidate — the dominant allocator in path queries; see search baseline). return includePath - ? (p, d) => EntryHelper.MakeFullPathPooled(p, d).Contains(pattern, StringComparison.OrdinalIgnoreCase) + ? (p, d) => EntryHelper.FullPathContains(p, d, pattern, StringComparison.OrdinalIgnoreCase) : (p, d) => d.Path.Contains(pattern, StringComparison.OrdinalIgnoreCase); } From 58481c845c74c52f39e77d1932537cdd7239ed3c Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sat, 6 Jun 2026 21:52:31 +1000 Subject: [PATCH 06/43] perf(catalog): lean files via ExtraData side-object (Phase 3, no format change) Move DirEntry's directory-only fields (Children + the two summary counts) into a lazily-allocated private ExtraData object, allocated only for directories. A file - the vast majority of entries - now carries one null 8-byte _extra reference instead of an always-null Children ref plus two count fields. Hash stays INLINE on purpose: moving it off-entry forces every hashed file to allocate an ExtraData whose header exceeds the 16-byte Hash16 (measured +16 B/entry regression for hashed catalogs). Keeping it inline means no catalog ever regresses. No .cde format change: Children keeps Key 3 via the property, counts are IgnoreMember. Round-trip is byte-identical to baseline (36,668,199 bytes). Measured 1M no-hash: 203.94 -> 198.85 B/entry; hashed has no regression. No find CPU regression (name 36.7ms, path 95.0ms). cdeLibTest 126 + cdeWinTest 32 passed. The full polymorphic split (~3x more saving) needs an abstract union root, IList type changes across lib/GUI/web, ~40 construction sites, a format bump, and GUI runtime validation - left as an explicit opt-in follow-up (see baseline/phase-results.md). --- src/cdeBenchmarks/baseline/phase-results.md | 42 ++++++++++++++ src/cdeLib/Entities/DirEntry.cs | 63 ++++++++++++++++++--- 2 files changed, 96 insertions(+), 9 deletions(-) diff --git a/src/cdeBenchmarks/baseline/phase-results.md b/src/cdeBenchmarks/baseline/phase-results.md index 832e4c5..be56a09 100644 --- a/src/cdeBenchmarks/baseline/phase-results.md +++ b/src/cdeBenchmarks/baseline/phase-results.md @@ -87,3 +87,45 @@ rejoin, same as name search). Name-search numbers are unchanged from Phase 1. - `cdeLibTest`: 126 passed (incl. all `EntryHelper` / `GetListFromRoot` / `RootEntry` path tests that assert exact path strings — confirms the single-pass build is identical). - Full solution builds clean. + +--- + +## Phase 3 — lean files via `ExtraData` side-object (no format change) + +**Change shipped** +- `DirEntry` moves its directory-only fields (`Children` + the two summary counts) into a + lazily-allocated private `ExtraData` object, allocated only for directories. A file — the vast + majority of entries — now carries a single 8-byte `_extra` reference (null) instead of an + always-null `Children` ref plus two count fields. `Hash` deliberately stays **inline** (moving it + off-entry forces every hashed file to allocate an `ExtraData` whose header costs more than the + 16-byte `Hash16`, a net regression for hashed catalogs — measured at +16 B/entry). No catalog + format change: `Children` keeps Key 3 via the property; counts are `[IgnoreMember]`. + (`src/cdeLib/Entities/DirEntry.cs`) + +### Footprint (retained managed heap, 1M fixture, ~10:1 file:dir) + +| Catalog | Phase 1 (post-R1a) | Phase 3 | Saved | +|---------|-------------------:|--------:|------:| +| no hashes | 203.94 | **198.85** | 5.09 B/entry | +| hashed | (≈209 post-R1a) | **203.98** | no regression | + +Round-trip verified: the serialized `.cde` is **byte-identical** to the baseline (36,668,199 bytes), +confirming the format is unchanged. Find shows no CPU regression (name 36.7 ms, path 95.0 ms). + +### Honest accounting of the win vs the planned split +The original Phase 3 was a polymorphic file/dir type split + hash side-table (format bump). The +`ExtraData` approach delivers the same *memory outcome direction* (lean files) **safely and with no +format change**, but a side-object has two costs a true type split avoids: every file still keeps an +8-byte `_extra` reference, and every directory pays a 16-byte object header. At this fixture's 10:1 +ratio that nets ~5 B/entry; at a real drive's ~6:1 (per the project's own notes) it is smaller still. + +The full polymorphic split (files as a lean type with no dir fields and no per-file ref; directories +with the fields inline and no extra object) would save ~3× more, but requires making `DirEntry` an +abstract MessagePack-union root, changing `IList` throughout lib + GUI + web + tests, +rewriting ~40 construction sites, a `.cde` format bump with migration, and FlatSharp/protobuf union +handling — a large change whose WinForms GUI behavior cannot be runtime-validated in this +environment (only via presenter unit tests). Left as an explicit, opt-in follow-up. + +### Correctness +- `cdeLibTest`: 126 passed; `cdeWinTest` (GUI presenter): 32 passed. +- Round-trip byte-identical; no find CPU regression. diff --git a/src/cdeLib/Entities/DirEntry.cs b/src/cdeLib/Entities/DirEntry.cs index 43b80cd..7cb35f1 100644 --- a/src/cdeLib/Entities/DirEntry.cs +++ b/src/cdeLib/Entities/DirEntry.cs @@ -18,6 +18,28 @@ public sealed class DirEntry : ICommonEntry { private string _path; + /// + /// Side-object holding directory-only state — the child list and the rolled-up summary counts. + /// Null on every file (the vast majority of entries), so a file no longer carries an always-null + /// Children reference plus two count fields. Only directories (~2% of entries) allocate it. + /// Serialization is unaffected: Children stays Key 3 via the property below, just backed here. + /// + /// Note: the content Hash deliberately stays INLINE on the entry. Moving it here too would force + /// every *hashed file* to allocate an ExtraData whose object header costs more than the 16-byte + /// Hash16 it replaced — a net regression for hashed catalogs. Keeping Hash inline means a hashed + /// file needs no side-object at all, so this change never increases footprint for any catalog. + /// + private sealed class ExtraData + { + public IList Children; + public uint FileEntryCount; + public uint DirEntryCount; + } + + private ExtraData _extra; + + private ExtraData EnsureExtra() => _extra ??= new ExtraData(); + [IgnoreMember] public DateTime Modified { @@ -160,13 +182,29 @@ public bool IsDefaultSort /// if this is a directory number of files contained in its hierarchy /// [IgnoreMember] - public uint FileEntryCount { get; set; } + public uint FileEntryCount + { + get => _extra?.FileEntryCount ?? 0; + set + { + if (value != 0) EnsureExtra().FileEntryCount = value; + else if (_extra != null) _extra.FileEntryCount = value; + } + } /// /// if this is a directory number of dirs contained in its hierarchy /// [IgnoreMember] - public uint DirEntryCount { get; set; } + public uint DirEntryCount + { + get => _extra?.DirEntryCount ?? 0; + set + { + if (value != 0) EnsureExtra().DirEntryCount = value; + else if (_extra != null) _extra.DirEntryCount = value; + } + } public void SetHash(byte[] hash) { @@ -337,17 +375,24 @@ public void SetSummaryFields() [ProtoMember(3, IsRequired = false)] [FlatBufferItem(3)] [Key(3)] - public IList Children { get; set; } - // ReSharper restore MemberCanBePrivate.Global - - public void AddChild(DirEntry child) + public IList Children { - if (Children == null) + get => _extra?.Children; + set { - Children = CollectionPool.GetDirEntryList(); + // A non-null child list (only directories have one) materialises ExtraData; files + // deserialize a nil Children and stay lean. + if (value != null) EnsureExtra().Children = value; + else if (_extra != null) _extra.Children = null; } + } + // ReSharper restore MemberCanBePrivate.Global - Children.Add(child); + public void AddChild(DirEntry child) + { + var extra = EnsureExtra(); + extra.Children ??= CollectionPool.GetDirEntryList(); + extra.Children.Add(child); } [ProtoMember(4, IsRequired = true)] From b6545c48812d765d98c53ba82cab35968304cc8b Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sat, 6 Jun 2026 21:58:23 +1000 Subject: [PATCH 07/43] docs(perf): investigate entry-name memory (names are ~35% of footprint) Add a --shared-names generator mode to isolate the per-file name-string cost, and document the finding: names cost ~70 B/entry (198.85 normal vs 129.23 with a shared name on a 1M fixture) - the single biggest remaining memory lever. Two options assessed with estimates + risks: UTF-8 byte[] per name (~15-20 B/entry, hot-path byte-matching rewrite) and a per-RootEntry name pool (~40 B/entry, largest win, very invasive). Both rewrite the hottest comparison path and bump the .cde format. See src/cdeBenchmarks/baseline/name-storage-investigation.md. --- .../baseline/name-storage-investigation.md | 62 +++++++++++++++++++ src/cdeMemProbe/Program.cs | 3 +- src/cdeMemProbe/SyntheticCatalog.cs | 18 ++++-- 3 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 src/cdeBenchmarks/baseline/name-storage-investigation.md diff --git a/src/cdeBenchmarks/baseline/name-storage-investigation.md b/src/cdeBenchmarks/baseline/name-storage-investigation.md new file mode 100644 index 0000000..df354ab --- /dev/null +++ b/src/cdeBenchmarks/baseline/name-storage-investigation.md @@ -0,0 +1,62 @@ +# Investigation: entry-name memory (UTF-8 / name-pool) + +## Measured: names are the dominant per-entry cost + +1M-entry fixture, no hashes, after Phases 1+3: + +| Name storage | Bytes/entry | Note | +|--------------|------------:|------| +| normal (~16-char unique names) | 198.85 | current | +| shared single 1-char name | **129.23** | isolates everything-except-name-strings | +| **name contribution** | **≈ 69.6 B/entry** | **~35% of total footprint** | + +Reproduce: `cdeMemProbe --generate 1000000 [--shared-names] --out x.cde` then `cdeMemProbe x.cde`. + +Names are stored today as two interned UTF-16 `string` objects per entry (`_path` = name without +extension, `field` = extension). For a 16-char ASCII name the `_path` object is ~54 B (16 B header + +4 B length + 32 B chars + 2) plus the 8 B reference in the entry. The extension is interned and +shared, so its amortised cost is small; almost all of the ~70 B is the per-file `_path` string. + +Two structural costs make this large: (a) UTF-16 spends **2 bytes per char** for names that are +overwhelmingly ASCII, and (b) every name is a **separate heap object** carrying a ~22 B header. + +## Option A — UTF-8 `byte[]` per name + +Store `_path` as a UTF-8 `byte[]` instead of a `string`. +- **Saves** the char halving only: ~16 B for a 16-char ASCII name → **est. ~15–20 B/entry** (~8–10%). +- **Keeps** the per-name object header (a `byte[]` header ≈ a `string` header). +- **Risk / work (high, hot path):** `DirEntry.Path` is read by the *hottest* operations — find + `Contains`/regex, `PathCompareWithDirTo` sort, path building, GUI display. If the getter + reconstructs a `string`, every access allocates and **search regresses massively** (we just made + it 48×/18× faster). Avoiding that means rewriting matching/sorting to work on `byte` spans: + - substring search → UTF-8 byte `IndexOf` (fine for ASCII; **case-insensitive non-ASCII is hard**). + - ordinal-ignore-case sort → byte compare (ASCII ok; Unicode ordering differs). + - regex → needs a `string`; would still allocate (or a UTF-8 regex engine). + - loses `string.Intern` dedup of repeated filenames (e.g. `index.html`, `__init__.py` recur a lot) + unless a `byte[]` dedup pool is added. +- **Format:** `Path` (Key 5) would serialize as bytes → `.cde` format bump (also shrinks the file). + +## Option B — name pool (offset into a shared UTF-8 buffer) ← biggest win + +Hold all names of a catalog in one big UTF-8 `byte[]` per `RootEntry`; each entry stores an +`int` offset + `short` length instead of string references. +- **Eliminates the per-name object header entirely** *and* halves char bytes. +- Per file: ~6 B (offset+len) in the entry + ~16 B in the pool = **~22 B vs ~62–70 B today → + est. ~40 B/entry saved (~20% of footprint)**. Largest lever available, by far. +- **Risk / work (very high):** same hot-path byte-matching rewrite as Option A, **plus** a pool + built at load, pool growth/lifetime management, dedup strategy (replacing interning), and a + bigger `.cde` format change. Touches lib + GUI + web + dupes. + +## Recommendation + +Names are the single biggest remaining memory lever (~35%), so the upside is real and larger than +the file/dir split. But both options rewrite the **hottest** code (the same name comparison that the +Phase 1–2 search wins depend on) and change the `.cde` format, with genuine Unicode-correctness +pitfalls in case-insensitive matching. This is a larger, riskier effort than Phases 1–3 combined and +warrants its own plan + prototype. + +Suggested path if pursued: prototype **Option A** behind the existing benchmarks first — convert +`_path` to UTF-8 with byte-based substring/sort matching — and measure BOTH the footprint delta and +the search-time delta on the 1M fixture before deciding whether the saving justifies the hot-path +risk. Only escalate to **Option B** (the pool) if the measured Option-A search impact is acceptable +and the extra ~20 B/entry is needed. diff --git a/src/cdeMemProbe/Program.cs b/src/cdeMemProbe/Program.cs index ff97c49..a070f13 100644 --- a/src/cdeMemProbe/Program.cs +++ b/src/cdeMemProbe/Program.cs @@ -61,6 +61,7 @@ private static async Task GenerateAsync(string[] args, string? countArg, IL } var withHashes = HasFlag(args, "--hashes", out _); + var sharedNames = HasFlag(args, "--shared-names", out _); var seed = HasFlag(args, "--seed", out var seedArg) && int.TryParse(seedArg, out var s) ? s : 42; var outPath = HasFlag(args, "--out", out var outArg) && !string.IsNullOrWhiteSpace(outArg) ? outArg! @@ -68,7 +69,7 @@ private static async Task GenerateAsync(string[] args, string? countArg, IL Console.Error.WriteLine($"Generating ~{count:N0} entries (hashes={withHashes}, seed={seed}) ..."); var sw = Stopwatch.StartNew(); - var root = SyntheticCatalog.Generate(count, withHashes, seed); + var root = SyntheticCatalog.Generate(count, withHashes, seed, sharedNames: sharedNames); root.ActualFileName = outPath; using (var repo = new CatalogRepository(logger)) { diff --git a/src/cdeMemProbe/SyntheticCatalog.cs b/src/cdeMemProbe/SyntheticCatalog.cs index 80b75eb..e012672 100644 --- a/src/cdeMemProbe/SyntheticCatalog.cs +++ b/src/cdeMemProbe/SyntheticCatalog.cs @@ -39,7 +39,8 @@ public static RootEntry Generate( bool withHashes = false, int seed = 42, int filesPerDir = 50, - int subDirsPerDir = 5) + int subDirsPerDir = 5, + bool sharedNames = false) { var random = new Random(seed); var root = new RootEntry @@ -71,9 +72,18 @@ void Populate(RootEntry? rootParent, DirEntry? dirParent) for (var i = 0; i < filesPerDir && created < targetEntryCount; i++) { var file = new DirEntry(false); - var ext = Extensions[random.Next(Extensions.Length)]; - var prefix = Prefixes[random.Next(Prefixes.Length)]; - file.SetPath($"{prefix}_{created:D7}{ext}"); + // sharedNames: every file shares one interned name, so the measured footprint + // excludes per-file name strings — the delta vs normal isolates the name cost. + if (sharedNames) + { + file.SetPath("x"); + } + else + { + var ext = Extensions[random.Next(Extensions.Length)]; + var prefix = Prefixes[random.Next(Prefixes.Length)]; + file.SetPath($"{prefix}_{created:D7}{ext}"); + } file.Size = random.Next(1, 50_000_000); file.Modified = BaseDate.AddMinutes(random.Next(0, 5_000_000)); if (withHashes) From 8efea3c25aa12b7ef0c421eef4fa0a3423c6d854 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sat, 6 Jun 2026 22:05:54 +1000 Subject: [PATCH 08/43] fix(find): make Cancel Search responsive again Cancellation was only checked inside the throttled progress-reporting block ("only check for cancel on progress reports"). Since the adaptive-frequency change (7d46506/a22539b2), progress reports fire only every ~50k entries on large catalogs, so an active search - especially a slow regex - ignored Worker.CancellationPending for tens of thousands of entries and felt like it never cancelled. Check the cheap CancellationPending volatile read every 4096 entries, independently of progress reporting, in both the sync (cdeWin / CLI) and legacy async find paths. Responsive even on a slow regex; negligible overhead on a fast full scan. cdeLibTest 126 passed; full solution builds clean. --- src/cdeLib/FindOptions.cs | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/src/cdeLib/FindOptions.cs b/src/cdeLib/FindOptions.cs index 81c74e0..5b1c8d1 100644 --- a/src/cdeLib/FindOptions.cs +++ b/src/cdeLib/FindOptions.cs @@ -65,6 +65,10 @@ public class FindOptions private readonly int[] _dummyProgressCount = new int[1]; + // Check Worker.CancellationPending every 4096 entries (mask = 4096-1). Frequent enough to feel + // instant even on a slow regex, cheap enough to be negligible on a fast full scan. + private const int CancelCheckMask = 4096 - 1; + public int SkipCount { get; set; } public int ProgressCount => _threadSafeProgressCount; @@ -200,6 +204,12 @@ private Func> CreateAsyncProcessor(int[] return true; // Skip enforced } + // Honour cancellation promptly, independently of throttled progress reporting (see GetFindFunc). + if ((currentCount & CancelCheckMask) == 0 && Worker?.CancellationPending == true) + { + return false; + } + // Rate-limited progress reporting with non-blocking UI update if (ProgressModifier > 0 && ShouldReportProgress(currentCount)) { @@ -309,15 +319,19 @@ bool FindFunc(ICommonEntry p, ICommonEntry dirEntry) return true; } + // Honour cancellation promptly. CancellationPending is a cheap volatile read, so check it + // often (every CancelCheckInterval entries) INDEPENDENTLY of progress reporting. Progress + // reporting is throttled to every ~50k entries to cut UI marshaling cost; tying the cancel + // check to it (as before) made a slow search ignore Cancel for tens of thousands of entries. + if ((currentCount & CancelCheckMask) == 0 && Worker?.CancellationPending == true) + { + return false; // end the find. + } + // Use lock-free progress reporting with reduced frequency if (ProgressModifier > 0 && ShouldReportProgress(currentCount)) { ProgressFunc(currentCount, ProgressEnd); - // only check for cancel on progress reports. - if (Worker?.CancellationPending == true) - { - return false; // end the find. - } } if (findPredicate(p, dirEntry)) From ef4a733daca63c843b1b9863d4fa29101fcd3386 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sat, 6 Jun 2026 22:15:20 +1000 Subject: [PATCH 09/43] perf(find): stream search progress on a 100ms timer (sync path responsiveness) Raw search throughput strongly favours the synchronous path at every catalog count (benchmarked: sync vs legacy async = 44x @ 1 catalog, 30x @ 10, 31x @ 100). The async path only ever "felt faster" because it streamed progress/results on a 100ms timer, while the sync path reported every ~50k entries - so on a long search results appeared in large infrequent chunks. Give the sync path the same time-based streaming: behind the cheap 4096-entry housekeeping gate, report progress at most every ~100ms (tick-threshold compare, no multiply/overflow). Fast AND smooth. Sync speed unchanged (33-55ms for 1M entries across 1/10/100 catalogs). Adds MultiCatalogSearchBenchmarks documenting the sync-vs-async comparison across catalog counts. --- .../MultiCatalogSearchBenchmarks.cs | 69 +++++++++++++++++++ src/cdeLib/FindOptions.cs | 42 +++++++---- 2 files changed, 98 insertions(+), 13 deletions(-) create mode 100644 src/cdeBenchmarks/MultiCatalogSearchBenchmarks.cs diff --git a/src/cdeBenchmarks/MultiCatalogSearchBenchmarks.cs b/src/cdeBenchmarks/MultiCatalogSearchBenchmarks.cs new file mode 100644 index 0000000..70a5ac9 --- /dev/null +++ b/src/cdeBenchmarks/MultiCatalogSearchBenchmarks.cs @@ -0,0 +1,69 @@ +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Threading; +using BenchmarkDotNet.Attributes; +using cdeLib; +using cdeLib.Entities; +using cdeMemProbe; + +namespace cdeBenchmarks; + +/// +/// Sync (production) vs legacy async find across DIFFERENT catalog counts at a fixed ~1M total +/// entries. Answers: does the synchronous path (parallel across roots) still win when there are +/// many catalogs (e.g. 100), where the work-stealing async path could parallelize within roots too? +/// +/// dotnet run -c Release --filter *MultiCatalog* +/// +[MemoryDiagnoser] +[SimpleJob(warmupCount: 2, iterationCount: 5)] +public class MultiCatalogSearchBenchmarks +{ + private const int TotalEntries = 1_000_000; + + private IList _roots = null!; + + // 1 = single big catalog; 100 = the user's many-catalogs scenario (10k entries each). + [Params(1, 10, 100)] + public int RootCount { get; set; } + + [GlobalSetup] + public void Setup() + { + var perRoot = TotalEntries / RootCount; + var roots = new List(RootCount); + for (var i = 0; i < RootCount; i++) + { + // Distinct seed per root so the catalogs differ, like a real multi-drive load. + roots.Add(SyntheticCatalog.Generate(perRoot, withHashes: false, seed: 1000 + i)); + } + _roots = roots; + } + + private FindOptions MakeOptions(StrongBox counter) => new() + { + Pattern = "zzzznomatchzzzz", // full scan — measures pure traversal throughput + RegexMode = false, + IncludePath = false, + IncludeFiles = true, + IncludeFolders = true, + LimitResultCount = int.MaxValue, + VisitorFunc = (_, _) => { Interlocked.Increment(ref counter.Value); return true; }, + }; + + [Benchmark(Baseline = true, Description = "sync Find (parallel across roots)")] + public int Sync() + { + var c = new StrongBox(); + MakeOptions(c).Find(_roots); + return c.Value; + } + + [Benchmark(Description = "legacy async FindAsync (work-stealing)")] + public int Async() + { + var c = new StrongBox(); + MakeOptions(c).FindAsync(_roots).GetAwaiter().GetResult(); + return c.Value; + } +} diff --git a/src/cdeLib/FindOptions.cs b/src/cdeLib/FindOptions.cs index 5b1c8d1..49b1162 100644 --- a/src/cdeLib/FindOptions.cs +++ b/src/cdeLib/FindOptions.cs @@ -65,10 +65,15 @@ public class FindOptions private readonly int[] _dummyProgressCount = new int[1]; - // Check Worker.CancellationPending every 4096 entries (mask = 4096-1). Frequent enough to feel - // instant even on a slow regex, cheap enough to be negligible on a fast full scan. + // Run cancellation + progress housekeeping every 4096 entries (mask = 4096-1). Frequent enough to + // feel instant even on a slow regex, cheap enough to be negligible on a fast full scan. private const int CancelCheckMask = 4096 - 1; + // Stream progress/results at most every ~100ms (time-based, like the old async path), so a long + // search updates the UI smoothly rather than in large infrequent entry-count-based chunks. + // Stored as a tick threshold so the hot-path check is a plain subtraction (no multiply/overflow). + private static readonly long ProgressIntervalTicks = Stopwatch.Frequency / 10; + public int SkipCount { get; set; } public int ProgressCount => _threadSafeProgressCount; @@ -319,19 +324,30 @@ bool FindFunc(ICommonEntry p, ICommonEntry dirEntry) return true; } - // Honour cancellation promptly. CancellationPending is a cheap volatile read, so check it - // often (every CancelCheckInterval entries) INDEPENDENTLY of progress reporting. Progress - // reporting is throttled to every ~50k entries to cut UI marshaling cost; tying the cancel - // check to it (as before) made a slow search ignore Cancel for tens of thousands of entries. - if ((currentCount & CancelCheckMask) == 0 && Worker?.CancellationPending == true) + // Periodic housekeeping behind a cheap entry-count gate (~every 4096 entries): + // 1. Honour cancellation promptly (was tied to the ~50k-entry progress throttle, which + // made a slow search ignore Cancel for tens of thousands of entries). + // 2. Stream progress/results on a ~100ms timer so results appear smoothly during a long + // search instead of in large infrequent chunks. This matches the responsiveness of + // the old async path (which felt faster purely because it streamed every 100ms), + // while keeping the synchronous path's much higher raw throughput. + if ((currentCount & CancelCheckMask) == 0) { - return false; // end the find. - } + if (Worker?.CancellationPending == true) + { + return false; // end the find. + } - // Use lock-free progress reporting with reduced frequency - if (ProgressModifier > 0 && ShouldReportProgress(currentCount)) - { - ProgressFunc(currentCount, ProgressEnd); + if (ProgressFunc != null && ProgressModifier > 0) + { + var now = Stopwatch.GetTimestamp(); + var last = Interlocked.Read(ref _lastProgressTimestamp); + if (now - last >= ProgressIntervalTicks + && Interlocked.CompareExchange(ref _lastProgressTimestamp, now, last) == last) + { + ProgressFunc(currentCount, ProgressEnd); + } + } } if (findPredicate(p, dirEntry)) From 9a6449c3328f1f200863fe6de9a448d250b91335 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sat, 6 Jun 2026 22:17:50 +1000 Subject: [PATCH 10/43] fix(cdeWin): make search result collection thread-safe across catalogs Find parallelizes across catalogs, so VisitorFunc runs on multiple threads. The result list used a plain List.Add, which is not thread-safe - searching many catalogs at once could drop results or throw as concurrent adds raced on the backing array. Guard Add with a lock, and hand the UI an immutable snapshot taken under that lock on each progress tick instead of the live list the worker threads are still mutating (the virtual ListView indexes into it on the UI thread). After Find() returns all threads are joined, so the final result uses the list directly with no extra copy. cdeWin builds clean; cdeWinTest 32 passed. --- src/cdeWin/CDEWinFormPresenter.cs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/cdeWin/CDEWinFormPresenter.cs b/src/cdeWin/CDEWinFormPresenter.cs index 4dd3225..a66b375 100644 --- a/src/cdeWin/CDEWinFormPresenter.cs +++ b/src/cdeWin/CDEWinFormPresenter.cs @@ -541,18 +541,32 @@ private void BgWorkerDoWork(object sender, DoWorkEventArgs e) var state = argument.State; var list = new List(500); - state.ListCount = list.Count; // 0 + var listLock = new object(); + state.ListCount = 0; state.List = list; worker.ReportProgress(0, state); + // Find parallelizes across catalogs, so VisitorFunc runs on multiple threads. List.Add is + // not thread-safe — without this lock, searching many catalogs at once could drop results or + // throw as concurrent adds race on the backing array. findOptions.VisitorFunc = (p, d) => { - list.Add(new PairDirEntry(p, d)); + lock (listLock) + { + list.Add(new PairDirEntry(p, d)); + } return true; }; + // Hand the UI an immutable snapshot taken under the lock — never the live list, which worker + // threads are still mutating while the (virtual) ListView indexes into it on the UI thread. findOptions.ProgressFunc = (counter, end) => { - state.ListCount = list.Count; // concurrency ! - state.List = list; // concurrency !!!! + List snapshot; + lock (listLock) + { + snapshot = new List(list); + } + state.ListCount = snapshot.Count; + state.List = snapshot; state.Counter = counter; state.End = end; worker.ReportProgress((int)(100.0 * counter / end), state); From e2c4bc0dd9cdb6a7305efb221b4df7d094a8bcdd Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sat, 6 Jun 2026 22:45:57 +1000 Subject: [PATCH 11/43] perf(spike): prototype + measure struct-of-arrays EntryStore Prototype a SoA catalog model (parallel value-type arrays; tree shape via int firstChild/nextSibling/parent indices) and a tree->SoA converter, with a cdeMemProbe --soa mode to measure its retained footprint. Measured (1M/10M fixtures): structural cost drops from 129.23 to 42.23 B/entry (-67%) - SoA holds the same catalog structure in ~1/3 the memory (10M: 421 MB vs ~1.29 GB). It is the only option that removes the per-entry object header. Search and full-path reconstruction validated on the store. Prototype only (lives in cdeMemProbe, production model untouched). A real migration is a core-model rewrite (serialization, search, dupes, hashing, GUI adapter); see src/cdeBenchmarks/baseline/soa-prototype.md for the proven numbers and phasing. --- src/cdeBenchmarks/baseline/soa-prototype.md | 56 +++++++++ src/cdeMemProbe/EntryStore.cs | 132 ++++++++++++++++++++ src/cdeMemProbe/Program.cs | 58 +++++++++ 3 files changed, 246 insertions(+) create mode 100644 src/cdeBenchmarks/baseline/soa-prototype.md create mode 100644 src/cdeMemProbe/EntryStore.cs diff --git a/src/cdeBenchmarks/baseline/soa-prototype.md b/src/cdeBenchmarks/baseline/soa-prototype.md new file mode 100644 index 0000000..6422fe7 --- /dev/null +++ b/src/cdeBenchmarks/baseline/soa-prototype.md @@ -0,0 +1,56 @@ +# Prototype: Struct-of-Arrays (SoA) EntryStore + +Feasibility + footprint measurement for replacing the pointer-based `DirEntry`/`RootEntry` tree +with parallel value-type arrays (one slot per entry; tree shape via `int` firstChild/nextSibling/ +parent indices). Prototype: `src/cdeMemProbe/EntryStore.cs`. Measure: +`cdeMemProbe --soa --generate N [--shared-names] [--hashes]`. + +## Measured footprint (1M & 10M fixtures) + +| Model | Structural (shared names) | Full (normal names) | +|-------|--------------------------:|--------------------:| +| Pointer tree (current) | 129.23 B/entry | 198.85 B/entry | +| **SoA prototype** | **42.23 B/entry** | 151.39 B/entry | +| **Saving** | **−87 B/entry (−67%)** | −47 B/entry (−24%) | + +- Scales linearly: SoA at 10M = **42.11 B/entry** (421 MB) vs the tree's ~1.29 GB of structural + memory — i.e. SoA holds the same 10M-entry catalog structure in **~1/3 the memory**. +- The "full / normal names" SoA number is inflated only because the prototype stores **un-interned, + un-split** full names. A production SoA would intern + split names like the tree does, landing the + total near **~112 B/entry (≈ −44%)**. The clean, name-independent result is the structural row. +- **Hashed catalogs:** the prototype's `--hashes` run did *not* allocate the `Hash16[]` side array + (the `root.IsHashDone` heuristic is wrong — only files are hashed), so it reads 42 B. A real hashed + SoA adds 16 B/entry → **~58 B/entry**, still far below the tree's ~145 B/entry hashed structural. + +## Why it wins + +Per entry, the tree pays a 16 B object header + 8 B-each reference fields + allocation rounding + +per-directory `List` overhead. SoA pays only packed array slots: +`long modified (8) + long size (8) + string name-ref (8) + byte flags (1) + int firstChild/nextSibling/parent (4+4+4)` +≈ **37 B + negligible per-array overhead** — no per-entry header, references shrink 8 B → 4 B. + +Validated working in the prototype: linear name-substring search over `Name[]` and full-path +reconstruction by walking `Parent[]` both produce correct results. + +## What a production migration would require (large, multi-day, high risk) + +The prototype proves the memory win; shipping it is a core-model rewrite: +- **Serialization** — read/write the arrays (or convert tree↔SoA at load/save). FlatSharp (already + wired) suits SoA well; format bump. +- **Search / sort / dupes / hashing** — operate on indices instead of objects. Likely *faster* + (cache locality), but every algorithm is re-pointed; hashing writes back into `Hash[]`. +- **GUI (cdeWin) + web** — navigate via `ICommonEntry`/`FullPath`/`GetListFromRoot`. Bridge with a + thin `readonly struct EntryRef(store, index) : ICommonEntry` adapter to minimize churn, or rewrite. +- **Construction sites + tests** — the ~40 `new DirEntry(...)` sites and the entity tests. + +Risk is highest of all options (it is *the* data model, on the billions-of-entries hot paths) and +the WinForms GUI can't be runtime-validated here. But the payoff is the largest by far: ~−67% +structural memory, the only option that removes the per-entry object header. + +## Recommendation + +SoA is decisively the highest-impact memory lever — proven, not estimated. Worth doing **if** the +team is prepared for a core-model migration. Suggested phasing: (1) land the SoA store + tree↔SoA +converter + a `EntryRef` adapter behind the existing `ICommonEntry` API so the GUI/dupes keep +working; (2) move search/serialization onto the store; (3) drop the tree. Each phase measured against +this prototype. diff --git a/src/cdeMemProbe/EntryStore.cs b/src/cdeMemProbe/EntryStore.cs new file mode 100644 index 0000000..773afc0 --- /dev/null +++ b/src/cdeMemProbe/EntryStore.cs @@ -0,0 +1,132 @@ +using System; +using System.Collections.Generic; +using cdeLib.Entities; + +namespace cdeMemProbe; + +/// +/// PROTOTYPE struct-of-arrays representation of a single catalog, to measure the memory a SoA model +/// would use versus the production pointer-based tree. One slot per entry +/// across parallel arrays; the tree shape is encoded with int indices (firstChild / nextSibling / +/// parent) instead of object references — so there is NO per-entry object header and references +/// become 4-byte ints. +/// +/// This is a measurement/feasibility prototype, not the production model. It deliberately keeps +/// names in a string[] (one ref per entry, exactly like today) so the comparison isolates the +/// STRUCTURAL saving (object headers + link fields), not name storage. +/// +public sealed class EntryStore +{ + public const int None = -1; + + public readonly int Count; + + // One slot per entry. Index 0 is the root. + public readonly long[] ModifiedTicks; + public readonly long[] Size; + public readonly string[] Name; + public readonly byte[] BitFields; + public readonly int[] FirstChild; // None if no children + public readonly int[] NextSibling; // None if last sibling + public readonly int[] Parent; // None for the root + + // Hashes are allocated only when the catalog actually has them (the common load-to-search + // catalog has none) — so an un-hashed catalog pays zero bytes here, unlike the inline 16-byte + // Hash16 on every DirEntry today. + public Hash16[] Hash; + + public string RootPath; + + private int _next; + + private EntryStore(int count) + { + Count = count; + ModifiedTicks = new long[count]; + Size = new long[count]; + Name = new string[count]; + BitFields = new byte[count]; + FirstChild = new int[count]; + NextSibling = new int[count]; + Parent = new int[count]; + } + + public bool IsDirectory(int i) => ((Flags)BitFields[i] & Flags.Directory) == Flags.Directory; + + /// + /// Build a SoA store from a loaded/generated catalog tree. The tree can be released afterwards; + /// the store is self-contained. + /// + public static EntryStore Build(RootEntry root) + { + var count = checked((int)(root.FileEntryCount + root.DirEntryCount) + 1); // +1 for the root itself + var store = new EntryStore(count) { RootPath = root.Path }; + + var hashed = root.IsHashDone; // catalogs are hashed wholesale; cheap heuristic for the prototype + if (hashed) store.Hash = new Hash16[count]; + + var rootIdx = store._next++; + store.Name[rootIdx] = root.Path; + store.ModifiedTicks[rootIdx] = root.ModifiedTicks; + store.Size[rootIdx] = root.Size; + store.BitFields[rootIdx] = (byte)root.BitFields; + store.Parent[rootIdx] = None; + store.FirstChild[rootIdx] = None; + store.NextSibling[rootIdx] = None; + + store.FillChildren(root.Children, rootIdx); + return store; + } + + private void FillChildren(IList children, int parentIdx) + { + if (children == null) return; + + var prevSibling = None; + foreach (var child in children) + { + var idx = _next++; + Name[idx] = child.Path; + ModifiedTicks[idx] = child.ModifiedTicks; + Size[idx] = child.Size; + BitFields[idx] = (byte)child.BitFields; + Parent[idx] = parentIdx; + FirstChild[idx] = None; + NextSibling[idx] = None; + if (Hash != null) Hash[idx] = child.Hash; + + if (prevSibling == None) FirstChild[parentIdx] = idx; + else NextSibling[prevSibling] = idx; + prevSibling = idx; + + if (child.IsDirectory) FillChildren(child.Children, idx); + } + } + + /// Full path of an entry, walking parent indices into a reused buffer (no per-node strings). + public string FullPath(int i) + { + // Collect ancestors (leaf -> root) then write root-first. + var stack = new Stack(); + for (var cur = i; cur != None; cur = Parent[cur]) stack.Push(cur); + var sb = new System.Text.StringBuilder(128); + while (stack.Count > 0) + { + var idx = stack.Pop(); + if (sb.Length > 0 && sb[^1] != '\\' && sb[^1] != '/') sb.Append('\\'); + sb.Append(Name[idx]); + } + return sb.ToString(); + } + + /// Linear name-substring search over the flat arrays. Returns match count. + public int CountNameMatches(string pattern) + { + var matches = 0; + for (var i = 1; i < Count; i++) // skip root at 0 + { + if (Name[i].Contains(pattern, StringComparison.OrdinalIgnoreCase)) matches++; + } + return matches; + } +} diff --git a/src/cdeMemProbe/Program.cs b/src/cdeMemProbe/Program.cs index a070f13..ab18118 100644 --- a/src/cdeMemProbe/Program.cs +++ b/src/cdeMemProbe/Program.cs @@ -2,6 +2,7 @@ using System.Diagnostics; using System.Globalization; using System.IO; +using System.Runtime.CompilerServices; using System.Threading.Tasks; using cdeLib.Catalog; using cdeLib.Entities; @@ -44,6 +45,11 @@ public static async Task Main(string[] args) // Silent Serilog logger (no sinks) so CatalogRepository stays quiet and out of the CSV. var logger = new LoggerConfiguration().CreateLogger(); + if (HasFlag(args, "--soa", out _)) + { + return MeasureSoa(args); + } + if (HasFlag(args, "--generate", out var genValue)) { return await GenerateAsync(args, genValue, logger); @@ -85,6 +91,58 @@ private static async Task GenerateAsync(string[] args, string? countArg, IL return 0; } + /// + /// Measure the retained footprint of the PROTOTYPE struct-of-arrays EntryStore, for comparison + /// with the pointer-tree model. Usage: cdeMemProbe --soa --generate N [--shared-names] [--hashes] + /// + private static int MeasureSoa(string[] args) + { + if (!HasFlag(args, "--generate", out var countArg) + || !int.TryParse(countArg, NumberStyles.Integer, CultureInfo.InvariantCulture, out var count) + || count <= 0) + { + Console.Error.WriteLine("--soa requires --generate , e.g. --soa --generate 1000000"); + return 1; + } + + var withHashes = HasFlag(args, "--hashes", out _); + var sharedNames = HasFlag(args, "--shared-names", out _); + + Console.Error.WriteLine( + $"Building SoA EntryStore for ~{count:N0} entries (hashes={withHashes}, sharedNames={sharedNames}) ..."); + + var store = BuildStoreReleasingTree(count, withHashes, sharedNames); + + // Settle so only the live EntryStore (not the now-dead source tree) is counted. + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + var managed = GC.GetTotalMemory(true); + GC.KeepAlive(store); + var bytesPerEntry = store.Count > 0 ? (double)managed / store.Count : 0; + + // Sanity-check the SoA actually works as a searchable structure. + var matches = store.CountNameMatches(sharedNames ? "x" : ".txt"); + + Console.WriteLine("model,entries,managedBytes,bytesPerEntry"); + Console.WriteLine(string.Join(',', "soa", + store.Count.ToString(CultureInfo.InvariantCulture), + managed.ToString(CultureInfo.InvariantCulture), + bytesPerEntry.ToString("F2", CultureInfo.InvariantCulture))); + Console.Error.WriteLine($"sanity: {matches:N0} name matches; full path[1] = {store.FullPath(1)}"); + return 0; + } + + // Separate non-inlined method so the source tree local is out of scope (collectable) before we + // measure the store in the caller. + [MethodImpl(MethodImplOptions.NoInlining)] + private static EntryStore BuildStoreReleasingTree(int count, bool withHashes, bool sharedNames) + { + var tree = SyntheticCatalog.Generate(count, withHashes, seed: 42, sharedNames: sharedNames); + return EntryStore.Build(tree); + } + private static async Task MeasureAsync(string file, bool printHeader, ILogger logger) { if (!File.Exists(file)) From 079149e3f3bd7a1152a301b608f7270f198c235d Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sat, 6 Jun 2026 22:54:09 +1000 Subject: [PATCH 12/43] perf(soa): production EntryStore + index search + equivalence tests (migration P1) First phase of the struct-of-arrays migration, additive - the production pointer tree is untouched. - cdeLib/Entities/Soa/EntryStore.cs: SoA catalog model (parallel arrays; int firstChild/nextSibling/parent links; lazy Hash[] only when hashed) plus a robust iterative tree->store converter (counts the real tree, not summary fields) and allocation-free full-path reconstruction by walking Parent[]. - cdeLib/Entities/Soa/EntryStoreSearch.cs: index-based find (substring/regex, name/path, file/folder filter) - a cache-friendly flat-array scan. - cdeLibTest/Soa/EntryStoreTests.cs: 12 tests proving store search results are IDENTICAL to the pointer-tree FindOptions across every query shape, plus full-path equivalence for every entry. - cdeMemProbe now uses the cdeLib EntryStore (prototype copy removed). Footprint reconfirmed: 42.23 B/entry structural vs 129.23 for the tree (-67%). cdeLibTest 138 passed. --- src/cdeLib/Entities/Soa/EntryStore.cs | 182 ++++++++++++++++++++ src/cdeLib/Entities/Soa/EntryStoreSearch.cs | 71 ++++++++ src/cdeLibTest/Soa/EntryStoreTests.cs | 138 +++++++++++++++ src/cdeMemProbe/EntryStore.cs | 132 -------------- src/cdeMemProbe/Program.cs | 5 +- 5 files changed, 395 insertions(+), 133 deletions(-) create mode 100644 src/cdeLib/Entities/Soa/EntryStore.cs create mode 100644 src/cdeLib/Entities/Soa/EntryStoreSearch.cs create mode 100644 src/cdeLibTest/Soa/EntryStoreTests.cs delete mode 100644 src/cdeMemProbe/EntryStore.cs diff --git a/src/cdeLib/Entities/Soa/EntryStore.cs b/src/cdeLib/Entities/Soa/EntryStore.cs new file mode 100644 index 0000000..6ccc7a1 --- /dev/null +++ b/src/cdeLib/Entities/Soa/EntryStore.cs @@ -0,0 +1,182 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace cdeLib.Entities.Soa; + +/// +/// Struct-of-arrays representation of a single catalog: one slot per entry across parallel arrays, +/// with the tree shape encoded as indices (firstChild / nextSibling / parent) +/// instead of object references. Holds the same catalog structure as a tree +/// in roughly one third of the structural memory — no per-entry object header, and references shrink +/// from 8-byte pointers to 4-byte indices. +/// +/// Phase 1 of the SoA migration: this is additive. converts an existing tree into +/// a store; the production tree model is unchanged. Later phases move load/search/serialization onto +/// the store and retire the tree. +/// +/// Index 0 is always the catalog root. (-1) terminates child/sibling chains and +/// marks the root's (absent) parent. +/// +public sealed class EntryStore +{ + public const int None = -1; + + public int Count { get; private set; } + + // One slot per entry (index 0 = root). + public long[] ModifiedTicks { get; private set; } + public long[] Size { get; private set; } + public string[] Name { get; private set; } + public byte[] BitFields { get; private set; } + public int[] FirstChild { get; private set; } + public int[] NextSibling { get; private set; } + public int[] Parent { get; private set; } + + /// + /// Content hashes, allocated lazily only when at least one entry is hashed. Null for the common + /// un-hashed catalog (load-to-search), so an un-hashed store pays zero bytes here — unlike the + /// inline 16-byte Hash16 carried by every DirEntry today. + /// + public Hash16[] Hash { get; private set; } + + private EntryStore(int count) + { + Count = count; + ModifiedTicks = new long[count]; + Size = new long[count]; + Name = new string[count]; + BitFields = new byte[count]; + FirstChild = new int[count]; + NextSibling = new int[count]; + Parent = new int[count]; + } + + public Flags Flags(int i) => (Flags)BitFields[i]; + public bool IsDirectory(int i) => (Flags(i) & Entities.Flags.Directory) == Entities.Flags.Directory; + public bool IsHashDone(int i) => (Flags(i) & Entities.Flags.HashDone) == Entities.Flags.HashDone; + public bool IsPartialHash(int i) => (Flags(i) & Entities.Flags.PartialHash) == Entities.Flags.PartialHash; + public DateTime Modified(int i) => DateTime.FromBinary(ModifiedTicks[i]); + + /// Enumerate the direct child indices of via the sibling chain. + public IEnumerable Children(int parent) + { + for (var c = FirstChild[parent]; c != None; c = NextSibling[c]) + { + yield return c; + } + } + + /// + /// Build the full path of entry into the supplied StringBuilder by walking + /// parent indices (root-first). No intermediate per-node strings. Pass a reused builder on hot paths. + /// + public void AppendFullPath(StringBuilder sb, int i) + { + // Walk leaf -> root collecting indices, then emit root-first. + var depth = 0; + for (var cur = i; cur != None; cur = Parent[cur]) depth++; + if (depth == 0) return; + + Span chain = depth <= 64 ? stackalloc int[depth] : new int[depth]; + var n = 0; + for (var cur = i; cur != None; cur = Parent[cur]) chain[n++] = cur; + + for (var k = depth - 1; k >= 0; k--) + { + var idx = chain[k]; + if (sb.Length > 0) + { + var last = sb[^1]; + if (last != '\\' && last != '/') sb.Append(System.IO.Path.DirectorySeparatorChar); + } + sb.Append(Name[idx] ?? string.Empty); + } + } + + public string FullPath(int i) + { + var sb = new StringBuilder(128); + AppendFullPath(sb, i); + return sb.ToString(); + } + + /// + /// Convert a loaded/generated catalog tree into a store. The tree may be released afterwards. + /// Iterative (explicit stack) so very deep trees cannot overflow. + /// + public static EntryStore Build(RootEntry root) + { + ArgumentNullException.ThrowIfNull(root); + + var count = CountEntries(root); // robust: counts the actual tree, not (possibly stale) summary fields + var store = new EntryStore(count); + + var next = 0; + var rootIdx = next++; + store.Name[rootIdx] = root.Path; + store.ModifiedTicks[rootIdx] = root.ModifiedTicks; + store.Size[rootIdx] = root.Size; + store.BitFields[rootIdx] = (byte)root.BitFields; + store.Parent[rootIdx] = None; + store.FirstChild[rootIdx] = None; + store.NextSibling[rootIdx] = None; + + // Stack of (children-of-a-directory, that directory's index). + var stack = new Stack<(IList Children, int ParentIdx)>(); + stack.Push((root.Children, rootIdx)); + + while (stack.Count > 0) + { + var (children, parentIdx) = stack.Pop(); + if (children == null) continue; + + var prevSibling = None; + foreach (var child in children) + { + var idx = next++; + store.Name[idx] = child.Path; + store.ModifiedTicks[idx] = child.ModifiedTicks; + store.Size[idx] = child.Size; + store.BitFields[idx] = (byte)child.BitFields; + store.Parent[idx] = parentIdx; + store.FirstChild[idx] = None; + store.NextSibling[idx] = None; + if (child.IsHashDone) store.SetHash(idx, child.Hash); + + if (prevSibling == None) store.FirstChild[parentIdx] = idx; + else store.NextSibling[prevSibling] = idx; + prevSibling = idx; + + if (child.IsDirectory) stack.Push((child.Children, idx)); + } + } + + return store; + } + + /// Count every entry in the tree (including the root) via an explicit stack. + private static int CountEntries(RootEntry root) + { + var count = 1; // the root + var stack = new Stack>(); + stack.Push(root.Children); + while (stack.Count > 0) + { + var children = stack.Pop(); + if (children == null) continue; + count += children.Count; + foreach (var child in children) + { + if (child.IsDirectory) stack.Push(child.Children); + } + } + return count; + } + + private void SetHash(int i, Hash16 hash) + { + Hash ??= new Hash16[Count]; + Hash[i] = hash; + } +} diff --git a/src/cdeLib/Entities/Soa/EntryStoreSearch.cs b/src/cdeLib/Entities/Soa/EntryStoreSearch.cs new file mode 100644 index 0000000..a713608 --- /dev/null +++ b/src/cdeLib/Entities/Soa/EntryStoreSearch.cs @@ -0,0 +1,71 @@ +using System; +using System.Text; +using System.Text.RegularExpressions; + +namespace cdeLib.Entities.Soa; + +/// +/// Index-based find over an — a linear scan of the flat arrays (cache +/// friendly, no per-entry indirection). Mirrors the core matching of the pointer-tree +/// FindOptions (substring/regex, name/path, file/folder filter) so the two can be proven +/// equivalent before the production search is moved onto the store. +/// +public static class EntryStoreSearch +{ + /// Invoke with the index of every entry matching the query. + public static void Find( + EntryStore store, + string pattern, + bool regexMode, + bool includePath, + bool includeFiles, + bool includeFolders, + Action onMatch) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(onMatch); + + if (!includeFiles && !includeFolders) return; + + Regex regex = null; + if (regexMode && !string.IsNullOrEmpty(pattern)) + { + regex = new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.Compiled); + } + + var sb = includePath ? new StringBuilder(260) : null; + var hasPattern = !string.IsNullOrEmpty(pattern); + + for (var i = 1; i < store.Count; i++) // index 0 is the root, never a result + { + var isDir = store.IsDirectory(i); + if (isDir ? !includeFolders : !includeFiles) continue; + + if (!hasPattern) + { + onMatch(i); + continue; + } + + bool match; + if (includePath) + { + sb.Clear(); + store.AppendFullPath(sb, i); + var path = sb.ToString(); + match = regexMode + ? regex.IsMatch(path) + : path.Contains(pattern, StringComparison.OrdinalIgnoreCase); + } + else + { + var name = store.Name[i] ?? string.Empty; + match = regexMode + ? regex.IsMatch(name) + : name.Contains(pattern, StringComparison.OrdinalIgnoreCase); + } + + if (match) onMatch(i); + } + } +} diff --git a/src/cdeLibTest/Soa/EntryStoreTests.cs b/src/cdeLibTest/Soa/EntryStoreTests.cs new file mode 100644 index 0000000..00af269 --- /dev/null +++ b/src/cdeLibTest/Soa/EntryStoreTests.cs @@ -0,0 +1,138 @@ +using System.Collections.Generic; +using System.Linq; +using cdeLib; +using cdeLib.Entities; +using cdeLib.Entities.Soa; +using NUnit.Framework; + +namespace cdeLibTest.Soa; + +/// +/// Proves the SoA + produce results identical +/// to the production pointer-tree find. This is the safety gate for moving search onto the store. +/// +[TestFixture] +public class EntryStoreTests +{ + // Builds a small known catalog: + // C:\test + // ├─ dir1\ (dir) + // │ ├─ alpha.txt + // │ └─ beta.log + // ├─ docs\ (dir) + // │ └─ alpha.md + // └─ root_file.txt + private static RootEntry BuildTree() + { + var root = new RootEntry { Path = @"C:\test" }; + + var dir1 = new DirEntry(true) { Path = "dir1" }; + dir1.AddChild(new DirEntry(false) { Path = "alpha.txt" }); + dir1.AddChild(new DirEntry(false) { Path = "beta.log" }); + + var docs = new DirEntry(true) { Path = "docs" }; + docs.AddChild(new DirEntry(false) { Path = "alpha.md" }); + + root.AddChild(dir1); + root.AddChild(docs); + root.AddChild(new DirEntry(false) { Path = "root_file.txt" }); + + root.SetInMemoryFields(); + return root; + } + + private static List TreeFind(RootEntry root, string pattern, bool regex, bool path, + bool files, bool folders) + { + var found = new List(); + var options = new FindOptions + { + Pattern = pattern, + RegexMode = regex, + IncludePath = path, + IncludeFiles = files, + IncludeFolders = folders, + LimitResultCount = int.MaxValue, + VisitorFunc = (p, d) => { found.Add(p.MakeFullPath(d)); return true; }, + }; + options.Find(new[] { root }); + return found; + } + + private static List StoreFind(EntryStore store, string pattern, bool regex, bool path, + bool files, bool folders) + { + var found = new List(); + EntryStoreSearch.Find(store, pattern, regex, path, files, folders, i => found.Add(store.FullPath(i))); + return found; + } + + [Test] + public void Build_CountsEveryEntryIncludingRoot() + { + var root = BuildTree(); + var store = EntryStore.Build(root); + // 3 dirs/files under root + 2 files under dir1 + 1 under docs = 6, plus the root = 7. + Assert.That(store.Count, Is.EqualTo(7)); + } + + [Test] + public void FullPath_MatchesTreeForEveryEntry() + { + var root = BuildTree(); + var store = EntryStore.Build(root); + + // Every non-root entry's store full path must appear in the tree's full path set. + var treePaths = TreeFind(root, "", false, false, true, true).OrderBy(x => x).ToList(); + var storePaths = new List(); + for (var i = 1; i < store.Count; i++) storePaths.Add(store.FullPath(i)); + storePaths.Sort(); + + Assert.That(storePaths, Is.EqualTo(treePaths)); + } + + [TestCase("alpha", false, false)] // name substring + [TestCase("txt", false, false)] + [TestCase("xyzzy", false, false)] // no matches + [TestCase("", false, false)] // match all + [TestCase(@"dir1\alpha", false, true)] // path substring + [TestCase("docs", false, true)] + [TestCase(@"\.md$", true, false)] // regex on name + [TestCase(@"test\\d", true, true)] // regex on path + public void Search_MatchesTreeFind(string pattern, bool regex, bool path) + { + var root = BuildTree(); + var store = EntryStore.Build(root); + + var tree = TreeFind(root, pattern, regex, path, true, true).OrderBy(x => x).ToList(); + var soa = StoreFind(store, pattern, regex, path, true, true).OrderBy(x => x).ToList(); + + Assert.That(soa, Is.EqualTo(tree), $"pattern='{pattern}' regex={regex} path={path}"); + } + + [Test] + public void Search_FilesOnly_MatchesTreeFind() + { + var root = BuildTree(); + var store = EntryStore.Build(root); + + var tree = TreeFind(root, "", false, false, files: true, folders: false).OrderBy(x => x).ToList(); + var soa = StoreFind(store, "", false, false, files: true, folders: false).OrderBy(x => x).ToList(); + + Assert.That(soa, Is.EqualTo(tree)); + Assert.That(soa, Has.Count.EqualTo(4)); // alpha.txt, beta.log, alpha.md, root_file.txt + } + + [Test] + public void Search_FoldersOnly_MatchesTreeFind() + { + var root = BuildTree(); + var store = EntryStore.Build(root); + + var tree = TreeFind(root, "", false, false, files: false, folders: true).OrderBy(x => x).ToList(); + var soa = StoreFind(store, "", false, false, files: false, folders: true).OrderBy(x => x).ToList(); + + Assert.That(soa, Is.EqualTo(tree)); + Assert.That(soa, Has.Count.EqualTo(2)); // dir1, docs + } +} diff --git a/src/cdeMemProbe/EntryStore.cs b/src/cdeMemProbe/EntryStore.cs deleted file mode 100644 index 773afc0..0000000 --- a/src/cdeMemProbe/EntryStore.cs +++ /dev/null @@ -1,132 +0,0 @@ -using System; -using System.Collections.Generic; -using cdeLib.Entities; - -namespace cdeMemProbe; - -/// -/// PROTOTYPE struct-of-arrays representation of a single catalog, to measure the memory a SoA model -/// would use versus the production pointer-based tree. One slot per entry -/// across parallel arrays; the tree shape is encoded with int indices (firstChild / nextSibling / -/// parent) instead of object references — so there is NO per-entry object header and references -/// become 4-byte ints. -/// -/// This is a measurement/feasibility prototype, not the production model. It deliberately keeps -/// names in a string[] (one ref per entry, exactly like today) so the comparison isolates the -/// STRUCTURAL saving (object headers + link fields), not name storage. -/// -public sealed class EntryStore -{ - public const int None = -1; - - public readonly int Count; - - // One slot per entry. Index 0 is the root. - public readonly long[] ModifiedTicks; - public readonly long[] Size; - public readonly string[] Name; - public readonly byte[] BitFields; - public readonly int[] FirstChild; // None if no children - public readonly int[] NextSibling; // None if last sibling - public readonly int[] Parent; // None for the root - - // Hashes are allocated only when the catalog actually has them (the common load-to-search - // catalog has none) — so an un-hashed catalog pays zero bytes here, unlike the inline 16-byte - // Hash16 on every DirEntry today. - public Hash16[] Hash; - - public string RootPath; - - private int _next; - - private EntryStore(int count) - { - Count = count; - ModifiedTicks = new long[count]; - Size = new long[count]; - Name = new string[count]; - BitFields = new byte[count]; - FirstChild = new int[count]; - NextSibling = new int[count]; - Parent = new int[count]; - } - - public bool IsDirectory(int i) => ((Flags)BitFields[i] & Flags.Directory) == Flags.Directory; - - /// - /// Build a SoA store from a loaded/generated catalog tree. The tree can be released afterwards; - /// the store is self-contained. - /// - public static EntryStore Build(RootEntry root) - { - var count = checked((int)(root.FileEntryCount + root.DirEntryCount) + 1); // +1 for the root itself - var store = new EntryStore(count) { RootPath = root.Path }; - - var hashed = root.IsHashDone; // catalogs are hashed wholesale; cheap heuristic for the prototype - if (hashed) store.Hash = new Hash16[count]; - - var rootIdx = store._next++; - store.Name[rootIdx] = root.Path; - store.ModifiedTicks[rootIdx] = root.ModifiedTicks; - store.Size[rootIdx] = root.Size; - store.BitFields[rootIdx] = (byte)root.BitFields; - store.Parent[rootIdx] = None; - store.FirstChild[rootIdx] = None; - store.NextSibling[rootIdx] = None; - - store.FillChildren(root.Children, rootIdx); - return store; - } - - private void FillChildren(IList children, int parentIdx) - { - if (children == null) return; - - var prevSibling = None; - foreach (var child in children) - { - var idx = _next++; - Name[idx] = child.Path; - ModifiedTicks[idx] = child.ModifiedTicks; - Size[idx] = child.Size; - BitFields[idx] = (byte)child.BitFields; - Parent[idx] = parentIdx; - FirstChild[idx] = None; - NextSibling[idx] = None; - if (Hash != null) Hash[idx] = child.Hash; - - if (prevSibling == None) FirstChild[parentIdx] = idx; - else NextSibling[prevSibling] = idx; - prevSibling = idx; - - if (child.IsDirectory) FillChildren(child.Children, idx); - } - } - - /// Full path of an entry, walking parent indices into a reused buffer (no per-node strings). - public string FullPath(int i) - { - // Collect ancestors (leaf -> root) then write root-first. - var stack = new Stack(); - for (var cur = i; cur != None; cur = Parent[cur]) stack.Push(cur); - var sb = new System.Text.StringBuilder(128); - while (stack.Count > 0) - { - var idx = stack.Pop(); - if (sb.Length > 0 && sb[^1] != '\\' && sb[^1] != '/') sb.Append('\\'); - sb.Append(Name[idx]); - } - return sb.ToString(); - } - - /// Linear name-substring search over the flat arrays. Returns match count. - public int CountNameMatches(string pattern) - { - var matches = 0; - for (var i = 1; i < Count; i++) // skip root at 0 - { - if (Name[i].Contains(pattern, StringComparison.OrdinalIgnoreCase)) matches++; - } - return matches; - } -} diff --git a/src/cdeMemProbe/Program.cs b/src/cdeMemProbe/Program.cs index ab18118..a8c2f72 100644 --- a/src/cdeMemProbe/Program.cs +++ b/src/cdeMemProbe/Program.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using cdeLib.Catalog; using cdeLib.Entities; +using cdeLib.Entities.Soa; using Serilog; namespace cdeMemProbe; @@ -123,7 +124,9 @@ private static int MeasureSoa(string[] args) var bytesPerEntry = store.Count > 0 ? (double)managed / store.Count : 0; // Sanity-check the SoA actually works as a searchable structure. - var matches = store.CountNameMatches(sharedNames ? "x" : ".txt"); + var matches = 0; + EntryStoreSearch.Find(store, sharedNames ? "x" : ".txt", + regexMode: false, includePath: false, includeFiles: true, includeFolders: true, _ => matches++); Console.WriteLine("model,entries,managedBytes,bytesPerEntry"); Console.WriteLine(string.Join(',', "soa", From 75697368f1b2a6a648f8936be881383f4c742e42 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sat, 6 Jun 2026 23:02:58 +1000 Subject: [PATCH 13/43] refactor(soa): abstract ICommonEntry.Children to IReadOnlyList (migration P2) Re-type the interface Children from IList to a covariant, read-only IReadOnlyList so a struct-of-arrays backing (EntryStore via a coming EntryRef adapter) can satisfy ICommonEntry without materialising DirEntry objects. The concrete tree classes keep their mutable IList Children (used by build/sort/MessagePack) and expose the abstract view via explicit interface implementation - a List satisfies IReadOnlyList at runtime through interface covariance, so almost all consumers (Count, foreach, indexing, LINQ) compile unchanged. Only three internal tree ops that mutated/keyed children via the interface needed updates: SortAllChildrenByPath (cast to concrete for Sort) and the two copy-hash lookups (Dictionary keyed by ICommonEntry). Behaviour preserved: cdeLibTest 138 + cdeWinTest 32 passed; full solution builds clean. This unblocks the EntryRef adapter (next) that lets the GUI/dupes run on SoA. --- src/cdeLib/Entities/DirEntry.cs | 6 +++++- src/cdeLib/Entities/ICommonEntry.cs | 8 +++++++- src/cdeLib/Entities/RootEntry.cs | 15 ++++++++++----- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/src/cdeLib/Entities/DirEntry.cs b/src/cdeLib/Entities/DirEntry.cs index 7cb35f1..1423012 100644 --- a/src/cdeLib/Entities/DirEntry.cs +++ b/src/cdeLib/Entities/DirEntry.cs @@ -388,6 +388,10 @@ public IList Children } // ReSharper restore MemberCanBePrivate.Global + // Covariant read-only view for ICommonEntry consumers. The backing List satisfies + // IReadOnlyList at runtime via interface covariance. + IReadOnlyList ICommonEntry.Children => _extra?.Children as IReadOnlyList; + public void AddChild(DirEntry child) { var extra = EnsureExtra(); @@ -514,7 +518,7 @@ public void TraverseTreesCopyHash(ICommonEntry destination) if (baseSourceEntry.Children != null && baseDestinationEntry.Children != null) { // Build dictionary for O(1) lookups instead of O(n) linear search - var destinationLookup = new Dictionary( + var destinationLookup = new Dictionary( baseDestinationEntry.Children.Count, StringComparer.OrdinalIgnoreCase); diff --git a/src/cdeLib/Entities/ICommonEntry.cs b/src/cdeLib/Entities/ICommonEntry.cs index ac39697..16edb2a 100644 --- a/src/cdeLib/Entities/ICommonEntry.cs +++ b/src/cdeLib/Entities/ICommonEntry.cs @@ -13,7 +13,13 @@ public interface ICommonEntry bool IsDefaultSort { get; set; } int PathCompareWithDirTo(ICommonEntry de); - IList Children { get; } + /// + /// Read-only, covariant view of child entries. Abstract so a struct-of-arrays backing + /// (EntryStore via EntryRef) can satisfy it without materialising DirEntry objects. The + /// concrete tree classes still expose a mutable IList<DirEntry> for build/sort/ + /// serialization; this is what consumers see when they hold an . + /// + IReadOnlyList Children { get; } public void AddChild(DirEntry child); diff --git a/src/cdeLib/Entities/RootEntry.cs b/src/cdeLib/Entities/RootEntry.cs index 97b06fc..0130352 100644 --- a/src/cdeLib/Entities/RootEntry.cs +++ b/src/cdeLib/Entities/RootEntry.cs @@ -475,10 +475,12 @@ public void SortAllChildrenByPath() TraverseTreePair((_, d) => { - if (d.IsDirectory && d.Children?.Count > 1) + // Sorting mutates the concrete child list, so work through the concrete DirEntry + // (the abstract ICommonEntry.Children is a read-only view). + if (d is DirEntry { IsDirectory: true } de && de.Children?.Count > 1) { - d.Children.Sort((de1, de2) => de1.PathCompareWithDirTo(de2)); - d.IsDefaultSort = true; + de.Children.Sort((de1, de2) => de1.PathCompareWithDirTo(de2)); + de.IsDefaultSort = true; } return true; @@ -805,6 +807,9 @@ public RootEntry GetRootEntry() [Key(15)] public IList Children { get; set; } + // Covariant read-only view for ICommonEntry consumers (see DirEntry for rationale). + IReadOnlyList ICommonEntry.Children => Children as IReadOnlyList; + public void AddChild(DirEntry child) { if (this.Children == null) @@ -988,9 +993,9 @@ private static void ProcessChildrenForHashCopy( /// /// Builds a dictionary for O(1) lookups of destination children by path. /// - private static Dictionary BuildDestinationLookup(IList children) + private static Dictionary BuildDestinationLookup(IReadOnlyList children) { - var lookup = new Dictionary(children.Count, StringComparer.OrdinalIgnoreCase); + var lookup = new Dictionary(children.Count, StringComparer.OrdinalIgnoreCase); foreach (var child in children) { From a4488b134be1dd4f1bd9dd924bddd6c85d70545e Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sat, 6 Jun 2026 23:09:29 +1000 Subject: [PATCH 14/43] perf(soa): EntryRef adapter - run ICommonEntry consumers on SoA (migration P3) EntryRef presents one EntryStore entry (by index) as an ICommonEntry: read members map onto the store arrays; build/mutate members throw (a store is produced wholesale by the loader, not edited entry-by-entry). This is the payoff of the Children abstraction: the existing tree-oriented, ICommonEntry-based code now runs on the struct-of-arrays model unchanged. Tests prove EntryHelper.TraverseTreePair over an EntryRef root yields the same full paths as the tree, children/counts/GetListFromRoot/FullPath match, and mutators throw. (Gate child access on having children, not the directory flag - the root, like RootEntry, isn't flagged a directory but has children.) cdeLibTest 143 + cdeWinTest 32 passed; full solution builds clean. Intended for occasional access (display, result rows); bulk traversal still uses index-based EntryStoreSearch to avoid per-entry wrapper allocations. --- src/cdeLib/Entities/Soa/EntryRef.cs | 190 ++++++++++++++++++++++++++++ src/cdeLibTest/Soa/EntryRefTests.cs | 118 +++++++++++++++++ 2 files changed, 308 insertions(+) create mode 100644 src/cdeLib/Entities/Soa/EntryRef.cs create mode 100644 src/cdeLibTest/Soa/EntryRefTests.cs diff --git a/src/cdeLib/Entities/Soa/EntryRef.cs b/src/cdeLib/Entities/Soa/EntryRef.cs new file mode 100644 index 0000000..6f31f26 --- /dev/null +++ b/src/cdeLib/Entities/Soa/EntryRef.cs @@ -0,0 +1,190 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace cdeLib.Entities.Soa; + +/// +/// Lightweight adapter presenting a single entry (by index) as an +/// , so existing tree-oriented consumers (GUI display, navigation, +/// dupes read paths) can run on the struct-of-arrays model without materialising the whole +/// pointer tree. Read members map onto the store's arrays; build/mutate members throw, since a +/// store is produced wholesale by the loader, not edited entry-by-entry. +/// +/// Intended for OCCASIONAL access (displaying a directory, a search result row). Bulk traversal +/// should use index-based APIs () to avoid per-entry wrapper allocs. +/// +public sealed class EntryRef : ICommonEntry +{ + private readonly EntryStore _store; + private readonly int _index; + + public EntryRef(EntryStore store, int index) + { + _store = store; + _index = index; + } + + public EntryStore Store => _store; + public int Index => _index; + + private static NotSupportedException ReadOnly([System.Runtime.CompilerServices.CallerMemberName] string m = null) + => new($"EntryRef is a read-only view over EntryStore; '{m}' is not supported."); + + public string Path { get => _store.Name[_index]; set => throw ReadOnly(); } + public long Size { get => _store.Size[_index]; set => throw ReadOnly(); } + public DateTime Modified { get => _store.Modified(_index); set => throw ReadOnly(); } + + public bool IsDirectory { get => _store.IsDirectory(_index); set => throw ReadOnly(); } + public bool IsHashDone { get => _store.IsHashDone(_index); set => throw ReadOnly(); } + public bool IsPartialHash { get => _store.IsPartialHash(_index); set => throw ReadOnly(); } + public bool IsModifiedBad + { + get => (_store.Flags(_index) & Flags.ModifiedBad) == Flags.ModifiedBad; + set => throw ReadOnly(); + } + public bool IsReparsePoint + { + get => (_store.Flags(_index) & Flags.ReparsePoint) == Flags.ReparsePoint; + set => throw ReadOnly(); + } + public bool IsDefaultSort { get => true; set => throw ReadOnly(); } // store is built in sorted order + + public Hash16 Hash + { + get => _store.Hash != null ? _store.Hash[_index] : default; + set => throw ReadOnly(); + } + + public string FullPath => _store.FullPath(_index); + + public bool PathProblem + { + get + { + for (var cur = _index; cur != EntryStore.None; cur = _store.Parent[cur]) + { + var name = _store.Name[cur]; + if (!string.IsNullOrEmpty(name) && (name.EndsWith(' ') || name.EndsWith('.'))) return true; + } + return false; + } + } + + public IReadOnlyList Children + { + get + { + // Gate on having children, not on the directory flag: the root is not flagged a + // directory yet has children (matching RootEntry), and a file simply has none. + if (_store.FirstChild[_index] == EntryStore.None) return null; + List list = null; + foreach (var c in _store.Children(_index)) + { + (list ??= new List()).Add(new EntryRef(_store, c)); + } + return list; + } + } + + public ICommonEntry ParentCommonEntry + { + get + { + var p = _store.Parent[_index]; + return p == EntryStore.None ? null : new EntryRef(_store, p); + } + set => throw ReadOnly(); + } + + public uint FileEntryCount { get => CountSubtree().Files; set => throw ReadOnly(); } + public uint DirEntryCount { get => CountSubtree().Dirs; set => throw ReadOnly(); } + + private (uint Files, uint Dirs) CountSubtree() + { + uint files = 0, dirs = 0; + var stack = new Stack(); + stack.Push(_index); + while (stack.Count > 0) + { + var n = stack.Pop(); + foreach (var c in _store.Children(n)) + { + if (_store.IsDirectory(c)) { dirs++; stack.Push(c); } + else files++; + } + } + return (files, dirs); + } + + public int PathCompareWithDirTo(ICommonEntry de) + { + if (de == null) return -1; + if (IsDirectory && !de.IsDirectory) return -1; + if (!IsDirectory && de.IsDirectory) return 1; + return string.Compare(Path, de.Path, StringComparison.OrdinalIgnoreCase); + } + + public int SizeCompareWithDirTo(ICommonEntry de) + { + if (de == null) return -1; + if (IsDirectory && !de.IsDirectory) return -1; + if (!IsDirectory && de.IsDirectory) return 1; + var c = Size.CompareTo(de.Size); + return c != 0 ? c : string.Compare(Path, de.Path, StringComparison.OrdinalIgnoreCase); + } + + public int ModifiedCompareTo(ICommonEntry de) + { + if (de == null) return -1; + if (IsModifiedBad && !de.IsModifiedBad) return -1; + if (!IsModifiedBad && de.IsModifiedBad) return 1; + if (IsModifiedBad && de.IsModifiedBad) return 0; + return DateTime.Compare(Modified, de.Modified); + } + + public string MakeFullPath(ICommonEntry dirEntry) + { + var parent = FullPath; + var name = dirEntry?.Path ?? "dnull"; + if (parent.Length > 0 && parent[^1] != '\\' && parent[^1] != '/') + return string.Concat(parent, System.IO.Path.DirectorySeparatorChar.ToString(), name); + return string.Concat(parent, name); + } + + public IList GetListFromRoot() + { + var list = new List(8); + for (var cur = _index; cur != EntryStore.None; cur = _store.Parent[cur]) + { + list.Add(new EntryRef(_store, cur)); + } + list.Reverse(); + return list; + } + + public bool ExistsOnFileSystem() => Directory.Exists(FullPath); + + public void TraverseTreePair(TraverseFunc func) + { + if (func == null) return; + var stack = new Stack(); + stack.Push(_index); + while (stack.Count > 0) + { + var n = stack.Pop(); + var parentRef = new EntryRef(_store, n); + foreach (var c in _store.Children(n)) + { + if (!func(parentRef, new EntryRef(_store, c))) return; + if (_store.IsDirectory(c)) stack.Push(c); + } + } + } + + // ----- build / mutate members: not supported on a read-only store view ----- + public void AddChild(DirEntry child) => throw ReadOnly(); + public void SetSummaryFields() => throw ReadOnly(); + public void SetHash(byte[] hashResponseHash) => throw ReadOnly(); + public void TraverseTreesCopyHash(ICommonEntry destination) => throw ReadOnly(); +} diff --git a/src/cdeLibTest/Soa/EntryRefTests.cs b/src/cdeLibTest/Soa/EntryRefTests.cs new file mode 100644 index 0000000..7747a37 --- /dev/null +++ b/src/cdeLibTest/Soa/EntryRefTests.cs @@ -0,0 +1,118 @@ +using System.Collections.Generic; +using System.Linq; +using cdeLib.Entities; +using cdeLib.Entities.Soa; +using NUnit.Framework; + +namespace cdeLibTest.Soa; + +/// +/// Proves the adapter lets the existing tree-oriented, ICommonEntry-based +/// code run unchanged on the struct-of-arrays — same traversal results, +/// same paths, same counts as the pointer tree. +/// +[TestFixture] +public class EntryRefTests +{ + private static RootEntry BuildTree() + { + var root = new RootEntry { Path = @"C:\test" }; + + var dir1 = new DirEntry(true) { Path = "dir1" }; + dir1.AddChild(new DirEntry(false) { Path = "alpha.txt", Size = 10 }); + dir1.AddChild(new DirEntry(false) { Path = "beta.log", Size = 20 }); + + var docs = new DirEntry(true) { Path = "docs" }; + docs.AddChild(new DirEntry(false) { Path = "alpha.md", Size = 30 }); + + root.AddChild(dir1); + root.AddChild(docs); + root.AddChild(new DirEntry(false) { Path = "root_file.txt", Size = 40 }); + + root.SetInMemoryFields(); + return root; + } + + private static List TraverseFullPaths(ICommonEntry root) + { + var paths = new List(); + EntryHelper.TraverseTreePair(root, (_, child) => { paths.Add(child.FullPath); return true; }); + paths.Sort(); + return paths; + } + + [Test] + public void TraverseTreePair_OverEntryRef_MatchesTreeTraversal() + { + var root = BuildTree(); + var store = EntryStore.Build(root); + var storeRoot = new EntryRef(store, 0); + + Assert.That(TraverseFullPaths(storeRoot), Is.EqualTo(TraverseFullPaths(root))); + } + + [Test] + public void Children_OverEntryRef_MatchTreeChildren() + { + var root = BuildTree(); + var store = EntryStore.Build(root); + var storeRoot = new EntryRef(store, 0); + + var treeChildNames = root.Children.Select(c => c.Path).OrderBy(x => x).ToList(); + var storeChildNames = storeRoot.Children.Select(c => c.Path).OrderBy(x => x).ToList(); + Assert.That(storeChildNames, Is.EqualTo(treeChildNames)); + + // A file has no children view. + var rootFile = storeRoot.Children.First(c => c.Path == "root_file.txt"); + Assert.That(rootFile.IsDirectory, Is.False); + Assert.That(rootFile.Children, Is.Null); + } + + [Test] + public void Counts_OverEntryRef_MatchTree() + { + var root = BuildTree(); + var store = EntryStore.Build(root); + var storeRoot = new EntryRef(store, 0); + + Assert.That(storeRoot.FileEntryCount, Is.EqualTo(root.FileEntryCount)); + Assert.That(storeRoot.DirEntryCount, Is.EqualTo(root.DirEntryCount)); + Assert.That(storeRoot.FileEntryCount, Is.EqualTo(4u)); + Assert.That(storeRoot.DirEntryCount, Is.EqualTo(2u)); + } + + [Test] + public void GetListFromRoot_OverEntryRef_GoesRootToLeaf() + { + var root = BuildTree(); + var store = EntryStore.Build(root); + var storeRoot = new EntryRef(store, 0); + + // Find alpha.txt under dir1 and walk back to root. + var alpha = TraverseFind(store, "alpha.txt"); + var chain = alpha.GetListFromRoot().Select(e => e.Path).ToList(); + Assert.That(chain, Is.EqualTo(new[] { @"C:\test", "dir1", "alpha.txt" })); + Assert.That(alpha.FullPath, Is.EqualTo(@"C:\test\dir1\alpha.txt")); + } + + [Test] + public void MutatingMembers_Throw() + { + var root = BuildTree(); + var store = EntryStore.Build(root); + var e = new EntryRef(store, 1); + + Assert.Throws(() => e.Path = "x"); + Assert.Throws(() => e.AddChild(new DirEntry(false))); + Assert.Throws(() => e.SetSummaryFields()); + } + + private static EntryRef TraverseFind(EntryStore store, string name) + { + for (var i = 1; i < store.Count; i++) + { + if (store.Name[i] == name) return new EntryRef(store, i); + } + return null; + } +} From b2c3375e243a5cf30a4bf15cb9a08b228a9bee79 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sat, 6 Jun 2026 23:25:23 +1000 Subject: [PATCH 15/43] perf(soa): wire CLI find onto EntryStore + reuse interned names (migration P4) Realizes the SoA memory win in an actual command. FindService (CLI find) now converts each loaded catalog to an EntryStore, releases the pointer tree, and searches the store via the index-based EntryStoreSearch. Verified end-to-end: `cde find` returns correct results incl. deep nested full paths. Critical fix found by measurement: the converter must REUSE the tree''s interned name strings, not rebuild them. DirEntry now holds its extension in a named _ext field and exposes NamePart/ExtPart; EntryStore stores names split (Name[]+Ext[]) pointing at the SAME interned objects, rejoining via FullName(i) and appending parts directly in path building (no full-name allocation). Without this the store allocated 1M fresh name strings on top of the pool-retained originals and footprint went UP. Measured on a real 1M-entry .cde (cdeMemProbe --store, synchronous load so the tree is fully released): tree 162.2 -> store 104.9 bytes/entry, -35%. (Earlier store numbers were polluted by an async state machine retaining the tree.) cdeLibTest 143 + cdeWinTest 32 passed; full solution builds clean. --- src/cdeLib/Entities/DirEntry.cs | 20 ++++++--- src/cdeLib/Entities/Soa/EntryRef.cs | 2 +- src/cdeLib/Entities/Soa/EntryStore.cs | 18 +++++++- src/cdeLib/Entities/Soa/EntryStoreSearch.cs | 2 +- src/cdeLib/FindService.cs | 43 ++++++++++--------- src/cdeLibTest/Soa/EntryRefTests.cs | 2 +- src/cdeMemProbe/Program.cs | 46 +++++++++++++++------ 7 files changed, 91 insertions(+), 42 deletions(-) diff --git a/src/cdeLib/Entities/DirEntry.cs b/src/cdeLib/Entities/DirEntry.cs index 1423012..c3c6b92 100644 --- a/src/cdeLib/Entities/DirEntry.cs +++ b/src/cdeLib/Entities/DirEntry.cs @@ -18,6 +18,16 @@ public sealed class DirEntry : ICommonEntry { private string _path; + // Extension (including the dot), interned and split from the name for memory efficiency. + // Null when the name has no extension. + private string _ext; + + /// Interned name without extension (the _path part). For SoA conversion reuse. + internal string NamePart => _path; + + /// Interned extension including the dot, or null. For SoA conversion reuse. + internal string ExtPart => _ext; + /// /// Side-object holding directory-only state — the child list and the rolled-up summary counts. /// Null on every file (the vast majority of entries), so a file no longer carries an always-null @@ -418,9 +428,9 @@ public string Path get { //return _path; - + // string.concat faster than string interpolation. - return string.IsNullOrEmpty(field) ? _path : string.Concat(_path, field); + return string.IsNullOrEmpty(_ext) ? _path : string.Concat(_path, _ext); } set { @@ -428,7 +438,7 @@ public string Path if (string.IsNullOrEmpty(value)) { _path = string.Intern(string.Empty); - field = null; + _ext = null; return; } @@ -438,13 +448,13 @@ public string Path if (lastDot > 0 && lastDot > valueSpan.LastIndexOfAny(PathSeparators)) { // Span slicing is zero-cost, allocate strings only for Intern - field = string.Intern(new string(valueSpan[lastDot..])); + _ext = string.Intern(new string(valueSpan[lastDot..])); _path = string.Intern(new string(valueSpan[..lastDot])); } else { _path = string.Intern(value); - field = null; + _ext = null; } // Simpler code but slightly less performance: diff --git a/src/cdeLib/Entities/Soa/EntryRef.cs b/src/cdeLib/Entities/Soa/EntryRef.cs index 6f31f26..254b2de 100644 --- a/src/cdeLib/Entities/Soa/EntryRef.cs +++ b/src/cdeLib/Entities/Soa/EntryRef.cs @@ -31,7 +31,7 @@ public EntryRef(EntryStore store, int index) private static NotSupportedException ReadOnly([System.Runtime.CompilerServices.CallerMemberName] string m = null) => new($"EntryRef is a read-only view over EntryStore; '{m}' is not supported."); - public string Path { get => _store.Name[_index]; set => throw ReadOnly(); } + public string Path { get => _store.FullName(_index); set => throw ReadOnly(); } public long Size { get => _store.Size[_index]; set => throw ReadOnly(); } public DateTime Modified { get => _store.Modified(_index); set => throw ReadOnly(); } diff --git a/src/cdeLib/Entities/Soa/EntryStore.cs b/src/cdeLib/Entities/Soa/EntryStore.cs index 6ccc7a1..c840290 100644 --- a/src/cdeLib/Entities/Soa/EntryStore.cs +++ b/src/cdeLib/Entities/Soa/EntryStore.cs @@ -27,7 +27,13 @@ public sealed class EntryStore // One slot per entry (index 0 = root). public long[] ModifiedTicks { get; private set; } public long[] Size { get; private set; } + + // Names are stored split (name-without-extension + extension) reusing the SAME interned string + // objects the source tree held — so conversion allocates no new name strings and the interned + // originals are shared, not duplicated. FullName(i) rejoins on demand. public string[] Name { get; private set; } + public string[] Ext { get; private set; } + public byte[] BitFields { get; private set; } public int[] FirstChild { get; private set; } public int[] NextSibling { get; private set; } @@ -46,12 +52,16 @@ private EntryStore(int count) ModifiedTicks = new long[count]; Size = new long[count]; Name = new string[count]; + Ext = new string[count]; BitFields = new byte[count]; FirstChild = new int[count]; NextSibling = new int[count]; Parent = new int[count]; } + /// Full entry name (name + extension), rejoined on demand like DirEntry.Path. + public string FullName(int i) => string.IsNullOrEmpty(Ext[i]) ? Name[i] : string.Concat(Name[i], Ext[i]); + public Flags Flags(int i) => (Flags)BitFields[i]; public bool IsDirectory(int i) => (Flags(i) & Entities.Flags.Directory) == Entities.Flags.Directory; public bool IsHashDone(int i) => (Flags(i) & Entities.Flags.HashDone) == Entities.Flags.HashDone; @@ -90,7 +100,9 @@ public void AppendFullPath(StringBuilder sb, int i) var last = sb[^1]; if (last != '\\' && last != '/') sb.Append(System.IO.Path.DirectorySeparatorChar); } + // Append the split name parts directly — no full-name string allocation for path building. sb.Append(Name[idx] ?? string.Empty); + if (!string.IsNullOrEmpty(Ext[idx])) sb.Append(Ext[idx]); } } @@ -114,7 +126,7 @@ public static EntryStore Build(RootEntry root) var next = 0; var rootIdx = next++; - store.Name[rootIdx] = root.Path; + store.Name[rootIdx] = root.Path; // root path is not split store.ModifiedTicks[rootIdx] = root.ModifiedTicks; store.Size[rootIdx] = root.Size; store.BitFields[rootIdx] = (byte)root.BitFields; @@ -135,7 +147,9 @@ public static EntryStore Build(RootEntry root) foreach (var child in children) { var idx = next++; - store.Name[idx] = child.Path; + // Reuse the child's already-interned name + extension objects (no fresh allocation). + store.Name[idx] = child.NamePart; + store.Ext[idx] = child.ExtPart; store.ModifiedTicks[idx] = child.ModifiedTicks; store.Size[idx] = child.Size; store.BitFields[idx] = (byte)child.BitFields; diff --git a/src/cdeLib/Entities/Soa/EntryStoreSearch.cs b/src/cdeLib/Entities/Soa/EntryStoreSearch.cs index a713608..4f57c30 100644 --- a/src/cdeLib/Entities/Soa/EntryStoreSearch.cs +++ b/src/cdeLib/Entities/Soa/EntryStoreSearch.cs @@ -59,7 +59,7 @@ public static void Find( } else { - var name = store.Name[i] ?? string.Empty; + var name = store.FullName(i); match = regexMode ? regex.IsMatch(name) : name.Contains(pattern, StringComparison.OrdinalIgnoreCase); diff --git a/src/cdeLib/FindService.cs b/src/cdeLib/FindService.cs index ec355df..176cde0 100644 --- a/src/cdeLib/FindService.cs +++ b/src/cdeLib/FindService.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Threading.Tasks; using cdeLib.Entities; +using cdeLib.Entities.Soa; using Serilog; namespace cdeLib; @@ -44,29 +46,30 @@ public void Find(string pattern, string param, IList rootEntries) public void Find(string pattern, bool regexMode, bool includePath, IList rootEntries) { - // Use the synchronous traversal: it is dramatically faster than the work-stealing async - // path, which ran every entry through an async Task state machine plus per-entry - // Task.Yield/Task.Delay. Measured on a 1M-entry catalog: ~49x faster for name search and - // ~7x for path search (see src/cdeBenchmarks/baseline/search-baseline.md). + // Convert each loaded catalog to the struct-of-arrays EntryStore and release its pointer + // tree before searching. The store holds the same catalog in ~1/3 the structural memory + // (42 vs 129 bytes/entry; see src/cdeBenchmarks/baseline/soa-prototype.md), and the + // index-based scan is cache friendly. The CLI find applies only pattern + name/path + + // file/folder filtering, all of which EntryStoreSearch supports. + var stores = new List(rootEntries.Count); + for (var i = 0; i < rootEntries.Count; i++) + { + if (rootEntries[i] != null) stores.Add(EntryStore.Build(rootEntries[i])); + rootEntries[i] = null; // drop the tree so it can be collected while we search the stores + } + var totalFound = 0L; - var findOptions = new FindOptions + var timer = Stopwatch.StartNew(); + foreach (var store in stores) { - Pattern = pattern, - RegexMode = regexMode, - IncludePath = includePath, - IncludeFiles = IncludeFiles, - IncludeFolders = IncludeFolders, - LimitResultCount = int.MaxValue, - VisitorFunc = (p, d) => - { - ++totalFound; - Console.WriteLine(" {0}", p.MakeFullPath(d)); - return true; - }, - }; + EntryStoreSearch.Find(store, pattern, regexMode, includePath, IncludeFiles, IncludeFolders, + idx => + { + ++totalFound; + Console.WriteLine(" {0}", store.FullPath(idx)); + }); + } - var timer = System.Diagnostics.Stopwatch.StartNew(); - findOptions.Find(rootEntries); timer.Stop(); Log.Logger.Information( "Search Execution Time: {ExecutionTime}, Matching pattern {Pattern}, Total found {TotalFound}", diff --git a/src/cdeLibTest/Soa/EntryRefTests.cs b/src/cdeLibTest/Soa/EntryRefTests.cs index 7747a37..53210f2 100644 --- a/src/cdeLibTest/Soa/EntryRefTests.cs +++ b/src/cdeLibTest/Soa/EntryRefTests.cs @@ -111,7 +111,7 @@ private static EntryRef TraverseFind(EntryStore store, string name) { for (var i = 1; i < store.Count; i++) { - if (store.Name[i] == name) return new EntryRef(store, i); + if (store.FullName(i) == name) return new EntryRef(store, i); } return null; } diff --git a/src/cdeMemProbe/Program.cs b/src/cdeMemProbe/Program.cs index a8c2f72..2ae6145 100644 --- a/src/cdeMemProbe/Program.cs +++ b/src/cdeMemProbe/Program.cs @@ -56,7 +56,8 @@ public static async Task Main(string[] args) return await GenerateAsync(args, genValue, logger); } - return await MeasureAsync(args[0], !HasFlag(args, "--no-header", out _), logger); + return await MeasureAsync(args[0], !HasFlag(args, "--no-header", out _), + asStore: HasFlag(args, "--store", out _), logger); } private static async Task GenerateAsync(string[] args, string? countArg, ILogger logger) @@ -146,7 +147,7 @@ private static EntryStore BuildStoreReleasingTree(int count, bool withHashes, bo return EntryStore.Build(tree); } - private static async Task MeasureAsync(string file, bool printHeader, ILogger logger) + private static async Task MeasureAsync(string file, bool printHeader, bool asStore, ILogger logger) { if (!File.Exists(file)) { @@ -155,21 +156,20 @@ private static async Task MeasureAsync(string file, bool printHeader, ILogg } var sw = Stopwatch.StartNew(); - RootEntry root; - using (var repo = new CatalogRepository(logger)) - { - root = await repo.LoadDirCacheAsync(file); - } + // Load (and for --store, convert to the SoA store) inside a synchronous helper so the source + // tree is a plain local that goes fully out of scope before we measure. (An async helper would + // capture the tree in its state machine and the store measurement would double-count it.) + var (measured, entries) = asStore + ? LoadAsStore(file, logger) + : LoadAsTree(file, logger); sw.Stop(); - if (root == null) + if (measured == null) { Console.Error.WriteLine($"failed to load catalog: {file}"); return 1; } - var entries = root.FileEntryCount + root.DirEntryCount; - // Settle the GC so GetTotalMemory reflects retained (live) objects, not transient load garbage. GC.Collect(); GC.WaitForPendingFinalizers(); @@ -180,8 +180,8 @@ private static async Task MeasureAsync(string file, bool printHeader, ILogg var peakWorkingSet = proc.PeakWorkingSet64; var privateBytes = proc.PrivateMemorySize64; - // Keep the tree alive across the measurement so it counts toward the live heap. - GC.KeepAlive(root); + // Keep the measured object (tree or store) alive across the measurement. + GC.KeepAlive(measured); var bytesPerEntry = entries > 0 ? (double)managedBytes / entries : 0; @@ -201,6 +201,28 @@ private static async Task MeasureAsync(string file, bool printHeader, ILogg return 0; } + [MethodImpl(MethodImplOptions.NoInlining)] + private static (object Measured, long Entries) LoadAsTree(string file, ILogger logger) + { + using var repo = new CatalogRepository(logger); + var root = repo.LoadDirCache(file); + if (root == null) return (null, 0); + return (root, root.FileEntryCount + root.DirEntryCount); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static (object Measured, long Entries) LoadAsStore(string file, ILogger logger) + { + using var repo = new CatalogRepository(logger); + var root = repo.LoadDirCache(file); + if (root == null) return (null, 0); + var entries = root.FileEntryCount + root.DirEntryCount; + var store = EntryStore.Build(root); + // root is a plain local; once this returns it is unreferenced and collectable, leaving only + // the store (which reuses the tree's interned name strings) for the caller to measure. + return (store, entries); + } + /// /// Returns true if is present. If the next token is not another flag it /// is returned as (so both --generate 100 and bare flags work). From d00a01a1f52f4c55c966e1c98595df03252f0c72 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sat, 6 Jun 2026 23:31:07 +1000 Subject: [PATCH 16/43] perf(soa): EntryStore carries catalog metadata (GUI wiring foundation) The cdeWin catalog list and search-result rows render root-level metadata (volume, default/actual .cde name, drive hint, avail/total space, scan dates, description, counts, size). EntryStore now captures all of it at conversion so a store is a complete, self-describing catalog - the prerequisite for holding catalogs as stores in the GUI. cdeLibTest 18 SoA tests passed (incl. metadata round-trip). --- src/cdeLib/Entities/Soa/EntryStore.cs | 32 +++++++++++++++++++++++- src/cdeLibTest/Soa/EntryStoreTests.cs | 35 +++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/src/cdeLib/Entities/Soa/EntryStore.cs b/src/cdeLib/Entities/Soa/EntryStore.cs index c840290..35e2d76 100644 --- a/src/cdeLib/Entities/Soa/EntryStore.cs +++ b/src/cdeLib/Entities/Soa/EntryStore.cs @@ -46,6 +46,21 @@ public sealed class EntryStore /// public Hash16[] Hash { get; private set; } + // ----- catalog-level metadata (what the GUI catalog list and search-result rows display) ----- + public string RootPath; // root path, e.g. C:\ (also Name[0]) + public string VolumeName; + public string DefaultFileName; // generated .cde name + public string ActualFileName; // path of the loaded .cde + public string DriveLetterHint; + public string Description; + public long AvailSpace; + public long TotalSpace; + public long ScanStartUtcTicks; + public long ScanEndUtcTicks; + public long RootSize; // total size of the catalog + public uint RootFileEntryCount; // total files in the catalog + public uint RootDirEntryCount; // total directories in the catalog + private EntryStore(int count) { Count = count; @@ -122,7 +137,22 @@ public static EntryStore Build(RootEntry root) ArgumentNullException.ThrowIfNull(root); var count = CountEntries(root); // robust: counts the actual tree, not (possibly stale) summary fields - var store = new EntryStore(count); + var store = new EntryStore(count) + { + RootPath = root.Path, + VolumeName = root.VolumeName, + DefaultFileName = root.DefaultFileName, + ActualFileName = root.ActualFileName, + DriveLetterHint = root.DriveLetterHint, + Description = root.Description, + AvailSpace = root.AvailSpace, + TotalSpace = root.TotalSpace, + ScanStartUtcTicks = root.ScanStartUtcTicks, + ScanEndUtcTicks = root.ScanEndUtcTicks, + RootSize = root.Size, + RootFileEntryCount = root.FileEntryCount, + RootDirEntryCount = root.DirEntryCount, + }; var next = 0; var rootIdx = next++; diff --git a/src/cdeLibTest/Soa/EntryStoreTests.cs b/src/cdeLibTest/Soa/EntryStoreTests.cs index 00af269..6a51cc4 100644 --- a/src/cdeLibTest/Soa/EntryStoreTests.cs +++ b/src/cdeLibTest/Soa/EntryStoreTests.cs @@ -76,6 +76,41 @@ public void Build_CountsEveryEntryIncludingRoot() Assert.That(store.Count, Is.EqualTo(7)); } + [Test] + public void Build_CapturesCatalogMetadata() + { + var root = new RootEntry + { + Path = @"D:\", + VolumeName = "DATA", + DefaultFileName = "D-DATA.cde", + ActualFileName = @"C:\cat\D-DATA.cde", + DriveLetterHint = "D", + Description = "data drive", + AvailSpace = 111, + TotalSpace = 222, + }; + root.AddChild(new DirEntry(false) { Path = "f.txt", Size = 7 }); + root.SetInMemoryFields(); + + var store = EntryStore.Build(root); + + Assert.Multiple(() => + { + Assert.That(store.RootPath, Is.EqualTo(@"D:\")); + Assert.That(store.VolumeName, Is.EqualTo("DATA")); + Assert.That(store.DefaultFileName, Is.EqualTo("D-DATA.cde")); + Assert.That(store.ActualFileName, Is.EqualTo(@"C:\cat\D-DATA.cde")); + Assert.That(store.DriveLetterHint, Is.EqualTo("D")); + Assert.That(store.Description, Is.EqualTo("data drive")); + Assert.That(store.AvailSpace, Is.EqualTo(111)); + Assert.That(store.TotalSpace, Is.EqualTo(222)); + Assert.That(store.RootFileEntryCount, Is.EqualTo(root.FileEntryCount)); + Assert.That(store.RootDirEntryCount, Is.EqualTo(root.DirEntryCount)); + Assert.That(store.RootSize, Is.EqualTo(root.Size)); + }); + } + [Test] public void FullPath_MatchesTreeForEveryEntry() { From cf8ccb7d84a24b5034bb47d63167f5662a300f1c Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sat, 6 Jun 2026 23:43:48 +1000 Subject: [PATCH 17/43] perf(soa): filter-capable EntryStoreSearch for GUI parity (cdeWin wiring foundation) Add EntryStoreFindOptions (pattern + name/path + file/folder + size/date/hour/ not-older-than ranges) and an EntryStoreSearch.Find overload that evaluates them directly against the store arrays - matching the filter set the cdeWin GUI search applies (which the CLI find did not need). Tested. This is the search foundation the GUI store-wiring requires; the presenter/form rewiring itself is the remaining step. --- .../Entities/Soa/EntryStoreFindOptions.cs | 35 +++++++++++ src/cdeLib/Entities/Soa/EntryStoreSearch.cs | 61 +++++++++++++++++++ src/cdeLibTest/Soa/EntryStoreTests.cs | 19 ++++++ 3 files changed, 115 insertions(+) create mode 100644 src/cdeLib/Entities/Soa/EntryStoreFindOptions.cs diff --git a/src/cdeLib/Entities/Soa/EntryStoreFindOptions.cs b/src/cdeLib/Entities/Soa/EntryStoreFindOptions.cs new file mode 100644 index 0000000..5dd3112 --- /dev/null +++ b/src/cdeLib/Entities/Soa/EntryStoreFindOptions.cs @@ -0,0 +1,35 @@ +using System; + +namespace cdeLib.Entities.Soa; + +/// +/// Filter parameters for , mirroring the subset of the tree-based +/// FindOptions that the cdeWin GUI search uses (pattern + name/path + file/folder + size/date/hour +/// ranges). Evaluated directly against the store's Size[] and ModifiedTicks[] arrays. +/// +public sealed class EntryStoreFindOptions +{ + public string Pattern { get; set; } + public bool RegexMode { get; set; } + public bool IncludePath { get; set; } + public bool IncludeFiles { get; set; } = true; + public bool IncludeFolders { get; set; } = true; + + public bool FromSizeEnable { get; set; } + public long FromSize { get; set; } + public bool ToSizeEnable { get; set; } + public long ToSize { get; set; } + + public bool FromDateEnable { get; set; } + public DateTime FromDate { get; set; } + public bool ToDateEnable { get; set; } + public DateTime ToDate { get; set; } + + public bool FromHourEnable { get; set; } + public TimeSpan FromHour { get; set; } + public bool ToHourEnable { get; set; } + public TimeSpan ToHour { get; set; } + + public bool NotOlderThanEnable { get; set; } + public DateTime NotOlderThan { get; set; } +} diff --git a/src/cdeLib/Entities/Soa/EntryStoreSearch.cs b/src/cdeLib/Entities/Soa/EntryStoreSearch.cs index 4f57c30..7502a27 100644 --- a/src/cdeLib/Entities/Soa/EntryStoreSearch.cs +++ b/src/cdeLib/Entities/Soa/EntryStoreSearch.cs @@ -12,6 +12,67 @@ namespace cdeLib.Entities.Soa; /// public static class EntryStoreSearch { + /// + /// Full-filter search (pattern + name/path + file/folder + size/date/hour ranges), mirroring the + /// cdeWin GUI search, evaluated directly against the store arrays. + /// + public static void Find(EntryStore store, EntryStoreFindOptions o, Action onMatch) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(o); + if (!o.IncludeFiles && !o.IncludeFolders) return; + + Regex regex = null; + if (o.RegexMode && !string.IsNullOrEmpty(o.Pattern)) + regex = new Regex(o.Pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.Compiled); + + var sb = o.IncludePath ? new StringBuilder(260) : null; + var hasPattern = !string.IsNullOrEmpty(o.Pattern); + + for (var i = 1; i < store.Count; i++) + { + var isDir = store.IsDirectory(i); + if (isDir ? !o.IncludeFolders : !o.IncludeFiles) continue; + + var size = store.Size[i]; + if (o.FromSizeEnable && size < o.FromSize) continue; + if (o.ToSizeEnable && size > o.ToSize) continue; + + if (o.FromDateEnable || o.ToDateEnable || o.FromHourEnable || o.ToHourEnable || o.NotOlderThanEnable) + { + var modified = store.Modified(i); + if (o.FromDateEnable && modified < o.FromDate) continue; + if (o.ToDateEnable && modified > o.ToDate) continue; + if (o.NotOlderThanEnable && modified < o.NotOlderThan) continue; + var tod = modified.TimeOfDay; + if (o.FromHourEnable && tod < o.FromHour) continue; + if (o.ToHourEnable && tod > o.ToHour) continue; + } + + if (!hasPattern) { onMatch(i); continue; } + + bool match; + if (o.IncludePath) + { + sb.Clear(); + store.AppendFullPath(sb, i); + var path = sb.ToString(); + match = o.RegexMode + ? regex.IsMatch(path) + : path.Contains(o.Pattern, StringComparison.OrdinalIgnoreCase); + } + else + { + var name = store.FullName(i); + match = o.RegexMode + ? regex.IsMatch(name) + : name.Contains(o.Pattern, StringComparison.OrdinalIgnoreCase); + } + + if (match) onMatch(i); + } + } + /// Invoke with the index of every entry matching the query. public static void Find( EntryStore store, diff --git a/src/cdeLibTest/Soa/EntryStoreTests.cs b/src/cdeLibTest/Soa/EntryStoreTests.cs index 6a51cc4..c27ef7f 100644 --- a/src/cdeLibTest/Soa/EntryStoreTests.cs +++ b/src/cdeLibTest/Soa/EntryStoreTests.cs @@ -145,6 +145,25 @@ public void Search_MatchesTreeFind(string pattern, bool regex, bool path) Assert.That(soa, Is.EqualTo(tree), $"pattern='{pattern}' regex={regex} path={path}"); } + [Test] + public void Search_WithSizeFilter_FiltersBySize() + { + var root = new RootEntry { Path = @"C:\s" }; + root.AddChild(new DirEntry(false) { Path = "small.txt", Size = 10 }); + root.AddChild(new DirEntry(false) { Path = "mid.txt", Size = 30 }); + root.AddChild(new DirEntry(false) { Path = "big.txt", Size = 40 }); + root.SetInMemoryFields(); + var store = EntryStore.Build(root); + + var found = new List(); + EntryStoreSearch.Find(store, + new EntryStoreFindOptions { IncludeFiles = true, IncludeFolders = false, FromSizeEnable = true, FromSize = 25 }, + i => found.Add(store.FullName(i))); + found.Sort(); + + Assert.That(found, Is.EqualTo(new[] { "big.txt", "mid.txt" })); // size >= 25 + } + [Test] public void Search_FilesOnly_MatchesTreeFind() { From 5d6e60f6e2dad84c3cd8c7ad64b279fe96c0fbca Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sun, 7 Jun 2026 00:04:29 +1000 Subject: [PATCH 18/43] perf(soa): wire cdeWin (GUI) to hold catalogs as EntryStores (migration P5) cdeWin now loads each catalog, converts it to an EntryStore, releases the pointer tree, and holds the catalog root as an EntryRef - so the GUI runs on the struct-of- arrays model (~1/3 the memory) for the many-catalog interactive workflow. - CatalogListViewHelper retyped IListViewHelper -> ; catalog list render + RootCompare read metadata from EntryRef.Store. - Search runs over the stores via the filter-capable EntryStoreSearch (size/date/hour parity) with cancel + 100ms streaming; results are EntryRef-based PairDirEntry. - Navigation (SetNewDirectoryRoot/BuildRootNode/Tag) and reload moved to ICommonEntry; catalog identity reads EntryRef.Store (falls back to GetRootEntry for tree pairs). - Added cancel/progress hooks to EntryStoreSearch; presenter tests updated for EntryRef catalog roots. cdeLibTest 145 + cdeWinTest 32 passed; full solution builds clean. NOTE: validated via build + unit tests only - the live WinForms UI was not run here and needs manual verification (catalog list, search, navigation, reload). --- src/cdeLib/Entities/Soa/EntryStoreSearch.cs | 11 +- src/cdeWin/CDEWinForm.cs | 4 +- src/cdeWin/CDEWinFormPresenter.cs | 224 ++++++++++++-------- src/cdeWin/ICDEWinForm.cs | 2 +- src/cdeWinTest/CDEWinFormPresenterTest.cs | 24 +-- src/cdeWinTest/TestCDEWinPresenterBase.cs | 11 +- 6 files changed, 169 insertions(+), 107 deletions(-) diff --git a/src/cdeLib/Entities/Soa/EntryStoreSearch.cs b/src/cdeLib/Entities/Soa/EntryStoreSearch.cs index 7502a27..33e12f1 100644 --- a/src/cdeLib/Entities/Soa/EntryStoreSearch.cs +++ b/src/cdeLib/Entities/Soa/EntryStoreSearch.cs @@ -16,7 +16,10 @@ public static class EntryStoreSearch /// Full-filter search (pattern + name/path + file/folder + size/date/hour ranges), mirroring the /// cdeWin GUI search, evaluated directly against the store arrays. /// - public static void Find(EntryStore store, EntryStoreFindOptions o, Action onMatch) + /// Polled every 4096 entries; return true to stop early (GUI cancel). + /// Called every 4096 entries with the running scanned count (GUI progress). + public static void Find(EntryStore store, EntryStoreFindOptions o, Action onMatch, + Func isCancelled = null, Action onScan = null) { ArgumentNullException.ThrowIfNull(store); ArgumentNullException.ThrowIfNull(o); @@ -31,6 +34,12 @@ public static void Find(EntryStore store, EntryStoreFindOptions o, Action o for (var i = 1; i < store.Count; i++) { + if ((i & 4095) == 0) + { + if (isCancelled != null && isCancelled()) return; + onScan?.Invoke(i); + } + var isDir = store.IsDirectory(i); if (isDir ? !o.IncludeFolders : !o.IncludeFiles) continue; diff --git a/src/cdeWin/CDEWinForm.cs b/src/cdeWin/CDEWinForm.cs index c985dca..516565d 100644 --- a/src/cdeWin/CDEWinForm.cs +++ b/src/cdeWin/CDEWinForm.cs @@ -78,7 +78,7 @@ public partial class CDEWinForm : Form, ICDEWinForm [DesignerSerializationVisibility(DesignerSerializationVisibility.Hidden)] public IListViewHelper DirectoryListViewHelper { get; set; } [DesignerSerializationVisibility(DesignerSerializationVisibility.Hidden)] - public IListViewHelper CatalogListViewHelper { get; set; } + public IListViewHelper CatalogListViewHelper { get; set; } [DesignerSerializationVisibility(DesignerSerializationVisibility.Hidden)] public CheckBoxDependentControlHelper FromDate { get; set; } @@ -223,7 +223,7 @@ private void RegisterClientEvents() patternComboBox.GotFocus += (_, _) => AcceptButton = searchButton; patternComboBox.LostFocus += (_, _) => AcceptButton = null; - CatalogListViewHelper = new ListViewHelper(catalogResultListView) + CatalogListViewHelper = new ListViewHelper(catalogResultListView) { MultiSelect = false, // ReSharper disable PossibleNullReferenceException diff --git a/src/cdeWin/CDEWinFormPresenter.cs b/src/cdeWin/CDEWinFormPresenter.cs index a66b375..f5faf96 100644 --- a/src/cdeWin/CDEWinFormPresenter.cs +++ b/src/cdeWin/CDEWinFormPresenter.cs @@ -10,6 +10,7 @@ using System.Windows.Forms; using cdeLib; using cdeLib.Entities; +using cdeLib.Entities.Soa; using cdeLib.Infrastructure; using cdeWin.Cfg; using JetBrains.Annotations; @@ -27,9 +28,29 @@ public class CDEWinFormPresenter : Presenter, ICDEWinFormPresenter private readonly Color _listViewDirForeColor = Color.DarkBlue; private readonly ICDEWinForm _clientForm; - private List _rootEntries; + + // Catalogs are held as struct-of-arrays EntryStores (≈1/3 the memory of the pointer tree); each + // catalog root is exposed as an EntryRef so the existing ICommonEntry-based GUI works unchanged. + private List _catalogRoots; private readonly IConfig _config; + private static List ToCatalogRoots(List trees) + { + var roots = new List(trees?.Count ?? 0); + if (trees == null) return roots; + for (var i = 0; i < trees.Count; i++) + { + roots.Add(new EntryRef(EntryStore.Build(trees[i]), 0)); + trees[i] = null; // release the tree so it can be collected + } + return roots; + } + + private static EntryStore StoreOf(ICommonEntry root) => ((EntryRef)root).Store; + + // Catalog of a search-result pair (its entries are EntryRefs into a store). + private static EntryStore StoreOfPair(PairDirEntry pde) => (pde.ChildDE as EntryRef)?.Store; + private readonly string[] _directoryVals; private readonly string[] _searchVals; private readonly string[] _catalogVals; @@ -71,7 +92,7 @@ public CDEWinFormPresenter( _clientForm = form; _config = config; _loadCatalogService = loadCatalogService; - _rootEntries = new List(); + _catalogRoots = new List(); _searchVals = new string[_config.DefaultSearchResultColumnCount]; _directoryVals = new string[_config.DefaultDirectoryColumnCount]; @@ -102,10 +123,10 @@ public async Task InitializeAsync() try { - _rootEntries = await _loadCatalogService.LoadRootEntriesAsync( + _catalogRoots = ToCatalogRoots(await _loadCatalogService.LoadRootEntriesAsync( _config, OnLoadProgress, - _loadingCts.Token); + _loadingCts.Token)); SetCatalogListView(); SetMemoryStatus(); @@ -194,10 +215,11 @@ private void RegisterListViewSorters() private void SetCatalogListView() { var catalogHelper = _clientForm.CatalogListViewHelper; - var count = catalogHelper.SetList(_rootEntries); + var count = catalogHelper.SetList(_catalogRoots); catalogHelper.SortList(); _clientForm.SetCatalogsLoadedStatus(count); - _clientForm.SetTotalFileEntriesLoadedStatus(_rootEntries.TotalFileEntries()); + _clientForm.SetTotalFileEntriesLoadedStatus( + (int)_catalogRoots.Sum(r => (long)StoreOf(r).RootFileEntryCount + StoreOf(r).RootDirEntryCount)); } private static double BytesToMb(long bytes) => bytes / (1024.0 * 1024.0); @@ -320,21 +342,24 @@ public void CatalogRetrieveVirtualItem() catalogHelper.RenderItem = lvi; } - private Color CreateRowValuesForRootEntry(IList vals, RootEntry rootEntry, Color listViewForeColor) - { - vals[0] = rootEntry.Path; - vals[1] = rootEntry.VolumeName; - vals[2] = rootEntry.DirEntryCount.ToString(); - vals[3] = rootEntry.FileEntryCount.ToString(); - vals[4] = (rootEntry.DirEntryCount + rootEntry.FileEntryCount).ToString(); - vals[5] = rootEntry.DriveLetterHint; - vals[6] = rootEntry.Size.ToHRString(); - vals[7] = rootEntry.AvailSpace.ToHRString(); - vals[8] = rootEntry.TotalSpace.ToHRString(); - vals[9] = string.Format(_config.DateFormatYMDHMS, rootEntry.ScanStartUtc.ToLocalTime()); - vals[10] = $"{TimeSpan.FromMilliseconds(rootEntry.ScanDurationMilliseconds).TotalSeconds:0.} sec"; - vals[11] = rootEntry.ActualFileName; - vals[12] = rootEntry.Description; + private Color CreateRowValuesForRootEntry(IList vals, ICommonEntry catalogRoot, Color listViewForeColor) + { + var s = StoreOf(catalogRoot); + var scanStart = new DateTime(s.ScanStartUtcTicks, DateTimeKind.Utc); + var scanDurationMs = (s.ScanEndUtcTicks - s.ScanStartUtcTicks) / TimeSpan.TicksPerMillisecond; + vals[0] = s.RootPath; + vals[1] = s.VolumeName; + vals[2] = s.RootDirEntryCount.ToString(); + vals[3] = s.RootFileEntryCount.ToString(); + vals[4] = (s.RootDirEntryCount + s.RootFileEntryCount).ToString(); + vals[5] = s.DriveLetterHint; + vals[6] = s.RootSize.ToHRString(); + vals[7] = s.AvailSpace.ToHRString(); + vals[8] = s.TotalSpace.ToHRString(); + vals[9] = string.Format(_config.DateFormatYMDHMS, scanStart.ToLocalTime()); + vals[10] = $"{TimeSpan.FromMilliseconds(scanDurationMs).TotalSeconds:0.} sec"; + vals[11] = s.ActualFileName; + vals[12] = s.Description; return listViewForeColor; } @@ -342,7 +367,7 @@ private Color CreateRowValuesForRootEntry(IList vals, RootEntry rootEntr public class BgWorkerParam { public FindOptions Options; - public IList RootEntries; + public IList RootEntries; public BgWorkerState State; } @@ -417,7 +442,7 @@ public void Search() var param = new BgWorkerParam { Options = findOptions, - RootEntries = _rootEntries, + RootEntries = _catalogRoots, State = new BgWorkerState() }; _bgWorker.RunWorkerAsync(param); @@ -537,56 +562,73 @@ private void BgWorkerDoWork(object sender, DoWorkEventArgs e) var worker = (BackgroundWorker)sender; var argument = (BgWorkerParam)e.Argument; var findOptions = argument.Options; - var rootEntries = argument.RootEntries; + var catalogRoots = argument.RootEntries; var state = argument.State; + // Translate the GUI FindOptions into the SoA search options and run over the EntryStores. + var opts = new EntryStoreFindOptions + { + Pattern = findOptions.Pattern, + RegexMode = findOptions.RegexMode, + IncludePath = findOptions.IncludePath, + IncludeFiles = findOptions.IncludeFiles, + IncludeFolders = findOptions.IncludeFolders, + FromSizeEnable = findOptions.FromSizeEnable, FromSize = findOptions.FromSize, + ToSizeEnable = findOptions.ToSizeEnable, ToSize = findOptions.ToSize, + FromDateEnable = findOptions.FromDateEnable, FromDate = findOptions.FromDate, + ToDateEnable = findOptions.ToDateEnable, ToDate = findOptions.ToDate, + FromHourEnable = findOptions.FromHourEnable, FromHour = findOptions.FromHour, + ToHourEnable = findOptions.ToHourEnable, ToHour = findOptions.ToHour, + NotOlderThanEnable = findOptions.NotOlderThanEnable, NotOlderThan = findOptions.NotOlderThan, + }; + var limit = findOptions.LimitResultCount; + + var stores = catalogRoots.Select(StoreOf).ToList(); + var grandTotal = stores.Sum(s => s.Count); + var scannedBase = 0; + var list = new List(500); - var listLock = new object(); state.ListCount = 0; state.List = list; + state.End = grandTotal; worker.ReportProgress(0, state); - // Find parallelizes across catalogs, so VisitorFunc runs on multiple threads. List.Add is - // not thread-safe — without this lock, searching many catalogs at once could drop results or - // throw as concurrent adds race on the backing array. - findOptions.VisitorFunc = (p, d) => - { - lock (listLock) - { - list.Add(new PairDirEntry(p, d)); - } - return true; - }; - // Hand the UI an immutable snapshot taken under the lock — never the live list, which worker - // threads are still mutating while the (virtual) ListView indexes into it on the UI thread. - findOptions.ProgressFunc = (counter, end) => + + var lastReport = Stopwatch.GetTimestamp(); + var reportTicks = Stopwatch.Frequency / 10; // ~100ms streaming + + void Report(int scanned) { - List snapshot; - lock (listLock) - { - snapshot = new List(list); - } - state.ListCount = snapshot.Count; - state.List = snapshot; - state.Counter = counter; - state.End = end; - worker.ReportProgress((int)(100.0 * counter / end), state); - }; + var now = Stopwatch.GetTimestamp(); + if (now - lastReport < reportTicks) return; + lastReport = now; + state.ListCount = list.Count; + state.List = new List(list); // immutable snapshot for the UI thread + state.Counter = scanned; + worker.ReportProgress(grandTotal > 0 ? (int)(100.0 * scanned / grandTotal) : 0, state); + } + var timer = Stopwatch.StartNew(); - findOptions.Find(rootEntries); - //findOptions.FindAsync(rootEntries).GetAwaiter().GetResult(); + foreach (var store in stores) + { + if (worker.CancellationPending || list.Count >= limit) break; + var baseScanned = scannedBase; + EntryStoreSearch.Find(store, opts, + onMatch: idx => + { + list.Add(new PairDirEntry(new EntryRef(store, store.Parent[idx]), new EntryRef(store, idx))); + }, + isCancelled: () => worker.CancellationPending || list.Count >= limit, + onScan: scanned => Report(baseScanned + scanned)); + scannedBase += store.Count; + } timer.Stop(); Log.Logger.Information( "Search execution time: {ExecutionTime} ms, Total found {TotalFound}", timer.ElapsedMilliseconds, list.Count); state.ListCount = list.Count; state.List = list; - var completePercent = (int)(100.0 * state.Counter / state.End); - if (state.End - state.Counter < findOptions.ProgressModifier) - { - completePercent = 100; - } - - worker.ReportProgress(completePercent, state); + state.Counter = grandTotal; + worker.ReportProgress(100, state); e.Result = list; } @@ -659,7 +701,8 @@ public void SearchResultRetrieveVirtualItem() _searchVals[(int)SearchResultColumn.FullPath] = pairDirEntry.ParentDE.FullPath; //TODO: Possibly wasting cycles traversing to the root for this, make smarter. - _searchVals[(int)SearchResultColumn.Catalog] = pairDirEntry.GetRootEntry().DefaultFileName; + _searchVals[(int)SearchResultColumn.Catalog] = + StoreOfPair(pairDirEntry)?.DefaultFileName ?? pairDirEntry.GetRootEntry()?.DefaultFileName ?? ""; searchHelper.RenderItem = BuildListViewItem(_searchVals, itemColor, pairDirEntry); } @@ -745,10 +788,10 @@ public void CatalogListViewItemActivate() _clientForm.CatalogListViewHelper.ActionOnActivateItem(GoToDirectoryRoot); } - private void GoToDirectoryRoot(RootEntry newRoot) + private void GoToDirectoryRoot(ICommonEntry newRoot) { - var currentRoot = (RootEntry)_clientForm.DirectoryTreeViewNodes?.Tag; - if (currentRoot == null || currentRoot != newRoot) + var currentRoot = (ICommonEntry)_clientForm.DirectoryTreeViewNodes?.Tag; + if (!SameRoot(currentRoot, newRoot)) { SetNewDirectoryRoot(newRoot); } @@ -756,7 +799,11 @@ private void GoToDirectoryRoot(RootEntry newRoot) _clientForm.SelectDirectoryPane(); } - private TreeNode SetNewDirectoryRoot(RootEntry newRoot) + // Catalog roots are EntryRef instances; two refs to the same catalog share a store. + private static bool SameRoot(ICommonEntry a, ICommonEntry b) + => a is EntryRef ea && b is EntryRef eb && ReferenceEquals(ea.Store, eb.Store); + + private TreeNode SetNewDirectoryRoot(ICommonEntry newRoot) { var newRootNode = BuildRootNode(newRoot); _clientForm.DirectoryTreeViewNodes = newRootNode; @@ -764,7 +811,7 @@ private TreeNode SetNewDirectoryRoot(RootEntry newRoot) return newRootNode; } - private static TreeNode BuildRootNode(RootEntry rootEntry) + private static TreeNode BuildRootNode(ICommonEntry rootEntry) { var rootTreeNode = NewTreeNode(rootEntry); SetDummyChildNode(rootTreeNode, rootEntry); @@ -792,16 +839,16 @@ private void SetDirectoryWithExpand(ICommonEntry dirEntry) private void SetDirectoryWithExpand(IEnumerable activatedDirEntryList) { var currentRootNode = _clientForm.DirectoryTreeViewNodes; - var currentRoot = (RootEntry)currentRootNode?.Tag; + var currentRoot = (ICommonEntry)currentRootNode?.Tag; TreeNode workingTreeNode = null; - RootEntry newRoot = null; + ICommonEntry newRoot = null; foreach (var entry in activatedDirEntryList) { if (newRoot == null) { - newRoot = (RootEntry)entry; - if (currentRoot != newRoot) + newRoot = entry; + if (!SameRoot(currentRoot, newRoot)) { currentRootNode = SetNewDirectoryRoot(newRoot); currentRoot = newRoot; @@ -885,8 +932,8 @@ private int SearchResultCompare(PairDirEntry pde1, PairDirEntry pde2) case 3: compareResult = string.Compare( - pde1.GetRootEntry().ActualFileName, - pde2.GetRootEntry().ActualFileName, + StoreOfPair(pde1)?.ActualFileName ?? pde1.GetRootEntry()?.ActualFileName, + StoreOfPair(pde2)?.ActualFileName ?? pde2.GetRootEntry()?.ActualFileName, StringComparison.OrdinalIgnoreCase); break; @@ -1158,26 +1205,28 @@ public void CatalogListViewColumnClick() _clientForm.CatalogListViewHelper.ListViewColumnClick(); } - private int RootCompare(RootEntry re1, RootEntry re2) + private int RootCompare(ICommonEntry root1, ICommonEntry root2) { + var re1 = StoreOf(root1); + var re2 = StoreOf(root2); var catalogHelper = _clientForm.CatalogListViewHelper; var column = catalogHelper.SortColumn; var compareResult = column switch { - 0 => string.Compare(re1.Path, re2.Path, StringComparison.Ordinal), + 0 => string.Compare(re1.RootPath, re2.RootPath, StringComparison.Ordinal), 1 => string.Compare(string.IsNullOrEmpty(re1.VolumeName) ? "" : re1.VolumeName, string.IsNullOrEmpty(re2.VolumeName) ? "" : re2.VolumeName, StringComparison.Ordinal), - 2 => re1.DirEntryCount.CompareTo(re2.DirEntryCount), - 3 => re1.FileEntryCount.CompareTo(re2.FileEntryCount), - 4 => (re1.DirEntryCount + re1.FileEntryCount).CompareTo(re2.DirEntryCount + re2.FileEntryCount), + 2 => re1.RootDirEntryCount.CompareTo(re2.RootDirEntryCount), + 3 => re1.RootFileEntryCount.CompareTo(re2.RootFileEntryCount), + 4 => (re1.RootDirEntryCount + re1.RootFileEntryCount).CompareTo(re2.RootDirEntryCount + re2.RootFileEntryCount), 5 => string.Compare(re1.DriveLetterHint, re2.DriveLetterHint, StringComparison.Ordinal), - 6 => re1.Size.CompareTo(re2.Size), + 6 => re1.RootSize.CompareTo(re2.RootSize), 7 => re1.AvailSpace.CompareTo(re2.AvailSpace), 8 => re1.TotalSpace.CompareTo(re2.TotalSpace), - 9 => re1.ScanStartUtc.CompareTo(re2.ScanStartUtc), - 10 => re1.ScanDurationMilliseconds.CompareTo(re2.ScanDurationMilliseconds), + 9 => re1.ScanStartUtcTicks.CompareTo(re2.ScanStartUtcTicks), + 10 => (re1.ScanEndUtcTicks - re1.ScanStartUtcTicks).CompareTo(re2.ScanEndUtcTicks - re2.ScanStartUtcTicks), 11 => string.Compare(re1.ActualFileName, re2.ActualFileName, StringComparison.Ordinal), - 12 => re1.DescriptionCompareTo(re2, _config), + 12 => string.Compare(re1.Description, re2.Description, StringComparison.Ordinal), _ => throw new Exception($"Problem column {column} not handled for sort.") }; @@ -1214,11 +1263,8 @@ public async void ReloadCatalogs() var directoryListHelper = _clientForm.DirectoryListViewHelper; directoryListHelper.SetList(null); - var previousRootEntries = _rootEntries; - foreach (var rootEntry in previousRootEntries) - { - rootEntry.ClearCommonEntryFields(); - } + // Drop the previous stores; releasing the references lets the GC reclaim them. + _catalogRoots = new List(); _clientForm.AddLine(string.Empty); _clientForm.AddLine("{0} v{1} reloading catalogs", _config.ProductName, _config.Version); @@ -1236,14 +1282,14 @@ public async void ReloadCatalogs() _clientForm.SetLoadingProgressValue(0); SetMemoryStatus(); - _rootEntries = await _loadCatalogService.LoadRootEntriesAsync( + _catalogRoots = ToCatalogRoots(await _loadCatalogService.LoadRootEntriesAsync( _config, OnLoadProgress, - _loadingCts.Token); + _loadingCts.Token)); - if (_rootEntries.Count > 0) + if (_catalogRoots.Count > 0) { - SetNewDirectoryRoot(_rootEntries.First()); + SetNewDirectoryRoot(_catalogRoots.First()); } SetCatalogListView(); diff --git a/src/cdeWin/ICDEWinForm.cs b/src/cdeWin/ICDEWinForm.cs index 8036bf3..dbdfcf2 100644 --- a/src/cdeWin/ICDEWinForm.cs +++ b/src/cdeWin/ICDEWinForm.cs @@ -92,7 +92,7 @@ public interface ICDEWinForm : IView IListViewHelper SearchResultListViewHelper { get; set; } IListViewHelper DirectoryListViewHelper { get; set; } - IListViewHelper CatalogListViewHelper { get; set; } + IListViewHelper CatalogListViewHelper { get; set; } CheckBoxDependentControlHelper FromDate { get; set; } CheckBoxDependentControlHelper ToDate { get; set; } diff --git a/src/cdeWinTest/CDEWinFormPresenterTest.cs b/src/cdeWinTest/CDEWinFormPresenterTest.cs index cc2ef09..b49243e 100644 --- a/src/cdeWinTest/CDEWinFormPresenterTest.cs +++ b/src/cdeWinTest/CDEWinFormPresenterTest.cs @@ -50,7 +50,7 @@ public void Always_Catalog_SortList() presenter.InitializeAsync().GetAwaiter().GetResult(); // SetList is called with empty list after InitializeAsync - _mockCatalogListViewHelper.Received().SetList(Arg.Any>()); + _mockCatalogListViewHelper.Received().SetList(Arg.Any>()); } [Test] @@ -59,14 +59,14 @@ public void Always_Register_Result_Sorters() var _ = new CDEWinFormPresenter(_mockForm, _stubConfig); _mockSearchResultListViewHelper.ColumnSortCompare = Arg.Any>(); - _mockCatalogListViewHelper.ColumnSortCompare = Arg.Any>(); + _mockCatalogListViewHelper.ColumnSortCompare = Arg.Any>(); _mockDirectoryListViewHelper.ColumnSortCompare = Arg.Any>(); } [Test] public void With_Null_RootEntry_List_SetsCatalogsLoaded() { - _mockCatalogListViewHelper.SetList(Arg.Any>()).Returns(3); + _mockCatalogListViewHelper.SetList(Arg.Any>()).Returns(3); var loadCatalogService = Substitute.For(); loadCatalogService.LoadRootEntriesAsync( @@ -263,9 +263,9 @@ public override void RunBeforeEveryTest() [Test] public void Catalog_Activate_On_Same_RootEntry_Does_Not_Set_Root() { - var testRootTreeNode = new TreeNode("Moo") { Tag = _rootEntry }; + var testRootTreeNode = new TreeNode("Moo") { Tag = _catalogRoot }; _mockForm.DirectoryTreeViewNodes.Returns(testRootTreeNode); - FakeItemActivateWithValue(_mockCatalogListViewHelper, _rootEntry); + FakeItemActivateWithValue(_mockCatalogListViewHelper, _catalogRoot); // ACT _sutPresenter.CatalogListViewItemActivate(); @@ -277,25 +277,25 @@ public void Catalog_Activate_On_Same_RootEntry_Does_Not_Set_Root() public void Catalog_Activate_On_Different_RootEntry_Sets_New_Root() { var alternateRootEntry = new RootEntry(_config) { Path = "alternate" }; - var testRootTreeNode = new TreeNode("Moo") { Tag = alternateRootEntry }; + var testRootTreeNode = new TreeNode("Moo") { Tag = CatalogRootOf(alternateRootEntry) }; _mockForm.DirectoryTreeViewNodes.Returns(testRootTreeNode); TreeNode treeNodeSet = null; _mockForm.DirectoryTreeViewNodes = Arg.Do(node => treeNodeSet = node); - FakeItemActivateWithValue(_mockCatalogListViewHelper, _rootEntry); + FakeItemActivateWithValue(_mockCatalogListViewHelper, _catalogRoot); // ACT _sutPresenter.CatalogListViewItemActivate(); // _mockForm.Received(1).DirectoryTreeViewNodes = Arg.Any(); - Assert.That(treeNodeSet.Tag, Is.EqualTo(_rootEntry)); + Assert.That(treeNodeSet.Tag, Is.EqualTo(_catalogRoot)); } [Test] public void Catalog_Activate_GoToDirectoryRoot_On_Null_RootNode_Sets_New_Root() { _mockForm.DirectoryTreeViewNodes.Returns((TreeNode)null); - FakeItemActivateWithValue(_mockCatalogListViewHelper, _rootEntry); + FakeItemActivateWithValue(_mockCatalogListViewHelper, _catalogRoot); // ACT _sutPresenter.CatalogListViewItemActivate(); @@ -306,7 +306,7 @@ public void Catalog_Activate_GoToDirectoryRoot_On_Null_RootNode_Sets_New_Root() [Test] public void Callback_GoToDirectoryRoot_Sets_Directory_Pane() { - FakeItemActivateWithValue(_mockCatalogListViewHelper, _rootEntry); + FakeItemActivateWithValue(_mockCatalogListViewHelper, _catalogRoot); _sutPresenter.CatalogListViewItemActivate(); @@ -316,7 +316,7 @@ public void Callback_GoToDirectoryRoot_Sets_Directory_Pane() [Test] public void Callback_GoToDirectoryRoot_Setting_RootNode_Calls_InitSort() { - FakeItemActivateWithValue(_mockCatalogListViewHelper, _rootEntry); + FakeItemActivateWithValue(_mockCatalogListViewHelper, _catalogRoot); _sutPresenter.CatalogListViewItemActivate(); @@ -360,7 +360,7 @@ public void Produces_ListView_Field() { _stubConfig.DateFormatYMDHMS.Returns("{0:yyyy/MM}"); // make local time zone irrelevant for test. _mockCatalogListViewHelper.RetrieveItemIndex.Returns(0); - _mockCatalogListViewHelper.GetItemAt(0).Returns(_rootEntry); + _mockCatalogListViewHelper.GetItemAt(0).Returns(_catalogRoot); ListViewItem setRenderItem = null; _mockCatalogListViewHelper.RenderItem = Arg.Do(lvi => setRenderItem = lvi); diff --git a/src/cdeWinTest/TestCDEWinPresenterBase.cs b/src/cdeWinTest/TestCDEWinPresenterBase.cs index 39eb17f..67fd605 100644 --- a/src/cdeWinTest/TestCDEWinPresenterBase.cs +++ b/src/cdeWinTest/TestCDEWinPresenterBase.cs @@ -18,7 +18,12 @@ public class TestCDEWinPresenterBase protected IConfig _stubConfig; protected IListViewHelper _mockSearchResultListViewHelper; protected IListViewHelper _mockDirectoryListViewHelper; - protected IListViewHelper _mockCatalogListViewHelper; + protected IListViewHelper _mockCatalogListViewHelper; + + // Catalog roots are presented to the GUI as EntryRef over an EntryStore (SoA model). + protected cdeLib.Entities.Soa.EntryRef _catalogRoot; + protected cdeLib.Entities.Soa.EntryRef CatalogRootOf(RootEntry re) + => new(cdeLib.Entities.Soa.EntryStore.Build(re), 0); protected RootEntry _rootEntry; protected DirEntry _dirEntry; @@ -42,7 +47,7 @@ public virtual void RunBeforeEveryTest() _mockForm.SearchResultListViewHelper.Returns(_mockSearchResultListViewHelper); _mockDirectoryListViewHelper = Substitute.For>(); _mockForm.DirectoryListViewHelper.Returns(_mockDirectoryListViewHelper); - _mockCatalogListViewHelper = Substitute.For>(); + _mockCatalogListViewHelper = Substitute.For>(); _mockForm.CatalogListViewHelper.Returns(_mockCatalogListViewHelper); } @@ -74,6 +79,7 @@ protected void InitRootWithFile() _rootEntry.AddChild(_dirEntry); _rootEntry.SetInMemoryFields(); _pairDirEntry = new PairDirEntry(_rootEntry, _dirEntry); + _catalogRoot = CatalogRootOf(_rootEntry); _rootList.Add(_rootEntry); } @@ -87,6 +93,7 @@ protected void InitRootWithDir() _rootEntry.AddChild(_dirEntry); _rootEntry.SetInMemoryFields(); _pairDirEntry = new PairDirEntry(_rootEntry, _dirEntry); + _catalogRoot = CatalogRootOf(_rootEntry); _rootList.Add(_rootEntry); } From a4c2e4150e525f8ef6d50959f8885e49fb69cbdb Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sun, 7 Jun 2026 08:17:24 +1000 Subject: [PATCH 19/43] spike(mmap): zero-copy columnar catalog format + one-way migration Prototype a struct-of-arrays catalog file read zero-copy over a memory map, so 'load' becomes mmap and search streams over the mapping. - ColumnarFormat: hand-rolled SoA .cdex writer (primitive columns + UTF-8 name blob), built from an EntryStore. - ColumnarReader: mmap reader exposing columns as ReadOnlySpan over the mapping; zero-alloc UTF-8 byte-scan name/path search. - cdeMemProbe --migrate (one-way MessagePack .cde -> .cdex) and --flat (mmap + search + measure) modes. Result on a real 649k-entry catalog: managed heap 82MB(tree)/44MB(store) -> 0.07MB; open 1123ms -> 3ms; full name scan allocates 40 bytes. Costs: +36% disk, working set ~= touched (reclaimable) pages, ASCII-only case fold + int offsets to address before production. See src/cdeBenchmarks/baseline/flatbuffer-mmap-spike.md. --- .../baseline/flatbuffer-mmap-spike.md | 74 +++++++ src/cdeMemProbe/Columnar/ColumnarFormat.cs | 185 +++++++++++++++++ src/cdeMemProbe/Columnar/ColumnarReader.cs | 190 ++++++++++++++++++ src/cdeMemProbe/Program.cs | 112 +++++++++++ src/cdeMemProbe/cdeMemProbe.csproj | 2 + 5 files changed, 563 insertions(+) create mode 100644 src/cdeBenchmarks/baseline/flatbuffer-mmap-spike.md create mode 100644 src/cdeMemProbe/Columnar/ColumnarFormat.cs create mode 100644 src/cdeMemProbe/Columnar/ColumnarReader.cs diff --git a/src/cdeBenchmarks/baseline/flatbuffer-mmap-spike.md b/src/cdeBenchmarks/baseline/flatbuffer-mmap-spike.md new file mode 100644 index 0000000..8e9ff1c --- /dev/null +++ b/src/cdeBenchmarks/baseline/flatbuffer-mmap-spike.md @@ -0,0 +1,74 @@ +# Spike: zero-copy columnar mmap catalog format + +Branch `spike/flatbuffer-mmap-search`. Goal: prove (or kill) the idea of replacing in-memory +catalog load with a **columnar struct-of-arrays file read zero-copy over a memory map**, so that +"load" becomes `mmap` and search streams over the mapping — managed heap ≈ 0, working set ≈ pages +actually touched (reclaimable OS page cache). + +Spike code lives in `src/cdeMemProbe/Columnar/` (`ColumnarFormat` writer, `ColumnarReader` mmap +reader) plus `--migrate` / `--flat` modes in `Program.cs`. It is intentionally **not** in +production `cdeLib` until the thesis was confirmed. Format magic `CDEX`, version 1. + +## Result: thesis confirmed, decisively + +Real catalog `C-System-C__program files.cde`, **649,377 entries**: + +| Model | Managed heap | B/entry (heap) | "Load"/open | Notes | +|---|---:|---:|---:|---| +| Tree (MessagePack, current) | 82.3 MB | 126.7 | 1123 ms | full pointer graph | +| In-memory SoA store | 43.8 MB | 67.4 | 1050 ms | parallel arrays on heap | +| **Columnar mmap** | **0.07 MB** | **0.11** | **3 ms** | catalog stays in the mapping | + +- **Managed heap ~0.** 72 KB total — just the reader object + tiny scan buffers. ~600× below the + store, ~1100× below the tree. The catalog never enters the GC heap. This is the primary goal + (footprint) hit as hard as it can be hit. +- **"Load" is ~free:** 3 ms to mmap vs ~1100 ms to deserialize. ~370× faster, and independent of + catalog size (no parse). +- **Search is zero-alloc:** a full name byte-scan of 649 K entries allocated **40 bytes** total + (path-walk variant 1088 B, from one growable buffer). Confirms UTF-8 byte-matching avoids the + FlatSharp lazy-string trap. +- **Search speed (single-threaded):** name `.dll` → 131 958 hits in **26.9 ms** (~24 M entries/s); + path `common` → 59 011 hits in **287 ms** (slower: rebuilds each full path). Both parallelize by + index range; path mode has obvious caching wins left on the table. + +Hashed 100 K synthetic round-trips correctly (Hash16 column is blittable); allocations still 40 B. + +## Honest costs / caveats + +1. **Disk +36 %** (35.7 MB vs 26.3 MB MessagePack). Causes: full names stored with repeated + extensions (no dedup), fixed-width 8-aligned columns, no varint packing. Mitigations: dedup + `Ext` via a shared-string column, optional per-column compression. Zero-copy wants fixed-width + numeric columns, so the numeric side stays uncompressed; the win is on the name blob. +2. **Working set ≈ touched pages.** A *full* scan touches the whole NameBlob, so working set rises + toward file size — but these are clean, file-backed, **reclaimable** pages (evictable, shared), + not committed GC heap. Selective queries and column-skipping (a name query never pages in + Size/Modified/Hash) keep it well below file size. The number that matters for GC pressure / + OOM — managed heap — is ~0 regardless. +3. **ASCII case-folding only** in the spike matcher. Production needs proper ordinal-ignore-case + over UTF-8 (or a stored case-folded name column). +4. **`int` entry count + `int` name offsets** cap at ~2.1 B entries / 2 GB name blob. The README + claims "billions"; production must widen offsets to `long` (and the count) to keep headroom. +5. **mmap lifetime / immutability.** The mapping must stay open while a catalog is "loaded"; + the buffer is read-only, so `hash`/`scan`/`dupes` write a *fresh* file (they already re-save). +6. **Hand-rolled, not FlatBuffers.** For dense homogeneous columns a hand-rolled layout gives a + genuinely alloc-free `MemoryMarshal.Cast` view with no vtable/string-materialization overhead; + FlatBuffers earns its keep on sparse/optional schemas, which a catalog is not. Same zero-copy + goal the user asked for, better fit for the data shape. + +## Reproduce + +``` +cdeMemProbe ".cde" # tree heap +cdeMemProbe ".cde" --store # in-memory SoA heap +cdeMemProbe --migrate ".cde" --out cat.cdex +cdeMemProbe --flat cat.cdex --pattern .dll # name search, zero-alloc +cdeMemProbe --flat cat.cdex --pattern common --path # path search +``` + +## Recommendation + +Promote to production behind a format-version gate: add the columnar writer/reader to `cdeLib`, +make `EntryRef : ICommonEntry` a view over `(buffer, offset)` (the seam already exists from the +SoA work), load `find`/GUI/`dump` straight off the mapping, and ship a one-way `cde migrate` +verb (the `--migrate` mode here is the prototype). `scan`/`hash`/`dupes` keep building a tree and +write the columnar file at save. Address caveats 3 & 4 before shipping. diff --git a/src/cdeMemProbe/Columnar/ColumnarFormat.cs b/src/cdeMemProbe/Columnar/ColumnarFormat.cs new file mode 100644 index 0000000..3e6a45a --- /dev/null +++ b/src/cdeMemProbe/Columnar/ColumnarFormat.cs @@ -0,0 +1,185 @@ +using System; +using System.IO; +using System.Runtime.InteropServices; +using System.Text; +using cdeLib.Entities.Soa; + +namespace cdeMemProbe.Columnar; + +/// +/// SPIKE — hand-rolled columnar (struct-of-arrays) on-disk catalog format, designed for +/// zero-copy reads over a memory-mapped file. The whole point is that "loading" a catalog +/// becomes mmap-ing the file: no managed object graph is built, so the working set is just the +/// file pages a query actually touches (in the reclaimable OS page cache), not GC heap. +/// +/// Layout (all little-endian; x64 assumed for the spike): +/// preamble: +/// [0] magic "CDEX" (4 bytes) +/// [4] int32 version +/// [8] int32 count (entries incl. root; index 0 = root) +/// [12] int32 flags (bit0 = hasHashes) +/// [16] (int64 offset, int64 length) x -- absolute, 8-aligned +/// column bodies (each padded to an 8-byte boundary), in order. +/// +/// Columns are dense and homogeneous — a name-only search sequentially scans just the NameBlob + +/// NameOffsets columns and never pages in Size/Modified/Hash. That column-skipping is the memory +/// lever the in-memory store cannot offer. +/// +/// Why hand-rolled rather than FlatBuffers/FlatSharp: for dense fixed-width columns this gives a +/// genuinely alloc-free view straight over the mapping, with +/// no vtable indirection or per-access string materialization (FlatSharp lazy mode's main pitfall). +/// FlatBuffers earns its keep for sparse/optional schemas; a catalog is the opposite of that. +/// +public static class ColumnarFormat +{ + public static readonly byte[] Magic = "CDEX"u8.ToArray(); + public const int Version = 1; + public const int FlagHasHashes = 1; + + /// Fixed column ordering. Hash/Meta lengths are 0 when absent. + public enum Col + { + ModifiedTicks = 0, // long[count] + Size, // long[count] + BitFields, // byte[count] + Parent, // int[count] + FirstChild, // int[count] + NextSibling, // int[count] + NameOffsets, // int[count+1] prefix offsets into NameBlob + NameBlob, // byte[] UTF-8 full names (name+ext) concatenated + Hash, // byte[16*count] (only when hasHashes) + Meta, // byte[] catalog metadata blob + } + + public const int ColumnCount = 10; + public const int PreambleFixed = 16; // magic+version+count+flags + public const int HeaderSize = PreambleFixed + ColumnCount * 16; + + private static long Align8(long v) => (v + 7) & ~7L; + + /// Convert an in-memory to the columnar file. One-time cost. + public static void Write(EntryStore store, string outPath) + { + var count = store.Count; + var hasHashes = store.Hash != null; + + // Build the variable-length name columns up front (UTF-8 full names + prefix offsets). + var nameOffsets = new int[count + 1]; + using var nameBlob = new MemoryStream(count * 12); + for (var i = 0; i < count; i++) + { + nameOffsets[i] = (int)nameBlob.Length; + WriteUtf8(nameBlob, store.Name[i]); + WriteUtf8(nameBlob, store.Ext[i]); // ext appended directly -> full name bytes, no separator + } + nameOffsets[count] = (int)nameBlob.Length; + var nameBlobBytes = nameBlob.GetBuffer().AsSpan(0, (int)nameBlob.Length); + + var meta = BuildMeta(store); + + // Lengths per column. + var len = new long[ColumnCount]; + len[(int)Col.ModifiedTicks] = (long)count * sizeof(long); + len[(int)Col.Size] = (long)count * sizeof(long); + len[(int)Col.BitFields] = count; + len[(int)Col.Parent] = (long)count * sizeof(int); + len[(int)Col.FirstChild] = (long)count * sizeof(int); + len[(int)Col.NextSibling] = (long)count * sizeof(int); + len[(int)Col.NameOffsets] = (long)(count + 1) * sizeof(int); + len[(int)Col.NameBlob] = nameBlobBytes.Length; + len[(int)Col.Hash] = hasHashes ? (long)count * 16 : 0; + len[(int)Col.Meta] = meta.Length; + + // Offsets: header first, then each column 8-aligned. + var off = new long[ColumnCount]; + var pos = (long)HeaderSize; + for (var c = 0; c < ColumnCount; c++) + { + pos = Align8(pos); + off[c] = pos; + pos += len[c]; + } + + using var fs = new FileStream(outPath, FileMode.Create, FileAccess.Write, FileShare.None, + 1 << 20, FileOptions.SequentialScan); + + // Preamble. + fs.Write(Magic); + WriteI32(fs, Version); + WriteI32(fs, count); + WriteI32(fs, hasHashes ? FlagHasHashes : 0); + for (var c = 0; c < ColumnCount; c++) + { + WriteI64(fs, off[c]); + WriteI64(fs, len[c]); + } + + // Column bodies (re-pad to each column's recorded offset). + WriteCol(fs, off[(int)Col.ModifiedTicks], MemoryMarshal.AsBytes(store.ModifiedTicks.AsSpan(0, count))); + WriteCol(fs, off[(int)Col.Size], MemoryMarshal.AsBytes(store.Size.AsSpan(0, count))); + WriteCol(fs, off[(int)Col.BitFields], store.BitFields.AsSpan(0, count)); + WriteCol(fs, off[(int)Col.Parent], MemoryMarshal.AsBytes(store.Parent.AsSpan(0, count))); + WriteCol(fs, off[(int)Col.FirstChild], MemoryMarshal.AsBytes(store.FirstChild.AsSpan(0, count))); + WriteCol(fs, off[(int)Col.NextSibling], MemoryMarshal.AsBytes(store.NextSibling.AsSpan(0, count))); + WriteCol(fs, off[(int)Col.NameOffsets], MemoryMarshal.AsBytes(nameOffsets.AsSpan())); + WriteCol(fs, off[(int)Col.NameBlob], nameBlobBytes); + if (hasHashes) + WriteCol(fs, off[(int)Col.Hash], MemoryMarshal.AsBytes(store.Hash.AsSpan(0, count))); + WriteCol(fs, off[(int)Col.Meta], meta); + } + + private static byte[] BuildMeta(EntryStore s) + { + using var ms = new MemoryStream(256); + WriteLenString(ms, s.RootPath); + WriteLenString(ms, s.VolumeName); + WriteLenString(ms, s.DefaultFileName); + WriteLenString(ms, s.ActualFileName); + WriteLenString(ms, s.DriveLetterHint); + WriteLenString(ms, s.Description); + WriteI64(ms, s.AvailSpace); + WriteI64(ms, s.TotalSpace); + WriteI64(ms, s.ScanStartUtcTicks); + WriteI64(ms, s.ScanEndUtcTicks); + WriteI64(ms, s.RootSize); + WriteI32(ms, (int)s.RootFileEntryCount); + WriteI32(ms, (int)s.RootDirEntryCount); + return ms.ToArray(); + } + + private static void WriteCol(FileStream fs, long offset, ReadOnlySpan body) + { + // Pad from current position up to the column's 8-aligned offset, then write the body. + var pad = offset - fs.Position; + for (var i = 0; i < pad; i++) fs.WriteByte(0); + fs.Write(body); + } + + private static void WriteUtf8(Stream s, string value) + { + if (string.IsNullOrEmpty(value)) return; + var bytes = Encoding.UTF8.GetBytes(value); + s.Write(bytes, 0, bytes.Length); + } + + private static void WriteLenString(Stream s, string value) + { + var bytes = string.IsNullOrEmpty(value) ? [] : Encoding.UTF8.GetBytes(value); + WriteI32(s, bytes.Length); + s.Write(bytes, 0, bytes.Length); + } + + private static void WriteI32(Stream s, int v) + { + Span b = stackalloc byte[4]; + System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(b, v); + s.Write(b); + } + + private static void WriteI64(Stream s, long v) + { + Span b = stackalloc byte[8]; + System.Buffers.Binary.BinaryPrimitives.WriteInt64LittleEndian(b, v); + s.Write(b); + } +} diff --git a/src/cdeMemProbe/Columnar/ColumnarReader.cs b/src/cdeMemProbe/Columnar/ColumnarReader.cs new file mode 100644 index 0000000..64d236d --- /dev/null +++ b/src/cdeMemProbe/Columnar/ColumnarReader.cs @@ -0,0 +1,190 @@ +using System; +using System.IO.MemoryMappedFiles; +using System.Runtime.InteropServices; +using System.Text; + +namespace cdeMemProbe.Columnar; + +/// +/// SPIKE — zero-copy reader over a file. "Loading" is just mmap-ing: +/// columns are exposed as straight over the mapping, so no catalog +/// data is copied to the managed heap. A name search byte-scans the UTF-8 NameBlob in place +/// (zero managed allocation per entry), and only the pages it touches fault into the working set. +/// +public sealed unsafe class ColumnarReader : IDisposable +{ + private readonly MemoryMappedFile _mmf; + private readonly MemoryMappedViewAccessor _view; + private byte* _base; + private readonly long _length; + + private readonly long[] _off = new long[ColumnarFormat.ColumnCount]; + private readonly long[] _len = new long[ColumnarFormat.ColumnCount]; + + public int Count { get; } + public bool HasHashes { get; } + + public ColumnarReader(string path) + { + var fileLen = new System.IO.FileInfo(path).Length; + _length = fileLen; + _mmf = MemoryMappedFile.CreateFromFile(path, System.IO.FileMode.Open, mapName: null, + capacity: 0, MemoryMappedFileAccess.Read); + _view = _mmf.CreateViewAccessor(0, 0, MemoryMappedFileAccess.Read); + _view.SafeMemoryMappedViewHandle.AcquirePointer(ref _base); + + var header = new ReadOnlySpan(_base, ColumnarFormat.HeaderSize); + if (!header[..4].SequenceEqual(ColumnarFormat.Magic)) + throw new InvalidDataException($"not a CDEX file: {path}"); + var version = BitConverter.ToInt32(header.Slice(4, 4)); + if (version != ColumnarFormat.Version) + throw new InvalidDataException($"unsupported CDEX version {version}"); + Count = BitConverter.ToInt32(header.Slice(8, 4)); + HasHashes = (BitConverter.ToInt32(header.Slice(12, 4)) & ColumnarFormat.FlagHasHashes) != 0; + + var p = ColumnarFormat.PreambleFixed; + for (var c = 0; c < ColumnarFormat.ColumnCount; c++) + { + _off[c] = BitConverter.ToInt64(header.Slice(p, 8)); p += 8; + _len[c] = BitConverter.ToInt64(header.Slice(p, 8)); p += 8; + } + } + + private ReadOnlySpan Bytes(ColumnarFormat.Col col) + => new(_base + _off[(int)col], (int)_len[(int)col]); + + private ReadOnlySpan As(ColumnarFormat.Col col) where T : struct + => MemoryMarshal.Cast(Bytes(col)); + + public ReadOnlySpan Parent => As(ColumnarFormat.Col.Parent); + public ReadOnlySpan Size => As(ColumnarFormat.Col.Size); + private ReadOnlySpan NameOffsets => As(ColumnarFormat.Col.NameOffsets); + private ReadOnlySpan NameBlob => Bytes(ColumnarFormat.Col.NameBlob); + + /// UTF-8 full-name bytes of entry , sliced in place from the mapping. + public ReadOnlySpan Name(int i) + { + var offs = NameOffsets; + return NameBlob.Slice(offs[i], offs[i + 1] - offs[i]); + } + + /// + /// Zero-allocation name search: byte-scan each entry's UTF-8 name for + /// (ASCII case-insensitive). Sequentially touches only the NameBlob + NameOffsets columns. + /// + public int FindName(ReadOnlySpan patternUtf8, Action onMatch = null) + { + var offs = NameOffsets; + var blob = NameBlob; + var matches = 0; + for (var i = 0; i < Count; i++) + { + var name = blob.Slice(offs[i], offs[i + 1] - offs[i]); + if (AsciiContainsIgnoreCase(name, patternUtf8)) + { + matches++; + onMatch?.Invoke(i); + } + } + return matches; + } + + /// + /// Path search: build each entry's full path bytes into a reused buffer by walking Parent[], + /// then byte-scan. Still no per-entry managed string allocation. + /// + public int FindPath(ReadOnlySpan patternUtf8, Action onMatch = null) + { + var parent = Parent; + var offs = NameOffsets; + var blob = NameBlob; + Span chain = stackalloc int[256]; + var buf = new byte[1024]; + var matches = 0; + + for (var i = 0; i < Count; i++) + { + var depth = 0; + for (var cur = i; cur != -1 && depth < chain.Length; cur = parent[cur]) chain[depth++] = cur; + + var n = 0; + for (var k = depth - 1; k >= 0; k--) + { + if (n > 0) buf = Append(buf, ref n, (byte)'\\'); + var idx = chain[k]; + var name = blob.Slice(offs[idx], offs[idx + 1] - offs[idx]); + buf = Append(buf, ref n, name); + } + + if (AsciiContainsIgnoreCase(buf.AsSpan(0, n), patternUtf8)) + { + matches++; + onMatch?.Invoke(i); + } + } + return matches; + } + + public string FullPath(int i) + { + var parent = Parent; + var offs = NameOffsets; + var blob = NameBlob; + Span chain = stackalloc int[256]; + var depth = 0; + for (var cur = i; cur != -1 && depth < chain.Length; cur = parent[cur]) chain[depth++] = cur; + var sb = new StringBuilder(128); + for (var k = depth - 1; k >= 0; k--) + { + if (sb.Length > 0) sb.Append('\\'); + var idx = chain[k]; + sb.Append(Encoding.UTF8.GetString(blob.Slice(offs[idx], offs[idx + 1] - offs[idx]))); + } + return sb.ToString(); + } + + private static byte[] Append(byte[] buf, ref int n, byte b) + { + if (n + 1 > buf.Length) Array.Resize(ref buf, buf.Length * 2); + buf[n++] = b; + return buf; + } + + private static byte[] Append(byte[] buf, ref int n, ReadOnlySpan src) + { + while (n + src.Length > buf.Length) Array.Resize(ref buf, buf.Length * 2); + src.CopyTo(buf.AsSpan(n)); + n += src.Length; + return buf; + } + + private static bool AsciiContainsIgnoreCase(ReadOnlySpan haystack, ReadOnlySpan needle) + { + if (needle.IsEmpty) return true; + if (haystack.Length < needle.Length) return false; + var last = haystack.Length - needle.Length; + for (var i = 0; i <= last; i++) + { + var k = 0; + for (; k < needle.Length; k++) + { + if (ToLower(haystack[i + k]) != ToLower(needle[k])) break; + } + if (k == needle.Length) return true; + } + return false; + } + + private static byte ToLower(byte b) => b is >= (byte)'A' and <= (byte)'Z' ? (byte)(b + 32) : b; + + public void Dispose() + { + if (_base != null) + { + _view.SafeMemoryMappedViewHandle.ReleasePointer(); + _base = null; + } + _view?.Dispose(); + _mmf?.Dispose(); + } +} diff --git a/src/cdeMemProbe/Program.cs b/src/cdeMemProbe/Program.cs index 2ae6145..f49e3de 100644 --- a/src/cdeMemProbe/Program.cs +++ b/src/cdeMemProbe/Program.cs @@ -51,6 +51,16 @@ public static async Task Main(string[] args) return MeasureSoa(args); } + if (HasFlag(args, "--migrate", out var migrateIn)) + { + return Migrate(args, migrateIn, logger); + } + + if (HasFlag(args, "--flat", out var flatFile)) + { + return MeasureFlat(args, flatFile); + } + if (HasFlag(args, "--generate", out var genValue)) { return await GenerateAsync(args, genValue, logger); @@ -223,6 +233,108 @@ private static (object Measured, long Entries) LoadAsStore(string file, ILogger return (store, entries); } + /// + /// One-way migration: load an existing MessagePack .cde, convert to the SoA EntryStore, and write + /// the columnar/mmap format. Usage: cdeMemProbe --migrate <in.cde> [--out <out.cdex>] + /// + private static int Migrate(string[] args, string? inFile, ILogger logger) + { + if (string.IsNullOrWhiteSpace(inFile) || !File.Exists(inFile)) + { + Console.Error.WriteLine("--migrate requires an existing , e.g. --migrate cat.cde --out cat.cdex"); + return 1; + } + + var outFile = HasFlag(args, "--out", out var outArg) && !string.IsNullOrWhiteSpace(outArg) + ? outArg! + : Path.ChangeExtension(inFile, ".cdex"); + + var sw = Stopwatch.StartNew(); + EntryStore store; + using (var repo = new CatalogRepository(logger)) + { + var root = repo.LoadDirCache(inFile); + if (root == null) + { + Console.Error.WriteLine($"failed to load catalog: {inFile}"); + return 1; + } + store = EntryStore.Build(root); + } + Columnar.ColumnarFormat.Write(store, outFile); + sw.Stop(); + + var srcLen = new FileInfo(inFile).Length; + var dstLen = new FileInfo(outFile).Length; + Console.Error.WriteLine( + $"migrated {store.Count:N0} entries: {Path.GetFileName(inFile)} ({srcLen:N0} B) -> " + + $"{Path.GetFileName(outFile)} ({dstLen:N0} B) in {sw.ElapsedMilliseconds:N0} ms"); + Console.WriteLine(outFile); + return 0; + } + + /// + /// Measure the zero-copy mmap path: open the columnar file (no managed load), run a search, and + /// report retained heap, allocations DURING the search, working set, and timing. + /// Usage: cdeMemProbe --flat <file.cdex> [--pattern X] [--path] + /// + private static int MeasureFlat(string[] args, string? flatFile) + { + if (string.IsNullOrWhiteSpace(flatFile) || !File.Exists(flatFile)) + { + Console.Error.WriteLine("--flat requires an existing "); + return 1; + } + + var pattern = HasFlag(args, "--pattern", out var p) && !string.IsNullOrEmpty(p) ? p! : ".txt"; + var pathMode = HasFlag(args, "--path", out _); + var patternUtf8 = System.Text.Encoding.UTF8.GetBytes(pattern); + + // Settle, then snapshot allocation + heap baselines so we can isolate the search's own cost. + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + var heapBefore = GC.GetTotalMemory(true); + + var openSw = Stopwatch.StartNew(); + using var reader = new Columnar.ColumnarReader(flatFile); + openSw.Stop(); + + var allocBefore = GC.GetTotalAllocatedBytes(precise: true); + var searchSw = Stopwatch.StartNew(); + var matches = pathMode + ? reader.FindPath(patternUtf8) + : reader.FindName(patternUtf8); + searchSw.Stop(); + var allocDuringSearch = GC.GetTotalAllocatedBytes(precise: true) - allocBefore; + + var heapAfter = GC.GetTotalMemory(false); // no forced collect: show what the search left live + using var proc = Process.GetCurrentProcess(); + var workingSet = proc.WorkingSet64; + var fileBytes = new FileInfo(flatFile).Length; + GC.KeepAlive(reader); + + var heapPerEntry = reader.Count > 0 ? (double)heapAfter / reader.Count : 0; + + Console.WriteLine( + "file,entries,mode,openMs,searchMs,matches,allocDuringSearch,heapBytes,heapPerEntry,workingSet,fileBytes"); + Console.WriteLine(string.Join(',', + Path.GetFileName(flatFile), + reader.Count.ToString(CultureInfo.InvariantCulture), + pathMode ? "path" : "name", + openSw.ElapsedMilliseconds.ToString(CultureInfo.InvariantCulture), + searchSw.Elapsed.TotalMilliseconds.ToString("F2", CultureInfo.InvariantCulture), + matches.ToString(CultureInfo.InvariantCulture), + allocDuringSearch.ToString(CultureInfo.InvariantCulture), + heapAfter.ToString(CultureInfo.InvariantCulture), + heapPerEntry.ToString("F2", CultureInfo.InvariantCulture), + workingSet.ToString(CultureInfo.InvariantCulture), + fileBytes.ToString(CultureInfo.InvariantCulture))); + Console.Error.WriteLine( + $"baseline heap before open: {heapBefore:N0} B; sample path[1] = {reader.FullPath(1)}"); + return 0; + } + /// /// Returns true if is present. If the next token is not another flag it /// is returned as (so both --generate 100 and bare flags work). diff --git a/src/cdeMemProbe/cdeMemProbe.csproj b/src/cdeMemProbe/cdeMemProbe.csproj index ea8ef5b..7b53141 100644 --- a/src/cdeMemProbe/cdeMemProbe.csproj +++ b/src/cdeMemProbe/cdeMemProbe.csproj @@ -6,6 +6,8 @@ enable enable latest + + true true From 183fadcc75fb26eb92c09e73b2de5e69d5edb40e Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sun, 7 Jun 2026 08:25:36 +1000 Subject: [PATCH 20/43] feat(columnar): promote zero-copy mmap catalog format to cdeLib + cde migrate Move the spike's columnar/mmap format into production cdeLib and add a one-way migration command. - cdeLib/Entities/Columnar: ColumnarFormat (writer), ColumnarCatalogReader (zero-copy mmap reader exposing columns as ReadOnlySpan over the mapping, parses catalog metadata once), Utf8Matcher (ordinal case-insensitive, zero-alloc ASCII fast path + decoded fallback for non-ASCII names). - Hardened vs spike: 64-bit name-blob offsets (no 2GB cap), Unicode-correct case folding, format magic+version gate, file/folder filtering. - 'cde migrate [file]' converts MessagePack .cde -> .cdex (current dir + one level down, or a single file). - cdeMemProbe now uses the cdeLib implementation (spike copy removed). - ColumnarCatalogTests: round-trip + search parity vs EntryStoreSearch (7 tests). cdeLib AllowUnsafeBlocks enabled for the mmap pointer acquisition. --- src/cde/CommandLine/CommandLineOptions.cs | 10 + src/cde/Program.cs | 67 +++++ .../Columnar/ColumnarCatalogReader.cs | 242 ++++++++++++++++++ .../Entities}/Columnar/ColumnarFormat.cs | 56 ++-- src/cdeLib/Entities/Columnar/Utf8Matcher.cs | 60 +++++ src/cdeLib/cdeLib.csproj | 3 + .../Columnar/ColumnarCatalogTests.cs | 136 ++++++++++ src/cdeMemProbe/Columnar/ColumnarReader.cs | 190 -------------- src/cdeMemProbe/Program.cs | 10 +- 9 files changed, 547 insertions(+), 227 deletions(-) create mode 100644 src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs rename src/{cdeMemProbe => cdeLib/Entities}/Columnar/ColumnarFormat.cs (72%) create mode 100644 src/cdeLib/Entities/Columnar/Utf8Matcher.cs create mode 100644 src/cdeLibTest/Columnar/ColumnarCatalogTests.cs delete mode 100644 src/cdeMemProbe/Columnar/ColumnarReader.cs diff --git a/src/cde/CommandLine/CommandLineOptions.cs b/src/cde/CommandLine/CommandLineOptions.cs index 9a31368..e9eca3b 100644 --- a/src/cde/CommandLine/CommandLineOptions.cs +++ b/src/cde/CommandLine/CommandLineOptions.cs @@ -66,6 +66,16 @@ public class ReplFindOptions public string Value { get; [UsedImplicitly] set; } } +[Verb("migrate", + HelpText = "Convert MessagePack .cde catalogs to the zero-copy columnar .cdex format (one-way).")] +public class MigrateOptions +{ + [Value(0, Required = false, + HelpText = "Specific .cde file to convert. If omitted, converts every catalog in the current " + + "directory and one level down, writing a .cdex beside each.")] + public string Path { get; [UsedImplicitly] set; } +} + [Verb("hash", HelpText = "Hash all catalogs in current directory")] public class HashOptions; diff --git a/src/cde/Program.cs b/src/cde/Program.cs index 51624b5..43abeb7 100644 --- a/src/cde/Program.cs +++ b/src/cde/Program.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.IO; using System.Linq; using System.Threading.Tasks; using Autofac; @@ -9,6 +10,8 @@ using cdeLib.Catalog; using cdeLib.Duplicates; using cdeLib.Entities; +using cdeLib.Entities.Columnar; +using cdeLib.Entities.Soa; using cdeLib.Hashing; using cdeLib.Upgrade; using CommandLine; @@ -52,6 +55,7 @@ private static ParserResult GetParserResult(IEnumerable args) ReplGrepPathOptions, ReplGrepOptions, ReplFindOptions, + MigrateOptions, HashOptions, DupesOptions, TreeDumpOptions, @@ -99,6 +103,7 @@ private static int Main(string[] args) .WithParsed(opts => FindRepl(FindService.ParamGrepPath, opts.Value)) .WithParsed(opts => FindRepl(FindService.ParamGrep, opts.Value)) .WithParsed(opts => FindRepl(FindService.ParamFind, opts.Value)) + .WithParsed(Migrate) .WithParsed(_ => HashCatalog()) .WithParsed(_ => FindDupes()) .WithParsed(_ => PrintPathsHaveHashEnumerator()) @@ -125,6 +130,68 @@ private static T Resolve() return _container.Resolve(); } + /// + /// One-way migration of MessagePack .cde catalogs to the zero-copy columnar .cdex format. With a + /// path argument, converts that file; otherwise converts every catalog discovered in the current + /// directory and one level down, writing a .cdex beside each source. + /// + private static void Migrate(MigrateOptions opts) + { + var repo = Resolve(); + + List files; + if (!string.IsNullOrWhiteSpace(opts.Path)) + { + if (!File.Exists(opts.Path)) + { + Console.WriteLine($"File not found: {opts.Path}"); + return; + } + files = [opts.Path]; + } + else + { + files = repo.GetCacheFileList(["./"]).ToList(); + } + + if (files.Count == 0) + { + Console.WriteLine("No .cde catalogs found to migrate."); + return; + } + + var converted = 0; + foreach (var file in files) + { + try + { + var root = repo.LoadDirCache(file); + if (root == null) + { + Console.WriteLine($" skip (could not load): {file}"); + continue; + } + + var store = EntryStore.Build(root); + var outFile = Path.ChangeExtension(file, ".cdex"); + ColumnarFormat.Write(store, outFile); + + var srcLen = new FileInfo(file).Length; + var dstLen = new FileInfo(outFile).Length; + Console.WriteLine( + $" {Path.GetFileName(file)} ({srcLen:N0} B) -> {Path.GetFileName(outFile)} " + + $"({dstLen:N0} B, {store.Count:N0} entries)"); + converted++; + } + catch (Exception ex) + { + Console.WriteLine($" error migrating {file}: {ex.Message}"); + } + } + + Console.WriteLine($"Migrated {converted} of {files.Count} catalog(s) to .cdex."); + } + private static void InvokeRepl() { var le = new LineEditor(name: null); diff --git a/src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs b/src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs new file mode 100644 index 0000000..e7d4659 --- /dev/null +++ b/src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs @@ -0,0 +1,242 @@ +using System; +using System.IO; +using System.IO.MemoryMappedFiles; +using System.Runtime.InteropServices; +using System.Text; +using cdeLib.Entities.Soa; + +namespace cdeLib.Entities.Columnar; + +/// +/// Zero-copy reader over a catalog file. "Loading" is just mmap-ing: +/// columns are exposed as straight over the mapping, so no catalog data +/// is copied to the managed heap. A name search byte-scans the UTF-8 NameBlob in place (zero managed +/// allocation per entry on the common ASCII path), and only the pages it touches fault into the +/// working set. +/// +/// The mapping stays open for the lifetime of the reader; hold it for as long as the catalog is +/// "loaded" and it on reload/exit. The file is read-only. +/// +public sealed unsafe class ColumnarCatalogReader : IDisposable +{ + private readonly MemoryMappedFile _mmf; + private readonly MemoryMappedViewAccessor _view; + private byte* _base; + + private readonly long[] _off = new long[ColumnarFormat.ColumnCount]; + private readonly long[] _len = new long[ColumnarFormat.ColumnCount]; + + public int Count { get; } + public bool HasHashes { get; } + + // Catalog-level metadata (parsed once from the small Meta blob — this is the only managed copy). + public string RootPath { get; } + public string VolumeName { get; } + public string DefaultFileName { get; } + public string ActualFileName { get; } + public string DriveLetterHint { get; } + public string Description { get; } + public long AvailSpace { get; } + public long TotalSpace { get; } + public long ScanStartUtcTicks { get; } + public long ScanEndUtcTicks { get; } + public long RootSize { get; } + public uint RootFileEntryCount { get; } + public uint RootDirEntryCount { get; } + + public ColumnarCatalogReader(string path) + { + _mmf = MemoryMappedFile.CreateFromFile(path, FileMode.Open, mapName: null, + capacity: 0, MemoryMappedFileAccess.Read); + _view = _mmf.CreateViewAccessor(0, 0, MemoryMappedFileAccess.Read); + _view.SafeMemoryMappedViewHandle.AcquirePointer(ref _base); + + var header = new ReadOnlySpan(_base, ColumnarFormat.HeaderSize); + if (!header[..4].SequenceEqual(ColumnarFormat.Magic)) + throw new InvalidDataException($"not a CDEX catalog: {path}"); + var version = BitConverter.ToInt32(header.Slice(4, 4)); + if (version != ColumnarFormat.Version) + throw new InvalidDataException( + $"unsupported CDEX version {version} (expected {ColumnarFormat.Version}): {path}"); + Count = BitConverter.ToInt32(header.Slice(8, 4)); + HasHashes = (BitConverter.ToInt32(header.Slice(12, 4)) & ColumnarFormat.FlagHasHashes) != 0; + + var p = ColumnarFormat.PreambleFixed; + for (var c = 0; c < ColumnarFormat.ColumnCount; c++) + { + _off[c] = BitConverter.ToInt64(header.Slice(p, 8)); p += 8; + _len[c] = BitConverter.ToInt64(header.Slice(p, 8)); p += 8; + } + + // Parse the metadata blob (small, read once). + var meta = Bytes(ColumnarFormat.Col.Meta); + var mp = 0; + RootPath = ReadLenString(meta, ref mp); + VolumeName = ReadLenString(meta, ref mp); + DefaultFileName = ReadLenString(meta, ref mp); + ActualFileName = ReadLenString(meta, ref mp); + DriveLetterHint = ReadLenString(meta, ref mp); + Description = ReadLenString(meta, ref mp); + AvailSpace = ReadI64(meta, ref mp); + TotalSpace = ReadI64(meta, ref mp); + ScanStartUtcTicks = ReadI64(meta, ref mp); + ScanEndUtcTicks = ReadI64(meta, ref mp); + RootSize = ReadI64(meta, ref mp); + RootFileEntryCount = (uint)ReadI32(meta, ref mp); + RootDirEntryCount = (uint)ReadI32(meta, ref mp); + } + + private ReadOnlySpan Bytes(ColumnarFormat.Col col) + => new(_base + _off[(int)col], (int)_len[(int)col]); + + private ReadOnlySpan As(ColumnarFormat.Col col) where T : struct + => MemoryMarshal.Cast(Bytes(col)); + + public ReadOnlySpan Size => As(ColumnarFormat.Col.Size); + public ReadOnlySpan ModifiedTicks => As(ColumnarFormat.Col.ModifiedTicks); + public ReadOnlySpan BitFields => Bytes(ColumnarFormat.Col.BitFields); + public ReadOnlySpan Parent => As(ColumnarFormat.Col.Parent); + private ReadOnlySpan NameOffsets => As(ColumnarFormat.Col.NameOffsets); + private ReadOnlySpan NameBlob => Bytes(ColumnarFormat.Col.NameBlob); + + public Flags Flags(int i) => (Flags)BitFields[i]; + public bool IsDirectory(int i) => (Flags(i) & Entities.Flags.Directory) == Entities.Flags.Directory; + + /// UTF-8 full-name bytes of entry , sliced in place from the mapping. + public ReadOnlySpan NameUtf8(int i) + { + var offs = NameOffsets; + return NameBlob.Slice((int)offs[i], (int)(offs[i + 1] - offs[i])); + } + + public string Name(int i) => Encoding.UTF8.GetString(NameUtf8(i)); + + /// + /// Name search: byte-scan each entry's UTF-8 name for (ordinal, + /// case-insensitive). Sequentially touches only the NameBlob + NameOffsets columns. Zero managed + /// allocation per entry for ASCII names; non-ASCII names fall back to a decoded comparison. + /// + public int FindName(string pattern, bool includeFiles, bool includeFolders, Action onMatch = null) + { + var matcher = new Utf8Matcher(pattern); + var offs = NameOffsets; + var blob = NameBlob; + var bits = BitFields; + var matches = 0; + for (var i = 0; i < Count; i++) + { + if (!Wanted(bits[i], includeFiles, includeFolders)) continue; + var name = blob.Slice((int)offs[i], (int)(offs[i + 1] - offs[i])); + if (matcher.Contains(name)) + { + matches++; + onMatch?.Invoke(i); + } + } + return matches; + } + + /// + /// Path search: build each entry's full path bytes into a reused buffer by walking Parent[], then + /// match. No per-entry managed string allocation on the ASCII path. + /// + public int FindPath(string pattern, bool includeFiles, bool includeFolders, Action onMatch = null) + { + var matcher = new Utf8Matcher(pattern); + var parent = Parent; + var offs = NameOffsets; + var blob = NameBlob; + var bits = BitFields; + Span chain = stackalloc int[256]; + var buf = new byte[1024]; + var matches = 0; + + for (var i = 0; i < Count; i++) + { + if (!Wanted(bits[i], includeFiles, includeFolders)) continue; + + var depth = 0; + for (var cur = i; cur != EntryStore.None && depth < chain.Length; cur = parent[cur]) + chain[depth++] = cur; + + var n = 0; + for (var k = depth - 1; k >= 0; k--) + { + if (n > 0) buf = Append(buf, ref n, (byte)Path.DirectorySeparatorChar); + var idx = chain[k]; + buf = Append(buf, ref n, blob.Slice((int)offs[idx], (int)(offs[idx + 1] - offs[idx]))); + } + + if (matcher.Contains(buf.AsSpan(0, n))) + { + matches++; + onMatch?.Invoke(i); + } + } + return matches; + } + + private static bool Wanted(byte bitField, bool includeFiles, bool includeFolders) + { + if (includeFiles && includeFolders) return true; + var isDir = ((Flags)bitField & Entities.Flags.Directory) == Entities.Flags.Directory; + return isDir ? includeFolders : includeFiles; + } + + public string FullPath(int i) + { + var parent = Parent; + var offs = NameOffsets; + var blob = NameBlob; + Span chain = stackalloc int[256]; + var depth = 0; + for (var cur = i; cur != EntryStore.None && depth < chain.Length; cur = parent[cur]) + chain[depth++] = cur; + var sb = new StringBuilder(128); + for (var k = depth - 1; k >= 0; k--) + { + if (sb.Length > 0) sb.Append(Path.DirectorySeparatorChar); + var idx = chain[k]; + sb.Append(Encoding.UTF8.GetString(blob.Slice((int)offs[idx], (int)(offs[idx + 1] - offs[idx])))); + } + return sb.ToString(); + } + + private static byte[] Append(byte[] buf, ref int n, byte b) + { + if (n + 1 > buf.Length) Array.Resize(ref buf, buf.Length * 2); + buf[n++] = b; + return buf; + } + + private static byte[] Append(byte[] buf, ref int n, ReadOnlySpan src) + { + while (n + src.Length > buf.Length) Array.Resize(ref buf, buf.Length * 2); + src.CopyTo(buf.AsSpan(n)); + n += src.Length; + return buf; + } + + private static string ReadLenString(ReadOnlySpan s, ref int p) + { + var len = BitConverter.ToInt32(s.Slice(p, 4)); p += 4; + if (len == 0) return string.Empty; + var str = Encoding.UTF8.GetString(s.Slice(p, len)); + p += len; + return str; + } + + private static long ReadI64(ReadOnlySpan s, ref int p) { var v = BitConverter.ToInt64(s.Slice(p, 8)); p += 8; return v; } + private static int ReadI32(ReadOnlySpan s, ref int p) { var v = BitConverter.ToInt32(s.Slice(p, 4)); p += 4; return v; } + + public void Dispose() + { + if (_base != null) + { + _view.SafeMemoryMappedViewHandle.ReleasePointer(); + _base = null; + } + _view?.Dispose(); + _mmf?.Dispose(); + } +} diff --git a/src/cdeMemProbe/Columnar/ColumnarFormat.cs b/src/cdeLib/Entities/Columnar/ColumnarFormat.cs similarity index 72% rename from src/cdeMemProbe/Columnar/ColumnarFormat.cs rename to src/cdeLib/Entities/Columnar/ColumnarFormat.cs index 3e6a45a..204e30e 100644 --- a/src/cdeMemProbe/Columnar/ColumnarFormat.cs +++ b/src/cdeLib/Entities/Columnar/ColumnarFormat.cs @@ -1,18 +1,19 @@ using System; +using System.Buffers.Binary; using System.IO; using System.Runtime.InteropServices; using System.Text; using cdeLib.Entities.Soa; -namespace cdeMemProbe.Columnar; +namespace cdeLib.Entities.Columnar; /// -/// SPIKE — hand-rolled columnar (struct-of-arrays) on-disk catalog format, designed for -/// zero-copy reads over a memory-mapped file. The whole point is that "loading" a catalog -/// becomes mmap-ing the file: no managed object graph is built, so the working set is just the -/// file pages a query actually touches (in the reclaimable OS page cache), not GC heap. +/// On-disk columnar (struct-of-arrays) catalog format, designed for zero-copy reads over a +/// memory-mapped file. "Loading" a catalog becomes mmap-ing the file — no managed object graph +/// is materialised, so the working set is only the file pages a query actually touches (in the +/// reclaimable OS page cache), not GC heap. See for the read side. /// -/// Layout (all little-endian; x64 assumed for the spike): +/// Layout (all little-endian): /// preamble: /// [0] magic "CDEX" (4 bytes) /// [4] int32 version @@ -21,22 +22,17 @@ namespace cdeMemProbe.Columnar; /// [16] (int64 offset, int64 length) x -- absolute, 8-aligned /// column bodies (each padded to an 8-byte boundary), in order. /// -/// Columns are dense and homogeneous — a name-only search sequentially scans just the NameBlob + -/// NameOffsets columns and never pages in Size/Modified/Hash. That column-skipping is the memory -/// lever the in-memory store cannot offer. -/// -/// Why hand-rolled rather than FlatBuffers/FlatSharp: for dense fixed-width columns this gives a -/// genuinely alloc-free view straight over the mapping, with -/// no vtable indirection or per-access string materialization (FlatSharp lazy mode's main pitfall). -/// FlatBuffers earns its keep for sparse/optional schemas; a catalog is the opposite of that. +/// Columns are dense and homogeneous, so a name-only search sequentially scans just the NameBlob + +/// NameOffsets columns and never pages in Size / Modified / Hash. NameOffsets are 64-bit so the name +/// blob is not capped at 2 GB. Entry count is 32-bit, matching 's int indexing. /// public static class ColumnarFormat { - public static readonly byte[] Magic = "CDEX"u8.ToArray(); + public static ReadOnlySpan Magic => "CDEX"u8; public const int Version = 1; public const int FlagHasHashes = 1; - /// Fixed column ordering. Hash/Meta lengths are 0 when absent. + /// Fixed column ordering. Hash length is 0 when the catalog is un-hashed. public enum Col { ModifiedTicks = 0, // long[count] @@ -45,10 +41,10 @@ public enum Col Parent, // int[count] FirstChild, // int[count] NextSibling, // int[count] - NameOffsets, // int[count+1] prefix offsets into NameBlob - NameBlob, // byte[] UTF-8 full names (name+ext) concatenated + NameOffsets, // long[count+1] prefix offsets into NameBlob + NameBlob, // byte[] UTF-8 full names (name+ext) concatenated Hash, // byte[16*count] (only when hasHashes) - Meta, // byte[] catalog metadata blob + Meta, // byte[] catalog metadata blob } public const int ColumnCount = 10; @@ -60,24 +56,24 @@ public enum Col /// Convert an in-memory to the columnar file. One-time cost. public static void Write(EntryStore store, string outPath) { + ArgumentNullException.ThrowIfNull(store); var count = store.Count; var hasHashes = store.Hash != null; - // Build the variable-length name columns up front (UTF-8 full names + prefix offsets). - var nameOffsets = new int[count + 1]; + // Build the variable-length name columns up front (UTF-8 full names + 64-bit prefix offsets). + var nameOffsets = new long[count + 1]; using var nameBlob = new MemoryStream(count * 12); for (var i = 0; i < count; i++) { - nameOffsets[i] = (int)nameBlob.Length; + nameOffsets[i] = nameBlob.Length; WriteUtf8(nameBlob, store.Name[i]); - WriteUtf8(nameBlob, store.Ext[i]); // ext appended directly -> full name bytes, no separator + WriteUtf8(nameBlob, store.Ext[i]); // ext appended directly -> full-name bytes, no separator } - nameOffsets[count] = (int)nameBlob.Length; + nameOffsets[count] = nameBlob.Length; var nameBlobBytes = nameBlob.GetBuffer().AsSpan(0, (int)nameBlob.Length); var meta = BuildMeta(store); - // Lengths per column. var len = new long[ColumnCount]; len[(int)Col.ModifiedTicks] = (long)count * sizeof(long); len[(int)Col.Size] = (long)count * sizeof(long); @@ -85,12 +81,11 @@ public static void Write(EntryStore store, string outPath) len[(int)Col.Parent] = (long)count * sizeof(int); len[(int)Col.FirstChild] = (long)count * sizeof(int); len[(int)Col.NextSibling] = (long)count * sizeof(int); - len[(int)Col.NameOffsets] = (long)(count + 1) * sizeof(int); + len[(int)Col.NameOffsets] = (long)(count + 1) * sizeof(long); len[(int)Col.NameBlob] = nameBlobBytes.Length; len[(int)Col.Hash] = hasHashes ? (long)count * 16 : 0; len[(int)Col.Meta] = meta.Length; - // Offsets: header first, then each column 8-aligned. var off = new long[ColumnCount]; var pos = (long)HeaderSize; for (var c = 0; c < ColumnCount; c++) @@ -103,7 +98,6 @@ public static void Write(EntryStore store, string outPath) using var fs = new FileStream(outPath, FileMode.Create, FileAccess.Write, FileShare.None, 1 << 20, FileOptions.SequentialScan); - // Preamble. fs.Write(Magic); WriteI32(fs, Version); WriteI32(fs, count); @@ -114,7 +108,6 @@ public static void Write(EntryStore store, string outPath) WriteI64(fs, len[c]); } - // Column bodies (re-pad to each column's recorded offset). WriteCol(fs, off[(int)Col.ModifiedTicks], MemoryMarshal.AsBytes(store.ModifiedTicks.AsSpan(0, count))); WriteCol(fs, off[(int)Col.Size], MemoryMarshal.AsBytes(store.Size.AsSpan(0, count))); WriteCol(fs, off[(int)Col.BitFields], store.BitFields.AsSpan(0, count)); @@ -149,7 +142,6 @@ private static byte[] BuildMeta(EntryStore s) private static void WriteCol(FileStream fs, long offset, ReadOnlySpan body) { - // Pad from current position up to the column's 8-aligned offset, then write the body. var pad = offset - fs.Position; for (var i = 0; i < pad; i++) fs.WriteByte(0); fs.Write(body); @@ -172,14 +164,14 @@ private static void WriteLenString(Stream s, string value) private static void WriteI32(Stream s, int v) { Span b = stackalloc byte[4]; - System.Buffers.Binary.BinaryPrimitives.WriteInt32LittleEndian(b, v); + BinaryPrimitives.WriteInt32LittleEndian(b, v); s.Write(b); } private static void WriteI64(Stream s, long v) { Span b = stackalloc byte[8]; - System.Buffers.Binary.BinaryPrimitives.WriteInt64LittleEndian(b, v); + BinaryPrimitives.WriteInt64LittleEndian(b, v); s.Write(b); } } diff --git a/src/cdeLib/Entities/Columnar/Utf8Matcher.cs b/src/cdeLib/Entities/Columnar/Utf8Matcher.cs new file mode 100644 index 0000000..0a6ff1d --- /dev/null +++ b/src/cdeLib/Entities/Columnar/Utf8Matcher.cs @@ -0,0 +1,60 @@ +using System; +using System.Text; + +namespace cdeLib.Entities.Columnar; + +/// +/// Ordinal, case-insensitive substring matcher over UTF-8 bytes, built once per query. The common +/// case — an ASCII pattern against an ASCII name — is matched by folding bytes in place with no +/// allocation. A pattern or name containing non-ASCII bytes falls back to a decoded +/// compare (allocates only for those rare names), +/// so results match the tree/store search path exactly. +/// +public readonly ref struct Utf8Matcher +{ + private readonly ReadOnlySpan _patternLowerAscii; // A-Z folded to a-z; valid only when _asciiPattern + private readonly string _pattern; + private readonly bool _asciiPattern; + private readonly bool _empty; + + public Utf8Matcher(string pattern) + { + _pattern = pattern ?? string.Empty; + _empty = _pattern.Length == 0; + var bytes = _empty ? [] : Encoding.UTF8.GetBytes(_pattern); + _asciiPattern = System.Text.Ascii.IsValid(bytes); + if (_asciiPattern && !_empty) + { + for (var i = 0; i < bytes.Length; i++) bytes[i] = ToLower(bytes[i]); + } + _patternLowerAscii = bytes; + } + + public bool Contains(ReadOnlySpan nameUtf8) + { + if (_empty) return true; + if (_asciiPattern && System.Text.Ascii.IsValid(nameUtf8)) + return AsciiContainsFolded(nameUtf8, _patternLowerAscii); + + // Rare path: non-ASCII somewhere. Decode and compare with real ordinal-ignore-case. + return Encoding.UTF8.GetString(nameUtf8).Contains(_pattern, StringComparison.OrdinalIgnoreCase); + } + + private static bool AsciiContainsFolded(ReadOnlySpan haystack, ReadOnlySpan needleLower) + { + if (haystack.Length < needleLower.Length) return false; + var last = haystack.Length - needleLower.Length; + for (var i = 0; i <= last; i++) + { + var k = 0; + for (; k < needleLower.Length; k++) + { + if (ToLower(haystack[i + k]) != needleLower[k]) break; + } + if (k == needleLower.Length) return true; + } + return false; + } + + private static byte ToLower(byte b) => b is >= (byte)'A' and <= (byte)'Z' ? (byte)(b + 32) : b; +} diff --git a/src/cdeLib/cdeLib.csproj b/src/cdeLib/cdeLib.csproj index d0cce39..eacb501 100644 --- a/src/cdeLib/cdeLib.csproj +++ b/src/cdeLib/cdeLib.csproj @@ -2,6 +2,9 @@ net10.0 + + true diff --git a/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs b/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs new file mode 100644 index 0000000..492f1c3 --- /dev/null +++ b/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs @@ -0,0 +1,136 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using cdeLib.Entities; +using cdeLib.Entities.Columnar; +using cdeLib.Entities.Soa; +using NUnit.Framework; + +namespace cdeLibTest.Columnar; + +/// +/// Proves the columnar/mmap format round-trips an and that its zero-copy +/// byte-scan search returns the same matches as the in-memory store search. This is the safety gate +/// for loading catalogs off the memory map instead of the managed heap. +/// +[TestFixture] +public class ColumnarCatalogTests +{ + // C:\test + // ├─ dir1\ (dir) + // │ ├─ alpha.txt + // │ └─ beta.log + // ├─ docs\ (dir) + // │ └─ alpha.md + // └─ root_file.txt + private static RootEntry BuildTree() + { + var root = new RootEntry + { + Path = @"C:\test", + VolumeName = "VOL", + DefaultFileName = "test.cde", + DriveLetterHint = "C", + Description = "desc", + AvailSpace = 123, + TotalSpace = 456, + }; + + var dir1 = new DirEntry(true) { Path = "dir1" }; + dir1.AddChild(new DirEntry(false) { Path = "alpha.txt" }); + dir1.AddChild(new DirEntry(false) { Path = "beta.log" }); + + var docs = new DirEntry(true) { Path = "docs" }; + docs.AddChild(new DirEntry(false) { Path = "alpha.md" }); + + root.AddChild(dir1); + root.AddChild(docs); + root.AddChild(new DirEntry(false) { Path = "root_file.txt" }); + + root.SetInMemoryFields(); + return root; + } + + private static string WriteTemp(EntryStore store) + { + var path = Path.Combine(Path.GetTempPath(), $"cdetest-{System.Guid.NewGuid():N}.cdex"); + ColumnarFormat.Write(store, path); + return path; + } + + [Test] + public void RoundTrip_PreservesCountAndMetadata() + { + var store = EntryStore.Build(BuildTree()); + var path = WriteTemp(store); + try + { + using var reader = new ColumnarCatalogReader(path); + Assert.That(reader.Count, Is.EqualTo(store.Count)); + Assert.That(reader.RootPath, Is.EqualTo(@"C:\test")); + Assert.That(reader.VolumeName, Is.EqualTo("VOL")); + Assert.That(reader.DefaultFileName, Is.EqualTo("test.cde")); + Assert.That(reader.Description, Is.EqualTo("desc")); + Assert.That(reader.AvailSpace, Is.EqualTo(123)); + Assert.That(reader.TotalSpace, Is.EqualTo(456)); + } + finally { File.Delete(path); } + } + + [Test] + public void FullPath_MatchesStore() + { + var store = EntryStore.Build(BuildTree()); + var path = WriteTemp(store); + try + { + using var reader = new ColumnarCatalogReader(path); + for (var i = 0; i < store.Count; i++) + Assert.That(reader.FullPath(i), Is.EqualTo(store.FullPath(i)), $"path mismatch at {i}"); + } + finally { File.Delete(path); } + } + + [TestCase("alpha", true, true)] // matches files in two dirs + [TestCase(".txt", true, true)] // extension match + [TestCase("DIR1", false, true)] // case-insensitive, folders + [TestCase("nope", true, true)] // no matches + public void FindName_MatchesStoreSearch(string pattern, bool files, bool folders) + { + var store = EntryStore.Build(BuildTree()); + var path = WriteTemp(store); + try + { + var expected = new List(); + EntryStoreSearch.Find(store, pattern, regexMode: false, includePath: false, + includeFiles: files, includeFolders: folders, i => expected.Add(store.FullPath(i))); + + using var reader = new ColumnarCatalogReader(path); + var actual = new List(); + reader.FindName(pattern, files, folders, i => actual.Add(reader.FullPath(i))); + + Assert.That(actual.OrderBy(x => x), Is.EqualTo(expected.OrderBy(x => x))); + } + finally { File.Delete(path); } + } + + [Test] + public void FindPath_MatchesStorePathSearch() + { + var store = EntryStore.Build(BuildTree()); + var path = WriteTemp(store); + try + { + var expected = new List(); + EntryStoreSearch.Find(store, "docs", regexMode: false, includePath: true, + includeFiles: true, includeFolders: true, i => expected.Add(store.FullPath(i))); + + using var reader = new ColumnarCatalogReader(path); + var actual = new List(); + reader.FindPath("docs", includeFiles: true, includeFolders: true, i => actual.Add(reader.FullPath(i))); + + Assert.That(actual.OrderBy(x => x), Is.EqualTo(expected.OrderBy(x => x))); + } + finally { File.Delete(path); } + } +} diff --git a/src/cdeMemProbe/Columnar/ColumnarReader.cs b/src/cdeMemProbe/Columnar/ColumnarReader.cs deleted file mode 100644 index 64d236d..0000000 --- a/src/cdeMemProbe/Columnar/ColumnarReader.cs +++ /dev/null @@ -1,190 +0,0 @@ -using System; -using System.IO.MemoryMappedFiles; -using System.Runtime.InteropServices; -using System.Text; - -namespace cdeMemProbe.Columnar; - -/// -/// SPIKE — zero-copy reader over a file. "Loading" is just mmap-ing: -/// columns are exposed as straight over the mapping, so no catalog -/// data is copied to the managed heap. A name search byte-scans the UTF-8 NameBlob in place -/// (zero managed allocation per entry), and only the pages it touches fault into the working set. -/// -public sealed unsafe class ColumnarReader : IDisposable -{ - private readonly MemoryMappedFile _mmf; - private readonly MemoryMappedViewAccessor _view; - private byte* _base; - private readonly long _length; - - private readonly long[] _off = new long[ColumnarFormat.ColumnCount]; - private readonly long[] _len = new long[ColumnarFormat.ColumnCount]; - - public int Count { get; } - public bool HasHashes { get; } - - public ColumnarReader(string path) - { - var fileLen = new System.IO.FileInfo(path).Length; - _length = fileLen; - _mmf = MemoryMappedFile.CreateFromFile(path, System.IO.FileMode.Open, mapName: null, - capacity: 0, MemoryMappedFileAccess.Read); - _view = _mmf.CreateViewAccessor(0, 0, MemoryMappedFileAccess.Read); - _view.SafeMemoryMappedViewHandle.AcquirePointer(ref _base); - - var header = new ReadOnlySpan(_base, ColumnarFormat.HeaderSize); - if (!header[..4].SequenceEqual(ColumnarFormat.Magic)) - throw new InvalidDataException($"not a CDEX file: {path}"); - var version = BitConverter.ToInt32(header.Slice(4, 4)); - if (version != ColumnarFormat.Version) - throw new InvalidDataException($"unsupported CDEX version {version}"); - Count = BitConverter.ToInt32(header.Slice(8, 4)); - HasHashes = (BitConverter.ToInt32(header.Slice(12, 4)) & ColumnarFormat.FlagHasHashes) != 0; - - var p = ColumnarFormat.PreambleFixed; - for (var c = 0; c < ColumnarFormat.ColumnCount; c++) - { - _off[c] = BitConverter.ToInt64(header.Slice(p, 8)); p += 8; - _len[c] = BitConverter.ToInt64(header.Slice(p, 8)); p += 8; - } - } - - private ReadOnlySpan Bytes(ColumnarFormat.Col col) - => new(_base + _off[(int)col], (int)_len[(int)col]); - - private ReadOnlySpan As(ColumnarFormat.Col col) where T : struct - => MemoryMarshal.Cast(Bytes(col)); - - public ReadOnlySpan Parent => As(ColumnarFormat.Col.Parent); - public ReadOnlySpan Size => As(ColumnarFormat.Col.Size); - private ReadOnlySpan NameOffsets => As(ColumnarFormat.Col.NameOffsets); - private ReadOnlySpan NameBlob => Bytes(ColumnarFormat.Col.NameBlob); - - /// UTF-8 full-name bytes of entry , sliced in place from the mapping. - public ReadOnlySpan Name(int i) - { - var offs = NameOffsets; - return NameBlob.Slice(offs[i], offs[i + 1] - offs[i]); - } - - /// - /// Zero-allocation name search: byte-scan each entry's UTF-8 name for - /// (ASCII case-insensitive). Sequentially touches only the NameBlob + NameOffsets columns. - /// - public int FindName(ReadOnlySpan patternUtf8, Action onMatch = null) - { - var offs = NameOffsets; - var blob = NameBlob; - var matches = 0; - for (var i = 0; i < Count; i++) - { - var name = blob.Slice(offs[i], offs[i + 1] - offs[i]); - if (AsciiContainsIgnoreCase(name, patternUtf8)) - { - matches++; - onMatch?.Invoke(i); - } - } - return matches; - } - - /// - /// Path search: build each entry's full path bytes into a reused buffer by walking Parent[], - /// then byte-scan. Still no per-entry managed string allocation. - /// - public int FindPath(ReadOnlySpan patternUtf8, Action onMatch = null) - { - var parent = Parent; - var offs = NameOffsets; - var blob = NameBlob; - Span chain = stackalloc int[256]; - var buf = new byte[1024]; - var matches = 0; - - for (var i = 0; i < Count; i++) - { - var depth = 0; - for (var cur = i; cur != -1 && depth < chain.Length; cur = parent[cur]) chain[depth++] = cur; - - var n = 0; - for (var k = depth - 1; k >= 0; k--) - { - if (n > 0) buf = Append(buf, ref n, (byte)'\\'); - var idx = chain[k]; - var name = blob.Slice(offs[idx], offs[idx + 1] - offs[idx]); - buf = Append(buf, ref n, name); - } - - if (AsciiContainsIgnoreCase(buf.AsSpan(0, n), patternUtf8)) - { - matches++; - onMatch?.Invoke(i); - } - } - return matches; - } - - public string FullPath(int i) - { - var parent = Parent; - var offs = NameOffsets; - var blob = NameBlob; - Span chain = stackalloc int[256]; - var depth = 0; - for (var cur = i; cur != -1 && depth < chain.Length; cur = parent[cur]) chain[depth++] = cur; - var sb = new StringBuilder(128); - for (var k = depth - 1; k >= 0; k--) - { - if (sb.Length > 0) sb.Append('\\'); - var idx = chain[k]; - sb.Append(Encoding.UTF8.GetString(blob.Slice(offs[idx], offs[idx + 1] - offs[idx]))); - } - return sb.ToString(); - } - - private static byte[] Append(byte[] buf, ref int n, byte b) - { - if (n + 1 > buf.Length) Array.Resize(ref buf, buf.Length * 2); - buf[n++] = b; - return buf; - } - - private static byte[] Append(byte[] buf, ref int n, ReadOnlySpan src) - { - while (n + src.Length > buf.Length) Array.Resize(ref buf, buf.Length * 2); - src.CopyTo(buf.AsSpan(n)); - n += src.Length; - return buf; - } - - private static bool AsciiContainsIgnoreCase(ReadOnlySpan haystack, ReadOnlySpan needle) - { - if (needle.IsEmpty) return true; - if (haystack.Length < needle.Length) return false; - var last = haystack.Length - needle.Length; - for (var i = 0; i <= last; i++) - { - var k = 0; - for (; k < needle.Length; k++) - { - if (ToLower(haystack[i + k]) != ToLower(needle[k])) break; - } - if (k == needle.Length) return true; - } - return false; - } - - private static byte ToLower(byte b) => b is >= (byte)'A' and <= (byte)'Z' ? (byte)(b + 32) : b; - - public void Dispose() - { - if (_base != null) - { - _view.SafeMemoryMappedViewHandle.ReleasePointer(); - _base = null; - } - _view?.Dispose(); - _mmf?.Dispose(); - } -} diff --git a/src/cdeMemProbe/Program.cs b/src/cdeMemProbe/Program.cs index f49e3de..5476d59 100644 --- a/src/cdeMemProbe/Program.cs +++ b/src/cdeMemProbe/Program.cs @@ -6,6 +6,7 @@ using System.Threading.Tasks; using cdeLib.Catalog; using cdeLib.Entities; +using cdeLib.Entities.Columnar; using cdeLib.Entities.Soa; using Serilog; @@ -261,7 +262,7 @@ private static int Migrate(string[] args, string? inFile, ILogger logger) } store = EntryStore.Build(root); } - Columnar.ColumnarFormat.Write(store, outFile); + ColumnarFormat.Write(store, outFile); sw.Stop(); var srcLen = new FileInfo(inFile).Length; @@ -288,7 +289,6 @@ private static int MeasureFlat(string[] args, string? flatFile) var pattern = HasFlag(args, "--pattern", out var p) && !string.IsNullOrEmpty(p) ? p! : ".txt"; var pathMode = HasFlag(args, "--path", out _); - var patternUtf8 = System.Text.Encoding.UTF8.GetBytes(pattern); // Settle, then snapshot allocation + heap baselines so we can isolate the search's own cost. GC.Collect(); @@ -297,14 +297,14 @@ private static int MeasureFlat(string[] args, string? flatFile) var heapBefore = GC.GetTotalMemory(true); var openSw = Stopwatch.StartNew(); - using var reader = new Columnar.ColumnarReader(flatFile); + using var reader = new ColumnarCatalogReader(flatFile); openSw.Stop(); var allocBefore = GC.GetTotalAllocatedBytes(precise: true); var searchSw = Stopwatch.StartNew(); var matches = pathMode - ? reader.FindPath(patternUtf8) - : reader.FindName(patternUtf8); + ? reader.FindPath(pattern, includeFiles: true, includeFolders: true) + : reader.FindName(pattern, includeFiles: true, includeFolders: true); searchSw.Stop(); var allocDuringSearch = GC.GetTotalAllocatedBytes(precise: true) - allocBefore; From 52c347d71b3c0df5f3b224b934876c58d1027757 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sun, 7 Jun 2026 08:32:08 +1000 Subject: [PATCH 21/43] feat(find): search .cdex catalogs zero-copy over mmap (promotion stage C) The find CLI now prefers the columnar format: if any .cdex catalogs exist in the current dir (or one level down) they are searched over their memory maps with no managed catalog load; otherwise it falls back to loading the .cde trees. - ColumnarCatalogReader.Find: unified pattern + name/path + file/folder search matching EntryStoreSearch semantics exactly (index 0/root skipped; substring byte-scan, regex via IgnoreCase|Singleline|Compiled decode). - ICatalogRepository.GetColumnarFileList discovers .cdex (current dir + one down), mirroring GetCacheFileList. - FindService.FindColumnar searches a set of open readers. - Program.RunFind prefers .cdex, disposes readers after. - Tests: regex parity vs EntryStoreSearch (10 columnar tests total). Verified end-to-end: find + greppath return correct nested paths from a .cdex with no .cde present. --- src/cde/Program.cs | 62 +++++++++++++------ src/cdeLib/Catalog/CatalogRepository.cs | 23 +++++++ src/cdeLib/Catalog/ICatalogRepository.cs | 6 ++ .../Columnar/ColumnarCatalogReader.cs | 40 +++++++++++- src/cdeLib/FindService.cs | 31 ++++++++++ .../Columnar/ColumnarCatalogTests.cs | 23 +++++++ 6 files changed, 163 insertions(+), 22 deletions(-) diff --git a/src/cde/Program.cs b/src/cde/Program.cs index 43abeb7..d9d9873 100644 --- a/src/cde/Program.cs +++ b/src/cde/Program.cs @@ -80,26 +80,10 @@ private static int Main(string[] args) var findService = Resolve(); var parsedResult = GetParserResult(args) .WithParsed(CreateCache) - .WithParsed(opts => - { - findService.Find(opts.Value, "--find", - Resolve().LoadCurrentDirCache()); - }) - .WithParsed(opts => - { - findService.Find(opts.Value, "--findpath", - Resolve().LoadCurrentDirCache()); - }) - .WithParsed(opts => - { - findService.Find(opts.Value, "--grep", - Resolve().LoadCurrentDirCache()); - }) - .WithParsed(opts => - { - findService.Find(opts.Value, "--greppath", - Resolve().LoadCurrentDirCache()); - }) + .WithParsed(opts => RunFind(findService, opts.Value, "--find")) + .WithParsed(opts => RunFind(findService, opts.Value, "--findpath")) + .WithParsed(opts => RunFind(findService, opts.Value, "--grep")) + .WithParsed(opts => RunFind(findService, opts.Value, "--greppath")) .WithParsed(opts => FindRepl(FindService.ParamGrepPath, opts.Value)) .WithParsed(opts => FindRepl(FindService.ParamGrep, opts.Value)) .WithParsed(opts => FindRepl(FindService.ParamFind, opts.Value)) @@ -130,6 +114,44 @@ private static T Resolve() return _container.Resolve(); } + /// + /// Run a find, preferring the zero-copy columnar format: if any .cdex catalogs exist in the + /// current dir (or one level down) they are searched over their memory maps with no managed catalog + /// load; otherwise we fall back to loading the MessagePack .cde trees. + /// + private static void RunFind(IFindService findService, string value, string param) + { + var repo = Resolve(); + var cdex = repo.GetColumnarFileList(["./"]); + if (cdex.Count == 0) + { + findService.Find(value, param, repo.LoadCurrentDirCache()); + return; + } + + var readers = new List(cdex.Count); + try + { + foreach (var file in cdex) + { + try + { + readers.Add(new ColumnarCatalogReader(file)); + } + catch (Exception ex) + { + Log.Logger.Warning(ex, "Skipping unreadable .cdex {File}", file); + } + } + + findService.FindColumnar(value, param, readers); + } + finally + { + foreach (var reader in readers) reader.Dispose(); + } + } + /// /// One-way migration of MessagePack .cde catalogs to the zero-copy columnar .cdex format. With a /// path argument, converts that file; otherwise converts every catalog discovered in the current diff --git a/src/cdeLib/Catalog/CatalogRepository.cs b/src/cdeLib/Catalog/CatalogRepository.cs index 6baec25..f567dec 100644 --- a/src/cdeLib/Catalog/CatalogRepository.cs +++ b/src/cdeLib/Catalog/CatalogRepository.cs @@ -197,6 +197,29 @@ private static IEnumerable GetCdeFiles(string path) return FileSystemHelper.GetFilesWithExtension(path, "cde"); } + public IList GetColumnarFileList(IEnumerable paths) + { + var result = new List(); + foreach (var path in paths) + { + result.AddRange(FileSystemHelper.GetFilesWithExtension(path, "cdex")); + + foreach (var childPath in Directory.GetDirectories(path)) + { + try + { + result.AddRange(FileSystemHelper.GetFilesWithExtension(childPath, "cdex")); + } + // ReSharper disable once EmptyGeneralCatchClause + catch + { + } // if cant list folders don't care. + } + } + + return result; + } + public RootEntry LoadDirCache(string file) { if (!File.Exists(file)) return null; diff --git a/src/cdeLib/Catalog/ICatalogRepository.cs b/src/cdeLib/Catalog/ICatalogRepository.cs index e9dc2e1..9d5c1e0 100644 --- a/src/cdeLib/Catalog/ICatalogRepository.cs +++ b/src/cdeLib/Catalog/ICatalogRepository.cs @@ -18,5 +18,11 @@ public interface ICatalogRepository /// IList GetCacheFileList(IEnumerable paths); + /// + /// Gets columnar .cdex catalogs in the current dir or one directory down — the zero-copy + /// mmap format produced by cde migrate. Mirrors for .cde. + /// + IList GetColumnarFileList(IEnumerable paths); + RootEntry LoadDirCache(string file); } \ No newline at end of file diff --git a/src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs b/src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs index e7d4659..df3a3e4 100644 --- a/src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs +++ b/src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs @@ -3,6 +3,7 @@ using System.IO.MemoryMappedFiles; using System.Runtime.InteropServices; using System.Text; +using System.Text.RegularExpressions; using cdeLib.Entities.Soa; namespace cdeLib.Entities.Columnar; @@ -111,6 +112,41 @@ public ReadOnlySpan NameUtf8(int i) public string Name(int i) => Encoding.UTF8.GetString(NameUtf8(i)); + /// + /// Find matching the production semantics exactly: pattern + + /// name/path + file/folder filter, index 0 (root) never a result. Substring matching byte-scans + /// the mapping (zero-alloc ASCII path); regex decodes per entry like the store search does. + /// + public int Find(string pattern, bool regexMode, bool includePath, bool includeFiles, + bool includeFolders, Action onMatch = null) + { + if (!includeFiles && !includeFolders) return 0; + if (regexMode && !string.IsNullOrEmpty(pattern)) + return FindRegex(pattern, includePath, includeFiles, includeFolders, onMatch); + return includePath + ? FindPath(pattern, includeFiles, includeFolders, onMatch) + : FindName(pattern, includeFiles, includeFolders, onMatch); + } + + private int FindRegex(string pattern, bool includePath, bool includeFiles, bool includeFolders, + Action onMatch) + { + var regex = new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.Compiled); + var bits = BitFields; + var matches = 0; + for (var i = 1; i < Count; i++) + { + if (!Wanted(bits[i], includeFiles, includeFolders)) continue; + var text = includePath ? FullPath(i) : Name(i); + if (regex.IsMatch(text)) + { + matches++; + onMatch?.Invoke(i); + } + } + return matches; + } + /// /// Name search: byte-scan each entry's UTF-8 name for (ordinal, /// case-insensitive). Sequentially touches only the NameBlob + NameOffsets columns. Zero managed @@ -123,7 +159,7 @@ public int FindName(string pattern, bool includeFiles, bool includeFolders, Acti var blob = NameBlob; var bits = BitFields; var matches = 0; - for (var i = 0; i < Count; i++) + for (var i = 1; i < Count; i++) // index 0 is the root, never a result { if (!Wanted(bits[i], includeFiles, includeFolders)) continue; var name = blob.Slice((int)offs[i], (int)(offs[i + 1] - offs[i])); @@ -151,7 +187,7 @@ public int FindPath(string pattern, bool includeFiles, bool includeFolders, Acti var buf = new byte[1024]; var matches = 0; - for (var i = 0; i < Count; i++) + for (var i = 1; i < Count; i++) // index 0 is the root, never a result { if (!Wanted(bits[i], includeFiles, includeFolders)) continue; diff --git a/src/cdeLib/FindService.cs b/src/cdeLib/FindService.cs index 176cde0..e0e80f6 100644 --- a/src/cdeLib/FindService.cs +++ b/src/cdeLib/FindService.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.Threading.Tasks; using cdeLib.Entities; +using cdeLib.Entities.Columnar; using cdeLib.Entities.Soa; using Serilog; @@ -12,6 +13,12 @@ public interface IFindService { void Find(string pattern, string param, IList rootEntries); void Find(string pattern, bool regexMode, bool includePath, IList rootEntries); + + /// + /// Search columnar .cdex catalogs zero-copy over their memory maps (no managed catalog + /// load). Mirrors result semantics. + /// + void FindColumnar(string pattern, string param, IList readers); Task FindAsync(string pattern, string param, IList rootEntries); Task FindAsync(string pattern, bool regexMode, bool includePath, IList rootEntries); @@ -76,6 +83,30 @@ public void Find(string pattern, bool regexMode, bool includePath, IList readers) + { + var regexMode = param is ParamGrep or ParamGrepPath; + var includePath = param is ParamGrepPath or ParamFindPath; + + var totalFound = 0L; + var timer = Stopwatch.StartNew(); + foreach (var reader in readers) + { + if (reader == null) continue; + reader.Find(pattern, regexMode, includePath, IncludeFiles, IncludeFolders, + idx => + { + ++totalFound; + Console.WriteLine(" {0}", reader.FullPath(idx)); + }); + } + + timer.Stop(); + Log.Logger.Information( + "Search Execution Time: {ExecutionTime}, Matching pattern {Pattern}, Total found {TotalFound}", + timer.ElapsedMilliseconds, pattern, totalFound); + } + public Task FindAsync(string pattern, string param, IList rootEntries) { var regexMode = param is ParamGrep or ParamGrepPath; diff --git a/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs b/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs index 492f1c3..a421dec 100644 --- a/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs +++ b/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs @@ -114,6 +114,29 @@ public void FindName_MatchesStoreSearch(string pattern, bool files, bool folders finally { File.Delete(path); } } + [TestCase(@"alpha\.txt", false)] // regex name + [TestCase("beta", false)] + [TestCase("alpha", true)] // regex on full path + public void Find_Regex_MatchesStoreSearch(string pattern, bool includePath) + { + var store = EntryStore.Build(BuildTree()); + var path = WriteTemp(store); + try + { + var expected = new List(); + EntryStoreSearch.Find(store, pattern, regexMode: true, includePath: includePath, + includeFiles: true, includeFolders: true, i => expected.Add(store.FullPath(i))); + + using var reader = new ColumnarCatalogReader(path); + var actual = new List(); + reader.Find(pattern, regexMode: true, includePath: includePath, + includeFiles: true, includeFolders: true, i => actual.Add(reader.FullPath(i))); + + Assert.That(actual.OrderBy(x => x), Is.EqualTo(expected.OrderBy(x => x))); + } + finally { File.Delete(path); } + } + [Test] public void FindPath_MatchesStorePathSearch() { From 164349c732309c93bdc5bc42efd98f5fd418d8f6 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sun, 7 Jun 2026 08:40:12 +1000 Subject: [PATCH 22/43] refactor(soa): unify EntryStore + mmap reader behind IEntrySource (stage D foundation) Introduce IEntrySource as the common read-only, index-addressed catalog surface over both the in-memory EntryStore and the zero-copy ColumnarCatalogReader, so EntryRef (and the GUI) work over either backing without caring whether the catalog is on the heap or in a memory map. - IEntrySource: per-entry accessors + full-filter search + catalog metadata. - EntryStore implements it (metadata fields -> properties; thin wrappers; instance Find delegates to EntryStoreSearch). - ColumnarCatalogReader implements it: child/sibling/hash column accessors, AppendFullPath, and a full-filter Find (size/date/hour + cancel/scan) matching EntryStoreSearch semantics exactly over the mapping. - EntryRef now backs onto IEntrySource (was EntryStore); exposes Source. - Tests: full-filter size-range parity + EntryRef navigating over a reader (12 columnar tests). 155 lib tests green. Pure cdeLib refactor; GUI rewiring to hold readers follows next. --- .../Columnar/ColumnarCatalogReader.cs | 142 +++++++++++++++++- src/cdeLib/Entities/IEntrySource.cs | 57 +++++++ src/cdeLib/Entities/Soa/EntryRef.cs | 80 +++++----- src/cdeLib/Entities/Soa/EntryStore.cs | 45 ++++-- .../Columnar/ColumnarCatalogTests.cs | 61 +++++++- 5 files changed, 327 insertions(+), 58 deletions(-) create mode 100644 src/cdeLib/Entities/IEntrySource.cs diff --git a/src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs b/src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs index df3a3e4..5bb74a3 100644 --- a/src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs +++ b/src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.IO.MemoryMappedFiles; using System.Runtime.InteropServices; @@ -18,7 +19,7 @@ namespace cdeLib.Entities.Columnar; /// The mapping stays open for the lifetime of the reader; hold it for as long as the catalog is /// "loaded" and it on reload/exit. The file is read-only. /// -public sealed unsafe class ColumnarCatalogReader : IDisposable +public sealed unsafe class ColumnarCatalogReader : IEntrySource, IDisposable { private readonly MemoryMappedFile _mmf; private readonly MemoryMappedViewAccessor _view; @@ -97,12 +98,62 @@ private ReadOnlySpan As(ColumnarFormat.Col col) where T : struct public ReadOnlySpan ModifiedTicks => As(ColumnarFormat.Col.ModifiedTicks); public ReadOnlySpan BitFields => Bytes(ColumnarFormat.Col.BitFields); public ReadOnlySpan Parent => As(ColumnarFormat.Col.Parent); + private ReadOnlySpan FirstChild => As(ColumnarFormat.Col.FirstChild); + private ReadOnlySpan NextSibling => As(ColumnarFormat.Col.NextSibling); + private ReadOnlySpan HashBytes => Bytes(ColumnarFormat.Col.Hash); private ReadOnlySpan NameOffsets => As(ColumnarFormat.Col.NameOffsets); private ReadOnlySpan NameBlob => Bytes(ColumnarFormat.Col.NameBlob); public Flags Flags(int i) => (Flags)BitFields[i]; public bool IsDirectory(int i) => (Flags(i) & Entities.Flags.Directory) == Entities.Flags.Directory; + // ----- IEntrySource: index-addressed accessors straight over the mapping ----- + public long SizeOf(int i) => Size[i]; + public DateTime ModifiedOf(int i) => DateTime.FromBinary(ModifiedTicks[i]); + public Flags FlagsOf(int i) => Flags(i); + public bool IsHashDone(int i) => (Flags(i) & Entities.Flags.HashDone) == Entities.Flags.HashDone; + public bool IsPartialHash(int i) => (Flags(i) & Entities.Flags.PartialHash) == Entities.Flags.PartialHash; + public bool HasHash => HasHashes; + public Hash16 HashOf(int i) => + HasHashes ? MemoryMarshal.Read(HashBytes.Slice(i * 16, 16)) : default; + public string FullName(int i) => Name(i); + public string NameOf(int i) => Name(i); // full name (not split) — fine for path-problem trailing checks + public int ParentOf(int i) => Parent[i]; + public int FirstChildOf(int i) => FirstChild[i]; + + public IEnumerable ChildrenOf(int parent) + { + // Materialise into a list (no yield): the sibling chain reads spans over the mapping, which a + // lazy iterator's state machine can't hold. Per-directory child fan-out is small. + var result = new List(); + var first = FirstChild[parent]; + if (first == EntryStore.None) return result; + var sib = NextSibling; + for (var c = first; c != EntryStore.None; c = sib[c]) result.Add(c); + return result; + } + + public void AppendFullPath(StringBuilder sb, int i) + { + var parent = Parent; + var offs = NameOffsets; + var blob = NameBlob; + Span chain = stackalloc int[256]; + var depth = 0; + for (var cur = i; cur != EntryStore.None && depth < chain.Length; cur = parent[cur]) + chain[depth++] = cur; + for (var k = depth - 1; k >= 0; k--) + { + if (sb.Length > 0) + { + var last = sb[^1]; + if (last != '\\' && last != '/') sb.Append(Path.DirectorySeparatorChar); + } + var idx = chain[k]; + sb.Append(Encoding.UTF8.GetString(blob.Slice((int)offs[idx], (int)(offs[idx + 1] - offs[idx])))); + } + } + /// UTF-8 full-name bytes of entry , sliced in place from the mapping. public ReadOnlySpan NameUtf8(int i) { @@ -128,6 +179,95 @@ public int Find(string pattern, bool regexMode, bool includePath, bool includeFi : FindName(pattern, includeFiles, includeFolders, onMatch); } + /// + /// Full-filter search (pattern + name/path + file/folder + size/date/hour ranges) matching the GUI + /// + /// semantics exactly, evaluated zero-copy over the mapping. + /// + public void Find(EntryStoreFindOptions o, Action onMatch, + Func isCancelled = null, Action onScan = null) + { + ArgumentNullException.ThrowIfNull(o); + ArgumentNullException.ThrowIfNull(onMatch); + if (!o.IncludeFiles && !o.IncludeFolders) return; + + var hasPattern = !string.IsNullOrEmpty(o.Pattern); + var regex = o.RegexMode && hasPattern + ? new Regex(o.Pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.Compiled) + : null; + var matcher = hasPattern && !o.RegexMode ? new Utf8Matcher(o.Pattern) : default; + + var size = Size; + var bits = BitFields; + var offs = NameOffsets; + var blob = NameBlob; + var pathBuf = o.IncludePath ? new byte[1024] : null; + Span chain = o.IncludePath ? stackalloc int[256] : default; + + for (var i = 1; i < Count; i++) // index 0 is the root, never a result + { + if ((i & 4095) == 0) + { + if (isCancelled != null && isCancelled()) return; + onScan?.Invoke(i); + } + + var isDir = ((Flags)bits[i] & Entities.Flags.Directory) == Entities.Flags.Directory; + if (isDir ? !o.IncludeFolders : !o.IncludeFiles) continue; + + if (o.FromSizeEnable && size[i] < o.FromSize) continue; + if (o.ToSizeEnable && size[i] > o.ToSize) continue; + + if (o.FromDateEnable || o.ToDateEnable || o.FromHourEnable || o.ToHourEnable || o.NotOlderThanEnable) + { + var modified = ModifiedOf(i); + if (o.FromDateEnable && modified < o.FromDate) continue; + if (o.ToDateEnable && modified > o.ToDate) continue; + if (o.NotOlderThanEnable && modified < o.NotOlderThan) continue; + var tod = modified.TimeOfDay; + if (o.FromHourEnable && tod < o.FromHour) continue; + if (o.ToHourEnable && tod > o.ToHour) continue; + } + + if (!hasPattern) { onMatch(i); continue; } + + bool match; + if (o.RegexMode) + { + match = regex.IsMatch(o.IncludePath ? FullPath(i) : Name(i)); + } + else if (o.IncludePath) + { + var n = BuildPathBytes(ref pathBuf, chain, i, offs, blob); + match = matcher.Contains(pathBuf.AsSpan(0, n)); + } + else + { + match = matcher.Contains(blob.Slice((int)offs[i], (int)(offs[i + 1] - offs[i]))); + } + + if (match) onMatch(i); + } + } + + // Build entry i's full-path UTF-8 bytes into buf (grown as needed); returns the byte length. + private int BuildPathBytes(ref byte[] buf, Span chain, int i, + ReadOnlySpan offs, ReadOnlySpan blob) + { + var parent = Parent; + var depth = 0; + for (var cur = i; cur != EntryStore.None && depth < chain.Length; cur = parent[cur]) + chain[depth++] = cur; + var n = 0; + for (var k = depth - 1; k >= 0; k--) + { + if (n > 0) buf = Append(buf, ref n, (byte)Path.DirectorySeparatorChar); + var idx = chain[k]; + buf = Append(buf, ref n, blob.Slice((int)offs[idx], (int)(offs[idx + 1] - offs[idx]))); + } + return n; + } + private int FindRegex(string pattern, bool includePath, bool includeFiles, bool includeFolders, Action onMatch) { diff --git a/src/cdeLib/Entities/IEntrySource.cs b/src/cdeLib/Entities/IEntrySource.cs new file mode 100644 index 0000000..6b1d0bf --- /dev/null +++ b/src/cdeLib/Entities/IEntrySource.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Text; +using cdeLib.Entities.Soa; + +namespace cdeLib.Entities; + +/// +/// A read-only, index-addressed catalog: the common surface over both the in-memory +/// (built from a loaded .cde tree) and the zero-copy +/// (mmap over a .cdex file). Lets the GUI and +/// present and search a catalog without caring whether it lives on the managed +/// heap or in a memory map. Index 0 is always the catalog root; child/sibling/parent chains terminate +/// at . +/// +public interface IEntrySource +{ + int Count { get; } + + // ----- per-entry accessors ----- + long SizeOf(int i); + DateTime ModifiedOf(int i); + Flags FlagsOf(int i); + bool IsDirectory(int i); + bool IsHashDone(int i); + bool IsPartialHash(int i); + bool HasHash { get; } + Hash16 HashOf(int i); + + string FullName(int i); // name + extension + string NameOf(int i); // name component used for path-problem checks (== FullName when not split) + string FullPath(int i); + void AppendFullPath(StringBuilder sb, int i); + + int ParentOf(int i); + int FirstChildOf(int i); + IEnumerable ChildrenOf(int i); + + // ----- full-filter search (pattern + name/path + file/folder + size/date/hour) ----- + void Find(EntryStoreFindOptions options, Action onMatch, + Func isCancelled = null, Action onScan = null); + + // ----- catalog-level metadata (what the GUI catalog list + result rows display) ----- + string RootPath { get; } + string VolumeName { get; } + string DefaultFileName { get; } + string ActualFileName { get; } + string DriveLetterHint { get; } + string Description { get; } + long AvailSpace { get; } + long TotalSpace { get; } + long ScanStartUtcTicks { get; } + long ScanEndUtcTicks { get; } + long RootSize { get; } + uint RootFileEntryCount { get; } + uint RootDirEntryCount { get; } +} diff --git a/src/cdeLib/Entities/Soa/EntryRef.cs b/src/cdeLib/Entities/Soa/EntryRef.cs index 254b2de..9a357e1 100644 --- a/src/cdeLib/Entities/Soa/EntryRef.cs +++ b/src/cdeLib/Entities/Soa/EntryRef.cs @@ -5,66 +5,68 @@ namespace cdeLib.Entities.Soa; /// -/// Lightweight adapter presenting a single entry (by index) as an -/// , so existing tree-oriented consumers (GUI display, navigation, -/// dupes read paths) can run on the struct-of-arrays model without materialising the whole -/// pointer tree. Read members map onto the store's arrays; build/mutate members throw, since a -/// store is produced wholesale by the loader, not edited entry-by-entry. +/// Lightweight adapter presenting a single catalog entry (by index) as an , +/// so existing tree-oriented consumers (GUI display, navigation, dupes read paths) can run on the +/// index-addressed model without materialising a pointer tree. Backs onto any +/// — the in-memory or the zero-copy +/// — so the GUI is agnostic to whether the catalog lives on the heap or in a memory map. Read members +/// map onto the source; build/mutate members throw, since a catalog is produced wholesale by the +/// loader, not edited entry-by-entry. /// -/// Intended for OCCASIONAL access (displaying a directory, a search result row). Bulk traversal -/// should use index-based APIs () to avoid per-entry wrapper allocs. +/// Intended for OCCASIONAL access (displaying a directory, a search result row). Bulk traversal should +/// use index-based search () to avoid per-entry wrapper allocs. /// public sealed class EntryRef : ICommonEntry { - private readonly EntryStore _store; + private readonly IEntrySource _source; private readonly int _index; - public EntryRef(EntryStore store, int index) + public EntryRef(IEntrySource source, int index) { - _store = store; + _source = source; _index = index; } - public EntryStore Store => _store; + public IEntrySource Source => _source; public int Index => _index; private static NotSupportedException ReadOnly([System.Runtime.CompilerServices.CallerMemberName] string m = null) - => new($"EntryRef is a read-only view over EntryStore; '{m}' is not supported."); + => new($"EntryRef is a read-only view over a catalog source; '{m}' is not supported."); - public string Path { get => _store.FullName(_index); set => throw ReadOnly(); } - public long Size { get => _store.Size[_index]; set => throw ReadOnly(); } - public DateTime Modified { get => _store.Modified(_index); set => throw ReadOnly(); } + public string Path { get => _source.FullName(_index); set => throw ReadOnly(); } + public long Size { get => _source.SizeOf(_index); set => throw ReadOnly(); } + public DateTime Modified { get => _source.ModifiedOf(_index); set => throw ReadOnly(); } - public bool IsDirectory { get => _store.IsDirectory(_index); set => throw ReadOnly(); } - public bool IsHashDone { get => _store.IsHashDone(_index); set => throw ReadOnly(); } - public bool IsPartialHash { get => _store.IsPartialHash(_index); set => throw ReadOnly(); } + public bool IsDirectory { get => _source.IsDirectory(_index); set => throw ReadOnly(); } + public bool IsHashDone { get => _source.IsHashDone(_index); set => throw ReadOnly(); } + public bool IsPartialHash { get => _source.IsPartialHash(_index); set => throw ReadOnly(); } public bool IsModifiedBad { - get => (_store.Flags(_index) & Flags.ModifiedBad) == Flags.ModifiedBad; + get => (_source.FlagsOf(_index) & Flags.ModifiedBad) == Flags.ModifiedBad; set => throw ReadOnly(); } public bool IsReparsePoint { - get => (_store.Flags(_index) & Flags.ReparsePoint) == Flags.ReparsePoint; + get => (_source.FlagsOf(_index) & Flags.ReparsePoint) == Flags.ReparsePoint; set => throw ReadOnly(); } - public bool IsDefaultSort { get => true; set => throw ReadOnly(); } // store is built in sorted order + public bool IsDefaultSort { get => true; set => throw ReadOnly(); } // source is built in sorted order public Hash16 Hash { - get => _store.Hash != null ? _store.Hash[_index] : default; + get => _source.HashOf(_index); set => throw ReadOnly(); } - public string FullPath => _store.FullPath(_index); + public string FullPath => _source.FullPath(_index); public bool PathProblem { get { - for (var cur = _index; cur != EntryStore.None; cur = _store.Parent[cur]) + for (var cur = _index; cur != EntryStore.None; cur = _source.ParentOf(cur)) { - var name = _store.Name[cur]; + var name = _source.NameOf(cur); if (!string.IsNullOrEmpty(name) && (name.EndsWith(' ') || name.EndsWith('.'))) return true; } return false; @@ -77,11 +79,11 @@ public IReadOnlyList Children { // Gate on having children, not on the directory flag: the root is not flagged a // directory yet has children (matching RootEntry), and a file simply has none. - if (_store.FirstChild[_index] == EntryStore.None) return null; + if (_source.FirstChildOf(_index) == EntryStore.None) return null; List list = null; - foreach (var c in _store.Children(_index)) + foreach (var c in _source.ChildrenOf(_index)) { - (list ??= new List()).Add(new EntryRef(_store, c)); + (list ??= new List()).Add(new EntryRef(_source, c)); } return list; } @@ -91,8 +93,8 @@ public ICommonEntry ParentCommonEntry { get { - var p = _store.Parent[_index]; - return p == EntryStore.None ? null : new EntryRef(_store, p); + var p = _source.ParentOf(_index); + return p == EntryStore.None ? null : new EntryRef(_source, p); } set => throw ReadOnly(); } @@ -108,9 +110,9 @@ public ICommonEntry ParentCommonEntry while (stack.Count > 0) { var n = stack.Pop(); - foreach (var c in _store.Children(n)) + foreach (var c in _source.ChildrenOf(n)) { - if (_store.IsDirectory(c)) { dirs++; stack.Push(c); } + if (_source.IsDirectory(c)) { dirs++; stack.Push(c); } else files++; } } @@ -155,9 +157,9 @@ public string MakeFullPath(ICommonEntry dirEntry) public IList GetListFromRoot() { var list = new List(8); - for (var cur = _index; cur != EntryStore.None; cur = _store.Parent[cur]) + for (var cur = _index; cur != EntryStore.None; cur = _source.ParentOf(cur)) { - list.Add(new EntryRef(_store, cur)); + list.Add(new EntryRef(_source, cur)); } list.Reverse(); return list; @@ -173,16 +175,16 @@ public void TraverseTreePair(TraverseFunc func) while (stack.Count > 0) { var n = stack.Pop(); - var parentRef = new EntryRef(_store, n); - foreach (var c in _store.Children(n)) + var parentRef = new EntryRef(_source, n); + foreach (var c in _source.ChildrenOf(n)) { - if (!func(parentRef, new EntryRef(_store, c))) return; - if (_store.IsDirectory(c)) stack.Push(c); + if (!func(parentRef, new EntryRef(_source, c))) return; + if (_source.IsDirectory(c)) stack.Push(c); } } } - // ----- build / mutate members: not supported on a read-only store view ----- + // ----- build / mutate members: not supported on a read-only source view ----- public void AddChild(DirEntry child) => throw ReadOnly(); public void SetSummaryFields() => throw ReadOnly(); public void SetHash(byte[] hashResponseHash) => throw ReadOnly(); diff --git a/src/cdeLib/Entities/Soa/EntryStore.cs b/src/cdeLib/Entities/Soa/EntryStore.cs index 35e2d76..8842415 100644 --- a/src/cdeLib/Entities/Soa/EntryStore.cs +++ b/src/cdeLib/Entities/Soa/EntryStore.cs @@ -18,7 +18,7 @@ namespace cdeLib.Entities.Soa; /// Index 0 is always the catalog root. (-1) terminates child/sibling chains and /// marks the root's (absent) parent. /// -public sealed class EntryStore +public sealed class EntryStore : IEntrySource { public const int None = -1; @@ -47,19 +47,19 @@ public sealed class EntryStore public Hash16[] Hash { get; private set; } // ----- catalog-level metadata (what the GUI catalog list and search-result rows display) ----- - public string RootPath; // root path, e.g. C:\ (also Name[0]) - public string VolumeName; - public string DefaultFileName; // generated .cde name - public string ActualFileName; // path of the loaded .cde - public string DriveLetterHint; - public string Description; - public long AvailSpace; - public long TotalSpace; - public long ScanStartUtcTicks; - public long ScanEndUtcTicks; - public long RootSize; // total size of the catalog - public uint RootFileEntryCount; // total files in the catalog - public uint RootDirEntryCount; // total directories in the catalog + public string RootPath { get; set; } // root path, e.g. C:\ (also Name[0]) + public string VolumeName { get; set; } + public string DefaultFileName { get; set; } // generated .cde name + public string ActualFileName { get; set; } // path of the loaded .cde + public string DriveLetterHint { get; set; } + public string Description { get; set; } + public long AvailSpace { get; set; } + public long TotalSpace { get; set; } + public long ScanStartUtcTicks { get; set; } + public long ScanEndUtcTicks { get; set; } + public long RootSize { get; set; } // total size of the catalog + public uint RootFileEntryCount { get; set; } // total files in the catalog + public uint RootDirEntryCount { get; set; } // total directories in the catalog private EntryStore(int count) { @@ -78,6 +78,23 @@ private EntryStore(int count) public string FullName(int i) => string.IsNullOrEmpty(Ext[i]) ? Name[i] : string.Concat(Name[i], Ext[i]); public Flags Flags(int i) => (Flags)BitFields[i]; + + // ----- IEntrySource: thin index-addressed accessors over the parallel arrays ----- + public long SizeOf(int i) => Size[i]; + public DateTime ModifiedOf(int i) => Modified(i); + public Flags FlagsOf(int i) => Flags(i); + public bool HasHash => Hash != null; + public Hash16 HashOf(int i) => Hash != null ? Hash[i] : default; + public string NameOf(int i) => Name[i]; + public int ParentOf(int i) => Parent[i]; + public int FirstChildOf(int i) => FirstChild[i]; + public IEnumerable ChildrenOf(int i) => Children(i); + + /// Full-filter search over this store (delegates to ). + public void Find(EntryStoreFindOptions options, Action onMatch, + Func isCancelled = null, Action onScan = null) + => EntryStoreSearch.Find(this, options, onMatch, isCancelled, onScan); + public bool IsDirectory(int i) => (Flags(i) & Entities.Flags.Directory) == Entities.Flags.Directory; public bool IsHashDone(int i) => (Flags(i) & Entities.Flags.HashDone) == Entities.Flags.HashDone; public bool IsPartialHash(int i) => (Flags(i) & Entities.Flags.PartialHash) == Entities.Flags.PartialHash; diff --git a/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs b/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs index a421dec..e2f97bd 100644 --- a/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs +++ b/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs @@ -37,15 +37,15 @@ private static RootEntry BuildTree() }; var dir1 = new DirEntry(true) { Path = "dir1" }; - dir1.AddChild(new DirEntry(false) { Path = "alpha.txt" }); - dir1.AddChild(new DirEntry(false) { Path = "beta.log" }); + dir1.AddChild(new DirEntry(false) { Path = "alpha.txt", Size = 100 }); + dir1.AddChild(new DirEntry(false) { Path = "beta.log", Size = 5000 }); var docs = new DirEntry(true) { Path = "docs" }; - docs.AddChild(new DirEntry(false) { Path = "alpha.md" }); + docs.AddChild(new DirEntry(false) { Path = "alpha.md", Size = 200 }); root.AddChild(dir1); root.AddChild(docs); - root.AddChild(new DirEntry(false) { Path = "root_file.txt" }); + root.AddChild(new DirEntry(false) { Path = "root_file.txt", Size = 50 }); root.SetInMemoryFields(); return root; @@ -137,6 +137,59 @@ public void Find_Regex_MatchesStoreSearch(string pattern, bool includePath) finally { File.Delete(path); } } + [Test] + public void Find_FullFilter_SizeRange_MatchesStore() + { + var store = EntryStore.Build(BuildTree()); + var path = WriteTemp(store); + try + { + var opts = new EntryStoreFindOptions + { + IncludeFiles = true, + IncludeFolders = true, + FromSizeEnable = true, + FromSize = 1000, // only beta.log (5000) qualifies + }; + + var expected = new List(); + EntryStoreSearch.Find(store, opts, i => expected.Add(store.FullPath(i))); + + using var reader = new ColumnarCatalogReader(path); + var actual = new List(); + reader.Find(opts, i => actual.Add(reader.FullPath(i))); + + // Reader and store must agree exactly (both include dir1, whose aggregated size >= 1000). + Assert.That(actual.OrderBy(x => x), Is.EqualTo(expected.OrderBy(x => x))); + Assert.That(actual, Does.Contain(@"C:\test\dir1\beta.log")); + Assert.That(actual, Does.Not.Contain(@"C:\test\root_file.txt")); // size 50, filtered out + } + finally { File.Delete(path); } + } + + [Test] + public void EntryRef_OverReader_NavigatesLikeStore() + { + var store = EntryStore.Build(BuildTree()); + var path = WriteTemp(store); + try + { + using var reader = new ColumnarCatalogReader(path); + // Same ICommonEntry adapter, backed by the mmap reader instead of the heap store. + ICommonEntry rootRef = new EntryRef(reader, 0); + ICommonEntry storeRootRef = new EntryRef(store, 0); + + Assert.That(rootRef.Children, Is.Not.Null); + Assert.That(rootRef.Children.Count, Is.EqualTo(storeRootRef.Children.Count)); + Assert.That(rootRef.FullPath, Is.EqualTo(storeRootRef.FullPath)); + + // Subtree counts (files/dirs) must match the heap-backed adapter. + Assert.That(rootRef.FileEntryCount, Is.EqualTo(storeRootRef.FileEntryCount)); + Assert.That(rootRef.DirEntryCount, Is.EqualTo(storeRootRef.DirEntryCount)); + } + finally { File.Delete(path); } + } + [Test] public void FindPath_MatchesStorePathSearch() { From 017a29e7c7c69ee23be6295b55efebf37156ce29 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sun, 7 Jun 2026 08:47:10 +1000 Subject: [PATCH 23/43] feat(cdeWin): hold catalogs as mmap .cdex readers, preferring zero-copy load (stage D) cdeWin now memory-maps columnar .cdex catalogs instead of loading .cde trees into the managed heap, for the 100-catalog low-memory case. - Catalogs held as IEntrySource via EntryRef: a ColumnarCatalogReader (mmap) when .cdex exist, else an EntryStore built from the .cde tree. - LoadCatalogService.GetColumnarFiles discovers .cdex (current dir + config path); presenter prefers it and only falls back to .cde load when none are found. - Search runs via source.Find (full size/date/hour filter parity); result pairs are EntryRefs over the source. - mmap sources disposed on reload and on form close (mappings released). - StoreOf/StoreOfPair -> SourceOf/SourceOfPair (IEntrySource); SameRoot compares sources. - Test: with a real migrated .cdex present, the presenter loads via mmap, does NOT call the .cde loader, and releases the mapping on close (33 presenter tests pass). Validated via build + unit tests only; the live WinForms UI was not run here and needs manual verification (catalog list, search, navigation, reload, exit). --- src/cdeWin/CDEWinFormPresenter.cs | 95 ++++++++++++++++------- src/cdeWin/LoadCatalogService.cs | 13 ++++ src/cdeWinTest/CDEWinFormPresenterTest.cs | 27 +++++++ 3 files changed, 108 insertions(+), 27 deletions(-) diff --git a/src/cdeWin/CDEWinFormPresenter.cs b/src/cdeWin/CDEWinFormPresenter.cs index f5faf96..f6c6e67 100644 --- a/src/cdeWin/CDEWinFormPresenter.cs +++ b/src/cdeWin/CDEWinFormPresenter.cs @@ -10,6 +10,7 @@ using System.Windows.Forms; using cdeLib; using cdeLib.Entities; +using cdeLib.Entities.Columnar; using cdeLib.Entities.Soa; using cdeLib.Infrastructure; using cdeWin.Cfg; @@ -29,8 +30,10 @@ public class CDEWinFormPresenter : Presenter, ICDEWinFormPresenter private readonly ICDEWinForm _clientForm; - // Catalogs are held as struct-of-arrays EntryStores (≈1/3 the memory of the pointer tree); each - // catalog root is exposed as an EntryRef so the existing ICommonEntry-based GUI works unchanged. + // Catalogs are held as IEntrySource — either a zero-copy ColumnarCatalogReader over a memory-mapped + // .cdex file (preferred: the catalog data stays in the OS page cache, not the managed heap) or, when + // no .cdex exists, an in-memory EntryStore built from the loaded .cde tree. Each catalog root is + // exposed as an EntryRef so the existing ICommonEntry-based GUI works unchanged either way. private List _catalogRoots; private readonly IConfig _config; @@ -46,10 +49,53 @@ private static List ToCatalogRoots(List trees) return roots; } - private static EntryStore StoreOf(ICommonEntry root) => ((EntryRef)root).Store; + // Open each .cdex as a zero-copy mmap reader. Unreadable files are skipped (logged by the caller). + private static List ReadersToCatalogRoots(IList cdexFiles) + { + var roots = new List(cdexFiles.Count); + foreach (var file in cdexFiles) + { + try + { + roots.Add(new EntryRef(new ColumnarCatalogReader(file), 0)); + } + catch (Exception ex) + { + Log.Logger.Warning(ex, "Skipping unreadable .cdex {File}", file); + } + } + return roots; + } + + private static IEntrySource SourceOf(ICommonEntry root) => ((EntryRef)root).Source; - // Catalog of a search-result pair (its entries are EntryRefs into a store). - private static EntryStore StoreOfPair(PairDirEntry pde) => (pde.ChildDE as EntryRef)?.Store; + // Catalog of a search-result pair (its entries are EntryRefs into a source). + private static IEntrySource SourceOfPair(PairDirEntry pde) => (pde.ChildDE as EntryRef)?.Source; + + // Prefer the zero-copy .cdex catalogs (mmap, near-zero managed heap); fall back to loading the + // .cde trees and building in-memory stores when no .cdex exists. Disposes any previously held + // mmap sources first so reloads don't leak mappings. + private async Task> LoadCatalogRootsAsync() + { + DisposeCatalogSources(); + var cdex = _loadCatalogService.GetColumnarFiles(_config); + if (cdex is { Count: > 0 }) + { + return ReadersToCatalogRoots(cdex); + } + return ToCatalogRoots(await _loadCatalogService.LoadRootEntriesAsync( + _config, OnLoadProgress, _loadingCts.Token)); + } + + // Memory-mapped catalog sources must be released on reload/exit so the mappings are closed. + private void DisposeCatalogSources() + { + if (_catalogRoots == null) return; + foreach (var root in _catalogRoots) + { + if (root is EntryRef { Source: IDisposable disposable }) disposable.Dispose(); + } + } private readonly string[] _directoryVals; private readonly string[] _searchVals; @@ -123,10 +169,7 @@ public async Task InitializeAsync() try { - _catalogRoots = ToCatalogRoots(await _loadCatalogService.LoadRootEntriesAsync( - _config, - OnLoadProgress, - _loadingCts.Token)); + _catalogRoots = await LoadCatalogRootsAsync(); SetCatalogListView(); SetMemoryStatus(); @@ -219,7 +262,7 @@ private void SetCatalogListView() catalogHelper.SortList(); _clientForm.SetCatalogsLoadedStatus(count); _clientForm.SetTotalFileEntriesLoadedStatus( - (int)_catalogRoots.Sum(r => (long)StoreOf(r).RootFileEntryCount + StoreOf(r).RootDirEntryCount)); + (int)_catalogRoots.Sum(r => (long)SourceOf(r).RootFileEntryCount + SourceOf(r).RootDirEntryCount)); } private static double BytesToMb(long bytes) => bytes / (1024.0 * 1024.0); @@ -344,7 +387,7 @@ public void CatalogRetrieveVirtualItem() private Color CreateRowValuesForRootEntry(IList vals, ICommonEntry catalogRoot, Color listViewForeColor) { - var s = StoreOf(catalogRoot); + var s = SourceOf(catalogRoot); var scanStart = new DateTime(s.ScanStartUtcTicks, DateTimeKind.Utc); var scanDurationMs = (s.ScanEndUtcTicks - s.ScanStartUtcTicks) / TimeSpan.TicksPerMillisecond; vals[0] = s.RootPath; @@ -583,8 +626,8 @@ private void BgWorkerDoWork(object sender, DoWorkEventArgs e) }; var limit = findOptions.LimitResultCount; - var stores = catalogRoots.Select(StoreOf).ToList(); - var grandTotal = stores.Sum(s => s.Count); + var sources = catalogRoots.Select(SourceOf).ToList(); + var grandTotal = sources.Sum(s => s.Count); var scannedBase = 0; var list = new List(500); @@ -608,18 +651,18 @@ void Report(int scanned) } var timer = Stopwatch.StartNew(); - foreach (var store in stores) + foreach (var source in sources) { if (worker.CancellationPending || list.Count >= limit) break; var baseScanned = scannedBase; - EntryStoreSearch.Find(store, opts, + source.Find(opts, onMatch: idx => { - list.Add(new PairDirEntry(new EntryRef(store, store.Parent[idx]), new EntryRef(store, idx))); + list.Add(new PairDirEntry(new EntryRef(source, source.ParentOf(idx)), new EntryRef(source, idx))); }, isCancelled: () => worker.CancellationPending || list.Count >= limit, onScan: scanned => Report(baseScanned + scanned)); - scannedBase += store.Count; + scannedBase += source.Count; } timer.Stop(); Log.Logger.Information( @@ -702,7 +745,7 @@ public void SearchResultRetrieveVirtualItem() //TODO: Possibly wasting cycles traversing to the root for this, make smarter. _searchVals[(int)SearchResultColumn.Catalog] = - StoreOfPair(pairDirEntry)?.DefaultFileName ?? pairDirEntry.GetRootEntry()?.DefaultFileName ?? ""; + SourceOfPair(pairDirEntry)?.DefaultFileName ?? pairDirEntry.GetRootEntry()?.DefaultFileName ?? ""; searchHelper.RenderItem = BuildListViewItem(_searchVals, itemColor, pairDirEntry); } @@ -780,6 +823,7 @@ public void MyFormClosing() { CancelLoading(); _config.RecordConfig(_clientForm); + DisposeCatalogSources(); // close any memory-mapped .cdex catalogs _clientForm.CleanUp(); } @@ -801,7 +845,7 @@ private void GoToDirectoryRoot(ICommonEntry newRoot) // Catalog roots are EntryRef instances; two refs to the same catalog share a store. private static bool SameRoot(ICommonEntry a, ICommonEntry b) - => a is EntryRef ea && b is EntryRef eb && ReferenceEquals(ea.Store, eb.Store); + => a is EntryRef ea && b is EntryRef eb && ReferenceEquals(ea.Source, eb.Source); private TreeNode SetNewDirectoryRoot(ICommonEntry newRoot) { @@ -932,8 +976,8 @@ private int SearchResultCompare(PairDirEntry pde1, PairDirEntry pde2) case 3: compareResult = string.Compare( - StoreOfPair(pde1)?.ActualFileName ?? pde1.GetRootEntry()?.ActualFileName, - StoreOfPair(pde2)?.ActualFileName ?? pde2.GetRootEntry()?.ActualFileName, + SourceOfPair(pde1)?.ActualFileName ?? pde1.GetRootEntry()?.ActualFileName, + SourceOfPair(pde2)?.ActualFileName ?? pde2.GetRootEntry()?.ActualFileName, StringComparison.OrdinalIgnoreCase); break; @@ -1207,8 +1251,8 @@ public void CatalogListViewColumnClick() private int RootCompare(ICommonEntry root1, ICommonEntry root2) { - var re1 = StoreOf(root1); - var re2 = StoreOf(root2); + var re1 = SourceOf(root1); + var re2 = SourceOf(root2); var catalogHelper = _clientForm.CatalogListViewHelper; var column = catalogHelper.SortColumn; var compareResult = column switch @@ -1282,10 +1326,7 @@ public async void ReloadCatalogs() _clientForm.SetLoadingProgressValue(0); SetMemoryStatus(); - _catalogRoots = ToCatalogRoots(await _loadCatalogService.LoadRootEntriesAsync( - _config, - OnLoadProgress, - _loadingCts.Token)); + _catalogRoots = await LoadCatalogRootsAsync(); if (_catalogRoots.Count > 0) { diff --git a/src/cdeWin/LoadCatalogService.cs b/src/cdeWin/LoadCatalogService.cs index 2d3fcec..fe537c4 100644 --- a/src/cdeWin/LoadCatalogService.cs +++ b/src/cdeWin/LoadCatalogService.cs @@ -19,6 +19,12 @@ Task> LoadRootEntriesAsync( IConfig config, Action progressCallback, CancellationToken cancellationToken = default); + + /// + /// Discover columnar .cdex catalogs (current dir + config path, one level down). When any + /// exist the GUI memory-maps them instead of loading .cde trees. Empty/none ⇒ fall back. + /// + IList GetColumnarFiles(IConfig config); } public class LoadCatalogService : ILoadCatalogService @@ -30,6 +36,13 @@ public LoadCatalogService(ILogger logger) _logger = logger; } + public IList GetColumnarFiles(IConfig config) + { + var cachePathList = new[] { ".", config.ConfigPath }; + using var repo = new CatalogRepository(_logger); + return repo.GetColumnarFileList(cachePathList); + } + public List LoadRootEntries(IConfig config) { List rootEntries; diff --git a/src/cdeWinTest/CDEWinFormPresenterTest.cs b/src/cdeWinTest/CDEWinFormPresenterTest.cs index b49243e..4a6c39d 100644 --- a/src/cdeWinTest/CDEWinFormPresenterTest.cs +++ b/src/cdeWinTest/CDEWinFormPresenterTest.cs @@ -113,6 +113,33 @@ public void With_RootEntry_List_SetsTotalFileEntries() _mockForm.Received().SetTotalFileEntriesLoadedStatus(1); } + [Test] + public void With_Columnar_Cdex_Present_LoadsViaMmap_AndSkipsCdeLoad() + { + // Migrate the test root to a real .cdex, then prove the presenter memory-maps it (and does + // NOT fall back to loading .cde trees) when GetColumnarFiles reports one. + var store = cdeLib.Entities.Soa.EntryStore.Build(_rootEntry); + var cdex = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"cdewintest-{Guid.NewGuid():N}.cdex"); + cdeLib.Entities.Columnar.ColumnarFormat.Write(store, cdex); + try + { + var loadCatalogsService = Substitute.For(); + loadCatalogsService.GetColumnarFiles(Arg.Any()).Returns(new List { cdex }); + + var presenter = new CDEWinFormPresenter(_mockForm, _stubConfig, loadCatalogsService); + presenter.InitializeAsync().GetAwaiter().GetResult(); + + _mockForm.Received().SetTotalFileEntriesLoadedStatus(1); + loadCatalogsService.DidNotReceive().LoadRootEntriesAsync( + Arg.Any(), Arg.Any>(), Arg.Any()); + + // The catalog is held memory-mapped: the file is locked until the presenter closes, + // which disposes the mmap source. This both proves mmap and exercises the cleanup path. + presenter.MyFormClosing(); + } + finally { System.IO.File.Delete(cdex); } + } + [Ignore("This cant really happen in a real TreeView, as the event to be triggered means there is a node")] [Test] public void With_TreeViewRoot_Null_Throws_Exception() From 4c5724ffa8cca2c746b0ca86ef06e3d2a83058c1 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sun, 7 Jun 2026 09:04:35 +1000 Subject: [PATCH 24/43] feat(hash,dupes): operate on columnar .cdex catalogs hash and dupes now consume (and hash produces) the .cdex format instead of .cde. The tree-based hashing engine (Duplication) is reused unchanged: each .cdex is reconstructed into a mutable tree, hashed, and written back as a fresh .cdex; dupes reconstructs trees and reuses the existing detection. - CatalogTreeBuilder.FromSource/FromColumnarFiles: rebuild a mutable RootEntry tree from an IEntrySource (inverse of EntryStore.Build), copying modified-ticks/flags/hash verbatim for exact round-trip. - IEntrySource.ModifiedTicksOf exposes raw stored ticks for reconstruction. - HashCatalogCommandHandler: source .cdex (GetColumnarFileList), ApplyHash, write each back via ColumnarFormat.Write. No .cdex found -> warn + no-op. - FindDuplicateCommandHandler: source .cdex, reuse FindDuplicates. - Tests: tree<->columnar round-trip incl. hashes; DuplicationTest exercise migrated to scan -> migrate -> hash(.cdex) -> dupes flow. 158 lib tests. Verified end-to-end: scan -> migrate -> rm .cde -> hash -> dupes reports the correct duplicate files from .cdex only. --- .../Duplicates/FindDuplicateCommandHandler.cs | 18 +++- src/cdeLib/Entities/CatalogTreeBuilder.cs | 82 +++++++++++++++++++ .../Columnar/ColumnarCatalogReader.cs | 1 + src/cdeLib/Entities/IEntrySource.cs | 1 + src/cdeLib/Entities/Soa/EntryStore.cs | 1 + .../Hashing/HashCatalogCommandHandler.cs | 18 +++- .../Columnar/ColumnarCatalogTests.cs | 34 ++++++++ src/cdeLibTest/DuplicationTest.cs | 14 +++- 8 files changed, 161 insertions(+), 8 deletions(-) create mode 100644 src/cdeLib/Entities/CatalogTreeBuilder.cs diff --git a/src/cdeLib/Duplicates/FindDuplicateCommandHandler.cs b/src/cdeLib/Duplicates/FindDuplicateCommandHandler.cs index 6ae299e..844b8d3 100644 --- a/src/cdeLib/Duplicates/FindDuplicateCommandHandler.cs +++ b/src/cdeLib/Duplicates/FindDuplicateCommandHandler.cs @@ -1,7 +1,9 @@ using System.Threading; using System.Threading.Tasks; using cdeLib.Catalog; +using cdeLib.Entities; using JetBrains.Annotations; +using Serilog; using SlimMessageBus; namespace cdeLib.Duplicates; @@ -11,16 +13,28 @@ public class FindDuplicateCommandHandler : IRequestHandler +/// Reconstructs a mutable tree from a read-only +/// (an or a memory-mapped ). This is +/// the inverse of , used by the batch hash/dupes +/// commands: they need the existing tree-based hashing engine (two-phase partial→full hashing, +/// cross-catalog size pairing, parallel-by-volume), so they rebuild a tree from the columnar catalog, +/// mutate it (hashes), and write a fresh .cdex back. Field values (modified ticks, flags, hash) +/// are copied verbatim so the rewritten catalog round-trips exactly. +/// +public static class CatalogTreeBuilder +{ + public static RootEntry FromSource(IEntrySource s) + { + var root = new RootEntry + { + Path = s.RootPath, + VolumeName = s.VolumeName, + DefaultFileName = s.DefaultFileName, + ActualFileName = s.ActualFileName, + DriveLetterHint = s.DriveLetterHint, + Description = s.Description, + AvailSpace = s.AvailSpace, + TotalSpace = s.TotalSpace, + ScanStartUtcTicks = s.ScanStartUtcTicks, + ScanEndUtcTicks = s.ScanEndUtcTicks, + ModifiedTicks = s.ModifiedTicksOf(0), + BitFields = s.FlagsOf(0), + }; + + var created = new ICommonEntry[s.Count]; + created[0] = root; + + // Walk the index tree (root = 0) creating a DirEntry per entry, preserving sibling order. + var stack = new Stack(); + stack.Push(0); + while (stack.Count > 0) + { + var p = stack.Pop(); + foreach (var c in s.ChildrenOf(p)) + { + var d = new DirEntry(s.IsDirectory(c)) + { + Path = s.FullName(c), + Size = s.SizeOf(c), + ModifiedTicks = s.ModifiedTicksOf(c), + BitFields = s.FlagsOf(c), + }; + if (s.HasHash && s.IsHashDone(c)) d.Hash = s.HashOf(c); + + created[c] = d; + created[p].AddChild(d); + if (s.IsDirectory(c)) stack.Push(c); + } + } + + root.SetInMemoryFields(); + return root; + } + + /// + /// Open each columnar .cdex file, reconstruct its tree, and tag it with the source path + /// () so a mutated catalog can be written straight back. + /// Each mapping is closed before returning — the tree is a full in-memory copy. + /// + public static List FromColumnarFiles(IEnumerable cdexFiles) + { + var trees = new List(); + foreach (var file in cdexFiles) + { + using var reader = new ColumnarCatalogReader(file); + var tree = FromSource(reader); + tree.ActualFileName = file; + trees.Add(tree); + } + return trees; + } +} diff --git a/src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs b/src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs index 5bb74a3..59138d4 100644 --- a/src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs +++ b/src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs @@ -110,6 +110,7 @@ private ReadOnlySpan As(ColumnarFormat.Col col) where T : struct // ----- IEntrySource: index-addressed accessors straight over the mapping ----- public long SizeOf(int i) => Size[i]; public DateTime ModifiedOf(int i) => DateTime.FromBinary(ModifiedTicks[i]); + public long ModifiedTicksOf(int i) => ModifiedTicks[i]; public Flags FlagsOf(int i) => Flags(i); public bool IsHashDone(int i) => (Flags(i) & Entities.Flags.HashDone) == Entities.Flags.HashDone; public bool IsPartialHash(int i) => (Flags(i) & Entities.Flags.PartialHash) == Entities.Flags.PartialHash; diff --git a/src/cdeLib/Entities/IEntrySource.cs b/src/cdeLib/Entities/IEntrySource.cs index 6b1d0bf..51cde7e 100644 --- a/src/cdeLib/Entities/IEntrySource.cs +++ b/src/cdeLib/Entities/IEntrySource.cs @@ -20,6 +20,7 @@ public interface IEntrySource // ----- per-entry accessors ----- long SizeOf(int i); DateTime ModifiedOf(int i); + long ModifiedTicksOf(int i); // raw stored ticks (for exact tree reconstruction) Flags FlagsOf(int i); bool IsDirectory(int i); bool IsHashDone(int i); diff --git a/src/cdeLib/Entities/Soa/EntryStore.cs b/src/cdeLib/Entities/Soa/EntryStore.cs index 8842415..bf4d28a 100644 --- a/src/cdeLib/Entities/Soa/EntryStore.cs +++ b/src/cdeLib/Entities/Soa/EntryStore.cs @@ -82,6 +82,7 @@ private EntryStore(int count) // ----- IEntrySource: thin index-addressed accessors over the parallel arrays ----- public long SizeOf(int i) => Size[i]; public DateTime ModifiedOf(int i) => Modified(i); + public long ModifiedTicksOf(int i) => ModifiedTicks[i]; public Flags FlagsOf(int i) => Flags(i); public bool HasHash => Hash != null; public Hash16 HashOf(int i) => Hash != null ? Hash[i] : default; diff --git a/src/cdeLib/Hashing/HashCatalogCommandHandler.cs b/src/cdeLib/Hashing/HashCatalogCommandHandler.cs index 290f432..94516be 100644 --- a/src/cdeLib/Hashing/HashCatalogCommandHandler.cs +++ b/src/cdeLib/Hashing/HashCatalogCommandHandler.cs @@ -3,6 +3,9 @@ using System.Threading.Tasks; using cdeLib.Catalog; using cdeLib.Duplicates; +using cdeLib.Entities; +using cdeLib.Entities.Columnar; +using cdeLib.Entities.Soa; using cdeLib.Infrastructure; using JetBrains.Annotations; using SlimMessageBus; @@ -28,9 +31,18 @@ public HashCatalogCommandHandler(Serilog.ILogger logger, IApplicationDiagnostics public async Task OnHandle(HashCatalogCommand request, CancellationToken cancellationToken) { + // Hash operates on the columnar .cdex catalogs. The hashing engine is tree-based, so each + // catalog is reconstructed into a mutable tree, hashed, then written back as a fresh .cdex. + var cdexFiles = _catalogRepository.GetColumnarFileList(["./"]); + if (cdexFiles.Count == 0) + { + _logger.Warning("No .cdex catalogs found. Run 'cde migrate' to create them first."); + return; + } + _logger.Information("Memory pre-catalog load: {MemoryAllocated}", _applicationDiagnostics.GetMemoryAllocated().FormatAsBytes()); - var rootEntries = _catalogRepository.LoadCurrentDirCache(); + var rootEntries = CatalogTreeBuilder.FromColumnarFiles(cdexFiles); _logger.Information("Memory post-catalog load: {MemoryAllocated}", _applicationDiagnostics.GetMemoryAllocated().FormatAsBytes()); var stopwatch = Stopwatch.StartNew(); @@ -38,8 +50,8 @@ public async Task OnHandle(HashCatalogCommand request, CancellationToken cancell foreach (var rootEntry in rootEntries) { - _logger.Information("Saving catalog {Filename}", rootEntry.DefaultFileName); - await _catalogRepository.Save(rootEntry).ConfigureAwait(false); + _logger.Information("Saving catalog {Filename}", rootEntry.ActualFileName); + ColumnarFormat.Write(EntryStore.Build(rootEntry), rootEntry.ActualFileName); } var ts = stopwatch.Elapsed; diff --git a/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs b/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs index e2f97bd..7927eb1 100644 --- a/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs +++ b/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs @@ -167,6 +167,40 @@ public void Find_FullFilter_SizeRange_MatchesStore() finally { File.Delete(path); } } + [Test] + public void CatalogTreeBuilder_RoundTrips_StructureSizesAndHashes() + { + // Set a hash on one file so the hash column round-trips too (the hash/dupes path). + var root = BuildTree(); + var beta = root.Children.First(c => c.Path == "dir1").Children.First(c => c.Path == "beta.log"); + beta.SetHash(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 }); + root.SetInMemoryFields(); + + var store0 = EntryStore.Build(root); + var path = WriteTemp(store0); + try + { + // .cdex -> mutable tree (as hash/dupes do) -> store again must match the original store. + var trees = CatalogTreeBuilder.FromColumnarFiles(new[] { path }); + Assert.That(trees, Has.Count.EqualTo(1)); + var store2 = EntryStore.Build(trees[0]); + + Assert.That(store2.Count, Is.EqualTo(store0.Count)); + Assert.That(store2.HasHash, Is.True); + for (var i = 0; i < store0.Count; i++) + { + Assert.That(store2.FullPath(i), Is.EqualTo(store0.FullPath(i)), $"path {i}"); + Assert.That(store2.Size[i], Is.EqualTo(store0.Size[i]), $"size {i}"); + Assert.That(store2.ModifiedTicks[i], Is.EqualTo(store0.ModifiedTicks[i]), $"modified {i}"); + Assert.That(store2.IsDirectory(i), Is.EqualTo(store0.IsDirectory(i)), $"isDir {i}"); + Assert.That(store2.IsHashDone(i), Is.EqualTo(store0.IsHashDone(i)), $"hashDone {i}"); + if (store0.IsHashDone(i)) + Assert.That(store2.HashOf(i), Is.EqualTo(store0.HashOf(i)), $"hash {i}"); + } + } + finally { File.Delete(path); } + } + [Test] public void EntryRef_OverReader_NavigatesLikeStore() { diff --git a/src/cdeLibTest/DuplicationTest.cs b/src/cdeLibTest/DuplicationTest.cs index 9b38a07..ff505cc 100644 --- a/src/cdeLibTest/DuplicationTest.cs +++ b/src/cdeLibTest/DuplicationTest.cs @@ -178,12 +178,20 @@ public void GetSizePairs_CheckSanityOfDupeSizeCountAndDupeFileCount_Exercise() Program.InitProgram(Array.Empty()); Program.CreateCache(new ScanOptions {Path = testPath}); + + // hash + dupes now operate on the columnar .cdex format, so migrate the freshly scanned + // .cde catalogs to .cdex first, then hash them. + var catalogRepository = new CatalogRepository(Log.Logger); + foreach (var cde in catalogRepository.GetCacheFileList(["./"])) + { + var store = cdeLib.Entities.Soa.EntryStore.Build(catalogRepository.LoadDirCache(cde)); + cdeLib.Entities.Columnar.ColumnarFormat.Write(store, System.IO.Path.ChangeExtension(cde, ".cdex")); + } Program.HashCatalog(); - // run tests. + // run tests. Load the hashed catalogs back from .cdex (where the hashes now live). Console.WriteLine($"0 Directory.GetCurrentDirectory() {System.IO.Directory.GetCurrentDirectory()}"); - var catalogRepository = new CatalogRepository(Log.Logger); - var rootEntries = catalogRepository.LoadCurrentDirCache(); + var rootEntries = CatalogTreeBuilder.FromColumnarFiles(catalogRepository.GetColumnarFileList(["./"])); if (rootEntries.Count == 0) { From 6c2c12f533786f2e472a9cde7c5b2bdb4e95b287 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sun, 7 Jun 2026 09:21:53 +1000 Subject: [PATCH 25/43] feat(scan): write columnar .cdex directly; fix hash-reuse on rescan scan now produces the zero-copy .cdex format instead of .cde, and reuses hashes from an existing .cdex on re-scan (reconstructed into a tree). With this, .cdex is the only format any command writes; 'cde migrate' becomes a one-time upgrade tool for legacy .cde catalogs rather than a routine step. - Both scan handlers (cde/ScanProgress active CLI override + cdeLib default) reuse hashes from .cdex (CatalogTreeBuilder.FromSource) and save via ColumnarFormat.Write(EntryStore.Build(re)). - Fix a latent bug surfaced by this: TraverseTreesCopyHash copied Hash + IsPartialHash but never set IsHashDone, so a reused hash was silently dropped on the next save (affected both DirEntry and RootEntry copy paths, for .cde too). Now sets IsHashDone on copy. - Tests: reconstruct-from-.cdex -> TraverseTreesCopyHash carries the hash (159 lib tests); DuplicationTest simplified now scan emits .cdex directly. Verified end-to-end: scan -> .cdex (no .cde); hash; dupes finds dupes; re-scan WITHOUT re-hash -> dupes still finds them (hash reuse persists). --- .../ScanProgress/CreateCacheCommandHandler.cs | 24 +++++++++---- .../Catalog/CreateCacheCommandHandler.cs | 21 ++++++++--- src/cdeLib/Entities/DirEntry.cs | 4 +++ src/cdeLib/Entities/RootEntry.cs | 3 ++ .../Columnar/ColumnarCatalogTests.cs | 36 +++++++++++++++++++ src/cdeLibTest/DuplicationTest.cs | 14 ++------ 6 files changed, 80 insertions(+), 22 deletions(-) diff --git a/src/cde/ScanProgress/CreateCacheCommandHandler.cs b/src/cde/ScanProgress/CreateCacheCommandHandler.cs index 30f8744..31e6253 100644 --- a/src/cde/ScanProgress/CreateCacheCommandHandler.cs +++ b/src/cde/ScanProgress/CreateCacheCommandHandler.cs @@ -1,10 +1,13 @@ using System; using System.Globalization; +using System.IO; using System.Threading; using System.Threading.Tasks; using cdeLib; using cdeLib.Catalog; using cdeLib.Entities; +using cdeLib.Entities.Columnar; +using cdeLib.Entities.Soa; using cdeLib.Infrastructure.Config; using Humanizer; using JetBrains.Annotations; @@ -54,10 +57,17 @@ private async Task MainLoop(CreateCacheCommand request, CancellationToken cancel return; } - var oldRoot = _catalogRepository.LoadDirCache(re.DefaultFileName); - if (oldRoot != null) + // Catalogs are stored in the zero-copy columnar .cdex format. Reuse hashes from an existing + // .cdex (reconstructed into a tree) for this scan path when one is found. + var cdexName = Path.ChangeExtension(re.DefaultFileName, ".cdex"); + if (File.Exists(cdexName)) { - Log.Information("Found cache \"{FileName}\", Updating hashes for new scan from cache file", re.DefaultFileName); + Log.Information("Found cache \"{FileName}\", Updating hashes for new scan from cache file", cdexName); + RootEntry oldRoot; + using (var reader = new ColumnarCatalogReader(cdexName)) + { + oldRoot = CatalogTreeBuilder.FromSource(reader); + } oldRoot.TraverseTreesCopyHash(re); } @@ -69,8 +79,10 @@ private async Task MainLoop(CreateCacheCommand request, CancellationToken cancel } ScanProgressConsole.EnqueueMessage("Saving catalog..."); - await _catalogRepository.Save(re).ConfigureAwait(false); - ScanProgressConsole.EnqueueMessage($"Saved to {re.DefaultFileName}"); + re.ActualFileName = cdexName; + await Task.Run(() => ColumnarFormat.Write(EntryStore.Build(re), cdexName), cancellationToken) + .ConfigureAwait(false); + ScanProgressConsole.EnqueueMessage($"Saved to {cdexName}"); // Calculate and display final scan summary sw.Stop(); @@ -83,7 +95,7 @@ private async Task MainLoop(CreateCacheCommand request, CancellationToken cancel var scansPerSecText = scansPerSec.ToString("N0", defaultNumberFormat); ScanProgressConsole.EnqueueMessage($"Total files scanned: {scanCountText}, Average: {scansPerSecText}/sec"); - Log.Information("Scanned path {Path}, Saved to {SavePath}", re.Path,re.DefaultFileName); + Log.Information("Scanned path {Path}, Saved to {SavePath}", re.Path, cdexName); Log.Information( "Scanned Files {FileCount:0,0}, Dirs {DirCount:0,0}, Total size {Size:0,0}", re.FileEntryCount, re.DirEntryCount, re.Size.Bytes().Humanize(CultureInfo.CurrentCulture)); diff --git a/src/cdeLib/Catalog/CreateCacheCommandHandler.cs b/src/cdeLib/Catalog/CreateCacheCommandHandler.cs index 383249f..fa90ca7 100644 --- a/src/cdeLib/Catalog/CreateCacheCommandHandler.cs +++ b/src/cdeLib/Catalog/CreateCacheCommandHandler.cs @@ -1,7 +1,10 @@ using System; +using System.IO; using System.Threading; using System.Threading.Tasks; using cdeLib.Entities; +using cdeLib.Entities.Columnar; +using cdeLib.Entities.Soa; using cdeLib.Infrastructure.Config; using JetBrains.Annotations; using SlimMessageBus; @@ -40,11 +43,18 @@ public async Task OnHandle(CreateCacheCommand request, CancellationToken cancell return; } - var oldRoot = _catalogRepository.LoadDirCache(re.DefaultFileName); - if (oldRoot != null) + // Catalogs are stored in the zero-copy columnar .cdex format. Reuse hashes from an existing + // .cdex (reconstructed into a tree) when one is found for this scan path. + var cdexName = Path.ChangeExtension(re.DefaultFileName, ".cdex"); + if (File.Exists(cdexName)) { - Console.WriteLine($"Found cache \"{re.DefaultFileName}\""); + Console.WriteLine($"Found cache \"{cdexName}\""); Console.WriteLine("Updating hashes on new scan from found cache file."); + RootEntry oldRoot; + using (var reader = new ColumnarCatalogReader(cdexName)) + { + oldRoot = CatalogTreeBuilder.FromSource(reader); + } oldRoot.TraverseTreesCopyHash(re); } @@ -55,11 +65,12 @@ public async Task OnHandle(CreateCacheCommand request, CancellationToken cancell re.Description = request.Description; } - await _catalogRepository.Save(re); + re.ActualFileName = cdexName; + await Task.Run(() => ColumnarFormat.Write(EntryStore.Build(re), cdexName), cancellationToken); var scanTimeSpan = re.ScanEndUtc - re.ScanStartUtc; Console.WriteLine($"Scanned path {re.Path}"); Console.WriteLine($"Scan time {scanTimeSpan.TotalMilliseconds:0.00} msecs"); - Console.WriteLine($"Saved scanned path {re.DefaultFileName}"); + Console.WriteLine($"Saved scanned path {cdexName}"); Console.WriteLine( $"Files {re.FileEntryCount:0,0} Dirs {re.DirEntryCount:0,0} Total Size of Files {re.Size:0,0} bytes"); } diff --git a/src/cdeLib/Entities/DirEntry.cs b/src/cdeLib/Entities/DirEntry.cs index c3c6b92..ecd014f 100644 --- a/src/cdeLib/Entities/DirEntry.cs +++ b/src/cdeLib/Entities/DirEntry.cs @@ -561,6 +561,10 @@ public void TraverseTreesCopyHash(ICommonEntry destination) { destinationDirEntry.IsPartialHash = sourceIsPartial; destinationDirEntry.Hash = sourceDirEntry.Hash; + // IsHashDone is a separate BitFields flag; without it the copied hash is + // ignored by hashing/serialization (the reused hash would be silently lost + // on the next save). Mark the destination hashed. + destinationDirEntry.IsHashDone = true; } } // Directory: Push to stack for traversal diff --git a/src/cdeLib/Entities/RootEntry.cs b/src/cdeLib/Entities/RootEntry.cs index 0130352..aaf9d0f 100644 --- a/src/cdeLib/Entities/RootEntry.cs +++ b/src/cdeLib/Entities/RootEntry.cs @@ -1041,6 +1041,9 @@ private static void TryCopyHashIfBeneficial(ICommonEntry source, ICommonEntry de { destination.IsPartialHash = source.IsPartialHash; destination.Hash = source.Hash; + // IsHashDone is a separate BitFields flag; without it the copied hash is ignored by + // hashing/serialization (the reused hash would be silently lost on the next save). + destination.IsHashDone = true; } } diff --git a/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs b/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs index 7927eb1..1aa0e0b 100644 --- a/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs +++ b/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs @@ -201,6 +201,42 @@ public void CatalogTreeBuilder_RoundTrips_StructureSizesAndHashes() finally { File.Delete(path); } } + private static RootEntry BuildForCopy(bool withHash) + { + var root = new RootEntry { Path = @"C:\test" }; + var f = new DirEntry(false) + { + Path = "a.txt", + Size = 100, + Modified = new System.DateTime(2020, 1, 1, 0, 0, 0, System.DateTimeKind.Utc), + }; + if (withHash) f.SetHash(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 }); + root.AddChild(f); + root.SetInMemoryFields(); + return root; + } + + [Test] + public void TraverseTreesCopyHash_FromReconstructedCdex_CopiesHashToFreshScan() + { + // Mirrors the re-scan hash-reuse path: old hashed catalog comes from a .cdex (reconstructed), + // fresh scan tree has no hash; the hash must copy across. + var store = EntryStore.Build(BuildForCopy(withHash: true)); + var path = WriteTemp(store); + try + { + var oldRoot = CatalogTreeBuilder.FromColumnarFiles(new[] { path })[0]; + var fresh = BuildForCopy(withHash: false); + + oldRoot.TraverseTreesCopyHash(fresh); + + var file = fresh.Children.First(c => c.Path == "a.txt"); + Assert.That(file.IsHashDone, Is.True, "hash should have been copied from the reconstructed .cdex"); + Assert.That(file.Hash, Is.EqualTo(oldRoot.Children.First(c => c.Path == "a.txt").Hash)); + } + finally { File.Delete(path); } + } + [Test] public void EntryRef_OverReader_NavigatesLikeStore() { diff --git a/src/cdeLibTest/DuplicationTest.cs b/src/cdeLibTest/DuplicationTest.cs index ff505cc..0cd286a 100644 --- a/src/cdeLibTest/DuplicationTest.cs +++ b/src/cdeLibTest/DuplicationTest.cs @@ -177,20 +177,12 @@ public void GetSizePairs_CheckSanityOfDupeSizeCountAndDupeFileCount_Exercise() } Program.InitProgram(Array.Empty()); - Program.CreateCache(new ScanOptions {Path = testPath}); - - // hash + dupes now operate on the columnar .cdex format, so migrate the freshly scanned - // .cde catalogs to .cdex first, then hash them. - var catalogRepository = new CatalogRepository(Log.Logger); - foreach (var cde in catalogRepository.GetCacheFileList(["./"])) - { - var store = cdeLib.Entities.Soa.EntryStore.Build(catalogRepository.LoadDirCache(cde)); - cdeLib.Entities.Columnar.ColumnarFormat.Write(store, System.IO.Path.ChangeExtension(cde, ".cdex")); - } - Program.HashCatalog(); + Program.CreateCache(new ScanOptions {Path = testPath}); // scan writes a columnar .cdex + Program.HashCatalog(); // hash operates on the .cdex // run tests. Load the hashed catalogs back from .cdex (where the hashes now live). Console.WriteLine($"0 Directory.GetCurrentDirectory() {System.IO.Directory.GetCurrentDirectory()}"); + var catalogRepository = new CatalogRepository(Log.Logger); var rootEntries = CatalogTreeBuilder.FromColumnarFiles(catalogRepository.GetColumnarFileList(["./"])); if (rootEntries.Count == 0) From 2d4631ea48e48670a66e61d55c4a7d1c157e6374 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sun, 7 Jun 2026 11:59:26 +1000 Subject: [PATCH 26/43] feat(hash): live on-screen progress for hash command Mirror the scan progress UX for `cde hash`. Hashing work emits progress and status events that a Spectre.Console status spinner renders live, instead of progress only going to the log. - Duplication: add opt-in ProgressEvent/StatusMessageEvent callbacks; fall back to the existing logger when unset (dupes path/tests unchanged). - cde/HashProgress: override handler runs work on a background task and drives HashProgressConsole; consumers update the display state. - AppContainerBuilder: register the override and exclude the cdeLib hash handler from message-bus auto-declaration (same pattern as scan). Fix percentage exceeding 100%: FilesProcessed is cumulative across the partial and full hash passes, but the denominator was only the partial-pass set size. Each phase now reports its own 0..100% progress (full phase uses a captured baseline), with a display-side clamp as a safety net. --- src/cde/AppContainerBuilder.cs | 8 +- .../HashProgress/HashCatalogCommandHandler.cs | 97 ++++++++++++++ .../HashProgress/HashCompletedEventHandler.cs | 17 +++ src/cde/HashProgress/HashProgressConsole.cs | 121 ++++++++++++++++++ .../HashProgressNotificationHandler.cs | 19 +++ .../HashProgress/HashStatusMessageHandler.cs | 17 +++ src/cdeLib/Duplicates/Duplication.cs | 96 ++++++++++---- src/cdeLib/Hashing/HashCompletedEvent.cs | 3 + src/cdeLib/Hashing/HashProgressEvent.cs | 3 + src/cdeLib/Hashing/HashStatusMessageEvent.cs | 3 + 10 files changed, 358 insertions(+), 26 deletions(-) create mode 100644 src/cde/HashProgress/HashCatalogCommandHandler.cs create mode 100644 src/cde/HashProgress/HashCompletedEventHandler.cs create mode 100644 src/cde/HashProgress/HashProgressConsole.cs create mode 100644 src/cde/HashProgress/HashProgressNotificationHandler.cs create mode 100644 src/cde/HashProgress/HashStatusMessageHandler.cs create mode 100644 src/cdeLib/Hashing/HashCompletedEvent.cs create mode 100644 src/cdeLib/Hashing/HashProgressEvent.cs create mode 100644 src/cdeLib/Hashing/HashStatusMessageEvent.cs diff --git a/src/cde/AppContainerBuilder.cs b/src/cde/AppContainerBuilder.cs index f3fbe4f..1ef5b87 100644 --- a/src/cde/AppContainerBuilder.cs +++ b/src/cde/AppContainerBuilder.cs @@ -47,7 +47,8 @@ public static IContainer BuildContainer(string[] args) mbb.WithProviderMemory() // Filter out cdeLib CreateCacheCommandHandler since cde assembly overrides it .AutoDeclareFrom(typeof(CdelibModule).Assembly, - consumerTypeFilter: t => t != typeof(cdeLib.Catalog.CreateCacheCommandHandler)) + consumerTypeFilter: t => t != typeof(cdeLib.Catalog.CreateCacheCommandHandler) + && t != typeof(cdeLib.Hashing.HashCatalogCommandHandler)) .AutoDeclareFrom(typeof(AppContainerBuilder).Assembly); }); @@ -67,7 +68,10 @@ public static IContainer BuildContainer(string[] args) builder.RegisterType().AsSelf(); builder.RegisterType().AsSelf(); builder.RegisterType().AsSelf(); - builder.RegisterType().AsSelf(); + builder.RegisterType().AsSelf(); + builder.RegisterType().AsSelf(); + builder.RegisterType().AsSelf(); + builder.RegisterType().AsSelf(); builder.RegisterType().AsSelf(); builder.RegisterType().AsSelf(); diff --git a/src/cde/HashProgress/HashCatalogCommandHandler.cs b/src/cde/HashProgress/HashCatalogCommandHandler.cs new file mode 100644 index 0000000..46eb76c --- /dev/null +++ b/src/cde/HashProgress/HashCatalogCommandHandler.cs @@ -0,0 +1,97 @@ +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using cdeLib; +using cdeLib.Catalog; +using cdeLib.Duplicates; +using cdeLib.Entities; +using cdeLib.Entities.Columnar; +using cdeLib.Entities.Soa; +using cdeLib.Hashing; +using cdeLib.Infrastructure; +using JetBrains.Annotations; +using SlimMessageBus; + +namespace cde.HashProgress; + +/// +/// cde-layer override of the hash handler. Mirrors +/// but renders a live Spectre progress display by publishing progress events emitted from . +/// The cdeLib handler is excluded from message-bus auto-declaration so this one is used by the CLI. +/// +[UsedImplicitly] +public class HashCatalogCommandHandler : IRequestHandler +{ + private readonly Duplication _duplication; + private readonly Serilog.ILogger _logger; + private readonly IApplicationDiagnostics _applicationDiagnostics; + private readonly ICatalogRepository _catalogRepository; + private readonly IMessageBus _messageBus; + + public HashCatalogCommandHandler(Serilog.ILogger logger, IApplicationDiagnostics applicationDiagnostics, + Duplication duplication, ICatalogRepository catalogRepository, IMessageBus messageBus) + { + _logger = logger; + _applicationDiagnostics = applicationDiagnostics; + _duplication = duplication; + _catalogRepository = catalogRepository; + _messageBus = messageBus; + } + + public async Task OnHandle(HashCatalogCommand request, CancellationToken cancellationToken) + { + // Hash operates on the columnar .cdex catalogs. The hashing engine is tree-based, so each + // catalog is reconstructed into a mutable tree, hashed, then written back as a fresh .cdex. + var cdexFiles = _catalogRepository.GetColumnarFileList(["./"]); + if (cdexFiles.Count == 0) + { + _logger.Warning("No .cdex catalogs found. Run 'cde migrate' to create them first."); + return; + } + + var mainLoopTask = Task.Run(() => MainLoop(cdexFiles, cancellationToken), cancellationToken); + var console = new HashProgressConsole(); + console.Start(mainLoopTask, cancellationToken); + await mainLoopTask.ConfigureAwait(false); + } + + private async Task MainLoop(IList cdexFiles, CancellationToken cancellationToken) + { + _logger.Information("Memory pre-catalog load: {MemoryAllocated}", + _applicationDiagnostics.GetMemoryAllocated().FormatAsBytes()); + var rootEntries = CatalogTreeBuilder.FromColumnarFiles(cdexFiles); + _logger.Information("Memory post-catalog load: {MemoryAllocated}", + _applicationDiagnostics.GetMemoryAllocated().FormatAsBytes()); + + // Route hashing progress and status lines to the Spectre console via the message bus. + _duplication.ProgressEvent = (processed, toHash, phase) => + _messageBus.Publish(new HashProgressEvent(processed, toHash, phase), cancellationToken: cancellationToken); + _duplication.StatusMessageEvent = message => + _messageBus.Publish(new HashStatusMessageEvent(message), cancellationToken: cancellationToken); + + var stopwatch = Stopwatch.StartNew(); + try + { + await _duplication.ApplyHash(rootEntries).ConfigureAwait(false); + + foreach (var rootEntry in rootEntries) + { + HashProgressConsole.EnqueueMessage($"Saving catalog {rootEntry.ActualFileName}"); + ColumnarFormat.Write(EntryStore.Build(rootEntry), rootEntry.ActualFileName); + } + + var ts = stopwatch.Elapsed; + var elapsedTime = $"{ts.Hours:00}:{ts.Minutes:00}:{ts.Seconds:00}.{ts.Milliseconds / 10:00}"; + HashProgressConsole.EnqueueMessage( + $"Hash Took {elapsedTime}, Memory: {_applicationDiagnostics.GetMemoryAllocated().FormatAsBytes()}"); + _logger.Information("Hash Took {ElapsedTime}, Memory: {Memory}", elapsedTime, + _applicationDiagnostics.GetMemoryAllocated().FormatAsBytes()); + } + finally + { + await _messageBus.Publish(new HashCompletedEvent(), cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + } +} diff --git a/src/cde/HashProgress/HashCompletedEventHandler.cs b/src/cde/HashProgress/HashCompletedEventHandler.cs new file mode 100644 index 0000000..cbae176 --- /dev/null +++ b/src/cde/HashProgress/HashCompletedEventHandler.cs @@ -0,0 +1,17 @@ +using System.Threading; +using System.Threading.Tasks; +using cdeLib.Hashing; +using JetBrains.Annotations; +using SlimMessageBus; + +namespace cde.HashProgress; + +[UsedImplicitly] +public class HashCompletedEventHandler : IConsumer +{ + public async Task OnHandle(HashCompletedEvent message, CancellationToken cancellationToken) + { + HashProgressConsole.HashIsComplete = true; + await Task.Yield(); + } +} diff --git a/src/cde/HashProgress/HashProgressConsole.cs b/src/cde/HashProgress/HashProgressConsole.cs new file mode 100644 index 0000000..3e1a900 --- /dev/null +++ b/src/cde/HashProgress/HashProgressConsole.cs @@ -0,0 +1,121 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using Serilog; +using Spectre.Console; + +namespace cde.HashProgress; + +public class HashProgressConsole +{ + public static long FilesProcessed { get; set; } + + public static long FilesToHash { get; set; } + + public static string Phase { get; set; } + + public static bool HashIsComplete { get; set; } + + private static readonly Queue Messages = new(); + + private static void WriteLogMessage(string message) + { + var width = AnsiConsole.Profile.Out.Width; + // Pad raw message to console width to clear previous content + var paddedMessage = $"LOG:{message}".PadRight(width); + // Use \r to return to start of line (status update behavior) + AnsiConsole.Markup($"\r[grey]{Markup.Escape(paddedMessage)}[/]\n"); + } + + /// + /// Enqueue a message to be displayed in the console progress UI + /// + public static void EnqueueMessage(string message) + { + Messages.Enqueue(message); + } + + public void Start(Task mainLoopTask, CancellationToken cancellationToken) + { + var sw = new Stopwatch(); + sw.Start(); + AnsiConsole.Status() + .AutoRefresh(enabled: true) + .Spinner(Spinner.Known.Default) + .Start("Hashing...", ctx => + { + while (!mainLoopTask.IsCompleted && !cancellationToken.IsCancellationRequested) + { + ShowProgress(sw, ctx); + } + + // Flush any remaining messages after the task completes + FlushMessages(); + }); + } + + private static void FlushMessages() + { + while (Messages.TryDequeue(out var msg)) + { + if (!string.IsNullOrEmpty(msg)) + { + WriteLogMessage(msg); + } + } + } + + private static long CalculateFilesPerSecond(Stopwatch sw) + { + var elapsedSec = sw.ElapsedMilliseconds / 1000; + if (elapsedSec < 1) elapsedSec = 1; + return FilesProcessed / elapsedSec; + } + + private void ShowProgress(Stopwatch sw, StatusContext ctx) + { + var defaultNumberFormat = new NumberFormatInfo(); + var filesPerSec = CalculateFilesPerSecond(sw); + + // Clamp to the total: FilesProcessed is cumulative across hash passes, so guard against + // any accounting edge case rendering more than the total or over 100%. + var processed = FilesToHash > 0 && FilesProcessed > FilesToHash ? FilesToHash : FilesProcessed; + var processedText = processed.ToString("N0", defaultNumberFormat); + var toHashText = FilesToHash.ToString("N0", defaultNumberFormat); + var perSecText = filesPerSec.ToString("N0", defaultNumberFormat); + var percent = FilesToHash > 0 ? 100.0 * processed / FilesToHash : 0.0; + var phase = string.IsNullOrEmpty(Phase) ? "Hashing" : Phase; + + var msg = + $"[yellow]{Markup.Escape(phase)}[/] [yellow]{processedText}[/] of [yellow]{toHashText}[/] ([yellow]{percent:F1}%[/]) Avg [yellow]{perSecText}[/]/sec"; + + try + { + ctx.Status(msg); + } + catch (Exception ex) + { + Log.Logger.Error(ex, "Error writing Status"); + } + + ctx.Spinner(Spinner.Known.Star); + ctx.SpinnerStyle(Style.Parse("green")); + + var dequeueMessages = true; + while (dequeueMessages) + { + Messages.TryDequeue(out var queued); + if (string.IsNullOrEmpty(queued)) + { + dequeueMessages = false; + } + else + { + WriteLogMessage(queued); + } + } + } +} diff --git a/src/cde/HashProgress/HashProgressNotificationHandler.cs b/src/cde/HashProgress/HashProgressNotificationHandler.cs new file mode 100644 index 0000000..2d0025e --- /dev/null +++ b/src/cde/HashProgress/HashProgressNotificationHandler.cs @@ -0,0 +1,19 @@ +using System.Threading; +using System.Threading.Tasks; +using cdeLib.Hashing; +using JetBrains.Annotations; +using SlimMessageBus; + +namespace cde.HashProgress; + +[UsedImplicitly] +public class HashProgressNotificationHandler : IConsumer +{ + public Task OnHandle(HashProgressEvent message, CancellationToken cancellationToken) + { + HashProgressConsole.FilesProcessed = message.FilesProcessed; + HashProgressConsole.FilesToHash = message.FilesToHash; + HashProgressConsole.Phase = message.Phase; + return Task.CompletedTask; + } +} diff --git a/src/cde/HashProgress/HashStatusMessageHandler.cs b/src/cde/HashProgress/HashStatusMessageHandler.cs new file mode 100644 index 0000000..99ad762 --- /dev/null +++ b/src/cde/HashProgress/HashStatusMessageHandler.cs @@ -0,0 +1,17 @@ +using System.Threading; +using System.Threading.Tasks; +using cdeLib.Hashing; +using JetBrains.Annotations; +using SlimMessageBus; + +namespace cde.HashProgress; + +[UsedImplicitly] +public class HashStatusMessageHandler : IConsumer +{ + public Task OnHandle(HashStatusMessageEvent message, CancellationToken cancellationToken) + { + HashProgressConsole.EnqueueMessage(message.Message); + return Task.CompletedTask; + } +} diff --git a/src/cdeLib/Duplicates/Duplication.cs b/src/cdeLib/Duplicates/Duplication.cs index 20c821f..0699d9b 100644 --- a/src/cdeLib/Duplicates/Duplication.cs +++ b/src/cdeLib/Duplicates/Duplication.cs @@ -26,6 +26,11 @@ public class Duplication private readonly HashSet _dirEntriesRequiringFullHashing = new(); protected readonly DuplicationStatistics _duplicationStatistics; + + // Cumulative FilesProcessed at the moment the full-hash phase begins, so that phase can report + // its own 0-based progress rather than continuing the partial phase's running total. + private long _processedAtFullHashStart; + private readonly ILogger _logger; private readonly IApplicationDiagnostics _applicationDiagnostics; private readonly HashHelper _hashHelper; @@ -40,6 +45,26 @@ public Duplication(ILogger logger, IConfiguration configuration, IApplicationDia _logger.LogDebug("Dupe Constructor Memory: {0}", _applicationDiagnostics.GetMemoryAllocated().FormatAsBytes()); } + /// + /// Optional callback raised periodically during so a UI can render live progress. + /// Arguments are (filesProcessed, filesToHash, phase). When null the progress is written to the log instead. + /// + public Action ProgressEvent { get; set; } + + /// + /// Optional callback raised with human readable status/summary lines during . + /// When null the message is written to the log instead. + /// + public Action StatusMessageEvent { get; set; } + + private void ReportStatus(string message) + { + if (StatusMessageEvent is not null) + StatusMessageEvent(message); + else + _logger.LogInfo(message); + } + /// /// Apply an Hash Checksum to all rootEntries /// @@ -70,10 +95,10 @@ public async Task ApplyHash(IList rootEntries) longestListSize = kvp.Key; } } - _logger.LogInfo("Found {0} sets of files matched by file size", newMatches.Count); - _logger.LogInfo("Total files processed for the file size matches is {0}", totalFilesInRootEntries); - _logger.LogInfo("Total files found with at least 1 other file of same length {0}", totalEntriesInSizeDupes); - _logger.LogInfo("Longest list of same sized files is {0} for size {1} ", longestListLength, longestListSize); + ReportStatus($"Found {newMatches.Count} sets of files matched by file size"); + ReportStatus($"Total files processed for the file size matches is {totalFilesInRootEntries}"); + ReportStatus($"Total files found with at least 1 other file of same length {totalEntriesInSizeDupes}"); + ReportStatus($"Longest list of same sized files is {longestListLength} for size {longestListSize} "); // flatten - optimized without LINQ _logger.LogDebug("Flatten List.."); @@ -169,24 +194,23 @@ public async Task ApplyHash(IList rootEntries) return; } - _logger.LogInfo("After initial partial hashing phase."); + ReportStatus("After initial partial hashing phase."); var perf = $"{_duplicationStatistics.BytesProcessed * (1000.0 / timer.ElapsedMilliseconds) / (1024.0 * 1024.0):F2} MB/s"; var statsMessage = $"FullHash: {_duplicationStatistics.FullHashes} PartialHash: {_duplicationStatistics.PartialHashes} Processed: {_duplicationStatistics.BytesProcessed / (1024 * 1024):F2} MB NotProcessed: {_duplicationStatistics.BytesNotProcessed / (1024 * 1024):F2} MB Perf: {perf}\nTotal Data Encountered: {_duplicationStatistics.TotalFileBytes / (1024 * 1024):F2} MB\nFailedHash: {_duplicationStatistics.FailedToHash} (almost always because cannot open to read file)"; - _logger.LogInfo(statsMessage); + ReportStatus(statsMessage); Hack.BreakConsoleFlag = false; // require you to press break again to stop the full hash phase. CheckDupesAndCompleteFullHash(rootEntries); - _logger.LogInfo(string.Empty); - _logger.LogInfo("After hashing completed."); + ReportStatus("After hashing completed."); timer.Stop(); perf = $"{_duplicationStatistics.BytesProcessed * (1000.0 / timer.ElapsedMilliseconds) / (1024.0 * 1024.0):F2} MB/s"; statsMessage = $"FullHash: {_duplicationStatistics.FullHashes} PartialHash: {_duplicationStatistics.PartialHashes} Processed: {_duplicationStatistics.BytesProcessed / (1024 * 1024):F2} MB Perf: {perf}\nFailedHash: {_duplicationStatistics.FailedToHash} (almost always because cannot open to read file)"; - _logger.LogInfo(statsMessage); + ReportStatus(statsMessage); await Task.CompletedTask; } @@ -282,10 +306,16 @@ private void CheckDupesAndCompleteFullHash(IEnumerable rootEntries) } } - _logger.LogInfo("Found {0} duplication collections.", foundDupes.Count); - _logger.LogInfo("Total files found with at least 1 other file duplicate {0}", - totalEntriesInDupes); - _logger.LogInfo("Longest list of duplicate files is {0}", longestListLength); + ReportStatus($"Found {foundDupes.Count} duplication collections."); + ReportStatus($"Total files found with at least 1 other file duplicate {totalEntriesInDupes}"); + ReportStatus($"Longest list of duplicate files is {longestListLength}"); + + // Switch progress reporting to the full-hash phase: FilesProcessed is cumulative across both + // passes, so capture a baseline here and re-target the denominator to this phase's own work. + // Each phase then reports its own 0..100% progress (the phase label distinguishes them) and the + // percentage can never exceed 100% as it did when the partial-pass denominator was reused. + _processedAtFullHashStart = _duplicationStatistics.FilesProcessed; + _duplicationStatistics.FilesToCheckForDuplicatesCount = totalEntriesInDupes; // Populate HashSet with entries requiring full hash foreach (var kvp in foundDupes) @@ -400,13 +430,21 @@ private async Task CalculateHash(string fullPath, ICommonEntry de, bool doPartia _duplicationStatistics.FullHashes++; if (_duplicationStatistics.FilesProcessed % displayCounterInterval == 0) { - _logger.LogInfo( - "Progress through duplicate files at {0} of {1} which is {2:F2}% Largest {3:F2} MB, Smallest {4:F2} MB", - _duplicationStatistics.FilesProcessed, _duplicationStatistics.FilesToCheckForDuplicatesCount, - 100 * (1.0 * _duplicationStatistics.FilesProcessed / - _duplicationStatistics.FilesToCheckForDuplicatesCount), - 1.0 * _duplicationStatistics.LargestFileSize / (1024 * 1024), - 1.0 * _duplicationStatistics.SmallestFileSize / (1024 * 1024)); + if (ProgressEvent is not null) + { + ProgressEvent(_duplicationStatistics.FilesProcessed, + _duplicationStatistics.FilesToCheckForDuplicatesCount, "Partial hash"); + } + else + { + _logger.LogInfo( + "Progress through duplicate files at {0} of {1} which is {2:F2}% Largest {3:F2} MB, Smallest {4:F2} MB", + _duplicationStatistics.FilesProcessed, _duplicationStatistics.FilesToCheckForDuplicatesCount, + 100 * (1.0 * _duplicationStatistics.FilesProcessed / + _duplicationStatistics.FilesToCheckForDuplicatesCount), + 1.0 * _duplicationStatistics.LargestFileSize / (1024 * 1024), + 1.0 * _duplicationStatistics.SmallestFileSize / (1024 * 1024)); + } } } else @@ -431,10 +469,20 @@ private async Task CalculateHash(string fullPath, ICommonEntry de, bool doPartia _duplicationStatistics.BytesProcessed += hashResponse.BytesHashed; if (_duplicationStatistics.FilesProcessed % displayCounterInterval == 0) { - _logger.LogInfo("Progress through duplicate files at {0} of {1} which is {2:.0}%", - _duplicationStatistics.FilesProcessed, _duplicationStatistics.FilesToCheckForDuplicatesCount, - 100 * (1.0 * _duplicationStatistics.FilesProcessed / - _duplicationStatistics.FilesToCheckForDuplicatesCount)); + // Report progress relative to the start of the full-hash phase so it reads 0..100%. + var processedThisPhase = _duplicationStatistics.FilesProcessed - _processedAtFullHashStart; + if (ProgressEvent is not null) + { + ProgressEvent(processedThisPhase, + _duplicationStatistics.FilesToCheckForDuplicatesCount, "Full hash"); + } + else + { + _logger.LogInfo("Progress through duplicate files at {0} of {1} which is {2:.0}%", + processedThisPhase, _duplicationStatistics.FilesToCheckForDuplicatesCount, + 100 * (1.0 * processedThisPhase / + _duplicationStatistics.FilesToCheckForDuplicatesCount)); + } } } else diff --git a/src/cdeLib/Hashing/HashCompletedEvent.cs b/src/cdeLib/Hashing/HashCompletedEvent.cs new file mode 100644 index 0000000..4586040 --- /dev/null +++ b/src/cdeLib/Hashing/HashCompletedEvent.cs @@ -0,0 +1,3 @@ +namespace cdeLib.Hashing; + +public record HashCompletedEvent; diff --git a/src/cdeLib/Hashing/HashProgressEvent.cs b/src/cdeLib/Hashing/HashProgressEvent.cs new file mode 100644 index 0000000..ebc0e58 --- /dev/null +++ b/src/cdeLib/Hashing/HashProgressEvent.cs @@ -0,0 +1,3 @@ +namespace cdeLib.Hashing; + +public record HashProgressEvent(long FilesProcessed, long FilesToHash, string Phase); diff --git a/src/cdeLib/Hashing/HashStatusMessageEvent.cs b/src/cdeLib/Hashing/HashStatusMessageEvent.cs new file mode 100644 index 0000000..0ee1585 --- /dev/null +++ b/src/cdeLib/Hashing/HashStatusMessageEvent.cs @@ -0,0 +1,3 @@ +namespace cdeLib.Hashing; + +public record HashStatusMessageEvent(string Message); From 2431eaa5853eec8454a502ecf158613eeba5f69a Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sun, 7 Jun 2026 22:54:45 +1000 Subject: [PATCH 27/43] docs: add cde migrate, fix .NET version and Nuke->Fallout - Document the 'cde migrate' command (one-way .cde -> .cdex conversion) - Update stale .Net7 reference to .NET 10 in Readme.md - developer.md: build now uses Fallout (replaced Nuke); add build details --- Readme.md | 26 +++++++++++++++++++++++++- developer.md | 12 +++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/Readme.md b/Readme.md index e3386a7..4a33b78 100644 --- a/Readme.md +++ b/Readme.md @@ -30,7 +30,7 @@ This application reads and writes a configuration file `cdeWinView.cfg`. - size of all the columns in list views - values of fields in the search parameters -The executable `cdeWin.exe` can be copied around by it self to be used anywhere that .Net7 is available with the behavior of the cdeWinView.cfg file as described just above. +The executable `cdeWin.exe` can be copied around by it self to be used anywhere that .NET 10 is available with the behavior of the cdeWinView.cfg file as described just above. #### cdeWeb (unreleased) @@ -186,6 +186,30 @@ Consider using -minHourAge to limit Hash and Dupes work if your are cleanign up Output the full tree of file entries in the catologs in text format. +### cde migrate \[Path\] + +#### Valid Options for this mode + +`No filter options supported.` + +This mode performs a one-way conversion of the original MessagePack `.cde` catalogs into the newer zero-copy columnar `.cdex` format. The `.cdex` format is laid out so it can be memory-mapped and searched without first deserialising the whole catalog into objects, which lowers memory use and speeds up load on large catalogs. + +- With a path argument, only that single `.cde` file is converted: + + ```batch + cde migrate C-V3Win7-C__users.cde + ``` + +- With no argument, every catalog discovered in the current directory and one directory below is converted (the same discovery rule used when loading catalogs): + + ```batch + cde migrate + ``` + +For each catalog a `.cdex` file is written beside the source `.cde`, keeping the same base name. The original `.cde` file is left in place and is not deleted, so the conversion is non-destructive. Re-running migrate simply overwrites the `.cdex` output. + +For each file converted cde prints the source and destination file names, their byte sizes and the entry count, then a summary of how many catalogs were migrated. Files that cannot be loaded are skipped and reported. + ### Parameter Options | | Parameter                  | Description | diff --git a/developer.md b/developer.md index 1ef6a5a..26e9a2c 100644 --- a/developer.md +++ b/developer.md @@ -2,7 +2,11 @@ ## Building the app -The build uses Nuke to perform the steps. +The build uses Fallout to perform the steps. (It replaced Nuke.) + +The build definition lives in `build/Build.cs` (built on `Fallout.Common`), and the +`build.cmd` / `build.ps1` / `build.sh` scripts bootstrap `build/_build.csproj`. +Fallout config, parameters and temp/log output are kept under `.fallout/`. To build the app on windows run: @@ -10,5 +14,11 @@ To build the app on windows run: build.cmd publish ``` +On Linux/macOS run: + +```shell +./build.sh publish +``` + Artifacts from the build will be built to `.\artifacts` From 8f90904da25c4d64dcf757d95af83ffce8f1ffbc Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Sun, 7 Jun 2026 23:06:02 +1000 Subject: [PATCH 28/43] docs: correct CLAUDE.md serialization and messaging libs The project does not use MediatR; CQRS/messaging is via SlimMessageBus (IRequestHandler/IMessageBus). Update serialization docs to reflect the current columnar .cdex format (zero-copy mmap) as primary, with the legacy .cde tree format using MessagePack (default), FlatSharp, or protobuf-net. --- claude.md | 47 +++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/claude.md b/claude.md index fe3db4a..35d198e 100644 --- a/claude.md +++ b/claude.md @@ -16,14 +16,14 @@ CDE is a high-performance file system cataloging utility written in C# that crea - Target: .NET 10 - Cross-platform: win-x64, linux-x64, osx-x64 - Entry point for scan, find, hash, dupes, dump commands - - Dependencies: Autofac, MediatR, CommandLineParser, Spectre.Console + - Dependencies: Autofac, SlimMessageBus, CommandLineParser, Spectre.Console - **cdeLib** - Core library containing business logic - Target: .NET 10 - Contains all catalog operations, hashing, duplicate detection - - Uses CQRS pattern with MediatR - - Serialization: MessagePack, FlatSharp - - Key dependencies: Autofac, MediatR, Serilog + - Uses CQRS pattern with SlimMessageBus + - Serialization: columnar `.cdex` (zero-copy, memory-mapped), plus MessagePack/FlatSharp/protobuf-net for the legacy `.cde` tree format + - Key dependencies: Autofac, SlimMessageBus, Serilog - **cdeWin** - Windows Forms GUI application - Target: .NET 10 (Windows) @@ -52,7 +52,7 @@ CDE is a high-performance file system cataloging utility written in C# that crea ### Architecture Patterns -- **CQRS (Command Query Responsibility Segregation)**: Commands and queries handled via MediatR +- **CQRS (Command Query Responsibility Segregation)**: Commands and queries handled via SlimMessageBus (`IRequestHandler`, `IMessageBus`) - Commands: `CreateCacheCommand`, `HashCatalogCommand`, `FindDuplicatesCommand`, `UpdateCommand` - Handlers: Separate handlers for each command - Events: `ScanProgressEvent` for progress tracking @@ -69,9 +69,12 @@ CDE is a high-performance file system cataloging utility written in C# that crea ### Serialization -Multiple serialization formats supported: -- **MessagePack** - Primary catalog file format (.cde files) -- **FlatSharp** - FlatBuffers support (alternative) +Two on-disk catalog formats: + +- **Columnar `.cdex`** (`Entities/Columnar/ColumnarFormat.cs`) - **Current/primary format.** A struct-of-arrays layout designed for *zero-copy reads over a memory-mapped file*: "loading" a catalog is mmap-ing it, so no managed object graph is materialised and the working set is only the file pages a query touches (reclaimable OS page cache, not GC heap). Custom binary layout with a `"CDEX"` magic header and dense, homogeneous columns (names, sizes, timestamps, hashes, tree links) — a name-only search scans just the name columns and never pages in the rest. Written directly by `scan`; read via `ColumnarCatalogReader`. +- **Legacy `.cde` tree format** - The original materialised directory-tree format, serialized via a pluggable `SerializerProtocol` in `Catalog/CatalogRepository.cs`: + - **MessagePack** - default protocol for `.cde` (`MessagePackConfig.Options`, custom `Hash16Formatter`/resolver) + - **FlatSharp** (FlatBuffers) and **protobuf-net** - alternative protocols selectable via `SerializerProtocol` ### Hashing @@ -135,7 +138,7 @@ Entry (base class) ### Key Command Handlers Located in `cdeLib/`: -- `Catalog/CreateCacheCommandHandler.cs` - Scans file systems, creates .cde files +- `Catalog/CreateCacheCommandHandler.cs` - Scans file systems, writes columnar `.cdex` catalogs (reusing hashes from an existing `.cdex` when present) - `Hashing/HashCatalogCommandHandler.cs` - Adds MD5 hashes to catalogs - `Duplicates/FindDuplicateCommandHandler.cs` - Identifies duplicate files - `FindService.cs` - File search functionality @@ -149,16 +152,18 @@ Located in `cdeLib/Infrastructure/`: - `WorkStealingTreeTraversal.cs` - Parallel directory traversal - `Config/` - Configuration classes -## Catalog File Format (.cde) +## Catalog File Formats (.cdex / .cde) -- **Extension**: `.cde` +- **Extensions**: `.cdex` (current columnar format) and `.cde` (legacy tree format) - **Naming**: Derived from drive letter, volume name, and path - - Example: `C-V3Win7-C__users.cde` for `C:\users\` - - Example: `UNC-toothless_c__users_.cde` for `\\unc\toothless\c$\users` -- **Loading**: All .cde files in current directory or one level down are loaded -- **Content**: Directory tree with optional MD5 hashes + - Example: `C-V3Win7-C__users.cdex` for `C:\users\` + - Example: `UNC-toothless_c__users_.cdex` for `\\unc\toothless\c$\users` +- **Loading**: All catalog files in the current directory or one level down are loaded (`GetColumnarFileList` for `.cdex`, `GetCacheFileList` for `.cde`) +- **Content**: Directory tree (or columns) with optional MD5 hashes - **Size**: Highly efficient - 500MB for 11 billion entries -- **Format**: MessagePack binary serialization (not compressed) +- **Format**: + - `.cdex` - custom columnar binary, memory-mapped for zero-copy loads (not compressed) + - `.cde` - MessagePack binary serialization by default (not compressed); protobuf/FlatBuffers selectable in code ## Common Operations @@ -229,7 +234,7 @@ This branch focuses on performance improvements and refactoring. Recent commits - Use object pooling for frequently allocated objects - Benchmark changes that affect hot paths -2. **Catalog Compatibility**: Changes to serialization affect .cde file format +2. **Catalog Compatibility**: Changes to serialization affect the `.cdex`/`.cde` file formats - Hash size changes require catalog recreation - Document breaking changes @@ -243,7 +248,7 @@ This branch focuses on performance improvements and refactoring. Recent commits ### Code Patterns -- **MediatR Commands**: Business operations are commands/queries +- **SlimMessageBus Commands**: Business operations are commands/queries (`IRequestHandler.OnHandle`) - **Dependency Injection**: Constructor injection via Autofac - **Logging**: Serilog with structured logging - **Configuration**: Microsoft.Extensions.Configuration with appsettings.json @@ -269,8 +274,10 @@ Standard .NET test runners (tests use NUnit, xUnit) ## Dependencies to Note - **Autofac** - Dependency injection -- **MediatR** - Command/query pattern -- **MessagePack** - Primary serialization +- **SlimMessageBus** - Command/query and pub/sub messaging (in-memory) +- **MessagePack** - Serialization for the legacy `.cde` tree format (current `.cdex` format uses a custom columnar layout) +- **FlatSharp** - FlatBuffers serialization (alternative `.cde` protocol) +- **protobuf-net** - Protobuf serialization (alternative `.cde` protocol) - **Serilog** - Structured logging - **CommandLineParser** - CLI argument parsing - **Spectre.Console** - Rich console output From 4ba5a3e6db4c48f4f29cddbadc4d40dec2bfef55 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Mon, 8 Jun 2026 11:21:10 +1000 Subject: [PATCH 29/43] feat(scan): add --follow-junctions option (default off) Directory junctions and symbolic links (reparse points) carry the Directory attribute, so the scan previously descended into them. This could cause scan cycles (e.g. a junction pointing at an ancestor) and duplicated catalog content. By default the scan now records reparse points but does not follow them. Pass --follow-junctions to restore the previous descend-into behaviour. The flag is threaded CLI -> CreateCacheCommand -> RootEntry.RecurseTree. --- Readme.md | 10 ++++++++++ src/cde/CommandLine/CommandLineOptions.cs | 5 +++++ src/cde/Program.cs | 3 ++- src/cde/ScanProgress/CreateCacheCommandHandler.cs | 2 +- src/cdeLib/Catalog/CreateCacheCommand.cs | 6 ++++++ src/cdeLib/Catalog/CreateCacheCommandHandler.cs | 2 +- src/cdeLib/Entities/RootEntry.cs | 15 +++++++++++---- 7 files changed, 36 insertions(+), 7 deletions(-) diff --git a/Readme.md b/Readme.md index 4a33b78..53f6445 100644 --- a/Readme.md +++ b/Readme.md @@ -109,6 +109,8 @@ cde path -find afilename [`-maxDateTime`](#parameter-options) [`-minTime`](#parameter-options) [`-maxTime`](#parameter-options) +[`--desc`](#parameter-options) +[`--follow-junctions`](#parameter-options) This is the mode of operation that creates and updates catalog files. @@ -116,6 +118,12 @@ When it creates new catalog files it will detect an old catalog file for the giv Only Last Modified Time of file system entries is captured into .cde files. +By default cde records directory junctions and symbolic links (reparse points) in the catalog but does **not** descend into them, which avoids scan cycles (for example a junction pointing back at an ancestor directory) and duplicated content. Pass `--follow-junctions` to descend into them. + +``` + cde scan C:\ --follow-junctions +``` + ### cde find String #### Valid Options for this mode @@ -228,6 +236,8 @@ For each file converted cde prints the source and destination file names, their | | `-maxResults {Int}` | Maximum number of results returned by cde. | | | `-exclude {Regex}` | A filter to exclude only entries that match these regexes for processing. | | | `-include {Regex}` | A filter to include only entries that match these Regexes for processing. | +| | `--desc {Text}` | (scan) Description text to store in the catalog file. | +| | `--follow-junctions` | (scan) Descend into directory junctions and symbolic links (reparse points). Off by default; reparse points are recorded but not followed, avoiding scan cycles and duplicated content. | ##### Date Time Format for parameters diff --git a/src/cde/CommandLine/CommandLineOptions.cs b/src/cde/CommandLine/CommandLineOptions.cs index e9eca3b..a66cea8 100644 --- a/src/cde/CommandLine/CommandLineOptions.cs +++ b/src/cde/CommandLine/CommandLineOptions.cs @@ -13,6 +13,11 @@ public class ScanOptions [Option("desc", HelpText = "Description to set")] public string Description { get; [UsedImplicitly] set; } + + [Option("follow-junctions", + Default = false, + HelpText = "Descend into directory junctions / symbolic links. Off by default to avoid cycles and duplicate content.")] + public bool FollowJunctions { get; [UsedImplicitly] set; } } [Verb("find", HelpText = "Uses all cache files available searches for ")] diff --git a/src/cde/Program.cs b/src/cde/Program.cs index d9d9873..50eb7fa 100644 --- a/src/cde/Program.cs +++ b/src/cde/Program.cs @@ -336,7 +336,8 @@ public static void HashCatalog() public static void CreateCache(ScanOptions opts) { var task = Task.Run(async () => - await MessageBus.Send(new CreateCacheCommand(opts.Path) { Description = opts.Description }) + await MessageBus.Send(new CreateCacheCommand(opts.Path) + { Description = opts.Description, FollowJunctions = opts.FollowJunctions }) .ConfigureAwait(false)); task.Wait(); } diff --git a/src/cde/ScanProgress/CreateCacheCommandHandler.cs b/src/cde/ScanProgress/CreateCacheCommandHandler.cs index 31e6253..72633be 100644 --- a/src/cde/ScanProgress/CreateCacheCommandHandler.cs +++ b/src/cde/ScanProgress/CreateCacheCommandHandler.cs @@ -50,7 +50,7 @@ private async Task MainLoop(CreateCacheCommand request, CancellationToken cancel re.SimpleScanEndEvent = () => _messageBus.Publish(new ScanCompletedEvent(), cancellationToken: cancellationToken); re.ExceptionEvent = PrintException; - re.PopulateRoot(request.Path); + re.PopulateRoot(request.Path, request.FollowJunctions); if (Hack.BreakConsoleFlag) { Console.WriteLine(" * Break key detected incomplete scan will not be saved."); diff --git a/src/cdeLib/Catalog/CreateCacheCommand.cs b/src/cdeLib/Catalog/CreateCacheCommand.cs index 6496fd7..94a2375 100644 --- a/src/cdeLib/Catalog/CreateCacheCommand.cs +++ b/src/cdeLib/Catalog/CreateCacheCommand.cs @@ -11,4 +11,10 @@ public CreateCacheCommand(string path) public string Path { get; } public string Description { get; set; } + + /// + /// When false (default), directory reparse points (junctions / symbolic links) are recorded + /// in the catalog but not descended into, avoiding cycles and duplicate content. + /// + public bool FollowJunctions { get; set; } } \ No newline at end of file diff --git a/src/cdeLib/Catalog/CreateCacheCommandHandler.cs b/src/cdeLib/Catalog/CreateCacheCommandHandler.cs index fa90ca7..cec055f 100644 --- a/src/cdeLib/Catalog/CreateCacheCommandHandler.cs +++ b/src/cdeLib/Catalog/CreateCacheCommandHandler.cs @@ -36,7 +36,7 @@ public async Task OnHandle(CreateCacheCommand request, CancellationToken cancell re.SimpleScanEndEvent = ScanEndOfEntries; re.ExceptionEvent = PrintExceptions; - re.PopulateRoot(request.Path); + re.PopulateRoot(request.Path, request.FollowJunctions); if (Hack.BreakConsoleFlag) { Console.WriteLine(" * Break key detected incomplete scan will not be saved."); diff --git a/src/cdeLib/Entities/RootEntry.cs b/src/cdeLib/Entities/RootEntry.cs index aaf9d0f..7c40633 100644 --- a/src/cdeLib/Entities/RootEntry.cs +++ b/src/cdeLib/Entities/RootEntry.cs @@ -24,6 +24,9 @@ public sealed class RootEntry : object, ICommonEntry private readonly IDriveInfoService _driveInfoService; private readonly IFileSystemAdapter _fileSystemAdapter; + // Set per-scan by RecurseTree; when false, directory reparse points are not descended into. + private bool _followJunctions; + [ProtoMember(2, IsRequired = true)] [FlatBufferItem(2)] [Key(2)] @@ -120,11 +123,11 @@ public RootEntry(IConfiguration configuration, IFileSystemAdapter fileSystemAdap } } - public void PopulateRoot(string startPath) + public void PopulateRoot(string startPath, bool followJunctions = false) { startPath = GetRootEntry(startPath); ScanStartUtc = DateTime.UtcNow; - RecurseTree(startPath); + RecurseTree(startPath, followJunctions); ScanEndUtc = DateTime.UtcNow; SetInMemoryFields(); } @@ -286,8 +289,9 @@ private static string SafeFileName(string path) /// /// Iteratively scans a directory tree using a stack-based approach for optimal performance. /// - public void RecurseTree(string startPath) + public void RecurseTree(string startPath, bool followJunctions = false) { + _followJunctions = followJunctions; var entryCount = 0; var stack = new Stack<(ICommonEntry, string)>(capacity: 64); stack.Push((this, startPath)); @@ -361,7 +365,10 @@ private void ProcessFileSystemEntry( var dirEntry = new DirEntry(fsInfo); parent.AddChild(dirEntry); - if (dirEntry.IsDirectory) + // Reparse points (junctions / directory symlinks) carry the Directory attribute, so they + // would otherwise be descended into. By default we record them but do not follow them, + // avoiding cycles (e.g. a junction pointing at an ancestor) and duplicate content. + if (dirEntry.IsDirectory && (_followJunctions || !dirEntry.IsReparsePoint)) { stack.Push((dirEntry, fsInfo.FullName)); } From 8d893636e512cb32f7baa473ae0ce59167d85492 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Mon, 8 Jun 2026 11:23:39 +1000 Subject: [PATCH 30/43] docs(claude): add shell here-string rule to prevent tooling mistakes Document that PowerShell here-strings (@'...'@) and Bash here-docs (<<'EOF') are not interchangeable across the two shell tools, after a stray @ leaked into a git commit message from using the wrong syntax. --- claude.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/claude.md b/claude.md index 35d198e..d20f451 100644 --- a/claude.md +++ b/claude.md @@ -246,6 +246,15 @@ This branch focuses on performance improvements and refactoring. Recent commits - Unit tests in cdeLibTest - Specification tests in cdeLibSpec/cdeLibSpec2 +### Shell & Tooling + +This is a Windows environment with both PowerShell and Bash available. The two shells have **incompatible** here-string / quoting syntax — never mix them. + +- **PowerShell here-string** is `@'` ... `'@` (closing `'@` must be at column 0). Only valid in the PowerShell tool. +- **Bash here-doc** is `<<'EOF'` ... `EOF`. Only valid in the Bash tool. +- Passing `@'...'@` to the Bash tool does **not** create a here-string — Bash treats the `@` characters as literal text, which (for example) prepends a stray `@` to git commit messages. +- For multi-line text (commit messages, file content) prefer the matching syntax for the tool you're calling, or write the text to a file and pass it with `-F `. + ### Code Patterns - **SlimMessageBus Commands**: Business operations are commands/queries (`IRequestHandler.OnHandle`) From d01e07cc682b4b6e45360ec9c6d5188dc154dc31 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Mon, 8 Jun 2026 15:57:53 +1000 Subject: [PATCH 31/43] refactor(di): simplify AppContainerBuilder registration - Drop the redundant explicit handler RegisterType<>().AsSelf() block; SlimMessageBus AutoDeclareFrom already self-registers handlers and builder.Populate surfaces them into Autofac. - Extract focused helpers (ConfigureMessageBus, RegisterCoreServices, WarnMissingConfig) and convert BuildContainer to TryBuildContainer. - Replace the hardcoded cdeLib-override exclusion list with a convention: skip any cdeLib request handler whose request type the cde assembly also handles (IConsumer pub/sub events stay additive). Self-maintaining. - Move cdeLib Logger->ILogger registration into CdelibModule where it belongs. Note: RegisterCoreServices takes IConfigurationRoot (not IConfiguration) so the instance registers as IConfigurationRoot, which Configuration depends on. --- src/cde/AppContainerBuilder.cs | 94 ++++++++++++++++++++----------- src/cde/Program.cs | 3 +- src/cdeLib/Module/CdelibModule.cs | 1 + 3 files changed, 64 insertions(+), 34 deletions(-) diff --git a/src/cde/AppContainerBuilder.cs b/src/cde/AppContainerBuilder.cs index 1ef5b87..6b42396 100644 --- a/src/cde/AppContainerBuilder.cs +++ b/src/cde/AppContainerBuilder.cs @@ -1,4 +1,8 @@ +using System; +using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Reflection; using Autofac; using Autofac.Extensions.DependencyInjection; using AutofacSerilogIntegration; @@ -8,6 +12,7 @@ using Microsoft.Extensions.DependencyInjection; using Serilog; using Serilog.Events; +using SlimMessageBus; using SlimMessageBus.Host; using SlimMessageBus.Host.Memory; @@ -19,63 +24,88 @@ namespace cde; public static class AppContainerBuilder { /// - /// Build the DI container. Returns null if appsettings.json is missing. + /// Builds the DI container. Returns false (and leaves null) + /// if appsettings.json is missing; otherwise returns true with the built container. /// - public static IContainer BuildContainer(string[] args) + public static bool TryBuildContainer(string[] args, out IContainer container) { + container = null; ConfigureBootstrapLogger(); // Check for appsettings.json before attempting to build if (!ConfigBuilder.AppSettingsExists()) { - var currentDir = Directory.GetCurrentDirectory(); - Log.Logger.Warning( - "Configuration file '{FileName}' not found in '{Directory}'", - ConfigBuilder.AppSettingsFileName, currentDir); - Log.Logger.Warning( - "Please ensure appsettings.json is in the same directory as the executable"); - return null; + WarnMissingConfig(); + return false; } + var config = ConfigBuilder.Build(args); + ConfigureLogger(config); + var services = new ServiceCollection(); + ConfigureMessageBus(services); + + var builder = new ContainerBuilder(); + RegisterCoreServices(builder, config); + builder.Populate(services); // surfaces SlimMessageBus + auto-declared handlers into Autofac + container = builder.Build(); + return true; + } + + private static void ConfigureMessageBus(IServiceCollection services) + { // Add logging (required by SlimMessageBus) services.AddLogging(loggingBuilder => loggingBuilder.AddSerilog(dispose: false)); + // The cde CLI replaces some cdeLib request handlers with Spectre-progress variants. A request + // type can only have one handler, so skip any cdeLib handler whose request type the cde assembly + // also handles — the cde override then binds alone. New overrides are detected automatically; + // there is no hand-maintained exclusion list. (IConsumer pub/sub events are additive, never skipped.) + var cdeRequestTypes = RequestTypesHandledIn(typeof(AppContainerBuilder).Assembly); + services.AddSlimMessageBus(mbb => { mbb.WithProviderMemory() - // Filter out cdeLib CreateCacheCommandHandler since cde assembly overrides it .AutoDeclareFrom(typeof(CdelibModule).Assembly, - consumerTypeFilter: t => t != typeof(cdeLib.Catalog.CreateCacheCommandHandler) - && t != typeof(cdeLib.Hashing.HashCatalogCommandHandler)) + consumerTypeFilter: t => !HandlesAnyRequest(t, cdeRequestTypes)) .AutoDeclareFrom(typeof(AppContainerBuilder).Assembly); }); + } - var builder = new ContainerBuilder(); - var config = ConfigBuilder.Build(args); - ConfigureLogger(config); + /// Request types handled by implementations in the assembly. + private static HashSet RequestTypesHandledIn(Assembly assembly) + => assembly.GetTypes().SelectMany(RequestTypesOf).ToHashSet(); + + private static bool HandlesAnyRequest(Type handler, HashSet requestTypes) + => RequestTypesOf(handler).Any(requestTypes.Contains); + + private static IEnumerable RequestTypesOf(Type handler) + => handler.GetInterfaces() + .Where(i => i.IsGenericType + && (i.GetGenericTypeDefinition() == typeof(IRequestHandler<>) + || i.GetGenericTypeDefinition() == typeof(IRequestHandler<,>))) + .Select(i => i.GetGenericArguments()[0]); + + private static void RegisterCoreServices(ContainerBuilder builder, IConfigurationRoot config) + { + // Register as IConfigurationRoot (its compile-time type) — cdeLib.Infrastructure.Configuration + // depends on IConfigurationRoot, so widening this to IConfiguration would break resolution. builder.RegisterInstance(config); - builder.RegisterType().As(); builder.RegisterLogger(); - builder.RegisterModule(); + // Handlers are registered by SlimMessageBus AutoDeclareFrom (addServicesFromAssembly: true) + // and surfaced into Autofac via builder.Populate(services) — no explicit handler registration needed. + } - // Populate Autofac from ServiceCollection (for SlimMessageBus) - builder.Populate(services); - - // Register handlers explicitly in Autofac to ensure they can be resolved - builder.RegisterType().AsSelf(); - builder.RegisterType().AsSelf(); - builder.RegisterType().AsSelf(); - builder.RegisterType().AsSelf(); - builder.RegisterType().AsSelf(); - builder.RegisterType().AsSelf(); - builder.RegisterType().AsSelf(); - builder.RegisterType().AsSelf(); - builder.RegisterType().AsSelf(); - - return builder.Build(); + private static void WarnMissingConfig() + { + var currentDir = Directory.GetCurrentDirectory(); + Log.Logger.Warning( + "Configuration file '{FileName}' not found in '{Directory}'", + ConfigBuilder.AppSettingsFileName, currentDir); + Log.Logger.Warning( + "Please ensure appsettings.json is in the same directory as the executable"); } private static void ConfigureBootstrapLogger() diff --git a/src/cde/Program.cs b/src/cde/Program.cs index 50eb7fa..cff0607 100644 --- a/src/cde/Program.cs +++ b/src/cde/Program.cs @@ -35,8 +35,7 @@ public static class Program /// public static bool InitProgram(string[] args) { - _container = AppContainerBuilder.BuildContainer(args); - if (_container == null) + if (!AppContainerBuilder.TryBuildContainer(args, out _container)) { return false; } diff --git a/src/cdeLib/Module/CdelibModule.cs b/src/cdeLib/Module/CdelibModule.cs index a512327..f7dd4f5 100644 --- a/src/cdeLib/Module/CdelibModule.cs +++ b/src/cdeLib/Module/CdelibModule.cs @@ -13,6 +13,7 @@ protected override void Load(ContainerBuilder builder) { // singletons. builder.RegisterType().As().SingleInstance(); + builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterType().As().SingleInstance(); From a0f3f180fe117cd6a854a0a2f3053189df77ffc7 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Mon, 8 Jun 2026 15:58:43 +1000 Subject: [PATCH 32/43] tidy: Remove unused parameter --- src/cde/ScanProgress/CreateCacheCommandHandler.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/cde/ScanProgress/CreateCacheCommandHandler.cs b/src/cde/ScanProgress/CreateCacheCommandHandler.cs index 72633be..ec86de6 100644 --- a/src/cde/ScanProgress/CreateCacheCommandHandler.cs +++ b/src/cde/ScanProgress/CreateCacheCommandHandler.cs @@ -20,14 +20,11 @@ namespace cde.ScanProgress; public class CreateCacheCommandHandler : IRequestHandler { private readonly IConfiguration _configuration; - private readonly ICatalogRepository _catalogRepository; private readonly IMessageBus _messageBus; - public CreateCacheCommandHandler(IConfiguration configuration, ICatalogRepository catalogRepository, - IMessageBus messageBus) + public CreateCacheCommandHandler(IConfiguration configuration, IMessageBus messageBus) { _configuration = configuration; - _catalogRepository = catalogRepository; _messageBus = messageBus; } From 73abbf921ea8c94610c184f57bf06a8504312cc2 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Mon, 8 Jun 2026 16:10:45 +1000 Subject: [PATCH 33/43] refactor(cde): extract command logic into injected CdeApp Program was a god class: composition root, CLI parser, and every command's implementation, pulling dependencies ad-hoc via Resolve() (service locator). - Add CdeApp: command implementations with IFindService/ICatalogRepository/ IMessageBus injected via constructor. Removes all 8 scattered Resolve() calls; Program now resolves one CdeApp and dispatches parsed verbs to it (382 -> ~110 lines, single responsibility). - Drop the Task.Run().Wait() sync-over-async wrappers around MessageBus.Send; call GetAwaiter().GetResult() directly (no threadpool hop). - FindPopulous: remove the redundant second Where filter and the CompareDirEntries comparator; single Where + OrderByDescending. - Extract MigrateOne, HandleReplCommand, PrintReplHelp from long methods. Static Program.CreateCache/HashCatalog shims retained (delegate to CdeApp) so cdeLibTest/DuplicationTest keeps driving scans through Program. --- src/cde/AppContainerBuilder.cs | 1 + src/cde/CdeApp.cs | 295 ++++++++++++++++++++ src/cde/Program.cs | 484 +++++++-------------------------- 3 files changed, 398 insertions(+), 382 deletions(-) create mode 100644 src/cde/CdeApp.cs diff --git a/src/cde/AppContainerBuilder.cs b/src/cde/AppContainerBuilder.cs index 6b42396..a65b305 100644 --- a/src/cde/AppContainerBuilder.cs +++ b/src/cde/AppContainerBuilder.cs @@ -94,6 +94,7 @@ private static void RegisterCoreServices(ContainerBuilder builder, IConfiguratio builder.RegisterInstance(config); builder.RegisterLogger(); builder.RegisterModule(); + builder.RegisterType(); // Handlers are registered by SlimMessageBus AutoDeclareFrom (addServicesFromAssembly: true) // and surfaced into Autofac via builder.Populate(services) — no explicit handler registration needed. } diff --git a/src/cde/CdeApp.cs b/src/cde/CdeApp.cs new file mode 100644 index 0000000..9882d0b --- /dev/null +++ b/src/cde/CdeApp.cs @@ -0,0 +1,295 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using cde.CommandLine; +using cdeLib; +using cdeLib.Catalog; +using cdeLib.Duplicates; +using cdeLib.Entities; +using cdeLib.Entities.Columnar; +using cdeLib.Entities.Soa; +using cdeLib.Hashing; +using cdeLib.Upgrade; +using Mono.Terminal; +using Serilog; +using SlimMessageBus; + +namespace cde; + +/// +/// Hosts the cde command implementations with their dependencies injected, keeping the logic free of +/// service-locator lookups and unit-testable in isolation. resolves a single +/// instance and dispatches parsed CLI verbs to it. +/// +public sealed class CdeApp( + IFindService findService, + ICatalogRepository repository, + IMessageBus messageBus) +{ + // ---- catalog commands (routed through the message bus) ---- + + public void CreateCache(ScanOptions opts) => + messageBus.Send(new CreateCacheCommand(opts.Path) + { Description = opts.Description, FollowJunctions = opts.FollowJunctions }) + .GetAwaiter().GetResult(); + + public void HashCatalog() => + messageBus.Send(new HashCatalogCommand()).GetAwaiter().GetResult(); + + public void FindDupes() => + messageBus.Send(new FindDuplicatesCommand()).GetAwaiter().GetResult(); + + public void Update(UpdateOptions opts) => + messageBus.Send(new UpdateCommand { FileName = opts.FileName, Description = opts.Description }) + .GetAwaiter().GetResult(); + + // ---- find ---- + + /// + /// Run a find, preferring the zero-copy columnar format: if any .cdex catalogs exist in the + /// current dir (or one level down) they are searched over their memory maps with no managed catalog + /// load; otherwise we fall back to loading the MessagePack .cde trees. + /// + public void RunFind(string value, string param) + { + var cdex = repository.GetColumnarFileList(["./"]); + if (cdex.Count == 0) + { + findService.Find(value, param, repository.LoadCurrentDirCache()); + return; + } + + var readers = new List(cdex.Count); + try + { + foreach (var file in cdex) + { + try + { + readers.Add(new ColumnarCatalogReader(file)); + } + catch (Exception ex) + { + Log.Logger.Warning(ex, "Skipping unreadable .cdex {File}", file); + } + } + + findService.FindColumnar(value, param, readers); + } + finally + { + foreach (var reader in readers) reader.Dispose(); + } + } + + // repl = read-eval-print-loop + public void FindRepl(string paramString, string firstPattern) + { + var rootEntries = repository.LoadCurrentDirCache(); + + if (!string.IsNullOrEmpty(firstPattern)) + findService.Find(firstPattern, paramString, rootEntries); + + Console.WriteLine("Issue --help for available params"); + + while (true) + { + if (Hack.BreakConsoleFlag) + Hack.BreakConsoleFlag = false; //reset otherwise we'll get some weird behaviour in loop. + Console.Write("Enter string to search : "); + var pattern = Console.ReadLine(); + if (string.IsNullOrEmpty(pattern)) + { + Console.WriteLine("Exiting..."); + break; + } + + if (pattern.StartsWith("--", StringComparison.CurrentCulture)) + { + HandleReplCommand(pattern[2..]); + } + else + { + findService.Find(pattern, paramString, rootEntries); + } + } + } + + private void HandleReplCommand(string command) + { + switch (command.ToLower(CultureInfo.CurrentCulture)) + { + case "includefiles": + findService.IncludeFiles = !findService.IncludeFiles; + Console.WriteLine($"IncludeFiles:{findService.IncludeFiles}"); + break; + case "includefolders": + findService.IncludeFolders = !findService.IncludeFolders; + Console.WriteLine($"IncludeFolders:{findService.IncludeFolders}"); + break; + case "help": + Console.WriteLine("Valid options are"); + Console.WriteLine("--includefiles"); + Console.WriteLine("--includefolders"); + break; + case "clear": + Console.Clear(); + break; + default: + Console.WriteLine($"unknown command {command}"); + break; + } + } + + // ---- migrate ---- + + /// + /// One-way migration of MessagePack .cde catalogs to the zero-copy columnar .cdex format. With a + /// path argument, converts that file; otherwise converts every catalog discovered in the current + /// directory and one level down, writing a .cdex beside each source. + /// + public void Migrate(MigrateOptions opts) + { + List files; + if (!string.IsNullOrWhiteSpace(opts.Path)) + { + if (!File.Exists(opts.Path)) + { + Console.WriteLine($"File not found: {opts.Path}"); + return; + } + files = [opts.Path]; + } + else + { + files = repository.GetCacheFileList(["./"]).ToList(); + } + + if (files.Count == 0) + { + Console.WriteLine("No .cde catalogs found to migrate."); + return; + } + + var converted = 0; + foreach (var file in files) + { + if (MigrateOne(file)) converted++; + } + + Console.WriteLine($"Migrated {converted} of {files.Count} catalog(s) to .cdex."); + } + + private bool MigrateOne(string file) + { + try + { + var root = repository.LoadDirCache(file); + if (root == null) + { + Console.WriteLine($" skip (could not load): {file}"); + return false; + } + + var store = EntryStore.Build(root); + var outFile = Path.ChangeExtension(file, ".cdex"); + ColumnarFormat.Write(store, outFile); + + var srcLen = new FileInfo(file).Length; + var dstLen = new FileInfo(outFile).Length; + Console.WriteLine( + $" {Path.GetFileName(file)} ({srcLen:N0} B) -> {Path.GetFileName(outFile)} " + + $"({dstLen:N0} B, {store.Count:N0} entries)"); + return true; + } + catch (Exception ex) + { + Console.WriteLine($" error migrating {file}: {ex.Message}"); + return false; + } + } + + // ---- repl / inspection ---- + + public void InvokeRepl() + { + var le = new LineEditor(name: null); + var running = true; + + while (running && le.Edit("shell> ", string.Empty) is { } s) + { + Console.WriteLine($"----> [{s}]"); + switch (s) + { + case "quit": + running = false; + break; + case "history": + case "!": + le.CmdHistoryDump(); + break; + case "help": + case "?": + PrintReplHelp(); + break; + } + } + } + + private static void PrintReplHelp() + { + Console.WriteLine("Builtin Commands:"); + Console.WriteLine(" quit - quit,"); + Console.WriteLine(" help - show help, ? - show help"); + Console.WriteLine(" history - show history, ! - show history"); + Console.WriteLine("Keystrokes:"); + Console.WriteLine(" Home, End, Left, Right, Up, Down, Back, Del, Tab"); + Console.WriteLine(" C-a, C-e, C-b, C-f, C-p, C-n, C-d"); + Console.WriteLine(" C-l - clear console to top"); + Console.WriteLine(" C-r - reverse search history"); + Console.WriteLine(" A-b - move backward word"); + Console.WriteLine(" A-f - move forward word"); + Console.WriteLine(" A-d - delete word forward"); + Console.WriteLine(" A-Backspace - delete word backward"); + } + + public void LoadWait() + { + repository.LoadCurrentDirCache(); + Console.ReadLine(); + } + + public void PrintPathsHaveHash() + { + var rootEntries = repository.LoadCurrentDirCache(); + foreach (var pairDirEntry in EntryHelper.GetPairDirEntries(rootEntries)) + { + var hash = pairDirEntry.ChildDE.IsHashDone ? "#" : " "; + var bang = pairDirEntry.PathProblem ? "!" : " "; + Console.WriteLine($"{hash}{bang}{pairDirEntry.FullPath}"); + if (Hack.BreakConsoleFlag) + { + break; + } + } + } + + public void FindPopulous(int minimumCount) + { + var largeEntries = EntryHelper.GetDirEntries(repository.LoadCurrentDirCache()) + .Where(e => e.Children is { } c && c.Count > minimumCount) + .OrderByDescending(e => e.Children.Count) + .ToList(); + + foreach (var e in largeEntries) + { + Console.WriteLine($"{e.FullPath} {e.Children.Count}"); + if (Hack.BreakConsoleFlag) + { + break; + } + } + } +} diff --git a/src/cde/Program.cs b/src/cde/Program.cs index cff0607..50e0def 100644 --- a/src/cde/Program.cs +++ b/src/cde/Program.cs @@ -1,382 +1,102 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Threading.Tasks; -using Autofac; -using cde.CommandLine; -using cdeLib; -using cdeLib.Catalog; -using cdeLib.Duplicates; -using cdeLib.Entities; -using cdeLib.Entities.Columnar; -using cdeLib.Entities.Soa; -using cdeLib.Hashing; -using cdeLib.Upgrade; -using CommandLine; -using SlimMessageBus; -using Mono.Terminal; -using Serilog; -using SerilogTimings; -using FindOptions = cde.CommandLine.FindOptions; -using IContainer = Autofac.IContainer; - -namespace cde; - -public static class Program -{ - private static IContainer _container; - - private static IMessageBus MessageBus { get; set; } - - /// - /// Initialize the program. Returns false if initialization failed (e.g., missing config). - /// - public static bool InitProgram(string[] args) - { - if (!AppContainerBuilder.TryBuildContainer(args, out _container)) - { - return false; - } - MessageBus = Resolve(); - return true; - } - - private static ParserResult GetParserResult(IEnumerable args) - { - var parser = CommandLineParserBuilder.Build(); - return parser.ParseArguments< - ScanOptions, - FindOptions, - GrepOptions, - GrepPathOptions, - ReplGrepPathOptions, - ReplGrepOptions, - ReplFindOptions, - MigrateOptions, - HashOptions, - DupesOptions, - TreeDumpOptions, - LoadWaitOptions, - ReplOptions, - PopulousFoldersOptions, - FindPathOptions, - UpdateOptions>(args); - } - - private static int Main(string[] args) - { - if (!InitProgram(args)) - { - return 1; // Exit with error code if initialization failed - } - Console.CancelKeyPress += BreakConsole; - try - { - using (Operation.Time("App")) - { - var findService = Resolve(); - var parsedResult = GetParserResult(args) - .WithParsed(CreateCache) - .WithParsed(opts => RunFind(findService, opts.Value, "--find")) - .WithParsed(opts => RunFind(findService, opts.Value, "--findpath")) - .WithParsed(opts => RunFind(findService, opts.Value, "--grep")) - .WithParsed(opts => RunFind(findService, opts.Value, "--greppath")) - .WithParsed(opts => FindRepl(FindService.ParamGrepPath, opts.Value)) - .WithParsed(opts => FindRepl(FindService.ParamGrep, opts.Value)) - .WithParsed(opts => FindRepl(FindService.ParamFind, opts.Value)) - .WithParsed(Migrate) - .WithParsed(_ => HashCatalog()) - .WithParsed(_ => FindDupes()) - .WithParsed(_ => PrintPathsHaveHashEnumerator()) - .WithParsed(_ => - { - Resolve().LoadCurrentDirCache(); - Console.ReadLine(); - }) - .WithParsed(_ => InvokeRepl()) - .WithParsed(opts => FindPopulous(opts.Count)) - .WithParsed(Update); - parsedResult.WithNotParsed(errs => CustomHelpText.DisplayHelp(parsedResult)); - return 0; - } - } - finally - { - Log.CloseAndFlush(); - } - } - - private static T Resolve() - { - return _container.Resolve(); - } - - /// - /// Run a find, preferring the zero-copy columnar format: if any .cdex catalogs exist in the - /// current dir (or one level down) they are searched over their memory maps with no managed catalog - /// load; otherwise we fall back to loading the MessagePack .cde trees. - /// - private static void RunFind(IFindService findService, string value, string param) - { - var repo = Resolve(); - var cdex = repo.GetColumnarFileList(["./"]); - if (cdex.Count == 0) - { - findService.Find(value, param, repo.LoadCurrentDirCache()); - return; - } - - var readers = new List(cdex.Count); - try - { - foreach (var file in cdex) - { - try - { - readers.Add(new ColumnarCatalogReader(file)); - } - catch (Exception ex) - { - Log.Logger.Warning(ex, "Skipping unreadable .cdex {File}", file); - } - } - - findService.FindColumnar(value, param, readers); - } - finally - { - foreach (var reader in readers) reader.Dispose(); - } - } - - /// - /// One-way migration of MessagePack .cde catalogs to the zero-copy columnar .cdex format. With a - /// path argument, converts that file; otherwise converts every catalog discovered in the current - /// directory and one level down, writing a .cdex beside each source. - /// - private static void Migrate(MigrateOptions opts) - { - var repo = Resolve(); - - List files; - if (!string.IsNullOrWhiteSpace(opts.Path)) - { - if (!File.Exists(opts.Path)) - { - Console.WriteLine($"File not found: {opts.Path}"); - return; - } - files = [opts.Path]; - } - else - { - files = repo.GetCacheFileList(["./"]).ToList(); - } - - if (files.Count == 0) - { - Console.WriteLine("No .cde catalogs found to migrate."); - return; - } - - var converted = 0; - foreach (var file in files) - { - try - { - var root = repo.LoadDirCache(file); - if (root == null) - { - Console.WriteLine($" skip (could not load): {file}"); - continue; - } - - var store = EntryStore.Build(root); - var outFile = Path.ChangeExtension(file, ".cdex"); - ColumnarFormat.Write(store, outFile); - - var srcLen = new FileInfo(file).Length; - var dstLen = new FileInfo(outFile).Length; - Console.WriteLine( - $" {Path.GetFileName(file)} ({srcLen:N0} B) -> {Path.GetFileName(outFile)} " + - $"({dstLen:N0} B, {store.Count:N0} entries)"); - converted++; - } - catch (Exception ex) - { - Console.WriteLine($" error migrating {file}: {ex.Message}"); - } - } - - Console.WriteLine($"Migrated {converted} of {files.Count} catalog(s) to .cdex."); - } - - private static void InvokeRepl() - { - var le = new LineEditor(name: null); - var running = true; - - while (running && le.Edit("shell> ", string.Empty) is { } s) - { - Console.WriteLine($"----> [{s}]"); - switch (s) - { - case "quit": - running = false; - break; - case "history": - case "!": - le.CmdHistoryDump(); - break; - case "help": - case "?": - Console.WriteLine("Builtin Commands:"); - Console.WriteLine(" quit - quit,"); - Console.WriteLine(" help - show help, ? - show help"); - Console.WriteLine(" history - show history, ! - show history"); - Console.WriteLine("Keystrokes:"); - Console.WriteLine(" Home, End, Left, Right, Up, Down, Back, Del, Tab"); - Console.WriteLine(" C-a, C-e, C-b, C-f, C-p, C-n, C-d"); - Console.WriteLine(" C-l - clear console to top"); - Console.WriteLine(" C-r - reverse search history"); - Console.WriteLine(" A-b - move backward word"); - Console.WriteLine(" A-f - move forward word"); - Console.WriteLine(" A-d - delete word forward"); - Console.WriteLine(" A-Backspace - delete word backward"); - break; - } - } - } - - private static void BreakConsole(object sender, ConsoleCancelEventArgs e) - { - Console.WriteLine("\n * Break key detected. will exit as soon as current file process is completed."); - Hack.BreakConsoleFlag = true; - e.Cancel = true; - } - - // repl = read-eval-print-loop - private static void FindRepl(string paramString, string firstPattern) - { - var rootEntries = Resolve().LoadCurrentDirCache(); - var findService = Resolve(); - - if (!string.IsNullOrEmpty(firstPattern)) - findService.Find(firstPattern, paramString, rootEntries); - - Console.WriteLine("Issue --help for available params"); - - while (true) - { - if (Hack.BreakConsoleFlag) - Hack.BreakConsoleFlag = false; //reset otherwise we'll get some weird behaviour in loop. - Console.Write("Enter string to search : "); - var pattern = Console.ReadLine(); - if (string.IsNullOrEmpty(pattern)) - { - Console.WriteLine("Exiting..."); - break; - } - - if (pattern.StartsWith("--", StringComparison.CurrentCulture)) - { - var command = pattern[2..]; - switch (command.ToLower(CultureInfo.CurrentCulture)) - { - case "includefiles": - findService.IncludeFiles = !findService.IncludeFiles; - Console.WriteLine($"IncludeFiles:{findService.IncludeFiles}"); - break; - case "includefolders": - findService.IncludeFolders = !findService.IncludeFolders; - Console.WriteLine($"IncludeFolders:{findService.IncludeFolders}"); - break; - case "help": - Console.WriteLine("Valid options are"); - Console.WriteLine("--includefiles"); - Console.WriteLine("--includefolders"); - break; - case "clear": - Console.Clear(); - break; - default: - Console.WriteLine($"unknown command {command}"); - break; - } - } - else - { - findService.Find(pattern, paramString, rootEntries); - } - } - } - - private static void Update(UpdateOptions opts) - { - var task = Task.Run(() => - MessageBus.Send(new UpdateCommand { FileName = opts.FileName, Description = opts.Description })); - task.Wait(); - } - - private static void FindDupes() - { - var task = Task.Run(() => MessageBus.Send(new FindDuplicatesCommand())); - task.Wait(); - } - - public static void HashCatalog() - { - var task = Task.Run(async () => await MessageBus.Send(new HashCatalogCommand()).ConfigureAwait(false)); - task.Wait(); - } - - public static void CreateCache(ScanOptions opts) - { - var task = Task.Run(async () => - await MessageBus.Send(new CreateCacheCommand(opts.Path) - { Description = opts.Description, FollowJunctions = opts.FollowJunctions }) - .ConfigureAwait(false)); - task.Wait(); - } - - private static void PrintPathsHaveHashEnumerator() - { - var rootEntries = Resolve().LoadCurrentDirCache(); - foreach (var pairDirEntry in EntryHelper.GetPairDirEntries(rootEntries)) - { - var hash = pairDirEntry.ChildDE.IsHashDone ? "#" : " "; - var bang = pairDirEntry.PathProblem ? "!" : " "; - Console.WriteLine($"{hash}{bang}{pairDirEntry.FullPath}"); - if (Hack.BreakConsoleFlag) - { - break; - } - } - } - - private static void FindPopulous(int minimumCount) - { - var rootEntries = Resolve().LoadCurrentDirCache(); - var entries = EntryHelper.GetDirEntries(rootEntries); - var largeEntries = entries - .Where(e => e.Children != null && e.Children.Count > minimumCount) - .ToList(); - largeEntries.Sort(CompareDirEntries); - - foreach (var e in largeEntries.Where(e => e.Children != null && e.Children.Count > minimumCount)) - { - Console.WriteLine($"{e.FullPath} {e.Children.Count}"); - if (Hack.BreakConsoleFlag) - { - break; - } - } - } - - private static int CompareDirEntries(ICommonEntry x, ICommonEntry y) - { - return y.Children.Count - x.Children.Count; - } -} \ No newline at end of file +using System; +using System.Collections.Generic; +using Autofac; +using cde.CommandLine; +using cdeLib; +using CommandLine; +using Serilog; +using SerilogTimings; +using FindOptions = cde.CommandLine.FindOptions; +using IContainer = Autofac.IContainer; + +namespace cde; + +public static class Program +{ + private static IContainer _container; + private static CdeApp _app; + + /// + /// Initialize the program. Returns false if initialization failed (e.g., missing config). + /// + public static bool InitProgram(string[] args) + { + if (!AppContainerBuilder.TryBuildContainer(args, out _container)) + { + return false; + } + _app = _container.Resolve(); + return true; + } + + // Static entry points retained for cdeLibTest/DuplicationTest, which drives a scan+hash via Program. + public static void CreateCache(ScanOptions opts) => _app.CreateCache(opts); + public static void HashCatalog() => _app.HashCatalog(); + + private static ParserResult GetParserResult(IEnumerable args) + { + var parser = CommandLineParserBuilder.Build(); + return parser.ParseArguments< + ScanOptions, + FindOptions, + GrepOptions, + GrepPathOptions, + ReplGrepPathOptions, + ReplGrepOptions, + ReplFindOptions, + MigrateOptions, + HashOptions, + DupesOptions, + TreeDumpOptions, + LoadWaitOptions, + ReplOptions, + PopulousFoldersOptions, + FindPathOptions, + UpdateOptions>(args); + } + + private static int Main(string[] args) + { + if (!InitProgram(args)) + { + return 1; // Exit with error code if initialization failed + } + Console.CancelKeyPress += BreakConsole; + try + { + using (Operation.Time("App")) + { + var parsedResult = GetParserResult(args) + .WithParsed(_app.CreateCache) + .WithParsed(opts => _app.RunFind(opts.Value, "--find")) + .WithParsed(opts => _app.RunFind(opts.Value, "--findpath")) + .WithParsed(opts => _app.RunFind(opts.Value, "--grep")) + .WithParsed(opts => _app.RunFind(opts.Value, "--greppath")) + .WithParsed(opts => _app.FindRepl(FindService.ParamGrepPath, opts.Value)) + .WithParsed(opts => _app.FindRepl(FindService.ParamGrep, opts.Value)) + .WithParsed(opts => _app.FindRepl(FindService.ParamFind, opts.Value)) + .WithParsed(_app.Migrate) + .WithParsed(_ => _app.HashCatalog()) + .WithParsed(_ => _app.FindDupes()) + .WithParsed(_ => _app.PrintPathsHaveHash()) + .WithParsed(_ => _app.LoadWait()) + .WithParsed(_ => _app.InvokeRepl()) + .WithParsed(opts => _app.FindPopulous(opts.Count)) + .WithParsed(_app.Update); + parsedResult.WithNotParsed(errs => CustomHelpText.DisplayHelp(parsedResult)); + return 0; + } + } + finally + { + Log.CloseAndFlush(); + } + } + + private static void BreakConsole(object sender, ConsoleCancelEventArgs e) + { + Console.WriteLine("\n * Break key detected. will exit as soon as current file process is completed."); + Hack.BreakConsoleFlag = true; + e.Cancel = true; + } +} From 5bbffc0995a2cdb534ffc53a383498f3d0493cc7 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Mon, 8 Jun 2026 16:11:28 +1000 Subject: [PATCH 34/43] task: spelling --- src/cde.sln.DotSettings | 1 + 1 file changed, 1 insertion(+) diff --git a/src/cde.sln.DotSettings b/src/cde.sln.DotSettings index b7a4fb2..6e166e7 100644 --- a/src/cde.sln.DotSettings +++ b/src/cde.sln.DotSettings @@ -13,6 +13,7 @@ True True True + True True True True From 95e4fd4bea6dc6b0b98136cdc07e5a82768ce2b0 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Mon, 8 Jun 2026 16:15:58 +1000 Subject: [PATCH 35/43] refactor(cde): make CLI path fully async, drop sync-over-async Main is now async Task and dispatches the parsed verb via pattern matching. Bus-backed commands (CreateCache/Hash/Dupes/Update) return Task and are awaited directly, so the production path no longer blocks a thread on GetAwaiter().GetResult(). Synchronous interactive/inspection commands are adapted to a completed task via RunSync. The verb switch also replaces the long WithParsed fluent chain. The static Program.CreateCache/HashCatalog shims (used only by DuplicationTest) block on the async methods; that blocking is now confined to test-support code, not the CLI entry path. Test remains unchanged and green. --- src/cde/CdeApp.cs | 19 +++++++------ src/cde/Program.cs | 66 +++++++++++++++++++++++++++++++--------------- 2 files changed, 54 insertions(+), 31 deletions(-) diff --git a/src/cde/CdeApp.cs b/src/cde/CdeApp.cs index 9882d0b..95e9054 100644 --- a/src/cde/CdeApp.cs +++ b/src/cde/CdeApp.cs @@ -3,6 +3,7 @@ using System.Globalization; using System.IO; using System.Linq; +using System.Threading.Tasks; using cde.CommandLine; using cdeLib; using cdeLib.Catalog; @@ -30,20 +31,18 @@ public sealed class CdeApp( { // ---- catalog commands (routed through the message bus) ---- - public void CreateCache(ScanOptions opts) => + public Task CreateCacheAsync(ScanOptions opts) => messageBus.Send(new CreateCacheCommand(opts.Path) - { Description = opts.Description, FollowJunctions = opts.FollowJunctions }) - .GetAwaiter().GetResult(); + { Description = opts.Description, FollowJunctions = opts.FollowJunctions }); - public void HashCatalog() => - messageBus.Send(new HashCatalogCommand()).GetAwaiter().GetResult(); + public Task HashCatalogAsync() => + messageBus.Send(new HashCatalogCommand()); - public void FindDupes() => - messageBus.Send(new FindDuplicatesCommand()).GetAwaiter().GetResult(); + public Task FindDupesAsync() => + messageBus.Send(new FindDuplicatesCommand()); - public void Update(UpdateOptions opts) => - messageBus.Send(new UpdateCommand { FileName = opts.FileName, Description = opts.Description }) - .GetAwaiter().GetResult(); + public Task UpdateAsync(UpdateOptions opts) => + messageBus.Send(new UpdateCommand { FileName = opts.FileName, Description = opts.Description }); // ---- find ---- diff --git a/src/cde/Program.cs b/src/cde/Program.cs index 50e0def..f8e196a 100644 --- a/src/cde/Program.cs +++ b/src/cde/Program.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Threading.Tasks; using Autofac; using cde.CommandLine; using cdeLib; @@ -30,8 +31,9 @@ public static bool InitProgram(string[] args) } // Static entry points retained for cdeLibTest/DuplicationTest, which drives a scan+hash via Program. - public static void CreateCache(ScanOptions opts) => _app.CreateCache(opts); - public static void HashCatalog() => _app.HashCatalog(); + // These block on the async commands; the blocking is confined to this test-support path, not Main. + public static void CreateCache(ScanOptions opts) => _app.CreateCacheAsync(opts).GetAwaiter().GetResult(); + public static void HashCatalog() => _app.HashCatalogAsync().GetAwaiter().GetResult(); private static ParserResult GetParserResult(IEnumerable args) { @@ -55,7 +57,7 @@ private static ParserResult GetParserResult(IEnumerable args) UpdateOptions>(args); } - private static int Main(string[] args) + private static async Task Main(string[] args) { if (!InitProgram(args)) { @@ -66,24 +68,15 @@ private static int Main(string[] args) { using (Operation.Time("App")) { - var parsedResult = GetParserResult(args) - .WithParsed(_app.CreateCache) - .WithParsed(opts => _app.RunFind(opts.Value, "--find")) - .WithParsed(opts => _app.RunFind(opts.Value, "--findpath")) - .WithParsed(opts => _app.RunFind(opts.Value, "--grep")) - .WithParsed(opts => _app.RunFind(opts.Value, "--greppath")) - .WithParsed(opts => _app.FindRepl(FindService.ParamGrepPath, opts.Value)) - .WithParsed(opts => _app.FindRepl(FindService.ParamGrep, opts.Value)) - .WithParsed(opts => _app.FindRepl(FindService.ParamFind, opts.Value)) - .WithParsed(_app.Migrate) - .WithParsed(_ => _app.HashCatalog()) - .WithParsed(_ => _app.FindDupes()) - .WithParsed(_ => _app.PrintPathsHaveHash()) - .WithParsed(_ => _app.LoadWait()) - .WithParsed(_ => _app.InvokeRepl()) - .WithParsed(opts => _app.FindPopulous(opts.Count)) - .WithParsed(_app.Update); - parsedResult.WithNotParsed(errs => CustomHelpText.DisplayHelp(parsedResult)); + var parsed = GetParserResult(args); + if (parsed is Parsed ok) + { + await DispatchAsync(ok.Value).ConfigureAwait(false); + } + else + { + CustomHelpText.DisplayHelp(parsed); + } return 0; } } @@ -93,6 +86,37 @@ private static int Main(string[] args) } } + /// + /// Routes a parsed verb to its command. Bus-backed commands are awaited directly; the synchronous + /// (interactive / inspection) commands are adapted to a completed task via . + /// + private static Task DispatchAsync(object options) => options switch + { + ScanOptions o => _app.CreateCacheAsync(o), + FindOptions o => RunSync(() => _app.RunFind(o.Value, "--find")), + FindPathOptions o => RunSync(() => _app.RunFind(o.Value, "--findpath")), + GrepOptions o => RunSync(() => _app.RunFind(o.Value, "--grep")), + GrepPathOptions o => RunSync(() => _app.RunFind(o.Value, "--greppath")), + ReplGrepPathOptions o => RunSync(() => _app.FindRepl(FindService.ParamGrepPath, o.Value)), + ReplGrepOptions o => RunSync(() => _app.FindRepl(FindService.ParamGrep, o.Value)), + ReplFindOptions o => RunSync(() => _app.FindRepl(FindService.ParamFind, o.Value)), + MigrateOptions o => RunSync(() => _app.Migrate(o)), + HashOptions _ => _app.HashCatalogAsync(), + DupesOptions _ => _app.FindDupesAsync(), + TreeDumpOptions _ => RunSync(_app.PrintPathsHaveHash), + LoadWaitOptions _ => RunSync(_app.LoadWait), + ReplOptions _ => RunSync(_app.InvokeRepl), + PopulousFoldersOptions o => RunSync(() => _app.FindPopulous(o.Count)), + UpdateOptions o => _app.UpdateAsync(o), + _ => Task.CompletedTask, + }; + + private static Task RunSync(Action action) + { + action(); + return Task.CompletedTask; + } + private static void BreakConsole(object sender, ConsoleCancelEventArgs e) { Console.WriteLine("\n * Break key detected. will exit as soon as current file process is completed."); From 1480ec709a2a009d5e3f30f2f074bf6b30d94236 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Mon, 8 Jun 2026 16:29:30 +1000 Subject: [PATCH 36/43] refactor: replace global Hack.BreakConsoleFlag with injectable OperationCancellation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Ctrl-C break used a global mutable static (Hack.BreakConsoleFlag) checked across scan traversal, the two-phase hashing, and the CLI inspection loops. - Add OperationCancellation: an injectable, resettable cancellation signal that exposes a real CancellationToken (registered SingleInstance in CdelibModule). Reset() supports the existing "press break again per phase" semantics that a one-shot CancellationToken cannot. - RootEntry.PopulateRoot/RecurseTree/TryEnumerateDirectory take an optional CancellationToken and check token.IsCancellationRequested (Finder/test callers use the default and are unchanged). - Scan handlers (cde + cdeLib) inject the signal, pass its token into the scan, and check it cooperatively after PopulateRoot — preserving the graceful "incomplete scan will not be saved" behaviour (no OperationCanceledException). - Duplication injects the signal; the two hash phases check IsCancellationRequested and Reset() between them. Program's Ctrl-C handler calls Cancel(); CdeApp uses it for the REPL per-prompt reset and the inspection-loop breaks. - Delete Hack.cs. Tests updated to supply the new dependency. --- src/cde/CdeApp.cs | 10 +++--- src/cde/Program.cs | 4 ++- .../ScanProgress/CreateCacheCommandHandler.cs | 9 +++-- .../Catalog/CreateCacheCommandHandler.cs | 8 +++-- src/cdeLib/Duplicates/Duplication.cs | 11 +++--- src/cdeLib/Entities/RootEntry.cs | 16 +++++---- src/cdeLib/Hack.cs | 7 ---- src/cdeLib/Module/CdelibModule.cs | 1 + src/cdeLib/OperationCancellation.cs | 35 +++++++++++++++++++ src/cdeLibTest/DuplicationTest.cs | 10 +++--- .../Infrastructure/DuplicationTests.cs | 3 +- 11 files changed, 78 insertions(+), 36 deletions(-) delete mode 100644 src/cdeLib/Hack.cs create mode 100644 src/cdeLib/OperationCancellation.cs diff --git a/src/cde/CdeApp.cs b/src/cde/CdeApp.cs index 95e9054..5b669ea 100644 --- a/src/cde/CdeApp.cs +++ b/src/cde/CdeApp.cs @@ -27,7 +27,8 @@ namespace cde; public sealed class CdeApp( IFindService findService, ICatalogRepository repository, - IMessageBus messageBus) + IMessageBus messageBus, + OperationCancellation cancellation) { // ---- catalog commands (routed through the message bus) ---- @@ -95,8 +96,7 @@ public void FindRepl(string paramString, string firstPattern) while (true) { - if (Hack.BreakConsoleFlag) - Hack.BreakConsoleFlag = false; //reset otherwise we'll get some weird behaviour in loop. + cancellation.Reset(); // fresh token each prompt so a break during the previous search doesn't carry over. Console.Write("Enter string to search : "); var pattern = Console.ReadLine(); if (string.IsNullOrEmpty(pattern)) @@ -268,7 +268,7 @@ public void PrintPathsHaveHash() var hash = pairDirEntry.ChildDE.IsHashDone ? "#" : " "; var bang = pairDirEntry.PathProblem ? "!" : " "; Console.WriteLine($"{hash}{bang}{pairDirEntry.FullPath}"); - if (Hack.BreakConsoleFlag) + if (cancellation.IsCancellationRequested) { break; } @@ -285,7 +285,7 @@ public void FindPopulous(int minimumCount) foreach (var e in largeEntries) { Console.WriteLine($"{e.FullPath} {e.Children.Count}"); - if (Hack.BreakConsoleFlag) + if (cancellation.IsCancellationRequested) { break; } diff --git a/src/cde/Program.cs b/src/cde/Program.cs index f8e196a..2e5b228 100644 --- a/src/cde/Program.cs +++ b/src/cde/Program.cs @@ -16,6 +16,7 @@ public static class Program { private static IContainer _container; private static CdeApp _app; + private static OperationCancellation _cancellation; /// /// Initialize the program. Returns false if initialization failed (e.g., missing config). @@ -27,6 +28,7 @@ public static bool InitProgram(string[] args) return false; } _app = _container.Resolve(); + _cancellation = _container.Resolve(); return true; } @@ -120,7 +122,7 @@ private static Task RunSync(Action action) private static void BreakConsole(object sender, ConsoleCancelEventArgs e) { Console.WriteLine("\n * Break key detected. will exit as soon as current file process is completed."); - Hack.BreakConsoleFlag = true; + _cancellation.Cancel(); e.Cancel = true; } } diff --git a/src/cde/ScanProgress/CreateCacheCommandHandler.cs b/src/cde/ScanProgress/CreateCacheCommandHandler.cs index ec86de6..c93dc71 100644 --- a/src/cde/ScanProgress/CreateCacheCommandHandler.cs +++ b/src/cde/ScanProgress/CreateCacheCommandHandler.cs @@ -21,11 +21,14 @@ public class CreateCacheCommandHandler : IRequestHandler { private readonly IConfiguration _configuration; private readonly IMessageBus _messageBus; + private readonly OperationCancellation _cancellation; - public CreateCacheCommandHandler(IConfiguration configuration, IMessageBus messageBus) + public CreateCacheCommandHandler(IConfiguration configuration, IMessageBus messageBus, + OperationCancellation cancellation) { _configuration = configuration; _messageBus = messageBus; + _cancellation = cancellation; } public async Task OnHandle(CreateCacheCommand request, CancellationToken cancellationToken) @@ -47,8 +50,8 @@ private async Task MainLoop(CreateCacheCommand request, CancellationToken cancel re.SimpleScanEndEvent = () => _messageBus.Publish(new ScanCompletedEvent(), cancellationToken: cancellationToken); re.ExceptionEvent = PrintException; - re.PopulateRoot(request.Path, request.FollowJunctions); - if (Hack.BreakConsoleFlag) + re.PopulateRoot(request.Path, request.FollowJunctions, _cancellation.Token); + if (_cancellation.IsCancellationRequested) { Console.WriteLine(" * Break key detected incomplete scan will not be saved."); return; diff --git a/src/cdeLib/Catalog/CreateCacheCommandHandler.cs b/src/cdeLib/Catalog/CreateCacheCommandHandler.cs index cec055f..3dce121 100644 --- a/src/cdeLib/Catalog/CreateCacheCommandHandler.cs +++ b/src/cdeLib/Catalog/CreateCacheCommandHandler.cs @@ -17,13 +17,15 @@ public class CreateCacheCommandHandler : IRequestHandler private readonly IConfiguration _configuration; private readonly ICatalogRepository _catalogRepository; private readonly IMessageBus _messageBus; + private readonly OperationCancellation _cancellation; public CreateCacheCommandHandler(IConfiguration configuration, ICatalogRepository catalogRepository, - IMessageBus messageBus) + IMessageBus messageBus, OperationCancellation cancellation) { _configuration = configuration; _catalogRepository = catalogRepository; _messageBus = messageBus; + _cancellation = cancellation; } public async Task OnHandle(CreateCacheCommand request, CancellationToken cancellationToken) @@ -36,8 +38,8 @@ public async Task OnHandle(CreateCacheCommand request, CancellationToken cancell re.SimpleScanEndEvent = ScanEndOfEntries; re.ExceptionEvent = PrintExceptions; - re.PopulateRoot(request.Path, request.FollowJunctions); - if (Hack.BreakConsoleFlag) + re.PopulateRoot(request.Path, request.FollowJunctions, _cancellation.Token); + if (_cancellation.IsCancellationRequested) { Console.WriteLine(" * Break key detected incomplete scan will not be saved."); return; diff --git a/src/cdeLib/Duplicates/Duplication.cs b/src/cdeLib/Duplicates/Duplication.cs index 0699d9b..add8a4c 100644 --- a/src/cdeLib/Duplicates/Duplication.cs +++ b/src/cdeLib/Duplicates/Duplication.cs @@ -34,13 +34,16 @@ public class Duplication private readonly ILogger _logger; private readonly IApplicationDiagnostics _applicationDiagnostics; private readonly HashHelper _hashHelper; + private readonly OperationCancellation _cancellation; - public Duplication(ILogger logger, IConfiguration configuration, IApplicationDiagnostics applicationDiagnostics) + public Duplication(ILogger logger, IConfiguration configuration, IApplicationDiagnostics applicationDiagnostics, + OperationCancellation cancellation) { _logger = logger; _hashHelper = new HashHelper(logger); _configuration = configuration; _applicationDiagnostics = applicationDiagnostics; + _cancellation = cancellation; _duplicationStatistics = new DuplicationStatistics(); _logger.LogDebug("Dupe Constructor Memory: {0}", _applicationDiagnostics.GetMemoryAllocated().FormatAsBytes()); } @@ -176,7 +179,7 @@ public async Task ApplyHash(IList rootEntries) { _duplicationStatistics.SeenFileSize(flatFile.ChildDE.Size); await CalculatePartialHashAsync(flatFile.FullPath, flatFile.ChildDE); - if (Hack.BreakConsoleFlag) + if (_cancellation.IsCancellationRequested) { Console.WriteLine("\n * Break key detected exiting hashing phase inner."); await cts.CancelAsync(); @@ -201,7 +204,7 @@ public async Task ApplyHash(IList rootEntries) $"FullHash: {_duplicationStatistics.FullHashes} PartialHash: {_duplicationStatistics.PartialHashes} Processed: {_duplicationStatistics.BytesProcessed / (1024 * 1024):F2} MB NotProcessed: {_duplicationStatistics.BytesNotProcessed / (1024 * 1024):F2} MB Perf: {perf}\nTotal Data Encountered: {_duplicationStatistics.TotalFileBytes / (1024 * 1024):F2} MB\nFailedHash: {_duplicationStatistics.FailedToHash} (almost always because cannot open to read file)"; ReportStatus(statsMessage); - Hack.BreakConsoleFlag = false; // require you to press break again to stop the full hash phase. + _cancellation.Reset(); // require you to press break again to stop the full hash phase. CheckDupesAndCompleteFullHash(rootEntries); ReportStatus("After hashing completed."); @@ -381,7 +384,7 @@ await Task.Run(() => var fullPath = pde.FullPath; await CalculateHash(fullPath, dirEntry, false); - if (Hack.BreakConsoleFlag) + if (_cancellation.IsCancellationRequested) { _logger.LogInfo("Break key detected, exiting full hash phase."); await cts.CancelAsync(); diff --git a/src/cdeLib/Entities/RootEntry.cs b/src/cdeLib/Entities/RootEntry.cs index 7c40633..8171fc4 100644 --- a/src/cdeLib/Entities/RootEntry.cs +++ b/src/cdeLib/Entities/RootEntry.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Threading; using cdeLib.Extensions; using cdeLib.Infrastructure; using cdeLib.Infrastructure.Config; @@ -123,11 +124,11 @@ public RootEntry(IConfiguration configuration, IFileSystemAdapter fileSystemAdap } } - public void PopulateRoot(string startPath, bool followJunctions = false) + public void PopulateRoot(string startPath, bool followJunctions = false, CancellationToken token = default) { startPath = GetRootEntry(startPath); ScanStartUtc = DateTime.UtcNow; - RecurseTree(startPath, followJunctions); + RecurseTree(startPath, followJunctions, token); ScanEndUtc = DateTime.UtcNow; SetInMemoryFields(); } @@ -289,7 +290,7 @@ private static string SafeFileName(string path) /// /// Iteratively scans a directory tree using a stack-based approach for optimal performance. /// - public void RecurseTree(string startPath, bool followJunctions = false) + public void RecurseTree(string startPath, bool followJunctions = false, CancellationToken token = default) { _followJunctions = followJunctions; var entryCount = 0; @@ -302,12 +303,12 @@ public void RecurseTree(string startPath, bool followJunctions = false) { var (parent, directory) = stack.Pop(); - if (TryEnumerateDirectory(directory, parent, stack, ref entryCount, progressTracker)) + if (TryEnumerateDirectory(directory, parent, stack, ref entryCount, progressTracker, token)) { continue; } - if (Hack.BreakConsoleFlag) + if (token.IsCancellationRequested) { break; } @@ -325,7 +326,8 @@ private bool TryEnumerateDirectory( ICommonEntry parent, Stack<(ICommonEntry, string)> stack, ref int entryCount, - ScanProgressTracker progressTracker) + ScanProgressTracker progressTracker, + CancellationToken token) { try { @@ -336,7 +338,7 @@ private bool TryEnumerateDirectory( { ProcessFileSystemEntry(fsInfo, parent, stack, ref entryCount, directory, progressTracker); - if (Hack.BreakConsoleFlag) + if (token.IsCancellationRequested) { break; } diff --git a/src/cdeLib/Hack.cs b/src/cdeLib/Hack.cs deleted file mode 100644 index cdc8d20..0000000 --- a/src/cdeLib/Hack.cs +++ /dev/null @@ -1,7 +0,0 @@ - -namespace cdeLib; - -public static class Hack -{ - public static volatile bool BreakConsoleFlag; // False is default -} \ No newline at end of file diff --git a/src/cdeLib/Module/CdelibModule.cs b/src/cdeLib/Module/CdelibModule.cs index f7dd4f5..e9554d5 100644 --- a/src/cdeLib/Module/CdelibModule.cs +++ b/src/cdeLib/Module/CdelibModule.cs @@ -17,6 +17,7 @@ protected override void Load(ContainerBuilder builder) builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterType().As().SingleInstance(); + builder.RegisterType().SingleInstance(); builder.RegisterType().InstancePerLifetimeScope(); diff --git a/src/cdeLib/OperationCancellation.cs b/src/cdeLib/OperationCancellation.cs new file mode 100644 index 0000000..1f451b2 --- /dev/null +++ b/src/cdeLib/OperationCancellation.cs @@ -0,0 +1,35 @@ +using System.Threading; + +namespace cdeLib; + +/// +/// Cooperative cancellation for long-running console operations, driven by Ctrl-C. Replaces the old +/// global Hack.BreakConsoleFlag with an injectable signal that exposes a real +/// . +/// +/// It is resettable: a phase can call to obtain a fresh, un-cancelled token, +/// so the user can interrupt the next phase independently. This mirrors the original "press break again" +/// behaviour used by the two-phase hashing and the find REPL. +/// +/// +public sealed class OperationCancellation +{ + private CancellationTokenSource _cts = new(); + + /// Token for the current operation/phase. Cancelled when the user requests a break. + public CancellationToken Token => _cts.Token; + + /// True once a break has been requested for the current operation/phase. + public bool IsCancellationRequested => _cts.IsCancellationRequested; + + /// Request cancellation of the current operation/phase (wired to Ctrl-C). + public void Cancel() => _cts.Cancel(); + + /// Begin a fresh operation/phase with a new, un-cancelled token. + public void Reset() + { + var old = _cts; + _cts = new CancellationTokenSource(); + old.Dispose(); + } +} diff --git a/src/cdeLibTest/DuplicationTest.cs b/src/cdeLibTest/DuplicationTest.cs index 0cd286a..f79450f 100644 --- a/src/cdeLibTest/DuplicationTest.cs +++ b/src/cdeLibTest/DuplicationTest.cs @@ -71,7 +71,7 @@ public void GetSizePairs_HashIrrelevant_NullIsNotAHashValue_PartialNotAUniqueHas var roots = new List {re1}; re1.SetInMemoryFields(); - var d = new Duplication(_logger, _configuration, _applicationDiagnostics); + var d = new Duplication(_logger, _configuration, _applicationDiagnostics, new OperationCancellation()); var sizePairDictionary = d.GetSizePairs(roots); Console.WriteLine($"Number of Size Pairs {sizePairDictionary.Count}"); @@ -117,7 +117,7 @@ public void GetDupePairs_DupeHashDoesNotMatchDiffSizeFilesOrPartialHash_OK() re1.AddChild(de10); var roots = new List {re1}; - var d = new Duplication(_logger, _configuration, _applicationDiagnostics); + var d = new Duplication(_logger, _configuration, _applicationDiagnostics, new OperationCancellation()); var dp = d.GetDupePairs(roots); var dp1 = dp.First(); @@ -195,7 +195,7 @@ public void GetSizePairs_CheckSanityOfDupeSizeCountAndDupeFileCount_Exercise() Console.WriteLine($"loaded {r.DefaultFileName}"); } - var d = new Duplication(_logger, _configuration, _applicationDiagnostics); + var d = new Duplication(_logger, _configuration, _applicationDiagnostics, new OperationCancellation()); var sizePairDictionary = d.GetSizePairs(rootEntries); Console.WriteLine($"Number of Size Pairs {sizePairDictionary.Count}"); @@ -248,7 +248,7 @@ public void GetDupePairs_CheckAllDupeFilesHaveFullHash_OK() var catalogRepository = new CatalogRepository(Log.Logger); var rootEntries = catalogRepository.LoadCurrentDirCache(); - var d = new Duplication(_logger, _configuration, _applicationDiagnostics); + var d = new Duplication(_logger, _configuration, _applicationDiagnostics, new OperationCancellation()); var dupePairEnum = d.GetDupePairs(rootEntries); foreach (var dupe in dupePairEnum) @@ -273,7 +273,7 @@ public async Task var catalogRepository = new CatalogRepository(Log.Logger); var rootEntries = catalogRepository.LoadCurrentDirCache(); - var d = new Duplication(_logger, _configuration, _applicationDiagnostics); + var d = new Duplication(_logger, _configuration, _applicationDiagnostics, new OperationCancellation()); await d.ApplyHash(rootEntries).ConfigureAwait(false); } // ReSharper restore InconsistentNaming diff --git a/src/cdeLibTest/Infrastructure/DuplicationTests.cs b/src/cdeLibTest/Infrastructure/DuplicationTests.cs index aed93e7..68f22b4 100644 --- a/src/cdeLibTest/Infrastructure/DuplicationTests.cs +++ b/src/cdeLibTest/Infrastructure/DuplicationTests.cs @@ -80,7 +80,8 @@ public void SetUp() private class TestDuplication : Duplication { public TestDuplication(ILogger logger, IConfiguration configuration, - IApplicationDiagnostics applicationDiagnostics) : base(logger, configuration, applicationDiagnostics) + IApplicationDiagnostics applicationDiagnostics) + : base(logger, configuration, applicationDiagnostics, new cdeLib.OperationCancellation()) { } From 389137c675f738218b21f0c00196d2196405d9db Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Mon, 8 Jun 2026 21:17:02 +1000 Subject: [PATCH 37/43] build(cde): suppress MA0048 for MigrateOptions, drop stale UpgradeOptions MigrateOptions is co-located with the other verb option classes in CommandLineOptions.cs; add it to the existing MA0048 suppression list like its siblings. Also remove the suppression for UpgradeOptions, which no longer exists (renamed to UpdateOptions, now in its own file). --- src/cde/GlobalSuppressions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cde/GlobalSuppressions.cs b/src/cde/GlobalSuppressions.cs index 6d7b2b2..662bbfc 100644 --- a/src/cde/GlobalSuppressions.cs +++ b/src/cde/GlobalSuppressions.cs @@ -12,6 +12,7 @@ [assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.GrepOptions")] [assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.GrepPathOptions")] [assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.LoadWaitOptions")] +[assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.MigrateOptions")] [assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.PopulousFoldersOptions")] [assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.ReplFindOptions")] [assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.ReplGrepOptions")] @@ -19,4 +20,3 @@ [assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.ReplOptions")] [assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.ScanOptions")] [assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.TreeDumpOptions")] -[assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.UpgradeOptions")] From a0656b5485c8ad70cbb2effc583d3c97f4db1c70 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Mon, 8 Jun 2026 22:19:57 +1000 Subject: [PATCH 38/43] refactor(cde): split scan MainLoop to clear MA0051 (method too long) Extract the .cdex hash-reuse block into ReuseHashesFromExistingCatalog and the final summary calculation/logging into ReportScanSummary. MainLoop drops under the 60-line limit and reads as a clear sequence of steps. No behaviour change. --- .../ScanProgress/CreateCacheCommandHandler.cs | 67 +++++++++++-------- 1 file changed, 40 insertions(+), 27 deletions(-) diff --git a/src/cde/ScanProgress/CreateCacheCommandHandler.cs b/src/cde/ScanProgress/CreateCacheCommandHandler.cs index c93dc71..6224a6a 100644 --- a/src/cde/ScanProgress/CreateCacheCommandHandler.cs +++ b/src/cde/ScanProgress/CreateCacheCommandHandler.cs @@ -57,19 +57,8 @@ private async Task MainLoop(CreateCacheCommand request, CancellationToken cancel return; } - // Catalogs are stored in the zero-copy columnar .cdex format. Reuse hashes from an existing - // .cdex (reconstructed into a tree) for this scan path when one is found. var cdexName = Path.ChangeExtension(re.DefaultFileName, ".cdex"); - if (File.Exists(cdexName)) - { - Log.Information("Found cache \"{FileName}\", Updating hashes for new scan from cache file", cdexName); - RootEntry oldRoot; - using (var reader = new ColumnarCatalogReader(cdexName)) - { - oldRoot = CatalogTreeBuilder.FromSource(reader); - } - oldRoot.TraverseTreesCopyHash(re); - } + ReuseHashesFromExistingCatalog(re, cdexName); re.SortAllChildrenByPath(); re.SetSummaryFields(); @@ -84,21 +73,7 @@ await Task.Run(() => ColumnarFormat.Write(EntryStore.Build(re), cdexName), cance .ConfigureAwait(false); ScanProgressConsole.EnqueueMessage($"Saved to {cdexName}"); - // Calculate and display final scan summary - sw.Stop(); - var elapsedSec = sw.ElapsedMilliseconds / 1000.0; - if (elapsedSec < 1) elapsedSec = 1; - var totalCount = re.FileEntryCount + re.DirEntryCount; - var scansPerSec = (long)(totalCount / elapsedSec); - var defaultNumberFormat = new NumberFormatInfo(); - var scanCountText = totalCount.ToString("N0", defaultNumberFormat); - var scansPerSecText = scansPerSec.ToString("N0", defaultNumberFormat); - ScanProgressConsole.EnqueueMessage($"Total files scanned: {scanCountText}, Average: {scansPerSecText}/sec"); - - Log.Information("Scanned path {Path}, Saved to {SavePath}", re.Path, cdexName); - Log.Information( - "Scanned Files {FileCount:0,0}, Dirs {DirCount:0,0}, Total size {Size:0,0}", re.FileEntryCount, - re.DirEntryCount, re.Size.Bytes().Humanize(CultureInfo.CurrentCulture)); + ReportScanSummary(re, cdexName, sw); } catch (ArgumentException ex) { @@ -106,6 +81,44 @@ await Task.Run(() => ColumnarFormat.Write(EntryStore.Build(re), cdexName), cance } } + /// + /// Catalogs are stored in the zero-copy columnar .cdex format. Reuse hashes from an existing + /// .cdex (reconstructed into a tree) for this scan path when one is found. + /// + private static void ReuseHashesFromExistingCatalog(RootEntry re, string cdexName) + { + if (!File.Exists(cdexName)) + { + return; + } + + Log.Information("Found cache \"{FileName}\", Updating hashes for new scan from cache file", cdexName); + RootEntry oldRoot; + using (var reader = new ColumnarCatalogReader(cdexName)) + { + oldRoot = CatalogTreeBuilder.FromSource(reader); + } + oldRoot.TraverseTreesCopyHash(re); + } + + private static void ReportScanSummary(RootEntry re, string cdexName, System.Diagnostics.Stopwatch sw) + { + sw.Stop(); + var elapsedSec = sw.ElapsedMilliseconds / 1000.0; + if (elapsedSec < 1) elapsedSec = 1; + var totalCount = re.FileEntryCount + re.DirEntryCount; + var scansPerSec = (long)(totalCount / elapsedSec); + var defaultNumberFormat = new NumberFormatInfo(); + var scanCountText = totalCount.ToString("N0", defaultNumberFormat); + var scansPerSecText = scansPerSec.ToString("N0", defaultNumberFormat); + ScanProgressConsole.EnqueueMessage($"Total files scanned: {scanCountText}, Average: {scansPerSecText}/sec"); + + Log.Information("Scanned path {Path}, Saved to {SavePath}", re.Path, cdexName); + Log.Information( + "Scanned Files {FileCount:0,0}, Dirs {DirCount:0,0}, Total size {Size:0,0}", re.FileEntryCount, + re.DirEntryCount, re.Size.Bytes().Humanize(CultureInfo.CurrentCulture)); + } + private void PrintException(string path, Exception ex) { Console.WriteLine($"Exception {ex.GetType()}, Path \"{path}\""); From 61a84f95a50db20839a035212dacf860bbbc3364 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Mon, 8 Jun 2026 22:22:31 +1000 Subject: [PATCH 39/43] style: repo-wide IDE code cleanup Apply a consistent code-style/modernization pass across the solution: prefer const over readonly where applicable, pattern matching (is/or) over comparison chains, and blank-line/formatting normalization. No behaviour change; solution builds and the cdeLibTest suite (159 passed, 7 skipped) is green. --- src/Mono.Terminal/LineEditor.cs | 131 +++++++++--------- src/cde/CdeApp.cs | 8 +- src/cde/Program.cs | 12 +- src/cdeBenchmarks/CatalogFixture.cs | 2 - src/cdeBenchmarks/Hash16Benchmarks.cs | 18 +-- .../MultiCatalogSearchBenchmarks.cs | 2 - src/cdeBenchmarks/PoolingBenchmarks.cs | 18 ++- src/cdeBenchmarks/SearchBenchmarks.cs | 2 - src/cdeLib/Catalog/CatalogRepository.cs | 4 +- .../Catalog/CreateCacheCommandHandler.cs | 5 +- src/cdeLib/DateTimePartialParameter.cs | 29 ++-- src/cdeLib/Duplicates/Duplication.cs | 10 +- .../Duplicates/DuplicationStatistics.cs | 8 +- src/cdeLib/Entities/Columnar/Utf8Matcher.cs | 4 +- src/cdeLib/Entities/DirEntry.cs | 29 ++-- src/cdeLib/Entities/RootEntry.cs | 61 +++----- src/cdeLib/Entities/Soa/EntryRef.cs | 11 +- src/cdeLib/Extensions/ListExtensions.cs | 27 ++-- src/cdeLib/FindOptions.cs | 5 +- src/cdeLib/FindService.cs | 3 +- .../Config/AppConfigurationSection.cs | 4 +- .../Infrastructure/FileSystemAdapter.cs | 2 +- .../Infrastructure/Hashing/HashHelper.cs | 2 +- .../Infrastructure/Hashing/MurMurHash3.cs | 4 +- src/cdeLib/Infrastructure/ObjectPool.cs | 4 +- src/cdeLib/TimePartialParameter.cs | 17 +-- .../Columnar/ColumnarCatalogTests.cs | 8 +- src/cdeLibTest/DirEntryEnumeratorTest.cs | 2 +- src/cdeLibTest/DuplicationTest.cs | 11 +- src/cdeLibTest/IdeaStructNode.cs | 4 +- .../Infrastructure/DuplicationPerfTest.cs | 2 +- .../Infrastructure/Hashing/Crc32.cs | 5 +- .../Performance/PerformanceTreeTraversal.cs | 4 +- src/cdeLibTest/RootEntryTest.cs | 27 +--- src/cdeLibTest/Soa/EntryRefTests.cs | 3 +- src/cdeLibTest/Soa/EntryStoreTests.cs | 4 +- src/cdeLibTest/TimePartialParameterTest.cs | 4 +- src/cdeWin/CDEWinForm.cs | 12 +- src/cdeWin/CDEWinFormPresenter.cs | 96 ++++++------- src/cdeWin/ContextMenuHelper.cs | 6 +- src/cdeWin/KeyEqualityComparer.cs | 2 +- src/cdeWin/LoadCatalogService.cs | 2 +- src/cdeWin/LoaderForm.cs | 2 +- src/cdeWin/Program.cs | 4 +- src/cdeWin/SplitContainerExtensions.cs | 2 +- src/cdeWin/StringExtension.cs | 3 +- src/cdeWinTest/CDEWinFormPresenterTest.cs | 5 +- src/cdeWinTest/TestCDEWinPresenterBase.cs | 10 +- ...estCDEWinPresenter_OptimiseRegexPattern.cs | 6 +- 49 files changed, 281 insertions(+), 365 deletions(-) diff --git a/src/Mono.Terminal/LineEditor.cs b/src/Mono.Terminal/LineEditor.cs index 9dff0e1..82503bd 100644 --- a/src/Mono.Terminal/LineEditor.cs +++ b/src/Mono.Terminal/LineEditor.cs @@ -113,9 +113,9 @@ public Completion(string prefix, string[] result) // This contains a raw stream pointing to stdout, used to bypass the TermInfoDriver private static Stream _unixRawOutput; - delegate void KeyHandler(); + private delegate void KeyHandler(); - struct Handler + private struct Handler { public readonly ConsoleKeyInfo Cki; public readonly KeyHandler KeyHandler; @@ -229,7 +229,7 @@ public LineEditor(string name, int histsize) // BackgroundColor properties, so we have to use the terminfo driver in Mono to // fetch these values - void GetUnixConsoleReset() + private void GetUnixConsoleReset() { // // On Unix, we want to be able to reset the color for the pop-up completion @@ -268,7 +268,7 @@ private void CmdDebug() Render(); } - void Render() + private void Render() { Console.Write(_shownPrompt); Console.Write(_renderedText); @@ -288,7 +288,7 @@ void Render() private void UpdateHomeRow(int screenpos) { - var lines = 1 + (screenpos / Console.WindowWidth); + var lines = 1 + screenpos / Console.WindowWidth; _homeRow = Console.CursorTop - (lines - 1); if (_homeRow < 0) @@ -296,7 +296,7 @@ private void UpdateHomeRow(int screenpos) } - void RenderFrom(int pos) + private void RenderFrom(int pos) { var rpos = TextToRenderPos(pos); int i; @@ -304,7 +304,7 @@ void RenderFrom(int pos) for (i = rpos; i < _renderedText.Length; i++) Console.Write(_renderedText[i]); - if ((_shownPrompt.Length + _renderedText.Length) > _maxRendered) + if (_shownPrompt.Length + _renderedText.Length > _maxRendered) _maxRendered = _shownPrompt.Length + _renderedText.Length; else { @@ -314,7 +314,7 @@ void RenderFrom(int pos) } } - void ComputeRendered() + private void ComputeRendered() { _renderedText.Length = 0; @@ -336,7 +336,7 @@ void ComputeRendered() } } - int TextToRenderPos(int pos) + private int TextToRenderPos(int pos) { var p = 0; @@ -363,7 +363,7 @@ private int TextToScreenPos(int pos) return _shownPrompt.Length + TextToRenderPos(pos); } - string Prompt + private string Prompt { get => _prompt; set => _prompt = value; @@ -371,12 +371,12 @@ string Prompt private int LineCount => (_shownPrompt.Length + _renderedText.Length) / Console.WindowWidth; - void ForceCursor(int newpos) + private void ForceCursor(int newpos) { _cursor = newpos; var actualPos = _shownPrompt.Length + TextToRenderPos(_cursor); - var row = _homeRow + (actualPos / Console.WindowWidth); + var row = _homeRow + actualPos / Console.WindowWidth; var col = actualPos % Console.WindowWidth; if (row >= Console.BufferHeight) @@ -384,7 +384,7 @@ void ForceCursor(int newpos) Console.SetCursorPosition(col, row); } - void UpdateCursor(int newpos) + private void UpdateCursor(int newpos) { if (_cursor == newpos) return; @@ -392,7 +392,7 @@ void UpdateCursor(int newpos) ForceCursor(newpos); } - void InsertChar(char c) + private void InsertChar(char c) { var prevLines = LineCount; _text = _text.Insert(_cursor, c); @@ -411,7 +411,7 @@ void InsertChar(char c) } } - static void SaveExcursion(Action code) + private static void SaveExcursion(Action code) { var savedCol = Console.CursorLeft; var savedRow = Console.CursorTop; @@ -433,7 +433,7 @@ static void SaveExcursion(Action code) } } - class CompletionState + private class CompletionState { public string Prefix; public string[] Completions; @@ -457,19 +457,19 @@ public CompletionState(int col, int row, int width, int height) throw new ArgumentException("Cannot be less than one", "Height"); } - void DrawSelection() + private void DrawSelection() { for (var r = 0; r < Height; r++) { var itemIdx = _topItem + r; - var selected = (itemIdx == _selectedItem); + var selected = itemIdx == _selectedItem; Console.ForegroundColor = selected ? ConsoleColor.Black : ConsoleColor.Gray; Console.BackgroundColor = selected ? ConsoleColor.Cyan : ConsoleColor.Blue; var item = Prefix + Completions[itemIdx]; if (item.Length > Width) - item = item.Substring(0, Width); + item = item[..Width]; Console.CursorLeft = Col; Console.CursorTop = Row + r; @@ -508,7 +508,7 @@ public void SelectPrevious() } } - void Clear() + private void Clear() { for (var r = 0; r < Height; r++) { @@ -525,7 +525,7 @@ public void Remove() } } - void ShowCompletions(string prefix, string[] completions) + private void ShowCompletions(string prefix, string[] completions) { // Ensure we have space, determine window size var windowHeight = Math.Max(1, Math.Min(completions.Length, Console.WindowHeight / 5)); @@ -587,7 +587,7 @@ public void HideCompletions() // Triggers the completion engine, if insertBestMatch is true, then this will // insert the best match found, this behaves like the shell "tab" which will // complete as much as possible given the options. - void Complete() + private void Complete() { if (AutoCompleteEvent == null) return; @@ -639,12 +639,12 @@ void Complete() var displayCompletions = (string[])completions.Clone(); if (last != -1) { - InsertTextAtCursor(displayCompletions[0].Substring(0, last + 1)); + InsertTextAtCursor(displayCompletions[0][..(last + 1)]); // Adjust the completions to skip the common prefix - prefix += displayCompletions[0].Substring(0, last + 1); + prefix += displayCompletions[0][..(last + 1)]; for (var i = 0; i < displayCompletions.Length; i++) - displayCompletions[i] = displayCompletions[i].Substring(last + 1); + displayCompletions[i] = displayCompletions[i][(last + 1)..]; } ShowCompletions(prefix, displayCompletions); @@ -655,7 +655,7 @@ void Complete() // When the user has triggered a completion window, this will try to update // the contents of it. The completion window is assumed to be hidden at this point - void UpdateCompletionWindow() + private void UpdateCompletionWindow() { if (_currentCompletion != null) throw new Exception("This method should only be called if the window has been hidden"); @@ -677,7 +677,7 @@ void UpdateCompletionWindow() } // Commands - void CmdDone() + private void CmdDone() { if (_currentCompletion != null) { @@ -689,7 +689,7 @@ void CmdDone() _done = true; } - void CmdTabOrComplete() + private void CmdTabOrComplete() { var complete = false; @@ -723,17 +723,17 @@ public void CmdHistoryDump() _history.Dump(); } - void CmdHome() + private void CmdHome() { UpdateCursor(0); } - void CmdEnd() + private void CmdEnd() { UpdateCursor(_text.Length); } - void CmdLeft() + private void CmdLeft() { if (_cursor == 0) return; @@ -741,7 +741,7 @@ void CmdLeft() UpdateCursor(_cursor - 1); } - void CmdBackwardWord() + private void CmdBackwardWord() { var p = WordBackward(_cursor); if (p == -1) @@ -749,7 +749,7 @@ void CmdBackwardWord() UpdateCursor(p); } - void CmdForwardWord() + private void CmdForwardWord() { var p = WordForward(_cursor); if (p == -1) @@ -757,7 +757,7 @@ void CmdForwardWord() UpdateCursor(p); } - void CmdRight() + private void CmdRight() { if (_cursor == _text.Length) return; @@ -765,14 +765,14 @@ void CmdRight() UpdateCursor(_cursor + 1); } - void RenderAfter(int p) + private void RenderAfter(int p) { ForceCursor(p); RenderFrom(p); ForceCursor(_cursor); } - void CmdBackspace() + private void CmdBackspace() { if (_cursor == 0) return; @@ -787,7 +787,7 @@ void CmdBackspace() UpdateCompletionWindow(); } - void CmdDeleteChar() + private void CmdDeleteChar() { // If there is no input, this behaves like EOF if (_text.Length == 0) @@ -805,7 +805,7 @@ void CmdDeleteChar() RenderAfter(_cursor); } - int WordForward(int p) + private int WordForward(int p) { if (p >= _text.Length) return -1; @@ -839,7 +839,7 @@ int WordForward(int p) return -1; } - int WordBackward(int p) + private int WordBackward(int p) { if (p == 0) return -1; @@ -879,7 +879,7 @@ int WordBackward(int p) return -1; } - void CmdDeleteWord() + private void CmdDeleteWord() { var pos = WordForward(_cursor); @@ -898,7 +898,7 @@ void CmdDeleteWord() RenderAfter(_cursor); } - void CmdDeleteBackword() + private void CmdDeleteBackword() { var pos = WordBackward(_cursor); if (pos == -1) @@ -917,12 +917,12 @@ void CmdDeleteBackword() } // Adds the current line to the history if needed - void HistoryUpdateLine() + private void HistoryUpdateLine() { _history.Update(_text.ToString()); } - void CmdHistoryPrev() + private void CmdHistoryPrev() { if (!_history.PreviousAvailable()) return; @@ -932,7 +932,7 @@ void CmdHistoryPrev() SetText(_history.Previous()); } - void CmdHistoryNext() + private void CmdHistoryNext() { if (!_history.NextAvailable()) return; @@ -941,7 +941,7 @@ void CmdHistoryNext() SetText(_history.Next()); } - void CmdUp() + private void CmdUp() { if (_currentCompletion == null) CmdHistoryPrev(); @@ -949,7 +949,7 @@ void CmdUp() _currentCompletion.SelectPrevious(); } - void CmdDown() + private void CmdDown() { if (_currentCompletion == null) CmdHistoryNext(); @@ -957,7 +957,7 @@ void CmdDown() _currentCompletion.SelectNext(); } - void CmdKillToEOF() + private void CmdKillToEOF() { _killBuffer = _text.ToString(_cursor, _text.Length - _cursor); _text.Length = _cursor; @@ -965,12 +965,12 @@ void CmdKillToEOF() RenderAfter(_cursor); } - void CmdYank() + private void CmdYank() { InsertTextAtCursor(_killBuffer); } - void InsertTextAtCursor(string str) + private void InsertTextAtCursor(string str) { var prevLines = LineCount; _text.Insert(_cursor, str); @@ -991,12 +991,12 @@ void InsertTextAtCursor(string str) } } - void SetSearchPrompt(string s) + private void SetSearchPrompt(string s) { SetPrompt("(reverse-i-search)`" + s + "': "); } - void ReverseSearch() + private void ReverseSearch() { int p; @@ -1016,7 +1016,7 @@ void ReverseSearch() else { // The cursor is somewhere in the middle of the string - var start = (_cursor == _matchAt) ? _cursor - 1 : _cursor; + var start = _cursor == _matchAt ? _cursor - 1 : _cursor; if (start != -1) { p = _text.ToString().LastIndexOf(_search, start, StringComparison.Ordinal); @@ -1041,7 +1041,7 @@ void ReverseSearch() } } - void CmdReverseSearch() + private void CmdReverseSearch() { if (_searching == 0) { @@ -1070,9 +1070,9 @@ void CmdReverseSearch() } } - void SearchAppend(char c) + private void SearchAppend(char c) { - _search = _search + c; + _search += c; SetSearchPrompt(_search); // @@ -1088,7 +1088,7 @@ void SearchAppend(char c) ReverseSearch(); } - void CmdRefresh() + private void CmdRefresh() { Console.Clear(); _maxRendered = 0; @@ -1096,7 +1096,7 @@ void CmdRefresh() ForceCursor(_cursor); } - void InterruptEdit(object sender, ConsoleCancelEventArgs a) + private void InterruptEdit(object sender, ConsoleCancelEventArgs a) { // Do not abort our program: a.Cancel = true; @@ -1106,7 +1106,7 @@ void InterruptEdit(object sender, ConsoleCancelEventArgs a) } // Implements heuristics to show the completion window based on the mode - bool HeuristicAutoComplete(bool wasCompleting, char insertedChar) + private bool HeuristicAutoComplete(bool wasCompleting, char insertedChar) { if (HeuristicsMode == "csharp") { @@ -1143,7 +1143,7 @@ bool HeuristicAutoComplete(bool wasCompleting, char insertedChar) return false; } - void HandleChar(char c) + private void HandleChar(char c) { if (_searching != 0) SearchAppend(c); @@ -1205,7 +1205,8 @@ private void EditLoop(CancellationToken cancellationToken) _lastHandler = handler.KeyHandler; break; } - else if (t.KeyChar == cki.KeyChar && t.Key == ConsoleKey.Zoom) + + if (t.KeyChar == cki.KeyChar && t.Key == ConsoleKey.Zoom) { handled = true; if (handler.ResetCompletion) @@ -1238,7 +1239,7 @@ private void EditLoop(CancellationToken cancellationToken) } } - void InitText(string initial) + private void InitText(string initial) { _text = new StringBuilder(initial); ComputeRendered(); @@ -1247,13 +1248,13 @@ void InitText(string initial) ForceCursor(_cursor); } - void SetText(string newtext) + private void SetText(string newtext) { Console.SetCursorPosition(0, _homeRow); InitText(newtext); } - void SetPrompt(string newprompt) + private void SetPrompt(string newprompt) { _shownPrompt = newprompt; Console.SetCursorPosition(0, _homeRow); @@ -1329,7 +1330,7 @@ public void SaveHistory() // Emulates the bash-like behavior, where edits done to the // history are recorded - class History + private class History { private readonly string[] _history; private int _head, _tail; @@ -1384,7 +1385,7 @@ public void Close() try { using var sw = File.CreateText(_histfile); - var start = (_count == _history.Length) ? _head : _tail; + var start = _count == _history.Length ? _head : _tail; for (var i = start; i < start + _count; i++) { var p = i % _history.Length; diff --git a/src/cde/CdeApp.cs b/src/cde/CdeApp.cs index 5b669ea..3e30f72 100644 --- a/src/cde/CdeApp.cs +++ b/src/cde/CdeApp.cs @@ -34,16 +34,16 @@ public sealed class CdeApp( public Task CreateCacheAsync(ScanOptions opts) => messageBus.Send(new CreateCacheCommand(opts.Path) - { Description = opts.Description, FollowJunctions = opts.FollowJunctions }); + { Description = opts.Description, FollowJunctions = opts.FollowJunctions }, cancellationToken: cancellation.Token); public Task HashCatalogAsync() => - messageBus.Send(new HashCatalogCommand()); + messageBus.Send(new HashCatalogCommand(), cancellationToken: cancellation.Token); public Task FindDupesAsync() => - messageBus.Send(new FindDuplicatesCommand()); + messageBus.Send(new FindDuplicatesCommand(), cancellationToken: cancellation.Token); public Task UpdateAsync(UpdateOptions opts) => - messageBus.Send(new UpdateCommand { FileName = opts.FileName, Description = opts.Description }); + messageBus.Send(new UpdateCommand { FileName = opts.FileName, Description = opts.Description }, cancellationToken: cancellation.Token); // ---- find ---- diff --git a/src/cde/Program.cs b/src/cde/Program.cs index 2e5b228..801f9f4 100644 --- a/src/cde/Program.cs +++ b/src/cde/Program.cs @@ -84,7 +84,7 @@ private static async Task Main(string[] args) } finally { - Log.CloseAndFlush(); + await Log.CloseAndFlushAsync().ConfigureAwait(false); } } @@ -103,11 +103,11 @@ private static async Task Main(string[] args) ReplGrepOptions o => RunSync(() => _app.FindRepl(FindService.ParamGrep, o.Value)), ReplFindOptions o => RunSync(() => _app.FindRepl(FindService.ParamFind, o.Value)), MigrateOptions o => RunSync(() => _app.Migrate(o)), - HashOptions _ => _app.HashCatalogAsync(), - DupesOptions _ => _app.FindDupesAsync(), - TreeDumpOptions _ => RunSync(_app.PrintPathsHaveHash), - LoadWaitOptions _ => RunSync(_app.LoadWait), - ReplOptions _ => RunSync(_app.InvokeRepl), + HashOptions => _app.HashCatalogAsync(), + DupesOptions => _app.FindDupesAsync(), + TreeDumpOptions => RunSync(_app.PrintPathsHaveHash), + LoadWaitOptions => RunSync(_app.LoadWait), + ReplOptions => RunSync(_app.InvokeRepl), PopulousFoldersOptions o => RunSync(() => _app.FindPopulous(o.Count)), UpdateOptions o => _app.UpdateAsync(o), _ => Task.CompletedTask, diff --git a/src/cdeBenchmarks/CatalogFixture.cs b/src/cdeBenchmarks/CatalogFixture.cs index 94f46ff..6b51772 100644 --- a/src/cdeBenchmarks/CatalogFixture.cs +++ b/src/cdeBenchmarks/CatalogFixture.cs @@ -1,5 +1,3 @@ -using System; -using System.IO; using cdeLib.Catalog; using cdeLib.Entities; using cdeMemProbe; diff --git a/src/cdeBenchmarks/Hash16Benchmarks.cs b/src/cdeBenchmarks/Hash16Benchmarks.cs index 9eaa5ba..48fb1ae 100644 --- a/src/cdeBenchmarks/Hash16Benchmarks.cs +++ b/src/cdeBenchmarks/Hash16Benchmarks.cs @@ -34,9 +34,9 @@ public class Hash16Benchmarks public void Setup() { // Simulate real MD5 hashes (16 bytes each) - _hash1 = new byte[] { 0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE, 0xF0, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88 }; - _hash2 = new byte[] { 0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE, 0xF0, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88 }; - _hash3 = new byte[] { 0xFF, 0xEE, 0xDD, 0xCC, 0xBB, 0xAA, 0x99, 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11, 0x00 }; + _hash1 = [0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE, 0xF0, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + _hash2 = [0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE, 0xF0, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + _hash3 = [0xFF, 0xEE, 0xDD, 0xCC, 0xBB, 0xAA, 0x99, 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11, 0x00]; _structHash1 = new Hash16(_hash1); _structHash2 = new Hash16(_hash2); @@ -44,10 +44,10 @@ public void Setup() // Pre-populate dictionary for lookup benchmarks _hashDictionary = new Dictionary>(IterationCount); - for (int i = 0; i < IterationCount; i++) + for (var i = 0; i < IterationCount; i++) { var hash = new Hash16(CreateVariedHash(i)); - _hashDictionary[hash] = new List { $"file_{i}.txt" }; + _hashDictionary[hash] = [$"file_{i}.txt"]; } } @@ -95,10 +95,10 @@ public bool EqualityComparison() [Benchmark(Description = "Dictionary lookups with Hash16 keys")] public int DictionaryLookup() { - int count = 0; + var count = 0; var lookupHash = new Hash16(CreateVariedHash(IterationCount / 2)); - for (int i = 0; i < 100; i++) + for (var i = 0; i < 100; i++) { if (_hashDictionary.TryGetValue(lookupHash, out var files)) { @@ -120,12 +120,12 @@ public Dictionary> DictionaryInsertion() { var dict = new Dictionary>(IterationCount); - for (int i = 0; i < IterationCount; i++) + for (var i = 0; i < IterationCount; i++) { var hash = new Hash16(CreateVariedHash(i)); if (!dict.TryGetValue(hash, out var list)) { - list = new List(); + list = []; dict[hash] = list; } list.Add($"file_{i}.txt"); diff --git a/src/cdeBenchmarks/MultiCatalogSearchBenchmarks.cs b/src/cdeBenchmarks/MultiCatalogSearchBenchmarks.cs index 70a5ac9..19f2ffe 100644 --- a/src/cdeBenchmarks/MultiCatalogSearchBenchmarks.cs +++ b/src/cdeBenchmarks/MultiCatalogSearchBenchmarks.cs @@ -1,6 +1,4 @@ -using System.Collections.Generic; using System.Runtime.CompilerServices; -using System.Threading; using BenchmarkDotNet.Attributes; using cdeLib; using cdeLib.Entities; diff --git a/src/cdeBenchmarks/PoolingBenchmarks.cs b/src/cdeBenchmarks/PoolingBenchmarks.cs index 18f01d9..a239874 100644 --- a/src/cdeBenchmarks/PoolingBenchmarks.cs +++ b/src/cdeBenchmarks/PoolingBenchmarks.cs @@ -188,16 +188,14 @@ public int StackPooling() for (var j = 0; j < 10; j++) { - if (stack.Count > 0) - { - stack.Pop(); - totalCount++; + if (stack.Count <= 0) continue; + stack.Pop(); + totalCount++; - // Push more items - var child = new DirEntry(false); - child.SetPath($"child_{j}"); - stack.Push(child); - } + // Push more items + var child = new DirEntry(false); + child.SetPath($"child_{j}"); + stack.Push(child); } CollectionPool.ReturnCommonEntryStack(stack); @@ -256,7 +254,7 @@ public int StringListPooling() for (var j = 0; j < 10; j++) { - list.Add($"path\\to\\file_{j}.txt"); + list.Add($@"path\to\file_{j}.txt"); } totalCount += list.Count; diff --git a/src/cdeBenchmarks/SearchBenchmarks.cs b/src/cdeBenchmarks/SearchBenchmarks.cs index 3c3373c..f34199b 100644 --- a/src/cdeBenchmarks/SearchBenchmarks.cs +++ b/src/cdeBenchmarks/SearchBenchmarks.cs @@ -1,6 +1,4 @@ -using System.Collections.Generic; using System.Runtime.CompilerServices; -using System.Threading; using BenchmarkDotNet.Attributes; using cdeLib; using cdeLib.Entities; diff --git a/src/cdeLib/Catalog/CatalogRepository.cs b/src/cdeLib/Catalog/CatalogRepository.cs index f567dec..60423c7 100644 --- a/src/cdeLib/Catalog/CatalogRepository.cs +++ b/src/cdeLib/Catalog/CatalogRepository.cs @@ -20,7 +20,7 @@ namespace cdeLib.Catalog; public sealed class CatalogRepository : ICatalogRepository, IDisposable { - private readonly SerializerProtocol _serializerProtocol = SerializerProtocol.MessagePack; // hard coded for now. + private SerializerProtocol _serializerProtocol = SerializerProtocol.MessagePack; // hard coded for now. private readonly ILogger _logger; private static readonly BufferPool BufferPool = new(); private readonly FileStreamManager _fileStreamManager = FileStreams.Instance; @@ -293,7 +293,7 @@ private void Dispose(bool disposing) { // Dispose of managed resources BufferPool?.Clear(); - // Note: FileStreamManager is a singleton, don't dispose it here + // Note: FileStreamManager is a singleton, don't dispose of it here } _disposed = true; } diff --git a/src/cdeLib/Catalog/CreateCacheCommandHandler.cs b/src/cdeLib/Catalog/CreateCacheCommandHandler.cs index 3dce121..852d347 100644 --- a/src/cdeLib/Catalog/CreateCacheCommandHandler.cs +++ b/src/cdeLib/Catalog/CreateCacheCommandHandler.cs @@ -15,15 +15,12 @@ namespace cdeLib.Catalog; public class CreateCacheCommandHandler : IRequestHandler { private readonly IConfiguration _configuration; - private readonly ICatalogRepository _catalogRepository; private readonly IMessageBus _messageBus; private readonly OperationCancellation _cancellation; - public CreateCacheCommandHandler(IConfiguration configuration, ICatalogRepository catalogRepository, - IMessageBus messageBus, OperationCancellation cancellation) + public CreateCacheCommandHandler(IConfiguration configuration, IMessageBus messageBus, OperationCancellation cancellation) { _configuration = configuration; - _catalogRepository = catalogRepository; _messageBus = messageBus; _cancellation = cancellation; } diff --git a/src/cdeLib/DateTimePartialParameter.cs b/src/cdeLib/DateTimePartialParameter.cs index d6a9e2d..8d04c45 100644 --- a/src/cdeLib/DateTimePartialParameter.cs +++ b/src/cdeLib/DateTimePartialParameter.cs @@ -7,7 +7,7 @@ namespace cdeLib; /// public class DateTimePartialParameter { - private readonly string _format = "--
T::"; + private const string Format = "--
T::"; // a Parsing Expression Grammar might be a better way to do this. PEG // - http://en.wikipedia.org/wiki/Parsing_expression_grammar @@ -28,9 +28,10 @@ public DateTimePartialParameter(string str) if (year < 1000) // this is not 4 digits, its only value e.g., 4 digits 0982 is 4 digits. { _e = new ArgumentException( - $"Require Year parameter be a 4 Digit Year as part of format '{_format}'"); + $"Require Year parameter be a 4 Digit Year as part of format '{Format}'"); return; } + _year = year; if (splitOnDash.Length > 1) // may have a month specified @@ -48,12 +49,14 @@ public DateTimePartialParameter(string str) month = tmp.Month; } } - if (month == 0 || month > 12) + + if (month is 0 or > 12) { _e = new ArgumentException( - $"Require valid integer 1-12 or Month name for Month as part of format '{_format}'"); + $"Require valid integer 1-12 or Month name for Month as part of format '{Format}'"); return; } + _month = month; } @@ -68,7 +71,7 @@ public DateTimePartialParameter(string str) if (SeparatorIsNotValid(splitOnDash[2], 'T')) { _e = new ArgumentException( - $"The separator between Date and Time must be 'T' as part of format '{_format}'"); + $"The separator between Date and Time must be 'T' as part of format '{Format}'"); return; } @@ -87,14 +90,15 @@ public DateTimePartialParameter(string str) if (dayOfMonth is 0 or > 31) { _e = new ArgumentException( - $"Require valid Day of Month integer range 1-31 for Day
as part of format '{_format}'"); + $"Require valid Day of Month integer range 1-31 for Day
as part of format '{Format}'"); return; } + _dayOfMonth = dayOfMonth; if (splitOnT.Length > 1 && splitOnT[1].Length > 0) { - var t = new TimePartialParameter(splitOnT[1], _format); + var t = new TimePartialParameter(splitOnT[1], Format); _hour = t.Hour; _minute = t.Minute; _second = t.Second; @@ -111,21 +115,22 @@ private static bool SeparatorIsNotValid(string str, char validSeparator) { continue; } + if (c != validSeparator) { badSeparator = true; } + break; } + return badSeparator; } public DateTime GetDate() { - if (_e != null) - { - throw _e; - } - return new DateTime(_year, _month, _dayOfMonth, _hour, _minute, _second, DateTimeKind.Unspecified); + return _e != null + ? throw _e + : new DateTime(_year, _month, _dayOfMonth, _hour, _minute, _second, DateTimeKind.Unspecified); } } \ No newline at end of file diff --git a/src/cdeLib/Duplicates/Duplication.cs b/src/cdeLib/Duplicates/Duplication.cs index add8a4c..56278f8 100644 --- a/src/cdeLib/Duplicates/Duplication.cs +++ b/src/cdeLib/Duplicates/Duplication.cs @@ -23,7 +23,7 @@ public class Duplication private readonly Dictionary> _duplicateFileSize = new(); - private readonly HashSet _dirEntriesRequiringFullHashing = new(); + private readonly HashSet _dirEntriesRequiringFullHashing = []; protected readonly DuplicationStatistics _duplicationStatistics; @@ -79,11 +79,7 @@ public async Task ApplyHash(IList rootEntries) _logger.LogDebug("PostPairSize Memory: {0}", _applicationDiagnostics.GetMemoryAllocated().FormatAsBytes()); // Calculate all aggregations in single pass to avoid multiple enumerations - long totalFilesInRootEntries = 0; - foreach (var entry in rootEntries) - { - totalFilesInRootEntries += entry.FileEntryCount; - } + long totalFilesInRootEntries = rootEntries.Aggregate(0, (current, entry) => current + entry.FileEntryCount); int totalEntriesInSizeDupes = 0; int longestListLength = -1; @@ -139,7 +135,7 @@ public async Task ApplyHash(IList rootEntries) var root = System.IO.Directory.GetDirectoryRoot(pde.FullPath); if (!groupedByDirectoryRoot.TryGetValue(root, out var group)) { - group = new List(); + group = []; groupedByDirectoryRoot[root] = group; } diff --git a/src/cdeLib/Duplicates/DuplicationStatistics.cs b/src/cdeLib/Duplicates/DuplicationStatistics.cs index 333df96..09456e2 100644 --- a/src/cdeLib/Duplicates/DuplicationStatistics.cs +++ b/src/cdeLib/Duplicates/DuplicationStatistics.cs @@ -41,11 +41,5 @@ public void SeenFileSize(long value) SmallestFileSize = value < SmallestFileSize ? value : SmallestFileSize; } - public long FilesProcessed - { - get - { - return PartialHashes + FullHashes + AllreadyDonePartials + AllreadyDoneFulls + FailedToHash; - } - } + public long FilesProcessed => PartialHashes + FullHashes + AllreadyDonePartials + AllreadyDoneFulls + FailedToHash; } \ No newline at end of file diff --git a/src/cdeLib/Entities/Columnar/Utf8Matcher.cs b/src/cdeLib/Entities/Columnar/Utf8Matcher.cs index 0a6ff1d..5f14f57 100644 --- a/src/cdeLib/Entities/Columnar/Utf8Matcher.cs +++ b/src/cdeLib/Entities/Columnar/Utf8Matcher.cs @@ -22,7 +22,7 @@ public Utf8Matcher(string pattern) _pattern = pattern ?? string.Empty; _empty = _pattern.Length == 0; var bytes = _empty ? [] : Encoding.UTF8.GetBytes(_pattern); - _asciiPattern = System.Text.Ascii.IsValid(bytes); + _asciiPattern = Ascii.IsValid(bytes); if (_asciiPattern && !_empty) { for (var i = 0; i < bytes.Length; i++) bytes[i] = ToLower(bytes[i]); @@ -33,7 +33,7 @@ public Utf8Matcher(string pattern) public bool Contains(ReadOnlySpan nameUtf8) { if (_empty) return true; - if (_asciiPattern && System.Text.Ascii.IsValid(nameUtf8)) + if (_asciiPattern && Ascii.IsValid(nameUtf8)) return AsciiContainsFolded(nameUtf8, _patternLowerAscii); // Rare path: non-ASCII somewhere. Decode and compare with real ordinal-ignore-case. diff --git a/src/cdeLib/Entities/DirEntry.cs b/src/cdeLib/Entities/DirEntry.cs index ecd014f..27dbb74 100644 --- a/src/cdeLib/Entities/DirEntry.cs +++ b/src/cdeLib/Entities/DirEntry.cs @@ -53,8 +53,8 @@ private sealed class ExtraData [IgnoreMember] public DateTime Modified { - set => ModifiedTicks = value.Ticks; get => DateTime.FromBinary(ModifiedTicks); + set => ModifiedTicks = value.Ticks; } [ProtoMember(1, IsRequired = true)] @@ -198,7 +198,10 @@ public uint FileEntryCount set { if (value != 0) EnsureExtra().FileEntryCount = value; - else if (_extra != null) _extra.FileEntryCount = value; + else + { + _extra?.FileEntryCount = value; + } } } @@ -212,7 +215,10 @@ public uint DirEntryCount set { if (value != 0) EnsureExtra().DirEntryCount = value; - else if (_extra != null) _extra.DirEntryCount = value; + else + { + _extra?.DirEntryCount = value; + } } } @@ -391,9 +397,12 @@ public IList Children set { // A non-null child list (only directories have one) materialises ExtraData; files - // deserialize a nil Children and stay lean. + // deserialize nil Children and stay lean. if (value != null) EnsureExtra().Children = value; - else if (_extra != null) _extra.Children = null; + else + { + _extra?.Children = null; + } } } // ReSharper restore MemberCanBePrivate.Global @@ -425,13 +434,9 @@ public void AddChild(DirEntry child) public string Path { //NOTE: Separating the extension from the path is more memory efficient (300MB saved on 6000MB load) but slower. - get - { - //return _path; - + get => // string.concat faster than string interpolation. - return string.IsNullOrEmpty(_ext) ? _path : string.Concat(_path, _ext); - } + string.IsNullOrEmpty(_ext) ? _path : string.Concat(_path, _ext); set { @@ -605,7 +610,7 @@ public static IEnumerable GetPairDirEntries(IEnumerable public IList GetListFromRoot() { var activatedDirEntryList = new List(8); - for (var entry = (ICommonEntry)this; entry != null; entry = entry.ParentCommonEntry) + for (ICommonEntry entry = this; entry != null; entry = entry.ParentCommonEntry) { activatedDirEntryList.Add(entry); } diff --git a/src/cdeLib/Entities/RootEntry.cs b/src/cdeLib/Entities/RootEntry.cs index 8171fc4..ac6a3a2 100644 --- a/src/cdeLib/Entities/RootEntry.cs +++ b/src/cdeLib/Entities/RootEntry.cs @@ -64,8 +64,8 @@ public sealed class RootEntry : object, ICommonEntry [IgnoreMember] public DateTime ScanStartUtc { - set => ScanStartUtcTicks = value.Ticks; get => DateTime.FromBinary(ScanStartUtcTicks); + set => ScanStartUtcTicks = value.Ticks; } [FlatBufferItem(8)] @@ -76,8 +76,8 @@ public DateTime ScanStartUtc [IgnoreMember] public DateTime ScanEndUtc { - set => ScanEndUtcTicks = value.Ticks; get => DateTime.FromBinary(ScanEndUtcTicks); + set => ScanEndUtcTicks = value.Ticks; } [FlatBufferItem(9)] @@ -109,11 +109,7 @@ public RootEntry() : this(null, null) { } - public RootEntry(IConfiguration configuration) : this(configuration, null) - { - } - - public RootEntry(IConfiguration configuration, IFileSystemAdapter fileSystemAdapter) + public RootEntry(IConfiguration configuration, IFileSystemAdapter fileSystemAdapter = null) { TheRootEntry = this; _driveInfoService = new DriveInfoService(); @@ -150,7 +146,7 @@ private string GetRootEntry(string startPath) var driveInfo = _driveInfoService.GetDriveSpace(pathRoot); if (driveInfo.AvailableBytes != null) AvailSpace = driveInfo.AvailableBytes.Value; if (driveInfo.TotalBytes != null) TotalSpace = driveInfo.TotalBytes.Value; - VolumeName = this.GetVolumeName(GetDirectoryRoot(pathRoot)); + VolumeName = GetVolumeName(GetDirectoryRoot(pathRoot)); return startPath; } @@ -486,7 +482,7 @@ public void SortAllChildrenByPath() { // Sorting mutates the concrete child list, so work through the concrete DirEntry // (the abstract ICommonEntry.Children is a read-only view). - if (d is DirEntry { IsDirectory: true } de && de.Children?.Count > 1) + if (d is DirEntry { IsDirectory: true, Children.Count: > 1 } de) { de.Children.Sort((de1, de2) => de1.PathCompareWithDirTo(de2)); de.IsDefaultSort = true; @@ -520,8 +516,8 @@ public int DescriptionCompareTo(RootEntry re, IConfigCdeLib config) [IgnoreMember] public DateTime Modified { - set => ModifiedTicks = value.Ticks; get => DateTime.FromBinary(ModifiedTicks); + set => ModifiedTicks = value.Ticks; } [ProtoMember(12, IsRequired = false)] @@ -732,22 +728,13 @@ public int ModifiedCompareTo(ICommonEntry de) return -1; // this before de } - if (IsModifiedBad && !de.IsModifiedBad) - { - return -1; // this before de - } - - if (!IsModifiedBad && de.IsModifiedBad) + return IsModifiedBad switch { - return 1; // this after de - } - - if (IsModifiedBad && de.IsModifiedBad) - { - return 0; - } - - return DateTime.Compare(Modified, de.Modified); + true when !de.IsModifiedBad => -1, + false when de.IsModifiedBad => 1, + true when de.IsModifiedBad => 0, + _ => DateTime.Compare(Modified, de.Modified) + }; } // is this right ? for the simple compareResult invert we do in caller ? - maybe not ? keep dirs at top anyway ? @@ -758,20 +745,15 @@ public int PathCompareWithDirTo(ICommonEntry de) return -1; // this before de } - if (IsDirectory && !de.IsDirectory) + return IsDirectory switch { - return -1; // this before de - } - - if (!IsDirectory && de.IsDirectory) - { - return 1; // this after de - } - - return string.Compare(Path, de.Path, StringComparison.OrdinalIgnoreCase); + true when !de.IsDirectory => -1, + false when de.IsDirectory => 1, + _ => string.Compare(Path, de.Path, StringComparison.OrdinalIgnoreCase) + }; } - // can this be done with TraverseTree ? + // can this be done with TraverseTree? public void SetSummaryFields() { var size = 0L; @@ -821,8 +803,7 @@ public RootEntry GetRootEntry() public void AddChild(DirEntry child) { - if (this.Children == null) - Children = new List(); + Children ??= new List(); Children.Add(child); } @@ -938,7 +919,7 @@ public void TraverseTreesCopyHash(ICommonEntry destination) ValidateTreeCopyParameters(this, destination); var stack = new Stack<(string, ICommonEntry, ICommonEntry)>(capacity: 64); - stack.Push((this.Path, this, destination)); + stack.Push((Path, this, destination)); while (stack.Count > 0) { @@ -1044,7 +1025,7 @@ private static void TryCopyHashIfBeneficial(ICommonEntry source, ICommonEntry de } var shouldCopy = !destination.IsHashDone // Destination has no hash - || (source.IsPartialHash == false && destination.IsPartialHash); // Upgrading partial to full + || (!source.IsPartialHash && destination.IsPartialHash); // Upgrading partial to full if (shouldCopy) { diff --git a/src/cdeLib/Entities/Soa/EntryRef.cs b/src/cdeLib/Entities/Soa/EntryRef.cs index 9a357e1..d4ac9f7 100644 --- a/src/cdeLib/Entities/Soa/EntryRef.cs +++ b/src/cdeLib/Entities/Soa/EntryRef.cs @@ -83,7 +83,7 @@ public IReadOnlyList Children List list = null; foreach (var c in _source.ChildrenOf(_index)) { - (list ??= new List()).Add(new EntryRef(_source, c)); + (list ??= []).Add(new EntryRef(_source, c)); } return list; } @@ -122,9 +122,12 @@ public ICommonEntry ParentCommonEntry public int PathCompareWithDirTo(ICommonEntry de) { if (de == null) return -1; - if (IsDirectory && !de.IsDirectory) return -1; - if (!IsDirectory && de.IsDirectory) return 1; - return string.Compare(Path, de.Path, StringComparison.OrdinalIgnoreCase); + return IsDirectory switch + { + true when !de.IsDirectory => -1, + false when de.IsDirectory => 1, + _ => string.Compare(Path, de.Path, StringComparison.OrdinalIgnoreCase) + }; } public int SizeCompareWithDirTo(ICommonEntry de) diff --git a/src/cdeLib/Extensions/ListExtensions.cs b/src/cdeLib/Extensions/ListExtensions.cs index fcb7153..115f3cc 100644 --- a/src/cdeLib/Extensions/ListExtensions.cs +++ b/src/cdeLib/Extensions/ListExtensions.cs @@ -19,22 +19,21 @@ public static void TruncateList(this IList iList, int max) /// public static void Sort(this IList list, Comparison comparison) { - // Fast path: List has optimized Sort implementation - if (list is List concreteList) + switch (list) { - concreteList.Sort(comparison); - return; + // Fast path: List has optimized Sort implementation + case List concreteList: + concreteList.Sort(comparison); + return; + // Fast path: Array has optimized Sort implementation + case T[] array: + Array.Sort(array, comparison); + return; + default: + // Slow path: Generic IList - must copy, sort, and copy back + SortGenericList(list, comparison); + break; } - - // Fast path: Array has optimized Sort implementation - if (list is T[] array) - { - Array.Sort(array, comparison); - return; - } - - // Slow path: Generic IList - must copy, sort, and copy back - SortGenericList(list, comparison); } /// diff --git a/src/cdeLib/FindOptions.cs b/src/cdeLib/FindOptions.cs index 49b1162..49090e0 100644 --- a/src/cdeLib/FindOptions.cs +++ b/src/cdeLib/FindOptions.cs @@ -134,7 +134,7 @@ public void Find(IEnumerable rootEntries) var findFunc = GetFindFunc(_dummyProgressCount, limitCount); // ReSharper disable PossibleMultipleEnumeration - Parallel.ForEach(sortedRootEntries, parallelOptions, (rootEntry) => + Parallel.ForEach(sortedRootEntries, parallelOptions, rootEntry => { // Use single-entry overload to avoid array allocation EntryHelper.TraverseTreePair(rootEntry, findFunc); @@ -312,6 +312,7 @@ private Func GetPatternMatcher() private TraverseFunc GetFindFunc(int[] progressCount, int[] limitCount) { var findPredicate = GetFindPredicate(); + return FindFunc; bool FindFunc(ICommonEntry p, ICommonEntry dirEntry) { @@ -360,8 +361,6 @@ bool FindFunc(ICommonEntry p, ICommonEntry dirEntry) return true; } - - return FindFunc; } public void ResetProgress() diff --git a/src/cdeLib/FindService.cs b/src/cdeLib/FindService.cs index e0e80f6..5cf0c02 100644 --- a/src/cdeLib/FindService.cs +++ b/src/cdeLib/FindService.cs @@ -92,8 +92,7 @@ public void FindColumnar(string pattern, string param, IList { ++totalFound; diff --git a/src/cdeLib/Infrastructure/Config/AppConfigurationSection.cs b/src/cdeLib/Infrastructure/Config/AppConfigurationSection.cs index 84d7570..e531073 100644 --- a/src/cdeLib/Infrastructure/Config/AppConfigurationSection.cs +++ b/src/cdeLib/Infrastructure/Config/AppConfigurationSection.cs @@ -4,8 +4,8 @@ public class AppConfigurationSection { public AppConfigurationSection() { - this.Display = new DisplaySection(); - this.Hashing = new HashingSection(); + Display = new DisplaySection(); + Hashing = new HashingSection(); } public DisplaySection Display { get; set; } diff --git a/src/cdeLib/Infrastructure/FileSystemAdapter.cs b/src/cdeLib/Infrastructure/FileSystemAdapter.cs index 2dd7bbd..e11b91e 100644 --- a/src/cdeLib/Infrastructure/FileSystemAdapter.cs +++ b/src/cdeLib/Infrastructure/FileSystemAdapter.cs @@ -14,7 +14,7 @@ public string GetFullPath(string path) public bool IsUnc(string path) { - return Path.IsPathFullyQualified(path) && path.StartsWith("\\\\"); + return Path.IsPathFullyQualified(path) && path.StartsWith(@"\\"); } public string GetDirectoryRoot(string path) diff --git a/src/cdeLib/Infrastructure/Hashing/HashHelper.cs b/src/cdeLib/Infrastructure/Hashing/HashHelper.cs index b002ede..96d6b5f 100644 --- a/src/cdeLib/Infrastructure/Hashing/HashHelper.cs +++ b/src/cdeLib/Infrastructure/Hashing/HashHelper.cs @@ -45,7 +45,7 @@ public async Task GetHashResponseFromFile(string filename, int? by totalBytesRead = bytesRead; while (bytesRead > 0 && totalBytesRead <= bytesToHash) { - bytesRead = stream.Read(rentedBuffer, 0, bufferSize); + bytesRead = await stream.ReadAsync(rentedBuffer.AsMemory(0, bufferSize)); totalBytesRead += bytesRead; } diff --git a/src/cdeLib/Infrastructure/Hashing/MurMurHash3.cs b/src/cdeLib/Infrastructure/Hashing/MurMurHash3.cs index be3ec60..b2b3b90 100644 --- a/src/cdeLib/Infrastructure/Hashing/MurMurHash3.cs +++ b/src/cdeLib/Infrastructure/Hashing/MurMurHash3.cs @@ -42,7 +42,7 @@ public static uint Hash(ReadOnlySpan data, uint seed) h1 ^= k1; h1 = rotl32(h1, 13); - h1 = (h1 * 5) + 0xe6546b64; + h1 = h1 * 5 + 0xe6546b64; position += 4; } @@ -119,7 +119,7 @@ public static uint Hash(Stream stream, UInt32 seed) h1 ^= k1; h1 = rotl32(h1, 13); - h1 = (h1 * 5) + 0xe6546b64; + h1 = h1 * 5 + 0xe6546b64; break; case 3: k1 = (uint) diff --git a/src/cdeLib/Infrastructure/ObjectPool.cs b/src/cdeLib/Infrastructure/ObjectPool.cs index bf6641e..e639b13 100644 --- a/src/cdeLib/Infrastructure/ObjectPool.cs +++ b/src/cdeLib/Infrastructure/ObjectPool.cs @@ -37,7 +37,7 @@ public ObjectPool(Func objectGenerator, Action resetAction = null, int max public T Get() { - if (_disposed) throw new ObjectDisposedException(nameof(ObjectPool)); + if (_disposed) throw new ObjectDisposedException(nameof(ObjectPool<>)); if (_objects.TryDequeue(out var item)) { @@ -90,7 +90,7 @@ public static class PoolManager new(() => new StringBuilder(256), sb => sb.Clear(), 50); private static readonly ObjectPool> StringListPool = - new(() => new List(), list => list.Clear(), 30); + new(() => [], list => list.Clear(), 30); public static StringBuilder GetStringBuilder() => StringBuilderPool.Get(); public static void ReturnStringBuilder(StringBuilder sb) => StringBuilderPool.Return(sb); diff --git a/src/cdeLib/TimePartialParameter.cs b/src/cdeLib/TimePartialParameter.cs index e595cd9..febad30 100644 --- a/src/cdeLib/TimePartialParameter.cs +++ b/src/cdeLib/TimePartialParameter.cs @@ -10,33 +10,30 @@ public class TimePartialParameter // "HH:MM:SS"; example private const string Format = "::"; - private readonly int _hour; // 0 - 23 public int Hour { get { ThrowExceptionIfSet(); - return _hour; + return field; } } - private readonly int _minute; // 0 - 59 public int Minute { get { ThrowExceptionIfSet(); - return _minute; + return field; } } - private readonly int _second; // 0 - 59 public int Second { get { ThrowExceptionIfSet(); - return _second; + return field; } } @@ -55,13 +52,13 @@ public TimePartialParameter(string str, string activeFormat = Format) var activeFormat1 = activeFormat; var splitOnColon = str.Split(':'); int.TryParse(splitOnColon[0], out var hour); - if (hour == 0 || hour > 23) + if (hour is 0 or > 23) { _e = new ArgumentException( $"Require valid Integer 1-23 for Hour as part of format '{activeFormat1}'"); return; } - _hour = hour; + Hour = hour; if (splitOnColon.Length > 1) // may have an hour specified { @@ -77,7 +74,7 @@ public TimePartialParameter(string str, string activeFormat = Format) $"Require valid integer 1-59 or for Minute as part of format '{activeFormat1}'"); return; } - _minute = minute; + Minute = minute; } if (splitOnColon.Length > 2) // may have second specified @@ -94,7 +91,7 @@ public TimePartialParameter(string str, string activeFormat = Format) $"Require valid integer 1-59 or for Second as part of format '{activeFormat1}'"); return; } - _second = second; + Second = second; } } } \ No newline at end of file diff --git a/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs b/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs index 1aa0e0b..9a623e7 100644 --- a/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs +++ b/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs @@ -173,7 +173,7 @@ public void CatalogTreeBuilder_RoundTrips_StructureSizesAndHashes() // Set a hash on one file so the hash column round-trips too (the hash/dupes path). var root = BuildTree(); var beta = root.Children.First(c => c.Path == "dir1").Children.First(c => c.Path == "beta.log"); - beta.SetHash(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 }); + beta.SetHash([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); root.SetInMemoryFields(); var store0 = EntryStore.Build(root); @@ -181,7 +181,7 @@ public void CatalogTreeBuilder_RoundTrips_StructureSizesAndHashes() try { // .cdex -> mutable tree (as hash/dupes do) -> store again must match the original store. - var trees = CatalogTreeBuilder.FromColumnarFiles(new[] { path }); + var trees = CatalogTreeBuilder.FromColumnarFiles([path]); Assert.That(trees, Has.Count.EqualTo(1)); var store2 = EntryStore.Build(trees[0]); @@ -210,7 +210,7 @@ private static RootEntry BuildForCopy(bool withHash) Size = 100, Modified = new System.DateTime(2020, 1, 1, 0, 0, 0, System.DateTimeKind.Utc), }; - if (withHash) f.SetHash(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16 }); + if (withHash) f.SetHash([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); root.AddChild(f); root.SetInMemoryFields(); return root; @@ -225,7 +225,7 @@ public void TraverseTreesCopyHash_FromReconstructedCdex_CopiesHashToFreshScan() var path = WriteTemp(store); try { - var oldRoot = CatalogTreeBuilder.FromColumnarFiles(new[] { path })[0]; + var oldRoot = CatalogTreeBuilder.FromColumnarFiles([path])[0]; var fresh = BuildForCopy(withHash: false); oldRoot.TraverseTreesCopyHash(fresh); diff --git a/src/cdeLibTest/DirEntryEnumeratorTest.cs b/src/cdeLibTest/DirEntryEnumeratorTest.cs index 4c8df38..c80ca19 100644 --- a/src/cdeLibTest/DirEntryEnumeratorTest.cs +++ b/src/cdeLibTest/DirEntryEnumeratorTest.cs @@ -54,7 +54,7 @@ public void Constructor_Minimal_NoErrors() [Test] public void MoveNext_NoRootEntries_FirstMoveNextFalse() { - RootEntries = new List(); + RootEntries = []; var e = new DirEntryEnumerator(RootEntries); diff --git a/src/cdeLibTest/DuplicationTest.cs b/src/cdeLibTest/DuplicationTest.cs index f79450f..2ae9768 100644 --- a/src/cdeLibTest/DuplicationTest.cs +++ b/src/cdeLibTest/DuplicationTest.cs @@ -167,7 +167,7 @@ private string AssemblyPathLocation() public void GetSizePairs_CheckSanityOfDupeSizeCountAndDupeFileCount_Exercise() { const int dupeCount = 10; - var testPath = this.AssemblyPathLocation(); + var testPath = AssemblyPathLocation(); // Create some dummy duplicate data. // create a catalog var random = FileHelper.RandomString(4096 * 16); @@ -176,7 +176,7 @@ public void GetSizePairs_CheckSanityOfDupeSizeCountAndDupeFileCount_Exercise() FileHelper.WriteAllText(random, testPath, $"CDE_testFile{i}.txt"); } - Program.InitProgram(Array.Empty()); + Program.InitProgram([]); Program.CreateCache(new ScanOptions {Path = testPath}); // scan writes a columnar .cdex Program.HashCatalog(); // hash operates on the .cdex @@ -221,15 +221,10 @@ private static long GetSumOfUniqueHashesForEachSize_ExcludePartialHash( var seenHash = new Dictionary(); foreach (var flatDe in fdeListOfSize) { - // var hash = flatDe.ChildDE.Hash; if (flatDe.ChildDE.IsHashDone // because this is run on SizeDupe list it can have null hashes. && !flatDe.ChildDE.IsPartialHash) { - if (!seenHash.ContainsKey(flatDe.ChildDE.Hash)) - { - seenHash[flatDe.ChildDE.Hash] = 0; - } - else + if (!seenHash.TryAdd(flatDe.ChildDE.Hash, 0)) { ++seenHash[flatDe.ChildDE.Hash]; } diff --git a/src/cdeLibTest/IdeaStructNode.cs b/src/cdeLibTest/IdeaStructNode.cs index 3ad8794..c11f896 100644 --- a/src/cdeLibTest/IdeaStructNode.cs +++ b/src/cdeLibTest/IdeaStructNode.cs @@ -96,7 +96,5 @@ internal struct Node public Hash16 Hash; // (16) -- ..(52) } -internal class IdeaStructNode -{ -} +internal class IdeaStructNode; #pragma warning restore 0649 \ No newline at end of file diff --git a/src/cdeLibTest/Infrastructure/DuplicationPerfTest.cs b/src/cdeLibTest/Infrastructure/DuplicationPerfTest.cs index 04e4fe2..7232f46 100644 --- a/src/cdeLibTest/Infrastructure/DuplicationPerfTest.cs +++ b/src/cdeLibTest/Infrastructure/DuplicationPerfTest.cs @@ -51,7 +51,7 @@ public void PerformanceHashTest() timer.Stop(); Console.WriteLine( - $"{hashKey}:\t\t{(data.Length * (1000.0 / (timer.ElapsedMilliseconds / 9999.0))) / (1024.0 * 1024.0):F2} MB/s ({timer.ElapsedMilliseconds})"); + $"{hashKey}:\t\t{data.Length * (1000.0 / (timer.ElapsedMilliseconds / 9999.0)) / (1024.0 * 1024.0):F2} MB/s ({timer.ElapsedMilliseconds})"); } } } \ No newline at end of file diff --git a/src/cdeLibTest/Infrastructure/Hashing/Crc32.cs b/src/cdeLibTest/Infrastructure/Hashing/Crc32.cs index 00d23f7..6b35a7d 100644 --- a/src/cdeLibTest/Infrastructure/Hashing/Crc32.cs +++ b/src/cdeLibTest/Infrastructure/Hashing/Crc32.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Linq; using cdeLib.Infrastructure.Hashing; namespace cdeLibTest.Infrastructure.Hashing; @@ -35,9 +36,7 @@ private void Init(uint poly) public UInt64 Hash(byte[] data) { - uint hash = 0xFFFFFFFF; - foreach (byte b in data) - hash = (hash << 8) ^ _tab[b ^ (hash >> 24)]; + var hash = data.Aggregate(0xFFFFFFFF, (current, b) => (current << 8) ^ _tab[b ^ (current >> 24)]); return ~hash; } diff --git a/src/cdeLibTest/Performance/PerformanceTreeTraversal.cs b/src/cdeLibTest/Performance/PerformanceTreeTraversal.cs index dc236a9..29c2cea 100644 --- a/src/cdeLibTest/Performance/PerformanceTreeTraversal.cs +++ b/src/cdeLibTest/Performance/PerformanceTreeTraversal.cs @@ -30,8 +30,8 @@ internal class PerformanceTreeTraversal // ReSharper disable JoinDeclarationAndInitializer // ReSharper disable PossibleMultipleEnumeration - private readonly int _repeatSmall = 100; - private readonly int _repeatLarge = 25; + private const int _repeatSmall = 100; + private const int _repeatLarge = 25; [SetUpFixture] public class TestData diff --git a/src/cdeLibTest/RootEntryTest.cs b/src/cdeLibTest/RootEntryTest.cs index b9cdddc..4f679be 100644 --- a/src/cdeLibTest/RootEntryTest.cs +++ b/src/cdeLibTest/RootEntryTest.cs @@ -46,33 +46,10 @@ public void Constructor_GetTreeWithMoreThanOneLevel_OK() re.RecurseTree(FileHelper.TestDir); Assert.That(re, Is.Not.Null); - var found = re.Children.Any(x => x.Children != null && x.Children.Count > 0); + var found = re.Children.Any(x => x.Children is { Count: > 0 }); Assert.That(found, Is.True, "One of entries does not have children."); } - //[Test] - //public void FindDir_LookForDir_InRoot() - //{ - // const string rootPath = @"C:\"; - // var re = new RootEntry { Path = rootPath }; - - // var foundEntry = re.FindDir(rootPath, @"C:\Moo"); - - // Assert.That(foundEntry, Is.InstanceOf(typeof(RootEntry))); - //} - - //[Test] - //public void FindDir_NotExistinRoot_ReturnRE() - //{ - // const string rootPath = @"C:\"; - // const string testPath = @"C:\Groo"; - // var re = new RootEntry { Path = rootPath }; - - // var foundEntry = re.FindDir(rootPath, testPath); - - // Assert.That(foundEntry, Is.InstanceOf(typeof(RootEntry))); - //} - [Test] public void GetDriverLetterHint_SimpleRootPath_ReturnsDriveLetter() { @@ -288,9 +265,7 @@ public void SetFullPath_OnRootDirectory_SetsAllFullPaths() re.SetInMemoryFields(); Assert.That(re.FullPath, Is.EqualTo(@"C:\")); - //Assert.That(fe1.FullPath, Is.Null);// Is.EqualTo(@"C:\fe1")); FullPath only set on directories to save memory. Assert.That(de2.FullPath, Is.EqualTo(@"C:\de2")); - //Assert.That(fe3.FullPath, Is.Null);//Is.EqualTo(@"C:\de2\fe3")); } } // ReSharper restore InconsistentNaming diff --git a/src/cdeLibTest/Soa/EntryRefTests.cs b/src/cdeLibTest/Soa/EntryRefTests.cs index 53210f2..64ca4b1 100644 --- a/src/cdeLibTest/Soa/EntryRefTests.cs +++ b/src/cdeLibTest/Soa/EntryRefTests.cs @@ -86,12 +86,11 @@ public void GetListFromRoot_OverEntryRef_GoesRootToLeaf() { var root = BuildTree(); var store = EntryStore.Build(root); - var storeRoot = new EntryRef(store, 0); // Find alpha.txt under dir1 and walk back to root. var alpha = TraverseFind(store, "alpha.txt"); var chain = alpha.GetListFromRoot().Select(e => e.Path).ToList(); - Assert.That(chain, Is.EqualTo(new[] { @"C:\test", "dir1", "alpha.txt" })); + Assert.That(chain, Is.EqualTo([@"C:\test", "dir1", "alpha.txt"])); Assert.That(alpha.FullPath, Is.EqualTo(@"C:\test\dir1\alpha.txt")); } diff --git a/src/cdeLibTest/Soa/EntryStoreTests.cs b/src/cdeLibTest/Soa/EntryStoreTests.cs index c27ef7f..72b1373 100644 --- a/src/cdeLibTest/Soa/EntryStoreTests.cs +++ b/src/cdeLibTest/Soa/EntryStoreTests.cs @@ -55,7 +55,7 @@ private static List TreeFind(RootEntry root, string pattern, bool regex, LimitResultCount = int.MaxValue, VisitorFunc = (p, d) => { found.Add(p.MakeFullPath(d)); return true; }, }; - options.Find(new[] { root }); + options.Find([root]); return found; } @@ -161,7 +161,7 @@ public void Search_WithSizeFilter_FiltersBySize() i => found.Add(store.FullName(i))); found.Sort(); - Assert.That(found, Is.EqualTo(new[] { "big.txt", "mid.txt" })); // size >= 25 + Assert.That(found, Is.EqualTo(["big.txt", "mid.txt"])); // size >= 25 } [Test] diff --git a/src/cdeLibTest/TimePartialParameterTest.cs b/src/cdeLibTest/TimePartialParameterTest.cs index 1450651..1759ddb 100644 --- a/src/cdeLibTest/TimePartialParameterTest.cs +++ b/src/cdeLibTest/TimePartialParameterTest.cs @@ -37,7 +37,7 @@ public void Bad_Hour_Parameter() [Test] public void Hour_With_Minute_Parameter() { - var args = "3:34"; + const string args = "3:34"; var d = new TimePartialParameter(args); Assert.That(d.Hour, Is.EqualTo(3)); Assert.That(d.Minute, Is.EqualTo(34)); @@ -63,7 +63,7 @@ public void Hour_With_Too_Large_Minute_Parameter() [Test] public void Hour_With_Minute_With_Second_Parameter() { - var args = "3:34:10"; + const string args = "3:34:10"; var d = new TimePartialParameter(args); Assert.That(d.Hour, Is.EqualTo(3)); Assert.That(d.Minute, Is.EqualTo(34)); diff --git a/src/cdeWin/CDEWinForm.cs b/src/cdeWin/CDEWinForm.cs index 516565d..2c9c682 100644 --- a/src/cdeWin/CDEWinForm.cs +++ b/src/cdeWin/CDEWinForm.cs @@ -176,19 +176,19 @@ private void RegisterClientEvents() "Using reload catalogs will use more memory than quitting and starting again."); SetToolTip(regexCheckbox, "Disabling Regex makes search faster"); - whatToSearchComboBox.Items.AddRange(new object[] { "Include Path in Search", "Exclude Path from Search" }); + whatToSearchComboBox.Items.AddRange("Include Path in Search", "Exclude Path from Search"); whatToSearchComboBox.SelectedIndex = 0; // default Include whatToSearchComboBox.DropDownStyle = ComboBoxStyle.DropDownList; SetToolTip(whatToSearchComboBox, "Excluding Path so that only entry Names are searched makes search faster."); - findComboBox.Items.AddRange(new object[] { "Files and Folders", "Files Only", "Folders Only" }); + findComboBox.Items.AddRange("Files and Folders", "Files Only", "Folders Only"); findComboBox.SelectedIndex = 0; // default Files and Folders findComboBox.DropDownStyle = ComboBoxStyle.DropDownList; - // TODO having ListViewHelper setup in VIEW breaks passive view. i think. + // TODO having ListViewHelper set up in VIEW breaks passive view. i think. // * it does register a bunch of events which it fires.... ? so not real bad. - // - whats happening is im making view smarter... with specific behaviour. + // - whats happening is I'm making view smarter... with specific behaviour. // - but its not passive, passive would require ListViewHelper to raise events // - from gui actions.... and decisions from presenter... // - - at moment, ListViewHelper is small presenter ? @@ -492,9 +492,9 @@ public TreeNode DirectoryTreeViewNodes } } - public bool IncludeFiles => findComboBox.SelectedIndex == 0 || findComboBox.SelectedIndex == 1; + public bool IncludeFiles => findComboBox.SelectedIndex is 0 or 1; - public bool IncludeFolders => findComboBox.SelectedIndex == 0 || findComboBox.SelectedIndex == 2; + public bool IncludeFolders => findComboBox.SelectedIndex is 0 or 2; [DesignerSerializationVisibility(DesignerSerializationVisibility.Hidden)] public int FindEntryFilter diff --git a/src/cdeWin/CDEWinFormPresenter.cs b/src/cdeWin/CDEWinFormPresenter.cs index f6c6e67..e5eef54 100644 --- a/src/cdeWin/CDEWinFormPresenter.cs +++ b/src/cdeWin/CDEWinFormPresenter.cs @@ -200,7 +200,7 @@ public async Task InitializeAsync() private void OnLoadProgress(int current, int total, string message) { - if (_clientForm is Control control && control.InvokeRequired) + if (_clientForm is Control { InvokeRequired: true } control) { control.BeginInvoke(() => OnLoadProgress(current, total, message)); return; @@ -210,7 +210,7 @@ private void OnLoadProgress(int current, int total, string message) _clientForm.SetSearchTimeStatus(message); if (total > 0) { - _clientForm.SetLoadingProgressValue((current * 100) / total); + _clientForm.SetLoadingProgressValue(current * 100 / total); } SetMemoryStatus(); } @@ -326,7 +326,7 @@ private static void CreateNodesPreExpand(TreeNode parentNode) private static bool HasDummyChildNode(TreeNode parentNode) { - return parentNode.Nodes.Count == 1 && parentNode.Nodes[0].Text == DummyNodeName; + return parentNode.Nodes is [{ Text: DummyNodeName }]; } private static void AddAllDirectoriesChildren(TreeNode treeNode, ICommonEntry dirEntry) @@ -339,12 +339,10 @@ private static void AddAllDirectoriesChildren(TreeNode treeNode, ICommonEntry di private static void AddDirectoryChildren(TreeNode treeNode, ICommonEntry dirEntry) { - if (dirEntry.IsDirectory) - { - var newTreeNode = NewTreeNode(dirEntry); - treeNode.Nodes.Add(newTreeNode); - SetDummyChildNode(newTreeNode, dirEntry); - } + if (!dirEntry.IsDirectory) return; + var newTreeNode = NewTreeNode(dirEntry); + treeNode.Nodes.Add(newTreeNode); + SetDummyChildNode(newTreeNode, dirEntry); } /// @@ -493,16 +491,13 @@ public void Search() private bool FromToDateInvalid() { - if (_clientForm.FromDate.Checked - && _clientForm.ToDate.Checked - && _clientForm.FromDateValue.Date >= _clientForm.ToDateValue.Date) - { - _clientForm.MessageBox( - "The From Date Field is greater than the To Date field no search results possible."); - return true; - } + if (!_clientForm.FromDate.Checked + || !_clientForm.ToDate.Checked + || _clientForm.FromDateValue.Date < _clientForm.ToDateValue.Date) return false; + _clientForm.MessageBox( + "The From Date Field is greater than the To Date field no search results possible."); + return true; - return false; } private bool FromToHourInvalid() @@ -521,31 +516,23 @@ private bool FromToHourInvalid() private bool RegexIsBad() { - if (_clientForm.RegexMode) - { - var regexError = RegexHelper.GetRegexErrorMessage(_clientForm.Pattern); - if (!string.IsNullOrEmpty(regexError)) - { - _clientForm.MessageBox(regexError); - return true; - } - } + if (!_clientForm.RegexMode) return false; + var regexError = RegexHelper.GetRegexErrorMessage(_clientForm.Pattern); + if (string.IsNullOrEmpty(regexError)) return false; + _clientForm.MessageBox(regexError); + return true; - return false; } private bool FromToSizeInvalid() { - if (_clientForm.FromSize.Checked - && _clientForm.ToSize.Checked - && FromSizeValue() > ToSizeValue()) - { - _clientForm.MessageBox( - "The From Size Field is greater than the To Size field no search results possible."); - return true; - } + if (!_clientForm.FromSize.Checked + || !_clientForm.ToSize.Checked + || FromSizeValue() <= ToSizeValue()) return false; + _clientForm.MessageBox( + "The From Size Field is greater than the To Size field no search results possible."); + return true; - return false; } private long FromSizeValue() @@ -639,17 +626,6 @@ private void BgWorkerDoWork(object sender, DoWorkEventArgs e) var lastReport = Stopwatch.GetTimestamp(); var reportTicks = Stopwatch.Frequency / 10; // ~100ms streaming - void Report(int scanned) - { - var now = Stopwatch.GetTimestamp(); - if (now - lastReport < reportTicks) return; - lastReport = now; - state.ListCount = list.Count; - state.List = new List(list); // immutable snapshot for the UI thread - state.Counter = scanned; - worker.ReportProgress(grandTotal > 0 ? (int)(100.0 * scanned / grandTotal) : 0, state); - } - var timer = Stopwatch.StartNew(); foreach (var source in sources) { @@ -673,6 +649,18 @@ void Report(int scanned) state.Counter = grandTotal; worker.ReportProgress(100, state); e.Result = list; + return; + + void Report(int scanned) + { + var now = Stopwatch.GetTimestamp(); + if (now - lastReport < reportTicks) return; + lastReport = now; + state.ListCount = list.Count; + state.List = new List(list); // immutable snapshot for the UI thread + state.Counter = scanned; + worker.ReportProgress(grandTotal > 0 ? (int)(100.0 * scanned / grandTotal) : 0, state); + } } private void BgWorkerRunWorkerCompleted(object sender, RunWorkerCompletedEventArgs e) @@ -1113,12 +1101,10 @@ private void ViewFileInDirectoryTab(PairDirEntry pde) private void SelectFileInDirectoryTab(ICommonEntry dirEntry) { - if (!dirEntry.IsDirectory) - { - var index = _directoryList.IndexOf(dirEntry); - var directoryHelper = _clientForm.DirectoryListViewHelper; - directoryHelper.SelectItem(index); - } + if (dirEntry.IsDirectory) return; + var index = _directoryList.IndexOf(dirEntry); + var directoryHelper = _clientForm.DirectoryListViewHelper; + directoryHelper.SelectItem(index); } public void DirectoryContextMenuOpenClick() @@ -1157,7 +1143,7 @@ public void DirectoryContextMenuCopyFullPathClick() { DirectoryGetContextMenuPairDirEntries(enumerableDirEntry => { - // we dont have parent dir entry here ... + // we don't have parent dir entry here var s = new StringBuilder(); foreach (var dirEntry in enumerableDirEntry) { diff --git a/src/cdeWin/ContextMenuHelper.cs b/src/cdeWin/ContextMenuHelper.cs index 9e72284..e515505 100644 --- a/src/cdeWin/ContextMenuHelper.cs +++ b/src/cdeWin/ContextMenuHelper.cs @@ -145,16 +145,14 @@ public EventHandler ParentHandler /// public CancelEventHandler CancelOpeningEventHandler { - get => _cancelOpeningEventHandler; + get; set { - _cancelOpeningEventHandler = value; + field = value; _menu.Opening += value; } } - private CancelEventHandler _cancelOpeningEventHandler; - public ContextMenuHelper() { // set all keys here rather than in individual setters for handlers. diff --git a/src/cdeWin/KeyEqualityComparer.cs b/src/cdeWin/KeyEqualityComparer.cs index 9917acc..46729f1 100644 --- a/src/cdeWin/KeyEqualityComparer.cs +++ b/src/cdeWin/KeyEqualityComparer.cs @@ -10,7 +10,7 @@ public class KeyEqualityComparer : IEqualityComparer { protected readonly Func KeyExtractor; - public KeyEqualityComparer(Func keyExtractor) + protected KeyEqualityComparer(Func keyExtractor) { KeyExtractor = keyExtractor; } diff --git a/src/cdeWin/LoadCatalogService.cs b/src/cdeWin/LoadCatalogService.cs index fe537c4..9d79df7 100644 --- a/src/cdeWin/LoadCatalogService.cs +++ b/src/cdeWin/LoadCatalogService.cs @@ -104,7 +104,7 @@ public async Task> LoadRootEntriesAsync( var now = DateTime.UtcNow; if (currentCount % progressReportThreshold == 0 || - (now - lastProgressReport) > progressReportInterval) + now - lastProgressReport > progressReportInterval) { progressCallback?.Invoke(currentCount, totalFiles, $"Loading catalog {currentCount} of {totalFiles}..."); diff --git a/src/cdeWin/LoaderForm.cs b/src/cdeWin/LoaderForm.cs index c94429b..f22a164 100644 --- a/src/cdeWin/LoaderForm.cs +++ b/src/cdeWin/LoaderForm.cs @@ -138,7 +138,7 @@ private List LoadCatalogs(BackgroundWorker worker) // Time-based or count-based progress reporting (whichever comes first) var now = DateTime.UtcNow; if (currentCount % progressReportThreshold == 0 || - (now - lastProgressReport) > progressReportInterval) + now - lastProgressReport > progressReportInterval) { worker.ReportProgress((int)(currentCount / (float)totalFiles * 100), new LoadingState(currentCount, totalFiles)); diff --git a/src/cdeWin/Program.cs b/src/cdeWin/Program.cs index 2d0c2b2..ba092d7 100644 --- a/src/cdeWin/Program.cs +++ b/src/cdeWin/Program.cs @@ -12,9 +12,9 @@ internal static class Program { public static IConfigurationRoot Configuration; - public static string Version => Application.ProductVersion; + private static string Version => Application.ProductVersion; - public static string ProductName => Application.ProductName; + private static string ProductName => Application.ProductName; [STAThread] private static void Main() diff --git a/src/cdeWin/SplitContainerExtensions.cs b/src/cdeWin/SplitContainerExtensions.cs index feb7005..7a30385 100644 --- a/src/cdeWin/SplitContainerExtensions.cs +++ b/src/cdeWin/SplitContainerExtensions.cs @@ -20,7 +20,7 @@ public static void SetSplitterRatio(this SplitContainer splitter, float splitter } } - public static int GetSplitterSize(this SplitContainer splitter) + private static int GetSplitterSize(this SplitContainer splitter) { return splitter.Orientation == Orientation.Vertical ? splitter.Width diff --git a/src/cdeWin/StringExtension.cs b/src/cdeWin/StringExtension.cs index 89841aa..70a18df 100644 --- a/src/cdeWin/StringExtension.cs +++ b/src/cdeWin/StringExtension.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Threading; namespace cdeWin; @@ -9,7 +10,7 @@ public static class StringExtension // Cache for formatted size strings (key: size, value: formatted string) private static readonly Dictionary SizeCache = new(1024); - private static readonly object SizeCacheLock = new(); + private static readonly Lock SizeCacheLock = new(); public static string ToHRString(this long val) { diff --git a/src/cdeWinTest/CDEWinFormPresenterTest.cs b/src/cdeWinTest/CDEWinFormPresenterTest.cs index 4a6c39d..c255f8f 100644 --- a/src/cdeWinTest/CDEWinFormPresenterTest.cs +++ b/src/cdeWinTest/CDEWinFormPresenterTest.cs @@ -31,8 +31,7 @@ public override void RunBeforeEveryTest() [Test] public void Always_Set_Search_Button() { - var _ = new CDEWinFormPresenter(_mockForm, _stubConfig); - + _ = new CDEWinFormPresenter(_mockForm, _stubConfig); _mockForm.Received().SearchButtonText = "Search"; } @@ -56,7 +55,7 @@ public void Always_Catalog_SortList() [Test] public void Always_Register_Result_Sorters() { - var _ = new CDEWinFormPresenter(_mockForm, _stubConfig); + _ = new CDEWinFormPresenter(_mockForm, _stubConfig); _mockSearchResultListViewHelper.ColumnSortCompare = Arg.Any>(); _mockCatalogListViewHelper.ColumnSortCompare = Arg.Any>(); diff --git a/src/cdeWinTest/TestCDEWinPresenterBase.cs b/src/cdeWinTest/TestCDEWinPresenterBase.cs index 67fd605..ec809d0 100644 --- a/src/cdeWinTest/TestCDEWinPresenterBase.cs +++ b/src/cdeWinTest/TestCDEWinPresenterBase.cs @@ -26,19 +26,17 @@ protected cdeLib.Entities.Soa.EntryRef CatalogRootOf(RootEntry re) => new(cdeLib.Entities.Soa.EntryStore.Build(re), 0); protected RootEntry _rootEntry; - protected DirEntry _dirEntry; + private DirEntry _dirEntry; protected PairDirEntry _pairDirEntry; - protected List _emptyRootList = new(); - protected List _rootList = new(); + protected List _rootList = []; // protected TreeNode _treeViewAfterSelectNode; private readonly IConfiguration _config = Substitute.For(); [SetUp] public virtual void RunBeforeEveryTest() { - _emptyRootList = new List(); - _rootList = new List(); + _rootList = []; _config.ProgressUpdateInterval.Returns(5000); _mockForm = Substitute.For(); @@ -86,7 +84,7 @@ protected void InitRootWithFile() protected void InitRootWithDir() { - // massive assumption on path, this T:\ is windows only...... + // massive assumption on the path, this T:\ is windows only...... // is it a valid test on other platforms or behavior on other platforms? _rootEntry = new RootEntry(_config) { Path = @"T:\" }; _dirEntry = new DirEntry(true) { Path = "Test1" }; diff --git a/src/cdeWinTest/TestCDEWinPresenter_OptimiseRegexPattern.cs b/src/cdeWinTest/TestCDEWinPresenter_OptimiseRegexPattern.cs index a9301a2..17c88a8 100644 --- a/src/cdeWinTest/TestCDEWinPresenter_OptimiseRegexPattern.cs +++ b/src/cdeWinTest/TestCDEWinPresenter_OptimiseRegexPattern.cs @@ -8,7 +8,7 @@ namespace cdeWinTest; [TestFixture] public class TestCDEWinPresenter_OptimiseRegexPattern_NotRegex : TestCDEWinPresenterBase { - protected TestOptimise _presenter; + private TestOptimise _presenter; [SetUp] public override void RunBeforeEveryTest() @@ -55,10 +55,10 @@ public void OptimiseRegexPattern_TrailingWild_ReturnsUnchanged_WhenNotRegexMode( [TestFixture] public class TestCDEWinPresenter_OptimiseRegexPattern_Regex : TestCDEWinPresenterBase { - protected TestOptimise _presenter; + private TestOptimise _presenter; [SetUp] - override public void RunBeforeEveryTest() + public override void RunBeforeEveryTest() { base.RunBeforeEveryTest(); _mockForm.RegexMode = true; From 546df4df4745480649269311549d4615a9c65305 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Tue, 9 Jun 2026 10:43:13 +1000 Subject: [PATCH 40/43] tidy: minor tidy. Mostly spelling, separate type per file. --- src/cde/CommandLine/CommandLineOptions.cs | 2 +- src/cdeLib/Catalog/CatalogRepository.cs | 14 +-- .../CommonEntryTest_TraverseTreesCopyHash.cs | 4 +- src/cdeWin/IListViewHelper.cs | 42 ++++++++ src/cdeWin/KeyEqualityComparer.cs | 17 ---- src/cdeWin/ListViewExtensions.cs | 2 +- src/cdeWin/ListViewHelper.cs | 98 +++++-------------- src/cdeWin/StrictKeyEqualityComparer.cs | 20 ++++ src/cdeWin/UpDownHelper.cs | 2 +- 9 files changed, 97 insertions(+), 104 deletions(-) create mode 100644 src/cdeWin/IListViewHelper.cs create mode 100644 src/cdeWin/StrictKeyEqualityComparer.cs diff --git a/src/cde/CommandLine/CommandLineOptions.cs b/src/cde/CommandLine/CommandLineOptions.cs index a66cea8..b435e2f 100644 --- a/src/cde/CommandLine/CommandLineOptions.cs +++ b/src/cde/CommandLine/CommandLineOptions.cs @@ -16,7 +16,7 @@ public class ScanOptions [Option("follow-junctions", Default = false, - HelpText = "Descend into directory junctions / symbolic links. Off by default to avoid cycles and duplicate content.")] + HelpText = "[DANGER NOT EXTENSIVELY TESTED] Descend into directory junctions / symbolic links. Off by default to avoid cycles and duplicate content.")] public bool FollowJunctions { get; [UsedImplicitly] set; } } diff --git a/src/cdeLib/Catalog/CatalogRepository.cs b/src/cdeLib/Catalog/CatalogRepository.cs index 60423c7..6d9deff 100644 --- a/src/cdeLib/Catalog/CatalogRepository.cs +++ b/src/cdeLib/Catalog/CatalogRepository.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Threading; using System.Threading.Tasks; using cdeLib.Entities; using cdeLib.Infrastructure; @@ -54,7 +53,6 @@ public RootEntry Read(string file) using (Operation.Time("Deserialize")) { var serializer = new FlatBufferSerializer(new FlatBufferSerializerOptions()); - // Use ReadOnlyMemory overload to avoid defensive copy return serializer.Parse(bytes.AsMemory()); } case SerializerProtocol.MessagePack: @@ -71,7 +69,7 @@ public RootEntry Read(string file) } } - public async Task ReadAsync(string file) + private async Task ReadAsync(string file) { try { @@ -125,7 +123,7 @@ public IList Load(IList cdeList) } _logger.Information("Catalog [{file}] read on ThreadId: {ThreadId}", file, - Thread.CurrentThread.ManagedThreadId); + Environment.CurrentManagedThreadId); }); return results.ToList(); @@ -144,6 +142,7 @@ public async Task> LoadAsync(IList cdeList) _logger.Information("Catalog [{file}] read on ThreadId: {ThreadId}", file, Environment.CurrentManagedThreadId); } + return rootEntry; }).ToList(); @@ -285,16 +284,18 @@ public async Task Save(RootEntry rootEntry) } } + /// + /// Dispose of managed resources + /// private void Dispose(bool disposing) { if (!_disposed) { if (disposing) { - // Dispose of managed resources BufferPool?.Clear(); - // Note: FileStreamManager is a singleton, don't dispose of it here } + _disposed = true; } } @@ -302,6 +303,5 @@ private void Dispose(bool disposing) public void Dispose() { Dispose(true); - GC.SuppressFinalize(this); } } \ No newline at end of file diff --git a/src/cdeLibTest/CommonEntryTest_TraverseTreesCopyHash.cs b/src/cdeLibTest/CommonEntryTest_TraverseTreesCopyHash.cs index da58c7e..30d0e50 100644 --- a/src/cdeLibTest/CommonEntryTest_TraverseTreesCopyHash.cs +++ b/src/cdeLibTest/CommonEntryTest_TraverseTreesCopyHash.cs @@ -165,12 +165,10 @@ public void TraverseTreesCopyHash_CopyHashIfSourceHasFullGasgAndDestHasPartialHa public void TraverseTreesCopyHash_DontCopyHashIfDestHasFullHash() { RecreateTestTree(); - _dde1.SetHash(99); // _dde1.Hash = new byte[] { 99 }; + _dde1.SetHash(99); _dde1.IsPartialHash = false; _reSource.TraverseTreesCopyHash(_reDest); - - // Assert.That(_dde1.Hash, Is.Not.Null); Assert.That(_dde1.Hash[0], Is.EqualTo(99)); } private void RecreateTestTree() diff --git a/src/cdeWin/IListViewHelper.cs b/src/cdeWin/IListViewHelper.cs new file mode 100644 index 0000000..6721b82 --- /dev/null +++ b/src/cdeWin/IListViewHelper.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Windows.Forms; +using cdeWin.Cfg; + +namespace cdeWin; + +public interface IListViewHelper : IDisposable where T : class +{ + /// + /// Used by virtual mode ListView + /// + int RetrieveItemIndex { get; set; } + + ListViewItem RenderItem { get; set; } + + IEnumerable SelectedIndices { get; set; } + + int SelectedIndicesCount { get; set; } + + SortOrder ColumnSortOrder { get; set; } + + int SortColumn { get; set; } + + Comparison ColumnSortCompare { get; set; } + + void InitSort(); + IEnumerable ColumnConfigs(); + void SetColumnConfigs(IEnumerable columns); + void ForceDraw(); + void SelectItem(int index); + void DeselectAllItems(); + void SelectAllItems(); + int SetList(List list); + void ListViewColumnClick(); + void SortList(); + void ActionOnSelectedItems(Action> action); + void ActionOnSelectedItem(Action action); + void ActionOnActivateItem(Action action); + T GetItemAt(int index); + void SearchListContextMenuOpening(object sender, System.ComponentModel.CancelEventArgs e); +} \ No newline at end of file diff --git a/src/cdeWin/KeyEqualityComparer.cs b/src/cdeWin/KeyEqualityComparer.cs index 46729f1..5e179c7 100644 --- a/src/cdeWin/KeyEqualityComparer.cs +++ b/src/cdeWin/KeyEqualityComparer.cs @@ -24,21 +24,4 @@ public int GetHashCode(T obj) { return KeyExtractor(obj).GetHashCode(); } -} - -/// -/// see http://stackoverflow.com/questions/98033/wrap-a-delegate-in-an-iequalitycomparer -/// -public class StrictKeyEqualityComparer - : KeyEqualityComparer where TKey : IEquatable -{ - public StrictKeyEqualityComparer(Func keyExtractor) : base(keyExtractor) - { } - - public override bool Equals(T x, T y) - { - // This will use the overload that accepts a TKey parameter - // instead of an object parameter. - return KeyExtractor(x).Equals(KeyExtractor(y)); - } } \ No newline at end of file diff --git a/src/cdeWin/ListViewExtensions.cs b/src/cdeWin/ListViewExtensions.cs index 4ef0c57..690a070 100644 --- a/src/cdeWin/ListViewExtensions.cs +++ b/src/cdeWin/ListViewExtensions.cs @@ -5,7 +5,7 @@ namespace cdeWin; -// FROM http://stackoverflow.com/a/254139 +// From http://stackoverflow.com/a/254139 // ReSharper disable InconsistentNaming [EditorBrowsable(EditorBrowsableState.Never)] public static class ListViewExtensions diff --git a/src/cdeWin/ListViewHelper.cs b/src/cdeWin/ListViewHelper.cs index da1a884..c328887 100644 --- a/src/cdeWin/ListViewHelper.cs +++ b/src/cdeWin/ListViewHelper.cs @@ -5,73 +5,16 @@ using cdeWin.Cfg; namespace cdeWin; -// think about making Presenter<> look for all members that implement IPresenterHelper -// and hookup events to them as well with matching names ? +// Think about making Presenter<> look for all members that implement IPresenterHelper +// and hook up events to them as well with matching names ? // encapsulate ListView in VirtualMode handling -public interface IListViewHelper : IDisposable where T : class -{ - /// - /// Used by virtual mode ListView - /// - int RetrieveItemIndex { get; set; } - - ListViewItem RenderItem { get; set; } - int AfterActivateIndex { get; set; } - int ColumnClickIndex { get; set; } - IEnumerable SelectedIndices { get; set; } - int SelectedIndicesCount { get; set; } - SortOrder ColumnSortOrder { get; set; } - int SortColumn { get; set; } - Comparison ColumnSortCompare { get; set; } - - /// - /// Adds CacheVirtualItems, RetrieveVirtualItem handler which sets RetrieveItemIndex before EventAction. - /// - EventAction RetrieveVirtualItem { get; set; } - - /// - /// Adds ColumnClick handler which sets ColumnClickIndex before EventAction.. - /// - EventAction ColumnClick { get; set; } - - /// - /// Adds ItemActivate handler which sets AfterActivateIndex before EventAction.. - /// - EventAction ItemActivate { get; set; } - - ContextMenuStrip ContextMenu { get; set; } - - /// - /// Adds SelectedIndexChanged, VirtualItemsSelectionRangeChanged handlers. - /// - EventAction ItemSelectionChanged { get; set; } - - bool MultiSelect { get; set; } - - void InitSort(); - IEnumerable ColumnConfigs(); - void SetColumnConfigs(IEnumerable columns); - void ForceDraw(); - void SelectItem(int index); - void DeselectAllItems(); - void SelectAllItems(); - int SetList(List list); - void ListViewColumnClick(); - void SortList(); - void ActionOnSelectedItems(Action> action); - void ActionOnSelectedItem(Action action); - void ActionOnActivateItem(Action action); - T GetItemAt(int index); - void SearchListContextMenuOpening(object sender, System.ComponentModel.CancelEventArgs e); -} - /// /// Consolidated code for ListView operation in VirtualMode. /// Only ListView events required are enabled. -/// Several property setters add Event handlers as required so don't call them more than once. +/// Several property setters add Event handlers as required, so don't call them more than once. /// -public class ListViewHelper : IListViewHelper where T : class +public sealed class ListViewHelper : IListViewHelper where T : class { private bool _isDisposed; private int _listSize; @@ -95,11 +38,17 @@ public class ListViewHelper : IListViewHelper where T : class public ListViewItem RenderItem { get; set; } public int AfterActivateIndex { get; set; } + public int ColumnClickIndex { get; set; } + public IEnumerable SelectedIndices { get; set; } + public int SelectedIndicesCount { get; set; } + public SortOrder ColumnSortOrder { get; set; } + public int SortColumn { get; set; } + public Comparison ColumnSortCompare { get; set; } public ListViewHelper(DoubleBufferListView listView) @@ -129,7 +78,7 @@ public EventAction RetrieveVirtualItem { // not adding retrieve virtual item events here as _list may not be set // was getting some odd errors earlier, this may address the null - // ListViewItem we got outside of visual studio in release builds. + // ListViewItem we got outside Visual Studio in release builds. _retrieveVirtualItem = value; if (_retrieveVirtualItem == null) return; // TODO AUDIT - this should probably add if not null, and remove if null? @@ -141,7 +90,7 @@ public EventAction RetrieveVirtualItem private EventAction _retrieveVirtualItem; /// - /// Adds ColumnClick handler which sets ColumnClickIndex before EventAction.. + /// Adds ColumnClick handler which sets ColumnClickIndex before EventAction. /// public EventAction ColumnClick { @@ -160,7 +109,7 @@ public EventAction ColumnClick private EventAction _columnClick; /// - /// Adds ItemActivate handler which sets AfterActivateIndex before EventAction.. + /// Adds ItemActivate handler which sets AfterActivateIndex before EventAction. /// public EventAction ItemActivate { @@ -253,6 +202,7 @@ private void MyRetrieveVirtualItem(object sender, RetrieveVirtualItemEventArgs e var evictIndex = _cacheOrder.Dequeue(); _itemCache.Remove(evictIndex); } + _itemCache[itemIndex] = newItem; _cacheOrder.Enqueue(itemIndex); @@ -286,7 +236,7 @@ private void MyVirtualItemsSelectionRangeChanged(object sender, private void ListViewItemSelectionChanged() { - SelectedIndicesCount = _listView.SelectedIndices.Count; // todo can i lose this ? + SelectedIndicesCount = _listView.SelectedIndices.Count; // todo can I lose this ? SelectedIndices = _listView.SelectedIndices.OfType(); if (SelectedIndicesCount > 0) { @@ -338,7 +288,7 @@ public void DeselectAllItems() } // Cannot use SelectItem() in a loop as it does Focus on each item. - public void SelectItems(IEnumerable itemIndices) + private void SelectItems(IEnumerable itemIndices) { var minIndex = int.MaxValue; foreach (var i in itemIndices) @@ -358,7 +308,7 @@ public void SelectItems(IEnumerable itemIndices) _listView.Select(); } - public void SelectItems(IEnumerable itemList) + private void SelectItems(IEnumerable itemList) { var newIndices = itemList.Select(item => _list.FindIndex(sortedItem => item == sortedItem)); SelectItems(newIndices); @@ -423,7 +373,7 @@ public void SortList() _list.Sort(ColumnSortCompare); SelectItems(selectedItems); - // Clear cache AFTER all item manipulations but BEFORE ForceDraw + // Clear cache AFTER all item manipulations, but BEFORE ForceDraw // This ensures no stale items from DeselectAllItems/SelectItems remain _itemCache.Clear(); _cacheOrder.Clear(); @@ -436,7 +386,7 @@ private void SetColumnSortArrow() _listView.SetSortIcon(SortColumn, ColumnSortOrder == SortOrder.Ascending ? SortOrder.Descending - : SortOrder.Ascending); // column state is inverted some how ? + : SortOrder.Ascending); // column state is inverted somehow? } public void ActionOnSelectedItems(Action> action) @@ -506,16 +456,16 @@ public T GetItemAt(int index) { return null; } + return _list[index]; } public void Dispose() { Dispose(true); - GC.SuppressFinalize(this); } - protected virtual void Dispose(bool disposing) + private void Dispose(bool disposing) { if (_isDisposed) return; @@ -525,7 +475,7 @@ protected virtual void Dispose(bool disposing) { _listView.CacheVirtualItems -= MyCacheVirtualItems; // - // If we don't do this we don't get the weird crash on exist of cdeWin + // If we don't do this we don't get the weird crash on exit of cdeWin // NullReferenceException // System.Windows.Forms.ListView.ListViewNativeItemCollection.get_Item(Int32 displayIndex) // at @@ -533,7 +483,7 @@ protected virtual void Dispose(bool disposing) // // On dispose it must be trying to be called after we remove it and it kaboom // This is local method to this Class it will never be anything else but this method - // Removing it at dispose when we exit seems like it's not actually important anyway + // Removing it at disposal when we exit seems like it's not actually important anyway // by not removing this we don't get the odd crash. // // _listView.RetrieveVirtualItem -= MyRetrieveVirtualItem; @@ -571,7 +521,7 @@ public void SearchListContextMenuOpening(object sender, System.ComponentModel.Ca var listViewItem = GetListViewItemAtMouse(); if (listViewItem == null) { - // cancel context menu if no list view item at right click. + // cancel context menu if no list view item at right-click. e.Cancel = true; } } diff --git a/src/cdeWin/StrictKeyEqualityComparer.cs b/src/cdeWin/StrictKeyEqualityComparer.cs new file mode 100644 index 0000000..4bb1110 --- /dev/null +++ b/src/cdeWin/StrictKeyEqualityComparer.cs @@ -0,0 +1,20 @@ +using System; + +namespace cdeWin; + +/// +/// see http://stackoverflow.com/questions/98033/wrap-a-delegate-in-an-iequalitycomparer +/// +public class StrictKeyEqualityComparer + : KeyEqualityComparer where TKey : IEquatable +{ + public StrictKeyEqualityComparer(Func keyExtractor) : base(keyExtractor) + { } + + public override bool Equals(T x, T y) + { + // This will use the overload that accepts a TKey parameter + // instead of an object parameter. + return KeyExtractor(x).Equals(KeyExtractor(y)); + } +} \ No newline at end of file diff --git a/src/cdeWin/UpDownHelper.cs b/src/cdeWin/UpDownHelper.cs index 92a53b5..4a00c0e 100644 --- a/src/cdeWin/UpDownHelper.cs +++ b/src/cdeWin/UpDownHelper.cs @@ -7,7 +7,7 @@ public class UpDownHelper private readonly NumericUpDown _upDown; // have a field - text box / up-down - // have a drop down which modifies the the field... like a multiplier or offset. + // have a drop-down which modifies the field... like a multiplier or offset. public UpDownHelper(NumericUpDown upDown, int decimalPlaces = 2) { _upDown = upDown; From e4de8c57b47c82019f24d70b573360a53b3ccb35 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Wed, 10 Jun 2026 12:31:10 +1000 Subject: [PATCH 41/43] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/cdeLib/Entities/Columnar/ColumnarFormat.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cdeLib/Entities/Columnar/ColumnarFormat.cs b/src/cdeLib/Entities/Columnar/ColumnarFormat.cs index 204e30e..5146c8f 100644 --- a/src/cdeLib/Entities/Columnar/ColumnarFormat.cs +++ b/src/cdeLib/Entities/Columnar/ColumnarFormat.cs @@ -22,10 +22,10 @@ namespace cdeLib.Entities.Columnar; /// [16] (int64 offset, int64 length) x -- absolute, 8-aligned /// column bodies (each padded to an 8-byte boundary), in order. /// -/// Columns are dense and homogeneous, so a name-only search sequentially scans just the NameBlob + -/// NameOffsets columns and never pages in Size / Modified / Hash. NameOffsets are 64-bit so the name -/// blob is not capped at 2 GB. Entry count is 32-bit, matching 's int indexing. -/// + /// Columns are dense and homogeneous, so a name-only search sequentially scans just the NameBlob + + /// NameOffsets columns and never pages in Size / Modified / Hash. NameOffsets are 64-bit, so the on-disk + /// name blob can exceed 2 GB (though Write currently buffers it in-memory, which is limited to ~2 GB). + /// Entry count is 32-bit, matching 's int indexing. public static class ColumnarFormat { public static ReadOnlySpan Magic => "CDEX"u8; From cd2d7f06e834717ef15f95272c2de2360890c272 Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Wed, 10 Jun 2026 12:31:43 +1000 Subject: [PATCH 42/43] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- src/cdeLib/Entities/Columnar/ColumnarFormat.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/cdeLib/Entities/Columnar/ColumnarFormat.cs b/src/cdeLib/Entities/Columnar/ColumnarFormat.cs index 5146c8f..64f9a5f 100644 --- a/src/cdeLib/Entities/Columnar/ColumnarFormat.cs +++ b/src/cdeLib/Entities/Columnar/ColumnarFormat.cs @@ -62,7 +62,9 @@ public static void Write(EntryStore store, string outPath) // Build the variable-length name columns up front (UTF-8 full names + 64-bit prefix offsets). var nameOffsets = new long[count + 1]; - using var nameBlob = new MemoryStream(count * 12); + + var initialCapacity = (int)Math.Min((long)count * 12, int.MaxValue); + using var nameBlob = new MemoryStream(initialCapacity); for (var i = 0; i < count; i++) { nameOffsets[i] = nameBlob.Length; @@ -70,8 +72,9 @@ public static void Write(EntryStore store, string outPath) WriteUtf8(nameBlob, store.Ext[i]); // ext appended directly -> full-name bytes, no separator } nameOffsets[count] = nameBlob.Length; + if (nameBlob.Length > int.MaxValue) + throw new InvalidOperationException("Name blob exceeded 2 GB; ColumnarFormat.Write currently buffers names in-memory."); var nameBlobBytes = nameBlob.GetBuffer().AsSpan(0, (int)nameBlob.Length); - var meta = BuildMeta(store); var len = new long[ColumnCount]; From aba79b227be010b700ffd377ca8ee1451658f7ca Mon Sep 17 00:00:00 2001 From: Jason Finch Date: Wed, 10 Jun 2026 12:46:52 +1000 Subject: [PATCH 43/43] fix: review fixes for search allocations, hash progress threading, and compile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - TimePartialParameter: add private set to Hour/Minute/Second so the constructor can assign while keeping the API read-only (fixes compile against the get-only field-keyword properties). - EntryStore: add TryWriteFullPath(Span) — an allocation-free mirror of AppendFullPath that writes the path into a caller buffer. - EntryStoreSearch: substring path searches now build the path into a rented char buffer and run Span.Contains instead of allocating a full-path string per entry; only regex mode materialises a string. Applied to both Find overloads. - HashProgressConsole: use ConcurrentQueue for the cross-thread message queue (handlers enqueue while the UI loop dequeues), and add a 50ms sleep to the progress loop so it stops busy-spinning a core. --- src/cde/HashProgress/HashProgressConsole.cs | 6 +- src/cdeLib/Entities/Soa/EntryStore.cs | 50 ++++++ src/cdeLib/Entities/Soa/EntryStoreSearch.cs | 176 ++++++++++++-------- src/cdeLib/TimePartialParameter.cs | 3 + 4 files changed, 160 insertions(+), 75 deletions(-) diff --git a/src/cde/HashProgress/HashProgressConsole.cs b/src/cde/HashProgress/HashProgressConsole.cs index 3e1a900..a081ff7 100644 --- a/src/cde/HashProgress/HashProgressConsole.cs +++ b/src/cde/HashProgress/HashProgressConsole.cs @@ -1,5 +1,5 @@ using System; -using System.Collections.Generic; +using System.Collections.Concurrent; using System.Diagnostics; using System.Globalization; using System.Threading; @@ -19,7 +19,7 @@ public class HashProgressConsole public static bool HashIsComplete { get; set; } - private static readonly Queue Messages = new(); + private static readonly ConcurrentQueue Messages = new(); private static void WriteLogMessage(string message) { @@ -50,6 +50,8 @@ public void Start(Task mainLoopTask, CancellationToken cancellationToken) while (!mainLoopTask.IsCompleted && !cancellationToken.IsCancellationRequested) { ShowProgress(sw, ctx); + // Throttle the refresh loop so it doesn't busy-spin a core while hashing. + Thread.Sleep(50); } // Flush any remaining messages after the task completes diff --git a/src/cdeLib/Entities/Soa/EntryStore.cs b/src/cdeLib/Entities/Soa/EntryStore.cs index bf4d28a..48a95db 100644 --- a/src/cdeLib/Entities/Soa/EntryStore.cs +++ b/src/cdeLib/Entities/Soa/EntryStore.cs @@ -139,6 +139,56 @@ public void AppendFullPath(StringBuilder sb, int i) } } + /// + /// Write the full path of entry into (root-first), + /// mirroring but without allocating a string. Returns the number of + /// chars written, or -1 if is too small (the caller should grow and retry). + /// Lets hot-path callers run comparisons with no per-entry path allocation. + /// + public int TryWriteFullPath(Span dest, int i) + { + var depth = 0; + for (var cur = i; cur != None; cur = Parent[cur]) depth++; + if (depth == 0) return 0; + + Span chain = depth <= 64 ? stackalloc int[depth] : new int[depth]; + var n = 0; + for (var cur = i; cur != None; cur = Parent[cur]) chain[n++] = cur; + + var pos = 0; + for (var k = depth - 1; k >= 0; k--) + { + var idx = chain[k]; + if (pos > 0) + { + var last = dest[pos - 1]; + if (last != '\\' && last != '/') + { + if (pos >= dest.Length) return -1; + dest[pos++] = System.IO.Path.DirectorySeparatorChar; + } + } + + var name = Name[idx]; + if (!string.IsNullOrEmpty(name)) + { + if (pos + name.Length > dest.Length) return -1; + name.AsSpan().CopyTo(dest[pos..]); + pos += name.Length; + } + + var ext = Ext[idx]; + if (!string.IsNullOrEmpty(ext)) + { + if (pos + ext.Length > dest.Length) return -1; + ext.AsSpan().CopyTo(dest[pos..]); + pos += ext.Length; + } + } + + return pos; + } + public string FullPath(int i) { var sb = new StringBuilder(128); diff --git a/src/cdeLib/Entities/Soa/EntryStoreSearch.cs b/src/cdeLib/Entities/Soa/EntryStoreSearch.cs index 33e12f1..c066cf1 100644 --- a/src/cdeLib/Entities/Soa/EntryStoreSearch.cs +++ b/src/cdeLib/Entities/Soa/EntryStoreSearch.cs @@ -1,5 +1,5 @@ using System; -using System.Text; +using System.Buffers; using System.Text.RegularExpressions; namespace cdeLib.Entities.Soa; @@ -29,57 +29,82 @@ public static void Find(EntryStore store, EntryStoreFindOptions o, Action o if (o.RegexMode && !string.IsNullOrEmpty(o.Pattern)) regex = new Regex(o.Pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.Compiled); - var sb = o.IncludePath ? new StringBuilder(260) : null; var hasPattern = !string.IsNullOrEmpty(o.Pattern); - for (var i = 1; i < store.Count; i++) + // Substring path matching runs Span.Contains over a rented buffer; only regex mode + // (rarer) materialises a string. Avoids a full-path string allocation per scanned entry. + var pathBuffer = o.IncludePath ? ArrayPool.Shared.Rent(512) : null; + try { - if ((i & 4095) == 0) + for (var i = 1; i < store.Count; i++) { - if (isCancelled != null && isCancelled()) return; - onScan?.Invoke(i); - } - - var isDir = store.IsDirectory(i); - if (isDir ? !o.IncludeFolders : !o.IncludeFiles) continue; - - var size = store.Size[i]; - if (o.FromSizeEnable && size < o.FromSize) continue; - if (o.ToSizeEnable && size > o.ToSize) continue; - - if (o.FromDateEnable || o.ToDateEnable || o.FromHourEnable || o.ToHourEnable || o.NotOlderThanEnable) - { - var modified = store.Modified(i); - if (o.FromDateEnable && modified < o.FromDate) continue; - if (o.ToDateEnable && modified > o.ToDate) continue; - if (o.NotOlderThanEnable && modified < o.NotOlderThan) continue; - var tod = modified.TimeOfDay; - if (o.FromHourEnable && tod < o.FromHour) continue; - if (o.ToHourEnable && tod > o.ToHour) continue; - } - - if (!hasPattern) { onMatch(i); continue; } - - bool match; - if (o.IncludePath) - { - sb.Clear(); - store.AppendFullPath(sb, i); - var path = sb.ToString(); - match = o.RegexMode - ? regex.IsMatch(path) - : path.Contains(o.Pattern, StringComparison.OrdinalIgnoreCase); - } - else - { - var name = store.FullName(i); - match = o.RegexMode - ? regex.IsMatch(name) - : name.Contains(o.Pattern, StringComparison.OrdinalIgnoreCase); + if ((i & 4095) == 0) + { + if (isCancelled != null && isCancelled()) return; + onScan?.Invoke(i); + } + + var isDir = store.IsDirectory(i); + if (isDir ? !o.IncludeFolders : !o.IncludeFiles) continue; + + var size = store.Size[i]; + if (o.FromSizeEnable && size < o.FromSize) continue; + if (o.ToSizeEnable && size > o.ToSize) continue; + + if (o.FromDateEnable || o.ToDateEnable || o.FromHourEnable || o.ToHourEnable || o.NotOlderThanEnable) + { + var modified = store.Modified(i); + if (o.FromDateEnable && modified < o.FromDate) continue; + if (o.ToDateEnable && modified > o.ToDate) continue; + if (o.NotOlderThanEnable && modified < o.NotOlderThan) continue; + var tod = modified.TimeOfDay; + if (o.FromHourEnable && tod < o.FromHour) continue; + if (o.ToHourEnable && tod > o.ToHour) continue; + } + + if (!hasPattern) { onMatch(i); continue; } + + bool match; + if (o.IncludePath) + { + var path = WritePath(store, i, ref pathBuffer); + match = o.RegexMode + ? regex.IsMatch(path.ToString()) + : path.Contains(o.Pattern, StringComparison.OrdinalIgnoreCase); + } + else + { + var name = store.FullName(i); + match = o.RegexMode + ? regex.IsMatch(name) + : name.Contains(o.Pattern, StringComparison.OrdinalIgnoreCase); + } + + if (match) onMatch(i); } + } + finally + { + if (pathBuffer != null) ArrayPool.Shared.Return(pathBuffer); + } + } - if (match) onMatch(i); + /// + /// Write entry 's full path into (rented), growing + /// and re-renting if it doesn't fit, and return the written span. The grown buffer is passed back + /// via so the caller reuses it for subsequent entries. + /// + private static ReadOnlySpan WritePath(EntryStore store, int i, ref char[] buffer) + { + int len; + while ((len = store.TryWriteFullPath(buffer, i)) < 0) + { + var bigger = ArrayPool.Shared.Rent(buffer.Length * 2); + ArrayPool.Shared.Return(buffer); + buffer = bigger; } + + return buffer.AsSpan(0, len); } /// Invoke with the index of every entry matching the query. @@ -103,39 +128,44 @@ public static void Find( regex = new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.Compiled); } - var sb = includePath ? new StringBuilder(260) : null; var hasPattern = !string.IsNullOrEmpty(pattern); - for (var i = 1; i < store.Count; i++) // index 0 is the root, never a result + var pathBuffer = includePath ? ArrayPool.Shared.Rent(512) : null; + try { - var isDir = store.IsDirectory(i); - if (isDir ? !includeFolders : !includeFiles) continue; - - if (!hasPattern) - { - onMatch(i); - continue; - } - - bool match; - if (includePath) - { - sb.Clear(); - store.AppendFullPath(sb, i); - var path = sb.ToString(); - match = regexMode - ? regex.IsMatch(path) - : path.Contains(pattern, StringComparison.OrdinalIgnoreCase); - } - else + for (var i = 1; i < store.Count; i++) // index 0 is the root, never a result { - var name = store.FullName(i); - match = regexMode - ? regex.IsMatch(name) - : name.Contains(pattern, StringComparison.OrdinalIgnoreCase); + var isDir = store.IsDirectory(i); + if (isDir ? !includeFolders : !includeFiles) continue; + + if (!hasPattern) + { + onMatch(i); + continue; + } + + bool match; + if (includePath) + { + var path = WritePath(store, i, ref pathBuffer); + match = regexMode + ? regex.IsMatch(path.ToString()) + : path.Contains(pattern, StringComparison.OrdinalIgnoreCase); + } + else + { + var name = store.FullName(i); + match = regexMode + ? regex.IsMatch(name) + : name.Contains(pattern, StringComparison.OrdinalIgnoreCase); + } + + if (match) onMatch(i); } - - if (match) onMatch(i); + } + finally + { + if (pathBuffer != null) ArrayPool.Shared.Return(pathBuffer); } } } diff --git a/src/cdeLib/TimePartialParameter.cs b/src/cdeLib/TimePartialParameter.cs index febad30..7f7530c 100644 --- a/src/cdeLib/TimePartialParameter.cs +++ b/src/cdeLib/TimePartialParameter.cs @@ -17,6 +17,7 @@ public int Hour ThrowExceptionIfSet(); return field; } + private set; } public int Minute @@ -26,6 +27,7 @@ public int Minute ThrowExceptionIfSet(); return field; } + private set; } public int Second @@ -35,6 +37,7 @@ public int Second ThrowExceptionIfSet(); return field; } + private set; } private readonly Exception _e;