diff --git a/.nuke/build.schema.json b/.fallout/build.schema.json similarity index 97% rename from .nuke/build.schema.json rename to .fallout/build.schema.json index a1458c4..ca39e4c 100644 --- a/.nuke/build.schema.json +++ b/.fallout/build.schema.json @@ -44,7 +44,7 @@ "Quiet" ] }, - "NukeBuild": { + "FalloutBuild": { "properties": { "Continue": { "type": "boolean", @@ -120,7 +120,7 @@ } }, { - "$ref": "#/definitions/NukeBuild" + "$ref": "#/definitions/FalloutBuild" } ] } diff --git a/.nuke/parameters.json b/.fallout/parameters.json similarity index 100% rename from .nuke/parameters.json rename to .fallout/parameters.json diff --git a/.gitignore b/.gitignore index 5f48a8e..4319a55 100644 --- a/.gitignore +++ b/.gitignore @@ -71,3 +71,6 @@ src/.vs/config/applicationhost.config **/Properties/launchSettings.json /.claude/settings.local.json /src/.claude/settings.local.json + +# BenchmarkDotNet writes results here when running cdeBenchmarks +**/BenchmarkDotNet.Artifacts/ diff --git a/Readme.md b/Readme.md index e3386a7..53f6445 100644 --- a/Readme.md +++ b/Readme.md @@ -30,7 +30,7 @@ This application reads and writes a configuration file `cdeWinView.cfg`. - size of all the columns in list views - values of fields in the search parameters -The executable `cdeWin.exe` can be copied around by it self to be used anywhere that .Net7 is available with the behavior of the cdeWinView.cfg file as described just above. +The executable `cdeWin.exe` can be copied around by it self to be used anywhere that .NET 10 is available with the behavior of the cdeWinView.cfg file as described just above. #### cdeWeb (unreleased) @@ -109,6 +109,8 @@ cde path -find afilename [`-maxDateTime`](#parameter-options) [`-minTime`](#parameter-options) [`-maxTime`](#parameter-options) +[`--desc`](#parameter-options) +[`--follow-junctions`](#parameter-options) This is the mode of operation that creates and updates catalog files. @@ -116,6 +118,12 @@ When it creates new catalog files it will detect an old catalog file for the giv Only Last Modified Time of file system entries is captured into .cde files. +By default cde records directory junctions and symbolic links (reparse points) in the catalog but does **not** descend into them, which avoids scan cycles (for example a junction pointing back at an ancestor directory) and duplicated content. Pass `--follow-junctions` to descend into them. + +``` + cde scan C:\ --follow-junctions +``` + ### cde find String #### Valid Options for this mode @@ -186,6 +194,30 @@ Consider using -minHourAge to limit Hash and Dupes work if your are cleanign up Output the full tree of file entries in the catologs in text format. +### cde migrate \[Path\] + +#### Valid Options for this mode + +`No filter options supported.` + +This mode performs a one-way conversion of the original MessagePack `.cde` catalogs into the newer zero-copy columnar `.cdex` format. The `.cdex` format is laid out so it can be memory-mapped and searched without first deserialising the whole catalog into objects, which lowers memory use and speeds up load on large catalogs. + +- With a path argument, only that single `.cde` file is converted: + + ```batch + cde migrate C-V3Win7-C__users.cde + ``` + +- With no argument, every catalog discovered in the current directory and one directory below is converted (the same discovery rule used when loading catalogs): + + ```batch + cde migrate + ``` + +For each catalog a `.cdex` file is written beside the source `.cde`, keeping the same base name. The original `.cde` file is left in place and is not deleted, so the conversion is non-destructive. Re-running migrate simply overwrites the `.cdex` output. + +For each file converted cde prints the source and destination file names, their byte sizes and the entry count, then a summary of how many catalogs were migrated. Files that cannot be loaded are skipped and reported. + ### Parameter Options | | Parameter                  | Description | @@ -204,6 +236,8 @@ Output the full tree of file entries in the catologs in text format. | | `-maxResults {Int}` | Maximum number of results returned by cde. | | | `-exclude {Regex}` | A filter to exclude only entries that match these regexes for processing. | | | `-include {Regex}` | A filter to include only entries that match these Regexes for processing. | +| | `--desc {Text}` | (scan) Description text to store in the catalog file. | +| | `--follow-junctions` | (scan) Descend into directory junctions and symbolic links (reparse points). Off by default; reparse points are recorded but not followed, avoiding scan cycles and duplicated content. | ##### Date Time Format for parameters diff --git a/build.ps1 b/build.ps1 index bbaa118..6256a32 100644 --- a/build.ps1 +++ b/build.ps1 @@ -14,7 +14,7 @@ $PSScriptRoot = Split-Path $MyInvocation.MyCommand.Path -Parent ########################################################################### $BuildProjectFile = "$PSScriptRoot\build\_build.csproj" -$TempDirectory = "$PSScriptRoot\\.nuke\temp" +$TempDirectory = "$PSScriptRoot\\.fallout\temp" $DotNetGlobalFile = "$PSScriptRoot\\global.json" $DotNetInstallUrl = "https://dot.net/v1/dotnet-install.ps1" diff --git a/build.sh b/build.sh index e8961f9..2e66cb3 100755 --- a/build.sh +++ b/build.sh @@ -10,7 +10,7 @@ SCRIPT_DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd) ########################################################################### BUILD_PROJECT_FILE="$SCRIPT_DIR/build/_build.csproj" -TEMP_DIRECTORY="$SCRIPT_DIR//.nuke/temp" +TEMP_DIRECTORY="$SCRIPT_DIR//.fallout/temp" DOTNET_GLOBAL_FILE="$SCRIPT_DIR//global.json" DOTNET_INSTALL_URL="https://dot.net/v1/dotnet-install.sh" diff --git a/build/Build.cs b/build/Build.cs index 431fc50..f2709ec 100644 --- a/build/Build.cs +++ b/build/Build.cs @@ -2,18 +2,18 @@ using JetBrains.Annotations; using Microsoft.Build.Evaluation; using Microsoft.Build.Locator; -using Nuke.Common; -using Nuke.Common.CI; -using Nuke.Common.Git; -using Nuke.Common.IO; -using Nuke.Common.ProjectModel; -using Nuke.Common.Tools.DotNet; -using Nuke.Common.Utilities.Collections; -using static Nuke.Common.Tools.DotNet.DotNetTasks; +using Fallout.Common; +using Fallout.Common.CI; +using Fallout.Common.Git; +using Fallout.Common.IO; +using Fallout.Common.ProjectModel; +using Fallout.Common.Tools.DotNet; +using Fallout.Common.Utilities.Collections; +using static Fallout.Common.Tools.DotNet.DotNetTasks; // ReSharper disable AllUnderscoreLocalParameterName [ShutdownDotNetAfterServerBuild] -class Build : NukeBuild +class Build : FalloutBuild { public Build() { diff --git a/build/Configuration.cs b/build/Configuration.cs index 78049f7..fe274ae 100644 --- a/build/Configuration.cs +++ b/build/Configuration.cs @@ -1,5 +1,5 @@ using System.ComponentModel; -using Nuke.Common.Tooling; +using Fallout.Common.Tooling; [TypeConverter(typeof(TypeConverter))] public class Configuration : Enumeration diff --git a/build/_build.csproj b/build/_build.csproj index 318bb57..c57e7f4 100644 --- a/build/_build.csproj +++ b/build/_build.csproj @@ -5,19 +5,20 @@ net10.0 CS0649;CS0169 - .. - .. - 1 + .. + .. + 1 true default + - + compile; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/claude.md b/claude.md index a9ef7c2..d20f451 100644 --- a/claude.md +++ b/claude.md @@ -16,14 +16,14 @@ CDE is a high-performance file system cataloging utility written in C# that crea - Target: .NET 10 - Cross-platform: win-x64, linux-x64, osx-x64 - Entry point for scan, find, hash, dupes, dump commands - - Dependencies: Autofac, MediatR, CommandLineParser, Spectre.Console + - Dependencies: Autofac, SlimMessageBus, CommandLineParser, Spectre.Console - **cdeLib** - Core library containing business logic - Target: .NET 10 - Contains all catalog operations, hashing, duplicate detection - - Uses CQRS pattern with MediatR - - Serialization: MessagePack, FlatSharp - - Key dependencies: Autofac, MediatR, Serilog + - Uses CQRS pattern with SlimMessageBus + - Serialization: columnar `.cdex` (zero-copy, memory-mapped), plus MessagePack/FlatSharp/protobuf-net for the legacy `.cde` tree format + - Key dependencies: Autofac, SlimMessageBus, Serilog - **cdeWin** - Windows Forms GUI application - Target: .NET 10 (Windows) @@ -52,7 +52,7 @@ CDE is a high-performance file system cataloging utility written in C# that crea ### Architecture Patterns -- **CQRS (Command Query Responsibility Segregation)**: Commands and queries handled via MediatR +- **CQRS (Command Query Responsibility Segregation)**: Commands and queries handled via SlimMessageBus (`IRequestHandler`, `IMessageBus`) - Commands: `CreateCacheCommand`, `HashCatalogCommand`, `FindDuplicatesCommand`, `UpdateCommand` - Handlers: Separate handlers for each command - Events: `ScanProgressEvent` for progress tracking @@ -69,9 +69,12 @@ CDE is a high-performance file system cataloging utility written in C# that crea ### Serialization -Multiple serialization formats supported: -- **MessagePack** - Primary catalog file format (.cde files) -- **FlatSharp** - FlatBuffers support (alternative) +Two on-disk catalog formats: + +- **Columnar `.cdex`** (`Entities/Columnar/ColumnarFormat.cs`) - **Current/primary format.** A struct-of-arrays layout designed for *zero-copy reads over a memory-mapped file*: "loading" a catalog is mmap-ing it, so no managed object graph is materialised and the working set is only the file pages a query touches (reclaimable OS page cache, not GC heap). Custom binary layout with a `"CDEX"` magic header and dense, homogeneous columns (names, sizes, timestamps, hashes, tree links) — a name-only search scans just the name columns and never pages in the rest. Written directly by `scan`; read via `ColumnarCatalogReader`. +- **Legacy `.cde` tree format** - The original materialised directory-tree format, serialized via a pluggable `SerializerProtocol` in `Catalog/CatalogRepository.cs`: + - **MessagePack** - default protocol for `.cde` (`MessagePackConfig.Options`, custom `Hash16Formatter`/resolver) + - **FlatSharp** (FlatBuffers) and **protobuf-net** - alternative protocols selectable via `SerializerProtocol` ### Hashing @@ -126,15 +129,16 @@ Entry (base class) ### Build System -- **Nuke Build** - Build automation - - `build.cmd` / `build.ps1` / `build.sh` - Build scripts - - `build/Build.cs` - Build definition +- **Fallout Build** - Build automation (replaced Nuke) + - `build.cmd` / `build.ps1` / `build.sh` - Build scripts (bootstrap `build/_build.csproj`) + - `build/Build.cs` - Build definition (uses `Fallout.Common`) + - `.fallout/` - Fallout config, parameters, and temp/log output - Command: `build.cmd publish` - Creates artifacts in `./artifacts` ### Key Command Handlers Located in `cdeLib/`: -- `Catalog/CreateCacheCommandHandler.cs` - Scans file systems, creates .cde files +- `Catalog/CreateCacheCommandHandler.cs` - Scans file systems, writes columnar `.cdex` catalogs (reusing hashes from an existing `.cdex` when present) - `Hashing/HashCatalogCommandHandler.cs` - Adds MD5 hashes to catalogs - `Duplicates/FindDuplicateCommandHandler.cs` - Identifies duplicate files - `FindService.cs` - File search functionality @@ -148,16 +152,18 @@ Located in `cdeLib/Infrastructure/`: - `WorkStealingTreeTraversal.cs` - Parallel directory traversal - `Config/` - Configuration classes -## Catalog File Format (.cde) +## Catalog File Formats (.cdex / .cde) -- **Extension**: `.cde` +- **Extensions**: `.cdex` (current columnar format) and `.cde` (legacy tree format) - **Naming**: Derived from drive letter, volume name, and path - - Example: `C-V3Win7-C__users.cde` for `C:\users\` - - Example: `UNC-toothless_c__users_.cde` for `\\unc\toothless\c$\users` -- **Loading**: All .cde files in current directory or one level down are loaded -- **Content**: Directory tree with optional MD5 hashes + - Example: `C-V3Win7-C__users.cdex` for `C:\users\` + - Example: `UNC-toothless_c__users_.cdex` for `\\unc\toothless\c$\users` +- **Loading**: All catalog files in the current directory or one level down are loaded (`GetColumnarFileList` for `.cdex`, `GetCacheFileList` for `.cde`) +- **Content**: Directory tree (or columns) with optional MD5 hashes - **Size**: Highly efficient - 500MB for 11 billion entries -- **Format**: MessagePack binary serialization (not compressed) +- **Format**: + - `.cdex` - custom columnar binary, memory-mapped for zero-copy loads (not compressed) + - `.cde` - MessagePack binary serialization by default (not compressed); protobuf/FlatBuffers selectable in code ## Common Operations @@ -228,7 +234,7 @@ This branch focuses on performance improvements and refactoring. Recent commits - Use object pooling for frequently allocated objects - Benchmark changes that affect hot paths -2. **Catalog Compatibility**: Changes to serialization affect .cde file format +2. **Catalog Compatibility**: Changes to serialization affect the `.cdex`/`.cde` file formats - Hash size changes require catalog recreation - Document breaking changes @@ -240,9 +246,18 @@ This branch focuses on performance improvements and refactoring. Recent commits - Unit tests in cdeLibTest - Specification tests in cdeLibSpec/cdeLibSpec2 +### Shell & Tooling + +This is a Windows environment with both PowerShell and Bash available. The two shells have **incompatible** here-string / quoting syntax — never mix them. + +- **PowerShell here-string** is `@'` ... `'@` (closing `'@` must be at column 0). Only valid in the PowerShell tool. +- **Bash here-doc** is `<<'EOF'` ... `EOF`. Only valid in the Bash tool. +- Passing `@'...'@` to the Bash tool does **not** create a here-string — Bash treats the `@` characters as literal text, which (for example) prepends a stray `@` to git commit messages. +- For multi-line text (commit messages, file content) prefer the matching syntax for the tool you're calling, or write the text to a file and pass it with `-F `. + ### Code Patterns -- **MediatR Commands**: Business operations are commands/queries +- **SlimMessageBus Commands**: Business operations are commands/queries (`IRequestHandler.OnHandle`) - **Dependency Injection**: Constructor injection via Autofac - **Logging**: Serilog with structured logging - **Configuration**: Microsoft.Extensions.Configuration with appsettings.json @@ -268,8 +283,10 @@ Standard .NET test runners (tests use NUnit, xUnit) ## Dependencies to Note - **Autofac** - Dependency injection -- **MediatR** - Command/query pattern -- **MessagePack** - Primary serialization +- **SlimMessageBus** - Command/query and pub/sub messaging (in-memory) +- **MessagePack** - Serialization for the legacy `.cde` tree format (current `.cdex` format uses a custom columnar layout) +- **FlatSharp** - FlatBuffers serialization (alternative `.cde` protocol) +- **protobuf-net** - Protobuf serialization (alternative `.cde` protocol) - **Serilog** - Structured logging - **CommandLineParser** - CLI argument parsing - **Spectre.Console** - Rich console output diff --git a/developer.md b/developer.md index 1ef6a5a..26e9a2c 100644 --- a/developer.md +++ b/developer.md @@ -2,7 +2,11 @@ ## Building the app -The build uses Nuke to perform the steps. +The build uses Fallout to perform the steps. (It replaced Nuke.) + +The build definition lives in `build/Build.cs` (built on `Fallout.Common`), and the +`build.cmd` / `build.ps1` / `build.sh` scripts bootstrap `build/_build.csproj`. +Fallout config, parameters and temp/log output are kept under `.fallout/`. To build the app on windows run: @@ -10,5 +14,11 @@ To build the app on windows run: build.cmd publish ``` +On Linux/macOS run: + +```shell +./build.sh publish +``` + Artifacts from the build will be built to `.\artifacts` diff --git a/src/Mono.Terminal/LineEditor.cs b/src/Mono.Terminal/LineEditor.cs index 9dff0e1..82503bd 100644 --- a/src/Mono.Terminal/LineEditor.cs +++ b/src/Mono.Terminal/LineEditor.cs @@ -113,9 +113,9 @@ public Completion(string prefix, string[] result) // This contains a raw stream pointing to stdout, used to bypass the TermInfoDriver private static Stream _unixRawOutput; - delegate void KeyHandler(); + private delegate void KeyHandler(); - struct Handler + private struct Handler { public readonly ConsoleKeyInfo Cki; public readonly KeyHandler KeyHandler; @@ -229,7 +229,7 @@ public LineEditor(string name, int histsize) // BackgroundColor properties, so we have to use the terminfo driver in Mono to // fetch these values - void GetUnixConsoleReset() + private void GetUnixConsoleReset() { // // On Unix, we want to be able to reset the color for the pop-up completion @@ -268,7 +268,7 @@ private void CmdDebug() Render(); } - void Render() + private void Render() { Console.Write(_shownPrompt); Console.Write(_renderedText); @@ -288,7 +288,7 @@ void Render() private void UpdateHomeRow(int screenpos) { - var lines = 1 + (screenpos / Console.WindowWidth); + var lines = 1 + screenpos / Console.WindowWidth; _homeRow = Console.CursorTop - (lines - 1); if (_homeRow < 0) @@ -296,7 +296,7 @@ private void UpdateHomeRow(int screenpos) } - void RenderFrom(int pos) + private void RenderFrom(int pos) { var rpos = TextToRenderPos(pos); int i; @@ -304,7 +304,7 @@ void RenderFrom(int pos) for (i = rpos; i < _renderedText.Length; i++) Console.Write(_renderedText[i]); - if ((_shownPrompt.Length + _renderedText.Length) > _maxRendered) + if (_shownPrompt.Length + _renderedText.Length > _maxRendered) _maxRendered = _shownPrompt.Length + _renderedText.Length; else { @@ -314,7 +314,7 @@ void RenderFrom(int pos) } } - void ComputeRendered() + private void ComputeRendered() { _renderedText.Length = 0; @@ -336,7 +336,7 @@ void ComputeRendered() } } - int TextToRenderPos(int pos) + private int TextToRenderPos(int pos) { var p = 0; @@ -363,7 +363,7 @@ private int TextToScreenPos(int pos) return _shownPrompt.Length + TextToRenderPos(pos); } - string Prompt + private string Prompt { get => _prompt; set => _prompt = value; @@ -371,12 +371,12 @@ string Prompt private int LineCount => (_shownPrompt.Length + _renderedText.Length) / Console.WindowWidth; - void ForceCursor(int newpos) + private void ForceCursor(int newpos) { _cursor = newpos; var actualPos = _shownPrompt.Length + TextToRenderPos(_cursor); - var row = _homeRow + (actualPos / Console.WindowWidth); + var row = _homeRow + actualPos / Console.WindowWidth; var col = actualPos % Console.WindowWidth; if (row >= Console.BufferHeight) @@ -384,7 +384,7 @@ void ForceCursor(int newpos) Console.SetCursorPosition(col, row); } - void UpdateCursor(int newpos) + private void UpdateCursor(int newpos) { if (_cursor == newpos) return; @@ -392,7 +392,7 @@ void UpdateCursor(int newpos) ForceCursor(newpos); } - void InsertChar(char c) + private void InsertChar(char c) { var prevLines = LineCount; _text = _text.Insert(_cursor, c); @@ -411,7 +411,7 @@ void InsertChar(char c) } } - static void SaveExcursion(Action code) + private static void SaveExcursion(Action code) { var savedCol = Console.CursorLeft; var savedRow = Console.CursorTop; @@ -433,7 +433,7 @@ static void SaveExcursion(Action code) } } - class CompletionState + private class CompletionState { public string Prefix; public string[] Completions; @@ -457,19 +457,19 @@ public CompletionState(int col, int row, int width, int height) throw new ArgumentException("Cannot be less than one", "Height"); } - void DrawSelection() + private void DrawSelection() { for (var r = 0; r < Height; r++) { var itemIdx = _topItem + r; - var selected = (itemIdx == _selectedItem); + var selected = itemIdx == _selectedItem; Console.ForegroundColor = selected ? ConsoleColor.Black : ConsoleColor.Gray; Console.BackgroundColor = selected ? ConsoleColor.Cyan : ConsoleColor.Blue; var item = Prefix + Completions[itemIdx]; if (item.Length > Width) - item = item.Substring(0, Width); + item = item[..Width]; Console.CursorLeft = Col; Console.CursorTop = Row + r; @@ -508,7 +508,7 @@ public void SelectPrevious() } } - void Clear() + private void Clear() { for (var r = 0; r < Height; r++) { @@ -525,7 +525,7 @@ public void Remove() } } - void ShowCompletions(string prefix, string[] completions) + private void ShowCompletions(string prefix, string[] completions) { // Ensure we have space, determine window size var windowHeight = Math.Max(1, Math.Min(completions.Length, Console.WindowHeight / 5)); @@ -587,7 +587,7 @@ public void HideCompletions() // Triggers the completion engine, if insertBestMatch is true, then this will // insert the best match found, this behaves like the shell "tab" which will // complete as much as possible given the options. - void Complete() + private void Complete() { if (AutoCompleteEvent == null) return; @@ -639,12 +639,12 @@ void Complete() var displayCompletions = (string[])completions.Clone(); if (last != -1) { - InsertTextAtCursor(displayCompletions[0].Substring(0, last + 1)); + InsertTextAtCursor(displayCompletions[0][..(last + 1)]); // Adjust the completions to skip the common prefix - prefix += displayCompletions[0].Substring(0, last + 1); + prefix += displayCompletions[0][..(last + 1)]; for (var i = 0; i < displayCompletions.Length; i++) - displayCompletions[i] = displayCompletions[i].Substring(last + 1); + displayCompletions[i] = displayCompletions[i][(last + 1)..]; } ShowCompletions(prefix, displayCompletions); @@ -655,7 +655,7 @@ void Complete() // When the user has triggered a completion window, this will try to update // the contents of it. The completion window is assumed to be hidden at this point - void UpdateCompletionWindow() + private void UpdateCompletionWindow() { if (_currentCompletion != null) throw new Exception("This method should only be called if the window has been hidden"); @@ -677,7 +677,7 @@ void UpdateCompletionWindow() } // Commands - void CmdDone() + private void CmdDone() { if (_currentCompletion != null) { @@ -689,7 +689,7 @@ void CmdDone() _done = true; } - void CmdTabOrComplete() + private void CmdTabOrComplete() { var complete = false; @@ -723,17 +723,17 @@ public void CmdHistoryDump() _history.Dump(); } - void CmdHome() + private void CmdHome() { UpdateCursor(0); } - void CmdEnd() + private void CmdEnd() { UpdateCursor(_text.Length); } - void CmdLeft() + private void CmdLeft() { if (_cursor == 0) return; @@ -741,7 +741,7 @@ void CmdLeft() UpdateCursor(_cursor - 1); } - void CmdBackwardWord() + private void CmdBackwardWord() { var p = WordBackward(_cursor); if (p == -1) @@ -749,7 +749,7 @@ void CmdBackwardWord() UpdateCursor(p); } - void CmdForwardWord() + private void CmdForwardWord() { var p = WordForward(_cursor); if (p == -1) @@ -757,7 +757,7 @@ void CmdForwardWord() UpdateCursor(p); } - void CmdRight() + private void CmdRight() { if (_cursor == _text.Length) return; @@ -765,14 +765,14 @@ void CmdRight() UpdateCursor(_cursor + 1); } - void RenderAfter(int p) + private void RenderAfter(int p) { ForceCursor(p); RenderFrom(p); ForceCursor(_cursor); } - void CmdBackspace() + private void CmdBackspace() { if (_cursor == 0) return; @@ -787,7 +787,7 @@ void CmdBackspace() UpdateCompletionWindow(); } - void CmdDeleteChar() + private void CmdDeleteChar() { // If there is no input, this behaves like EOF if (_text.Length == 0) @@ -805,7 +805,7 @@ void CmdDeleteChar() RenderAfter(_cursor); } - int WordForward(int p) + private int WordForward(int p) { if (p >= _text.Length) return -1; @@ -839,7 +839,7 @@ int WordForward(int p) return -1; } - int WordBackward(int p) + private int WordBackward(int p) { if (p == 0) return -1; @@ -879,7 +879,7 @@ int WordBackward(int p) return -1; } - void CmdDeleteWord() + private void CmdDeleteWord() { var pos = WordForward(_cursor); @@ -898,7 +898,7 @@ void CmdDeleteWord() RenderAfter(_cursor); } - void CmdDeleteBackword() + private void CmdDeleteBackword() { var pos = WordBackward(_cursor); if (pos == -1) @@ -917,12 +917,12 @@ void CmdDeleteBackword() } // Adds the current line to the history if needed - void HistoryUpdateLine() + private void HistoryUpdateLine() { _history.Update(_text.ToString()); } - void CmdHistoryPrev() + private void CmdHistoryPrev() { if (!_history.PreviousAvailable()) return; @@ -932,7 +932,7 @@ void CmdHistoryPrev() SetText(_history.Previous()); } - void CmdHistoryNext() + private void CmdHistoryNext() { if (!_history.NextAvailable()) return; @@ -941,7 +941,7 @@ void CmdHistoryNext() SetText(_history.Next()); } - void CmdUp() + private void CmdUp() { if (_currentCompletion == null) CmdHistoryPrev(); @@ -949,7 +949,7 @@ void CmdUp() _currentCompletion.SelectPrevious(); } - void CmdDown() + private void CmdDown() { if (_currentCompletion == null) CmdHistoryNext(); @@ -957,7 +957,7 @@ void CmdDown() _currentCompletion.SelectNext(); } - void CmdKillToEOF() + private void CmdKillToEOF() { _killBuffer = _text.ToString(_cursor, _text.Length - _cursor); _text.Length = _cursor; @@ -965,12 +965,12 @@ void CmdKillToEOF() RenderAfter(_cursor); } - void CmdYank() + private void CmdYank() { InsertTextAtCursor(_killBuffer); } - void InsertTextAtCursor(string str) + private void InsertTextAtCursor(string str) { var prevLines = LineCount; _text.Insert(_cursor, str); @@ -991,12 +991,12 @@ void InsertTextAtCursor(string str) } } - void SetSearchPrompt(string s) + private void SetSearchPrompt(string s) { SetPrompt("(reverse-i-search)`" + s + "': "); } - void ReverseSearch() + private void ReverseSearch() { int p; @@ -1016,7 +1016,7 @@ void ReverseSearch() else { // The cursor is somewhere in the middle of the string - var start = (_cursor == _matchAt) ? _cursor - 1 : _cursor; + var start = _cursor == _matchAt ? _cursor - 1 : _cursor; if (start != -1) { p = _text.ToString().LastIndexOf(_search, start, StringComparison.Ordinal); @@ -1041,7 +1041,7 @@ void ReverseSearch() } } - void CmdReverseSearch() + private void CmdReverseSearch() { if (_searching == 0) { @@ -1070,9 +1070,9 @@ void CmdReverseSearch() } } - void SearchAppend(char c) + private void SearchAppend(char c) { - _search = _search + c; + _search += c; SetSearchPrompt(_search); // @@ -1088,7 +1088,7 @@ void SearchAppend(char c) ReverseSearch(); } - void CmdRefresh() + private void CmdRefresh() { Console.Clear(); _maxRendered = 0; @@ -1096,7 +1096,7 @@ void CmdRefresh() ForceCursor(_cursor); } - void InterruptEdit(object sender, ConsoleCancelEventArgs a) + private void InterruptEdit(object sender, ConsoleCancelEventArgs a) { // Do not abort our program: a.Cancel = true; @@ -1106,7 +1106,7 @@ void InterruptEdit(object sender, ConsoleCancelEventArgs a) } // Implements heuristics to show the completion window based on the mode - bool HeuristicAutoComplete(bool wasCompleting, char insertedChar) + private bool HeuristicAutoComplete(bool wasCompleting, char insertedChar) { if (HeuristicsMode == "csharp") { @@ -1143,7 +1143,7 @@ bool HeuristicAutoComplete(bool wasCompleting, char insertedChar) return false; } - void HandleChar(char c) + private void HandleChar(char c) { if (_searching != 0) SearchAppend(c); @@ -1205,7 +1205,8 @@ private void EditLoop(CancellationToken cancellationToken) _lastHandler = handler.KeyHandler; break; } - else if (t.KeyChar == cki.KeyChar && t.Key == ConsoleKey.Zoom) + + if (t.KeyChar == cki.KeyChar && t.Key == ConsoleKey.Zoom) { handled = true; if (handler.ResetCompletion) @@ -1238,7 +1239,7 @@ private void EditLoop(CancellationToken cancellationToken) } } - void InitText(string initial) + private void InitText(string initial) { _text = new StringBuilder(initial); ComputeRendered(); @@ -1247,13 +1248,13 @@ void InitText(string initial) ForceCursor(_cursor); } - void SetText(string newtext) + private void SetText(string newtext) { Console.SetCursorPosition(0, _homeRow); InitText(newtext); } - void SetPrompt(string newprompt) + private void SetPrompt(string newprompt) { _shownPrompt = newprompt; Console.SetCursorPosition(0, _homeRow); @@ -1329,7 +1330,7 @@ public void SaveHistory() // Emulates the bash-like behavior, where edits done to the // history are recorded - class History + private class History { private readonly string[] _history; private int _head, _tail; @@ -1384,7 +1385,7 @@ public void Close() try { using var sw = File.CreateText(_histfile); - var start = (_count == _history.Length) ? _head : _tail; + var start = _count == _history.Length ? _head : _tail; for (var i = start; i < start + _count; i++) { var p = i % _history.Length; diff --git a/src/cde.sln.DotSettings b/src/cde.sln.DotSettings index b7a4fb2..6e166e7 100644 --- a/src/cde.sln.DotSettings +++ b/src/cde.sln.DotSettings @@ -13,6 +13,7 @@ True True True + True True True True diff --git a/src/cde.slnx b/src/cde.slnx new file mode 100644 index 0000000..b328ed5 --- /dev/null +++ b/src/cde.slnx @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/src/cde/AppContainerBuilder.cs b/src/cde/AppContainerBuilder.cs index f3fbe4f..a65b305 100644 --- a/src/cde/AppContainerBuilder.cs +++ b/src/cde/AppContainerBuilder.cs @@ -1,4 +1,8 @@ +using System; +using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Reflection; using Autofac; using Autofac.Extensions.DependencyInjection; using AutofacSerilogIntegration; @@ -8,6 +12,7 @@ using Microsoft.Extensions.DependencyInjection; using Serilog; using Serilog.Events; +using SlimMessageBus; using SlimMessageBus.Host; using SlimMessageBus.Host.Memory; @@ -19,59 +24,89 @@ namespace cde; public static class AppContainerBuilder { /// - /// Build the DI container. Returns null if appsettings.json is missing. + /// Builds the DI container. Returns false (and leaves null) + /// if appsettings.json is missing; otherwise returns true with the built container. /// - public static IContainer BuildContainer(string[] args) + public static bool TryBuildContainer(string[] args, out IContainer container) { + container = null; ConfigureBootstrapLogger(); // Check for appsettings.json before attempting to build if (!ConfigBuilder.AppSettingsExists()) { - var currentDir = Directory.GetCurrentDirectory(); - Log.Logger.Warning( - "Configuration file '{FileName}' not found in '{Directory}'", - ConfigBuilder.AppSettingsFileName, currentDir); - Log.Logger.Warning( - "Please ensure appsettings.json is in the same directory as the executable"); - return null; + WarnMissingConfig(); + return false; } + var config = ConfigBuilder.Build(args); + ConfigureLogger(config); + var services = new ServiceCollection(); + ConfigureMessageBus(services); + var builder = new ContainerBuilder(); + RegisterCoreServices(builder, config); + builder.Populate(services); // surfaces SlimMessageBus + auto-declared handlers into Autofac + + container = builder.Build(); + return true; + } + + private static void ConfigureMessageBus(IServiceCollection services) + { // Add logging (required by SlimMessageBus) services.AddLogging(loggingBuilder => loggingBuilder.AddSerilog(dispose: false)); + // The cde CLI replaces some cdeLib request handlers with Spectre-progress variants. A request + // type can only have one handler, so skip any cdeLib handler whose request type the cde assembly + // also handles — the cde override then binds alone. New overrides are detected automatically; + // there is no hand-maintained exclusion list. (IConsumer pub/sub events are additive, never skipped.) + var cdeRequestTypes = RequestTypesHandledIn(typeof(AppContainerBuilder).Assembly); + services.AddSlimMessageBus(mbb => { mbb.WithProviderMemory() - // Filter out cdeLib CreateCacheCommandHandler since cde assembly overrides it .AutoDeclareFrom(typeof(CdelibModule).Assembly, - consumerTypeFilter: t => t != typeof(cdeLib.Catalog.CreateCacheCommandHandler)) + consumerTypeFilter: t => !HandlesAnyRequest(t, cdeRequestTypes)) .AutoDeclareFrom(typeof(AppContainerBuilder).Assembly); }); + } - var builder = new ContainerBuilder(); - var config = ConfigBuilder.Build(args); - ConfigureLogger(config); - builder.RegisterInstance(config); - builder.RegisterType().As(); - builder.RegisterLogger(); + /// Request types handled by implementations in the assembly. + private static HashSet RequestTypesHandledIn(Assembly assembly) + => assembly.GetTypes().SelectMany(RequestTypesOf).ToHashSet(); - builder.RegisterModule(); + private static bool HandlesAnyRequest(Type handler, HashSet requestTypes) + => RequestTypesOf(handler).Any(requestTypes.Contains); - // Populate Autofac from ServiceCollection (for SlimMessageBus) - builder.Populate(services); + private static IEnumerable RequestTypesOf(Type handler) + => handler.GetInterfaces() + .Where(i => i.IsGenericType + && (i.GetGenericTypeDefinition() == typeof(IRequestHandler<>) + || i.GetGenericTypeDefinition() == typeof(IRequestHandler<,>))) + .Select(i => i.GetGenericArguments()[0]); - // Register handlers explicitly in Autofac to ensure they can be resolved - builder.RegisterType().AsSelf(); - builder.RegisterType().AsSelf(); - builder.RegisterType().AsSelf(); - builder.RegisterType().AsSelf(); - builder.RegisterType().AsSelf(); - builder.RegisterType().AsSelf(); + private static void RegisterCoreServices(ContainerBuilder builder, IConfigurationRoot config) + { + // Register as IConfigurationRoot (its compile-time type) — cdeLib.Infrastructure.Configuration + // depends on IConfigurationRoot, so widening this to IConfiguration would break resolution. + builder.RegisterInstance(config); + builder.RegisterLogger(); + builder.RegisterModule(); + builder.RegisterType(); + // Handlers are registered by SlimMessageBus AutoDeclareFrom (addServicesFromAssembly: true) + // and surfaced into Autofac via builder.Populate(services) — no explicit handler registration needed. + } - return builder.Build(); + private static void WarnMissingConfig() + { + var currentDir = Directory.GetCurrentDirectory(); + Log.Logger.Warning( + "Configuration file '{FileName}' not found in '{Directory}'", + ConfigBuilder.AppSettingsFileName, currentDir); + Log.Logger.Warning( + "Please ensure appsettings.json is in the same directory as the executable"); } private static void ConfigureBootstrapLogger() diff --git a/src/cde/CdeApp.cs b/src/cde/CdeApp.cs new file mode 100644 index 0000000..3e30f72 --- /dev/null +++ b/src/cde/CdeApp.cs @@ -0,0 +1,294 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using cde.CommandLine; +using cdeLib; +using cdeLib.Catalog; +using cdeLib.Duplicates; +using cdeLib.Entities; +using cdeLib.Entities.Columnar; +using cdeLib.Entities.Soa; +using cdeLib.Hashing; +using cdeLib.Upgrade; +using Mono.Terminal; +using Serilog; +using SlimMessageBus; + +namespace cde; + +/// +/// Hosts the cde command implementations with their dependencies injected, keeping the logic free of +/// service-locator lookups and unit-testable in isolation. resolves a single +/// instance and dispatches parsed CLI verbs to it. +/// +public sealed class CdeApp( + IFindService findService, + ICatalogRepository repository, + IMessageBus messageBus, + OperationCancellation cancellation) +{ + // ---- catalog commands (routed through the message bus) ---- + + public Task CreateCacheAsync(ScanOptions opts) => + messageBus.Send(new CreateCacheCommand(opts.Path) + { Description = opts.Description, FollowJunctions = opts.FollowJunctions }, cancellationToken: cancellation.Token); + + public Task HashCatalogAsync() => + messageBus.Send(new HashCatalogCommand(), cancellationToken: cancellation.Token); + + public Task FindDupesAsync() => + messageBus.Send(new FindDuplicatesCommand(), cancellationToken: cancellation.Token); + + public Task UpdateAsync(UpdateOptions opts) => + messageBus.Send(new UpdateCommand { FileName = opts.FileName, Description = opts.Description }, cancellationToken: cancellation.Token); + + // ---- find ---- + + /// + /// Run a find, preferring the zero-copy columnar format: if any .cdex catalogs exist in the + /// current dir (or one level down) they are searched over their memory maps with no managed catalog + /// load; otherwise we fall back to loading the MessagePack .cde trees. + /// + public void RunFind(string value, string param) + { + var cdex = repository.GetColumnarFileList(["./"]); + if (cdex.Count == 0) + { + findService.Find(value, param, repository.LoadCurrentDirCache()); + return; + } + + var readers = new List(cdex.Count); + try + { + foreach (var file in cdex) + { + try + { + readers.Add(new ColumnarCatalogReader(file)); + } + catch (Exception ex) + { + Log.Logger.Warning(ex, "Skipping unreadable .cdex {File}", file); + } + } + + findService.FindColumnar(value, param, readers); + } + finally + { + foreach (var reader in readers) reader.Dispose(); + } + } + + // repl = read-eval-print-loop + public void FindRepl(string paramString, string firstPattern) + { + var rootEntries = repository.LoadCurrentDirCache(); + + if (!string.IsNullOrEmpty(firstPattern)) + findService.Find(firstPattern, paramString, rootEntries); + + Console.WriteLine("Issue --help for available params"); + + while (true) + { + cancellation.Reset(); // fresh token each prompt so a break during the previous search doesn't carry over. + Console.Write("Enter string to search : "); + var pattern = Console.ReadLine(); + if (string.IsNullOrEmpty(pattern)) + { + Console.WriteLine("Exiting..."); + break; + } + + if (pattern.StartsWith("--", StringComparison.CurrentCulture)) + { + HandleReplCommand(pattern[2..]); + } + else + { + findService.Find(pattern, paramString, rootEntries); + } + } + } + + private void HandleReplCommand(string command) + { + switch (command.ToLower(CultureInfo.CurrentCulture)) + { + case "includefiles": + findService.IncludeFiles = !findService.IncludeFiles; + Console.WriteLine($"IncludeFiles:{findService.IncludeFiles}"); + break; + case "includefolders": + findService.IncludeFolders = !findService.IncludeFolders; + Console.WriteLine($"IncludeFolders:{findService.IncludeFolders}"); + break; + case "help": + Console.WriteLine("Valid options are"); + Console.WriteLine("--includefiles"); + Console.WriteLine("--includefolders"); + break; + case "clear": + Console.Clear(); + break; + default: + Console.WriteLine($"unknown command {command}"); + break; + } + } + + // ---- migrate ---- + + /// + /// One-way migration of MessagePack .cde catalogs to the zero-copy columnar .cdex format. With a + /// path argument, converts that file; otherwise converts every catalog discovered in the current + /// directory and one level down, writing a .cdex beside each source. + /// + public void Migrate(MigrateOptions opts) + { + List files; + if (!string.IsNullOrWhiteSpace(opts.Path)) + { + if (!File.Exists(opts.Path)) + { + Console.WriteLine($"File not found: {opts.Path}"); + return; + } + files = [opts.Path]; + } + else + { + files = repository.GetCacheFileList(["./"]).ToList(); + } + + if (files.Count == 0) + { + Console.WriteLine("No .cde catalogs found to migrate."); + return; + } + + var converted = 0; + foreach (var file in files) + { + if (MigrateOne(file)) converted++; + } + + Console.WriteLine($"Migrated {converted} of {files.Count} catalog(s) to .cdex."); + } + + private bool MigrateOne(string file) + { + try + { + var root = repository.LoadDirCache(file); + if (root == null) + { + Console.WriteLine($" skip (could not load): {file}"); + return false; + } + + var store = EntryStore.Build(root); + var outFile = Path.ChangeExtension(file, ".cdex"); + ColumnarFormat.Write(store, outFile); + + var srcLen = new FileInfo(file).Length; + var dstLen = new FileInfo(outFile).Length; + Console.WriteLine( + $" {Path.GetFileName(file)} ({srcLen:N0} B) -> {Path.GetFileName(outFile)} " + + $"({dstLen:N0} B, {store.Count:N0} entries)"); + return true; + } + catch (Exception ex) + { + Console.WriteLine($" error migrating {file}: {ex.Message}"); + return false; + } + } + + // ---- repl / inspection ---- + + public void InvokeRepl() + { + var le = new LineEditor(name: null); + var running = true; + + while (running && le.Edit("shell> ", string.Empty) is { } s) + { + Console.WriteLine($"----> [{s}]"); + switch (s) + { + case "quit": + running = false; + break; + case "history": + case "!": + le.CmdHistoryDump(); + break; + case "help": + case "?": + PrintReplHelp(); + break; + } + } + } + + private static void PrintReplHelp() + { + Console.WriteLine("Builtin Commands:"); + Console.WriteLine(" quit - quit,"); + Console.WriteLine(" help - show help, ? - show help"); + Console.WriteLine(" history - show history, ! - show history"); + Console.WriteLine("Keystrokes:"); + Console.WriteLine(" Home, End, Left, Right, Up, Down, Back, Del, Tab"); + Console.WriteLine(" C-a, C-e, C-b, C-f, C-p, C-n, C-d"); + Console.WriteLine(" C-l - clear console to top"); + Console.WriteLine(" C-r - reverse search history"); + Console.WriteLine(" A-b - move backward word"); + Console.WriteLine(" A-f - move forward word"); + Console.WriteLine(" A-d - delete word forward"); + Console.WriteLine(" A-Backspace - delete word backward"); + } + + public void LoadWait() + { + repository.LoadCurrentDirCache(); + Console.ReadLine(); + } + + public void PrintPathsHaveHash() + { + var rootEntries = repository.LoadCurrentDirCache(); + foreach (var pairDirEntry in EntryHelper.GetPairDirEntries(rootEntries)) + { + var hash = pairDirEntry.ChildDE.IsHashDone ? "#" : " "; + var bang = pairDirEntry.PathProblem ? "!" : " "; + Console.WriteLine($"{hash}{bang}{pairDirEntry.FullPath}"); + if (cancellation.IsCancellationRequested) + { + break; + } + } + } + + public void FindPopulous(int minimumCount) + { + var largeEntries = EntryHelper.GetDirEntries(repository.LoadCurrentDirCache()) + .Where(e => e.Children is { } c && c.Count > minimumCount) + .OrderByDescending(e => e.Children.Count) + .ToList(); + + foreach (var e in largeEntries) + { + Console.WriteLine($"{e.FullPath} {e.Children.Count}"); + if (cancellation.IsCancellationRequested) + { + break; + } + } + } +} diff --git a/src/cde/CommandLine/CommandLineOptions.cs b/src/cde/CommandLine/CommandLineOptions.cs index 9a31368..b435e2f 100644 --- a/src/cde/CommandLine/CommandLineOptions.cs +++ b/src/cde/CommandLine/CommandLineOptions.cs @@ -13,6 +13,11 @@ public class ScanOptions [Option("desc", HelpText = "Description to set")] public string Description { get; [UsedImplicitly] set; } + + [Option("follow-junctions", + Default = false, + HelpText = "[DANGER NOT EXTENSIVELY TESTED] Descend into directory junctions / symbolic links. Off by default to avoid cycles and duplicate content.")] + public bool FollowJunctions { get; [UsedImplicitly] set; } } [Verb("find", HelpText = "Uses all cache files available searches for ")] @@ -66,6 +71,16 @@ public class ReplFindOptions public string Value { get; [UsedImplicitly] set; } } +[Verb("migrate", + HelpText = "Convert MessagePack .cde catalogs to the zero-copy columnar .cdex format (one-way).")] +public class MigrateOptions +{ + [Value(0, Required = false, + HelpText = "Specific .cde file to convert. If omitted, converts every catalog in the current " + + "directory and one level down, writing a .cdex beside each.")] + public string Path { get; [UsedImplicitly] set; } +} + [Verb("hash", HelpText = "Hash all catalogs in current directory")] public class HashOptions; diff --git a/src/cde/GlobalSuppressions.cs b/src/cde/GlobalSuppressions.cs index 6d7b2b2..662bbfc 100644 --- a/src/cde/GlobalSuppressions.cs +++ b/src/cde/GlobalSuppressions.cs @@ -12,6 +12,7 @@ [assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.GrepOptions")] [assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.GrepPathOptions")] [assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.LoadWaitOptions")] +[assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.MigrateOptions")] [assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.PopulousFoldersOptions")] [assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.ReplFindOptions")] [assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.ReplGrepOptions")] @@ -19,4 +20,3 @@ [assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.ReplOptions")] [assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.ScanOptions")] [assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.TreeDumpOptions")] -[assembly: SuppressMessage("Design", "MA0048:File name must match type name", Justification = "", Scope = "type", Target = "~T:cde.CommandLine.UpgradeOptions")] diff --git a/src/cde/HashProgress/HashCatalogCommandHandler.cs b/src/cde/HashProgress/HashCatalogCommandHandler.cs new file mode 100644 index 0000000..46eb76c --- /dev/null +++ b/src/cde/HashProgress/HashCatalogCommandHandler.cs @@ -0,0 +1,97 @@ +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using cdeLib; +using cdeLib.Catalog; +using cdeLib.Duplicates; +using cdeLib.Entities; +using cdeLib.Entities.Columnar; +using cdeLib.Entities.Soa; +using cdeLib.Hashing; +using cdeLib.Infrastructure; +using JetBrains.Annotations; +using SlimMessageBus; + +namespace cde.HashProgress; + +/// +/// cde-layer override of the hash handler. Mirrors +/// but renders a live Spectre progress display by publishing progress events emitted from . +/// The cdeLib handler is excluded from message-bus auto-declaration so this one is used by the CLI. +/// +[UsedImplicitly] +public class HashCatalogCommandHandler : IRequestHandler +{ + private readonly Duplication _duplication; + private readonly Serilog.ILogger _logger; + private readonly IApplicationDiagnostics _applicationDiagnostics; + private readonly ICatalogRepository _catalogRepository; + private readonly IMessageBus _messageBus; + + public HashCatalogCommandHandler(Serilog.ILogger logger, IApplicationDiagnostics applicationDiagnostics, + Duplication duplication, ICatalogRepository catalogRepository, IMessageBus messageBus) + { + _logger = logger; + _applicationDiagnostics = applicationDiagnostics; + _duplication = duplication; + _catalogRepository = catalogRepository; + _messageBus = messageBus; + } + + public async Task OnHandle(HashCatalogCommand request, CancellationToken cancellationToken) + { + // Hash operates on the columnar .cdex catalogs. The hashing engine is tree-based, so each + // catalog is reconstructed into a mutable tree, hashed, then written back as a fresh .cdex. + var cdexFiles = _catalogRepository.GetColumnarFileList(["./"]); + if (cdexFiles.Count == 0) + { + _logger.Warning("No .cdex catalogs found. Run 'cde migrate' to create them first."); + return; + } + + var mainLoopTask = Task.Run(() => MainLoop(cdexFiles, cancellationToken), cancellationToken); + var console = new HashProgressConsole(); + console.Start(mainLoopTask, cancellationToken); + await mainLoopTask.ConfigureAwait(false); + } + + private async Task MainLoop(IList cdexFiles, CancellationToken cancellationToken) + { + _logger.Information("Memory pre-catalog load: {MemoryAllocated}", + _applicationDiagnostics.GetMemoryAllocated().FormatAsBytes()); + var rootEntries = CatalogTreeBuilder.FromColumnarFiles(cdexFiles); + _logger.Information("Memory post-catalog load: {MemoryAllocated}", + _applicationDiagnostics.GetMemoryAllocated().FormatAsBytes()); + + // Route hashing progress and status lines to the Spectre console via the message bus. + _duplication.ProgressEvent = (processed, toHash, phase) => + _messageBus.Publish(new HashProgressEvent(processed, toHash, phase), cancellationToken: cancellationToken); + _duplication.StatusMessageEvent = message => + _messageBus.Publish(new HashStatusMessageEvent(message), cancellationToken: cancellationToken); + + var stopwatch = Stopwatch.StartNew(); + try + { + await _duplication.ApplyHash(rootEntries).ConfigureAwait(false); + + foreach (var rootEntry in rootEntries) + { + HashProgressConsole.EnqueueMessage($"Saving catalog {rootEntry.ActualFileName}"); + ColumnarFormat.Write(EntryStore.Build(rootEntry), rootEntry.ActualFileName); + } + + var ts = stopwatch.Elapsed; + var elapsedTime = $"{ts.Hours:00}:{ts.Minutes:00}:{ts.Seconds:00}.{ts.Milliseconds / 10:00}"; + HashProgressConsole.EnqueueMessage( + $"Hash Took {elapsedTime}, Memory: {_applicationDiagnostics.GetMemoryAllocated().FormatAsBytes()}"); + _logger.Information("Hash Took {ElapsedTime}, Memory: {Memory}", elapsedTime, + _applicationDiagnostics.GetMemoryAllocated().FormatAsBytes()); + } + finally + { + await _messageBus.Publish(new HashCompletedEvent(), cancellationToken: cancellationToken) + .ConfigureAwait(false); + } + } +} diff --git a/src/cde/HashProgress/HashCompletedEventHandler.cs b/src/cde/HashProgress/HashCompletedEventHandler.cs new file mode 100644 index 0000000..cbae176 --- /dev/null +++ b/src/cde/HashProgress/HashCompletedEventHandler.cs @@ -0,0 +1,17 @@ +using System.Threading; +using System.Threading.Tasks; +using cdeLib.Hashing; +using JetBrains.Annotations; +using SlimMessageBus; + +namespace cde.HashProgress; + +[UsedImplicitly] +public class HashCompletedEventHandler : IConsumer +{ + public async Task OnHandle(HashCompletedEvent message, CancellationToken cancellationToken) + { + HashProgressConsole.HashIsComplete = true; + await Task.Yield(); + } +} diff --git a/src/cde/HashProgress/HashProgressConsole.cs b/src/cde/HashProgress/HashProgressConsole.cs new file mode 100644 index 0000000..a081ff7 --- /dev/null +++ b/src/cde/HashProgress/HashProgressConsole.cs @@ -0,0 +1,123 @@ +using System; +using System.Collections.Concurrent; +using System.Diagnostics; +using System.Globalization; +using System.Threading; +using System.Threading.Tasks; +using Serilog; +using Spectre.Console; + +namespace cde.HashProgress; + +public class HashProgressConsole +{ + public static long FilesProcessed { get; set; } + + public static long FilesToHash { get; set; } + + public static string Phase { get; set; } + + public static bool HashIsComplete { get; set; } + + private static readonly ConcurrentQueue Messages = new(); + + private static void WriteLogMessage(string message) + { + var width = AnsiConsole.Profile.Out.Width; + // Pad raw message to console width to clear previous content + var paddedMessage = $"LOG:{message}".PadRight(width); + // Use \r to return to start of line (status update behavior) + AnsiConsole.Markup($"\r[grey]{Markup.Escape(paddedMessage)}[/]\n"); + } + + /// + /// Enqueue a message to be displayed in the console progress UI + /// + public static void EnqueueMessage(string message) + { + Messages.Enqueue(message); + } + + public void Start(Task mainLoopTask, CancellationToken cancellationToken) + { + var sw = new Stopwatch(); + sw.Start(); + AnsiConsole.Status() + .AutoRefresh(enabled: true) + .Spinner(Spinner.Known.Default) + .Start("Hashing...", ctx => + { + while (!mainLoopTask.IsCompleted && !cancellationToken.IsCancellationRequested) + { + ShowProgress(sw, ctx); + // Throttle the refresh loop so it doesn't busy-spin a core while hashing. + Thread.Sleep(50); + } + + // Flush any remaining messages after the task completes + FlushMessages(); + }); + } + + private static void FlushMessages() + { + while (Messages.TryDequeue(out var msg)) + { + if (!string.IsNullOrEmpty(msg)) + { + WriteLogMessage(msg); + } + } + } + + private static long CalculateFilesPerSecond(Stopwatch sw) + { + var elapsedSec = sw.ElapsedMilliseconds / 1000; + if (elapsedSec < 1) elapsedSec = 1; + return FilesProcessed / elapsedSec; + } + + private void ShowProgress(Stopwatch sw, StatusContext ctx) + { + var defaultNumberFormat = new NumberFormatInfo(); + var filesPerSec = CalculateFilesPerSecond(sw); + + // Clamp to the total: FilesProcessed is cumulative across hash passes, so guard against + // any accounting edge case rendering more than the total or over 100%. + var processed = FilesToHash > 0 && FilesProcessed > FilesToHash ? FilesToHash : FilesProcessed; + var processedText = processed.ToString("N0", defaultNumberFormat); + var toHashText = FilesToHash.ToString("N0", defaultNumberFormat); + var perSecText = filesPerSec.ToString("N0", defaultNumberFormat); + var percent = FilesToHash > 0 ? 100.0 * processed / FilesToHash : 0.0; + var phase = string.IsNullOrEmpty(Phase) ? "Hashing" : Phase; + + var msg = + $"[yellow]{Markup.Escape(phase)}[/] [yellow]{processedText}[/] of [yellow]{toHashText}[/] ([yellow]{percent:F1}%[/]) Avg [yellow]{perSecText}[/]/sec"; + + try + { + ctx.Status(msg); + } + catch (Exception ex) + { + Log.Logger.Error(ex, "Error writing Status"); + } + + ctx.Spinner(Spinner.Known.Star); + ctx.SpinnerStyle(Style.Parse("green")); + + var dequeueMessages = true; + while (dequeueMessages) + { + Messages.TryDequeue(out var queued); + if (string.IsNullOrEmpty(queued)) + { + dequeueMessages = false; + } + else + { + WriteLogMessage(queued); + } + } + } +} diff --git a/src/cde/HashProgress/HashProgressNotificationHandler.cs b/src/cde/HashProgress/HashProgressNotificationHandler.cs new file mode 100644 index 0000000..2d0025e --- /dev/null +++ b/src/cde/HashProgress/HashProgressNotificationHandler.cs @@ -0,0 +1,19 @@ +using System.Threading; +using System.Threading.Tasks; +using cdeLib.Hashing; +using JetBrains.Annotations; +using SlimMessageBus; + +namespace cde.HashProgress; + +[UsedImplicitly] +public class HashProgressNotificationHandler : IConsumer +{ + public Task OnHandle(HashProgressEvent message, CancellationToken cancellationToken) + { + HashProgressConsole.FilesProcessed = message.FilesProcessed; + HashProgressConsole.FilesToHash = message.FilesToHash; + HashProgressConsole.Phase = message.Phase; + return Task.CompletedTask; + } +} diff --git a/src/cde/HashProgress/HashStatusMessageHandler.cs b/src/cde/HashProgress/HashStatusMessageHandler.cs new file mode 100644 index 0000000..99ad762 --- /dev/null +++ b/src/cde/HashProgress/HashStatusMessageHandler.cs @@ -0,0 +1,17 @@ +using System.Threading; +using System.Threading.Tasks; +using cdeLib.Hashing; +using JetBrains.Annotations; +using SlimMessageBus; + +namespace cde.HashProgress; + +[UsedImplicitly] +public class HashStatusMessageHandler : IConsumer +{ + public Task OnHandle(HashStatusMessageEvent message, CancellationToken cancellationToken) + { + HashProgressConsole.EnqueueMessage(message.Message); + return Task.CompletedTask; + } +} diff --git a/src/cde/Program.cs b/src/cde/Program.cs index 51624b5..801f9f4 100644 --- a/src/cde/Program.cs +++ b/src/cde/Program.cs @@ -1,293 +1,128 @@ -using System; -using System.Collections.Generic; -using System.Globalization; -using System.Linq; -using System.Threading.Tasks; -using Autofac; -using cde.CommandLine; -using cdeLib; -using cdeLib.Catalog; -using cdeLib.Duplicates; -using cdeLib.Entities; -using cdeLib.Hashing; -using cdeLib.Upgrade; -using CommandLine; -using SlimMessageBus; -using Mono.Terminal; -using Serilog; -using SerilogTimings; -using FindOptions = cde.CommandLine.FindOptions; -using IContainer = Autofac.IContainer; - -namespace cde; - -public static class Program -{ - private static IContainer _container; - - private static IMessageBus MessageBus { get; set; } - - /// - /// Initialize the program. Returns false if initialization failed (e.g., missing config). - /// - public static bool InitProgram(string[] args) - { - _container = AppContainerBuilder.BuildContainer(args); - if (_container == null) - { - return false; - } - MessageBus = Resolve(); - return true; - } - - private static ParserResult GetParserResult(IEnumerable args) - { - var parser = CommandLineParserBuilder.Build(); - return parser.ParseArguments< - ScanOptions, - FindOptions, - GrepOptions, - GrepPathOptions, - ReplGrepPathOptions, - ReplGrepOptions, - ReplFindOptions, - HashOptions, - DupesOptions, - TreeDumpOptions, - LoadWaitOptions, - ReplOptions, - PopulousFoldersOptions, - FindPathOptions, - UpdateOptions>(args); - } - - private static int Main(string[] args) - { - if (!InitProgram(args)) - { - return 1; // Exit with error code if initialization failed - } - Console.CancelKeyPress += BreakConsole; - try - { - using (Operation.Time("App")) - { - var findService = Resolve(); - var parsedResult = GetParserResult(args) - .WithParsed(CreateCache) - .WithParsed(opts => - { - findService.Find(opts.Value, "--find", - Resolve().LoadCurrentDirCache()); - }) - .WithParsed(opts => - { - findService.Find(opts.Value, "--findpath", - Resolve().LoadCurrentDirCache()); - }) - .WithParsed(opts => - { - findService.Find(opts.Value, "--grep", - Resolve().LoadCurrentDirCache()); - }) - .WithParsed(opts => - { - findService.Find(opts.Value, "--greppath", - Resolve().LoadCurrentDirCache()); - }) - .WithParsed(opts => FindRepl(FindService.ParamGrepPath, opts.Value)) - .WithParsed(opts => FindRepl(FindService.ParamGrep, opts.Value)) - .WithParsed(opts => FindRepl(FindService.ParamFind, opts.Value)) - .WithParsed(_ => HashCatalog()) - .WithParsed(_ => FindDupes()) - .WithParsed(_ => PrintPathsHaveHashEnumerator()) - .WithParsed(_ => - { - Resolve().LoadCurrentDirCache(); - Console.ReadLine(); - }) - .WithParsed(_ => InvokeRepl()) - .WithParsed(opts => FindPopulous(opts.Count)) - .WithParsed(Update); - parsedResult.WithNotParsed(errs => CustomHelpText.DisplayHelp(parsedResult)); - return 0; - } - } - finally - { - Log.CloseAndFlush(); - } - } - - private static T Resolve() - { - return _container.Resolve(); - } - - private static void InvokeRepl() - { - var le = new LineEditor(name: null); - var running = true; - - while (running && le.Edit("shell> ", string.Empty) is { } s) - { - Console.WriteLine($"----> [{s}]"); - switch (s) - { - case "quit": - running = false; - break; - case "history": - case "!": - le.CmdHistoryDump(); - break; - case "help": - case "?": - Console.WriteLine("Builtin Commands:"); - Console.WriteLine(" quit - quit,"); - Console.WriteLine(" help - show help, ? - show help"); - Console.WriteLine(" history - show history, ! - show history"); - Console.WriteLine("Keystrokes:"); - Console.WriteLine(" Home, End, Left, Right, Up, Down, Back, Del, Tab"); - Console.WriteLine(" C-a, C-e, C-b, C-f, C-p, C-n, C-d"); - Console.WriteLine(" C-l - clear console to top"); - Console.WriteLine(" C-r - reverse search history"); - Console.WriteLine(" A-b - move backward word"); - Console.WriteLine(" A-f - move forward word"); - Console.WriteLine(" A-d - delete word forward"); - Console.WriteLine(" A-Backspace - delete word backward"); - break; - } - } - } - - private static void BreakConsole(object sender, ConsoleCancelEventArgs e) - { - Console.WriteLine("\n * Break key detected. will exit as soon as current file process is completed."); - Hack.BreakConsoleFlag = true; - e.Cancel = true; - } - - // repl = read-eval-print-loop - private static void FindRepl(string paramString, string firstPattern) - { - var rootEntries = Resolve().LoadCurrentDirCache(); - var findService = Resolve(); - - if (!string.IsNullOrEmpty(firstPattern)) - findService.Find(firstPattern, paramString, rootEntries); - - Console.WriteLine("Issue --help for available params"); - - while (true) - { - if (Hack.BreakConsoleFlag) - Hack.BreakConsoleFlag = false; //reset otherwise we'll get some weird behaviour in loop. - Console.Write("Enter string to search : "); - var pattern = Console.ReadLine(); - if (string.IsNullOrEmpty(pattern)) - { - Console.WriteLine("Exiting..."); - break; - } - - if (pattern.StartsWith("--", StringComparison.CurrentCulture)) - { - var command = pattern[2..]; - switch (command.ToLower(CultureInfo.CurrentCulture)) - { - case "includefiles": - findService.IncludeFiles = !findService.IncludeFiles; - Console.WriteLine($"IncludeFiles:{findService.IncludeFiles}"); - break; - case "includefolders": - findService.IncludeFolders = !findService.IncludeFolders; - Console.WriteLine($"IncludeFolders:{findService.IncludeFolders}"); - break; - case "help": - Console.WriteLine("Valid options are"); - Console.WriteLine("--includefiles"); - Console.WriteLine("--includefolders"); - break; - case "clear": - Console.Clear(); - break; - default: - Console.WriteLine($"unknown command {command}"); - break; - } - } - else - { - findService.Find(pattern, paramString, rootEntries); - } - } - } - - private static void Update(UpdateOptions opts) - { - var task = Task.Run(() => - MessageBus.Send(new UpdateCommand { FileName = opts.FileName, Description = opts.Description })); - task.Wait(); - } - - private static void FindDupes() - { - var task = Task.Run(() => MessageBus.Send(new FindDuplicatesCommand())); - task.Wait(); - } - - public static void HashCatalog() - { - var task = Task.Run(async () => await MessageBus.Send(new HashCatalogCommand()).ConfigureAwait(false)); - task.Wait(); - } - - public static void CreateCache(ScanOptions opts) - { - var task = Task.Run(async () => - await MessageBus.Send(new CreateCacheCommand(opts.Path) { Description = opts.Description }) - .ConfigureAwait(false)); - task.Wait(); - } - - private static void PrintPathsHaveHashEnumerator() - { - var rootEntries = Resolve().LoadCurrentDirCache(); - foreach (var pairDirEntry in EntryHelper.GetPairDirEntries(rootEntries)) - { - var hash = pairDirEntry.ChildDE.IsHashDone ? "#" : " "; - var bang = pairDirEntry.PathProblem ? "!" : " "; - Console.WriteLine($"{hash}{bang}{pairDirEntry.FullPath}"); - if (Hack.BreakConsoleFlag) - { - break; - } - } - } - - private static void FindPopulous(int minimumCount) - { - var rootEntries = Resolve().LoadCurrentDirCache(); - var entries = EntryHelper.GetDirEntries(rootEntries); - var largeEntries = entries - .Where(e => e.Children != null && e.Children.Count > minimumCount) - .ToList(); - largeEntries.Sort(CompareDirEntries); - - foreach (var e in largeEntries.Where(e => e.Children != null && e.Children.Count > minimumCount)) - { - Console.WriteLine($"{e.FullPath} {e.Children.Count}"); - if (Hack.BreakConsoleFlag) - { - break; - } - } - } - - private static int CompareDirEntries(ICommonEntry x, ICommonEntry y) - { - return y.Children.Count - x.Children.Count; - } -} \ No newline at end of file +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Autofac; +using cde.CommandLine; +using cdeLib; +using CommandLine; +using Serilog; +using SerilogTimings; +using FindOptions = cde.CommandLine.FindOptions; +using IContainer = Autofac.IContainer; + +namespace cde; + +public static class Program +{ + private static IContainer _container; + private static CdeApp _app; + private static OperationCancellation _cancellation; + + /// + /// Initialize the program. Returns false if initialization failed (e.g., missing config). + /// + public static bool InitProgram(string[] args) + { + if (!AppContainerBuilder.TryBuildContainer(args, out _container)) + { + return false; + } + _app = _container.Resolve(); + _cancellation = _container.Resolve(); + return true; + } + + // Static entry points retained for cdeLibTest/DuplicationTest, which drives a scan+hash via Program. + // These block on the async commands; the blocking is confined to this test-support path, not Main. + public static void CreateCache(ScanOptions opts) => _app.CreateCacheAsync(opts).GetAwaiter().GetResult(); + public static void HashCatalog() => _app.HashCatalogAsync().GetAwaiter().GetResult(); + + private static ParserResult GetParserResult(IEnumerable args) + { + var parser = CommandLineParserBuilder.Build(); + return parser.ParseArguments< + ScanOptions, + FindOptions, + GrepOptions, + GrepPathOptions, + ReplGrepPathOptions, + ReplGrepOptions, + ReplFindOptions, + MigrateOptions, + HashOptions, + DupesOptions, + TreeDumpOptions, + LoadWaitOptions, + ReplOptions, + PopulousFoldersOptions, + FindPathOptions, + UpdateOptions>(args); + } + + private static async Task Main(string[] args) + { + if (!InitProgram(args)) + { + return 1; // Exit with error code if initialization failed + } + Console.CancelKeyPress += BreakConsole; + try + { + using (Operation.Time("App")) + { + var parsed = GetParserResult(args); + if (parsed is Parsed ok) + { + await DispatchAsync(ok.Value).ConfigureAwait(false); + } + else + { + CustomHelpText.DisplayHelp(parsed); + } + return 0; + } + } + finally + { + await Log.CloseAndFlushAsync().ConfigureAwait(false); + } + } + + /// + /// Routes a parsed verb to its command. Bus-backed commands are awaited directly; the synchronous + /// (interactive / inspection) commands are adapted to a completed task via . + /// + private static Task DispatchAsync(object options) => options switch + { + ScanOptions o => _app.CreateCacheAsync(o), + FindOptions o => RunSync(() => _app.RunFind(o.Value, "--find")), + FindPathOptions o => RunSync(() => _app.RunFind(o.Value, "--findpath")), + GrepOptions o => RunSync(() => _app.RunFind(o.Value, "--grep")), + GrepPathOptions o => RunSync(() => _app.RunFind(o.Value, "--greppath")), + ReplGrepPathOptions o => RunSync(() => _app.FindRepl(FindService.ParamGrepPath, o.Value)), + ReplGrepOptions o => RunSync(() => _app.FindRepl(FindService.ParamGrep, o.Value)), + ReplFindOptions o => RunSync(() => _app.FindRepl(FindService.ParamFind, o.Value)), + MigrateOptions o => RunSync(() => _app.Migrate(o)), + HashOptions => _app.HashCatalogAsync(), + DupesOptions => _app.FindDupesAsync(), + TreeDumpOptions => RunSync(_app.PrintPathsHaveHash), + LoadWaitOptions => RunSync(_app.LoadWait), + ReplOptions => RunSync(_app.InvokeRepl), + PopulousFoldersOptions o => RunSync(() => _app.FindPopulous(o.Count)), + UpdateOptions o => _app.UpdateAsync(o), + _ => Task.CompletedTask, + }; + + private static Task RunSync(Action action) + { + action(); + return Task.CompletedTask; + } + + private static void BreakConsole(object sender, ConsoleCancelEventArgs e) + { + Console.WriteLine("\n * Break key detected. will exit as soon as current file process is completed."); + _cancellation.Cancel(); + e.Cancel = true; + } +} diff --git a/src/cde/ScanProgress/CreateCacheCommandHandler.cs b/src/cde/ScanProgress/CreateCacheCommandHandler.cs index 30f8744..6224a6a 100644 --- a/src/cde/ScanProgress/CreateCacheCommandHandler.cs +++ b/src/cde/ScanProgress/CreateCacheCommandHandler.cs @@ -1,10 +1,13 @@ using System; using System.Globalization; +using System.IO; using System.Threading; using System.Threading.Tasks; using cdeLib; using cdeLib.Catalog; using cdeLib.Entities; +using cdeLib.Entities.Columnar; +using cdeLib.Entities.Soa; using cdeLib.Infrastructure.Config; using Humanizer; using JetBrains.Annotations; @@ -17,15 +20,15 @@ namespace cde.ScanProgress; public class CreateCacheCommandHandler : IRequestHandler { private readonly IConfiguration _configuration; - private readonly ICatalogRepository _catalogRepository; private readonly IMessageBus _messageBus; + private readonly OperationCancellation _cancellation; - public CreateCacheCommandHandler(IConfiguration configuration, ICatalogRepository catalogRepository, - IMessageBus messageBus) + public CreateCacheCommandHandler(IConfiguration configuration, IMessageBus messageBus, + OperationCancellation cancellation) { _configuration = configuration; - _catalogRepository = catalogRepository; _messageBus = messageBus; + _cancellation = cancellation; } public async Task OnHandle(CreateCacheCommand request, CancellationToken cancellationToken) @@ -47,19 +50,15 @@ private async Task MainLoop(CreateCacheCommand request, CancellationToken cancel re.SimpleScanEndEvent = () => _messageBus.Publish(new ScanCompletedEvent(), cancellationToken: cancellationToken); re.ExceptionEvent = PrintException; - re.PopulateRoot(request.Path); - if (Hack.BreakConsoleFlag) + re.PopulateRoot(request.Path, request.FollowJunctions, _cancellation.Token); + if (_cancellation.IsCancellationRequested) { Console.WriteLine(" * Break key detected incomplete scan will not be saved."); return; } - var oldRoot = _catalogRepository.LoadDirCache(re.DefaultFileName); - if (oldRoot != null) - { - Log.Information("Found cache \"{FileName}\", Updating hashes for new scan from cache file", re.DefaultFileName); - oldRoot.TraverseTreesCopyHash(re); - } + var cdexName = Path.ChangeExtension(re.DefaultFileName, ".cdex"); + ReuseHashesFromExistingCatalog(re, cdexName); re.SortAllChildrenByPath(); re.SetSummaryFields(); @@ -69,24 +68,12 @@ private async Task MainLoop(CreateCacheCommand request, CancellationToken cancel } ScanProgressConsole.EnqueueMessage("Saving catalog..."); - await _catalogRepository.Save(re).ConfigureAwait(false); - ScanProgressConsole.EnqueueMessage($"Saved to {re.DefaultFileName}"); + re.ActualFileName = cdexName; + await Task.Run(() => ColumnarFormat.Write(EntryStore.Build(re), cdexName), cancellationToken) + .ConfigureAwait(false); + ScanProgressConsole.EnqueueMessage($"Saved to {cdexName}"); - // Calculate and display final scan summary - sw.Stop(); - var elapsedSec = sw.ElapsedMilliseconds / 1000.0; - if (elapsedSec < 1) elapsedSec = 1; - var totalCount = re.FileEntryCount + re.DirEntryCount; - var scansPerSec = (long)(totalCount / elapsedSec); - var defaultNumberFormat = new NumberFormatInfo(); - var scanCountText = totalCount.ToString("N0", defaultNumberFormat); - var scansPerSecText = scansPerSec.ToString("N0", defaultNumberFormat); - ScanProgressConsole.EnqueueMessage($"Total files scanned: {scanCountText}, Average: {scansPerSecText}/sec"); - - Log.Information("Scanned path {Path}, Saved to {SavePath}", re.Path,re.DefaultFileName); - Log.Information( - "Scanned Files {FileCount:0,0}, Dirs {DirCount:0,0}, Total size {Size:0,0}", re.FileEntryCount, - re.DirEntryCount, re.Size.Bytes().Humanize(CultureInfo.CurrentCulture)); + ReportScanSummary(re, cdexName, sw); } catch (ArgumentException ex) { @@ -94,6 +81,44 @@ private async Task MainLoop(CreateCacheCommand request, CancellationToken cancel } } + /// + /// Catalogs are stored in the zero-copy columnar .cdex format. Reuse hashes from an existing + /// .cdex (reconstructed into a tree) for this scan path when one is found. + /// + private static void ReuseHashesFromExistingCatalog(RootEntry re, string cdexName) + { + if (!File.Exists(cdexName)) + { + return; + } + + Log.Information("Found cache \"{FileName}\", Updating hashes for new scan from cache file", cdexName); + RootEntry oldRoot; + using (var reader = new ColumnarCatalogReader(cdexName)) + { + oldRoot = CatalogTreeBuilder.FromSource(reader); + } + oldRoot.TraverseTreesCopyHash(re); + } + + private static void ReportScanSummary(RootEntry re, string cdexName, System.Diagnostics.Stopwatch sw) + { + sw.Stop(); + var elapsedSec = sw.ElapsedMilliseconds / 1000.0; + if (elapsedSec < 1) elapsedSec = 1; + var totalCount = re.FileEntryCount + re.DirEntryCount; + var scansPerSec = (long)(totalCount / elapsedSec); + var defaultNumberFormat = new NumberFormatInfo(); + var scanCountText = totalCount.ToString("N0", defaultNumberFormat); + var scansPerSecText = scansPerSec.ToString("N0", defaultNumberFormat); + ScanProgressConsole.EnqueueMessage($"Total files scanned: {scanCountText}, Average: {scansPerSecText}/sec"); + + Log.Information("Scanned path {Path}, Saved to {SavePath}", re.Path, cdexName); + Log.Information( + "Scanned Files {FileCount:0,0}, Dirs {DirCount:0,0}, Total size {Size:0,0}", re.FileEntryCount, + re.DirEntryCount, re.Size.Bytes().Humanize(CultureInfo.CurrentCulture)); + } + private void PrintException(string path, Exception ex) { Console.WriteLine($"Exception {ex.GetType()}, Path \"{path}\""); diff --git a/src/cde/cde.csproj b/src/cde/cde.csproj index a874c1f..4a0d9fb 100644 --- a/src/cde/cde.csproj +++ b/src/cde/cde.csproj @@ -17,27 +17,27 @@ - + - + - + - - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - - + + + + - + diff --git a/src/cdeBenchmarks/CatalogFixture.cs b/src/cdeBenchmarks/CatalogFixture.cs new file mode 100644 index 0000000..6b51772 --- /dev/null +++ b/src/cdeBenchmarks/CatalogFixture.cs @@ -0,0 +1,52 @@ +using cdeLib.Catalog; +using cdeLib.Entities; +using cdeMemProbe; +using Serilog; + +namespace cdeBenchmarks; + +/// +/// Thin helper that turns the shared generator into the two shapes +/// the benchmarks need: an in-memory tree (for search) and an on-disk .cde file (for load). +/// Centralised so every benchmark and phase measures the identical fixture. +/// +internal static class CatalogFixture +{ + private static readonly ILogger Silent = new LoggerConfiguration().CreateLogger(); + + /// Build the synthetic tree in memory (no disk I/O). + public static RootEntry BuildInMemory(int entryCount, bool withHashes) + => SyntheticCatalog.Generate(entryCount, withHashes); + + /// Generate the synthetic tree and serialize it to a fresh temp .cde file; returns the path. + public static string WriteTemp(int entryCount, bool withHashes) + { + var root = SyntheticCatalog.Generate(entryCount, withHashes); + var path = Path.Combine( + Path.GetTempPath(), + $"cde-bench-{entryCount}{(withHashes ? "-hashed" : "")}-{Guid.NewGuid():N}.cde"); + root.ActualFileName = path; + using var repo = new CatalogRepository(Silent); + repo.Save(root).GetAwaiter().GetResult(); + return path; + } + + /// Load a catalog from disk through the real production load path. + public static RootEntry Load(string path) + { + using var repo = new CatalogRepository(Silent); + return repo.LoadDirCacheAsync(path).GetAwaiter().GetResult(); + } + + public static void TryDelete(string path) + { + try + { + if (path != null && File.Exists(path)) File.Delete(path); + } + catch (IOException) + { + // Best-effort temp cleanup; ignore. + } + } +} diff --git a/src/cdeBenchmarks/CatalogLoadBenchmarks.cs b/src/cdeBenchmarks/CatalogLoadBenchmarks.cs new file mode 100644 index 0000000..1ec58a7 --- /dev/null +++ b/src/cdeBenchmarks/CatalogLoadBenchmarks.cs @@ -0,0 +1,46 @@ +using BenchmarkDotNet.Attributes; +using cdeLib.Entities; + +namespace cdeBenchmarks; + +/// +/// End-to-end catalog LOAD benchmark: deserialize a real on-disk .cde through the production +/// load path (CatalogRepository.LoadDirCacheAsync → MessagePack → SetInMemoryFields). +/// +/// Measures load wall-clock and (via MemoryDiagnoser) allocations during load. Retained footprint +/// is measured separately by the standalone cdeMemProbe — BenchmarkDotNet cannot report +/// steady-state retained heap, only per-iteration allocations. +/// +/// dotnet run -c Release --filter *CatalogLoad* +/// +[MemoryDiagnoser] +[SimpleJob(warmupCount: 1, iterationCount: 3)] // Load is expensive; keep iteration count modest. +public class CatalogLoadBenchmarks +{ + private string _catalogPath = null!; + + // 10M takes real time/disk; start at 1M for routine runs and opt into 10M explicitly via filter. + [Params(1_000_000)] + public int EntryCount { get; set; } + + [Params(false, true)] + public bool WithHashes { get; set; } + + [GlobalSetup] + public void Setup() + { + _catalogPath = CatalogFixture.WriteTemp(EntryCount, WithHashes); + } + + [GlobalCleanup] + public void Cleanup() + { + CatalogFixture.TryDelete(_catalogPath); + } + + [Benchmark(Description = "Load catalog from disk")] + public RootEntry LoadCatalog() + { + return CatalogFixture.Load(_catalogPath); + } +} diff --git a/src/cdeBenchmarks/Hash16Benchmarks.cs b/src/cdeBenchmarks/Hash16Benchmarks.cs index 9eaa5ba..48fb1ae 100644 --- a/src/cdeBenchmarks/Hash16Benchmarks.cs +++ b/src/cdeBenchmarks/Hash16Benchmarks.cs @@ -34,9 +34,9 @@ public class Hash16Benchmarks public void Setup() { // Simulate real MD5 hashes (16 bytes each) - _hash1 = new byte[] { 0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE, 0xF0, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88 }; - _hash2 = new byte[] { 0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE, 0xF0, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88 }; - _hash3 = new byte[] { 0xFF, 0xEE, 0xDD, 0xCC, 0xBB, 0xAA, 0x99, 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11, 0x00 }; + _hash1 = [0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE, 0xF0, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + _hash2 = [0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC, 0xDE, 0xF0, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88]; + _hash3 = [0xFF, 0xEE, 0xDD, 0xCC, 0xBB, 0xAA, 0x99, 0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11, 0x00]; _structHash1 = new Hash16(_hash1); _structHash2 = new Hash16(_hash2); @@ -44,10 +44,10 @@ public void Setup() // Pre-populate dictionary for lookup benchmarks _hashDictionary = new Dictionary>(IterationCount); - for (int i = 0; i < IterationCount; i++) + for (var i = 0; i < IterationCount; i++) { var hash = new Hash16(CreateVariedHash(i)); - _hashDictionary[hash] = new List { $"file_{i}.txt" }; + _hashDictionary[hash] = [$"file_{i}.txt"]; } } @@ -95,10 +95,10 @@ public bool EqualityComparison() [Benchmark(Description = "Dictionary lookups with Hash16 keys")] public int DictionaryLookup() { - int count = 0; + var count = 0; var lookupHash = new Hash16(CreateVariedHash(IterationCount / 2)); - for (int i = 0; i < 100; i++) + for (var i = 0; i < 100; i++) { if (_hashDictionary.TryGetValue(lookupHash, out var files)) { @@ -120,12 +120,12 @@ public Dictionary> DictionaryInsertion() { var dict = new Dictionary>(IterationCount); - for (int i = 0; i < IterationCount; i++) + for (var i = 0; i < IterationCount; i++) { var hash = new Hash16(CreateVariedHash(i)); if (!dict.TryGetValue(hash, out var list)) { - list = new List(); + list = []; dict[hash] = list; } list.Add($"file_{i}.txt"); diff --git a/src/cdeBenchmarks/MultiCatalogSearchBenchmarks.cs b/src/cdeBenchmarks/MultiCatalogSearchBenchmarks.cs new file mode 100644 index 0000000..19f2ffe --- /dev/null +++ b/src/cdeBenchmarks/MultiCatalogSearchBenchmarks.cs @@ -0,0 +1,67 @@ +using System.Runtime.CompilerServices; +using BenchmarkDotNet.Attributes; +using cdeLib; +using cdeLib.Entities; +using cdeMemProbe; + +namespace cdeBenchmarks; + +/// +/// Sync (production) vs legacy async find across DIFFERENT catalog counts at a fixed ~1M total +/// entries. Answers: does the synchronous path (parallel across roots) still win when there are +/// many catalogs (e.g. 100), where the work-stealing async path could parallelize within roots too? +/// +/// dotnet run -c Release --filter *MultiCatalog* +/// +[MemoryDiagnoser] +[SimpleJob(warmupCount: 2, iterationCount: 5)] +public class MultiCatalogSearchBenchmarks +{ + private const int TotalEntries = 1_000_000; + + private IList _roots = null!; + + // 1 = single big catalog; 100 = the user's many-catalogs scenario (10k entries each). + [Params(1, 10, 100)] + public int RootCount { get; set; } + + [GlobalSetup] + public void Setup() + { + var perRoot = TotalEntries / RootCount; + var roots = new List(RootCount); + for (var i = 0; i < RootCount; i++) + { + // Distinct seed per root so the catalogs differ, like a real multi-drive load. + roots.Add(SyntheticCatalog.Generate(perRoot, withHashes: false, seed: 1000 + i)); + } + _roots = roots; + } + + private FindOptions MakeOptions(StrongBox counter) => new() + { + Pattern = "zzzznomatchzzzz", // full scan — measures pure traversal throughput + RegexMode = false, + IncludePath = false, + IncludeFiles = true, + IncludeFolders = true, + LimitResultCount = int.MaxValue, + VisitorFunc = (_, _) => { Interlocked.Increment(ref counter.Value); return true; }, + }; + + [Benchmark(Baseline = true, Description = "sync Find (parallel across roots)")] + public int Sync() + { + var c = new StrongBox(); + MakeOptions(c).Find(_roots); + return c.Value; + } + + [Benchmark(Description = "legacy async FindAsync (work-stealing)")] + public int Async() + { + var c = new StrongBox(); + MakeOptions(c).FindAsync(_roots).GetAwaiter().GetResult(); + return c.Value; + } +} diff --git a/src/cdeBenchmarks/PoolingBenchmarks.cs b/src/cdeBenchmarks/PoolingBenchmarks.cs index 18f01d9..a239874 100644 --- a/src/cdeBenchmarks/PoolingBenchmarks.cs +++ b/src/cdeBenchmarks/PoolingBenchmarks.cs @@ -188,16 +188,14 @@ public int StackPooling() for (var j = 0; j < 10; j++) { - if (stack.Count > 0) - { - stack.Pop(); - totalCount++; + if (stack.Count <= 0) continue; + stack.Pop(); + totalCount++; - // Push more items - var child = new DirEntry(false); - child.SetPath($"child_{j}"); - stack.Push(child); - } + // Push more items + var child = new DirEntry(false); + child.SetPath($"child_{j}"); + stack.Push(child); } CollectionPool.ReturnCommonEntryStack(stack); @@ -256,7 +254,7 @@ public int StringListPooling() for (var j = 0; j < 10; j++) { - list.Add($"path\\to\\file_{j}.txt"); + list.Add($@"path\to\file_{j}.txt"); } totalCount += list.Count; diff --git a/src/cdeBenchmarks/SearchBenchmarks.cs b/src/cdeBenchmarks/SearchBenchmarks.cs new file mode 100644 index 0000000..f34199b --- /dev/null +++ b/src/cdeBenchmarks/SearchBenchmarks.cs @@ -0,0 +1,93 @@ +using System.Runtime.CompilerServices; +using BenchmarkDotNet.Attributes; +using cdeLib; +using cdeLib.Entities; + +namespace cdeBenchmarks; + +/// +/// Search (find) throughput across the matrix {substring, regex} × {name, path}, against a fixed +/// in-memory synthetic catalog loaded once. The visitor only counts matches (no console I/O) so the +/// measurement reflects traversal + matching cost, not output. +/// +/// Patterns are chosen with known hit-rates: +/// *NoMatch – 0 hits → pure full-scan + match throughput (worst case, most representative of cost) +/// *Common – ~8% hits (".txt") → includes per-match visitor + full-path-build cost +/// +/// dotnet run -c Release --filter *Search* +/// +[MemoryDiagnoser] +[SimpleJob(warmupCount: 2, iterationCount: 5)] +public class SearchBenchmarks +{ + private IList _roots = null!; + + [Params(1_000_000)] + public int EntryCount { get; set; } + + [GlobalSetup] + public void Setup() + { + // Hashes are irrelevant to search cost; use the no-hash fixture. + _roots = new List { CatalogFixture.BuildInMemory(EntryCount, withHashes: false) }; + } + + private FindOptions MakeOptions(string pattern, bool regexMode, bool includePath, StrongBox counter) + => new() + { + Pattern = pattern, + RegexMode = regexMode, + IncludePath = includePath, + IncludeFiles = true, + IncludeFolders = true, + LimitResultCount = int.MaxValue, + VisitorFunc = (_, _) => + { + Interlocked.Increment(ref counter.Value); + return true; + }, + }; + + // Production path: synchronous Find (plain bool delegate, no per-entry async state machine). + // FindService routes the CLI through this as of Phase 1. + private int RunFind(string pattern, bool regexMode, bool includePath) + { + var counter = new StrongBox(); + MakeOptions(pattern, regexMode, includePath, counter).Find(_roots); + return counter.Value; + } + + // Legacy work-stealing async path, kept only to track the gap that motivated the Phase 1 switch. + private int RunFindAsyncLegacy(string pattern, bool regexMode, bool includePath) + { + var counter = new StrongBox(); + MakeOptions(pattern, regexMode, includePath, counter).FindAsync(_roots).GetAwaiter().GetResult(); + return counter.Value; + } + + [Benchmark(Description = "substring, name, no match (full scan)")] + public int SubstringNameNoMatch() => RunFind("zzzznomatchzzzz", regexMode: false, includePath: false); + + [Benchmark(Description = "substring, name, ~8% hits (.txt)")] + public int SubstringNameCommon() => RunFind(".txt", regexMode: false, includePath: false); + + [Benchmark(Description = "substring, path, no match (full scan + path build)")] + public int SubstringPathNoMatch() => RunFind("zzzznomatchzzzz", regexMode: false, includePath: true); + + [Benchmark(Description = "substring, path, ~8% hits (.txt)")] + public int SubstringPathCommon() => RunFind(".txt", regexMode: false, includePath: true); + + [Benchmark(Description = "regex, name, no match (full scan)")] + public int RegexNameNoMatch() => RunFind("zzzz[0-9]nomatch", regexMode: true, includePath: false); + + [Benchmark(Description = "regex, path, ~8% hits (\\.txt$)")] + public int RegexPathCommon() => RunFind(@"\.txt$", regexMode: true, includePath: true); + + // --- Legacy async path (deprecated for the CLI in Phase 1; kept to track the gap) --- + + [Benchmark(Description = "LEGACY-async substring, name, no match")] + public int LegacySubstringNameNoMatch() => RunFindAsyncLegacy("zzzznomatchzzzz", regexMode: false, includePath: false); + + [Benchmark(Description = "LEGACY-async substring, path, no match")] + public int LegacySubstringPathNoMatch() => RunFindAsyncLegacy("zzzznomatchzzzz", regexMode: false, includePath: true); +} diff --git a/src/cdeBenchmarks/baseline/README.md b/src/cdeBenchmarks/baseline/README.md new file mode 100644 index 0000000..efe743f --- /dev/null +++ b/src/cdeBenchmarks/baseline/README.md @@ -0,0 +1,64 @@ +# Memory & Search Baseline (Phase 0) + +This directory holds the committed baseline that every memory/search optimization phase is +measured against. The goal of the work: **reduce the in-memory footprint of a loaded catalog +(primary)** and **improve search performance (secondary)**. A catalog format-version bump is +permitted in later phases. + +All phases must re-run the *same* harness against the *same* synthetic fixture (same seed) so +deltas are attributable to the change under test, not to fixture drift. + +## Harness + +| Tool | Project | Measures | +|------|---------|----------| +| `cdeMemProbe` | `src/cdeMemProbe` | **Retained** managed heap + working set after load, bytes/entry, load wall-clock | +| `CatalogLoadBenchmarks` | `src/cdeBenchmarks` | Load wall-clock + allocations (BenchmarkDotNet) | +| `SearchBenchmarks` | `src/cdeBenchmarks` | Find throughput across {substring,regex} × {name,path} | +| `SyntheticCatalog` | `src/cdeMemProbe` | Shared deterministic fixture generator (seed 42, ~50:1 file:dir) | + +`cdeMemProbe` is a separate minimal process on purpose: BenchmarkDotNet's `MemoryDiagnoser` +reports *allocations during a run*, not the *steady-state retained heap* — which is the headline +metric here. The probe loads one catalog, settles the GC (`Collect` ×2 + `WaitForPendingFinalizers`), +then reports `GC.GetTotalMemory(true)` and process working/private set. + +## Reproduce + +```powershell +# Build +dotnet build src/cdeMemProbe/cdeMemProbe.csproj -c Release +dotnet build src/cdeBenchmarks/cdeBenchmarks.csproj -c Release + +# Footprint: generate a fixture, then measure it in a clean process +$probe = "src/cdeMemProbe/bin/Release/net10.0/cdeMemProbe.dll" +dotnet $probe --generate 1000000 --out $env:TEMP\fix-1m.cde +dotnet $probe $env:TEMP\fix-1m.cde # add --hashes to --generate for the hashed variant + +# Load timing + allocations +dotnet run --project src/cdeBenchmarks -c Release -- --filter *CatalogLoad* + +# Search timings +dotnet run --project src/cdeBenchmarks -c Release -- --filter *Search* +``` + +## Baseline results — 2026-06-06 + +Machine: Windows 11, .NET 10, Server GC. Synthetic fixture, seed 42, ~50 files per directory. +Raw footprint rows in [`footprint-baseline.csv`](./footprint-baseline.csv). + +### Footprint (retained managed heap after load) + +| Entries | Hashes | Managed heap | Working set | **Bytes / entry** | Load ms | +|--------:|:------:|-------------:|------------:|------------------:|--------:| +| 1,000,000 | no | 202.1 MB | 356.5 MB | **211.94** | 1385 | +| 1,000,000 | yes | 207.0 MB | 373.8 MB | **217.07** | 1290 | +| 10,000,000 | no | 2018.7 MB | 2887.0 MB | **211.67** | 8447 | + +Key observation: hashed vs no-hash footprint is nearly identical (+~5 B/entry), because the +16-byte `Hash16` struct is stored **inline on every entry whether or not a hash is set**. This is +the direct evidence behind the Phase 3 plan to move hashes off-entry into a side table +(~16 B/entry reclaimable in the common no-hash case). + +### Search timings (1,000,000-entry fixture) + +See [`search-baseline.md`](./search-baseline.md) for the full BenchmarkDotNet table. diff --git a/src/cdeBenchmarks/baseline/flatbuffer-mmap-spike.md b/src/cdeBenchmarks/baseline/flatbuffer-mmap-spike.md new file mode 100644 index 0000000..8e9ff1c --- /dev/null +++ b/src/cdeBenchmarks/baseline/flatbuffer-mmap-spike.md @@ -0,0 +1,74 @@ +# Spike: zero-copy columnar mmap catalog format + +Branch `spike/flatbuffer-mmap-search`. Goal: prove (or kill) the idea of replacing in-memory +catalog load with a **columnar struct-of-arrays file read zero-copy over a memory map**, so that +"load" becomes `mmap` and search streams over the mapping — managed heap ≈ 0, working set ≈ pages +actually touched (reclaimable OS page cache). + +Spike code lives in `src/cdeMemProbe/Columnar/` (`ColumnarFormat` writer, `ColumnarReader` mmap +reader) plus `--migrate` / `--flat` modes in `Program.cs`. It is intentionally **not** in +production `cdeLib` until the thesis was confirmed. Format magic `CDEX`, version 1. + +## Result: thesis confirmed, decisively + +Real catalog `C-System-C__program files.cde`, **649,377 entries**: + +| Model | Managed heap | B/entry (heap) | "Load"/open | Notes | +|---|---:|---:|---:|---| +| Tree (MessagePack, current) | 82.3 MB | 126.7 | 1123 ms | full pointer graph | +| In-memory SoA store | 43.8 MB | 67.4 | 1050 ms | parallel arrays on heap | +| **Columnar mmap** | **0.07 MB** | **0.11** | **3 ms** | catalog stays in the mapping | + +- **Managed heap ~0.** 72 KB total — just the reader object + tiny scan buffers. ~600× below the + store, ~1100× below the tree. The catalog never enters the GC heap. This is the primary goal + (footprint) hit as hard as it can be hit. +- **"Load" is ~free:** 3 ms to mmap vs ~1100 ms to deserialize. ~370× faster, and independent of + catalog size (no parse). +- **Search is zero-alloc:** a full name byte-scan of 649 K entries allocated **40 bytes** total + (path-walk variant 1088 B, from one growable buffer). Confirms UTF-8 byte-matching avoids the + FlatSharp lazy-string trap. +- **Search speed (single-threaded):** name `.dll` → 131 958 hits in **26.9 ms** (~24 M entries/s); + path `common` → 59 011 hits in **287 ms** (slower: rebuilds each full path). Both parallelize by + index range; path mode has obvious caching wins left on the table. + +Hashed 100 K synthetic round-trips correctly (Hash16 column is blittable); allocations still 40 B. + +## Honest costs / caveats + +1. **Disk +36 %** (35.7 MB vs 26.3 MB MessagePack). Causes: full names stored with repeated + extensions (no dedup), fixed-width 8-aligned columns, no varint packing. Mitigations: dedup + `Ext` via a shared-string column, optional per-column compression. Zero-copy wants fixed-width + numeric columns, so the numeric side stays uncompressed; the win is on the name blob. +2. **Working set ≈ touched pages.** A *full* scan touches the whole NameBlob, so working set rises + toward file size — but these are clean, file-backed, **reclaimable** pages (evictable, shared), + not committed GC heap. Selective queries and column-skipping (a name query never pages in + Size/Modified/Hash) keep it well below file size. The number that matters for GC pressure / + OOM — managed heap — is ~0 regardless. +3. **ASCII case-folding only** in the spike matcher. Production needs proper ordinal-ignore-case + over UTF-8 (or a stored case-folded name column). +4. **`int` entry count + `int` name offsets** cap at ~2.1 B entries / 2 GB name blob. The README + claims "billions"; production must widen offsets to `long` (and the count) to keep headroom. +5. **mmap lifetime / immutability.** The mapping must stay open while a catalog is "loaded"; + the buffer is read-only, so `hash`/`scan`/`dupes` write a *fresh* file (they already re-save). +6. **Hand-rolled, not FlatBuffers.** For dense homogeneous columns a hand-rolled layout gives a + genuinely alloc-free `MemoryMarshal.Cast` view with no vtable/string-materialization overhead; + FlatBuffers earns its keep on sparse/optional schemas, which a catalog is not. Same zero-copy + goal the user asked for, better fit for the data shape. + +## Reproduce + +``` +cdeMemProbe ".cde" # tree heap +cdeMemProbe ".cde" --store # in-memory SoA heap +cdeMemProbe --migrate ".cde" --out cat.cdex +cdeMemProbe --flat cat.cdex --pattern .dll # name search, zero-alloc +cdeMemProbe --flat cat.cdex --pattern common --path # path search +``` + +## Recommendation + +Promote to production behind a format-version gate: add the columnar writer/reader to `cdeLib`, +make `EntryRef : ICommonEntry` a view over `(buffer, offset)` (the seam already exists from the +SoA work), load `find`/GUI/`dump` straight off the mapping, and ship a one-way `cde migrate` +verb (the `--migrate` mode here is the prototype). `scan`/`hash`/`dupes` keep building a tree and +write the columnar file at save. Address caveats 3 & 4 before shipping. diff --git a/src/cdeBenchmarks/baseline/footprint-baseline.csv b/src/cdeBenchmarks/baseline/footprint-baseline.csv new file mode 100644 index 0000000..8cc9487 --- /dev/null +++ b/src/cdeBenchmarks/baseline/footprint-baseline.csv @@ -0,0 +1,4 @@ +file,entries,loadMs,managedBytes,peakWorkingSet,privateBytes,bytesPerEntry +cde-base-1000000.cde,1000000,1385,211940976,373788672,373096448,211.94 +cde-base-1000000-h.cde,1000000,1290,217069152,391979008,395849728,217.07 +cde-base-10000000.cde,10000000,8447,2116746320,3027185664,3121905664,211.67 diff --git a/src/cdeBenchmarks/baseline/name-storage-investigation.md b/src/cdeBenchmarks/baseline/name-storage-investigation.md new file mode 100644 index 0000000..df354ab --- /dev/null +++ b/src/cdeBenchmarks/baseline/name-storage-investigation.md @@ -0,0 +1,62 @@ +# Investigation: entry-name memory (UTF-8 / name-pool) + +## Measured: names are the dominant per-entry cost + +1M-entry fixture, no hashes, after Phases 1+3: + +| Name storage | Bytes/entry | Note | +|--------------|------------:|------| +| normal (~16-char unique names) | 198.85 | current | +| shared single 1-char name | **129.23** | isolates everything-except-name-strings | +| **name contribution** | **≈ 69.6 B/entry** | **~35% of total footprint** | + +Reproduce: `cdeMemProbe --generate 1000000 [--shared-names] --out x.cde` then `cdeMemProbe x.cde`. + +Names are stored today as two interned UTF-16 `string` objects per entry (`_path` = name without +extension, `field` = extension). For a 16-char ASCII name the `_path` object is ~54 B (16 B header + +4 B length + 32 B chars + 2) plus the 8 B reference in the entry. The extension is interned and +shared, so its amortised cost is small; almost all of the ~70 B is the per-file `_path` string. + +Two structural costs make this large: (a) UTF-16 spends **2 bytes per char** for names that are +overwhelmingly ASCII, and (b) every name is a **separate heap object** carrying a ~22 B header. + +## Option A — UTF-8 `byte[]` per name + +Store `_path` as a UTF-8 `byte[]` instead of a `string`. +- **Saves** the char halving only: ~16 B for a 16-char ASCII name → **est. ~15–20 B/entry** (~8–10%). +- **Keeps** the per-name object header (a `byte[]` header ≈ a `string` header). +- **Risk / work (high, hot path):** `DirEntry.Path` is read by the *hottest* operations — find + `Contains`/regex, `PathCompareWithDirTo` sort, path building, GUI display. If the getter + reconstructs a `string`, every access allocates and **search regresses massively** (we just made + it 48×/18× faster). Avoiding that means rewriting matching/sorting to work on `byte` spans: + - substring search → UTF-8 byte `IndexOf` (fine for ASCII; **case-insensitive non-ASCII is hard**). + - ordinal-ignore-case sort → byte compare (ASCII ok; Unicode ordering differs). + - regex → needs a `string`; would still allocate (or a UTF-8 regex engine). + - loses `string.Intern` dedup of repeated filenames (e.g. `index.html`, `__init__.py` recur a lot) + unless a `byte[]` dedup pool is added. +- **Format:** `Path` (Key 5) would serialize as bytes → `.cde` format bump (also shrinks the file). + +## Option B — name pool (offset into a shared UTF-8 buffer) ← biggest win + +Hold all names of a catalog in one big UTF-8 `byte[]` per `RootEntry`; each entry stores an +`int` offset + `short` length instead of string references. +- **Eliminates the per-name object header entirely** *and* halves char bytes. +- Per file: ~6 B (offset+len) in the entry + ~16 B in the pool = **~22 B vs ~62–70 B today → + est. ~40 B/entry saved (~20% of footprint)**. Largest lever available, by far. +- **Risk / work (very high):** same hot-path byte-matching rewrite as Option A, **plus** a pool + built at load, pool growth/lifetime management, dedup strategy (replacing interning), and a + bigger `.cde` format change. Touches lib + GUI + web + dupes. + +## Recommendation + +Names are the single biggest remaining memory lever (~35%), so the upside is real and larger than +the file/dir split. But both options rewrite the **hottest** code (the same name comparison that the +Phase 1–2 search wins depend on) and change the `.cde` format, with genuine Unicode-correctness +pitfalls in case-insensitive matching. This is a larger, riskier effort than Phases 1–3 combined and +warrants its own plan + prototype. + +Suggested path if pursued: prototype **Option A** behind the existing benchmarks first — convert +`_path` to UTF-8 with byte-based substring/sort matching — and measure BOTH the footprint delta and +the search-time delta on the 1M fixture before deciding whether the saving justifies the hot-path +risk. Only escalate to **Option B** (the pool) if the measured Option-A search impact is acceptable +and the extra ~20 B/entry is needed. diff --git a/src/cdeBenchmarks/baseline/phase-results.md b/src/cdeBenchmarks/baseline/phase-results.md new file mode 100644 index 0000000..be56a09 --- /dev/null +++ b/src/cdeBenchmarks/baseline/phase-results.md @@ -0,0 +1,131 @@ +# Phase Results vs Baseline + +Tracks the measured delta of each optimization phase against the committed Phase 0 baseline +(`footprint-baseline.csv`, `search-baseline.md`). Same fixture, same harness, every phase. + +--- + +## Phase 1 — free non-breaking wins (no format change) + +**Changes shipped** +- **S3** — `FindService` (the CLI `cde find` path) now runs the synchronous `FindOptions.Find` + instead of the work-stealing `FindAsync`. The async path ran every entry through an + `async Task` state machine plus `Task.Yield()`/`Task.Delay(0)` every 500/2000 entries. + (`src/cdeLib/FindService.cs`) +- **R1a** — `FileEntryCount` / `DirEntryCount` changed `long → uint` on `DirEntry`, `RootEntry`, + `ICommonEntry`. Values were already `(uint)`-truncated at assignment, so semantics are + unchanged; in-memory only, no catalog format impact. + +**Re-sequenced (not done in Phase 1)** +- **S1** (path-build allocation, the ~973 MB below) → moved to **Phase 2**, where carrying the + running path down the traversal stack makes it allocation-free without duplicated work. +- **S2** (`SearchValues` name prefilter) → dropped as low value: name search is already ~35 ms, + and its remaining 52 MB is the `DirEntry.Path` getter re-joining the split extension, which a + prefilter would not remove. + +### Search (1,000,000-entry fixture) + +| Query | Baseline (async) | Phase 1 (sync) | Speedup | +|-------|-----------------:|---------------:|--------:| +| substring, name, no match | 1746 ms | **36.3 ms** | 48× | +| substring, name, ~8% (.txt) | 1722 ms | **35.4 ms** | 49× | +| substring, path, no match | 1755 ms | **235.6 ms** | 7.4× | +| substring, path, ~8% (.txt) | 1774 ms | **243.0 ms** | 7.3× | +| regex, name, no match | 1817 ms | **59.6 ms** | 30× | +| regex, path, ~8% (`\.txt$`) | 1828 ms | **253.1 ms** | 7.2× | + +Allocation is unchanged by Phase 1 (name 52.7 MB, path 972.7 MB) — that is S1, targeted in Phase 2. +The retained `LEGACY-async*` benchmarks still measure ~1710–1764 ms, confirming the old path. + +### Footprint (retained managed heap) + +| Entries | Hashes | Baseline B/entry | Phase 1 B/entry | Saved | +|--------:|:------:|-----------------:|----------------:|------:| +| 1,000,000 | no | 211.94 | **203.94** | 8.00 B/entry | + +Exactly the predicted 8 bytes/entry (two `long`→`uint`). At 1M entries: 202.1 MB → 194.5 MB managed. + +### Correctness +- `cdeLibTest`: 126 passed, 0 failed (7 skipped). +- Full solution (`cde.slnx`) builds clean. +- Note: `cdeLibSpec` / `cdeLibSpec2` are legacy `net48` projects incompatible with the `net10` + library and do not restore — pre-existing, unrelated to this work, not in the solution. + +--- + +## Phase 2 — allocation-free path search (S1), no format change + +**Changes shipped** +- Reimplemented `EntryHelper.MakeFullPath` as a single pass that walks the `ParentCommonEntry` + chain into the shared `StringBuilder`, instead of the old recursion that allocated a fresh + full-path string at every ancestor level. All path-building callers (find, dupes, GUI, dump) + benefit. (`src/cdeLib/Entities/EntryHelper.cs`) +- Added `EntryHelper.FullPathContains` — builds the path into a pooled `char[]` and matches over a + `Span` (`MemoryExtensions.Contains`), allocating no result string. Wired into the find + substring path matcher. (`src/cdeLib/Entities/EntryHelper.cs`, `src/cdeLib/FindOptions.cs`) + +**Re-sequencing note:** the original Phase 2 also dropped `ParentCommonEntry` (~8 B/entry). Code +evidence shows `.FullPath` / `GetListFromRoot` are called only on **directories** or via +`PairDirEntry` (which carries an explicit parent) — files never need a standalone parent pointer. +The cdeWin GUI depends heavily on directory parent pointers, so removing the field from *all* +entries is a large, risky GUI refactor for the same 8 B that R1a already delivered safely. The +parent pointer can only be reclaimed from **files**, which requires the file/dir type split — so +`ParentCommonEntry` removal folds into **Phase 3**, validated by the split. + +### Search (1,000,000-entry fixture) — cumulative + +| Query | Baseline | Phase 1 | Phase 2 | Alloc (base → P2) | +|-------|---------:|--------:|--------:|------------------:| +| substring, path, no match | 1755 ms | 236 ms | **101.6 ms** | 978.7 MB → **52.7 MB** | +| substring, path, ~8% (.txt) | 1774 ms | 243 ms | **96.4 ms** | 978.7 MB → **52.7 MB** | +| regex, path, ~8% (`\.txt$`) | 1828 ms | 253 ms | **136 ms** | 1349 MB → **265 MB** | + +Path-search allocation cut ~94.6% (residual 52.7 MB is the per-file `DirEntry.Path` extension +rejoin, same as name search). Name-search numbers are unchanged from Phase 1. + +### Correctness +- `cdeLibTest`: 126 passed (incl. all `EntryHelper` / `GetListFromRoot` / `RootEntry` path tests + that assert exact path strings — confirms the single-pass build is identical). +- Full solution builds clean. + +--- + +## Phase 3 — lean files via `ExtraData` side-object (no format change) + +**Change shipped** +- `DirEntry` moves its directory-only fields (`Children` + the two summary counts) into a + lazily-allocated private `ExtraData` object, allocated only for directories. A file — the vast + majority of entries — now carries a single 8-byte `_extra` reference (null) instead of an + always-null `Children` ref plus two count fields. `Hash` deliberately stays **inline** (moving it + off-entry forces every hashed file to allocate an `ExtraData` whose header costs more than the + 16-byte `Hash16`, a net regression for hashed catalogs — measured at +16 B/entry). No catalog + format change: `Children` keeps Key 3 via the property; counts are `[IgnoreMember]`. + (`src/cdeLib/Entities/DirEntry.cs`) + +### Footprint (retained managed heap, 1M fixture, ~10:1 file:dir) + +| Catalog | Phase 1 (post-R1a) | Phase 3 | Saved | +|---------|-------------------:|--------:|------:| +| no hashes | 203.94 | **198.85** | 5.09 B/entry | +| hashed | (≈209 post-R1a) | **203.98** | no regression | + +Round-trip verified: the serialized `.cde` is **byte-identical** to the baseline (36,668,199 bytes), +confirming the format is unchanged. Find shows no CPU regression (name 36.7 ms, path 95.0 ms). + +### Honest accounting of the win vs the planned split +The original Phase 3 was a polymorphic file/dir type split + hash side-table (format bump). The +`ExtraData` approach delivers the same *memory outcome direction* (lean files) **safely and with no +format change**, but a side-object has two costs a true type split avoids: every file still keeps an +8-byte `_extra` reference, and every directory pays a 16-byte object header. At this fixture's 10:1 +ratio that nets ~5 B/entry; at a real drive's ~6:1 (per the project's own notes) it is smaller still. + +The full polymorphic split (files as a lean type with no dir fields and no per-file ref; directories +with the fields inline and no extra object) would save ~3× more, but requires making `DirEntry` an +abstract MessagePack-union root, changing `IList` throughout lib + GUI + web + tests, +rewriting ~40 construction sites, a `.cde` format bump with migration, and FlatSharp/protobuf union +handling — a large change whose WinForms GUI behavior cannot be runtime-validated in this +environment (only via presenter unit tests). Left as an explicit, opt-in follow-up. + +### Correctness +- `cdeLibTest`: 126 passed; `cdeWinTest` (GUI presenter): 32 passed. +- Round-trip byte-identical; no find CPU regression. diff --git a/src/cdeBenchmarks/baseline/search-baseline.md b/src/cdeBenchmarks/baseline/search-baseline.md new file mode 100644 index 0000000..1719032 --- /dev/null +++ b/src/cdeBenchmarks/baseline/search-baseline.md @@ -0,0 +1,35 @@ +# Search Baseline — 2026-06-06 + +BenchmarkDotNet, .NET 10, Server GC. Fixture: 1,000,000-entry synthetic catalog (seed 42), +in memory. Driven through the production path `FindOptions.FindAsync` (what `FindService` / +`cde find` uses). Visitor counts matches only — no console I/O. + +| Method | Mean | StdDev | Allocated | +|--------|-----:|-------:|----------:| +| substring, name, no match (full scan) | 1.746 s | 0.043 s | 58.7 MB | +| substring, name, ~8% hits (.txt) | 1.722 s | 0.007 s | 58.7 MB | +| substring, path, no match (full scan + path build) | 1.755 s | 0.026 s | 978.7 MB | +| substring, path, ~8% hits (.txt) | 1.774 s | 0.012 s | 978.7 MB | +| regex, name, no match (full scan) | 1.817 s | 0.079 s | 460.9 MB | +| regex, path, ~8% hits (`\.txt$`) | 1.828 s | 0.006 s | 1349.0 MB | + +## What this reveals (drives Phase 1) + +- **Path search allocates ~0.98–1.35 GB per single 1M-entry query.** `GetPatternMatcher` builds a + full-path string (`EntryHelper.MakeFullPathPooled`) for *every candidate* — and for a no-match + query that's every entry. **S1**: match over pooled `Span` (`MemoryExtensions.Contains`) + instead of allocating a string per entry; pass the already-built path to the visitor on a match + rather than rebuilding it in `FindService.FindAsync`. Target: path allocation → near-zero. + +- **Even name-only, no-match scan is ~1.7 s and allocates 58.7 MB** for what should be an + allocation-free linear scan. The cost is the per-entry `async Task` processor + (`CreateAsyncProcessor`) plus `Task.Yield()` / `Task.Delay(0)` every 500 / 2000 entries. The + synchronous `Find` path (`GetFindFunc`, a plain `bool` delegate) has none of this. **S3**: route + the CLI through the synchronous path (or make parallel/serial adaptive) and drop per-entry yields. + +- Mean time is dominated by async overhead, not matching: name-no-match (58 MB alloc) and + path-no-match (978 MB alloc) have nearly identical ~1.75 s means. Cutting the async overhead + should move the needle more than matching micro-opts. + +These numbers are the bar Phase 1 must beat. Re-run with +`dotnet run --project src/cdeBenchmarks -c Release -- --filter *Search*`. diff --git a/src/cdeBenchmarks/baseline/soa-prototype.md b/src/cdeBenchmarks/baseline/soa-prototype.md new file mode 100644 index 0000000..6422fe7 --- /dev/null +++ b/src/cdeBenchmarks/baseline/soa-prototype.md @@ -0,0 +1,56 @@ +# Prototype: Struct-of-Arrays (SoA) EntryStore + +Feasibility + footprint measurement for replacing the pointer-based `DirEntry`/`RootEntry` tree +with parallel value-type arrays (one slot per entry; tree shape via `int` firstChild/nextSibling/ +parent indices). Prototype: `src/cdeMemProbe/EntryStore.cs`. Measure: +`cdeMemProbe --soa --generate N [--shared-names] [--hashes]`. + +## Measured footprint (1M & 10M fixtures) + +| Model | Structural (shared names) | Full (normal names) | +|-------|--------------------------:|--------------------:| +| Pointer tree (current) | 129.23 B/entry | 198.85 B/entry | +| **SoA prototype** | **42.23 B/entry** | 151.39 B/entry | +| **Saving** | **−87 B/entry (−67%)** | −47 B/entry (−24%) | + +- Scales linearly: SoA at 10M = **42.11 B/entry** (421 MB) vs the tree's ~1.29 GB of structural + memory — i.e. SoA holds the same 10M-entry catalog structure in **~1/3 the memory**. +- The "full / normal names" SoA number is inflated only because the prototype stores **un-interned, + un-split** full names. A production SoA would intern + split names like the tree does, landing the + total near **~112 B/entry (≈ −44%)**. The clean, name-independent result is the structural row. +- **Hashed catalogs:** the prototype's `--hashes` run did *not* allocate the `Hash16[]` side array + (the `root.IsHashDone` heuristic is wrong — only files are hashed), so it reads 42 B. A real hashed + SoA adds 16 B/entry → **~58 B/entry**, still far below the tree's ~145 B/entry hashed structural. + +## Why it wins + +Per entry, the tree pays a 16 B object header + 8 B-each reference fields + allocation rounding + +per-directory `List` overhead. SoA pays only packed array slots: +`long modified (8) + long size (8) + string name-ref (8) + byte flags (1) + int firstChild/nextSibling/parent (4+4+4)` +≈ **37 B + negligible per-array overhead** — no per-entry header, references shrink 8 B → 4 B. + +Validated working in the prototype: linear name-substring search over `Name[]` and full-path +reconstruction by walking `Parent[]` both produce correct results. + +## What a production migration would require (large, multi-day, high risk) + +The prototype proves the memory win; shipping it is a core-model rewrite: +- **Serialization** — read/write the arrays (or convert tree↔SoA at load/save). FlatSharp (already + wired) suits SoA well; format bump. +- **Search / sort / dupes / hashing** — operate on indices instead of objects. Likely *faster* + (cache locality), but every algorithm is re-pointed; hashing writes back into `Hash[]`. +- **GUI (cdeWin) + web** — navigate via `ICommonEntry`/`FullPath`/`GetListFromRoot`. Bridge with a + thin `readonly struct EntryRef(store, index) : ICommonEntry` adapter to minimize churn, or rewrite. +- **Construction sites + tests** — the ~40 `new DirEntry(...)` sites and the entity tests. + +Risk is highest of all options (it is *the* data model, on the billions-of-entries hot paths) and +the WinForms GUI can't be runtime-validated here. But the payoff is the largest by far: ~−67% +structural memory, the only option that removes the per-entry object header. + +## Recommendation + +SoA is decisively the highest-impact memory lever — proven, not estimated. Worth doing **if** the +team is prepared for a core-model migration. Suggested phasing: (1) land the SoA store + tree↔SoA +converter + a `EntryRef` adapter behind the existing `ICommonEntry` API so the GUI/dupes keep +working; (2) move search/serialization onto the store; (3) drop the tree. Each phase measured against +this prototype. diff --git a/src/cdeBenchmarks/cdeBenchmarks.csproj b/src/cdeBenchmarks/cdeBenchmarks.csproj index 693f219..149b349 100644 --- a/src/cdeBenchmarks/cdeBenchmarks.csproj +++ b/src/cdeBenchmarks/cdeBenchmarks.csproj @@ -11,10 +11,18 @@ + + + + + diff --git a/src/cdeLib/Catalog/CatalogRepository.cs b/src/cdeLib/Catalog/CatalogRepository.cs index 6baec25..6d9deff 100644 --- a/src/cdeLib/Catalog/CatalogRepository.cs +++ b/src/cdeLib/Catalog/CatalogRepository.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Threading; using System.Threading.Tasks; using cdeLib.Entities; using cdeLib.Infrastructure; @@ -20,7 +19,7 @@ namespace cdeLib.Catalog; public sealed class CatalogRepository : ICatalogRepository, IDisposable { - private readonly SerializerProtocol _serializerProtocol = SerializerProtocol.MessagePack; // hard coded for now. + private SerializerProtocol _serializerProtocol = SerializerProtocol.MessagePack; // hard coded for now. private readonly ILogger _logger; private static readonly BufferPool BufferPool = new(); private readonly FileStreamManager _fileStreamManager = FileStreams.Instance; @@ -54,7 +53,6 @@ public RootEntry Read(string file) using (Operation.Time("Deserialize")) { var serializer = new FlatBufferSerializer(new FlatBufferSerializerOptions()); - // Use ReadOnlyMemory overload to avoid defensive copy return serializer.Parse(bytes.AsMemory()); } case SerializerProtocol.MessagePack: @@ -71,7 +69,7 @@ public RootEntry Read(string file) } } - public async Task ReadAsync(string file) + private async Task ReadAsync(string file) { try { @@ -125,7 +123,7 @@ public IList Load(IList cdeList) } _logger.Information("Catalog [{file}] read on ThreadId: {ThreadId}", file, - Thread.CurrentThread.ManagedThreadId); + Environment.CurrentManagedThreadId); }); return results.ToList(); @@ -144,6 +142,7 @@ public async Task> LoadAsync(IList cdeList) _logger.Information("Catalog [{file}] read on ThreadId: {ThreadId}", file, Environment.CurrentManagedThreadId); } + return rootEntry; }).ToList(); @@ -197,6 +196,29 @@ private static IEnumerable GetCdeFiles(string path) return FileSystemHelper.GetFilesWithExtension(path, "cde"); } + public IList GetColumnarFileList(IEnumerable paths) + { + var result = new List(); + foreach (var path in paths) + { + result.AddRange(FileSystemHelper.GetFilesWithExtension(path, "cdex")); + + foreach (var childPath in Directory.GetDirectories(path)) + { + try + { + result.AddRange(FileSystemHelper.GetFilesWithExtension(childPath, "cdex")); + } + // ReSharper disable once EmptyGeneralCatchClause + catch + { + } // if cant list folders don't care. + } + } + + return result; + } + public RootEntry LoadDirCache(string file) { if (!File.Exists(file)) return null; @@ -262,16 +284,18 @@ public async Task Save(RootEntry rootEntry) } } + /// + /// Dispose of managed resources + /// private void Dispose(bool disposing) { if (!_disposed) { if (disposing) { - // Dispose of managed resources BufferPool?.Clear(); - // Note: FileStreamManager is a singleton, don't dispose it here } + _disposed = true; } } @@ -279,6 +303,5 @@ private void Dispose(bool disposing) public void Dispose() { Dispose(true); - GC.SuppressFinalize(this); } } \ No newline at end of file diff --git a/src/cdeLib/Catalog/CreateCacheCommand.cs b/src/cdeLib/Catalog/CreateCacheCommand.cs index 6496fd7..94a2375 100644 --- a/src/cdeLib/Catalog/CreateCacheCommand.cs +++ b/src/cdeLib/Catalog/CreateCacheCommand.cs @@ -11,4 +11,10 @@ public CreateCacheCommand(string path) public string Path { get; } public string Description { get; set; } + + /// + /// When false (default), directory reparse points (junctions / symbolic links) are recorded + /// in the catalog but not descended into, avoiding cycles and duplicate content. + /// + public bool FollowJunctions { get; set; } } \ No newline at end of file diff --git a/src/cdeLib/Catalog/CreateCacheCommandHandler.cs b/src/cdeLib/Catalog/CreateCacheCommandHandler.cs index 383249f..852d347 100644 --- a/src/cdeLib/Catalog/CreateCacheCommandHandler.cs +++ b/src/cdeLib/Catalog/CreateCacheCommandHandler.cs @@ -1,7 +1,10 @@ using System; +using System.IO; using System.Threading; using System.Threading.Tasks; using cdeLib.Entities; +using cdeLib.Entities.Columnar; +using cdeLib.Entities.Soa; using cdeLib.Infrastructure.Config; using JetBrains.Annotations; using SlimMessageBus; @@ -12,15 +15,14 @@ namespace cdeLib.Catalog; public class CreateCacheCommandHandler : IRequestHandler { private readonly IConfiguration _configuration; - private readonly ICatalogRepository _catalogRepository; private readonly IMessageBus _messageBus; + private readonly OperationCancellation _cancellation; - public CreateCacheCommandHandler(IConfiguration configuration, ICatalogRepository catalogRepository, - IMessageBus messageBus) + public CreateCacheCommandHandler(IConfiguration configuration, IMessageBus messageBus, OperationCancellation cancellation) { _configuration = configuration; - _catalogRepository = catalogRepository; _messageBus = messageBus; + _cancellation = cancellation; } public async Task OnHandle(CreateCacheCommand request, CancellationToken cancellationToken) @@ -33,18 +35,25 @@ public async Task OnHandle(CreateCacheCommand request, CancellationToken cancell re.SimpleScanEndEvent = ScanEndOfEntries; re.ExceptionEvent = PrintExceptions; - re.PopulateRoot(request.Path); - if (Hack.BreakConsoleFlag) + re.PopulateRoot(request.Path, request.FollowJunctions, _cancellation.Token); + if (_cancellation.IsCancellationRequested) { Console.WriteLine(" * Break key detected incomplete scan will not be saved."); return; } - var oldRoot = _catalogRepository.LoadDirCache(re.DefaultFileName); - if (oldRoot != null) + // Catalogs are stored in the zero-copy columnar .cdex format. Reuse hashes from an existing + // .cdex (reconstructed into a tree) when one is found for this scan path. + var cdexName = Path.ChangeExtension(re.DefaultFileName, ".cdex"); + if (File.Exists(cdexName)) { - Console.WriteLine($"Found cache \"{re.DefaultFileName}\""); + Console.WriteLine($"Found cache \"{cdexName}\""); Console.WriteLine("Updating hashes on new scan from found cache file."); + RootEntry oldRoot; + using (var reader = new ColumnarCatalogReader(cdexName)) + { + oldRoot = CatalogTreeBuilder.FromSource(reader); + } oldRoot.TraverseTreesCopyHash(re); } @@ -55,11 +64,12 @@ public async Task OnHandle(CreateCacheCommand request, CancellationToken cancell re.Description = request.Description; } - await _catalogRepository.Save(re); + re.ActualFileName = cdexName; + await Task.Run(() => ColumnarFormat.Write(EntryStore.Build(re), cdexName), cancellationToken); var scanTimeSpan = re.ScanEndUtc - re.ScanStartUtc; Console.WriteLine($"Scanned path {re.Path}"); Console.WriteLine($"Scan time {scanTimeSpan.TotalMilliseconds:0.00} msecs"); - Console.WriteLine($"Saved scanned path {re.DefaultFileName}"); + Console.WriteLine($"Saved scanned path {cdexName}"); Console.WriteLine( $"Files {re.FileEntryCount:0,0} Dirs {re.DirEntryCount:0,0} Total Size of Files {re.Size:0,0} bytes"); } diff --git a/src/cdeLib/Catalog/ICatalogRepository.cs b/src/cdeLib/Catalog/ICatalogRepository.cs index e9dc2e1..9d5c1e0 100644 --- a/src/cdeLib/Catalog/ICatalogRepository.cs +++ b/src/cdeLib/Catalog/ICatalogRepository.cs @@ -18,5 +18,11 @@ public interface ICatalogRepository /// IList GetCacheFileList(IEnumerable paths); + /// + /// Gets columnar .cdex catalogs in the current dir or one directory down — the zero-copy + /// mmap format produced by cde migrate. Mirrors for .cde. + /// + IList GetColumnarFileList(IEnumerable paths); + RootEntry LoadDirCache(string file); } \ No newline at end of file diff --git a/src/cdeLib/DateTimePartialParameter.cs b/src/cdeLib/DateTimePartialParameter.cs index d6a9e2d..8d04c45 100644 --- a/src/cdeLib/DateTimePartialParameter.cs +++ b/src/cdeLib/DateTimePartialParameter.cs @@ -7,7 +7,7 @@ namespace cdeLib; /// public class DateTimePartialParameter { - private readonly string _format = "--
T::"; + private const string Format = "--
T::"; // a Parsing Expression Grammar might be a better way to do this. PEG // - http://en.wikipedia.org/wiki/Parsing_expression_grammar @@ -28,9 +28,10 @@ public DateTimePartialParameter(string str) if (year < 1000) // this is not 4 digits, its only value e.g., 4 digits 0982 is 4 digits. { _e = new ArgumentException( - $"Require Year parameter be a 4 Digit Year as part of format '{_format}'"); + $"Require Year parameter be a 4 Digit Year as part of format '{Format}'"); return; } + _year = year; if (splitOnDash.Length > 1) // may have a month specified @@ -48,12 +49,14 @@ public DateTimePartialParameter(string str) month = tmp.Month; } } - if (month == 0 || month > 12) + + if (month is 0 or > 12) { _e = new ArgumentException( - $"Require valid integer 1-12 or Month name for Month as part of format '{_format}'"); + $"Require valid integer 1-12 or Month name for Month as part of format '{Format}'"); return; } + _month = month; } @@ -68,7 +71,7 @@ public DateTimePartialParameter(string str) if (SeparatorIsNotValid(splitOnDash[2], 'T')) { _e = new ArgumentException( - $"The separator between Date and Time must be 'T' as part of format '{_format}'"); + $"The separator between Date and Time must be 'T' as part of format '{Format}'"); return; } @@ -87,14 +90,15 @@ public DateTimePartialParameter(string str) if (dayOfMonth is 0 or > 31) { _e = new ArgumentException( - $"Require valid Day of Month integer range 1-31 for Day
as part of format '{_format}'"); + $"Require valid Day of Month integer range 1-31 for Day
as part of format '{Format}'"); return; } + _dayOfMonth = dayOfMonth; if (splitOnT.Length > 1 && splitOnT[1].Length > 0) { - var t = new TimePartialParameter(splitOnT[1], _format); + var t = new TimePartialParameter(splitOnT[1], Format); _hour = t.Hour; _minute = t.Minute; _second = t.Second; @@ -111,21 +115,22 @@ private static bool SeparatorIsNotValid(string str, char validSeparator) { continue; } + if (c != validSeparator) { badSeparator = true; } + break; } + return badSeparator; } public DateTime GetDate() { - if (_e != null) - { - throw _e; - } - return new DateTime(_year, _month, _dayOfMonth, _hour, _minute, _second, DateTimeKind.Unspecified); + return _e != null + ? throw _e + : new DateTime(_year, _month, _dayOfMonth, _hour, _minute, _second, DateTimeKind.Unspecified); } } \ No newline at end of file diff --git a/src/cdeLib/Duplicates/Duplication.cs b/src/cdeLib/Duplicates/Duplication.cs index 20c821f..56278f8 100644 --- a/src/cdeLib/Duplicates/Duplication.cs +++ b/src/cdeLib/Duplicates/Duplication.cs @@ -23,23 +23,51 @@ public class Duplication private readonly Dictionary> _duplicateFileSize = new(); - private readonly HashSet _dirEntriesRequiringFullHashing = new(); + private readonly HashSet _dirEntriesRequiringFullHashing = []; protected readonly DuplicationStatistics _duplicationStatistics; + + // Cumulative FilesProcessed at the moment the full-hash phase begins, so that phase can report + // its own 0-based progress rather than continuing the partial phase's running total. + private long _processedAtFullHashStart; + private readonly ILogger _logger; private readonly IApplicationDiagnostics _applicationDiagnostics; private readonly HashHelper _hashHelper; + private readonly OperationCancellation _cancellation; - public Duplication(ILogger logger, IConfiguration configuration, IApplicationDiagnostics applicationDiagnostics) + public Duplication(ILogger logger, IConfiguration configuration, IApplicationDiagnostics applicationDiagnostics, + OperationCancellation cancellation) { _logger = logger; _hashHelper = new HashHelper(logger); _configuration = configuration; _applicationDiagnostics = applicationDiagnostics; + _cancellation = cancellation; _duplicationStatistics = new DuplicationStatistics(); _logger.LogDebug("Dupe Constructor Memory: {0}", _applicationDiagnostics.GetMemoryAllocated().FormatAsBytes()); } + /// + /// Optional callback raised periodically during so a UI can render live progress. + /// Arguments are (filesProcessed, filesToHash, phase). When null the progress is written to the log instead. + /// + public Action ProgressEvent { get; set; } + + /// + /// Optional callback raised with human readable status/summary lines during . + /// When null the message is written to the log instead. + /// + public Action StatusMessageEvent { get; set; } + + private void ReportStatus(string message) + { + if (StatusMessageEvent is not null) + StatusMessageEvent(message); + else + _logger.LogInfo(message); + } + /// /// Apply an Hash Checksum to all rootEntries /// @@ -51,11 +79,7 @@ public async Task ApplyHash(IList rootEntries) _logger.LogDebug("PostPairSize Memory: {0}", _applicationDiagnostics.GetMemoryAllocated().FormatAsBytes()); // Calculate all aggregations in single pass to avoid multiple enumerations - long totalFilesInRootEntries = 0; - foreach (var entry in rootEntries) - { - totalFilesInRootEntries += entry.FileEntryCount; - } + long totalFilesInRootEntries = rootEntries.Aggregate(0, (current, entry) => current + entry.FileEntryCount); int totalEntriesInSizeDupes = 0; int longestListLength = -1; @@ -70,10 +94,10 @@ public async Task ApplyHash(IList rootEntries) longestListSize = kvp.Key; } } - _logger.LogInfo("Found {0} sets of files matched by file size", newMatches.Count); - _logger.LogInfo("Total files processed for the file size matches is {0}", totalFilesInRootEntries); - _logger.LogInfo("Total files found with at least 1 other file of same length {0}", totalEntriesInSizeDupes); - _logger.LogInfo("Longest list of same sized files is {0} for size {1} ", longestListLength, longestListSize); + ReportStatus($"Found {newMatches.Count} sets of files matched by file size"); + ReportStatus($"Total files processed for the file size matches is {totalFilesInRootEntries}"); + ReportStatus($"Total files found with at least 1 other file of same length {totalEntriesInSizeDupes}"); + ReportStatus($"Longest list of same sized files is {longestListLength} for size {longestListSize} "); // flatten - optimized without LINQ _logger.LogDebug("Flatten List.."); @@ -111,7 +135,7 @@ public async Task ApplyHash(IList rootEntries) var root = System.IO.Directory.GetDirectoryRoot(pde.FullPath); if (!groupedByDirectoryRoot.TryGetValue(root, out var group)) { - group = new List(); + group = []; groupedByDirectoryRoot[root] = group; } @@ -151,7 +175,7 @@ public async Task ApplyHash(IList rootEntries) { _duplicationStatistics.SeenFileSize(flatFile.ChildDE.Size); await CalculatePartialHashAsync(flatFile.FullPath, flatFile.ChildDE); - if (Hack.BreakConsoleFlag) + if (_cancellation.IsCancellationRequested) { Console.WriteLine("\n * Break key detected exiting hashing phase inner."); await cts.CancelAsync(); @@ -169,24 +193,23 @@ public async Task ApplyHash(IList rootEntries) return; } - _logger.LogInfo("After initial partial hashing phase."); + ReportStatus("After initial partial hashing phase."); var perf = $"{_duplicationStatistics.BytesProcessed * (1000.0 / timer.ElapsedMilliseconds) / (1024.0 * 1024.0):F2} MB/s"; var statsMessage = $"FullHash: {_duplicationStatistics.FullHashes} PartialHash: {_duplicationStatistics.PartialHashes} Processed: {_duplicationStatistics.BytesProcessed / (1024 * 1024):F2} MB NotProcessed: {_duplicationStatistics.BytesNotProcessed / (1024 * 1024):F2} MB Perf: {perf}\nTotal Data Encountered: {_duplicationStatistics.TotalFileBytes / (1024 * 1024):F2} MB\nFailedHash: {_duplicationStatistics.FailedToHash} (almost always because cannot open to read file)"; - _logger.LogInfo(statsMessage); + ReportStatus(statsMessage); - Hack.BreakConsoleFlag = false; // require you to press break again to stop the full hash phase. + _cancellation.Reset(); // require you to press break again to stop the full hash phase. CheckDupesAndCompleteFullHash(rootEntries); - _logger.LogInfo(string.Empty); - _logger.LogInfo("After hashing completed."); + ReportStatus("After hashing completed."); timer.Stop(); perf = $"{_duplicationStatistics.BytesProcessed * (1000.0 / timer.ElapsedMilliseconds) / (1024.0 * 1024.0):F2} MB/s"; statsMessage = $"FullHash: {_duplicationStatistics.FullHashes} PartialHash: {_duplicationStatistics.PartialHashes} Processed: {_duplicationStatistics.BytesProcessed / (1024 * 1024):F2} MB Perf: {perf}\nFailedHash: {_duplicationStatistics.FailedToHash} (almost always because cannot open to read file)"; - _logger.LogInfo(statsMessage); + ReportStatus(statsMessage); await Task.CompletedTask; } @@ -282,10 +305,16 @@ private void CheckDupesAndCompleteFullHash(IEnumerable rootEntries) } } - _logger.LogInfo("Found {0} duplication collections.", foundDupes.Count); - _logger.LogInfo("Total files found with at least 1 other file duplicate {0}", - totalEntriesInDupes); - _logger.LogInfo("Longest list of duplicate files is {0}", longestListLength); + ReportStatus($"Found {foundDupes.Count} duplication collections."); + ReportStatus($"Total files found with at least 1 other file duplicate {totalEntriesInDupes}"); + ReportStatus($"Longest list of duplicate files is {longestListLength}"); + + // Switch progress reporting to the full-hash phase: FilesProcessed is cumulative across both + // passes, so capture a baseline here and re-target the denominator to this phase's own work. + // Each phase then reports its own 0..100% progress (the phase label distinguishes them) and the + // percentage can never exceed 100% as it did when the partial-pass denominator was reused. + _processedAtFullHashStart = _duplicationStatistics.FilesProcessed; + _duplicationStatistics.FilesToCheckForDuplicatesCount = totalEntriesInDupes; // Populate HashSet with entries requiring full hash foreach (var kvp in foundDupes) @@ -351,7 +380,7 @@ await Task.Run(() => var fullPath = pde.FullPath; await CalculateHash(fullPath, dirEntry, false); - if (Hack.BreakConsoleFlag) + if (_cancellation.IsCancellationRequested) { _logger.LogInfo("Break key detected, exiting full hash phase."); await cts.CancelAsync(); @@ -400,13 +429,21 @@ private async Task CalculateHash(string fullPath, ICommonEntry de, bool doPartia _duplicationStatistics.FullHashes++; if (_duplicationStatistics.FilesProcessed % displayCounterInterval == 0) { - _logger.LogInfo( - "Progress through duplicate files at {0} of {1} which is {2:F2}% Largest {3:F2} MB, Smallest {4:F2} MB", - _duplicationStatistics.FilesProcessed, _duplicationStatistics.FilesToCheckForDuplicatesCount, - 100 * (1.0 * _duplicationStatistics.FilesProcessed / - _duplicationStatistics.FilesToCheckForDuplicatesCount), - 1.0 * _duplicationStatistics.LargestFileSize / (1024 * 1024), - 1.0 * _duplicationStatistics.SmallestFileSize / (1024 * 1024)); + if (ProgressEvent is not null) + { + ProgressEvent(_duplicationStatistics.FilesProcessed, + _duplicationStatistics.FilesToCheckForDuplicatesCount, "Partial hash"); + } + else + { + _logger.LogInfo( + "Progress through duplicate files at {0} of {1} which is {2:F2}% Largest {3:F2} MB, Smallest {4:F2} MB", + _duplicationStatistics.FilesProcessed, _duplicationStatistics.FilesToCheckForDuplicatesCount, + 100 * (1.0 * _duplicationStatistics.FilesProcessed / + _duplicationStatistics.FilesToCheckForDuplicatesCount), + 1.0 * _duplicationStatistics.LargestFileSize / (1024 * 1024), + 1.0 * _duplicationStatistics.SmallestFileSize / (1024 * 1024)); + } } } else @@ -431,10 +468,20 @@ private async Task CalculateHash(string fullPath, ICommonEntry de, bool doPartia _duplicationStatistics.BytesProcessed += hashResponse.BytesHashed; if (_duplicationStatistics.FilesProcessed % displayCounterInterval == 0) { - _logger.LogInfo("Progress through duplicate files at {0} of {1} which is {2:.0}%", - _duplicationStatistics.FilesProcessed, _duplicationStatistics.FilesToCheckForDuplicatesCount, - 100 * (1.0 * _duplicationStatistics.FilesProcessed / - _duplicationStatistics.FilesToCheckForDuplicatesCount)); + // Report progress relative to the start of the full-hash phase so it reads 0..100%. + var processedThisPhase = _duplicationStatistics.FilesProcessed - _processedAtFullHashStart; + if (ProgressEvent is not null) + { + ProgressEvent(processedThisPhase, + _duplicationStatistics.FilesToCheckForDuplicatesCount, "Full hash"); + } + else + { + _logger.LogInfo("Progress through duplicate files at {0} of {1} which is {2:.0}%", + processedThisPhase, _duplicationStatistics.FilesToCheckForDuplicatesCount, + 100 * (1.0 * processedThisPhase / + _duplicationStatistics.FilesToCheckForDuplicatesCount)); + } } } else diff --git a/src/cdeLib/Duplicates/DuplicationStatistics.cs b/src/cdeLib/Duplicates/DuplicationStatistics.cs index 333df96..09456e2 100644 --- a/src/cdeLib/Duplicates/DuplicationStatistics.cs +++ b/src/cdeLib/Duplicates/DuplicationStatistics.cs @@ -41,11 +41,5 @@ public void SeenFileSize(long value) SmallestFileSize = value < SmallestFileSize ? value : SmallestFileSize; } - public long FilesProcessed - { - get - { - return PartialHashes + FullHashes + AllreadyDonePartials + AllreadyDoneFulls + FailedToHash; - } - } + public long FilesProcessed => PartialHashes + FullHashes + AllreadyDonePartials + AllreadyDoneFulls + FailedToHash; } \ No newline at end of file diff --git a/src/cdeLib/Duplicates/FindDuplicateCommandHandler.cs b/src/cdeLib/Duplicates/FindDuplicateCommandHandler.cs index 6ae299e..844b8d3 100644 --- a/src/cdeLib/Duplicates/FindDuplicateCommandHandler.cs +++ b/src/cdeLib/Duplicates/FindDuplicateCommandHandler.cs @@ -1,7 +1,9 @@ using System.Threading; using System.Threading.Tasks; using cdeLib.Catalog; +using cdeLib.Entities; using JetBrains.Annotations; +using Serilog; using SlimMessageBus; namespace cdeLib.Duplicates; @@ -11,16 +13,28 @@ public class FindDuplicateCommandHandler : IRequestHandler +/// Reconstructs a mutable tree from a read-only +/// (an or a memory-mapped ). This is +/// the inverse of , used by the batch hash/dupes +/// commands: they need the existing tree-based hashing engine (two-phase partial→full hashing, +/// cross-catalog size pairing, parallel-by-volume), so they rebuild a tree from the columnar catalog, +/// mutate it (hashes), and write a fresh .cdex back. Field values (modified ticks, flags, hash) +/// are copied verbatim so the rewritten catalog round-trips exactly. +/// +public static class CatalogTreeBuilder +{ + public static RootEntry FromSource(IEntrySource s) + { + var root = new RootEntry + { + Path = s.RootPath, + VolumeName = s.VolumeName, + DefaultFileName = s.DefaultFileName, + ActualFileName = s.ActualFileName, + DriveLetterHint = s.DriveLetterHint, + Description = s.Description, + AvailSpace = s.AvailSpace, + TotalSpace = s.TotalSpace, + ScanStartUtcTicks = s.ScanStartUtcTicks, + ScanEndUtcTicks = s.ScanEndUtcTicks, + ModifiedTicks = s.ModifiedTicksOf(0), + BitFields = s.FlagsOf(0), + }; + + var created = new ICommonEntry[s.Count]; + created[0] = root; + + // Walk the index tree (root = 0) creating a DirEntry per entry, preserving sibling order. + var stack = new Stack(); + stack.Push(0); + while (stack.Count > 0) + { + var p = stack.Pop(); + foreach (var c in s.ChildrenOf(p)) + { + var d = new DirEntry(s.IsDirectory(c)) + { + Path = s.FullName(c), + Size = s.SizeOf(c), + ModifiedTicks = s.ModifiedTicksOf(c), + BitFields = s.FlagsOf(c), + }; + if (s.HasHash && s.IsHashDone(c)) d.Hash = s.HashOf(c); + + created[c] = d; + created[p].AddChild(d); + if (s.IsDirectory(c)) stack.Push(c); + } + } + + root.SetInMemoryFields(); + return root; + } + + /// + /// Open each columnar .cdex file, reconstruct its tree, and tag it with the source path + /// () so a mutated catalog can be written straight back. + /// Each mapping is closed before returning — the tree is a full in-memory copy. + /// + public static List FromColumnarFiles(IEnumerable cdexFiles) + { + var trees = new List(); + foreach (var file in cdexFiles) + { + using var reader = new ColumnarCatalogReader(file); + var tree = FromSource(reader); + tree.ActualFileName = file; + trees.Add(tree); + } + return trees; + } +} diff --git a/src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs b/src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs new file mode 100644 index 0000000..59138d4 --- /dev/null +++ b/src/cdeLib/Entities/Columnar/ColumnarCatalogReader.cs @@ -0,0 +1,419 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.MemoryMappedFiles; +using System.Runtime.InteropServices; +using System.Text; +using System.Text.RegularExpressions; +using cdeLib.Entities.Soa; + +namespace cdeLib.Entities.Columnar; + +/// +/// Zero-copy reader over a catalog file. "Loading" is just mmap-ing: +/// columns are exposed as straight over the mapping, so no catalog data +/// is copied to the managed heap. A name search byte-scans the UTF-8 NameBlob in place (zero managed +/// allocation per entry on the common ASCII path), and only the pages it touches fault into the +/// working set. +/// +/// The mapping stays open for the lifetime of the reader; hold it for as long as the catalog is +/// "loaded" and it on reload/exit. The file is read-only. +/// +public sealed unsafe class ColumnarCatalogReader : IEntrySource, IDisposable +{ + private readonly MemoryMappedFile _mmf; + private readonly MemoryMappedViewAccessor _view; + private byte* _base; + + private readonly long[] _off = new long[ColumnarFormat.ColumnCount]; + private readonly long[] _len = new long[ColumnarFormat.ColumnCount]; + + public int Count { get; } + public bool HasHashes { get; } + + // Catalog-level metadata (parsed once from the small Meta blob — this is the only managed copy). + public string RootPath { get; } + public string VolumeName { get; } + public string DefaultFileName { get; } + public string ActualFileName { get; } + public string DriveLetterHint { get; } + public string Description { get; } + public long AvailSpace { get; } + public long TotalSpace { get; } + public long ScanStartUtcTicks { get; } + public long ScanEndUtcTicks { get; } + public long RootSize { get; } + public uint RootFileEntryCount { get; } + public uint RootDirEntryCount { get; } + + public ColumnarCatalogReader(string path) + { + _mmf = MemoryMappedFile.CreateFromFile(path, FileMode.Open, mapName: null, + capacity: 0, MemoryMappedFileAccess.Read); + _view = _mmf.CreateViewAccessor(0, 0, MemoryMappedFileAccess.Read); + _view.SafeMemoryMappedViewHandle.AcquirePointer(ref _base); + + var header = new ReadOnlySpan(_base, ColumnarFormat.HeaderSize); + if (!header[..4].SequenceEqual(ColumnarFormat.Magic)) + throw new InvalidDataException($"not a CDEX catalog: {path}"); + var version = BitConverter.ToInt32(header.Slice(4, 4)); + if (version != ColumnarFormat.Version) + throw new InvalidDataException( + $"unsupported CDEX version {version} (expected {ColumnarFormat.Version}): {path}"); + Count = BitConverter.ToInt32(header.Slice(8, 4)); + HasHashes = (BitConverter.ToInt32(header.Slice(12, 4)) & ColumnarFormat.FlagHasHashes) != 0; + + var p = ColumnarFormat.PreambleFixed; + for (var c = 0; c < ColumnarFormat.ColumnCount; c++) + { + _off[c] = BitConverter.ToInt64(header.Slice(p, 8)); p += 8; + _len[c] = BitConverter.ToInt64(header.Slice(p, 8)); p += 8; + } + + // Parse the metadata blob (small, read once). + var meta = Bytes(ColumnarFormat.Col.Meta); + var mp = 0; + RootPath = ReadLenString(meta, ref mp); + VolumeName = ReadLenString(meta, ref mp); + DefaultFileName = ReadLenString(meta, ref mp); + ActualFileName = ReadLenString(meta, ref mp); + DriveLetterHint = ReadLenString(meta, ref mp); + Description = ReadLenString(meta, ref mp); + AvailSpace = ReadI64(meta, ref mp); + TotalSpace = ReadI64(meta, ref mp); + ScanStartUtcTicks = ReadI64(meta, ref mp); + ScanEndUtcTicks = ReadI64(meta, ref mp); + RootSize = ReadI64(meta, ref mp); + RootFileEntryCount = (uint)ReadI32(meta, ref mp); + RootDirEntryCount = (uint)ReadI32(meta, ref mp); + } + + private ReadOnlySpan Bytes(ColumnarFormat.Col col) + => new(_base + _off[(int)col], (int)_len[(int)col]); + + private ReadOnlySpan As(ColumnarFormat.Col col) where T : struct + => MemoryMarshal.Cast(Bytes(col)); + + public ReadOnlySpan Size => As(ColumnarFormat.Col.Size); + public ReadOnlySpan ModifiedTicks => As(ColumnarFormat.Col.ModifiedTicks); + public ReadOnlySpan BitFields => Bytes(ColumnarFormat.Col.BitFields); + public ReadOnlySpan Parent => As(ColumnarFormat.Col.Parent); + private ReadOnlySpan FirstChild => As(ColumnarFormat.Col.FirstChild); + private ReadOnlySpan NextSibling => As(ColumnarFormat.Col.NextSibling); + private ReadOnlySpan HashBytes => Bytes(ColumnarFormat.Col.Hash); + private ReadOnlySpan NameOffsets => As(ColumnarFormat.Col.NameOffsets); + private ReadOnlySpan NameBlob => Bytes(ColumnarFormat.Col.NameBlob); + + public Flags Flags(int i) => (Flags)BitFields[i]; + public bool IsDirectory(int i) => (Flags(i) & Entities.Flags.Directory) == Entities.Flags.Directory; + + // ----- IEntrySource: index-addressed accessors straight over the mapping ----- + public long SizeOf(int i) => Size[i]; + public DateTime ModifiedOf(int i) => DateTime.FromBinary(ModifiedTicks[i]); + public long ModifiedTicksOf(int i) => ModifiedTicks[i]; + public Flags FlagsOf(int i) => Flags(i); + public bool IsHashDone(int i) => (Flags(i) & Entities.Flags.HashDone) == Entities.Flags.HashDone; + public bool IsPartialHash(int i) => (Flags(i) & Entities.Flags.PartialHash) == Entities.Flags.PartialHash; + public bool HasHash => HasHashes; + public Hash16 HashOf(int i) => + HasHashes ? MemoryMarshal.Read(HashBytes.Slice(i * 16, 16)) : default; + public string FullName(int i) => Name(i); + public string NameOf(int i) => Name(i); // full name (not split) — fine for path-problem trailing checks + public int ParentOf(int i) => Parent[i]; + public int FirstChildOf(int i) => FirstChild[i]; + + public IEnumerable ChildrenOf(int parent) + { + // Materialise into a list (no yield): the sibling chain reads spans over the mapping, which a + // lazy iterator's state machine can't hold. Per-directory child fan-out is small. + var result = new List(); + var first = FirstChild[parent]; + if (first == EntryStore.None) return result; + var sib = NextSibling; + for (var c = first; c != EntryStore.None; c = sib[c]) result.Add(c); + return result; + } + + public void AppendFullPath(StringBuilder sb, int i) + { + var parent = Parent; + var offs = NameOffsets; + var blob = NameBlob; + Span chain = stackalloc int[256]; + var depth = 0; + for (var cur = i; cur != EntryStore.None && depth < chain.Length; cur = parent[cur]) + chain[depth++] = cur; + for (var k = depth - 1; k >= 0; k--) + { + if (sb.Length > 0) + { + var last = sb[^1]; + if (last != '\\' && last != '/') sb.Append(Path.DirectorySeparatorChar); + } + var idx = chain[k]; + sb.Append(Encoding.UTF8.GetString(blob.Slice((int)offs[idx], (int)(offs[idx + 1] - offs[idx])))); + } + } + + /// UTF-8 full-name bytes of entry , sliced in place from the mapping. + public ReadOnlySpan NameUtf8(int i) + { + var offs = NameOffsets; + return NameBlob.Slice((int)offs[i], (int)(offs[i + 1] - offs[i])); + } + + public string Name(int i) => Encoding.UTF8.GetString(NameUtf8(i)); + + /// + /// Find matching the production semantics exactly: pattern + + /// name/path + file/folder filter, index 0 (root) never a result. Substring matching byte-scans + /// the mapping (zero-alloc ASCII path); regex decodes per entry like the store search does. + /// + public int Find(string pattern, bool regexMode, bool includePath, bool includeFiles, + bool includeFolders, Action onMatch = null) + { + if (!includeFiles && !includeFolders) return 0; + if (regexMode && !string.IsNullOrEmpty(pattern)) + return FindRegex(pattern, includePath, includeFiles, includeFolders, onMatch); + return includePath + ? FindPath(pattern, includeFiles, includeFolders, onMatch) + : FindName(pattern, includeFiles, includeFolders, onMatch); + } + + /// + /// Full-filter search (pattern + name/path + file/folder + size/date/hour ranges) matching the GUI + /// + /// semantics exactly, evaluated zero-copy over the mapping. + /// + public void Find(EntryStoreFindOptions o, Action onMatch, + Func isCancelled = null, Action onScan = null) + { + ArgumentNullException.ThrowIfNull(o); + ArgumentNullException.ThrowIfNull(onMatch); + if (!o.IncludeFiles && !o.IncludeFolders) return; + + var hasPattern = !string.IsNullOrEmpty(o.Pattern); + var regex = o.RegexMode && hasPattern + ? new Regex(o.Pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.Compiled) + : null; + var matcher = hasPattern && !o.RegexMode ? new Utf8Matcher(o.Pattern) : default; + + var size = Size; + var bits = BitFields; + var offs = NameOffsets; + var blob = NameBlob; + var pathBuf = o.IncludePath ? new byte[1024] : null; + Span chain = o.IncludePath ? stackalloc int[256] : default; + + for (var i = 1; i < Count; i++) // index 0 is the root, never a result + { + if ((i & 4095) == 0) + { + if (isCancelled != null && isCancelled()) return; + onScan?.Invoke(i); + } + + var isDir = ((Flags)bits[i] & Entities.Flags.Directory) == Entities.Flags.Directory; + if (isDir ? !o.IncludeFolders : !o.IncludeFiles) continue; + + if (o.FromSizeEnable && size[i] < o.FromSize) continue; + if (o.ToSizeEnable && size[i] > o.ToSize) continue; + + if (o.FromDateEnable || o.ToDateEnable || o.FromHourEnable || o.ToHourEnable || o.NotOlderThanEnable) + { + var modified = ModifiedOf(i); + if (o.FromDateEnable && modified < o.FromDate) continue; + if (o.ToDateEnable && modified > o.ToDate) continue; + if (o.NotOlderThanEnable && modified < o.NotOlderThan) continue; + var tod = modified.TimeOfDay; + if (o.FromHourEnable && tod < o.FromHour) continue; + if (o.ToHourEnable && tod > o.ToHour) continue; + } + + if (!hasPattern) { onMatch(i); continue; } + + bool match; + if (o.RegexMode) + { + match = regex.IsMatch(o.IncludePath ? FullPath(i) : Name(i)); + } + else if (o.IncludePath) + { + var n = BuildPathBytes(ref pathBuf, chain, i, offs, blob); + match = matcher.Contains(pathBuf.AsSpan(0, n)); + } + else + { + match = matcher.Contains(blob.Slice((int)offs[i], (int)(offs[i + 1] - offs[i]))); + } + + if (match) onMatch(i); + } + } + + // Build entry i's full-path UTF-8 bytes into buf (grown as needed); returns the byte length. + private int BuildPathBytes(ref byte[] buf, Span chain, int i, + ReadOnlySpan offs, ReadOnlySpan blob) + { + var parent = Parent; + var depth = 0; + for (var cur = i; cur != EntryStore.None && depth < chain.Length; cur = parent[cur]) + chain[depth++] = cur; + var n = 0; + for (var k = depth - 1; k >= 0; k--) + { + if (n > 0) buf = Append(buf, ref n, (byte)Path.DirectorySeparatorChar); + var idx = chain[k]; + buf = Append(buf, ref n, blob.Slice((int)offs[idx], (int)(offs[idx + 1] - offs[idx]))); + } + return n; + } + + private int FindRegex(string pattern, bool includePath, bool includeFiles, bool includeFolders, + Action onMatch) + { + var regex = new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.Compiled); + var bits = BitFields; + var matches = 0; + for (var i = 1; i < Count; i++) + { + if (!Wanted(bits[i], includeFiles, includeFolders)) continue; + var text = includePath ? FullPath(i) : Name(i); + if (regex.IsMatch(text)) + { + matches++; + onMatch?.Invoke(i); + } + } + return matches; + } + + /// + /// Name search: byte-scan each entry's UTF-8 name for (ordinal, + /// case-insensitive). Sequentially touches only the NameBlob + NameOffsets columns. Zero managed + /// allocation per entry for ASCII names; non-ASCII names fall back to a decoded comparison. + /// + public int FindName(string pattern, bool includeFiles, bool includeFolders, Action onMatch = null) + { + var matcher = new Utf8Matcher(pattern); + var offs = NameOffsets; + var blob = NameBlob; + var bits = BitFields; + var matches = 0; + for (var i = 1; i < Count; i++) // index 0 is the root, never a result + { + if (!Wanted(bits[i], includeFiles, includeFolders)) continue; + var name = blob.Slice((int)offs[i], (int)(offs[i + 1] - offs[i])); + if (matcher.Contains(name)) + { + matches++; + onMatch?.Invoke(i); + } + } + return matches; + } + + /// + /// Path search: build each entry's full path bytes into a reused buffer by walking Parent[], then + /// match. No per-entry managed string allocation on the ASCII path. + /// + public int FindPath(string pattern, bool includeFiles, bool includeFolders, Action onMatch = null) + { + var matcher = new Utf8Matcher(pattern); + var parent = Parent; + var offs = NameOffsets; + var blob = NameBlob; + var bits = BitFields; + Span chain = stackalloc int[256]; + var buf = new byte[1024]; + var matches = 0; + + for (var i = 1; i < Count; i++) // index 0 is the root, never a result + { + if (!Wanted(bits[i], includeFiles, includeFolders)) continue; + + var depth = 0; + for (var cur = i; cur != EntryStore.None && depth < chain.Length; cur = parent[cur]) + chain[depth++] = cur; + + var n = 0; + for (var k = depth - 1; k >= 0; k--) + { + if (n > 0) buf = Append(buf, ref n, (byte)Path.DirectorySeparatorChar); + var idx = chain[k]; + buf = Append(buf, ref n, blob.Slice((int)offs[idx], (int)(offs[idx + 1] - offs[idx]))); + } + + if (matcher.Contains(buf.AsSpan(0, n))) + { + matches++; + onMatch?.Invoke(i); + } + } + return matches; + } + + private static bool Wanted(byte bitField, bool includeFiles, bool includeFolders) + { + if (includeFiles && includeFolders) return true; + var isDir = ((Flags)bitField & Entities.Flags.Directory) == Entities.Flags.Directory; + return isDir ? includeFolders : includeFiles; + } + + public string FullPath(int i) + { + var parent = Parent; + var offs = NameOffsets; + var blob = NameBlob; + Span chain = stackalloc int[256]; + var depth = 0; + for (var cur = i; cur != EntryStore.None && depth < chain.Length; cur = parent[cur]) + chain[depth++] = cur; + var sb = new StringBuilder(128); + for (var k = depth - 1; k >= 0; k--) + { + if (sb.Length > 0) sb.Append(Path.DirectorySeparatorChar); + var idx = chain[k]; + sb.Append(Encoding.UTF8.GetString(blob.Slice((int)offs[idx], (int)(offs[idx + 1] - offs[idx])))); + } + return sb.ToString(); + } + + private static byte[] Append(byte[] buf, ref int n, byte b) + { + if (n + 1 > buf.Length) Array.Resize(ref buf, buf.Length * 2); + buf[n++] = b; + return buf; + } + + private static byte[] Append(byte[] buf, ref int n, ReadOnlySpan src) + { + while (n + src.Length > buf.Length) Array.Resize(ref buf, buf.Length * 2); + src.CopyTo(buf.AsSpan(n)); + n += src.Length; + return buf; + } + + private static string ReadLenString(ReadOnlySpan s, ref int p) + { + var len = BitConverter.ToInt32(s.Slice(p, 4)); p += 4; + if (len == 0) return string.Empty; + var str = Encoding.UTF8.GetString(s.Slice(p, len)); + p += len; + return str; + } + + private static long ReadI64(ReadOnlySpan s, ref int p) { var v = BitConverter.ToInt64(s.Slice(p, 8)); p += 8; return v; } + private static int ReadI32(ReadOnlySpan s, ref int p) { var v = BitConverter.ToInt32(s.Slice(p, 4)); p += 4; return v; } + + public void Dispose() + { + if (_base != null) + { + _view.SafeMemoryMappedViewHandle.ReleasePointer(); + _base = null; + } + _view?.Dispose(); + _mmf?.Dispose(); + } +} diff --git a/src/cdeLib/Entities/Columnar/ColumnarFormat.cs b/src/cdeLib/Entities/Columnar/ColumnarFormat.cs new file mode 100644 index 0000000..64f9a5f --- /dev/null +++ b/src/cdeLib/Entities/Columnar/ColumnarFormat.cs @@ -0,0 +1,180 @@ +using System; +using System.Buffers.Binary; +using System.IO; +using System.Runtime.InteropServices; +using System.Text; +using cdeLib.Entities.Soa; + +namespace cdeLib.Entities.Columnar; + +/// +/// On-disk columnar (struct-of-arrays) catalog format, designed for zero-copy reads over a +/// memory-mapped file. "Loading" a catalog becomes mmap-ing the file — no managed object graph +/// is materialised, so the working set is only the file pages a query actually touches (in the +/// reclaimable OS page cache), not GC heap. See for the read side. +/// +/// Layout (all little-endian): +/// preamble: +/// [0] magic "CDEX" (4 bytes) +/// [4] int32 version +/// [8] int32 count (entries incl. root; index 0 = root) +/// [12] int32 flags (bit0 = hasHashes) +/// [16] (int64 offset, int64 length) x -- absolute, 8-aligned +/// column bodies (each padded to an 8-byte boundary), in order. +/// + /// Columns are dense and homogeneous, so a name-only search sequentially scans just the NameBlob + + /// NameOffsets columns and never pages in Size / Modified / Hash. NameOffsets are 64-bit, so the on-disk + /// name blob can exceed 2 GB (though Write currently buffers it in-memory, which is limited to ~2 GB). + /// Entry count is 32-bit, matching 's int indexing. +public static class ColumnarFormat +{ + public static ReadOnlySpan Magic => "CDEX"u8; + public const int Version = 1; + public const int FlagHasHashes = 1; + + /// Fixed column ordering. Hash length is 0 when the catalog is un-hashed. + public enum Col + { + ModifiedTicks = 0, // long[count] + Size, // long[count] + BitFields, // byte[count] + Parent, // int[count] + FirstChild, // int[count] + NextSibling, // int[count] + NameOffsets, // long[count+1] prefix offsets into NameBlob + NameBlob, // byte[] UTF-8 full names (name+ext) concatenated + Hash, // byte[16*count] (only when hasHashes) + Meta, // byte[] catalog metadata blob + } + + public const int ColumnCount = 10; + public const int PreambleFixed = 16; // magic+version+count+flags + public const int HeaderSize = PreambleFixed + ColumnCount * 16; + + private static long Align8(long v) => (v + 7) & ~7L; + + /// Convert an in-memory to the columnar file. One-time cost. + public static void Write(EntryStore store, string outPath) + { + ArgumentNullException.ThrowIfNull(store); + var count = store.Count; + var hasHashes = store.Hash != null; + + // Build the variable-length name columns up front (UTF-8 full names + 64-bit prefix offsets). + var nameOffsets = new long[count + 1]; + + var initialCapacity = (int)Math.Min((long)count * 12, int.MaxValue); + using var nameBlob = new MemoryStream(initialCapacity); + for (var i = 0; i < count; i++) + { + nameOffsets[i] = nameBlob.Length; + WriteUtf8(nameBlob, store.Name[i]); + WriteUtf8(nameBlob, store.Ext[i]); // ext appended directly -> full-name bytes, no separator + } + nameOffsets[count] = nameBlob.Length; + if (nameBlob.Length > int.MaxValue) + throw new InvalidOperationException("Name blob exceeded 2 GB; ColumnarFormat.Write currently buffers names in-memory."); + var nameBlobBytes = nameBlob.GetBuffer().AsSpan(0, (int)nameBlob.Length); + var meta = BuildMeta(store); + + var len = new long[ColumnCount]; + len[(int)Col.ModifiedTicks] = (long)count * sizeof(long); + len[(int)Col.Size] = (long)count * sizeof(long); + len[(int)Col.BitFields] = count; + len[(int)Col.Parent] = (long)count * sizeof(int); + len[(int)Col.FirstChild] = (long)count * sizeof(int); + len[(int)Col.NextSibling] = (long)count * sizeof(int); + len[(int)Col.NameOffsets] = (long)(count + 1) * sizeof(long); + len[(int)Col.NameBlob] = nameBlobBytes.Length; + len[(int)Col.Hash] = hasHashes ? (long)count * 16 : 0; + len[(int)Col.Meta] = meta.Length; + + var off = new long[ColumnCount]; + var pos = (long)HeaderSize; + for (var c = 0; c < ColumnCount; c++) + { + pos = Align8(pos); + off[c] = pos; + pos += len[c]; + } + + using var fs = new FileStream(outPath, FileMode.Create, FileAccess.Write, FileShare.None, + 1 << 20, FileOptions.SequentialScan); + + fs.Write(Magic); + WriteI32(fs, Version); + WriteI32(fs, count); + WriteI32(fs, hasHashes ? FlagHasHashes : 0); + for (var c = 0; c < ColumnCount; c++) + { + WriteI64(fs, off[c]); + WriteI64(fs, len[c]); + } + + WriteCol(fs, off[(int)Col.ModifiedTicks], MemoryMarshal.AsBytes(store.ModifiedTicks.AsSpan(0, count))); + WriteCol(fs, off[(int)Col.Size], MemoryMarshal.AsBytes(store.Size.AsSpan(0, count))); + WriteCol(fs, off[(int)Col.BitFields], store.BitFields.AsSpan(0, count)); + WriteCol(fs, off[(int)Col.Parent], MemoryMarshal.AsBytes(store.Parent.AsSpan(0, count))); + WriteCol(fs, off[(int)Col.FirstChild], MemoryMarshal.AsBytes(store.FirstChild.AsSpan(0, count))); + WriteCol(fs, off[(int)Col.NextSibling], MemoryMarshal.AsBytes(store.NextSibling.AsSpan(0, count))); + WriteCol(fs, off[(int)Col.NameOffsets], MemoryMarshal.AsBytes(nameOffsets.AsSpan())); + WriteCol(fs, off[(int)Col.NameBlob], nameBlobBytes); + if (hasHashes) + WriteCol(fs, off[(int)Col.Hash], MemoryMarshal.AsBytes(store.Hash.AsSpan(0, count))); + WriteCol(fs, off[(int)Col.Meta], meta); + } + + private static byte[] BuildMeta(EntryStore s) + { + using var ms = new MemoryStream(256); + WriteLenString(ms, s.RootPath); + WriteLenString(ms, s.VolumeName); + WriteLenString(ms, s.DefaultFileName); + WriteLenString(ms, s.ActualFileName); + WriteLenString(ms, s.DriveLetterHint); + WriteLenString(ms, s.Description); + WriteI64(ms, s.AvailSpace); + WriteI64(ms, s.TotalSpace); + WriteI64(ms, s.ScanStartUtcTicks); + WriteI64(ms, s.ScanEndUtcTicks); + WriteI64(ms, s.RootSize); + WriteI32(ms, (int)s.RootFileEntryCount); + WriteI32(ms, (int)s.RootDirEntryCount); + return ms.ToArray(); + } + + private static void WriteCol(FileStream fs, long offset, ReadOnlySpan body) + { + var pad = offset - fs.Position; + for (var i = 0; i < pad; i++) fs.WriteByte(0); + fs.Write(body); + } + + private static void WriteUtf8(Stream s, string value) + { + if (string.IsNullOrEmpty(value)) return; + var bytes = Encoding.UTF8.GetBytes(value); + s.Write(bytes, 0, bytes.Length); + } + + private static void WriteLenString(Stream s, string value) + { + var bytes = string.IsNullOrEmpty(value) ? [] : Encoding.UTF8.GetBytes(value); + WriteI32(s, bytes.Length); + s.Write(bytes, 0, bytes.Length); + } + + private static void WriteI32(Stream s, int v) + { + Span b = stackalloc byte[4]; + BinaryPrimitives.WriteInt32LittleEndian(b, v); + s.Write(b); + } + + private static void WriteI64(Stream s, long v) + { + Span b = stackalloc byte[8]; + BinaryPrimitives.WriteInt64LittleEndian(b, v); + s.Write(b); + } +} diff --git a/src/cdeLib/Entities/Columnar/Utf8Matcher.cs b/src/cdeLib/Entities/Columnar/Utf8Matcher.cs new file mode 100644 index 0000000..5f14f57 --- /dev/null +++ b/src/cdeLib/Entities/Columnar/Utf8Matcher.cs @@ -0,0 +1,60 @@ +using System; +using System.Text; + +namespace cdeLib.Entities.Columnar; + +/// +/// Ordinal, case-insensitive substring matcher over UTF-8 bytes, built once per query. The common +/// case — an ASCII pattern against an ASCII name — is matched by folding bytes in place with no +/// allocation. A pattern or name containing non-ASCII bytes falls back to a decoded +/// compare (allocates only for those rare names), +/// so results match the tree/store search path exactly. +/// +public readonly ref struct Utf8Matcher +{ + private readonly ReadOnlySpan _patternLowerAscii; // A-Z folded to a-z; valid only when _asciiPattern + private readonly string _pattern; + private readonly bool _asciiPattern; + private readonly bool _empty; + + public Utf8Matcher(string pattern) + { + _pattern = pattern ?? string.Empty; + _empty = _pattern.Length == 0; + var bytes = _empty ? [] : Encoding.UTF8.GetBytes(_pattern); + _asciiPattern = Ascii.IsValid(bytes); + if (_asciiPattern && !_empty) + { + for (var i = 0; i < bytes.Length; i++) bytes[i] = ToLower(bytes[i]); + } + _patternLowerAscii = bytes; + } + + public bool Contains(ReadOnlySpan nameUtf8) + { + if (_empty) return true; + if (_asciiPattern && Ascii.IsValid(nameUtf8)) + return AsciiContainsFolded(nameUtf8, _patternLowerAscii); + + // Rare path: non-ASCII somewhere. Decode and compare with real ordinal-ignore-case. + return Encoding.UTF8.GetString(nameUtf8).Contains(_pattern, StringComparison.OrdinalIgnoreCase); + } + + private static bool AsciiContainsFolded(ReadOnlySpan haystack, ReadOnlySpan needleLower) + { + if (haystack.Length < needleLower.Length) return false; + var last = haystack.Length - needleLower.Length; + for (var i = 0; i <= last; i++) + { + var k = 0; + for (; k < needleLower.Length; k++) + { + if (ToLower(haystack[i + k]) != needleLower[k]) break; + } + if (k == needleLower.Length) return true; + } + return false; + } + + private static byte ToLower(byte b) => b is >= (byte)'A' and <= (byte)'Z' ? (byte)(b + 32) : b; +} diff --git a/src/cdeLib/Entities/DirEntry.cs b/src/cdeLib/Entities/DirEntry.cs index 0af273e..27dbb74 100644 --- a/src/cdeLib/Entities/DirEntry.cs +++ b/src/cdeLib/Entities/DirEntry.cs @@ -18,11 +18,43 @@ public sealed class DirEntry : ICommonEntry { private string _path; + // Extension (including the dot), interned and split from the name for memory efficiency. + // Null when the name has no extension. + private string _ext; + + /// Interned name without extension (the _path part). For SoA conversion reuse. + internal string NamePart => _path; + + /// Interned extension including the dot, or null. For SoA conversion reuse. + internal string ExtPart => _ext; + + /// + /// Side-object holding directory-only state — the child list and the rolled-up summary counts. + /// Null on every file (the vast majority of entries), so a file no longer carries an always-null + /// Children reference plus two count fields. Only directories (~2% of entries) allocate it. + /// Serialization is unaffected: Children stays Key 3 via the property below, just backed here. + /// + /// Note: the content Hash deliberately stays INLINE on the entry. Moving it here too would force + /// every *hashed file* to allocate an ExtraData whose object header costs more than the 16-byte + /// Hash16 it replaced — a net regression for hashed catalogs. Keeping Hash inline means a hashed + /// file needs no side-object at all, so this change never increases footprint for any catalog. + /// + private sealed class ExtraData + { + public IList Children; + public uint FileEntryCount; + public uint DirEntryCount; + } + + private ExtraData _extra; + + private ExtraData EnsureExtra() => _extra ??= new ExtraData(); + [IgnoreMember] public DateTime Modified { - set => ModifiedTicks = value.Ticks; get => DateTime.FromBinary(ModifiedTicks); + set => ModifiedTicks = value.Ticks; } [ProtoMember(1, IsRequired = true)] @@ -160,13 +192,35 @@ public bool IsDefaultSort /// if this is a directory number of files contained in its hierarchy /// [IgnoreMember] - public long FileEntryCount { get; set; } + public uint FileEntryCount + { + get => _extra?.FileEntryCount ?? 0; + set + { + if (value != 0) EnsureExtra().FileEntryCount = value; + else + { + _extra?.FileEntryCount = value; + } + } + } /// /// if this is a directory number of dirs contained in its hierarchy /// [IgnoreMember] - public long DirEntryCount { get; set; } + public uint DirEntryCount + { + get => _extra?.DirEntryCount ?? 0; + set + { + if (value != 0) EnsureExtra().DirEntryCount = value; + else + { + _extra?.DirEntryCount = value; + } + } + } public void SetHash(byte[] hash) { @@ -337,17 +391,31 @@ public void SetSummaryFields() [ProtoMember(3, IsRequired = false)] [FlatBufferItem(3)] [Key(3)] - public IList Children { get; set; } - // ReSharper restore MemberCanBePrivate.Global - - public void AddChild(DirEntry child) + public IList Children { - if (Children == null) + get => _extra?.Children; + set { - Children = CollectionPool.GetDirEntryList(); + // A non-null child list (only directories have one) materialises ExtraData; files + // deserialize nil Children and stay lean. + if (value != null) EnsureExtra().Children = value; + else + { + _extra?.Children = null; + } } + } + // ReSharper restore MemberCanBePrivate.Global - Children.Add(child); + // Covariant read-only view for ICommonEntry consumers. The backing List satisfies + // IReadOnlyList at runtime via interface covariance. + IReadOnlyList ICommonEntry.Children => _extra?.Children as IReadOnlyList; + + public void AddChild(DirEntry child) + { + var extra = EnsureExtra(); + extra.Children ??= CollectionPool.GetDirEntryList(); + extra.Children.Add(child); } [ProtoMember(4, IsRequired = true)] @@ -366,20 +434,16 @@ public void AddChild(DirEntry child) public string Path { //NOTE: Separating the extension from the path is more memory efficient (300MB saved on 6000MB load) but slower. - get - { - //return _path; - + get => // string.concat faster than string interpolation. - return string.IsNullOrEmpty(field) ? _path : string.Concat(_path, field); - } + string.IsNullOrEmpty(_ext) ? _path : string.Concat(_path, _ext); set { if (string.IsNullOrEmpty(value)) { _path = string.Intern(string.Empty); - field = null; + _ext = null; return; } @@ -389,13 +453,13 @@ public string Path if (lastDot > 0 && lastDot > valueSpan.LastIndexOfAny(PathSeparators)) { // Span slicing is zero-cost, allocate strings only for Intern - field = string.Intern(new string(valueSpan[lastDot..])); + _ext = string.Intern(new string(valueSpan[lastDot..])); _path = string.Intern(new string(valueSpan[..lastDot])); } else { _path = string.Intern(value); - field = null; + _ext = null; } // Simpler code but slightly less performance: @@ -469,7 +533,7 @@ public void TraverseTreesCopyHash(ICommonEntry destination) if (baseSourceEntry.Children != null && baseDestinationEntry.Children != null) { // Build dictionary for O(1) lookups instead of O(n) linear search - var destinationLookup = new Dictionary( + var destinationLookup = new Dictionary( baseDestinationEntry.Children.Count, StringComparer.OrdinalIgnoreCase); @@ -502,6 +566,10 @@ public void TraverseTreesCopyHash(ICommonEntry destination) { destinationDirEntry.IsPartialHash = sourceIsPartial; destinationDirEntry.Hash = sourceDirEntry.Hash; + // IsHashDone is a separate BitFields flag; without it the copied hash is + // ignored by hashing/serialization (the reused hash would be silently lost + // on the next save). Mark the destination hashed. + destinationDirEntry.IsHashDone = true; } } // Directory: Push to stack for traversal @@ -542,7 +610,7 @@ public static IEnumerable GetPairDirEntries(IEnumerable public IList GetListFromRoot() { var activatedDirEntryList = new List(8); - for (var entry = (ICommonEntry)this; entry != null; entry = entry.ParentCommonEntry) + for (ICommonEntry entry = this; entry != null; entry = entry.ParentCommonEntry) { activatedDirEntryList.Add(entry); } diff --git a/src/cdeLib/Entities/EntryHelper.cs b/src/cdeLib/Entities/EntryHelper.cs index 0a61bdc..f4d3142 100644 --- a/src/cdeLib/Entities/EntryHelper.cs +++ b/src/cdeLib/Entities/EntryHelper.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading; @@ -25,38 +26,73 @@ public static IEnumerable GetPairDirEntries(IEnumerable return new PairDirEntryEnumerator(rootEntries); } + // Reused per-thread scratch buffer for allocation-free path matching (see FullPathContains). + private static readonly ThreadLocal PathMatchBuffer = + new(() => new char[512]); + /// - /// Creates a full path using a ThreadLocal StringBuilder to reduce allocations. - /// Still allocates the final string, but avoids intermediate allocations from Path.Combine. + /// Builds the full path of (whose directory parent is + /// ) into the shared ThreadLocal StringBuilder in a single pass, + /// walking the parent chain via ParentCommonEntry. Avoids the recursive per-level full-path + /// string allocations of the old approach (which called parent.FullPath at every level) — only + /// each segment's name is materialised. Returns the StringBuilder for the caller to consume. /// - public static string MakeFullPath(ICommonEntry parentEntry, ICommonEntry dirEntry) + private static StringBuilder BuildFullPath(ICommonEntry parentEntry, ICommonEntry dirEntry) { var sb = PathBuilder.Value!; sb.Clear(); - var parentPath = parentEntry.FullPath; - if (parentPath != null) - { - sb.Append(parentPath); - if (sb.Length > 0) - { - var lastChar = sb[^1]; - if (lastChar != '\\' && lastChar != '/') - sb.Append(System.IO.Path.DirectorySeparatorChar); - } - } - + AppendAncestorPath(sb, parentEntry); + AppendSeparatorIfNeeded(sb); sb.Append(dirEntry.Path ?? "dnull"); - var result = sb.ToString(); - // Prevent StringBuilder from growing unbounded in long-running processes - // Only shrink if the capacity is large AND current content fits in target size + // Prevent the pooled StringBuilder from growing unbounded in long-running processes. if (sb.Capacity > 1024 && sb.Length <= 512) { sb.Capacity = 512; } - return result; + return sb; + } + + /// + /// Append the full path of to root-first by + /// recursing up the ParentCommonEntry chain. The root (null parent) contributes its stored + /// FullPath; every descendant contributes its own Path segment. No intermediate path strings. + /// + private static void AppendAncestorPath(StringBuilder sb, ICommonEntry entry) + { + if (entry == null) return; + + var parent = entry.ParentCommonEntry; + if (parent == null) + { + // Root entry: FullPath is a stored field on RootEntry (no recursion/allocation). + sb.Append(entry.FullPath ?? entry.Path ?? string.Empty); + return; + } + + AppendAncestorPath(sb, parent); + AppendSeparatorIfNeeded(sb); + sb.Append(entry.Path ?? "dnull"); + } + + private static void AppendSeparatorIfNeeded(StringBuilder sb) + { + if (sb.Length == 0) return; + var last = sb[^1]; + if (last != '\\' && last != '/') + sb.Append(System.IO.Path.DirectorySeparatorChar); + } + + /// + /// Creates a full path using a ThreadLocal StringBuilder to reduce allocations. + /// Still allocates the final string, but avoids intermediate allocations from Path.Combine + /// and from recursive per-level full-path string building. + /// + public static string MakeFullPath(ICommonEntry parentEntry, ICommonEntry dirEntry) + { + return BuildFullPath(parentEntry, dirEntry).ToString(); } /// @@ -68,6 +104,28 @@ public static string MakeFullPathPooled(ICommonEntry parentEntry, ICommonEntry d return MakeFullPath(parentEntry, dirEntry); } + /// + /// Allocation-free full-path substring test: builds the full path into pooled buffers and + /// matches over a span, without allocating the result string. + /// Used by find's substring path search, the dominant allocator in path queries. + /// + public static bool FullPathContains(ICommonEntry parentEntry, ICommonEntry dirEntry, + string pattern, StringComparison comparison) + { + var sb = BuildFullPath(parentEntry, dirEntry); + var length = sb.Length; + + var buffer = PathMatchBuffer.Value!; + if (buffer.Length < length) + { + buffer = new char[Math.Max(length, buffer.Length * 2)]; + PathMatchBuffer.Value = buffer; + } + + sb.CopyTo(0, buffer, 0, length); + return buffer.AsSpan(0, length).Contains(pattern, comparison); + } + /// /// Recursive traversal /// diff --git a/src/cdeLib/Entities/ICommonEntry.cs b/src/cdeLib/Entities/ICommonEntry.cs index a6fbf43..16edb2a 100644 --- a/src/cdeLib/Entities/ICommonEntry.cs +++ b/src/cdeLib/Entities/ICommonEntry.cs @@ -13,7 +13,13 @@ public interface ICommonEntry bool IsDefaultSort { get; set; } int PathCompareWithDirTo(ICommonEntry de); - IList Children { get; } + /// + /// Read-only, covariant view of child entries. Abstract so a struct-of-arrays backing + /// (EntryStore via EntryRef) can satisfy it without materialising DirEntry objects. The + /// concrete tree classes still expose a mutable IList<DirEntry> for build/sort/ + /// serialization; this is what consumers see when they hold an . + /// + IReadOnlyList Children { get; } public void AddChild(DirEntry child); @@ -33,8 +39,8 @@ public interface ICommonEntry bool IsDirectory { get; set; } bool PathProblem { get; } - long FileEntryCount { get; set; } - long DirEntryCount { get; set; } + uint FileEntryCount { get; set; } + uint DirEntryCount { get; set; } DateTime Modified { get; set; } bool IsHashDone { get; set; } bool IsPartialHash { get; set; } diff --git a/src/cdeLib/Entities/IEntrySource.cs b/src/cdeLib/Entities/IEntrySource.cs new file mode 100644 index 0000000..51cde7e --- /dev/null +++ b/src/cdeLib/Entities/IEntrySource.cs @@ -0,0 +1,58 @@ +using System; +using System.Collections.Generic; +using System.Text; +using cdeLib.Entities.Soa; + +namespace cdeLib.Entities; + +/// +/// A read-only, index-addressed catalog: the common surface over both the in-memory +/// (built from a loaded .cde tree) and the zero-copy +/// (mmap over a .cdex file). Lets the GUI and +/// present and search a catalog without caring whether it lives on the managed +/// heap or in a memory map. Index 0 is always the catalog root; child/sibling/parent chains terminate +/// at . +/// +public interface IEntrySource +{ + int Count { get; } + + // ----- per-entry accessors ----- + long SizeOf(int i); + DateTime ModifiedOf(int i); + long ModifiedTicksOf(int i); // raw stored ticks (for exact tree reconstruction) + Flags FlagsOf(int i); + bool IsDirectory(int i); + bool IsHashDone(int i); + bool IsPartialHash(int i); + bool HasHash { get; } + Hash16 HashOf(int i); + + string FullName(int i); // name + extension + string NameOf(int i); // name component used for path-problem checks (== FullName when not split) + string FullPath(int i); + void AppendFullPath(StringBuilder sb, int i); + + int ParentOf(int i); + int FirstChildOf(int i); + IEnumerable ChildrenOf(int i); + + // ----- full-filter search (pattern + name/path + file/folder + size/date/hour) ----- + void Find(EntryStoreFindOptions options, Action onMatch, + Func isCancelled = null, Action onScan = null); + + // ----- catalog-level metadata (what the GUI catalog list + result rows display) ----- + string RootPath { get; } + string VolumeName { get; } + string DefaultFileName { get; } + string ActualFileName { get; } + string DriveLetterHint { get; } + string Description { get; } + long AvailSpace { get; } + long TotalSpace { get; } + long ScanStartUtcTicks { get; } + long ScanEndUtcTicks { get; } + long RootSize { get; } + uint RootFileEntryCount { get; } + uint RootDirEntryCount { get; } +} diff --git a/src/cdeLib/Entities/RootEntry.cs b/src/cdeLib/Entities/RootEntry.cs index c425206..ac6a3a2 100644 --- a/src/cdeLib/Entities/RootEntry.cs +++ b/src/cdeLib/Entities/RootEntry.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using System.IO; using System.Linq; +using System.Threading; using cdeLib.Extensions; using cdeLib.Infrastructure; using cdeLib.Infrastructure.Config; @@ -24,6 +25,9 @@ public sealed class RootEntry : object, ICommonEntry private readonly IDriveInfoService _driveInfoService; private readonly IFileSystemAdapter _fileSystemAdapter; + // Set per-scan by RecurseTree; when false, directory reparse points are not descended into. + private bool _followJunctions; + [ProtoMember(2, IsRequired = true)] [FlatBufferItem(2)] [Key(2)] @@ -60,8 +64,8 @@ public sealed class RootEntry : object, ICommonEntry [IgnoreMember] public DateTime ScanStartUtc { - set => ScanStartUtcTicks = value.Ticks; get => DateTime.FromBinary(ScanStartUtcTicks); + set => ScanStartUtcTicks = value.Ticks; } [FlatBufferItem(8)] @@ -72,8 +76,8 @@ public DateTime ScanStartUtc [IgnoreMember] public DateTime ScanEndUtc { - set => ScanEndUtcTicks = value.Ticks; get => DateTime.FromBinary(ScanEndUtcTicks); + set => ScanEndUtcTicks = value.Ticks; } [FlatBufferItem(9)] @@ -105,11 +109,7 @@ public RootEntry() : this(null, null) { } - public RootEntry(IConfiguration configuration) : this(configuration, null) - { - } - - public RootEntry(IConfiguration configuration, IFileSystemAdapter fileSystemAdapter) + public RootEntry(IConfiguration configuration, IFileSystemAdapter fileSystemAdapter = null) { TheRootEntry = this; _driveInfoService = new DriveInfoService(); @@ -120,11 +120,11 @@ public RootEntry(IConfiguration configuration, IFileSystemAdapter fileSystemAdap } } - public void PopulateRoot(string startPath) + public void PopulateRoot(string startPath, bool followJunctions = false, CancellationToken token = default) { startPath = GetRootEntry(startPath); ScanStartUtc = DateTime.UtcNow; - RecurseTree(startPath); + RecurseTree(startPath, followJunctions, token); ScanEndUtc = DateTime.UtcNow; SetInMemoryFields(); } @@ -146,7 +146,7 @@ private string GetRootEntry(string startPath) var driveInfo = _driveInfoService.GetDriveSpace(pathRoot); if (driveInfo.AvailableBytes != null) AvailSpace = driveInfo.AvailableBytes.Value; if (driveInfo.TotalBytes != null) TotalSpace = driveInfo.TotalBytes.Value; - VolumeName = this.GetVolumeName(GetDirectoryRoot(pathRoot)); + VolumeName = GetVolumeName(GetDirectoryRoot(pathRoot)); return startPath; } @@ -286,8 +286,9 @@ private static string SafeFileName(string path) /// /// Iteratively scans a directory tree using a stack-based approach for optimal performance. /// - public void RecurseTree(string startPath) + public void RecurseTree(string startPath, bool followJunctions = false, CancellationToken token = default) { + _followJunctions = followJunctions; var entryCount = 0; var stack = new Stack<(ICommonEntry, string)>(capacity: 64); stack.Push((this, startPath)); @@ -298,12 +299,12 @@ public void RecurseTree(string startPath) { var (parent, directory) = stack.Pop(); - if (TryEnumerateDirectory(directory, parent, stack, ref entryCount, progressTracker)) + if (TryEnumerateDirectory(directory, parent, stack, ref entryCount, progressTracker, token)) { continue; } - if (Hack.BreakConsoleFlag) + if (token.IsCancellationRequested) { break; } @@ -321,7 +322,8 @@ private bool TryEnumerateDirectory( ICommonEntry parent, Stack<(ICommonEntry, string)> stack, ref int entryCount, - ScanProgressTracker progressTracker) + ScanProgressTracker progressTracker, + CancellationToken token) { try { @@ -332,7 +334,7 @@ private bool TryEnumerateDirectory( { ProcessFileSystemEntry(fsInfo, parent, stack, ref entryCount, directory, progressTracker); - if (Hack.BreakConsoleFlag) + if (token.IsCancellationRequested) { break; } @@ -361,7 +363,10 @@ private void ProcessFileSystemEntry( var dirEntry = new DirEntry(fsInfo); parent.AddChild(dirEntry); - if (dirEntry.IsDirectory) + // Reparse points (junctions / directory symlinks) carry the Directory attribute, so they + // would otherwise be descended into. By default we record them but do not follow them, + // avoiding cycles (e.g. a junction pointing at an ancestor) and duplicate content. + if (dirEntry.IsDirectory && (_followJunctions || !dirEntry.IsReparsePoint)) { stack.Push((dirEntry, fsInfo.FullName)); } @@ -475,10 +480,12 @@ public void SortAllChildrenByPath() TraverseTreePair((_, d) => { - if (d.IsDirectory && d.Children?.Count > 1) + // Sorting mutates the concrete child list, so work through the concrete DirEntry + // (the abstract ICommonEntry.Children is a read-only view). + if (d is DirEntry { IsDirectory: true, Children.Count: > 1 } de) { - d.Children.Sort((de1, de2) => de1.PathCompareWithDirTo(de2)); - d.IsDefaultSort = true; + de.Children.Sort((de1, de2) => de1.PathCompareWithDirTo(de2)); + de.IsDefaultSort = true; } return true; @@ -509,8 +516,8 @@ public int DescriptionCompareTo(RootEntry re, IConfigCdeLib config) [IgnoreMember] public DateTime Modified { - set => ModifiedTicks = value.Ticks; get => DateTime.FromBinary(ModifiedTicks); + set => ModifiedTicks = value.Ticks; } [ProtoMember(12, IsRequired = false)] @@ -639,13 +646,13 @@ public bool IsDefaultSort /// if this is a directory number of files contained in its hierarchy /// [IgnoreMember] - public long FileEntryCount { get; set; } + public uint FileEntryCount { get; set; } /// /// if this is a directory number of dirs contained in its hierarchy /// [IgnoreMember] - public long DirEntryCount { get; set; } + public uint DirEntryCount { get; set; } public void SetHash(byte[] hash) { @@ -721,22 +728,13 @@ public int ModifiedCompareTo(ICommonEntry de) return -1; // this before de } - if (IsModifiedBad && !de.IsModifiedBad) - { - return -1; // this before de - } - - if (!IsModifiedBad && de.IsModifiedBad) - { - return 1; // this after de - } - - if (IsModifiedBad && de.IsModifiedBad) + return IsModifiedBad switch { - return 0; - } - - return DateTime.Compare(Modified, de.Modified); + true when !de.IsModifiedBad => -1, + false when de.IsModifiedBad => 1, + true when de.IsModifiedBad => 0, + _ => DateTime.Compare(Modified, de.Modified) + }; } // is this right ? for the simple compareResult invert we do in caller ? - maybe not ? keep dirs at top anyway ? @@ -747,20 +745,15 @@ public int PathCompareWithDirTo(ICommonEntry de) return -1; // this before de } - if (IsDirectory && !de.IsDirectory) + return IsDirectory switch { - return -1; // this before de - } - - if (!IsDirectory && de.IsDirectory) - { - return 1; // this after de - } - - return string.Compare(Path, de.Path, StringComparison.OrdinalIgnoreCase); + true when !de.IsDirectory => -1, + false when de.IsDirectory => 1, + _ => string.Compare(Path, de.Path, StringComparison.OrdinalIgnoreCase) + }; } - // can this be done with TraverseTree ? + // can this be done with TraverseTree? public void SetSummaryFields() { var size = 0L; @@ -805,10 +798,12 @@ public RootEntry GetRootEntry() [Key(15)] public IList Children { get; set; } + // Covariant read-only view for ICommonEntry consumers (see DirEntry for rationale). + IReadOnlyList ICommonEntry.Children => Children as IReadOnlyList; + public void AddChild(DirEntry child) { - if (this.Children == null) - Children = new List(); + Children ??= new List(); Children.Add(child); } @@ -924,7 +919,7 @@ public void TraverseTreesCopyHash(ICommonEntry destination) ValidateTreeCopyParameters(this, destination); var stack = new Stack<(string, ICommonEntry, ICommonEntry)>(capacity: 64); - stack.Push((this.Path, this, destination)); + stack.Push((Path, this, destination)); while (stack.Count > 0) { @@ -988,9 +983,9 @@ private static void ProcessChildrenForHashCopy( /// /// Builds a dictionary for O(1) lookups of destination children by path. /// - private static Dictionary BuildDestinationLookup(IList children) + private static Dictionary BuildDestinationLookup(IReadOnlyList children) { - var lookup = new Dictionary(children.Count, StringComparer.OrdinalIgnoreCase); + var lookup = new Dictionary(children.Count, StringComparer.OrdinalIgnoreCase); foreach (var child in children) { @@ -1030,12 +1025,15 @@ private static void TryCopyHashIfBeneficial(ICommonEntry source, ICommonEntry de } var shouldCopy = !destination.IsHashDone // Destination has no hash - || (source.IsPartialHash == false && destination.IsPartialHash); // Upgrading partial to full + || (!source.IsPartialHash && destination.IsPartialHash); // Upgrading partial to full if (shouldCopy) { destination.IsPartialHash = source.IsPartialHash; destination.Hash = source.Hash; + // IsHashDone is a separate BitFields flag; without it the copied hash is ignored by + // hashing/serialization (the reused hash would be silently lost on the next save). + destination.IsHashDone = true; } } diff --git a/src/cdeLib/Entities/Soa/EntryRef.cs b/src/cdeLib/Entities/Soa/EntryRef.cs new file mode 100644 index 0000000..d4ac9f7 --- /dev/null +++ b/src/cdeLib/Entities/Soa/EntryRef.cs @@ -0,0 +1,195 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace cdeLib.Entities.Soa; + +/// +/// Lightweight adapter presenting a single catalog entry (by index) as an , +/// so existing tree-oriented consumers (GUI display, navigation, dupes read paths) can run on the +/// index-addressed model without materialising a pointer tree. Backs onto any +/// — the in-memory or the zero-copy +/// — so the GUI is agnostic to whether the catalog lives on the heap or in a memory map. Read members +/// map onto the source; build/mutate members throw, since a catalog is produced wholesale by the +/// loader, not edited entry-by-entry. +/// +/// Intended for OCCASIONAL access (displaying a directory, a search result row). Bulk traversal should +/// use index-based search () to avoid per-entry wrapper allocs. +/// +public sealed class EntryRef : ICommonEntry +{ + private readonly IEntrySource _source; + private readonly int _index; + + public EntryRef(IEntrySource source, int index) + { + _source = source; + _index = index; + } + + public IEntrySource Source => _source; + public int Index => _index; + + private static NotSupportedException ReadOnly([System.Runtime.CompilerServices.CallerMemberName] string m = null) + => new($"EntryRef is a read-only view over a catalog source; '{m}' is not supported."); + + public string Path { get => _source.FullName(_index); set => throw ReadOnly(); } + public long Size { get => _source.SizeOf(_index); set => throw ReadOnly(); } + public DateTime Modified { get => _source.ModifiedOf(_index); set => throw ReadOnly(); } + + public bool IsDirectory { get => _source.IsDirectory(_index); set => throw ReadOnly(); } + public bool IsHashDone { get => _source.IsHashDone(_index); set => throw ReadOnly(); } + public bool IsPartialHash { get => _source.IsPartialHash(_index); set => throw ReadOnly(); } + public bool IsModifiedBad + { + get => (_source.FlagsOf(_index) & Flags.ModifiedBad) == Flags.ModifiedBad; + set => throw ReadOnly(); + } + public bool IsReparsePoint + { + get => (_source.FlagsOf(_index) & Flags.ReparsePoint) == Flags.ReparsePoint; + set => throw ReadOnly(); + } + public bool IsDefaultSort { get => true; set => throw ReadOnly(); } // source is built in sorted order + + public Hash16 Hash + { + get => _source.HashOf(_index); + set => throw ReadOnly(); + } + + public string FullPath => _source.FullPath(_index); + + public bool PathProblem + { + get + { + for (var cur = _index; cur != EntryStore.None; cur = _source.ParentOf(cur)) + { + var name = _source.NameOf(cur); + if (!string.IsNullOrEmpty(name) && (name.EndsWith(' ') || name.EndsWith('.'))) return true; + } + return false; + } + } + + public IReadOnlyList Children + { + get + { + // Gate on having children, not on the directory flag: the root is not flagged a + // directory yet has children (matching RootEntry), and a file simply has none. + if (_source.FirstChildOf(_index) == EntryStore.None) return null; + List list = null; + foreach (var c in _source.ChildrenOf(_index)) + { + (list ??= []).Add(new EntryRef(_source, c)); + } + return list; + } + } + + public ICommonEntry ParentCommonEntry + { + get + { + var p = _source.ParentOf(_index); + return p == EntryStore.None ? null : new EntryRef(_source, p); + } + set => throw ReadOnly(); + } + + public uint FileEntryCount { get => CountSubtree().Files; set => throw ReadOnly(); } + public uint DirEntryCount { get => CountSubtree().Dirs; set => throw ReadOnly(); } + + private (uint Files, uint Dirs) CountSubtree() + { + uint files = 0, dirs = 0; + var stack = new Stack(); + stack.Push(_index); + while (stack.Count > 0) + { + var n = stack.Pop(); + foreach (var c in _source.ChildrenOf(n)) + { + if (_source.IsDirectory(c)) { dirs++; stack.Push(c); } + else files++; + } + } + return (files, dirs); + } + + public int PathCompareWithDirTo(ICommonEntry de) + { + if (de == null) return -1; + return IsDirectory switch + { + true when !de.IsDirectory => -1, + false when de.IsDirectory => 1, + _ => string.Compare(Path, de.Path, StringComparison.OrdinalIgnoreCase) + }; + } + + public int SizeCompareWithDirTo(ICommonEntry de) + { + if (de == null) return -1; + if (IsDirectory && !de.IsDirectory) return -1; + if (!IsDirectory && de.IsDirectory) return 1; + var c = Size.CompareTo(de.Size); + return c != 0 ? c : string.Compare(Path, de.Path, StringComparison.OrdinalIgnoreCase); + } + + public int ModifiedCompareTo(ICommonEntry de) + { + if (de == null) return -1; + if (IsModifiedBad && !de.IsModifiedBad) return -1; + if (!IsModifiedBad && de.IsModifiedBad) return 1; + if (IsModifiedBad && de.IsModifiedBad) return 0; + return DateTime.Compare(Modified, de.Modified); + } + + public string MakeFullPath(ICommonEntry dirEntry) + { + var parent = FullPath; + var name = dirEntry?.Path ?? "dnull"; + if (parent.Length > 0 && parent[^1] != '\\' && parent[^1] != '/') + return string.Concat(parent, System.IO.Path.DirectorySeparatorChar.ToString(), name); + return string.Concat(parent, name); + } + + public IList GetListFromRoot() + { + var list = new List(8); + for (var cur = _index; cur != EntryStore.None; cur = _source.ParentOf(cur)) + { + list.Add(new EntryRef(_source, cur)); + } + list.Reverse(); + return list; + } + + public bool ExistsOnFileSystem() => Directory.Exists(FullPath); + + public void TraverseTreePair(TraverseFunc func) + { + if (func == null) return; + var stack = new Stack(); + stack.Push(_index); + while (stack.Count > 0) + { + var n = stack.Pop(); + var parentRef = new EntryRef(_source, n); + foreach (var c in _source.ChildrenOf(n)) + { + if (!func(parentRef, new EntryRef(_source, c))) return; + if (_source.IsDirectory(c)) stack.Push(c); + } + } + } + + // ----- build / mutate members: not supported on a read-only source view ----- + public void AddChild(DirEntry child) => throw ReadOnly(); + public void SetSummaryFields() => throw ReadOnly(); + public void SetHash(byte[] hashResponseHash) => throw ReadOnly(); + public void TraverseTreesCopyHash(ICommonEntry destination) => throw ReadOnly(); +} diff --git a/src/cdeLib/Entities/Soa/EntryStore.cs b/src/cdeLib/Entities/Soa/EntryStore.cs new file mode 100644 index 0000000..48a95db --- /dev/null +++ b/src/cdeLib/Entities/Soa/EntryStore.cs @@ -0,0 +1,294 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace cdeLib.Entities.Soa; + +/// +/// Struct-of-arrays representation of a single catalog: one slot per entry across parallel arrays, +/// with the tree shape encoded as indices (firstChild / nextSibling / parent) +/// instead of object references. Holds the same catalog structure as a tree +/// in roughly one third of the structural memory — no per-entry object header, and references shrink +/// from 8-byte pointers to 4-byte indices. +/// +/// Phase 1 of the SoA migration: this is additive. converts an existing tree into +/// a store; the production tree model is unchanged. Later phases move load/search/serialization onto +/// the store and retire the tree. +/// +/// Index 0 is always the catalog root. (-1) terminates child/sibling chains and +/// marks the root's (absent) parent. +/// +public sealed class EntryStore : IEntrySource +{ + public const int None = -1; + + public int Count { get; private set; } + + // One slot per entry (index 0 = root). + public long[] ModifiedTicks { get; private set; } + public long[] Size { get; private set; } + + // Names are stored split (name-without-extension + extension) reusing the SAME interned string + // objects the source tree held — so conversion allocates no new name strings and the interned + // originals are shared, not duplicated. FullName(i) rejoins on demand. + public string[] Name { get; private set; } + public string[] Ext { get; private set; } + + public byte[] BitFields { get; private set; } + public int[] FirstChild { get; private set; } + public int[] NextSibling { get; private set; } + public int[] Parent { get; private set; } + + /// + /// Content hashes, allocated lazily only when at least one entry is hashed. Null for the common + /// un-hashed catalog (load-to-search), so an un-hashed store pays zero bytes here — unlike the + /// inline 16-byte Hash16 carried by every DirEntry today. + /// + public Hash16[] Hash { get; private set; } + + // ----- catalog-level metadata (what the GUI catalog list and search-result rows display) ----- + public string RootPath { get; set; } // root path, e.g. C:\ (also Name[0]) + public string VolumeName { get; set; } + public string DefaultFileName { get; set; } // generated .cde name + public string ActualFileName { get; set; } // path of the loaded .cde + public string DriveLetterHint { get; set; } + public string Description { get; set; } + public long AvailSpace { get; set; } + public long TotalSpace { get; set; } + public long ScanStartUtcTicks { get; set; } + public long ScanEndUtcTicks { get; set; } + public long RootSize { get; set; } // total size of the catalog + public uint RootFileEntryCount { get; set; } // total files in the catalog + public uint RootDirEntryCount { get; set; } // total directories in the catalog + + private EntryStore(int count) + { + Count = count; + ModifiedTicks = new long[count]; + Size = new long[count]; + Name = new string[count]; + Ext = new string[count]; + BitFields = new byte[count]; + FirstChild = new int[count]; + NextSibling = new int[count]; + Parent = new int[count]; + } + + /// Full entry name (name + extension), rejoined on demand like DirEntry.Path. + public string FullName(int i) => string.IsNullOrEmpty(Ext[i]) ? Name[i] : string.Concat(Name[i], Ext[i]); + + public Flags Flags(int i) => (Flags)BitFields[i]; + + // ----- IEntrySource: thin index-addressed accessors over the parallel arrays ----- + public long SizeOf(int i) => Size[i]; + public DateTime ModifiedOf(int i) => Modified(i); + public long ModifiedTicksOf(int i) => ModifiedTicks[i]; + public Flags FlagsOf(int i) => Flags(i); + public bool HasHash => Hash != null; + public Hash16 HashOf(int i) => Hash != null ? Hash[i] : default; + public string NameOf(int i) => Name[i]; + public int ParentOf(int i) => Parent[i]; + public int FirstChildOf(int i) => FirstChild[i]; + public IEnumerable ChildrenOf(int i) => Children(i); + + /// Full-filter search over this store (delegates to ). + public void Find(EntryStoreFindOptions options, Action onMatch, + Func isCancelled = null, Action onScan = null) + => EntryStoreSearch.Find(this, options, onMatch, isCancelled, onScan); + + public bool IsDirectory(int i) => (Flags(i) & Entities.Flags.Directory) == Entities.Flags.Directory; + public bool IsHashDone(int i) => (Flags(i) & Entities.Flags.HashDone) == Entities.Flags.HashDone; + public bool IsPartialHash(int i) => (Flags(i) & Entities.Flags.PartialHash) == Entities.Flags.PartialHash; + public DateTime Modified(int i) => DateTime.FromBinary(ModifiedTicks[i]); + + /// Enumerate the direct child indices of via the sibling chain. + public IEnumerable Children(int parent) + { + for (var c = FirstChild[parent]; c != None; c = NextSibling[c]) + { + yield return c; + } + } + + /// + /// Build the full path of entry into the supplied StringBuilder by walking + /// parent indices (root-first). No intermediate per-node strings. Pass a reused builder on hot paths. + /// + public void AppendFullPath(StringBuilder sb, int i) + { + // Walk leaf -> root collecting indices, then emit root-first. + var depth = 0; + for (var cur = i; cur != None; cur = Parent[cur]) depth++; + if (depth == 0) return; + + Span chain = depth <= 64 ? stackalloc int[depth] : new int[depth]; + var n = 0; + for (var cur = i; cur != None; cur = Parent[cur]) chain[n++] = cur; + + for (var k = depth - 1; k >= 0; k--) + { + var idx = chain[k]; + if (sb.Length > 0) + { + var last = sb[^1]; + if (last != '\\' && last != '/') sb.Append(System.IO.Path.DirectorySeparatorChar); + } + // Append the split name parts directly — no full-name string allocation for path building. + sb.Append(Name[idx] ?? string.Empty); + if (!string.IsNullOrEmpty(Ext[idx])) sb.Append(Ext[idx]); + } + } + + /// + /// Write the full path of entry into (root-first), + /// mirroring but without allocating a string. Returns the number of + /// chars written, or -1 if is too small (the caller should grow and retry). + /// Lets hot-path callers run comparisons with no per-entry path allocation. + /// + public int TryWriteFullPath(Span dest, int i) + { + var depth = 0; + for (var cur = i; cur != None; cur = Parent[cur]) depth++; + if (depth == 0) return 0; + + Span chain = depth <= 64 ? stackalloc int[depth] : new int[depth]; + var n = 0; + for (var cur = i; cur != None; cur = Parent[cur]) chain[n++] = cur; + + var pos = 0; + for (var k = depth - 1; k >= 0; k--) + { + var idx = chain[k]; + if (pos > 0) + { + var last = dest[pos - 1]; + if (last != '\\' && last != '/') + { + if (pos >= dest.Length) return -1; + dest[pos++] = System.IO.Path.DirectorySeparatorChar; + } + } + + var name = Name[idx]; + if (!string.IsNullOrEmpty(name)) + { + if (pos + name.Length > dest.Length) return -1; + name.AsSpan().CopyTo(dest[pos..]); + pos += name.Length; + } + + var ext = Ext[idx]; + if (!string.IsNullOrEmpty(ext)) + { + if (pos + ext.Length > dest.Length) return -1; + ext.AsSpan().CopyTo(dest[pos..]); + pos += ext.Length; + } + } + + return pos; + } + + public string FullPath(int i) + { + var sb = new StringBuilder(128); + AppendFullPath(sb, i); + return sb.ToString(); + } + + /// + /// Convert a loaded/generated catalog tree into a store. The tree may be released afterwards. + /// Iterative (explicit stack) so very deep trees cannot overflow. + /// + public static EntryStore Build(RootEntry root) + { + ArgumentNullException.ThrowIfNull(root); + + var count = CountEntries(root); // robust: counts the actual tree, not (possibly stale) summary fields + var store = new EntryStore(count) + { + RootPath = root.Path, + VolumeName = root.VolumeName, + DefaultFileName = root.DefaultFileName, + ActualFileName = root.ActualFileName, + DriveLetterHint = root.DriveLetterHint, + Description = root.Description, + AvailSpace = root.AvailSpace, + TotalSpace = root.TotalSpace, + ScanStartUtcTicks = root.ScanStartUtcTicks, + ScanEndUtcTicks = root.ScanEndUtcTicks, + RootSize = root.Size, + RootFileEntryCount = root.FileEntryCount, + RootDirEntryCount = root.DirEntryCount, + }; + + var next = 0; + var rootIdx = next++; + store.Name[rootIdx] = root.Path; // root path is not split + store.ModifiedTicks[rootIdx] = root.ModifiedTicks; + store.Size[rootIdx] = root.Size; + store.BitFields[rootIdx] = (byte)root.BitFields; + store.Parent[rootIdx] = None; + store.FirstChild[rootIdx] = None; + store.NextSibling[rootIdx] = None; + + // Stack of (children-of-a-directory, that directory's index). + var stack = new Stack<(IList Children, int ParentIdx)>(); + stack.Push((root.Children, rootIdx)); + + while (stack.Count > 0) + { + var (children, parentIdx) = stack.Pop(); + if (children == null) continue; + + var prevSibling = None; + foreach (var child in children) + { + var idx = next++; + // Reuse the child's already-interned name + extension objects (no fresh allocation). + store.Name[idx] = child.NamePart; + store.Ext[idx] = child.ExtPart; + store.ModifiedTicks[idx] = child.ModifiedTicks; + store.Size[idx] = child.Size; + store.BitFields[idx] = (byte)child.BitFields; + store.Parent[idx] = parentIdx; + store.FirstChild[idx] = None; + store.NextSibling[idx] = None; + if (child.IsHashDone) store.SetHash(idx, child.Hash); + + if (prevSibling == None) store.FirstChild[parentIdx] = idx; + else store.NextSibling[prevSibling] = idx; + prevSibling = idx; + + if (child.IsDirectory) stack.Push((child.Children, idx)); + } + } + + return store; + } + + /// Count every entry in the tree (including the root) via an explicit stack. + private static int CountEntries(RootEntry root) + { + var count = 1; // the root + var stack = new Stack>(); + stack.Push(root.Children); + while (stack.Count > 0) + { + var children = stack.Pop(); + if (children == null) continue; + count += children.Count; + foreach (var child in children) + { + if (child.IsDirectory) stack.Push(child.Children); + } + } + return count; + } + + private void SetHash(int i, Hash16 hash) + { + Hash ??= new Hash16[Count]; + Hash[i] = hash; + } +} diff --git a/src/cdeLib/Entities/Soa/EntryStoreFindOptions.cs b/src/cdeLib/Entities/Soa/EntryStoreFindOptions.cs new file mode 100644 index 0000000..5dd3112 --- /dev/null +++ b/src/cdeLib/Entities/Soa/EntryStoreFindOptions.cs @@ -0,0 +1,35 @@ +using System; + +namespace cdeLib.Entities.Soa; + +/// +/// Filter parameters for , mirroring the subset of the tree-based +/// FindOptions that the cdeWin GUI search uses (pattern + name/path + file/folder + size/date/hour +/// ranges). Evaluated directly against the store's Size[] and ModifiedTicks[] arrays. +/// +public sealed class EntryStoreFindOptions +{ + public string Pattern { get; set; } + public bool RegexMode { get; set; } + public bool IncludePath { get; set; } + public bool IncludeFiles { get; set; } = true; + public bool IncludeFolders { get; set; } = true; + + public bool FromSizeEnable { get; set; } + public long FromSize { get; set; } + public bool ToSizeEnable { get; set; } + public long ToSize { get; set; } + + public bool FromDateEnable { get; set; } + public DateTime FromDate { get; set; } + public bool ToDateEnable { get; set; } + public DateTime ToDate { get; set; } + + public bool FromHourEnable { get; set; } + public TimeSpan FromHour { get; set; } + public bool ToHourEnable { get; set; } + public TimeSpan ToHour { get; set; } + + public bool NotOlderThanEnable { get; set; } + public DateTime NotOlderThan { get; set; } +} diff --git a/src/cdeLib/Entities/Soa/EntryStoreSearch.cs b/src/cdeLib/Entities/Soa/EntryStoreSearch.cs new file mode 100644 index 0000000..c066cf1 --- /dev/null +++ b/src/cdeLib/Entities/Soa/EntryStoreSearch.cs @@ -0,0 +1,171 @@ +using System; +using System.Buffers; +using System.Text.RegularExpressions; + +namespace cdeLib.Entities.Soa; + +/// +/// Index-based find over an — a linear scan of the flat arrays (cache +/// friendly, no per-entry indirection). Mirrors the core matching of the pointer-tree +/// FindOptions (substring/regex, name/path, file/folder filter) so the two can be proven +/// equivalent before the production search is moved onto the store. +/// +public static class EntryStoreSearch +{ + /// + /// Full-filter search (pattern + name/path + file/folder + size/date/hour ranges), mirroring the + /// cdeWin GUI search, evaluated directly against the store arrays. + /// + /// Polled every 4096 entries; return true to stop early (GUI cancel). + /// Called every 4096 entries with the running scanned count (GUI progress). + public static void Find(EntryStore store, EntryStoreFindOptions o, Action onMatch, + Func isCancelled = null, Action onScan = null) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(o); + if (!o.IncludeFiles && !o.IncludeFolders) return; + + Regex regex = null; + if (o.RegexMode && !string.IsNullOrEmpty(o.Pattern)) + regex = new Regex(o.Pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.Compiled); + + var hasPattern = !string.IsNullOrEmpty(o.Pattern); + + // Substring path matching runs Span.Contains over a rented buffer; only regex mode + // (rarer) materialises a string. Avoids a full-path string allocation per scanned entry. + var pathBuffer = o.IncludePath ? ArrayPool.Shared.Rent(512) : null; + try + { + for (var i = 1; i < store.Count; i++) + { + if ((i & 4095) == 0) + { + if (isCancelled != null && isCancelled()) return; + onScan?.Invoke(i); + } + + var isDir = store.IsDirectory(i); + if (isDir ? !o.IncludeFolders : !o.IncludeFiles) continue; + + var size = store.Size[i]; + if (o.FromSizeEnable && size < o.FromSize) continue; + if (o.ToSizeEnable && size > o.ToSize) continue; + + if (o.FromDateEnable || o.ToDateEnable || o.FromHourEnable || o.ToHourEnable || o.NotOlderThanEnable) + { + var modified = store.Modified(i); + if (o.FromDateEnable && modified < o.FromDate) continue; + if (o.ToDateEnable && modified > o.ToDate) continue; + if (o.NotOlderThanEnable && modified < o.NotOlderThan) continue; + var tod = modified.TimeOfDay; + if (o.FromHourEnable && tod < o.FromHour) continue; + if (o.ToHourEnable && tod > o.ToHour) continue; + } + + if (!hasPattern) { onMatch(i); continue; } + + bool match; + if (o.IncludePath) + { + var path = WritePath(store, i, ref pathBuffer); + match = o.RegexMode + ? regex.IsMatch(path.ToString()) + : path.Contains(o.Pattern, StringComparison.OrdinalIgnoreCase); + } + else + { + var name = store.FullName(i); + match = o.RegexMode + ? regex.IsMatch(name) + : name.Contains(o.Pattern, StringComparison.OrdinalIgnoreCase); + } + + if (match) onMatch(i); + } + } + finally + { + if (pathBuffer != null) ArrayPool.Shared.Return(pathBuffer); + } + } + + /// + /// Write entry 's full path into (rented), growing + /// and re-renting if it doesn't fit, and return the written span. The grown buffer is passed back + /// via so the caller reuses it for subsequent entries. + /// + private static ReadOnlySpan WritePath(EntryStore store, int i, ref char[] buffer) + { + int len; + while ((len = store.TryWriteFullPath(buffer, i)) < 0) + { + var bigger = ArrayPool.Shared.Rent(buffer.Length * 2); + ArrayPool.Shared.Return(buffer); + buffer = bigger; + } + + return buffer.AsSpan(0, len); + } + + /// Invoke with the index of every entry matching the query. + public static void Find( + EntryStore store, + string pattern, + bool regexMode, + bool includePath, + bool includeFiles, + bool includeFolders, + Action onMatch) + { + ArgumentNullException.ThrowIfNull(store); + ArgumentNullException.ThrowIfNull(onMatch); + + if (!includeFiles && !includeFolders) return; + + Regex regex = null; + if (regexMode && !string.IsNullOrEmpty(pattern)) + { + regex = new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.Singleline | RegexOptions.Compiled); + } + + var hasPattern = !string.IsNullOrEmpty(pattern); + + var pathBuffer = includePath ? ArrayPool.Shared.Rent(512) : null; + try + { + for (var i = 1; i < store.Count; i++) // index 0 is the root, never a result + { + var isDir = store.IsDirectory(i); + if (isDir ? !includeFolders : !includeFiles) continue; + + if (!hasPattern) + { + onMatch(i); + continue; + } + + bool match; + if (includePath) + { + var path = WritePath(store, i, ref pathBuffer); + match = regexMode + ? regex.IsMatch(path.ToString()) + : path.Contains(pattern, StringComparison.OrdinalIgnoreCase); + } + else + { + var name = store.FullName(i); + match = regexMode + ? regex.IsMatch(name) + : name.Contains(pattern, StringComparison.OrdinalIgnoreCase); + } + + if (match) onMatch(i); + } + } + finally + { + if (pathBuffer != null) ArrayPool.Shared.Return(pathBuffer); + } + } +} diff --git a/src/cdeLib/Extensions/ListExtensions.cs b/src/cdeLib/Extensions/ListExtensions.cs index fcb7153..115f3cc 100644 --- a/src/cdeLib/Extensions/ListExtensions.cs +++ b/src/cdeLib/Extensions/ListExtensions.cs @@ -19,22 +19,21 @@ public static void TruncateList(this IList iList, int max) /// public static void Sort(this IList list, Comparison comparison) { - // Fast path: List has optimized Sort implementation - if (list is List concreteList) + switch (list) { - concreteList.Sort(comparison); - return; + // Fast path: List has optimized Sort implementation + case List concreteList: + concreteList.Sort(comparison); + return; + // Fast path: Array has optimized Sort implementation + case T[] array: + Array.Sort(array, comparison); + return; + default: + // Slow path: Generic IList - must copy, sort, and copy back + SortGenericList(list, comparison); + break; } - - // Fast path: Array has optimized Sort implementation - if (list is T[] array) - { - Array.Sort(array, comparison); - return; - } - - // Slow path: Generic IList - must copy, sort, and copy back - SortGenericList(list, comparison); } /// diff --git a/src/cdeLib/FindOptions.cs b/src/cdeLib/FindOptions.cs index f94b1cc..49090e0 100644 --- a/src/cdeLib/FindOptions.cs +++ b/src/cdeLib/FindOptions.cs @@ -65,6 +65,15 @@ public class FindOptions private readonly int[] _dummyProgressCount = new int[1]; + // Run cancellation + progress housekeeping every 4096 entries (mask = 4096-1). Frequent enough to + // feel instant even on a slow regex, cheap enough to be negligible on a fast full scan. + private const int CancelCheckMask = 4096 - 1; + + // Stream progress/results at most every ~100ms (time-based, like the old async path), so a long + // search updates the UI smoothly rather than in large infrequent entry-count-based chunks. + // Stored as a tick threshold so the hot-path check is a plain subtraction (no multiply/overflow). + private static readonly long ProgressIntervalTicks = Stopwatch.Frequency / 10; + public int SkipCount { get; set; } public int ProgressCount => _threadSafeProgressCount; @@ -125,7 +134,7 @@ public void Find(IEnumerable rootEntries) var findFunc = GetFindFunc(_dummyProgressCount, limitCount); // ReSharper disable PossibleMultipleEnumeration - Parallel.ForEach(sortedRootEntries, parallelOptions, (rootEntry) => + Parallel.ForEach(sortedRootEntries, parallelOptions, rootEntry => { // Use single-entry overload to avoid array allocation EntryHelper.TraverseTreePair(rootEntry, findFunc); @@ -200,6 +209,12 @@ private Func> CreateAsyncProcessor(int[] return true; // Skip enforced } + // Honour cancellation promptly, independently of throttled progress reporting (see GetFindFunc). + if ((currentCount & CancelCheckMask) == 0 && Worker?.CancellationPending == true) + { + return false; + } + // Rate-limited progress reporting with non-blocking UI update if (ProgressModifier > 0 && ShouldReportProgress(currentCount)) { @@ -286,15 +301,18 @@ private Func GetPatternMatcher() : (p, d) => regex.IsMatch(d.Path); } - // String matching with StringComparison for better performance + // String matching with StringComparison for better performance. + // Path mode uses the allocation-free span matcher (avoids building a full-path string per + // candidate — the dominant allocator in path queries; see search baseline). return includePath - ? (p, d) => EntryHelper.MakeFullPathPooled(p, d).Contains(pattern, StringComparison.OrdinalIgnoreCase) + ? (p, d) => EntryHelper.FullPathContains(p, d, pattern, StringComparison.OrdinalIgnoreCase) : (p, d) => d.Path.Contains(pattern, StringComparison.OrdinalIgnoreCase); } private TraverseFunc GetFindFunc(int[] progressCount, int[] limitCount) { var findPredicate = GetFindPredicate(); + return FindFunc; bool FindFunc(ICommonEntry p, ICommonEntry dirEntry) { @@ -307,15 +325,30 @@ bool FindFunc(ICommonEntry p, ICommonEntry dirEntry) return true; } - // Use lock-free progress reporting with reduced frequency - if (ProgressModifier > 0 && ShouldReportProgress(currentCount)) + // Periodic housekeeping behind a cheap entry-count gate (~every 4096 entries): + // 1. Honour cancellation promptly (was tied to the ~50k-entry progress throttle, which + // made a slow search ignore Cancel for tens of thousands of entries). + // 2. Stream progress/results on a ~100ms timer so results appear smoothly during a long + // search instead of in large infrequent chunks. This matches the responsiveness of + // the old async path (which felt faster purely because it streamed every 100ms), + // while keeping the synchronous path's much higher raw throughput. + if ((currentCount & CancelCheckMask) == 0) { - ProgressFunc(currentCount, ProgressEnd); - // only check for cancel on progress reports. if (Worker?.CancellationPending == true) { return false; // end the find. } + + if (ProgressFunc != null && ProgressModifier > 0) + { + var now = Stopwatch.GetTimestamp(); + var last = Interlocked.Read(ref _lastProgressTimestamp); + if (now - last >= ProgressIntervalTicks + && Interlocked.CompareExchange(ref _lastProgressTimestamp, now, last) == last) + { + ProgressFunc(currentCount, ProgressEnd); + } + } } if (findPredicate(p, dirEntry)) @@ -328,8 +361,6 @@ bool FindFunc(ICommonEntry p, ICommonEntry dirEntry) return true; } - - return FindFunc; } public void ResetProgress() diff --git a/src/cdeLib/FindService.cs b/src/cdeLib/FindService.cs index 6ee59dc..5cf0c02 100644 --- a/src/cdeLib/FindService.cs +++ b/src/cdeLib/FindService.cs @@ -1,7 +1,10 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Threading.Tasks; using cdeLib.Entities; +using cdeLib.Entities.Columnar; +using cdeLib.Entities.Soa; using Serilog; namespace cdeLib; @@ -10,6 +13,12 @@ public interface IFindService { void Find(string pattern, string param, IList rootEntries); void Find(string pattern, bool regexMode, bool includePath, IList rootEntries); + + /// + /// Search columnar .cdex catalogs zero-copy over their memory maps (no managed catalog + /// load). Mirrors result semantics. + /// + void FindColumnar(string pattern, string param, IList readers); Task FindAsync(string pattern, string param, IList rootEntries); Task FindAsync(string pattern, bool regexMode, bool includePath, IList rootEntries); @@ -44,41 +53,72 @@ public void Find(string pattern, string param, IList rootEntries) public void Find(string pattern, bool regexMode, bool includePath, IList rootEntries) { - // Use async version for better performance - FindAsync(pattern, regexMode, includePath, rootEntries).GetAwaiter().GetResult(); + // Convert each loaded catalog to the struct-of-arrays EntryStore and release its pointer + // tree before searching. The store holds the same catalog in ~1/3 the structural memory + // (42 vs 129 bytes/entry; see src/cdeBenchmarks/baseline/soa-prototype.md), and the + // index-based scan is cache friendly. The CLI find applies only pattern + name/path + + // file/folder filtering, all of which EntryStoreSearch supports. + var stores = new List(rootEntries.Count); + for (var i = 0; i < rootEntries.Count; i++) + { + if (rootEntries[i] != null) stores.Add(EntryStore.Build(rootEntries[i])); + rootEntries[i] = null; // drop the tree so it can be collected while we search the stores + } + + var totalFound = 0L; + var timer = Stopwatch.StartNew(); + foreach (var store in stores) + { + EntryStoreSearch.Find(store, pattern, regexMode, includePath, IncludeFiles, IncludeFolders, + idx => + { + ++totalFound; + Console.WriteLine(" {0}", store.FullPath(idx)); + }); + } + + timer.Stop(); + Log.Logger.Information( + "Search Execution Time: {ExecutionTime}, Matching pattern {Pattern}, Total found {TotalFound}", + timer.ElapsedMilliseconds, pattern, totalFound); } - public async Task FindAsync(string pattern, string param, IList rootEntries) + public void FindColumnar(string pattern, string param, IList readers) { var regexMode = param is ParamGrep or ParamGrepPath; var includePath = param is ParamGrepPath or ParamFindPath; - await FindAsync(pattern, regexMode, includePath, rootEntries); - } - public async Task FindAsync(string pattern, bool regexMode, bool includePath, IList rootEntries) - { var totalFound = 0L; - var findOptions = new FindOptions + var timer = Stopwatch.StartNew(); + foreach (var reader in readers) { - Pattern = pattern, - RegexMode = regexMode, - IncludePath = includePath, - IncludeFiles = IncludeFiles, - IncludeFolders = IncludeFolders, - LimitResultCount = int.MaxValue, - VisitorFunc = (p, d) => - { - ++totalFound; - Console.WriteLine(" {0}", p.MakeFullPath(d)); - return true; - }, - }; - - var timer = System.Diagnostics.Stopwatch.StartNew(); - await findOptions.FindAsync(rootEntries); + reader?.Find(pattern, regexMode, includePath, IncludeFiles, IncludeFolders, + idx => + { + ++totalFound; + Console.WriteLine(" {0}", reader.FullPath(idx)); + }); + } + timer.Stop(); Log.Logger.Information( "Search Execution Time: {ExecutionTime}, Matching pattern {Pattern}, Total found {TotalFound}", timer.ElapsedMilliseconds, pattern, totalFound); } + + public Task FindAsync(string pattern, string param, IList rootEntries) + { + var regexMode = param is ParamGrep or ParamGrepPath; + var includePath = param is ParamGrepPath or ParamFindPath; + return FindAsync(pattern, regexMode, includePath, rootEntries); + } + + public Task FindAsync(string pattern, bool regexMode, bool includePath, IList rootEntries) + { + // Search is CPU-bound; the synchronous path is the fast one. Keep the async signature for + // API compatibility but run the fast core. Callers wanting off-thread execution should + // wrap this in Task.Run themselves. + Find(pattern, regexMode, includePath, rootEntries); + return Task.CompletedTask; + } } \ No newline at end of file diff --git a/src/cdeLib/Hack.cs b/src/cdeLib/Hack.cs deleted file mode 100644 index cdc8d20..0000000 --- a/src/cdeLib/Hack.cs +++ /dev/null @@ -1,7 +0,0 @@ - -namespace cdeLib; - -public static class Hack -{ - public static volatile bool BreakConsoleFlag; // False is default -} \ No newline at end of file diff --git a/src/cdeLib/Hashing/HashCatalogCommandHandler.cs b/src/cdeLib/Hashing/HashCatalogCommandHandler.cs index 290f432..94516be 100644 --- a/src/cdeLib/Hashing/HashCatalogCommandHandler.cs +++ b/src/cdeLib/Hashing/HashCatalogCommandHandler.cs @@ -3,6 +3,9 @@ using System.Threading.Tasks; using cdeLib.Catalog; using cdeLib.Duplicates; +using cdeLib.Entities; +using cdeLib.Entities.Columnar; +using cdeLib.Entities.Soa; using cdeLib.Infrastructure; using JetBrains.Annotations; using SlimMessageBus; @@ -28,9 +31,18 @@ public HashCatalogCommandHandler(Serilog.ILogger logger, IApplicationDiagnostics public async Task OnHandle(HashCatalogCommand request, CancellationToken cancellationToken) { + // Hash operates on the columnar .cdex catalogs. The hashing engine is tree-based, so each + // catalog is reconstructed into a mutable tree, hashed, then written back as a fresh .cdex. + var cdexFiles = _catalogRepository.GetColumnarFileList(["./"]); + if (cdexFiles.Count == 0) + { + _logger.Warning("No .cdex catalogs found. Run 'cde migrate' to create them first."); + return; + } + _logger.Information("Memory pre-catalog load: {MemoryAllocated}", _applicationDiagnostics.GetMemoryAllocated().FormatAsBytes()); - var rootEntries = _catalogRepository.LoadCurrentDirCache(); + var rootEntries = CatalogTreeBuilder.FromColumnarFiles(cdexFiles); _logger.Information("Memory post-catalog load: {MemoryAllocated}", _applicationDiagnostics.GetMemoryAllocated().FormatAsBytes()); var stopwatch = Stopwatch.StartNew(); @@ -38,8 +50,8 @@ public async Task OnHandle(HashCatalogCommand request, CancellationToken cancell foreach (var rootEntry in rootEntries) { - _logger.Information("Saving catalog {Filename}", rootEntry.DefaultFileName); - await _catalogRepository.Save(rootEntry).ConfigureAwait(false); + _logger.Information("Saving catalog {Filename}", rootEntry.ActualFileName); + ColumnarFormat.Write(EntryStore.Build(rootEntry), rootEntry.ActualFileName); } var ts = stopwatch.Elapsed; diff --git a/src/cdeLib/Hashing/HashCompletedEvent.cs b/src/cdeLib/Hashing/HashCompletedEvent.cs new file mode 100644 index 0000000..4586040 --- /dev/null +++ b/src/cdeLib/Hashing/HashCompletedEvent.cs @@ -0,0 +1,3 @@ +namespace cdeLib.Hashing; + +public record HashCompletedEvent; diff --git a/src/cdeLib/Hashing/HashProgressEvent.cs b/src/cdeLib/Hashing/HashProgressEvent.cs new file mode 100644 index 0000000..ebc0e58 --- /dev/null +++ b/src/cdeLib/Hashing/HashProgressEvent.cs @@ -0,0 +1,3 @@ +namespace cdeLib.Hashing; + +public record HashProgressEvent(long FilesProcessed, long FilesToHash, string Phase); diff --git a/src/cdeLib/Hashing/HashStatusMessageEvent.cs b/src/cdeLib/Hashing/HashStatusMessageEvent.cs new file mode 100644 index 0000000..0ee1585 --- /dev/null +++ b/src/cdeLib/Hashing/HashStatusMessageEvent.cs @@ -0,0 +1,3 @@ +namespace cdeLib.Hashing; + +public record HashStatusMessageEvent(string Message); diff --git a/src/cdeLib/Infrastructure/Config/AppConfigurationSection.cs b/src/cdeLib/Infrastructure/Config/AppConfigurationSection.cs index 84d7570..e531073 100644 --- a/src/cdeLib/Infrastructure/Config/AppConfigurationSection.cs +++ b/src/cdeLib/Infrastructure/Config/AppConfigurationSection.cs @@ -4,8 +4,8 @@ public class AppConfigurationSection { public AppConfigurationSection() { - this.Display = new DisplaySection(); - this.Hashing = new HashingSection(); + Display = new DisplaySection(); + Hashing = new HashingSection(); } public DisplaySection Display { get; set; } diff --git a/src/cdeLib/Infrastructure/FileSystemAdapter.cs b/src/cdeLib/Infrastructure/FileSystemAdapter.cs index 2dd7bbd..e11b91e 100644 --- a/src/cdeLib/Infrastructure/FileSystemAdapter.cs +++ b/src/cdeLib/Infrastructure/FileSystemAdapter.cs @@ -14,7 +14,7 @@ public string GetFullPath(string path) public bool IsUnc(string path) { - return Path.IsPathFullyQualified(path) && path.StartsWith("\\\\"); + return Path.IsPathFullyQualified(path) && path.StartsWith(@"\\"); } public string GetDirectoryRoot(string path) diff --git a/src/cdeLib/Infrastructure/Hashing/HashHelper.cs b/src/cdeLib/Infrastructure/Hashing/HashHelper.cs index b002ede..96d6b5f 100644 --- a/src/cdeLib/Infrastructure/Hashing/HashHelper.cs +++ b/src/cdeLib/Infrastructure/Hashing/HashHelper.cs @@ -45,7 +45,7 @@ public async Task GetHashResponseFromFile(string filename, int? by totalBytesRead = bytesRead; while (bytesRead > 0 && totalBytesRead <= bytesToHash) { - bytesRead = stream.Read(rentedBuffer, 0, bufferSize); + bytesRead = await stream.ReadAsync(rentedBuffer.AsMemory(0, bufferSize)); totalBytesRead += bytesRead; } diff --git a/src/cdeLib/Infrastructure/Hashing/MurMurHash3.cs b/src/cdeLib/Infrastructure/Hashing/MurMurHash3.cs index be3ec60..b2b3b90 100644 --- a/src/cdeLib/Infrastructure/Hashing/MurMurHash3.cs +++ b/src/cdeLib/Infrastructure/Hashing/MurMurHash3.cs @@ -42,7 +42,7 @@ public static uint Hash(ReadOnlySpan data, uint seed) h1 ^= k1; h1 = rotl32(h1, 13); - h1 = (h1 * 5) + 0xe6546b64; + h1 = h1 * 5 + 0xe6546b64; position += 4; } @@ -119,7 +119,7 @@ public static uint Hash(Stream stream, UInt32 seed) h1 ^= k1; h1 = rotl32(h1, 13); - h1 = (h1 * 5) + 0xe6546b64; + h1 = h1 * 5 + 0xe6546b64; break; case 3: k1 = (uint) diff --git a/src/cdeLib/Infrastructure/ObjectPool.cs b/src/cdeLib/Infrastructure/ObjectPool.cs index bf6641e..e639b13 100644 --- a/src/cdeLib/Infrastructure/ObjectPool.cs +++ b/src/cdeLib/Infrastructure/ObjectPool.cs @@ -37,7 +37,7 @@ public ObjectPool(Func objectGenerator, Action resetAction = null, int max public T Get() { - if (_disposed) throw new ObjectDisposedException(nameof(ObjectPool)); + if (_disposed) throw new ObjectDisposedException(nameof(ObjectPool<>)); if (_objects.TryDequeue(out var item)) { @@ -90,7 +90,7 @@ public static class PoolManager new(() => new StringBuilder(256), sb => sb.Clear(), 50); private static readonly ObjectPool> StringListPool = - new(() => new List(), list => list.Clear(), 30); + new(() => [], list => list.Clear(), 30); public static StringBuilder GetStringBuilder() => StringBuilderPool.Get(); public static void ReturnStringBuilder(StringBuilder sb) => StringBuilderPool.Return(sb); diff --git a/src/cdeLib/Module/CdelibModule.cs b/src/cdeLib/Module/CdelibModule.cs index a512327..e9554d5 100644 --- a/src/cdeLib/Module/CdelibModule.cs +++ b/src/cdeLib/Module/CdelibModule.cs @@ -13,9 +13,11 @@ protected override void Load(ContainerBuilder builder) { // singletons. builder.RegisterType().As().SingleInstance(); + builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterType().As(); builder.RegisterType().As().SingleInstance(); + builder.RegisterType().SingleInstance(); builder.RegisterType().InstancePerLifetimeScope(); diff --git a/src/cdeLib/OperationCancellation.cs b/src/cdeLib/OperationCancellation.cs new file mode 100644 index 0000000..1f451b2 --- /dev/null +++ b/src/cdeLib/OperationCancellation.cs @@ -0,0 +1,35 @@ +using System.Threading; + +namespace cdeLib; + +/// +/// Cooperative cancellation for long-running console operations, driven by Ctrl-C. Replaces the old +/// global Hack.BreakConsoleFlag with an injectable signal that exposes a real +/// . +/// +/// It is resettable: a phase can call to obtain a fresh, un-cancelled token, +/// so the user can interrupt the next phase independently. This mirrors the original "press break again" +/// behaviour used by the two-phase hashing and the find REPL. +/// +/// +public sealed class OperationCancellation +{ + private CancellationTokenSource _cts = new(); + + /// Token for the current operation/phase. Cancelled when the user requests a break. + public CancellationToken Token => _cts.Token; + + /// True once a break has been requested for the current operation/phase. + public bool IsCancellationRequested => _cts.IsCancellationRequested; + + /// Request cancellation of the current operation/phase (wired to Ctrl-C). + public void Cancel() => _cts.Cancel(); + + /// Begin a fresh operation/phase with a new, un-cancelled token. + public void Reset() + { + var old = _cts; + _cts = new CancellationTokenSource(); + old.Dispose(); + } +} diff --git a/src/cdeLib/TimePartialParameter.cs b/src/cdeLib/TimePartialParameter.cs index e595cd9..7f7530c 100644 --- a/src/cdeLib/TimePartialParameter.cs +++ b/src/cdeLib/TimePartialParameter.cs @@ -10,34 +10,34 @@ public class TimePartialParameter // "HH:MM:SS"; example private const string Format = "::"; - private readonly int _hour; // 0 - 23 public int Hour { get { ThrowExceptionIfSet(); - return _hour; + return field; } + private set; } - private readonly int _minute; // 0 - 59 public int Minute { get { ThrowExceptionIfSet(); - return _minute; + return field; } + private set; } - private readonly int _second; // 0 - 59 public int Second { get { ThrowExceptionIfSet(); - return _second; + return field; } + private set; } private readonly Exception _e; @@ -55,13 +55,13 @@ public TimePartialParameter(string str, string activeFormat = Format) var activeFormat1 = activeFormat; var splitOnColon = str.Split(':'); int.TryParse(splitOnColon[0], out var hour); - if (hour == 0 || hour > 23) + if (hour is 0 or > 23) { _e = new ArgumentException( $"Require valid Integer 1-23 for Hour as part of format '{activeFormat1}'"); return; } - _hour = hour; + Hour = hour; if (splitOnColon.Length > 1) // may have an hour specified { @@ -77,7 +77,7 @@ public TimePartialParameter(string str, string activeFormat = Format) $"Require valid integer 1-59 or for Minute as part of format '{activeFormat1}'"); return; } - _minute = minute; + Minute = minute; } if (splitOnColon.Length > 2) // may have second specified @@ -94,7 +94,7 @@ public TimePartialParameter(string str, string activeFormat = Format) $"Require valid integer 1-59 or for Second as part of format '{activeFormat1}'"); return; } - _second = second; + Second = second; } } } \ No newline at end of file diff --git a/src/cdeLib/cdeLib.csproj b/src/cdeLib/cdeLib.csproj index 6141f9e..eacb501 100644 --- a/src/cdeLib/cdeLib.csproj +++ b/src/cdeLib/cdeLib.csproj @@ -2,32 +2,35 @@ net10.0 + + true - + - - - - - + + + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - + + + - + diff --git a/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs b/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs new file mode 100644 index 0000000..9a623e7 --- /dev/null +++ b/src/cdeLibTest/Columnar/ColumnarCatalogTests.cs @@ -0,0 +1,282 @@ +using System.Collections.Generic; +using System.IO; +using System.Linq; +using cdeLib.Entities; +using cdeLib.Entities.Columnar; +using cdeLib.Entities.Soa; +using NUnit.Framework; + +namespace cdeLibTest.Columnar; + +/// +/// Proves the columnar/mmap format round-trips an and that its zero-copy +/// byte-scan search returns the same matches as the in-memory store search. This is the safety gate +/// for loading catalogs off the memory map instead of the managed heap. +/// +[TestFixture] +public class ColumnarCatalogTests +{ + // C:\test + // ├─ dir1\ (dir) + // │ ├─ alpha.txt + // │ └─ beta.log + // ├─ docs\ (dir) + // │ └─ alpha.md + // └─ root_file.txt + private static RootEntry BuildTree() + { + var root = new RootEntry + { + Path = @"C:\test", + VolumeName = "VOL", + DefaultFileName = "test.cde", + DriveLetterHint = "C", + Description = "desc", + AvailSpace = 123, + TotalSpace = 456, + }; + + var dir1 = new DirEntry(true) { Path = "dir1" }; + dir1.AddChild(new DirEntry(false) { Path = "alpha.txt", Size = 100 }); + dir1.AddChild(new DirEntry(false) { Path = "beta.log", Size = 5000 }); + + var docs = new DirEntry(true) { Path = "docs" }; + docs.AddChild(new DirEntry(false) { Path = "alpha.md", Size = 200 }); + + root.AddChild(dir1); + root.AddChild(docs); + root.AddChild(new DirEntry(false) { Path = "root_file.txt", Size = 50 }); + + root.SetInMemoryFields(); + return root; + } + + private static string WriteTemp(EntryStore store) + { + var path = Path.Combine(Path.GetTempPath(), $"cdetest-{System.Guid.NewGuid():N}.cdex"); + ColumnarFormat.Write(store, path); + return path; + } + + [Test] + public void RoundTrip_PreservesCountAndMetadata() + { + var store = EntryStore.Build(BuildTree()); + var path = WriteTemp(store); + try + { + using var reader = new ColumnarCatalogReader(path); + Assert.That(reader.Count, Is.EqualTo(store.Count)); + Assert.That(reader.RootPath, Is.EqualTo(@"C:\test")); + Assert.That(reader.VolumeName, Is.EqualTo("VOL")); + Assert.That(reader.DefaultFileName, Is.EqualTo("test.cde")); + Assert.That(reader.Description, Is.EqualTo("desc")); + Assert.That(reader.AvailSpace, Is.EqualTo(123)); + Assert.That(reader.TotalSpace, Is.EqualTo(456)); + } + finally { File.Delete(path); } + } + + [Test] + public void FullPath_MatchesStore() + { + var store = EntryStore.Build(BuildTree()); + var path = WriteTemp(store); + try + { + using var reader = new ColumnarCatalogReader(path); + for (var i = 0; i < store.Count; i++) + Assert.That(reader.FullPath(i), Is.EqualTo(store.FullPath(i)), $"path mismatch at {i}"); + } + finally { File.Delete(path); } + } + + [TestCase("alpha", true, true)] // matches files in two dirs + [TestCase(".txt", true, true)] // extension match + [TestCase("DIR1", false, true)] // case-insensitive, folders + [TestCase("nope", true, true)] // no matches + public void FindName_MatchesStoreSearch(string pattern, bool files, bool folders) + { + var store = EntryStore.Build(BuildTree()); + var path = WriteTemp(store); + try + { + var expected = new List(); + EntryStoreSearch.Find(store, pattern, regexMode: false, includePath: false, + includeFiles: files, includeFolders: folders, i => expected.Add(store.FullPath(i))); + + using var reader = new ColumnarCatalogReader(path); + var actual = new List(); + reader.FindName(pattern, files, folders, i => actual.Add(reader.FullPath(i))); + + Assert.That(actual.OrderBy(x => x), Is.EqualTo(expected.OrderBy(x => x))); + } + finally { File.Delete(path); } + } + + [TestCase(@"alpha\.txt", false)] // regex name + [TestCase("beta", false)] + [TestCase("alpha", true)] // regex on full path + public void Find_Regex_MatchesStoreSearch(string pattern, bool includePath) + { + var store = EntryStore.Build(BuildTree()); + var path = WriteTemp(store); + try + { + var expected = new List(); + EntryStoreSearch.Find(store, pattern, regexMode: true, includePath: includePath, + includeFiles: true, includeFolders: true, i => expected.Add(store.FullPath(i))); + + using var reader = new ColumnarCatalogReader(path); + var actual = new List(); + reader.Find(pattern, regexMode: true, includePath: includePath, + includeFiles: true, includeFolders: true, i => actual.Add(reader.FullPath(i))); + + Assert.That(actual.OrderBy(x => x), Is.EqualTo(expected.OrderBy(x => x))); + } + finally { File.Delete(path); } + } + + [Test] + public void Find_FullFilter_SizeRange_MatchesStore() + { + var store = EntryStore.Build(BuildTree()); + var path = WriteTemp(store); + try + { + var opts = new EntryStoreFindOptions + { + IncludeFiles = true, + IncludeFolders = true, + FromSizeEnable = true, + FromSize = 1000, // only beta.log (5000) qualifies + }; + + var expected = new List(); + EntryStoreSearch.Find(store, opts, i => expected.Add(store.FullPath(i))); + + using var reader = new ColumnarCatalogReader(path); + var actual = new List(); + reader.Find(opts, i => actual.Add(reader.FullPath(i))); + + // Reader and store must agree exactly (both include dir1, whose aggregated size >= 1000). + Assert.That(actual.OrderBy(x => x), Is.EqualTo(expected.OrderBy(x => x))); + Assert.That(actual, Does.Contain(@"C:\test\dir1\beta.log")); + Assert.That(actual, Does.Not.Contain(@"C:\test\root_file.txt")); // size 50, filtered out + } + finally { File.Delete(path); } + } + + [Test] + public void CatalogTreeBuilder_RoundTrips_StructureSizesAndHashes() + { + // Set a hash on one file so the hash column round-trips too (the hash/dupes path). + var root = BuildTree(); + var beta = root.Children.First(c => c.Path == "dir1").Children.First(c => c.Path == "beta.log"); + beta.SetHash([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); + root.SetInMemoryFields(); + + var store0 = EntryStore.Build(root); + var path = WriteTemp(store0); + try + { + // .cdex -> mutable tree (as hash/dupes do) -> store again must match the original store. + var trees = CatalogTreeBuilder.FromColumnarFiles([path]); + Assert.That(trees, Has.Count.EqualTo(1)); + var store2 = EntryStore.Build(trees[0]); + + Assert.That(store2.Count, Is.EqualTo(store0.Count)); + Assert.That(store2.HasHash, Is.True); + for (var i = 0; i < store0.Count; i++) + { + Assert.That(store2.FullPath(i), Is.EqualTo(store0.FullPath(i)), $"path {i}"); + Assert.That(store2.Size[i], Is.EqualTo(store0.Size[i]), $"size {i}"); + Assert.That(store2.ModifiedTicks[i], Is.EqualTo(store0.ModifiedTicks[i]), $"modified {i}"); + Assert.That(store2.IsDirectory(i), Is.EqualTo(store0.IsDirectory(i)), $"isDir {i}"); + Assert.That(store2.IsHashDone(i), Is.EqualTo(store0.IsHashDone(i)), $"hashDone {i}"); + if (store0.IsHashDone(i)) + Assert.That(store2.HashOf(i), Is.EqualTo(store0.HashOf(i)), $"hash {i}"); + } + } + finally { File.Delete(path); } + } + + private static RootEntry BuildForCopy(bool withHash) + { + var root = new RootEntry { Path = @"C:\test" }; + var f = new DirEntry(false) + { + Path = "a.txt", + Size = 100, + Modified = new System.DateTime(2020, 1, 1, 0, 0, 0, System.DateTimeKind.Utc), + }; + if (withHash) f.SetHash([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]); + root.AddChild(f); + root.SetInMemoryFields(); + return root; + } + + [Test] + public void TraverseTreesCopyHash_FromReconstructedCdex_CopiesHashToFreshScan() + { + // Mirrors the re-scan hash-reuse path: old hashed catalog comes from a .cdex (reconstructed), + // fresh scan tree has no hash; the hash must copy across. + var store = EntryStore.Build(BuildForCopy(withHash: true)); + var path = WriteTemp(store); + try + { + var oldRoot = CatalogTreeBuilder.FromColumnarFiles([path])[0]; + var fresh = BuildForCopy(withHash: false); + + oldRoot.TraverseTreesCopyHash(fresh); + + var file = fresh.Children.First(c => c.Path == "a.txt"); + Assert.That(file.IsHashDone, Is.True, "hash should have been copied from the reconstructed .cdex"); + Assert.That(file.Hash, Is.EqualTo(oldRoot.Children.First(c => c.Path == "a.txt").Hash)); + } + finally { File.Delete(path); } + } + + [Test] + public void EntryRef_OverReader_NavigatesLikeStore() + { + var store = EntryStore.Build(BuildTree()); + var path = WriteTemp(store); + try + { + using var reader = new ColumnarCatalogReader(path); + // Same ICommonEntry adapter, backed by the mmap reader instead of the heap store. + ICommonEntry rootRef = new EntryRef(reader, 0); + ICommonEntry storeRootRef = new EntryRef(store, 0); + + Assert.That(rootRef.Children, Is.Not.Null); + Assert.That(rootRef.Children.Count, Is.EqualTo(storeRootRef.Children.Count)); + Assert.That(rootRef.FullPath, Is.EqualTo(storeRootRef.FullPath)); + + // Subtree counts (files/dirs) must match the heap-backed adapter. + Assert.That(rootRef.FileEntryCount, Is.EqualTo(storeRootRef.FileEntryCount)); + Assert.That(rootRef.DirEntryCount, Is.EqualTo(storeRootRef.DirEntryCount)); + } + finally { File.Delete(path); } + } + + [Test] + public void FindPath_MatchesStorePathSearch() + { + var store = EntryStore.Build(BuildTree()); + var path = WriteTemp(store); + try + { + var expected = new List(); + EntryStoreSearch.Find(store, "docs", regexMode: false, includePath: true, + includeFiles: true, includeFolders: true, i => expected.Add(store.FullPath(i))); + + using var reader = new ColumnarCatalogReader(path); + var actual = new List(); + reader.FindPath("docs", includeFiles: true, includeFolders: true, i => actual.Add(reader.FullPath(i))); + + Assert.That(actual.OrderBy(x => x), Is.EqualTo(expected.OrderBy(x => x))); + } + finally { File.Delete(path); } + } +} diff --git a/src/cdeLibTest/CommonEntryTest_TraverseTreesCopyHash.cs b/src/cdeLibTest/CommonEntryTest_TraverseTreesCopyHash.cs index da58c7e..30d0e50 100644 --- a/src/cdeLibTest/CommonEntryTest_TraverseTreesCopyHash.cs +++ b/src/cdeLibTest/CommonEntryTest_TraverseTreesCopyHash.cs @@ -165,12 +165,10 @@ public void TraverseTreesCopyHash_CopyHashIfSourceHasFullGasgAndDestHasPartialHa public void TraverseTreesCopyHash_DontCopyHashIfDestHasFullHash() { RecreateTestTree(); - _dde1.SetHash(99); // _dde1.Hash = new byte[] { 99 }; + _dde1.SetHash(99); _dde1.IsPartialHash = false; _reSource.TraverseTreesCopyHash(_reDest); - - // Assert.That(_dde1.Hash, Is.Not.Null); Assert.That(_dde1.Hash[0], Is.EqualTo(99)); } private void RecreateTestTree() diff --git a/src/cdeLibTest/DirEntryEnumeratorTest.cs b/src/cdeLibTest/DirEntryEnumeratorTest.cs index 4c8df38..c80ca19 100644 --- a/src/cdeLibTest/DirEntryEnumeratorTest.cs +++ b/src/cdeLibTest/DirEntryEnumeratorTest.cs @@ -54,7 +54,7 @@ public void Constructor_Minimal_NoErrors() [Test] public void MoveNext_NoRootEntries_FirstMoveNextFalse() { - RootEntries = new List(); + RootEntries = []; var e = new DirEntryEnumerator(RootEntries); diff --git a/src/cdeLibTest/DuplicationTest.cs b/src/cdeLibTest/DuplicationTest.cs index 9b38a07..2ae9768 100644 --- a/src/cdeLibTest/DuplicationTest.cs +++ b/src/cdeLibTest/DuplicationTest.cs @@ -71,7 +71,7 @@ public void GetSizePairs_HashIrrelevant_NullIsNotAHashValue_PartialNotAUniqueHas var roots = new List {re1}; re1.SetInMemoryFields(); - var d = new Duplication(_logger, _configuration, _applicationDiagnostics); + var d = new Duplication(_logger, _configuration, _applicationDiagnostics, new OperationCancellation()); var sizePairDictionary = d.GetSizePairs(roots); Console.WriteLine($"Number of Size Pairs {sizePairDictionary.Count}"); @@ -117,7 +117,7 @@ public void GetDupePairs_DupeHashDoesNotMatchDiffSizeFilesOrPartialHash_OK() re1.AddChild(de10); var roots = new List {re1}; - var d = new Duplication(_logger, _configuration, _applicationDiagnostics); + var d = new Duplication(_logger, _configuration, _applicationDiagnostics, new OperationCancellation()); var dp = d.GetDupePairs(roots); var dp1 = dp.First(); @@ -167,7 +167,7 @@ private string AssemblyPathLocation() public void GetSizePairs_CheckSanityOfDupeSizeCountAndDupeFileCount_Exercise() { const int dupeCount = 10; - var testPath = this.AssemblyPathLocation(); + var testPath = AssemblyPathLocation(); // Create some dummy duplicate data. // create a catalog var random = FileHelper.RandomString(4096 * 16); @@ -176,14 +176,14 @@ public void GetSizePairs_CheckSanityOfDupeSizeCountAndDupeFileCount_Exercise() FileHelper.WriteAllText(random, testPath, $"CDE_testFile{i}.txt"); } - Program.InitProgram(Array.Empty()); - Program.CreateCache(new ScanOptions {Path = testPath}); - Program.HashCatalog(); + Program.InitProgram([]); + Program.CreateCache(new ScanOptions {Path = testPath}); // scan writes a columnar .cdex + Program.HashCatalog(); // hash operates on the .cdex - // run tests. + // run tests. Load the hashed catalogs back from .cdex (where the hashes now live). Console.WriteLine($"0 Directory.GetCurrentDirectory() {System.IO.Directory.GetCurrentDirectory()}"); var catalogRepository = new CatalogRepository(Log.Logger); - var rootEntries = catalogRepository.LoadCurrentDirCache(); + var rootEntries = CatalogTreeBuilder.FromColumnarFiles(catalogRepository.GetColumnarFileList(["./"])); if (rootEntries.Count == 0) { @@ -195,7 +195,7 @@ public void GetSizePairs_CheckSanityOfDupeSizeCountAndDupeFileCount_Exercise() Console.WriteLine($"loaded {r.DefaultFileName}"); } - var d = new Duplication(_logger, _configuration, _applicationDiagnostics); + var d = new Duplication(_logger, _configuration, _applicationDiagnostics, new OperationCancellation()); var sizePairDictionary = d.GetSizePairs(rootEntries); Console.WriteLine($"Number of Size Pairs {sizePairDictionary.Count}"); @@ -221,15 +221,10 @@ private static long GetSumOfUniqueHashesForEachSize_ExcludePartialHash( var seenHash = new Dictionary(); foreach (var flatDe in fdeListOfSize) { - // var hash = flatDe.ChildDE.Hash; if (flatDe.ChildDE.IsHashDone // because this is run on SizeDupe list it can have null hashes. && !flatDe.ChildDE.IsPartialHash) { - if (!seenHash.ContainsKey(flatDe.ChildDE.Hash)) - { - seenHash[flatDe.ChildDE.Hash] = 0; - } - else + if (!seenHash.TryAdd(flatDe.ChildDE.Hash, 0)) { ++seenHash[flatDe.ChildDE.Hash]; } @@ -248,7 +243,7 @@ public void GetDupePairs_CheckAllDupeFilesHaveFullHash_OK() var catalogRepository = new CatalogRepository(Log.Logger); var rootEntries = catalogRepository.LoadCurrentDirCache(); - var d = new Duplication(_logger, _configuration, _applicationDiagnostics); + var d = new Duplication(_logger, _configuration, _applicationDiagnostics, new OperationCancellation()); var dupePairEnum = d.GetDupePairs(rootEntries); foreach (var dupe in dupePairEnum) @@ -273,7 +268,7 @@ public async Task var catalogRepository = new CatalogRepository(Log.Logger); var rootEntries = catalogRepository.LoadCurrentDirCache(); - var d = new Duplication(_logger, _configuration, _applicationDiagnostics); + var d = new Duplication(_logger, _configuration, _applicationDiagnostics, new OperationCancellation()); await d.ApplyHash(rootEntries).ConfigureAwait(false); } // ReSharper restore InconsistentNaming diff --git a/src/cdeLibTest/EntryHelperTest.cs b/src/cdeLibTest/EntryHelperTest.cs index 5ef647f..4ec5421 100644 --- a/src/cdeLibTest/EntryHelperTest.cs +++ b/src/cdeLibTest/EntryHelperTest.cs @@ -113,6 +113,26 @@ public void MakeFullPath_MultipleCallsWithVaryingLengths_DoesNotThrow() shortResult2.ShouldBe(@"C:\short"); } + [Test] + public void FullPath_NestedDirEntries_DoesNotDuplicateParentPath() + { + // Regression: DirEntry.FullPath recurses into MakeFullPath using a shared + // ThreadLocal StringBuilder. Previously the buffer was cleared before the + // recursive parent lookup, so the parent path was left in the buffer and + // appended twice (e.g. "D:\ArchiveD:\Archive\Apps2021"). + var re = new RootEntry(_config) { Path = @"D:\" }; + var archive = new DirEntry(true) { Path = "Archive" }; + var apps = new DirEntry(true) { Path = "Apps2021" }; + re.AddChild(archive); + archive.AddChild(apps); + re.SetInMemoryFields(); + + // Depths 1-3 should all produce a single, correct full path. + re.FullPath.ShouldBe(@"D:\"); + archive.FullPath.ShouldBe(@"D:\Archive"); + apps.FullPath.ShouldBe(@"D:\Archive\Apps2021"); + } + [Test] public void MakeFullPathPooled_IsAliasForMakeFullPath() { diff --git a/src/cdeLibTest/IdeaStructNode.cs b/src/cdeLibTest/IdeaStructNode.cs index 3ad8794..c11f896 100644 --- a/src/cdeLibTest/IdeaStructNode.cs +++ b/src/cdeLibTest/IdeaStructNode.cs @@ -96,7 +96,5 @@ internal struct Node public Hash16 Hash; // (16) -- ..(52) } -internal class IdeaStructNode -{ -} +internal class IdeaStructNode; #pragma warning restore 0649 \ No newline at end of file diff --git a/src/cdeLibTest/Infrastructure/DuplicationPerfTest.cs b/src/cdeLibTest/Infrastructure/DuplicationPerfTest.cs index 04e4fe2..7232f46 100644 --- a/src/cdeLibTest/Infrastructure/DuplicationPerfTest.cs +++ b/src/cdeLibTest/Infrastructure/DuplicationPerfTest.cs @@ -51,7 +51,7 @@ public void PerformanceHashTest() timer.Stop(); Console.WriteLine( - $"{hashKey}:\t\t{(data.Length * (1000.0 / (timer.ElapsedMilliseconds / 9999.0))) / (1024.0 * 1024.0):F2} MB/s ({timer.ElapsedMilliseconds})"); + $"{hashKey}:\t\t{data.Length * (1000.0 / (timer.ElapsedMilliseconds / 9999.0)) / (1024.0 * 1024.0):F2} MB/s ({timer.ElapsedMilliseconds})"); } } } \ No newline at end of file diff --git a/src/cdeLibTest/Infrastructure/DuplicationTests.cs b/src/cdeLibTest/Infrastructure/DuplicationTests.cs index aed93e7..68f22b4 100644 --- a/src/cdeLibTest/Infrastructure/DuplicationTests.cs +++ b/src/cdeLibTest/Infrastructure/DuplicationTests.cs @@ -80,7 +80,8 @@ public void SetUp() private class TestDuplication : Duplication { public TestDuplication(ILogger logger, IConfiguration configuration, - IApplicationDiagnostics applicationDiagnostics) : base(logger, configuration, applicationDiagnostics) + IApplicationDiagnostics applicationDiagnostics) + : base(logger, configuration, applicationDiagnostics, new cdeLib.OperationCancellation()) { } diff --git a/src/cdeLibTest/Infrastructure/Hashing/Crc32.cs b/src/cdeLibTest/Infrastructure/Hashing/Crc32.cs index 00d23f7..6b35a7d 100644 --- a/src/cdeLibTest/Infrastructure/Hashing/Crc32.cs +++ b/src/cdeLibTest/Infrastructure/Hashing/Crc32.cs @@ -1,5 +1,6 @@ using System; using System.IO; +using System.Linq; using cdeLib.Infrastructure.Hashing; namespace cdeLibTest.Infrastructure.Hashing; @@ -35,9 +36,7 @@ private void Init(uint poly) public UInt64 Hash(byte[] data) { - uint hash = 0xFFFFFFFF; - foreach (byte b in data) - hash = (hash << 8) ^ _tab[b ^ (hash >> 24)]; + var hash = data.Aggregate(0xFFFFFFFF, (current, b) => (current << 8) ^ _tab[b ^ (current >> 24)]); return ~hash; } diff --git a/src/cdeLibTest/Performance/PerformanceTreeTraversal.cs b/src/cdeLibTest/Performance/PerformanceTreeTraversal.cs index dc236a9..29c2cea 100644 --- a/src/cdeLibTest/Performance/PerformanceTreeTraversal.cs +++ b/src/cdeLibTest/Performance/PerformanceTreeTraversal.cs @@ -30,8 +30,8 @@ internal class PerformanceTreeTraversal // ReSharper disable JoinDeclarationAndInitializer // ReSharper disable PossibleMultipleEnumeration - private readonly int _repeatSmall = 100; - private readonly int _repeatLarge = 25; + private const int _repeatSmall = 100; + private const int _repeatLarge = 25; [SetUpFixture] public class TestData diff --git a/src/cdeLibTest/RootEntryTest.cs b/src/cdeLibTest/RootEntryTest.cs index b9cdddc..4f679be 100644 --- a/src/cdeLibTest/RootEntryTest.cs +++ b/src/cdeLibTest/RootEntryTest.cs @@ -46,33 +46,10 @@ public void Constructor_GetTreeWithMoreThanOneLevel_OK() re.RecurseTree(FileHelper.TestDir); Assert.That(re, Is.Not.Null); - var found = re.Children.Any(x => x.Children != null && x.Children.Count > 0); + var found = re.Children.Any(x => x.Children is { Count: > 0 }); Assert.That(found, Is.True, "One of entries does not have children."); } - //[Test] - //public void FindDir_LookForDir_InRoot() - //{ - // const string rootPath = @"C:\"; - // var re = new RootEntry { Path = rootPath }; - - // var foundEntry = re.FindDir(rootPath, @"C:\Moo"); - - // Assert.That(foundEntry, Is.InstanceOf(typeof(RootEntry))); - //} - - //[Test] - //public void FindDir_NotExistinRoot_ReturnRE() - //{ - // const string rootPath = @"C:\"; - // const string testPath = @"C:\Groo"; - // var re = new RootEntry { Path = rootPath }; - - // var foundEntry = re.FindDir(rootPath, testPath); - - // Assert.That(foundEntry, Is.InstanceOf(typeof(RootEntry))); - //} - [Test] public void GetDriverLetterHint_SimpleRootPath_ReturnsDriveLetter() { @@ -288,9 +265,7 @@ public void SetFullPath_OnRootDirectory_SetsAllFullPaths() re.SetInMemoryFields(); Assert.That(re.FullPath, Is.EqualTo(@"C:\")); - //Assert.That(fe1.FullPath, Is.Null);// Is.EqualTo(@"C:\fe1")); FullPath only set on directories to save memory. Assert.That(de2.FullPath, Is.EqualTo(@"C:\de2")); - //Assert.That(fe3.FullPath, Is.Null);//Is.EqualTo(@"C:\de2\fe3")); } } // ReSharper restore InconsistentNaming diff --git a/src/cdeLibTest/Soa/EntryRefTests.cs b/src/cdeLibTest/Soa/EntryRefTests.cs new file mode 100644 index 0000000..64ca4b1 --- /dev/null +++ b/src/cdeLibTest/Soa/EntryRefTests.cs @@ -0,0 +1,117 @@ +using System.Collections.Generic; +using System.Linq; +using cdeLib.Entities; +using cdeLib.Entities.Soa; +using NUnit.Framework; + +namespace cdeLibTest.Soa; + +/// +/// Proves the adapter lets the existing tree-oriented, ICommonEntry-based +/// code run unchanged on the struct-of-arrays — same traversal results, +/// same paths, same counts as the pointer tree. +/// +[TestFixture] +public class EntryRefTests +{ + private static RootEntry BuildTree() + { + var root = new RootEntry { Path = @"C:\test" }; + + var dir1 = new DirEntry(true) { Path = "dir1" }; + dir1.AddChild(new DirEntry(false) { Path = "alpha.txt", Size = 10 }); + dir1.AddChild(new DirEntry(false) { Path = "beta.log", Size = 20 }); + + var docs = new DirEntry(true) { Path = "docs" }; + docs.AddChild(new DirEntry(false) { Path = "alpha.md", Size = 30 }); + + root.AddChild(dir1); + root.AddChild(docs); + root.AddChild(new DirEntry(false) { Path = "root_file.txt", Size = 40 }); + + root.SetInMemoryFields(); + return root; + } + + private static List TraverseFullPaths(ICommonEntry root) + { + var paths = new List(); + EntryHelper.TraverseTreePair(root, (_, child) => { paths.Add(child.FullPath); return true; }); + paths.Sort(); + return paths; + } + + [Test] + public void TraverseTreePair_OverEntryRef_MatchesTreeTraversal() + { + var root = BuildTree(); + var store = EntryStore.Build(root); + var storeRoot = new EntryRef(store, 0); + + Assert.That(TraverseFullPaths(storeRoot), Is.EqualTo(TraverseFullPaths(root))); + } + + [Test] + public void Children_OverEntryRef_MatchTreeChildren() + { + var root = BuildTree(); + var store = EntryStore.Build(root); + var storeRoot = new EntryRef(store, 0); + + var treeChildNames = root.Children.Select(c => c.Path).OrderBy(x => x).ToList(); + var storeChildNames = storeRoot.Children.Select(c => c.Path).OrderBy(x => x).ToList(); + Assert.That(storeChildNames, Is.EqualTo(treeChildNames)); + + // A file has no children view. + var rootFile = storeRoot.Children.First(c => c.Path == "root_file.txt"); + Assert.That(rootFile.IsDirectory, Is.False); + Assert.That(rootFile.Children, Is.Null); + } + + [Test] + public void Counts_OverEntryRef_MatchTree() + { + var root = BuildTree(); + var store = EntryStore.Build(root); + var storeRoot = new EntryRef(store, 0); + + Assert.That(storeRoot.FileEntryCount, Is.EqualTo(root.FileEntryCount)); + Assert.That(storeRoot.DirEntryCount, Is.EqualTo(root.DirEntryCount)); + Assert.That(storeRoot.FileEntryCount, Is.EqualTo(4u)); + Assert.That(storeRoot.DirEntryCount, Is.EqualTo(2u)); + } + + [Test] + public void GetListFromRoot_OverEntryRef_GoesRootToLeaf() + { + var root = BuildTree(); + var store = EntryStore.Build(root); + + // Find alpha.txt under dir1 and walk back to root. + var alpha = TraverseFind(store, "alpha.txt"); + var chain = alpha.GetListFromRoot().Select(e => e.Path).ToList(); + Assert.That(chain, Is.EqualTo([@"C:\test", "dir1", "alpha.txt"])); + Assert.That(alpha.FullPath, Is.EqualTo(@"C:\test\dir1\alpha.txt")); + } + + [Test] + public void MutatingMembers_Throw() + { + var root = BuildTree(); + var store = EntryStore.Build(root); + var e = new EntryRef(store, 1); + + Assert.Throws(() => e.Path = "x"); + Assert.Throws(() => e.AddChild(new DirEntry(false))); + Assert.Throws(() => e.SetSummaryFields()); + } + + private static EntryRef TraverseFind(EntryStore store, string name) + { + for (var i = 1; i < store.Count; i++) + { + if (store.FullName(i) == name) return new EntryRef(store, i); + } + return null; + } +} diff --git a/src/cdeLibTest/Soa/EntryStoreTests.cs b/src/cdeLibTest/Soa/EntryStoreTests.cs new file mode 100644 index 0000000..72b1373 --- /dev/null +++ b/src/cdeLibTest/Soa/EntryStoreTests.cs @@ -0,0 +1,192 @@ +using System.Collections.Generic; +using System.Linq; +using cdeLib; +using cdeLib.Entities; +using cdeLib.Entities.Soa; +using NUnit.Framework; + +namespace cdeLibTest.Soa; + +/// +/// Proves the SoA + produce results identical +/// to the production pointer-tree find. This is the safety gate for moving search onto the store. +/// +[TestFixture] +public class EntryStoreTests +{ + // Builds a small known catalog: + // C:\test + // ├─ dir1\ (dir) + // │ ├─ alpha.txt + // │ └─ beta.log + // ├─ docs\ (dir) + // │ └─ alpha.md + // └─ root_file.txt + private static RootEntry BuildTree() + { + var root = new RootEntry { Path = @"C:\test" }; + + var dir1 = new DirEntry(true) { Path = "dir1" }; + dir1.AddChild(new DirEntry(false) { Path = "alpha.txt" }); + dir1.AddChild(new DirEntry(false) { Path = "beta.log" }); + + var docs = new DirEntry(true) { Path = "docs" }; + docs.AddChild(new DirEntry(false) { Path = "alpha.md" }); + + root.AddChild(dir1); + root.AddChild(docs); + root.AddChild(new DirEntry(false) { Path = "root_file.txt" }); + + root.SetInMemoryFields(); + return root; + } + + private static List TreeFind(RootEntry root, string pattern, bool regex, bool path, + bool files, bool folders) + { + var found = new List(); + var options = new FindOptions + { + Pattern = pattern, + RegexMode = regex, + IncludePath = path, + IncludeFiles = files, + IncludeFolders = folders, + LimitResultCount = int.MaxValue, + VisitorFunc = (p, d) => { found.Add(p.MakeFullPath(d)); return true; }, + }; + options.Find([root]); + return found; + } + + private static List StoreFind(EntryStore store, string pattern, bool regex, bool path, + bool files, bool folders) + { + var found = new List(); + EntryStoreSearch.Find(store, pattern, regex, path, files, folders, i => found.Add(store.FullPath(i))); + return found; + } + + [Test] + public void Build_CountsEveryEntryIncludingRoot() + { + var root = BuildTree(); + var store = EntryStore.Build(root); + // 3 dirs/files under root + 2 files under dir1 + 1 under docs = 6, plus the root = 7. + Assert.That(store.Count, Is.EqualTo(7)); + } + + [Test] + public void Build_CapturesCatalogMetadata() + { + var root = new RootEntry + { + Path = @"D:\", + VolumeName = "DATA", + DefaultFileName = "D-DATA.cde", + ActualFileName = @"C:\cat\D-DATA.cde", + DriveLetterHint = "D", + Description = "data drive", + AvailSpace = 111, + TotalSpace = 222, + }; + root.AddChild(new DirEntry(false) { Path = "f.txt", Size = 7 }); + root.SetInMemoryFields(); + + var store = EntryStore.Build(root); + + Assert.Multiple(() => + { + Assert.That(store.RootPath, Is.EqualTo(@"D:\")); + Assert.That(store.VolumeName, Is.EqualTo("DATA")); + Assert.That(store.DefaultFileName, Is.EqualTo("D-DATA.cde")); + Assert.That(store.ActualFileName, Is.EqualTo(@"C:\cat\D-DATA.cde")); + Assert.That(store.DriveLetterHint, Is.EqualTo("D")); + Assert.That(store.Description, Is.EqualTo("data drive")); + Assert.That(store.AvailSpace, Is.EqualTo(111)); + Assert.That(store.TotalSpace, Is.EqualTo(222)); + Assert.That(store.RootFileEntryCount, Is.EqualTo(root.FileEntryCount)); + Assert.That(store.RootDirEntryCount, Is.EqualTo(root.DirEntryCount)); + Assert.That(store.RootSize, Is.EqualTo(root.Size)); + }); + } + + [Test] + public void FullPath_MatchesTreeForEveryEntry() + { + var root = BuildTree(); + var store = EntryStore.Build(root); + + // Every non-root entry's store full path must appear in the tree's full path set. + var treePaths = TreeFind(root, "", false, false, true, true).OrderBy(x => x).ToList(); + var storePaths = new List(); + for (var i = 1; i < store.Count; i++) storePaths.Add(store.FullPath(i)); + storePaths.Sort(); + + Assert.That(storePaths, Is.EqualTo(treePaths)); + } + + [TestCase("alpha", false, false)] // name substring + [TestCase("txt", false, false)] + [TestCase("xyzzy", false, false)] // no matches + [TestCase("", false, false)] // match all + [TestCase(@"dir1\alpha", false, true)] // path substring + [TestCase("docs", false, true)] + [TestCase(@"\.md$", true, false)] // regex on name + [TestCase(@"test\\d", true, true)] // regex on path + public void Search_MatchesTreeFind(string pattern, bool regex, bool path) + { + var root = BuildTree(); + var store = EntryStore.Build(root); + + var tree = TreeFind(root, pattern, regex, path, true, true).OrderBy(x => x).ToList(); + var soa = StoreFind(store, pattern, regex, path, true, true).OrderBy(x => x).ToList(); + + Assert.That(soa, Is.EqualTo(tree), $"pattern='{pattern}' regex={regex} path={path}"); + } + + [Test] + public void Search_WithSizeFilter_FiltersBySize() + { + var root = new RootEntry { Path = @"C:\s" }; + root.AddChild(new DirEntry(false) { Path = "small.txt", Size = 10 }); + root.AddChild(new DirEntry(false) { Path = "mid.txt", Size = 30 }); + root.AddChild(new DirEntry(false) { Path = "big.txt", Size = 40 }); + root.SetInMemoryFields(); + var store = EntryStore.Build(root); + + var found = new List(); + EntryStoreSearch.Find(store, + new EntryStoreFindOptions { IncludeFiles = true, IncludeFolders = false, FromSizeEnable = true, FromSize = 25 }, + i => found.Add(store.FullName(i))); + found.Sort(); + + Assert.That(found, Is.EqualTo(["big.txt", "mid.txt"])); // size >= 25 + } + + [Test] + public void Search_FilesOnly_MatchesTreeFind() + { + var root = BuildTree(); + var store = EntryStore.Build(root); + + var tree = TreeFind(root, "", false, false, files: true, folders: false).OrderBy(x => x).ToList(); + var soa = StoreFind(store, "", false, false, files: true, folders: false).OrderBy(x => x).ToList(); + + Assert.That(soa, Is.EqualTo(tree)); + Assert.That(soa, Has.Count.EqualTo(4)); // alpha.txt, beta.log, alpha.md, root_file.txt + } + + [Test] + public void Search_FoldersOnly_MatchesTreeFind() + { + var root = BuildTree(); + var store = EntryStore.Build(root); + + var tree = TreeFind(root, "", false, false, files: false, folders: true).OrderBy(x => x).ToList(); + var soa = StoreFind(store, "", false, false, files: false, folders: true).OrderBy(x => x).ToList(); + + Assert.That(soa, Is.EqualTo(tree)); + Assert.That(soa, Has.Count.EqualTo(2)); // dir1, docs + } +} diff --git a/src/cdeLibTest/TimePartialParameterTest.cs b/src/cdeLibTest/TimePartialParameterTest.cs index 1450651..1759ddb 100644 --- a/src/cdeLibTest/TimePartialParameterTest.cs +++ b/src/cdeLibTest/TimePartialParameterTest.cs @@ -37,7 +37,7 @@ public void Bad_Hour_Parameter() [Test] public void Hour_With_Minute_Parameter() { - var args = "3:34"; + const string args = "3:34"; var d = new TimePartialParameter(args); Assert.That(d.Hour, Is.EqualTo(3)); Assert.That(d.Minute, Is.EqualTo(34)); @@ -63,7 +63,7 @@ public void Hour_With_Too_Large_Minute_Parameter() [Test] public void Hour_With_Minute_With_Second_Parameter() { - var args = "3:34:10"; + const string args = "3:34:10"; var d = new TimePartialParameter(args); Assert.That(d.Hour, Is.EqualTo(3)); Assert.That(d.Minute, Is.EqualTo(34)); diff --git a/src/cdeLibTest/cdeLibTest.csproj b/src/cdeLibTest/cdeLibTest.csproj index cba3f6e..410cce2 100644 --- a/src/cdeLibTest/cdeLibTest.csproj +++ b/src/cdeLibTest/cdeLibTest.csproj @@ -15,12 +15,12 @@ - - + + - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/src/cdeMemProbe/Program.cs b/src/cdeMemProbe/Program.cs new file mode 100644 index 0000000..5476d59 --- /dev/null +++ b/src/cdeMemProbe/Program.cs @@ -0,0 +1,356 @@ +using System; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; +using cdeLib.Catalog; +using cdeLib.Entities; +using cdeLib.Entities.Columnar; +using cdeLib.Entities.Soa; +using Serilog; + +namespace cdeMemProbe; + +/// +/// Standalone footprint probe for cde catalogs. Measures the RETAINED managed heap and process +/// working set after a catalog is fully loaded — the headline metric for the memory-reduction +/// effort. BenchmarkDotNet's MemoryDiagnoser measures allocations during a run, not steady-state +/// retained size, so this lives in its own minimal process. +/// +/// Usage: +/// cdeMemProbe --generate <N> [--hashes] [--out <path.cde>] [--seed N] +/// Generate a synthetic catalog of ~N entries and save it. Prints the path. No measurement. +/// +/// cdeMemProbe <path.cde> [--no-header] +/// Load the catalog, settle the GC, and emit one CSV row of footprint metrics. +/// +/// Typical flow (clean measurement = generate and measure in separate processes): +/// cdeMemProbe --generate 1000000 --out fixture-1m.cde +/// cdeMemProbe fixture-1m.cde +/// +public static class Program +{ + private const string CsvHeader = + "file,entries,loadMs,managedBytes,peakWorkingSet,privateBytes,bytesPerEntry"; + + public static async Task Main(string[] args) + { + if (args.Length == 0) + { + Console.Error.WriteLine( + "usage: cdeMemProbe --generate [--hashes] [--out ] [--seed N]\n" + + " cdeMemProbe [--no-header]"); + return 1; + } + + // Silent Serilog logger (no sinks) so CatalogRepository stays quiet and out of the CSV. + var logger = new LoggerConfiguration().CreateLogger(); + + if (HasFlag(args, "--soa", out _)) + { + return MeasureSoa(args); + } + + if (HasFlag(args, "--migrate", out var migrateIn)) + { + return Migrate(args, migrateIn, logger); + } + + if (HasFlag(args, "--flat", out var flatFile)) + { + return MeasureFlat(args, flatFile); + } + + if (HasFlag(args, "--generate", out var genValue)) + { + return await GenerateAsync(args, genValue, logger); + } + + return await MeasureAsync(args[0], !HasFlag(args, "--no-header", out _), + asStore: HasFlag(args, "--store", out _), logger); + } + + private static async Task GenerateAsync(string[] args, string? countArg, ILogger logger) + { + if (!int.TryParse(countArg, NumberStyles.Integer, CultureInfo.InvariantCulture, out var count) || count <= 0) + { + Console.Error.WriteLine("--generate requires a positive entry count, e.g. --generate 1000000"); + return 1; + } + + var withHashes = HasFlag(args, "--hashes", out _); + var sharedNames = HasFlag(args, "--shared-names", out _); + var seed = HasFlag(args, "--seed", out var seedArg) && int.TryParse(seedArg, out var s) ? s : 42; + var outPath = HasFlag(args, "--out", out var outArg) && !string.IsNullOrWhiteSpace(outArg) + ? outArg! + : Path.Combine(Path.GetTempPath(), $"cde-synthetic-{count}{(withHashes ? "-hashed" : "")}.cde"); + + Console.Error.WriteLine($"Generating ~{count:N0} entries (hashes={withHashes}, seed={seed}) ..."); + var sw = Stopwatch.StartNew(); + var root = SyntheticCatalog.Generate(count, withHashes, seed, sharedNames: sharedNames); + root.ActualFileName = outPath; + using (var repo = new CatalogRepository(logger)) + { + await repo.Save(root); + } + sw.Stop(); + + var actual = root.FileEntryCount + root.DirEntryCount; + var fileInfo = new FileInfo(outPath); + Console.Error.WriteLine( + $"Wrote {actual:N0} entries to {outPath} ({fileInfo.Length:N0} bytes on disk) in {sw.ElapsedMilliseconds:N0} ms"); + Console.WriteLine(outPath); + return 0; + } + + /// + /// Measure the retained footprint of the PROTOTYPE struct-of-arrays EntryStore, for comparison + /// with the pointer-tree model. Usage: cdeMemProbe --soa --generate N [--shared-names] [--hashes] + /// + private static int MeasureSoa(string[] args) + { + if (!HasFlag(args, "--generate", out var countArg) + || !int.TryParse(countArg, NumberStyles.Integer, CultureInfo.InvariantCulture, out var count) + || count <= 0) + { + Console.Error.WriteLine("--soa requires --generate , e.g. --soa --generate 1000000"); + return 1; + } + + var withHashes = HasFlag(args, "--hashes", out _); + var sharedNames = HasFlag(args, "--shared-names", out _); + + Console.Error.WriteLine( + $"Building SoA EntryStore for ~{count:N0} entries (hashes={withHashes}, sharedNames={sharedNames}) ..."); + + var store = BuildStoreReleasingTree(count, withHashes, sharedNames); + + // Settle so only the live EntryStore (not the now-dead source tree) is counted. + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + var managed = GC.GetTotalMemory(true); + GC.KeepAlive(store); + var bytesPerEntry = store.Count > 0 ? (double)managed / store.Count : 0; + + // Sanity-check the SoA actually works as a searchable structure. + var matches = 0; + EntryStoreSearch.Find(store, sharedNames ? "x" : ".txt", + regexMode: false, includePath: false, includeFiles: true, includeFolders: true, _ => matches++); + + Console.WriteLine("model,entries,managedBytes,bytesPerEntry"); + Console.WriteLine(string.Join(',', "soa", + store.Count.ToString(CultureInfo.InvariantCulture), + managed.ToString(CultureInfo.InvariantCulture), + bytesPerEntry.ToString("F2", CultureInfo.InvariantCulture))); + Console.Error.WriteLine($"sanity: {matches:N0} name matches; full path[1] = {store.FullPath(1)}"); + return 0; + } + + // Separate non-inlined method so the source tree local is out of scope (collectable) before we + // measure the store in the caller. + [MethodImpl(MethodImplOptions.NoInlining)] + private static EntryStore BuildStoreReleasingTree(int count, bool withHashes, bool sharedNames) + { + var tree = SyntheticCatalog.Generate(count, withHashes, seed: 42, sharedNames: sharedNames); + return EntryStore.Build(tree); + } + + private static async Task MeasureAsync(string file, bool printHeader, bool asStore, ILogger logger) + { + if (!File.Exists(file)) + { + Console.Error.WriteLine($"catalog not found: {file}"); + return 1; + } + + var sw = Stopwatch.StartNew(); + // Load (and for --store, convert to the SoA store) inside a synchronous helper so the source + // tree is a plain local that goes fully out of scope before we measure. (An async helper would + // capture the tree in its state machine and the store measurement would double-count it.) + var (measured, entries) = asStore + ? LoadAsStore(file, logger) + : LoadAsTree(file, logger); + sw.Stop(); + + if (measured == null) + { + Console.Error.WriteLine($"failed to load catalog: {file}"); + return 1; + } + + // Settle the GC so GetTotalMemory reflects retained (live) objects, not transient load garbage. + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + var managedBytes = GC.GetTotalMemory(true); + using var proc = Process.GetCurrentProcess(); + var peakWorkingSet = proc.PeakWorkingSet64; + var privateBytes = proc.PrivateMemorySize64; + + // Keep the measured object (tree or store) alive across the measurement. + GC.KeepAlive(measured); + + var bytesPerEntry = entries > 0 ? (double)managedBytes / entries : 0; + + if (printHeader) + { + Console.WriteLine(CsvHeader); + } + + Console.WriteLine(string.Join(',', + Path.GetFileName(file), + entries.ToString(CultureInfo.InvariantCulture), + sw.ElapsedMilliseconds.ToString(CultureInfo.InvariantCulture), + managedBytes.ToString(CultureInfo.InvariantCulture), + peakWorkingSet.ToString(CultureInfo.InvariantCulture), + privateBytes.ToString(CultureInfo.InvariantCulture), + bytesPerEntry.ToString("F2", CultureInfo.InvariantCulture))); + return 0; + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static (object Measured, long Entries) LoadAsTree(string file, ILogger logger) + { + using var repo = new CatalogRepository(logger); + var root = repo.LoadDirCache(file); + if (root == null) return (null, 0); + return (root, root.FileEntryCount + root.DirEntryCount); + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private static (object Measured, long Entries) LoadAsStore(string file, ILogger logger) + { + using var repo = new CatalogRepository(logger); + var root = repo.LoadDirCache(file); + if (root == null) return (null, 0); + var entries = root.FileEntryCount + root.DirEntryCount; + var store = EntryStore.Build(root); + // root is a plain local; once this returns it is unreferenced and collectable, leaving only + // the store (which reuses the tree's interned name strings) for the caller to measure. + return (store, entries); + } + + /// + /// One-way migration: load an existing MessagePack .cde, convert to the SoA EntryStore, and write + /// the columnar/mmap format. Usage: cdeMemProbe --migrate <in.cde> [--out <out.cdex>] + /// + private static int Migrate(string[] args, string? inFile, ILogger logger) + { + if (string.IsNullOrWhiteSpace(inFile) || !File.Exists(inFile)) + { + Console.Error.WriteLine("--migrate requires an existing , e.g. --migrate cat.cde --out cat.cdex"); + return 1; + } + + var outFile = HasFlag(args, "--out", out var outArg) && !string.IsNullOrWhiteSpace(outArg) + ? outArg! + : Path.ChangeExtension(inFile, ".cdex"); + + var sw = Stopwatch.StartNew(); + EntryStore store; + using (var repo = new CatalogRepository(logger)) + { + var root = repo.LoadDirCache(inFile); + if (root == null) + { + Console.Error.WriteLine($"failed to load catalog: {inFile}"); + return 1; + } + store = EntryStore.Build(root); + } + ColumnarFormat.Write(store, outFile); + sw.Stop(); + + var srcLen = new FileInfo(inFile).Length; + var dstLen = new FileInfo(outFile).Length; + Console.Error.WriteLine( + $"migrated {store.Count:N0} entries: {Path.GetFileName(inFile)} ({srcLen:N0} B) -> " + + $"{Path.GetFileName(outFile)} ({dstLen:N0} B) in {sw.ElapsedMilliseconds:N0} ms"); + Console.WriteLine(outFile); + return 0; + } + + /// + /// Measure the zero-copy mmap path: open the columnar file (no managed load), run a search, and + /// report retained heap, allocations DURING the search, working set, and timing. + /// Usage: cdeMemProbe --flat <file.cdex> [--pattern X] [--path] + /// + private static int MeasureFlat(string[] args, string? flatFile) + { + if (string.IsNullOrWhiteSpace(flatFile) || !File.Exists(flatFile)) + { + Console.Error.WriteLine("--flat requires an existing "); + return 1; + } + + var pattern = HasFlag(args, "--pattern", out var p) && !string.IsNullOrEmpty(p) ? p! : ".txt"; + var pathMode = HasFlag(args, "--path", out _); + + // Settle, then snapshot allocation + heap baselines so we can isolate the search's own cost. + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + var heapBefore = GC.GetTotalMemory(true); + + var openSw = Stopwatch.StartNew(); + using var reader = new ColumnarCatalogReader(flatFile); + openSw.Stop(); + + var allocBefore = GC.GetTotalAllocatedBytes(precise: true); + var searchSw = Stopwatch.StartNew(); + var matches = pathMode + ? reader.FindPath(pattern, includeFiles: true, includeFolders: true) + : reader.FindName(pattern, includeFiles: true, includeFolders: true); + searchSw.Stop(); + var allocDuringSearch = GC.GetTotalAllocatedBytes(precise: true) - allocBefore; + + var heapAfter = GC.GetTotalMemory(false); // no forced collect: show what the search left live + using var proc = Process.GetCurrentProcess(); + var workingSet = proc.WorkingSet64; + var fileBytes = new FileInfo(flatFile).Length; + GC.KeepAlive(reader); + + var heapPerEntry = reader.Count > 0 ? (double)heapAfter / reader.Count : 0; + + Console.WriteLine( + "file,entries,mode,openMs,searchMs,matches,allocDuringSearch,heapBytes,heapPerEntry,workingSet,fileBytes"); + Console.WriteLine(string.Join(',', + Path.GetFileName(flatFile), + reader.Count.ToString(CultureInfo.InvariantCulture), + pathMode ? "path" : "name", + openSw.ElapsedMilliseconds.ToString(CultureInfo.InvariantCulture), + searchSw.Elapsed.TotalMilliseconds.ToString("F2", CultureInfo.InvariantCulture), + matches.ToString(CultureInfo.InvariantCulture), + allocDuringSearch.ToString(CultureInfo.InvariantCulture), + heapAfter.ToString(CultureInfo.InvariantCulture), + heapPerEntry.ToString("F2", CultureInfo.InvariantCulture), + workingSet.ToString(CultureInfo.InvariantCulture), + fileBytes.ToString(CultureInfo.InvariantCulture))); + Console.Error.WriteLine( + $"baseline heap before open: {heapBefore:N0} B; sample path[1] = {reader.FullPath(1)}"); + return 0; + } + + /// + /// Returns true if is present. If the next token is not another flag it + /// is returned as (so both --generate 100 and bare flags work). + /// + private static bool HasFlag(string[] args, string name, out string? value) + { + value = null; + for (var i = 0; i < args.Length; i++) + { + if (!string.Equals(args[i], name, StringComparison.OrdinalIgnoreCase)) continue; + if (i + 1 < args.Length && !args[i + 1].StartsWith("--", StringComparison.Ordinal)) + { + value = args[i + 1]; + } + return true; + } + return false; + } +} diff --git a/src/cdeMemProbe/SyntheticCatalog.cs b/src/cdeMemProbe/SyntheticCatalog.cs new file mode 100644 index 0000000..e012672 --- /dev/null +++ b/src/cdeMemProbe/SyntheticCatalog.cs @@ -0,0 +1,112 @@ +using System; +using System.Collections.Generic; +using cdeLib.Entities; + +namespace cdeMemProbe; + +/// +/// Deterministic synthetic .cde catalog generator used to measure memory footprint and +/// search performance reproducibly across optimization phases. Builds a +/// tree in memory with a realistic file:directory ratio, then the caller saves/loads it. +/// +/// The SAME fixture (same seed + parameters) must be used for every phase so per-entry-byte and +/// search-timing deltas are attributable to the change under test, not to fixture drift. +/// +public static class SyntheticCatalog +{ + // Representative extension/prefix spread; interning means duplicates are deduplicated in memory + // exactly as a real catalog would be. + private static readonly string[] Extensions = + [".txt", ".jpg", ".pdf", ".doc", ".mp4", ".zip", ".exe", ".dll", ".cs", ".json", ".png", ".log"]; + + private static readonly string[] Prefixes = + ["Document", "Image", "Video", "Archive", "Config", "Data", "Log", "Report", "Backup", "Cache"]; + + private static readonly DateTime BaseDate = new(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + /// + /// Build a synthetic tree of approximately entries + /// (files + directories combined), using a ~:1 file/dir ratio. + /// Generation is breadth-first so the tree is bushy and shallow like a real file system. + /// + /// Approximate total of files + directories to generate. + /// When true, every file gets an MD5-sized hash set (worst case for memory). + /// Deterministic RNG seed; keep fixed across phases. + /// Files added per directory (drives the file:dir ratio). + /// Sub-directories added per directory (drives breadth). + public static RootEntry Generate( + int targetEntryCount, + bool withHashes = false, + int seed = 42, + int filesPerDir = 50, + int subDirsPerDir = 5, + bool sharedNames = false) + { + var random = new Random(seed); + var root = new RootEntry + { + Path = @"C:\synthetic", + DefaultFileName = "synthetic.cde", + DriveLetterHint = "C", + VolumeName = "SYNTHETIC", + Description = $"Synthetic catalog ~{targetEntryCount} entries (hashes={withHashes})", + Children = new List(), + }; + + var created = 0; + // Directories that can still receive children. Breadth-first never drains before the target + // is hit because each directory enqueues more directories than it dequeues. + var queue = new Queue(); + + Populate(root, null); + while (queue.Count > 0 && created < targetEntryCount) + { + Populate(null, queue.Dequeue()); + } + + root.SetInMemoryFields(); + return root; + + void Populate(RootEntry? rootParent, DirEntry? dirParent) + { + for (var i = 0; i < filesPerDir && created < targetEntryCount; i++) + { + var file = new DirEntry(false); + // sharedNames: every file shares one interned name, so the measured footprint + // excludes per-file name strings — the delta vs normal isolates the name cost. + if (sharedNames) + { + file.SetPath("x"); + } + else + { + var ext = Extensions[random.Next(Extensions.Length)]; + var prefix = Prefixes[random.Next(Prefixes.Length)]; + file.SetPath($"{prefix}_{created:D7}{ext}"); + } + file.Size = random.Next(1, 50_000_000); + file.Modified = BaseDate.AddMinutes(random.Next(0, 5_000_000)); + if (withHashes) + { + // Mix of unique and duplicate hashes (every 17th repeats) to exercise dupe paths. + file.SetHash(created % 17 == 0 ? 17 : created); + } + + if (rootParent != null) rootParent.AddChild(file); + else dirParent!.AddChild(file); + created++; + } + + for (var i = 0; i < subDirsPerDir && created < targetEntryCount; i++) + { + var dir = new DirEntry(true); + dir.SetPath($"dir_{created:D7}"); + dir.Modified = BaseDate.AddMinutes(random.Next(0, 5_000_000)); + if (rootParent != null) rootParent.AddChild(dir); + else dirParent!.AddChild(dir); + created++; + queue.Enqueue(dir); + } + } + } +} diff --git a/src/cdeMemProbe/cdeMemProbe.csproj b/src/cdeMemProbe/cdeMemProbe.csproj new file mode 100644 index 0000000..7b53141 --- /dev/null +++ b/src/cdeMemProbe/cdeMemProbe.csproj @@ -0,0 +1,21 @@ + + + + Exe + net10.0 + enable + enable + latest + + true + + true + true + + + + + + + diff --git a/src/cdeWin/CDEWinForm.cs b/src/cdeWin/CDEWinForm.cs index c985dca..2c9c682 100644 --- a/src/cdeWin/CDEWinForm.cs +++ b/src/cdeWin/CDEWinForm.cs @@ -78,7 +78,7 @@ public partial class CDEWinForm : Form, ICDEWinForm [DesignerSerializationVisibility(DesignerSerializationVisibility.Hidden)] public IListViewHelper DirectoryListViewHelper { get; set; } [DesignerSerializationVisibility(DesignerSerializationVisibility.Hidden)] - public IListViewHelper CatalogListViewHelper { get; set; } + public IListViewHelper CatalogListViewHelper { get; set; } [DesignerSerializationVisibility(DesignerSerializationVisibility.Hidden)] public CheckBoxDependentControlHelper FromDate { get; set; } @@ -176,19 +176,19 @@ private void RegisterClientEvents() "Using reload catalogs will use more memory than quitting and starting again."); SetToolTip(regexCheckbox, "Disabling Regex makes search faster"); - whatToSearchComboBox.Items.AddRange(new object[] { "Include Path in Search", "Exclude Path from Search" }); + whatToSearchComboBox.Items.AddRange("Include Path in Search", "Exclude Path from Search"); whatToSearchComboBox.SelectedIndex = 0; // default Include whatToSearchComboBox.DropDownStyle = ComboBoxStyle.DropDownList; SetToolTip(whatToSearchComboBox, "Excluding Path so that only entry Names are searched makes search faster."); - findComboBox.Items.AddRange(new object[] { "Files and Folders", "Files Only", "Folders Only" }); + findComboBox.Items.AddRange("Files and Folders", "Files Only", "Folders Only"); findComboBox.SelectedIndex = 0; // default Files and Folders findComboBox.DropDownStyle = ComboBoxStyle.DropDownList; - // TODO having ListViewHelper setup in VIEW breaks passive view. i think. + // TODO having ListViewHelper set up in VIEW breaks passive view. i think. // * it does register a bunch of events which it fires.... ? so not real bad. - // - whats happening is im making view smarter... with specific behaviour. + // - whats happening is I'm making view smarter... with specific behaviour. // - but its not passive, passive would require ListViewHelper to raise events // - from gui actions.... and decisions from presenter... // - - at moment, ListViewHelper is small presenter ? @@ -223,7 +223,7 @@ private void RegisterClientEvents() patternComboBox.GotFocus += (_, _) => AcceptButton = searchButton; patternComboBox.LostFocus += (_, _) => AcceptButton = null; - CatalogListViewHelper = new ListViewHelper(catalogResultListView) + CatalogListViewHelper = new ListViewHelper(catalogResultListView) { MultiSelect = false, // ReSharper disable PossibleNullReferenceException @@ -492,9 +492,9 @@ public TreeNode DirectoryTreeViewNodes } } - public bool IncludeFiles => findComboBox.SelectedIndex == 0 || findComboBox.SelectedIndex == 1; + public bool IncludeFiles => findComboBox.SelectedIndex is 0 or 1; - public bool IncludeFolders => findComboBox.SelectedIndex == 0 || findComboBox.SelectedIndex == 2; + public bool IncludeFolders => findComboBox.SelectedIndex is 0 or 2; [DesignerSerializationVisibility(DesignerSerializationVisibility.Hidden)] public int FindEntryFilter diff --git a/src/cdeWin/CDEWinFormPresenter.cs b/src/cdeWin/CDEWinFormPresenter.cs index 4dd3225..e5eef54 100644 --- a/src/cdeWin/CDEWinFormPresenter.cs +++ b/src/cdeWin/CDEWinFormPresenter.cs @@ -10,6 +10,8 @@ using System.Windows.Forms; using cdeLib; using cdeLib.Entities; +using cdeLib.Entities.Columnar; +using cdeLib.Entities.Soa; using cdeLib.Infrastructure; using cdeWin.Cfg; using JetBrains.Annotations; @@ -27,9 +29,74 @@ public class CDEWinFormPresenter : Presenter, ICDEWinFormPresenter private readonly Color _listViewDirForeColor = Color.DarkBlue; private readonly ICDEWinForm _clientForm; - private List _rootEntries; + + // Catalogs are held as IEntrySource — either a zero-copy ColumnarCatalogReader over a memory-mapped + // .cdex file (preferred: the catalog data stays in the OS page cache, not the managed heap) or, when + // no .cdex exists, an in-memory EntryStore built from the loaded .cde tree. Each catalog root is + // exposed as an EntryRef so the existing ICommonEntry-based GUI works unchanged either way. + private List _catalogRoots; private readonly IConfig _config; + private static List ToCatalogRoots(List trees) + { + var roots = new List(trees?.Count ?? 0); + if (trees == null) return roots; + for (var i = 0; i < trees.Count; i++) + { + roots.Add(new EntryRef(EntryStore.Build(trees[i]), 0)); + trees[i] = null; // release the tree so it can be collected + } + return roots; + } + + // Open each .cdex as a zero-copy mmap reader. Unreadable files are skipped (logged by the caller). + private static List ReadersToCatalogRoots(IList cdexFiles) + { + var roots = new List(cdexFiles.Count); + foreach (var file in cdexFiles) + { + try + { + roots.Add(new EntryRef(new ColumnarCatalogReader(file), 0)); + } + catch (Exception ex) + { + Log.Logger.Warning(ex, "Skipping unreadable .cdex {File}", file); + } + } + return roots; + } + + private static IEntrySource SourceOf(ICommonEntry root) => ((EntryRef)root).Source; + + // Catalog of a search-result pair (its entries are EntryRefs into a source). + private static IEntrySource SourceOfPair(PairDirEntry pde) => (pde.ChildDE as EntryRef)?.Source; + + // Prefer the zero-copy .cdex catalogs (mmap, near-zero managed heap); fall back to loading the + // .cde trees and building in-memory stores when no .cdex exists. Disposes any previously held + // mmap sources first so reloads don't leak mappings. + private async Task> LoadCatalogRootsAsync() + { + DisposeCatalogSources(); + var cdex = _loadCatalogService.GetColumnarFiles(_config); + if (cdex is { Count: > 0 }) + { + return ReadersToCatalogRoots(cdex); + } + return ToCatalogRoots(await _loadCatalogService.LoadRootEntriesAsync( + _config, OnLoadProgress, _loadingCts.Token)); + } + + // Memory-mapped catalog sources must be released on reload/exit so the mappings are closed. + private void DisposeCatalogSources() + { + if (_catalogRoots == null) return; + foreach (var root in _catalogRoots) + { + if (root is EntryRef { Source: IDisposable disposable }) disposable.Dispose(); + } + } + private readonly string[] _directoryVals; private readonly string[] _searchVals; private readonly string[] _catalogVals; @@ -71,7 +138,7 @@ public CDEWinFormPresenter( _clientForm = form; _config = config; _loadCatalogService = loadCatalogService; - _rootEntries = new List(); + _catalogRoots = new List(); _searchVals = new string[_config.DefaultSearchResultColumnCount]; _directoryVals = new string[_config.DefaultDirectoryColumnCount]; @@ -102,10 +169,7 @@ public async Task InitializeAsync() try { - _rootEntries = await _loadCatalogService.LoadRootEntriesAsync( - _config, - OnLoadProgress, - _loadingCts.Token); + _catalogRoots = await LoadCatalogRootsAsync(); SetCatalogListView(); SetMemoryStatus(); @@ -136,7 +200,7 @@ public async Task InitializeAsync() private void OnLoadProgress(int current, int total, string message) { - if (_clientForm is Control control && control.InvokeRequired) + if (_clientForm is Control { InvokeRequired: true } control) { control.BeginInvoke(() => OnLoadProgress(current, total, message)); return; @@ -146,7 +210,7 @@ private void OnLoadProgress(int current, int total, string message) _clientForm.SetSearchTimeStatus(message); if (total > 0) { - _clientForm.SetLoadingProgressValue((current * 100) / total); + _clientForm.SetLoadingProgressValue(current * 100 / total); } SetMemoryStatus(); } @@ -194,10 +258,11 @@ private void RegisterListViewSorters() private void SetCatalogListView() { var catalogHelper = _clientForm.CatalogListViewHelper; - var count = catalogHelper.SetList(_rootEntries); + var count = catalogHelper.SetList(_catalogRoots); catalogHelper.SortList(); _clientForm.SetCatalogsLoadedStatus(count); - _clientForm.SetTotalFileEntriesLoadedStatus(_rootEntries.TotalFileEntries()); + _clientForm.SetTotalFileEntriesLoadedStatus( + (int)_catalogRoots.Sum(r => (long)SourceOf(r).RootFileEntryCount + SourceOf(r).RootDirEntryCount)); } private static double BytesToMb(long bytes) => bytes / (1024.0 * 1024.0); @@ -261,7 +326,7 @@ private static void CreateNodesPreExpand(TreeNode parentNode) private static bool HasDummyChildNode(TreeNode parentNode) { - return parentNode.Nodes.Count == 1 && parentNode.Nodes[0].Text == DummyNodeName; + return parentNode.Nodes is [{ Text: DummyNodeName }]; } private static void AddAllDirectoriesChildren(TreeNode treeNode, ICommonEntry dirEntry) @@ -274,12 +339,10 @@ private static void AddAllDirectoriesChildren(TreeNode treeNode, ICommonEntry di private static void AddDirectoryChildren(TreeNode treeNode, ICommonEntry dirEntry) { - if (dirEntry.IsDirectory) - { - var newTreeNode = NewTreeNode(dirEntry); - treeNode.Nodes.Add(newTreeNode); - SetDummyChildNode(newTreeNode, dirEntry); - } + if (!dirEntry.IsDirectory) return; + var newTreeNode = NewTreeNode(dirEntry); + treeNode.Nodes.Add(newTreeNode); + SetDummyChildNode(newTreeNode, dirEntry); } /// @@ -320,21 +383,24 @@ public void CatalogRetrieveVirtualItem() catalogHelper.RenderItem = lvi; } - private Color CreateRowValuesForRootEntry(IList vals, RootEntry rootEntry, Color listViewForeColor) - { - vals[0] = rootEntry.Path; - vals[1] = rootEntry.VolumeName; - vals[2] = rootEntry.DirEntryCount.ToString(); - vals[3] = rootEntry.FileEntryCount.ToString(); - vals[4] = (rootEntry.DirEntryCount + rootEntry.FileEntryCount).ToString(); - vals[5] = rootEntry.DriveLetterHint; - vals[6] = rootEntry.Size.ToHRString(); - vals[7] = rootEntry.AvailSpace.ToHRString(); - vals[8] = rootEntry.TotalSpace.ToHRString(); - vals[9] = string.Format(_config.DateFormatYMDHMS, rootEntry.ScanStartUtc.ToLocalTime()); - vals[10] = $"{TimeSpan.FromMilliseconds(rootEntry.ScanDurationMilliseconds).TotalSeconds:0.} sec"; - vals[11] = rootEntry.ActualFileName; - vals[12] = rootEntry.Description; + private Color CreateRowValuesForRootEntry(IList vals, ICommonEntry catalogRoot, Color listViewForeColor) + { + var s = SourceOf(catalogRoot); + var scanStart = new DateTime(s.ScanStartUtcTicks, DateTimeKind.Utc); + var scanDurationMs = (s.ScanEndUtcTicks - s.ScanStartUtcTicks) / TimeSpan.TicksPerMillisecond; + vals[0] = s.RootPath; + vals[1] = s.VolumeName; + vals[2] = s.RootDirEntryCount.ToString(); + vals[3] = s.RootFileEntryCount.ToString(); + vals[4] = (s.RootDirEntryCount + s.RootFileEntryCount).ToString(); + vals[5] = s.DriveLetterHint; + vals[6] = s.RootSize.ToHRString(); + vals[7] = s.AvailSpace.ToHRString(); + vals[8] = s.TotalSpace.ToHRString(); + vals[9] = string.Format(_config.DateFormatYMDHMS, scanStart.ToLocalTime()); + vals[10] = $"{TimeSpan.FromMilliseconds(scanDurationMs).TotalSeconds:0.} sec"; + vals[11] = s.ActualFileName; + vals[12] = s.Description; return listViewForeColor; } @@ -342,7 +408,7 @@ private Color CreateRowValuesForRootEntry(IList vals, RootEntry rootEntr public class BgWorkerParam { public FindOptions Options; - public IList RootEntries; + public IList RootEntries; public BgWorkerState State; } @@ -417,7 +483,7 @@ public void Search() var param = new BgWorkerParam { Options = findOptions, - RootEntries = _rootEntries, + RootEntries = _catalogRoots, State = new BgWorkerState() }; _bgWorker.RunWorkerAsync(param); @@ -425,16 +491,13 @@ public void Search() private bool FromToDateInvalid() { - if (_clientForm.FromDate.Checked - && _clientForm.ToDate.Checked - && _clientForm.FromDateValue.Date >= _clientForm.ToDateValue.Date) - { - _clientForm.MessageBox( - "The From Date Field is greater than the To Date field no search results possible."); - return true; - } + if (!_clientForm.FromDate.Checked + || !_clientForm.ToDate.Checked + || _clientForm.FromDateValue.Date < _clientForm.ToDateValue.Date) return false; + _clientForm.MessageBox( + "The From Date Field is greater than the To Date field no search results possible."); + return true; - return false; } private bool FromToHourInvalid() @@ -453,31 +516,23 @@ private bool FromToHourInvalid() private bool RegexIsBad() { - if (_clientForm.RegexMode) - { - var regexError = RegexHelper.GetRegexErrorMessage(_clientForm.Pattern); - if (!string.IsNullOrEmpty(regexError)) - { - _clientForm.MessageBox(regexError); - return true; - } - } + if (!_clientForm.RegexMode) return false; + var regexError = RegexHelper.GetRegexErrorMessage(_clientForm.Pattern); + if (string.IsNullOrEmpty(regexError)) return false; + _clientForm.MessageBox(regexError); + return true; - return false; } private bool FromToSizeInvalid() { - if (_clientForm.FromSize.Checked - && _clientForm.ToSize.Checked - && FromSizeValue() > ToSizeValue()) - { - _clientForm.MessageBox( - "The From Size Field is greater than the To Size field no search results possible."); - return true; - } + if (!_clientForm.FromSize.Checked + || !_clientForm.ToSize.Checked + || FromSizeValue() <= ToSizeValue()) return false; + _clientForm.MessageBox( + "The From Size Field is greater than the To Size field no search results possible."); + return true; - return false; } private long FromSizeValue() @@ -537,43 +592,75 @@ private void BgWorkerDoWork(object sender, DoWorkEventArgs e) var worker = (BackgroundWorker)sender; var argument = (BgWorkerParam)e.Argument; var findOptions = argument.Options; - var rootEntries = argument.RootEntries; + var catalogRoots = argument.RootEntries; var state = argument.State; + // Translate the GUI FindOptions into the SoA search options and run over the EntryStores. + var opts = new EntryStoreFindOptions + { + Pattern = findOptions.Pattern, + RegexMode = findOptions.RegexMode, + IncludePath = findOptions.IncludePath, + IncludeFiles = findOptions.IncludeFiles, + IncludeFolders = findOptions.IncludeFolders, + FromSizeEnable = findOptions.FromSizeEnable, FromSize = findOptions.FromSize, + ToSizeEnable = findOptions.ToSizeEnable, ToSize = findOptions.ToSize, + FromDateEnable = findOptions.FromDateEnable, FromDate = findOptions.FromDate, + ToDateEnable = findOptions.ToDateEnable, ToDate = findOptions.ToDate, + FromHourEnable = findOptions.FromHourEnable, FromHour = findOptions.FromHour, + ToHourEnable = findOptions.ToHourEnable, ToHour = findOptions.ToHour, + NotOlderThanEnable = findOptions.NotOlderThanEnable, NotOlderThan = findOptions.NotOlderThan, + }; + var limit = findOptions.LimitResultCount; + + var sources = catalogRoots.Select(SourceOf).ToList(); + var grandTotal = sources.Sum(s => s.Count); + var scannedBase = 0; + var list = new List(500); - state.ListCount = list.Count; // 0 + state.ListCount = 0; state.List = list; + state.End = grandTotal; worker.ReportProgress(0, state); - findOptions.VisitorFunc = (p, d) => - { - list.Add(new PairDirEntry(p, d)); - return true; - }; - findOptions.ProgressFunc = (counter, end) => - { - state.ListCount = list.Count; // concurrency ! - state.List = list; // concurrency !!!! - state.Counter = counter; - state.End = end; - worker.ReportProgress((int)(100.0 * counter / end), state); - }; + + var lastReport = Stopwatch.GetTimestamp(); + var reportTicks = Stopwatch.Frequency / 10; // ~100ms streaming + var timer = Stopwatch.StartNew(); - findOptions.Find(rootEntries); - //findOptions.FindAsync(rootEntries).GetAwaiter().GetResult(); + foreach (var source in sources) + { + if (worker.CancellationPending || list.Count >= limit) break; + var baseScanned = scannedBase; + source.Find(opts, + onMatch: idx => + { + list.Add(new PairDirEntry(new EntryRef(source, source.ParentOf(idx)), new EntryRef(source, idx))); + }, + isCancelled: () => worker.CancellationPending || list.Count >= limit, + onScan: scanned => Report(baseScanned + scanned)); + scannedBase += source.Count; + } timer.Stop(); Log.Logger.Information( "Search execution time: {ExecutionTime} ms, Total found {TotalFound}", timer.ElapsedMilliseconds, list.Count); state.ListCount = list.Count; state.List = list; - var completePercent = (int)(100.0 * state.Counter / state.End); - if (state.End - state.Counter < findOptions.ProgressModifier) + state.Counter = grandTotal; + worker.ReportProgress(100, state); + e.Result = list; + return; + + void Report(int scanned) { - completePercent = 100; + var now = Stopwatch.GetTimestamp(); + if (now - lastReport < reportTicks) return; + lastReport = now; + state.ListCount = list.Count; + state.List = new List(list); // immutable snapshot for the UI thread + state.Counter = scanned; + worker.ReportProgress(grandTotal > 0 ? (int)(100.0 * scanned / grandTotal) : 0, state); } - - worker.ReportProgress(completePercent, state); - e.Result = list; } private void BgWorkerRunWorkerCompleted(object sender, RunWorkerCompletedEventArgs e) @@ -645,7 +732,8 @@ public void SearchResultRetrieveVirtualItem() _searchVals[(int)SearchResultColumn.FullPath] = pairDirEntry.ParentDE.FullPath; //TODO: Possibly wasting cycles traversing to the root for this, make smarter. - _searchVals[(int)SearchResultColumn.Catalog] = pairDirEntry.GetRootEntry().DefaultFileName; + _searchVals[(int)SearchResultColumn.Catalog] = + SourceOfPair(pairDirEntry)?.DefaultFileName ?? pairDirEntry.GetRootEntry()?.DefaultFileName ?? ""; searchHelper.RenderItem = BuildListViewItem(_searchVals, itemColor, pairDirEntry); } @@ -723,6 +811,7 @@ public void MyFormClosing() { CancelLoading(); _config.RecordConfig(_clientForm); + DisposeCatalogSources(); // close any memory-mapped .cdex catalogs _clientForm.CleanUp(); } @@ -731,10 +820,10 @@ public void CatalogListViewItemActivate() _clientForm.CatalogListViewHelper.ActionOnActivateItem(GoToDirectoryRoot); } - private void GoToDirectoryRoot(RootEntry newRoot) + private void GoToDirectoryRoot(ICommonEntry newRoot) { - var currentRoot = (RootEntry)_clientForm.DirectoryTreeViewNodes?.Tag; - if (currentRoot == null || currentRoot != newRoot) + var currentRoot = (ICommonEntry)_clientForm.DirectoryTreeViewNodes?.Tag; + if (!SameRoot(currentRoot, newRoot)) { SetNewDirectoryRoot(newRoot); } @@ -742,7 +831,11 @@ private void GoToDirectoryRoot(RootEntry newRoot) _clientForm.SelectDirectoryPane(); } - private TreeNode SetNewDirectoryRoot(RootEntry newRoot) + // Catalog roots are EntryRef instances; two refs to the same catalog share a store. + private static bool SameRoot(ICommonEntry a, ICommonEntry b) + => a is EntryRef ea && b is EntryRef eb && ReferenceEquals(ea.Source, eb.Source); + + private TreeNode SetNewDirectoryRoot(ICommonEntry newRoot) { var newRootNode = BuildRootNode(newRoot); _clientForm.DirectoryTreeViewNodes = newRootNode; @@ -750,7 +843,7 @@ private TreeNode SetNewDirectoryRoot(RootEntry newRoot) return newRootNode; } - private static TreeNode BuildRootNode(RootEntry rootEntry) + private static TreeNode BuildRootNode(ICommonEntry rootEntry) { var rootTreeNode = NewTreeNode(rootEntry); SetDummyChildNode(rootTreeNode, rootEntry); @@ -778,16 +871,16 @@ private void SetDirectoryWithExpand(ICommonEntry dirEntry) private void SetDirectoryWithExpand(IEnumerable activatedDirEntryList) { var currentRootNode = _clientForm.DirectoryTreeViewNodes; - var currentRoot = (RootEntry)currentRootNode?.Tag; + var currentRoot = (ICommonEntry)currentRootNode?.Tag; TreeNode workingTreeNode = null; - RootEntry newRoot = null; + ICommonEntry newRoot = null; foreach (var entry in activatedDirEntryList) { if (newRoot == null) { - newRoot = (RootEntry)entry; - if (currentRoot != newRoot) + newRoot = entry; + if (!SameRoot(currentRoot, newRoot)) { currentRootNode = SetNewDirectoryRoot(newRoot); currentRoot = newRoot; @@ -871,8 +964,8 @@ private int SearchResultCompare(PairDirEntry pde1, PairDirEntry pde2) case 3: compareResult = string.Compare( - pde1.GetRootEntry().ActualFileName, - pde2.GetRootEntry().ActualFileName, + SourceOfPair(pde1)?.ActualFileName ?? pde1.GetRootEntry()?.ActualFileName, + SourceOfPair(pde2)?.ActualFileName ?? pde2.GetRootEntry()?.ActualFileName, StringComparison.OrdinalIgnoreCase); break; @@ -1008,12 +1101,10 @@ private void ViewFileInDirectoryTab(PairDirEntry pde) private void SelectFileInDirectoryTab(ICommonEntry dirEntry) { - if (!dirEntry.IsDirectory) - { - var index = _directoryList.IndexOf(dirEntry); - var directoryHelper = _clientForm.DirectoryListViewHelper; - directoryHelper.SelectItem(index); - } + if (dirEntry.IsDirectory) return; + var index = _directoryList.IndexOf(dirEntry); + var directoryHelper = _clientForm.DirectoryListViewHelper; + directoryHelper.SelectItem(index); } public void DirectoryContextMenuOpenClick() @@ -1052,7 +1143,7 @@ public void DirectoryContextMenuCopyFullPathClick() { DirectoryGetContextMenuPairDirEntries(enumerableDirEntry => { - // we dont have parent dir entry here ... + // we don't have parent dir entry here var s = new StringBuilder(); foreach (var dirEntry in enumerableDirEntry) { @@ -1144,26 +1235,28 @@ public void CatalogListViewColumnClick() _clientForm.CatalogListViewHelper.ListViewColumnClick(); } - private int RootCompare(RootEntry re1, RootEntry re2) + private int RootCompare(ICommonEntry root1, ICommonEntry root2) { + var re1 = SourceOf(root1); + var re2 = SourceOf(root2); var catalogHelper = _clientForm.CatalogListViewHelper; var column = catalogHelper.SortColumn; var compareResult = column switch { - 0 => string.Compare(re1.Path, re2.Path, StringComparison.Ordinal), + 0 => string.Compare(re1.RootPath, re2.RootPath, StringComparison.Ordinal), 1 => string.Compare(string.IsNullOrEmpty(re1.VolumeName) ? "" : re1.VolumeName, string.IsNullOrEmpty(re2.VolumeName) ? "" : re2.VolumeName, StringComparison.Ordinal), - 2 => re1.DirEntryCount.CompareTo(re2.DirEntryCount), - 3 => re1.FileEntryCount.CompareTo(re2.FileEntryCount), - 4 => (re1.DirEntryCount + re1.FileEntryCount).CompareTo(re2.DirEntryCount + re2.FileEntryCount), + 2 => re1.RootDirEntryCount.CompareTo(re2.RootDirEntryCount), + 3 => re1.RootFileEntryCount.CompareTo(re2.RootFileEntryCount), + 4 => (re1.RootDirEntryCount + re1.RootFileEntryCount).CompareTo(re2.RootDirEntryCount + re2.RootFileEntryCount), 5 => string.Compare(re1.DriveLetterHint, re2.DriveLetterHint, StringComparison.Ordinal), - 6 => re1.Size.CompareTo(re2.Size), + 6 => re1.RootSize.CompareTo(re2.RootSize), 7 => re1.AvailSpace.CompareTo(re2.AvailSpace), 8 => re1.TotalSpace.CompareTo(re2.TotalSpace), - 9 => re1.ScanStartUtc.CompareTo(re2.ScanStartUtc), - 10 => re1.ScanDurationMilliseconds.CompareTo(re2.ScanDurationMilliseconds), + 9 => re1.ScanStartUtcTicks.CompareTo(re2.ScanStartUtcTicks), + 10 => (re1.ScanEndUtcTicks - re1.ScanStartUtcTicks).CompareTo(re2.ScanEndUtcTicks - re2.ScanStartUtcTicks), 11 => string.Compare(re1.ActualFileName, re2.ActualFileName, StringComparison.Ordinal), - 12 => re1.DescriptionCompareTo(re2, _config), + 12 => string.Compare(re1.Description, re2.Description, StringComparison.Ordinal), _ => throw new Exception($"Problem column {column} not handled for sort.") }; @@ -1200,11 +1293,8 @@ public async void ReloadCatalogs() var directoryListHelper = _clientForm.DirectoryListViewHelper; directoryListHelper.SetList(null); - var previousRootEntries = _rootEntries; - foreach (var rootEntry in previousRootEntries) - { - rootEntry.ClearCommonEntryFields(); - } + // Drop the previous stores; releasing the references lets the GC reclaim them. + _catalogRoots = new List(); _clientForm.AddLine(string.Empty); _clientForm.AddLine("{0} v{1} reloading catalogs", _config.ProductName, _config.Version); @@ -1222,14 +1312,11 @@ public async void ReloadCatalogs() _clientForm.SetLoadingProgressValue(0); SetMemoryStatus(); - _rootEntries = await _loadCatalogService.LoadRootEntriesAsync( - _config, - OnLoadProgress, - _loadingCts.Token); + _catalogRoots = await LoadCatalogRootsAsync(); - if (_rootEntries.Count > 0) + if (_catalogRoots.Count > 0) { - SetNewDirectoryRoot(_rootEntries.First()); + SetNewDirectoryRoot(_catalogRoots.First()); } SetCatalogListView(); diff --git a/src/cdeWin/ContextMenuHelper.cs b/src/cdeWin/ContextMenuHelper.cs index 9e72284..e515505 100644 --- a/src/cdeWin/ContextMenuHelper.cs +++ b/src/cdeWin/ContextMenuHelper.cs @@ -145,16 +145,14 @@ public EventHandler ParentHandler /// public CancelEventHandler CancelOpeningEventHandler { - get => _cancelOpeningEventHandler; + get; set { - _cancelOpeningEventHandler = value; + field = value; _menu.Opening += value; } } - private CancelEventHandler _cancelOpeningEventHandler; - public ContextMenuHelper() { // set all keys here rather than in individual setters for handlers. diff --git a/src/cdeWin/ICDEWinForm.cs b/src/cdeWin/ICDEWinForm.cs index 8036bf3..dbdfcf2 100644 --- a/src/cdeWin/ICDEWinForm.cs +++ b/src/cdeWin/ICDEWinForm.cs @@ -92,7 +92,7 @@ public interface ICDEWinForm : IView IListViewHelper SearchResultListViewHelper { get; set; } IListViewHelper DirectoryListViewHelper { get; set; } - IListViewHelper CatalogListViewHelper { get; set; } + IListViewHelper CatalogListViewHelper { get; set; } CheckBoxDependentControlHelper FromDate { get; set; } CheckBoxDependentControlHelper ToDate { get; set; } diff --git a/src/cdeWin/IListViewHelper.cs b/src/cdeWin/IListViewHelper.cs new file mode 100644 index 0000000..6721b82 --- /dev/null +++ b/src/cdeWin/IListViewHelper.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Windows.Forms; +using cdeWin.Cfg; + +namespace cdeWin; + +public interface IListViewHelper : IDisposable where T : class +{ + /// + /// Used by virtual mode ListView + /// + int RetrieveItemIndex { get; set; } + + ListViewItem RenderItem { get; set; } + + IEnumerable SelectedIndices { get; set; } + + int SelectedIndicesCount { get; set; } + + SortOrder ColumnSortOrder { get; set; } + + int SortColumn { get; set; } + + Comparison ColumnSortCompare { get; set; } + + void InitSort(); + IEnumerable ColumnConfigs(); + void SetColumnConfigs(IEnumerable columns); + void ForceDraw(); + void SelectItem(int index); + void DeselectAllItems(); + void SelectAllItems(); + int SetList(List list); + void ListViewColumnClick(); + void SortList(); + void ActionOnSelectedItems(Action> action); + void ActionOnSelectedItem(Action action); + void ActionOnActivateItem(Action action); + T GetItemAt(int index); + void SearchListContextMenuOpening(object sender, System.ComponentModel.CancelEventArgs e); +} \ No newline at end of file diff --git a/src/cdeWin/KeyEqualityComparer.cs b/src/cdeWin/KeyEqualityComparer.cs index 9917acc..5e179c7 100644 --- a/src/cdeWin/KeyEqualityComparer.cs +++ b/src/cdeWin/KeyEqualityComparer.cs @@ -10,7 +10,7 @@ public class KeyEqualityComparer : IEqualityComparer { protected readonly Func KeyExtractor; - public KeyEqualityComparer(Func keyExtractor) + protected KeyEqualityComparer(Func keyExtractor) { KeyExtractor = keyExtractor; } @@ -24,21 +24,4 @@ public int GetHashCode(T obj) { return KeyExtractor(obj).GetHashCode(); } -} - -/// -/// see http://stackoverflow.com/questions/98033/wrap-a-delegate-in-an-iequalitycomparer -/// -public class StrictKeyEqualityComparer - : KeyEqualityComparer where TKey : IEquatable -{ - public StrictKeyEqualityComparer(Func keyExtractor) : base(keyExtractor) - { } - - public override bool Equals(T x, T y) - { - // This will use the overload that accepts a TKey parameter - // instead of an object parameter. - return KeyExtractor(x).Equals(KeyExtractor(y)); - } } \ No newline at end of file diff --git a/src/cdeWin/ListViewExtensions.cs b/src/cdeWin/ListViewExtensions.cs index 4ef0c57..690a070 100644 --- a/src/cdeWin/ListViewExtensions.cs +++ b/src/cdeWin/ListViewExtensions.cs @@ -5,7 +5,7 @@ namespace cdeWin; -// FROM http://stackoverflow.com/a/254139 +// From http://stackoverflow.com/a/254139 // ReSharper disable InconsistentNaming [EditorBrowsable(EditorBrowsableState.Never)] public static class ListViewExtensions diff --git a/src/cdeWin/ListViewHelper.cs b/src/cdeWin/ListViewHelper.cs index da1a884..c328887 100644 --- a/src/cdeWin/ListViewHelper.cs +++ b/src/cdeWin/ListViewHelper.cs @@ -5,73 +5,16 @@ using cdeWin.Cfg; namespace cdeWin; -// think about making Presenter<> look for all members that implement IPresenterHelper -// and hookup events to them as well with matching names ? +// Think about making Presenter<> look for all members that implement IPresenterHelper +// and hook up events to them as well with matching names ? // encapsulate ListView in VirtualMode handling -public interface IListViewHelper : IDisposable where T : class -{ - /// - /// Used by virtual mode ListView - /// - int RetrieveItemIndex { get; set; } - - ListViewItem RenderItem { get; set; } - int AfterActivateIndex { get; set; } - int ColumnClickIndex { get; set; } - IEnumerable SelectedIndices { get; set; } - int SelectedIndicesCount { get; set; } - SortOrder ColumnSortOrder { get; set; } - int SortColumn { get; set; } - Comparison ColumnSortCompare { get; set; } - - /// - /// Adds CacheVirtualItems, RetrieveVirtualItem handler which sets RetrieveItemIndex before EventAction. - /// - EventAction RetrieveVirtualItem { get; set; } - - /// - /// Adds ColumnClick handler which sets ColumnClickIndex before EventAction.. - /// - EventAction ColumnClick { get; set; } - - /// - /// Adds ItemActivate handler which sets AfterActivateIndex before EventAction.. - /// - EventAction ItemActivate { get; set; } - - ContextMenuStrip ContextMenu { get; set; } - - /// - /// Adds SelectedIndexChanged, VirtualItemsSelectionRangeChanged handlers. - /// - EventAction ItemSelectionChanged { get; set; } - - bool MultiSelect { get; set; } - - void InitSort(); - IEnumerable ColumnConfigs(); - void SetColumnConfigs(IEnumerable columns); - void ForceDraw(); - void SelectItem(int index); - void DeselectAllItems(); - void SelectAllItems(); - int SetList(List list); - void ListViewColumnClick(); - void SortList(); - void ActionOnSelectedItems(Action> action); - void ActionOnSelectedItem(Action action); - void ActionOnActivateItem(Action action); - T GetItemAt(int index); - void SearchListContextMenuOpening(object sender, System.ComponentModel.CancelEventArgs e); -} - /// /// Consolidated code for ListView operation in VirtualMode. /// Only ListView events required are enabled. -/// Several property setters add Event handlers as required so don't call them more than once. +/// Several property setters add Event handlers as required, so don't call them more than once. /// -public class ListViewHelper : IListViewHelper where T : class +public sealed class ListViewHelper : IListViewHelper where T : class { private bool _isDisposed; private int _listSize; @@ -95,11 +38,17 @@ public class ListViewHelper : IListViewHelper where T : class public ListViewItem RenderItem { get; set; } public int AfterActivateIndex { get; set; } + public int ColumnClickIndex { get; set; } + public IEnumerable SelectedIndices { get; set; } + public int SelectedIndicesCount { get; set; } + public SortOrder ColumnSortOrder { get; set; } + public int SortColumn { get; set; } + public Comparison ColumnSortCompare { get; set; } public ListViewHelper(DoubleBufferListView listView) @@ -129,7 +78,7 @@ public EventAction RetrieveVirtualItem { // not adding retrieve virtual item events here as _list may not be set // was getting some odd errors earlier, this may address the null - // ListViewItem we got outside of visual studio in release builds. + // ListViewItem we got outside Visual Studio in release builds. _retrieveVirtualItem = value; if (_retrieveVirtualItem == null) return; // TODO AUDIT - this should probably add if not null, and remove if null? @@ -141,7 +90,7 @@ public EventAction RetrieveVirtualItem private EventAction _retrieveVirtualItem; /// - /// Adds ColumnClick handler which sets ColumnClickIndex before EventAction.. + /// Adds ColumnClick handler which sets ColumnClickIndex before EventAction. /// public EventAction ColumnClick { @@ -160,7 +109,7 @@ public EventAction ColumnClick private EventAction _columnClick; /// - /// Adds ItemActivate handler which sets AfterActivateIndex before EventAction.. + /// Adds ItemActivate handler which sets AfterActivateIndex before EventAction. /// public EventAction ItemActivate { @@ -253,6 +202,7 @@ private void MyRetrieveVirtualItem(object sender, RetrieveVirtualItemEventArgs e var evictIndex = _cacheOrder.Dequeue(); _itemCache.Remove(evictIndex); } + _itemCache[itemIndex] = newItem; _cacheOrder.Enqueue(itemIndex); @@ -286,7 +236,7 @@ private void MyVirtualItemsSelectionRangeChanged(object sender, private void ListViewItemSelectionChanged() { - SelectedIndicesCount = _listView.SelectedIndices.Count; // todo can i lose this ? + SelectedIndicesCount = _listView.SelectedIndices.Count; // todo can I lose this ? SelectedIndices = _listView.SelectedIndices.OfType(); if (SelectedIndicesCount > 0) { @@ -338,7 +288,7 @@ public void DeselectAllItems() } // Cannot use SelectItem() in a loop as it does Focus on each item. - public void SelectItems(IEnumerable itemIndices) + private void SelectItems(IEnumerable itemIndices) { var minIndex = int.MaxValue; foreach (var i in itemIndices) @@ -358,7 +308,7 @@ public void SelectItems(IEnumerable itemIndices) _listView.Select(); } - public void SelectItems(IEnumerable itemList) + private void SelectItems(IEnumerable itemList) { var newIndices = itemList.Select(item => _list.FindIndex(sortedItem => item == sortedItem)); SelectItems(newIndices); @@ -423,7 +373,7 @@ public void SortList() _list.Sort(ColumnSortCompare); SelectItems(selectedItems); - // Clear cache AFTER all item manipulations but BEFORE ForceDraw + // Clear cache AFTER all item manipulations, but BEFORE ForceDraw // This ensures no stale items from DeselectAllItems/SelectItems remain _itemCache.Clear(); _cacheOrder.Clear(); @@ -436,7 +386,7 @@ private void SetColumnSortArrow() _listView.SetSortIcon(SortColumn, ColumnSortOrder == SortOrder.Ascending ? SortOrder.Descending - : SortOrder.Ascending); // column state is inverted some how ? + : SortOrder.Ascending); // column state is inverted somehow? } public void ActionOnSelectedItems(Action> action) @@ -506,16 +456,16 @@ public T GetItemAt(int index) { return null; } + return _list[index]; } public void Dispose() { Dispose(true); - GC.SuppressFinalize(this); } - protected virtual void Dispose(bool disposing) + private void Dispose(bool disposing) { if (_isDisposed) return; @@ -525,7 +475,7 @@ protected virtual void Dispose(bool disposing) { _listView.CacheVirtualItems -= MyCacheVirtualItems; // - // If we don't do this we don't get the weird crash on exist of cdeWin + // If we don't do this we don't get the weird crash on exit of cdeWin // NullReferenceException // System.Windows.Forms.ListView.ListViewNativeItemCollection.get_Item(Int32 displayIndex) // at @@ -533,7 +483,7 @@ protected virtual void Dispose(bool disposing) // // On dispose it must be trying to be called after we remove it and it kaboom // This is local method to this Class it will never be anything else but this method - // Removing it at dispose when we exit seems like it's not actually important anyway + // Removing it at disposal when we exit seems like it's not actually important anyway // by not removing this we don't get the odd crash. // // _listView.RetrieveVirtualItem -= MyRetrieveVirtualItem; @@ -571,7 +521,7 @@ public void SearchListContextMenuOpening(object sender, System.ComponentModel.Ca var listViewItem = GetListViewItemAtMouse(); if (listViewItem == null) { - // cancel context menu if no list view item at right click. + // cancel context menu if no list view item at right-click. e.Cancel = true; } } diff --git a/src/cdeWin/LoadCatalogService.cs b/src/cdeWin/LoadCatalogService.cs index 2d3fcec..9d79df7 100644 --- a/src/cdeWin/LoadCatalogService.cs +++ b/src/cdeWin/LoadCatalogService.cs @@ -19,6 +19,12 @@ Task> LoadRootEntriesAsync( IConfig config, Action progressCallback, CancellationToken cancellationToken = default); + + /// + /// Discover columnar .cdex catalogs (current dir + config path, one level down). When any + /// exist the GUI memory-maps them instead of loading .cde trees. Empty/none ⇒ fall back. + /// + IList GetColumnarFiles(IConfig config); } public class LoadCatalogService : ILoadCatalogService @@ -30,6 +36,13 @@ public LoadCatalogService(ILogger logger) _logger = logger; } + public IList GetColumnarFiles(IConfig config) + { + var cachePathList = new[] { ".", config.ConfigPath }; + using var repo = new CatalogRepository(_logger); + return repo.GetColumnarFileList(cachePathList); + } + public List LoadRootEntries(IConfig config) { List rootEntries; @@ -91,7 +104,7 @@ public async Task> LoadRootEntriesAsync( var now = DateTime.UtcNow; if (currentCount % progressReportThreshold == 0 || - (now - lastProgressReport) > progressReportInterval) + now - lastProgressReport > progressReportInterval) { progressCallback?.Invoke(currentCount, totalFiles, $"Loading catalog {currentCount} of {totalFiles}..."); diff --git a/src/cdeWin/LoaderForm.cs b/src/cdeWin/LoaderForm.cs index c94429b..f22a164 100644 --- a/src/cdeWin/LoaderForm.cs +++ b/src/cdeWin/LoaderForm.cs @@ -138,7 +138,7 @@ private List LoadCatalogs(BackgroundWorker worker) // Time-based or count-based progress reporting (whichever comes first) var now = DateTime.UtcNow; if (currentCount % progressReportThreshold == 0 || - (now - lastProgressReport) > progressReportInterval) + now - lastProgressReport > progressReportInterval) { worker.ReportProgress((int)(currentCount / (float)totalFiles * 100), new LoadingState(currentCount, totalFiles)); diff --git a/src/cdeWin/Program.cs b/src/cdeWin/Program.cs index 2d0c2b2..ba092d7 100644 --- a/src/cdeWin/Program.cs +++ b/src/cdeWin/Program.cs @@ -12,9 +12,9 @@ internal static class Program { public static IConfigurationRoot Configuration; - public static string Version => Application.ProductVersion; + private static string Version => Application.ProductVersion; - public static string ProductName => Application.ProductName; + private static string ProductName => Application.ProductName; [STAThread] private static void Main() diff --git a/src/cdeWin/SplitContainerExtensions.cs b/src/cdeWin/SplitContainerExtensions.cs index feb7005..7a30385 100644 --- a/src/cdeWin/SplitContainerExtensions.cs +++ b/src/cdeWin/SplitContainerExtensions.cs @@ -20,7 +20,7 @@ public static void SetSplitterRatio(this SplitContainer splitter, float splitter } } - public static int GetSplitterSize(this SplitContainer splitter) + private static int GetSplitterSize(this SplitContainer splitter) { return splitter.Orientation == Orientation.Vertical ? splitter.Width diff --git a/src/cdeWin/StrictKeyEqualityComparer.cs b/src/cdeWin/StrictKeyEqualityComparer.cs new file mode 100644 index 0000000..4bb1110 --- /dev/null +++ b/src/cdeWin/StrictKeyEqualityComparer.cs @@ -0,0 +1,20 @@ +using System; + +namespace cdeWin; + +/// +/// see http://stackoverflow.com/questions/98033/wrap-a-delegate-in-an-iequalitycomparer +/// +public class StrictKeyEqualityComparer + : KeyEqualityComparer where TKey : IEquatable +{ + public StrictKeyEqualityComparer(Func keyExtractor) : base(keyExtractor) + { } + + public override bool Equals(T x, T y) + { + // This will use the overload that accepts a TKey parameter + // instead of an object parameter. + return KeyExtractor(x).Equals(KeyExtractor(y)); + } +} \ No newline at end of file diff --git a/src/cdeWin/StringExtension.cs b/src/cdeWin/StringExtension.cs index 89841aa..70a18df 100644 --- a/src/cdeWin/StringExtension.cs +++ b/src/cdeWin/StringExtension.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Threading; namespace cdeWin; @@ -9,7 +10,7 @@ public static class StringExtension // Cache for formatted size strings (key: size, value: formatted string) private static readonly Dictionary SizeCache = new(1024); - private static readonly object SizeCacheLock = new(); + private static readonly Lock SizeCacheLock = new(); public static string ToHRString(this long val) { diff --git a/src/cdeWin/UpDownHelper.cs b/src/cdeWin/UpDownHelper.cs index 92a53b5..4a00c0e 100644 --- a/src/cdeWin/UpDownHelper.cs +++ b/src/cdeWin/UpDownHelper.cs @@ -7,7 +7,7 @@ public class UpDownHelper private readonly NumericUpDown _upDown; // have a field - text box / up-down - // have a drop down which modifies the the field... like a multiplier or offset. + // have a drop-down which modifies the field... like a multiplier or offset. public UpDownHelper(NumericUpDown upDown, int decimalPlaces = 2) { _upDown = upDown; diff --git a/src/cdeWin/cdeWin.csproj b/src/cdeWin/cdeWin.csproj index ac09834..c1de4f0 100644 --- a/src/cdeWin/cdeWin.csproj +++ b/src/cdeWin/cdeWin.csproj @@ -24,10 +24,10 @@ - + - + diff --git a/src/cdeWinTest/CDEWinFormPresenterTest.cs b/src/cdeWinTest/CDEWinFormPresenterTest.cs index cc2ef09..c255f8f 100644 --- a/src/cdeWinTest/CDEWinFormPresenterTest.cs +++ b/src/cdeWinTest/CDEWinFormPresenterTest.cs @@ -31,8 +31,7 @@ public override void RunBeforeEveryTest() [Test] public void Always_Set_Search_Button() { - var _ = new CDEWinFormPresenter(_mockForm, _stubConfig); - + _ = new CDEWinFormPresenter(_mockForm, _stubConfig); _mockForm.Received().SearchButtonText = "Search"; } @@ -50,23 +49,23 @@ public void Always_Catalog_SortList() presenter.InitializeAsync().GetAwaiter().GetResult(); // SetList is called with empty list after InitializeAsync - _mockCatalogListViewHelper.Received().SetList(Arg.Any>()); + _mockCatalogListViewHelper.Received().SetList(Arg.Any>()); } [Test] public void Always_Register_Result_Sorters() { - var _ = new CDEWinFormPresenter(_mockForm, _stubConfig); + _ = new CDEWinFormPresenter(_mockForm, _stubConfig); _mockSearchResultListViewHelper.ColumnSortCompare = Arg.Any>(); - _mockCatalogListViewHelper.ColumnSortCompare = Arg.Any>(); + _mockCatalogListViewHelper.ColumnSortCompare = Arg.Any>(); _mockDirectoryListViewHelper.ColumnSortCompare = Arg.Any>(); } [Test] public void With_Null_RootEntry_List_SetsCatalogsLoaded() { - _mockCatalogListViewHelper.SetList(Arg.Any>()).Returns(3); + _mockCatalogListViewHelper.SetList(Arg.Any>()).Returns(3); var loadCatalogService = Substitute.For(); loadCatalogService.LoadRootEntriesAsync( @@ -113,6 +112,33 @@ public void With_RootEntry_List_SetsTotalFileEntries() _mockForm.Received().SetTotalFileEntriesLoadedStatus(1); } + [Test] + public void With_Columnar_Cdex_Present_LoadsViaMmap_AndSkipsCdeLoad() + { + // Migrate the test root to a real .cdex, then prove the presenter memory-maps it (and does + // NOT fall back to loading .cde trees) when GetColumnarFiles reports one. + var store = cdeLib.Entities.Soa.EntryStore.Build(_rootEntry); + var cdex = System.IO.Path.Combine(System.IO.Path.GetTempPath(), $"cdewintest-{Guid.NewGuid():N}.cdex"); + cdeLib.Entities.Columnar.ColumnarFormat.Write(store, cdex); + try + { + var loadCatalogsService = Substitute.For(); + loadCatalogsService.GetColumnarFiles(Arg.Any()).Returns(new List { cdex }); + + var presenter = new CDEWinFormPresenter(_mockForm, _stubConfig, loadCatalogsService); + presenter.InitializeAsync().GetAwaiter().GetResult(); + + _mockForm.Received().SetTotalFileEntriesLoadedStatus(1); + loadCatalogsService.DidNotReceive().LoadRootEntriesAsync( + Arg.Any(), Arg.Any>(), Arg.Any()); + + // The catalog is held memory-mapped: the file is locked until the presenter closes, + // which disposes the mmap source. This both proves mmap and exercises the cleanup path. + presenter.MyFormClosing(); + } + finally { System.IO.File.Delete(cdex); } + } + [Ignore("This cant really happen in a real TreeView, as the event to be triggered means there is a node")] [Test] public void With_TreeViewRoot_Null_Throws_Exception() @@ -263,9 +289,9 @@ public override void RunBeforeEveryTest() [Test] public void Catalog_Activate_On_Same_RootEntry_Does_Not_Set_Root() { - var testRootTreeNode = new TreeNode("Moo") { Tag = _rootEntry }; + var testRootTreeNode = new TreeNode("Moo") { Tag = _catalogRoot }; _mockForm.DirectoryTreeViewNodes.Returns(testRootTreeNode); - FakeItemActivateWithValue(_mockCatalogListViewHelper, _rootEntry); + FakeItemActivateWithValue(_mockCatalogListViewHelper, _catalogRoot); // ACT _sutPresenter.CatalogListViewItemActivate(); @@ -277,25 +303,25 @@ public void Catalog_Activate_On_Same_RootEntry_Does_Not_Set_Root() public void Catalog_Activate_On_Different_RootEntry_Sets_New_Root() { var alternateRootEntry = new RootEntry(_config) { Path = "alternate" }; - var testRootTreeNode = new TreeNode("Moo") { Tag = alternateRootEntry }; + var testRootTreeNode = new TreeNode("Moo") { Tag = CatalogRootOf(alternateRootEntry) }; _mockForm.DirectoryTreeViewNodes.Returns(testRootTreeNode); TreeNode treeNodeSet = null; _mockForm.DirectoryTreeViewNodes = Arg.Do(node => treeNodeSet = node); - FakeItemActivateWithValue(_mockCatalogListViewHelper, _rootEntry); + FakeItemActivateWithValue(_mockCatalogListViewHelper, _catalogRoot); // ACT _sutPresenter.CatalogListViewItemActivate(); // _mockForm.Received(1).DirectoryTreeViewNodes = Arg.Any(); - Assert.That(treeNodeSet.Tag, Is.EqualTo(_rootEntry)); + Assert.That(treeNodeSet.Tag, Is.EqualTo(_catalogRoot)); } [Test] public void Catalog_Activate_GoToDirectoryRoot_On_Null_RootNode_Sets_New_Root() { _mockForm.DirectoryTreeViewNodes.Returns((TreeNode)null); - FakeItemActivateWithValue(_mockCatalogListViewHelper, _rootEntry); + FakeItemActivateWithValue(_mockCatalogListViewHelper, _catalogRoot); // ACT _sutPresenter.CatalogListViewItemActivate(); @@ -306,7 +332,7 @@ public void Catalog_Activate_GoToDirectoryRoot_On_Null_RootNode_Sets_New_Root() [Test] public void Callback_GoToDirectoryRoot_Sets_Directory_Pane() { - FakeItemActivateWithValue(_mockCatalogListViewHelper, _rootEntry); + FakeItemActivateWithValue(_mockCatalogListViewHelper, _catalogRoot); _sutPresenter.CatalogListViewItemActivate(); @@ -316,7 +342,7 @@ public void Callback_GoToDirectoryRoot_Sets_Directory_Pane() [Test] public void Callback_GoToDirectoryRoot_Setting_RootNode_Calls_InitSort() { - FakeItemActivateWithValue(_mockCatalogListViewHelper, _rootEntry); + FakeItemActivateWithValue(_mockCatalogListViewHelper, _catalogRoot); _sutPresenter.CatalogListViewItemActivate(); @@ -360,7 +386,7 @@ public void Produces_ListView_Field() { _stubConfig.DateFormatYMDHMS.Returns("{0:yyyy/MM}"); // make local time zone irrelevant for test. _mockCatalogListViewHelper.RetrieveItemIndex.Returns(0); - _mockCatalogListViewHelper.GetItemAt(0).Returns(_rootEntry); + _mockCatalogListViewHelper.GetItemAt(0).Returns(_catalogRoot); ListViewItem setRenderItem = null; _mockCatalogListViewHelper.RenderItem = Arg.Do(lvi => setRenderItem = lvi); diff --git a/src/cdeWinTest/TestCDEWinPresenterBase.cs b/src/cdeWinTest/TestCDEWinPresenterBase.cs index 39eb17f..ec809d0 100644 --- a/src/cdeWinTest/TestCDEWinPresenterBase.cs +++ b/src/cdeWinTest/TestCDEWinPresenterBase.cs @@ -18,22 +18,25 @@ public class TestCDEWinPresenterBase protected IConfig _stubConfig; protected IListViewHelper _mockSearchResultListViewHelper; protected IListViewHelper _mockDirectoryListViewHelper; - protected IListViewHelper _mockCatalogListViewHelper; + protected IListViewHelper _mockCatalogListViewHelper; + + // Catalog roots are presented to the GUI as EntryRef over an EntryStore (SoA model). + protected cdeLib.Entities.Soa.EntryRef _catalogRoot; + protected cdeLib.Entities.Soa.EntryRef CatalogRootOf(RootEntry re) + => new(cdeLib.Entities.Soa.EntryStore.Build(re), 0); protected RootEntry _rootEntry; - protected DirEntry _dirEntry; + private DirEntry _dirEntry; protected PairDirEntry _pairDirEntry; - protected List _emptyRootList = new(); - protected List _rootList = new(); + protected List _rootList = []; // protected TreeNode _treeViewAfterSelectNode; private readonly IConfiguration _config = Substitute.For(); [SetUp] public virtual void RunBeforeEveryTest() { - _emptyRootList = new List(); - _rootList = new List(); + _rootList = []; _config.ProgressUpdateInterval.Returns(5000); _mockForm = Substitute.For(); @@ -42,7 +45,7 @@ public virtual void RunBeforeEveryTest() _mockForm.SearchResultListViewHelper.Returns(_mockSearchResultListViewHelper); _mockDirectoryListViewHelper = Substitute.For>(); _mockForm.DirectoryListViewHelper.Returns(_mockDirectoryListViewHelper); - _mockCatalogListViewHelper = Substitute.For>(); + _mockCatalogListViewHelper = Substitute.For>(); _mockForm.CatalogListViewHelper.Returns(_mockCatalogListViewHelper); } @@ -74,19 +77,21 @@ protected void InitRootWithFile() _rootEntry.AddChild(_dirEntry); _rootEntry.SetInMemoryFields(); _pairDirEntry = new PairDirEntry(_rootEntry, _dirEntry); + _catalogRoot = CatalogRootOf(_rootEntry); _rootList.Add(_rootEntry); } protected void InitRootWithDir() { - // massive assumption on path, this T:\ is windows only...... + // massive assumption on the path, this T:\ is windows only...... // is it a valid test on other platforms or behavior on other platforms? _rootEntry = new RootEntry(_config) { Path = @"T:\" }; _dirEntry = new DirEntry(true) { Path = "Test1" }; _rootEntry.AddChild(_dirEntry); _rootEntry.SetInMemoryFields(); _pairDirEntry = new PairDirEntry(_rootEntry, _dirEntry); + _catalogRoot = CatalogRootOf(_rootEntry); _rootList.Add(_rootEntry); } diff --git a/src/cdeWinTest/TestCDEWinPresenter_OptimiseRegexPattern.cs b/src/cdeWinTest/TestCDEWinPresenter_OptimiseRegexPattern.cs index a9301a2..17c88a8 100644 --- a/src/cdeWinTest/TestCDEWinPresenter_OptimiseRegexPattern.cs +++ b/src/cdeWinTest/TestCDEWinPresenter_OptimiseRegexPattern.cs @@ -8,7 +8,7 @@ namespace cdeWinTest; [TestFixture] public class TestCDEWinPresenter_OptimiseRegexPattern_NotRegex : TestCDEWinPresenterBase { - protected TestOptimise _presenter; + private TestOptimise _presenter; [SetUp] public override void RunBeforeEveryTest() @@ -55,10 +55,10 @@ public void OptimiseRegexPattern_TrailingWild_ReturnsUnchanged_WhenNotRegexMode( [TestFixture] public class TestCDEWinPresenter_OptimiseRegexPattern_Regex : TestCDEWinPresenterBase { - protected TestOptimise _presenter; + private TestOptimise _presenter; [SetUp] - override public void RunBeforeEveryTest() + public override void RunBeforeEveryTest() { base.RunBeforeEveryTest(); _mockForm.RegexMode = true; diff --git a/src/cdeWinTest/cdeWinTest.csproj b/src/cdeWinTest/cdeWinTest.csproj index cd030f7..b67c271 100644 --- a/src/cdeWinTest/cdeWinTest.csproj +++ b/src/cdeWinTest/cdeWinTest.csproj @@ -8,10 +8,10 @@ - + - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive