From 2bf6f8b3fbff0998e7a45f41c13bb8c477a9c84c Mon Sep 17 00:00:00 2001 From: Lillian Gensolin Date: Mon, 8 Jan 2024 15:09:12 -0500 Subject: [PATCH 1/4] added new settings to make rollover methods configurable --- .../Services/Writer/ArchiveInitializer.cs | 58 +++++++++++++++++-- .../Writer/ArchiveInitializerSettings.cs | 40 +++++++++++++ 2 files changed, 94 insertions(+), 4 deletions(-) diff --git a/src/SnapDB/Snap/Services/Writer/ArchiveInitializer.cs b/src/SnapDB/Snap/Services/Writer/ArchiveInitializer.cs index 8d33fe6d..70c01ab5 100644 --- a/src/SnapDB/Snap/Services/Writer/ArchiveInitializer.cs +++ b/src/SnapDB/Snap/Services/Writer/ArchiveInitializer.cs @@ -211,14 +211,64 @@ private string GetPathWithEnoughSpace(long estimatedSize) long remainingSpace = Settings.DesiredRemainingSpace; - foreach (string path in Settings.WritePath) + if (Settings.BalancingMethod == BalancingMethod.FillToDesired) { - FilePath.GetAvailableFreeSpace(path, out long freeSpace, out _); + foreach (string path in Settings.WritePath) + { + FilePath.GetAvailableFreeSpace(path, out long freeSpace, out _); + + if (freeSpace - estimatedSize > remainingSpace) + return path; + + } + } + + if (Settings.BalancingMethod == BalancingMethod.FillSmallestAvailable) + { + long current = 0; + string smallest = null; + + foreach (string path in Settings.WritePath) + { + FilePath.GetAvailableFreeSpace(path, out long freeSpace, out _); + + if (freeSpace - estimatedSize < remainingSpace) + continue; + if (freeSpace < current) + { + smallest = path; + current = freeSpace; + } + } + + if (!string.IsNullOrEmpty(smallest)) + return smallest; + } + + if (Settings.BalancingMethod == BalancingMethod.FillLargestAvailable) + { + long current = 0; + string smallest = null; - if (freeSpace - estimatedSize > remainingSpace) - return path; + foreach (string path in Settings.WritePath) + { + FilePath.GetAvailableFreeSpace(path, out long freeSpace, out _); + + if (freeSpace - estimatedSize < remainingSpace) + continue; + if (freeSpace < current) + { + smallest = path; + current = freeSpace; + } + } + + if (!string.IsNullOrEmpty(smallest)) + return smallest; } + //for pct, calculate percentage as freespace/totalspace then pick the largest to fill to a point + throw new InvalidOperationException("Out of free space"); } diff --git a/src/SnapDB/Snap/Services/Writer/ArchiveInitializerSettings.cs b/src/SnapDB/Snap/Services/Writer/ArchiveInitializerSettings.cs index ccb3b826..e9ce71a6 100644 --- a/src/SnapDB/Snap/Services/Writer/ArchiveInitializerSettings.cs +++ b/src/SnapDB/Snap/Services/Writer/ArchiveInitializerSettings.cs @@ -31,6 +31,30 @@ namespace SnapDB.Snap.Services.Writer; +public enum BalancingMethod +{ + /// + /// Fills in order + /// + FillToDesired, + /// + /// Fills the one with the smallest available space + /// + FillSmallestAvailable, + /// + /// Fills the one with the most available space + /// + FillLargestAvailable, + /// + /// Fills the one with the largest total space + /// + FillLargestTotal, + /// + /// Fills to the same percentage across the board + /// + FillToMatchingPercentage +} + /// /// Settings for . /// @@ -39,6 +63,7 @@ public class ArchiveInitializerSettings : SettingsBase + /// The method used to balance file share load. + /// + public BalancingMethod BalancingMethod + { + get => m_balancingMethod; + set + { + TestForEditable(); + + m_balancingMethod = value; + } + } + /// /// Gets the method that the directory structure will follow when writing a new file. /// @@ -316,6 +355,7 @@ private void Initialize() m_fileExtension = ".d2i"; m_desiredRemainingSpace = 5 * 1024 * 1024 * 1024L; //5GB m_encodingMethod = EncodingDefinition.FixedSizeCombinedEncoding; + m_balancingMethod = BalancingMethod.FillToDesired; WritePath = new ImmutableList(x => { PathHelpers.ValidatePathName(x); From eaf908c768767fc06e7ed211eb0b178a469f2f0a Mon Sep 17 00:00:00 2001 From: Lillian Gensolin Date: Tue, 9 Jan 2024 17:16:49 -0500 Subject: [PATCH 2/4] completed all balancing methods v1 --- .../Services/Writer/ArchiveInitializer.cs | 65 +++++++++++++++++-- 1 file changed, 60 insertions(+), 5 deletions(-) diff --git a/src/SnapDB/Snap/Services/Writer/ArchiveInitializer.cs b/src/SnapDB/Snap/Services/Writer/ArchiveInitializer.cs index 70c01ab5..f8aedfb3 100644 --- a/src/SnapDB/Snap/Services/Writer/ArchiveInitializer.cs +++ b/src/SnapDB/Snap/Services/Writer/ArchiveInitializer.cs @@ -234,6 +234,7 @@ private string GetPathWithEnoughSpace(long estimatedSize) if (freeSpace - estimatedSize < remainingSpace) continue; + if (freeSpace < current) { smallest = path; @@ -248,7 +249,7 @@ private string GetPathWithEnoughSpace(long estimatedSize) if (Settings.BalancingMethod == BalancingMethod.FillLargestAvailable) { long current = 0; - string smallest = null; + string largest = null; foreach (string path in Settings.WritePath) { @@ -256,15 +257,69 @@ private string GetPathWithEnoughSpace(long estimatedSize) if (freeSpace - estimatedSize < remainingSpace) continue; - if (freeSpace < current) + + if (freeSpace > current) { - smallest = path; + largest = path; current = freeSpace; } } - if (!string.IsNullOrEmpty(smallest)) - return smallest; + if(!string.IsNullOrEmpty(largest)) + return largest; + + } + + if (Settings.BalancingMethod == BalancingMethod.FillLargestTotal) + { + long current = 0; + string largestTotal = null; + + foreach (string path in Settings.WritePath) + { + FilePath.GetAvailableFreeSpace(path, out long freeSpace, out long totalSize); + + if (freeSpace - estimatedSize < totalSize) + continue; + if (current < totalSize) + { + largestTotal = path; + current = totalSize; + } + } + + if (!string.IsNullOrEmpty(largestTotal)) + return largestTotal; + } + + if (Settings.BalancingMethod == BalancingMethod.FillToMatchingPercentage) + { + // how full the current path is (percentage format) + long currentPercentage; + // how full the fullest one is, which every other path will be compared to + long highestPercentage = 0; + // the location with the highest percentage + string fillStandard = null; + + foreach (string path in Settings.WritePath) + { + FilePath.GetAvailableFreeSpace(path, out long freeSpace, out long totalSize); + + currentPercentage = freeSpace/totalSize; + + if (freeSpace - estimatedSize < remainingSpace) + continue; + + if (currentPercentage > highestPercentage) + { + fillStandard = path; + highestPercentage = currentPercentage; + } + + } + + if (!string.IsNullOrEmpty(fillStandard)) + return fillStandard; } //for pct, calculate percentage as freespace/totalspace then pick the largest to fill to a point From 2b73846032265fbefb43f9ad1123f732898aa80e Mon Sep 17 00:00:00 2001 From: Lillian Gensolin Date: Wed, 10 Jan 2024 11:35:35 -0500 Subject: [PATCH 3/4] improved filltomatchingpercentage logic --- .../Services/Writer/ArchiveInitializer.cs | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/SnapDB/Snap/Services/Writer/ArchiveInitializer.cs b/src/SnapDB/Snap/Services/Writer/ArchiveInitializer.cs index f8aedfb3..c62765ec 100644 --- a/src/SnapDB/Snap/Services/Writer/ArchiveInitializer.cs +++ b/src/SnapDB/Snap/Services/Writer/ArchiveInitializer.cs @@ -294,32 +294,34 @@ private string GetPathWithEnoughSpace(long estimatedSize) if (Settings.BalancingMethod == BalancingMethod.FillToMatchingPercentage) { - // how full the current path is (percentage format) - long currentPercentage; - // how full the fullest one is, which every other path will be compared to - long highestPercentage = 0; - // the location with the highest percentage - string fillStandard = null; + // what percentage of the current path is left free + long currentRemainingPercentage; + // the occupancy percentage of the most empty space + long emptiestPercentage = 0; + // the location that needs to be filled, i.e. the location with the emptiestPercentage + string toFill = null; foreach (string path in Settings.WritePath) { FilePath.GetAvailableFreeSpace(path, out long freeSpace, out long totalSize); - currentPercentage = freeSpace/totalSize; - if (freeSpace - estimatedSize < remainingSpace) continue; - if (currentPercentage > highestPercentage) - { - fillStandard = path; - highestPercentage = currentPercentage; - } + currentRemainingPercentage = freeSpace / totalSize; + + if (currentRemainingPercentage == emptiestPercentage) + continue; + + if (currentRemainingPercentage > emptiestPercentage) + emptiestPercentage = currentRemainingPercentage; + if (currentRemainingPercentage < emptiestPercentage) + toFill = path; } - if (!string.IsNullOrEmpty(fillStandard)) - return fillStandard; + if (!string.IsNullOrEmpty(toFill)) + return toFill; } //for pct, calculate percentage as freespace/totalspace then pick the largest to fill to a point From c6b9b1d2efd01e425f475bb412c2d96c644565af Mon Sep 17 00:00:00 2001 From: Lillian Gensolin Date: Wed, 24 Jan 2024 18:17:17 -0500 Subject: [PATCH 4/4] improved rollover logic --- .../Services/Writer/ArchiveInitializer.cs | 29 ++++++++++++++----- .../Writer/ArchiveInitializerSettings.cs | 10 +++---- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/src/SnapDB/Snap/Services/Writer/ArchiveInitializer.cs b/src/SnapDB/Snap/Services/Writer/ArchiveInitializer.cs index c62765ec..e7b8ebc3 100644 --- a/src/SnapDB/Snap/Services/Writer/ArchiveInitializer.cs +++ b/src/SnapDB/Snap/Services/Writer/ArchiveInitializer.cs @@ -217,6 +217,7 @@ private string GetPathWithEnoughSpace(long estimatedSize) { FilePath.GetAvailableFreeSpace(path, out long freeSpace, out _); + //If there is space, fill it. if (freeSpace - estimatedSize > remainingSpace) return path; @@ -232,9 +233,11 @@ private string GetPathWithEnoughSpace(long estimatedSize) { FilePath.GetAvailableFreeSpace(path, out long freeSpace, out _); + // Checks to ensure that this path has enough available space to write to if (freeSpace - estimatedSize < remainingSpace) continue; + // If the path we are checking has less free space than the previous path, that becomes the path with the least available space. if (freeSpace < current) { smallest = path; @@ -258,6 +261,7 @@ private string GetPathWithEnoughSpace(long estimatedSize) if (freeSpace - estimatedSize < remainingSpace) continue; + // If the path we are checking has more free space than the previous path, that becomes the path with the most available space. if (freeSpace > current) { largest = path; @@ -273,19 +277,23 @@ private string GetPathWithEnoughSpace(long estimatedSize) if (Settings.BalancingMethod == BalancingMethod.FillLargestTotal) { long current = 0; + string largestTotal = null; foreach (string path in Settings.WritePath) { FilePath.GetAvailableFreeSpace(path, out long freeSpace, out long totalSize); - if (freeSpace - estimatedSize < totalSize) + if (freeSpace - estimatedSize < remainingSpace) continue; - if (current < totalSize) + + // If the path we are looking at has a greater total size than the previous one, it becomes the largestTotal. + if (totalSize > current) { largestTotal = path; - current = totalSize; + current = freeSpace; } + } if (!string.IsNullOrEmpty(largestTotal)) @@ -297,7 +305,7 @@ private string GetPathWithEnoughSpace(long estimatedSize) // what percentage of the current path is left free long currentRemainingPercentage; // the occupancy percentage of the most empty space - long emptiestPercentage = 0; + long fullestPercentage = 0; // the location that needs to be filled, i.e. the location with the emptiestPercentage string toFill = null; @@ -310,13 +318,18 @@ private string GetPathWithEnoughSpace(long estimatedSize) currentRemainingPercentage = freeSpace / totalSize; - if (currentRemainingPercentage == emptiestPercentage) + // Compares the percentage that is free to the fullestPercentage + if (currentRemainingPercentage == fullestPercentage) continue; - if (currentRemainingPercentage > emptiestPercentage) - emptiestPercentage = currentRemainingPercentage; + // If the remaining percentage of available space for this path is greater than the remaining percentage of the + // previous one, this occupancy percentage becomes the standard for the rest to be compared to. + if (currentRemainingPercentage > fullestPercentage) + fullestPercentage = currentRemainingPercentage; - if (currentRemainingPercentage < emptiestPercentage) + // If the remaining percentage of available space for this path is smaller than the remaining percentage of the + // fullest path, this path must be filled to that standard. + if (currentRemainingPercentage < fullestPercentage) toFill = path; } diff --git a/src/SnapDB/Snap/Services/Writer/ArchiveInitializerSettings.cs b/src/SnapDB/Snap/Services/Writer/ArchiveInitializerSettings.cs index e9ce71a6..7aac922e 100644 --- a/src/SnapDB/Snap/Services/Writer/ArchiveInitializerSettings.cs +++ b/src/SnapDB/Snap/Services/Writer/ArchiveInitializerSettings.cs @@ -34,23 +34,23 @@ namespace SnapDB.Snap.Services.Writer; public enum BalancingMethod { /// - /// Fills in order + /// Fills the path to meet the desired free space, prioritizing paths with the most available space /// FillToDesired, /// - /// Fills the one with the smallest available space + /// Fills the path with the least available space /// FillSmallestAvailable, /// - /// Fills the one with the most available space + /// Fills the path with the most available space /// FillLargestAvailable, /// - /// Fills the one with the largest total space + /// Fills the path with the most space, available or not, as long as there is room /// FillLargestTotal, /// - /// Fills to the same percentage across the board + /// Fills each path to the same amount across the board /// FillToMatchingPercentage }