diff --git a/.github/workflows/publish-nuget.yml b/.github/workflows/publish-nuget.yml new file mode 100644 index 0000000..85ecbea --- /dev/null +++ b/.github/workflows/publish-nuget.yml @@ -0,0 +1,37 @@ +name: Publish NuGet package + +on: + push: + branches: + - master + paths: + - TDigest/** + - README.md + - .github/workflows/publish-nuget.yml + +permissions: + contents: read + +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: 10.0.x + + - name: Test + run: dotnet test TDigest.Tests/TDigest.Tests.csproj --configuration Release --framework net10.0 + + - name: Pack + run: dotnet pack TDigest/TDigest.csproj --configuration Release --output artifacts + + - name: Publish + run: | + for package in artifacts/*.nupkg; do + dotnet nuget push "$package" --api-key "$NUGET_API_KEY" --source https://api.nuget.org/v3/index.json --skip-duplicate + done + env: + NUGET_API_KEY: ${{ secrets.NUGET_API_KEY }} diff --git a/README.md b/README.md index 54e2aa3..f734a21 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ The Nuget package for this Implementation can be found here +The library targets .NET Framework 4.5 and 4.7.2, .NET Standard 2.0, .NET 8, and .NET 10. + The T-Digest white paper can be found here Example Code: diff --git a/TDigest.CMD/TDigest.CMD.csproj b/TDigest.CMD/TDigest.CMD.csproj index b7f2efd..4599005 100644 --- a/TDigest.CMD/TDigest.CMD.csproj +++ b/TDigest.CMD/TDigest.CMD.csproj @@ -9,7 +9,7 @@ Properties TDigest.CMD TDigest.CMD - v4.5 + v4.7.2 512 diff --git a/TDigest.Tests/TDigest.Tests.csproj b/TDigest.Tests/TDigest.Tests.csproj index 41cfda4..60672f5 100644 --- a/TDigest.Tests/TDigest.Tests.csproj +++ b/TDigest.Tests/TDigest.Tests.csproj @@ -1,89 +1,18 @@ - - + - Debug - AnyCPU - {7BCAFD59-C058-4E80-BE5A-B842522C951E} - Library - Properties - TDigest.Tests - TDigest.Tests - v4.5 - 512 - {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} - 10.0 - $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) - $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages - False - UnitTest + net472;net10.0 + true + false - - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - - - - - - - - - - - - - + - - + + + + + - - {909364f5-4f9a-444d-be6a-ce2a6ef592b0} - TDigest - + - - - - - False - - - False - - - False - - - False - - - - - - - - \ No newline at end of file + diff --git a/TDigest.Tests/Tests.cs b/TDigest.Tests/Tests.cs index e4708d4..e07db29 100644 --- a/TDigest.Tests/Tests.cs +++ b/TDigest.Tests/Tests.cs @@ -24,6 +24,20 @@ public void TestFixForNegativeQuantileBug() } } + [TestMethod] + public void AddingValuesCanReorderCentroidsWithoutInvalidatingEnumeration() + { + var digest = new TDigest(.001); + + for (var i = 0; i < 100000; i++) + { + digest.Add((i * 7919) % 10000); + } + + Assert.AreEqual(100000, digest.Count); + Assert.IsTrue(digest.Quantile(.5) >= 0); + } + [TestMethod] public void TestUniformDistribution() { Random r = new Random(); @@ -207,6 +221,17 @@ public void TestSerialization() { Assert.IsTrue(areEqual, "Serialized TDigest is not the same as original"); } + [TestMethod] + public void DeserializationRejectsDuplicateCentroidMeans() { + byte[] serialized = new double[] { + 0, 0, .02, 25, 1, 1, + 1, 2, + 1, 3 + }.SelectMany(BitConverter.GetBytes).ToArray(); + + Assert.ThrowsException(() => new TDigest(serialized)); + } + private double GetAvgPercentileError(IList all, TDigest digest) { return Enumerable.Range(1, 999) .Select(n => n / 1000.0) diff --git a/TDigest/TDigest.cs b/TDigest/TDigest.cs index 9f7c0ed..86b4b36 100644 --- a/TDigest/TDigest.cs +++ b/TDigest/TDigest.cs @@ -1,5 +1,4 @@ -using C5; -using System; +using System; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -10,7 +9,7 @@ namespace StatsLib { public class TDigest { - private C5.TreeDictionary _centroids; + private SortedSet _centroids; private static Random _rand; private double _count; @@ -72,7 +71,7 @@ public double Average { /// A T-Digest created by merging the specified T-Digests public static TDigest Merge(TDigest a, TDigest b) { TDigest merged = new TDigest(); - Centroid[] combined = a._centroids.Values.Concat(b._centroids.Values).ToArray(); + Centroid[] combined = a._centroids.Concat(b._centroids).ToArray(); combined.Shuffle(); foreach (var c in combined) { merged.Add(c.Mean, c.Count); @@ -94,7 +93,7 @@ public TDigest(double accuracy = 0.02, double compression = 25) { if (accuracy <= 0) throw new ArgumentOutOfRangeException("Accuracy must be greater than 0"); if (compression < 15) throw new ArgumentOutOfRangeException("Compression constant must be 15 or greater"); - _centroids = new TreeDictionary(); + _centroids = new SortedSet(CentroidMeanComparer.Instance); _rand = new Random(); _count = 0; Accuracy = accuracy; @@ -126,13 +125,14 @@ public TDigest(byte[] serialized) Mean = BitConverter.ToDouble(serialized, i * 16 + 48), Count = BitConverter.ToDouble(serialized, i * 16 + 8 + 48) }) - .Select(d => new Centroid(d.Mean, d.Count)); - - var kvPairs = centroids - .Select(c => new C5.KeyValuePair(c.Mean, c)) - .ToArray(); + .Select(d => new Centroid(d.Mean, d.Count)) + .ToList(); - _centroids.AddAll(kvPairs); + foreach (var centroid in centroids) { + if (!_centroids.Add(centroid)) { + throw new ArgumentException("Serialized data is invalid or corrupted"); + } + } _count = centroids.Sum(c => c.Count); } @@ -161,7 +161,7 @@ public void Add(double value, double weight = 1) { } if (_centroids.Count == 0) { - _centroids.Add(value, new Centroid(value, weight)); + _centroids.Add(new Centroid(value, weight)); return; } @@ -179,10 +179,7 @@ public void Add(double value, double weight = 1) { var cData = candidates[_rand.Next() % candidates.Count]; var delta_w = Math.Min(cData.Threshold - cData.Centroid.Count, weight); - double oldMean; - if (cData.Centroid.Update(delta_w, value, out oldMean)) { - ReInsertCentroid(oldMean, cData.Centroid); - } + UpdateCentroid(cData.Centroid, delta_w, value); weight -= delta_w; candidates.Remove(cData); @@ -191,12 +188,12 @@ public void Add(double value, double weight = 1) { if (weight > 0) { var toAdd = new Centroid(value, weight); - if (_centroids.FindOrAdd(value, ref toAdd)) { - double oldMean; - - if (toAdd.Update(weight, toAdd.Mean, out oldMean)) { - ReInsertCentroid(oldMean, toAdd); - } + Centroid existing; + if (TryGetCentroid(value, out existing)) { + UpdateCentroid(existing, weight, value); + } + else { + _centroids.Add(toAdd); } } @@ -221,7 +218,7 @@ public double Quantile(double quantile) { if (_centroids.Count == 1) { - return _centroids.First().Value.Mean; + return _centroids.First().Mean; } double index = quantile * _count; @@ -234,8 +231,8 @@ public double Quantile(double quantile) { return Max; } - Centroid currentNode = _centroids.First().Value; - Centroid lastNode = _centroids.Last().Value; + Centroid currentNode = _centroids.First(); + Centroid lastNode = _centroids.Last(); double currentWeight = currentNode.Count; if (currentWeight == 2 && index <= 2) { @@ -243,7 +240,7 @@ public double Quantile(double quantile) { return 2 * currentNode.Mean - Min; } - if (_centroids.Last().Value.Count == 2 && index > Count - 2) + if (_centroids.Last().Count == 2 && index > Count - 2) { // likewise for last centroid return 2 * lastNode.Mean - Max; @@ -256,7 +253,7 @@ public double Quantile(double quantile) { return WeightedAvg(Min, weightSoFar - index, currentNode.Mean, index - 1); } - foreach (Centroid nextNode in _centroids.Values.Skip(1)) + foreach (Centroid nextNode in _centroids.Skip(1)) { double nextWeight = nextNode.Count; double dw = (currentWeight + nextWeight) / 2.0; @@ -317,7 +314,7 @@ private double WeightedAvg(double m1, double w1, double m2, double w2) /// An array of objects that contain a value (x-axis) and a count (y-axis) /// which can be used to plot a distribution of the data set public DistributionPoint[] GetDistribution() { - return _centroids.Values + return _centroids .Select(c => new DistributionPoint(c.Mean, c.Count)) .ToArray(); } @@ -327,12 +324,12 @@ public DistributionPoint[] GetDistribution() { /// /// public Byte[] Serialize() { - var count = _centroids.Values.Sum(c => c.Count); + var count = _centroids.Sum(c => c.Count); var fields = new[] { _newAvg, _oldAvg, Accuracy, CompressionConstant, Min, Max } .SelectMany(f => BitConverter.GetBytes(f)); - var data = _centroids.Values + var data = _centroids .SelectMany(c => BitConverter.GetBytes(c.Mean).Concat(BitConverter.GetBytes(c.Count))); return fields @@ -342,7 +339,7 @@ public Byte[] Serialize() { private bool Compress() { TDigest newTDigest = new TDigest(Accuracy, CompressionConstant); - List temp = _centroids.Values.ToList(); + List temp = _centroids.ToList(); temp.Shuffle(); foreach (Centroid centroid in temp) { @@ -357,7 +354,7 @@ private bool Compress() { private double ComputeCentroidQuantile(Centroid centroid) { double sum = 0; - foreach (Centroid c in _centroids.Values) { + foreach (Centroid c in _centroids) { if (c.Mean > centroid.Mean) break; sum += c.Count; } @@ -366,38 +363,54 @@ private double ComputeCentroidQuantile(Centroid centroid) { return (centroid.Count / 2 + sum) / denom; } - private IEnumerable GetClosestCentroids(double x) { - C5.KeyValuePair successor; - C5.KeyValuePair predecessor; + private IList GetClosestCentroids(double x) { + var probe = new Centroid(x, 0); + Centroid exact; + if (TryGetCentroid(x, out exact)) { + return new[] { exact }; + } - if (!_centroids.TryWeakSuccessor(x, out successor)) { - yield return _centroids.Predecessor(x).Value; - yield break; + Centroid first = _centroids.Min; + if (CentroidMeanComparer.Instance.Compare(probe, first) < 0) { + return new[] { first }; } - if (successor.Value.Mean == x || !_centroids.TryPredecessor(x, out predecessor)) { - yield return successor.Value; - yield break; + Centroid last = _centroids.Max; + if (CentroidMeanComparer.Instance.Compare(probe, last) > 0) { + return new[] { last }; } - double sDiff = Math.Abs(successor.Value.Mean - x); - double pDiff = Math.Abs(successor.Value.Mean - x); + Centroid successor = _centroids.GetViewBetween(probe, last).Min; + Centroid predecessor = _centroids.GetViewBetween(first, probe).Max; + double sDiff = Math.Abs(successor.Mean - x); + double pDiff = Math.Abs(predecessor.Mean - x); - if (sDiff < pDiff) yield return successor.Value; - else if (pDiff < sDiff) yield return predecessor.Value; - else { - yield return successor.Value; - yield return predecessor.Value; + if (sDiff < pDiff) return new[] { successor }; + if (pDiff < sDiff) return new[] { predecessor }; + return new[] { successor, predecessor }; + } + + private bool TryGetCentroid(double mean, out Centroid centroid) { + var probe = new Centroid(mean, 0); + SortedSet matches = _centroids.GetViewBetween(probe, probe); + if (matches.Count == 0) { + centroid = null; + return false; } + + centroid = matches.Min; + return true; } private double GetThreshold(double q) { return 4 * _count * Accuracy * q * (1 - q); } - private void ReInsertCentroid(double oldMean, Centroid c) { - var ret = _centroids.Remove(oldMean); - _centroids.Add(c.Mean, c); + private void UpdateCentroid(Centroid c, double weight, double value) { + _centroids.Remove(c); + double oldMean; + c.Update(weight, value, out oldMean); + _centroids.Add(c); } } @@ -428,4 +441,15 @@ public bool Update(double delta_w, double value, out double oldMean) { return oldMean != Mean; } } + + internal sealed class CentroidMeanComparer : IComparer { + internal static readonly CentroidMeanComparer Instance = new CentroidMeanComparer(); + + private CentroidMeanComparer() { + } + + public int Compare(Centroid x, Centroid y) { + return x.Mean.CompareTo(y.Mean); + } + } } diff --git a/TDigest/TDigest.csproj b/TDigest/TDigest.csproj index ebbc098..0b957d2 100644 --- a/TDigest/TDigest.csproj +++ b/TDigest/TDigest.csproj @@ -1,77 +1,23 @@ - - - + - Release - AnyCPU - {909364F5-4F9A-444D-BE6A-CE2A6EF592B0} - Library - Properties + net45;net472;netstandard2.0;net8.0;net10.0 StatsLib TDigest - v4.5 - 512 - - - AnyCPU - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - AnyCPU - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - - - + 2.0.0 + Graham Henry + Streaming Percentile Estimation + https://github.com/quantumtunneling/T-Digest.NET + MIT + README.md + Streaming Percentile Quantile Estimation + Copyright 2019 + false true - - Key.snk + - - ..\packages\C5.2.3.0.1\lib\net40\C5.dll - True - - - - - - - - - - - - - - - - - - - - - + + - - - \ No newline at end of file + diff --git a/TDigest/packages.config b/TDigest/packages.config deleted file mode 100644 index ea9cb3f..0000000 --- a/TDigest/packages.config +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file