From aa638eee9d0ebd8a6676c51de11a7b96907f35bd Mon Sep 17 00:00:00 2001 From: Vahid Date: Sat, 28 Mar 2026 09:23:46 -0400 Subject: [PATCH 01/30] bug fix in cli. --- EBA/CLI/CLI.cs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/EBA/CLI/CLI.cs b/EBA/CLI/CLI.cs index 002fa55d..e225f0ec 100644 --- a/EBA/CLI/CLI.cs +++ b/EBA/CLI/CLI.cs @@ -81,14 +81,6 @@ e is NotSupportedException || _workingDirOption, _statusFilenameOption, GetBitcoinCmd( - defOps, - bitcoinTraverseCmdHandlerAsync, - bitcoinDeDupCmdHandlerAsync, - bitcoinImportCmdHandlerAsync, - bitcoinSampleCmdHandlerAsync, - bitcoinAddressStatsHandlerAsync, - bitcoinImportCypherQueriesAsync, - bitcoinPostProcessGraphHandlerAsync) defaultOptions: defOps, traverseHandlerAsync: bitcoinTraverseCmdHandlerAsync, dedupHandlerAsync: bitcoinDeDupCmdHandlerAsync, @@ -96,7 +88,8 @@ e is NotSupportedException || sampleHandlerAsync: bitcoinSampleCmdHandlerAsync, mapMarketHandlerAsync: bitcoinMapMarketHandlerAsync, addressStatsHandlerAsync: bitcoinAddressStatsHandlerAsync, - importCypherQueriesAsync: bitcoinImportCypherQueriesAsync) + importCypherQueriesAsync: bitcoinImportCypherQueriesAsync, + postProcessGraphHandlerAsync: bitcoinPostProcessGraphHandlerAsync) }; for (int i = 0; i < _rootCmd.Options.Count; i++) From 24ceeca9664569756810bd67d58fc7f4223b9c21 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sat, 28 Mar 2026 19:40:15 -0400 Subject: [PATCH 02/30] Separate OHLCV into an independent type and udpate market mapper accordingly. --- .../Bitcoin/Utilities/MarketMapper.cs | 64 ++------- EBA/Utilities/OHLCV.cs | 133 ++++++++++++++++++ 2 files changed, 146 insertions(+), 51 deletions(-) create mode 100644 EBA/Utilities/OHLCV.cs diff --git a/EBA/Blockchains/Bitcoin/Utilities/MarketMapper.cs b/EBA/Blockchains/Bitcoin/Utilities/MarketMapper.cs index 1fff361e..3150cfca 100644 --- a/EBA/Blockchains/Bitcoin/Utilities/MarketMapper.cs +++ b/EBA/Blockchains/Bitcoin/Utilities/MarketMapper.cs @@ -1,4 +1,4 @@ -using System.Globalization; +using EBA.Utilities; namespace EBA.Blockchains.Bitcoin.Utilities; @@ -7,39 +7,7 @@ public class MarketMapper(BitcoinChainAgent agent, ILogger private readonly BitcoinChainAgent _agent = agent; private readonly ILogger _logger = logger; - private record OHLCV(long Timestamp, float Open, float High, float Low, float Close, float Volume) - { - public static bool TryParse(string csvLine, out OHLCV candle) - { - candle = null; - - var values = csvLine.Split(','); - if (values.Length < 6) return false; - - if (!double.TryParse(values[0], NumberStyles.Any, CultureInfo.InvariantCulture, out var ts) || - !float.TryParse(values[1], NumberStyles.Any, CultureInfo.InvariantCulture, out var open) || - !float.TryParse(values[2], NumberStyles.Any, CultureInfo.InvariantCulture, out var high) || - !float.TryParse(values[3], NumberStyles.Any, CultureInfo.InvariantCulture, out var low) || - !float.TryParse(values[4], NumberStyles.Any, CultureInfo.InvariantCulture, out var close) || - !float.TryParse(values[5], NumberStyles.Any, CultureInfo.InvariantCulture, out var volume)) - { - return false; - } - - candle = new OHLCV( - Timestamp: (long)ts, - Open: open, - High: high, - Low: low, - Close: close, - Volume: volume - ); - - return true; - } - } - - private record BlockOHLC(BlockMetadata Metadata, OHLCV OHLC); + private record BlockOHLC(BlockMetadata Metadata, OHLCV ohlcv); public async Task MapAsync(Options options, CancellationToken cT) { @@ -49,17 +17,10 @@ public async Task MapAsync(Options options, CancellationToken cT) var matchedBlockMarket = MatchBlockAndMarketData(blocks, options.Bitcoin.MapMarket.OhlcvSourceFilename); using var writer = new StreamWriter(options.Bitcoin.MapMarket.BlockOhlcvMappedFilename); + writer.WriteLine(string.Join('\t', ["Height", .. OHLCV.GetFeaturesName()])); + foreach (var x in matchedBlockMarket) - writer.WriteLine( - string.Join( - '\t', - x.Metadata.Height, - x.Metadata.MedianTime, - x.OHLC.Open, - x.OHLC.High, - x.OHLC.Low, - x.OHLC.Close, - x.OHLC.Volume)); + writer.WriteLine(string.Join('\t', [x.Metadata.Height.ToString(), .. x.ohlcv.GetFeatures()])); _logger.LogInformation( "Finished writing mapped block and market data to {MappedOutputFilename}", @@ -89,7 +50,7 @@ private List MatchBlockAndMarketData(List blocks, stri while ((line = reader.ReadLine()) != null) { - if (!OHLCV.TryParse(line, out var candle)) + if (!OHLCV.TryParse(line, out var candle) || candle == null) continue; if (candle.Timestamp < startTime) @@ -107,12 +68,13 @@ private List MatchBlockAndMarketData(List blocks, stri new BlockOHLC( sortedBlocks[i], new OHLCV( - sortedBlocks[i].MedianTime, - data.First().Open, - data.Max(x => x.High), - data.Min(x => x.Low), - data.Last().Close, - data.Average(x => x.Volume)))); + timestamp: sortedBlocks[i].MedianTime, + open: data.First().Open, + high: data.Max(x => x.High), + low: data.Min(x => x.Low), + close: data.Last().Close, + volume: (long)Math.Round(data.Average(x => x.Volume)), + vwap: OHLCV.GetVWAP(data)))); } } diff --git a/EBA/Utilities/OHLCV.cs b/EBA/Utilities/OHLCV.cs new file mode 100644 index 00000000..91d522ba --- /dev/null +++ b/EBA/Utilities/OHLCV.cs @@ -0,0 +1,133 @@ +using System.Globalization; + +namespace EBA.Utilities; + +public class OHLCV(long timestamp, decimal open, decimal high, decimal low, decimal close, long volume, decimal vwap) +{ + public long Timestamp { get; } = timestamp; + public decimal Open { get; } = open; + public decimal High { get; } = high; + public decimal Low { get; } = low; + public decimal Close { get; } = close; + public long Volume { get; } = volume; + + /// + /// Volume-Weighted Average Price (VWAP) + /// + public decimal VWAP { get; } = vwap; + + public decimal OHLC4 { get { return (Open + High + Low + Close) / 4; } } + + public static bool TryParse(string csvLine, out OHLCV? candle) + { + candle = default; + + var values = csvLine.Split(','); + if (values.Length < 6) return false; + + if (!double.TryParse(values[0], NumberStyles.Any, CultureInfo.InvariantCulture, out var timestamp) || + !decimal.TryParse(values[1], NumberStyles.Any, CultureInfo.InvariantCulture, out var open) || + !decimal.TryParse(values[2], NumberStyles.Any, CultureInfo.InvariantCulture, out var high) || + !decimal.TryParse(values[3], NumberStyles.Any, CultureInfo.InvariantCulture, out var low) || + !decimal.TryParse(values[4], NumberStyles.Any, CultureInfo.InvariantCulture, out var close) || + !decimal.TryParse(values[5], NumberStyles.Any, CultureInfo.InvariantCulture, out var volume)) + { + return false; + } + + candle = new OHLCV( + timestamp: (long)timestamp, + open: open, + high: high, + low: low, + close: close, + volume: (long)(volume * BitcoinChainAgent.Coin), + vwap: 0); + + return true; + } + + public static decimal GetVWAP(List candles) + { + decimal sumVolume = 0; + decimal vwapSum = 0; + + foreach (var c in candles) + { + sumVolume += c.Volume; + vwapSum += c.OHLC4 * c.Volume; + } + + if (sumVolume == 0) + return 0; + + return vwapSum / sumVolume; + } + + public static bool TryParseFile(string filename, out Dictionary candles) + { + candles = []; + + using var reader = new StreamReader(filename); + reader.ReadLine(); + string? line; + while ((line = reader.ReadLine()) != null) + { + var cols = line.Split('\t'); + if (cols.Length < 8) return false; + + if (!long.TryParse(cols[0], NumberStyles.Any, CultureInfo.InvariantCulture, out var height) || + !long.TryParse(cols[1], NumberStyles.Any, CultureInfo.InvariantCulture, out var timestamp) || + !decimal.TryParse(cols[2], NumberStyles.Any, CultureInfo.InvariantCulture, out var open) || + !decimal.TryParse(cols[3], NumberStyles.Any, CultureInfo.InvariantCulture, out var high) || + !decimal.TryParse(cols[4], NumberStyles.Any, CultureInfo.InvariantCulture, out var low) || + !decimal.TryParse(cols[5], NumberStyles.Any, CultureInfo.InvariantCulture, out var close) || + !decimal.TryParse(cols[6], NumberStyles.Any, CultureInfo.InvariantCulture, out var vwap) || + !decimal.TryParse(cols[7], NumberStyles.Any, CultureInfo.InvariantCulture, out var ohlc4) || + !long.TryParse(cols[8], NumberStyles.Any, CultureInfo.InvariantCulture, out var volume)) + { + return false; + } + + candles[height] = new OHLCV( + timestamp: timestamp, + open: open, + high: high, + low: low, + close: close, + volume: volume, + vwap: vwap); + } + return true; + } + + public static string[] GetFeaturesName() + { + return + [ + nameof(Timestamp), + nameof(Open), + nameof(High), + nameof(Low), + nameof(Close), + nameof(VWAP), + nameof(OHLC4), + $"{nameof(Volume)}(Satoshi)" + ]; + } + + public string[] GetFeatures() + { + return + [ + Timestamp.ToString(CultureInfo.InvariantCulture), + Open.ToString(CultureInfo.InvariantCulture), + High.ToString(CultureInfo.InvariantCulture), + Low.ToString(CultureInfo.InvariantCulture), + Close.ToString(CultureInfo.InvariantCulture), + VWAP.ToString(CultureInfo.InvariantCulture), + OHLC4.ToString(CultureInfo.InvariantCulture), + Volume.ToString(CultureInfo.InvariantCulture) + ]; + } +} From a665bb8518c20fba4fad0aa8df955379301be3f3 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sat, 28 Mar 2026 20:14:43 -0400 Subject: [PATCH 03/30] use arg names when calling initializers --- EBA/Blockchains/Bitcoin/GraphModel/B2BEdge.cs | 8 +++++++- EBA/Blockchains/Bitcoin/GraphModel/B2TEdge.cs | 8 +++++++- EBA/Blockchains/Bitcoin/GraphModel/C2TEdge.cs | 12 ++++++------ EBA/Blockchains/Bitcoin/GraphModel/S2TEdge.cs | 8 +++++++- EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs | 16 ++++++++++++++-- EBA/Blockchains/Bitcoin/GraphModel/T2TEdge.cs | 8 +++++++- EBA/Graph/Model/Edge.cs | 9 ++++++--- 7 files changed, 54 insertions(+), 15 deletions(-) diff --git a/EBA/Blockchains/Bitcoin/GraphModel/B2BEdge.cs b/EBA/Blockchains/Bitcoin/GraphModel/B2BEdge.cs index 8be9ab94..bdc6b33d 100644 --- a/EBA/Blockchains/Bitcoin/GraphModel/B2BEdge.cs +++ b/EBA/Blockchains/Bitcoin/GraphModel/B2BEdge.cs @@ -3,7 +3,13 @@ public class B2BEdge( BlockNode source, BlockNode target) - : Edge(source, target, 0, Kind.Relation, 0, 0) + : Edge( + source: source, + target: target, + value: 0, + relation: Kind.Relation, + timestamp: 0, + blockHeight: 0) { public static new EdgeKind Kind => new(BlockNode.Kind, BlockNode.Kind, RelationType.Follows); diff --git a/EBA/Blockchains/Bitcoin/GraphModel/B2TEdge.cs b/EBA/Blockchains/Bitcoin/GraphModel/B2TEdge.cs index 4bd5dca8..f08bd8a9 100644 --- a/EBA/Blockchains/Bitcoin/GraphModel/B2TEdge.cs +++ b/EBA/Blockchains/Bitcoin/GraphModel/B2TEdge.cs @@ -8,7 +8,13 @@ public B2TEdge( BlockNode source, TxNode target, long value, uint timestamp, long blockHeight) : - base(source, target, value, Kind.Relation, timestamp, blockHeight) + base( + source: source, + target: target, + value: value, + relation: Kind.Relation, + timestamp: timestamp, + blockHeight: blockHeight) { } public B2TEdge Update(long value) diff --git a/EBA/Blockchains/Bitcoin/GraphModel/C2TEdge.cs b/EBA/Blockchains/Bitcoin/GraphModel/C2TEdge.cs index d02ca10d..f9053168 100644 --- a/EBA/Blockchains/Bitcoin/GraphModel/C2TEdge.cs +++ b/EBA/Blockchains/Bitcoin/GraphModel/C2TEdge.cs @@ -6,12 +6,12 @@ public class C2TEdge( uint timestamp, long blockHeight) : Edge( - new CoinbaseNode(), - target, - value, - RelationType.Mints, - timestamp, - blockHeight) + source: new CoinbaseNode(), + target: target, + value: value, + relation: RelationType.Mints, + timestamp: timestamp, + blockHeight: blockHeight) { public new static EdgeKind Kind => new(CoinbaseNode.Kind, TxNode.Kind, RelationType.Mints); diff --git a/EBA/Blockchains/Bitcoin/GraphModel/S2TEdge.cs b/EBA/Blockchains/Bitcoin/GraphModel/S2TEdge.cs index 07a4eb02..b442b3f4 100644 --- a/EBA/Blockchains/Bitcoin/GraphModel/S2TEdge.cs +++ b/EBA/Blockchains/Bitcoin/GraphModel/S2TEdge.cs @@ -36,7 +36,13 @@ public S2TEdge( string txid, int vout, bool generated) : - base(source, target, value, Kind.Relation, timestamp, spentHeight) + base( + source: source, + target: target, + value: value, + relation: Kind.Relation, + timestamp: timestamp, + blockHeight: spentHeight) { Txid = txid; Vout = vout; diff --git a/EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs b/EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs index c87359ee..894e058c 100644 --- a/EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs +++ b/EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs @@ -17,7 +17,13 @@ public T2SEdge( long creationHeight, Output output, long spentHeight = long.MaxValue) - : base(source, target, output.Value, Kind.Relation, timestamp, creationHeight) + : base( + source: source, + target: target, + relation: Kind.Relation, + value: output.Value, + timestamp: timestamp, + blockHeight: creationHeight) { Vout = output.N; SpentHeight = spentHeight; @@ -31,7 +37,13 @@ public T2SEdge( long value, int outputIndex, long spentHeight = long.MaxValue) : - base(source, target, value, Kind.Relation, timestamp, creationHeight) + base( + source: source, + target: target, + relation: Kind.Relation, + value: value, + timestamp: timestamp, + blockHeight: creationHeight) { Vout = outputIndex; SpentHeight = spentHeight; diff --git a/EBA/Blockchains/Bitcoin/GraphModel/T2TEdge.cs b/EBA/Blockchains/Bitcoin/GraphModel/T2TEdge.cs index de1e5930..e9e47420 100644 --- a/EBA/Blockchains/Bitcoin/GraphModel/T2TEdge.cs +++ b/EBA/Blockchains/Bitcoin/GraphModel/T2TEdge.cs @@ -5,7 +5,13 @@ public class T2TEdge : Edge public T2TEdge( TxNode source, TxNode target, long value, RelationType type, uint timestamp, long blockHeight) : - base(source, target, value, type, timestamp, blockHeight) + base( + source: source, + target: target, + value: value, + relation: type, + timestamp: timestamp, + blockHeight: blockHeight) { } public static EdgeKind KindTransfers => new(TxNode.Kind, TxNode.Kind, RelationType.Transfers); diff --git a/EBA/Graph/Model/Edge.cs b/EBA/Graph/Model/Edge.cs index 65d4de24..956e73f1 100644 --- a/EBA/Graph/Model/Edge.cs +++ b/EBA/Graph/Model/Edge.cs @@ -37,9 +37,12 @@ public static string Header private const string _delimiter = "\t"; public Edge( - TSource source, TTarget target, - long value, RelationType relation, - uint timestamp, long blockHeight) + TSource source, + TTarget target, + RelationType relation, + long value, + uint timestamp, + long blockHeight) { Source = source; Target = target; From b4b13a35bfb06f365112f0415905290bbf2f6e23 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sun, 29 Mar 2026 12:48:31 -0400 Subject: [PATCH 04/30] Add satoshi to fiat calculator --- EBA/Utilities/OHLCV.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/EBA/Utilities/OHLCV.cs b/EBA/Utilities/OHLCV.cs index 91d522ba..dfc98f7e 100644 --- a/EBA/Utilities/OHLCV.cs +++ b/EBA/Utilities/OHLCV.cs @@ -18,6 +18,11 @@ public class OHLCV(long timestamp, decimal open, decimal high, decimal low, deci public decimal OHLC4 { get { return (Open + High + Low + Close) / 4; } } + public decimal GetFiatValue(long satoshiAmount) + { + return satoshiAmount / (decimal)BitcoinChainAgent.Coin * VWAP; + } + public static bool TryParse(string csvLine, out OHLCV? candle) { candle = default; From 63f2db85ef067c094ddbacae91b5a3cb077067a3 Mon Sep 17 00:00:00 2001 From: Vahid Date: Tue, 31 Mar 2026 21:03:21 -0400 Subject: [PATCH 05/30] Add GetSchemaConfig and GetSeedingCommands methods --- .../Bitcoin/Strategies/B2BEdgeStrategy.cs | 16 ++++++++++--- .../Bitcoin/Strategies/B2TEdgeStrategy.cs | 10 ++++---- .../Bitcoin/Strategies/BitcoinStrategyBase.cs | 2 +- .../Strategies/BitcoinStrategyFactory.cs | 3 ++- .../Bitcoin/Strategies/BlockNodeStrategy.cs | 19 +++++++++++---- .../Bitcoin/Strategies/C2TEdgeStrategy.cs | 10 ++++---- .../Bitcoin/Strategies/S2TEdgeStrategy.cs | 16 ++++++------- .../Bitcoin/Strategies/ScriptNodeStrategy.cs | 18 ++++++++++---- .../Bitcoin/Strategies/T2SEdgeStrategy.cs | 24 +++++++++++++------ .../Bitcoin/Strategies/T2TEdgeStrategy.cs | 10 ++++---- .../Bitcoin/Strategies/TxNodeStrategy.cs | 22 +++++++++++++---- EBA/Graph/Db/Neo4jDb/StrategyBase.cs | 10 ++++++++ 12 files changed, 113 insertions(+), 47 deletions(-) diff --git a/EBA/Graph/Bitcoin/Strategies/B2BEdgeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/B2BEdgeStrategy.cs index 3b829ffc..a1fc0850 100644 --- a/EBA/Graph/Bitcoin/Strategies/B2BEdgeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/B2BEdgeStrategy.cs @@ -7,7 +7,7 @@ public class B2BEdgeStrategy(bool serializeCompressed) $"edges_{B2BEdge.Kind.Source}_{B2BEdge.Kind.Relation}_{B2BEdge.Kind.Target}", serializeCompressed) { - private static readonly PropertyMapping[] _mappings = + public static readonly PropertyMapping[] Mappings = [ Factory.SourceId(BlockNodeStrategy.IdSpace, e => e.BlockHeight), Factory.TargetId(BlockNodeStrategy.IdSpace, e => e.BlockHeight), @@ -16,7 +16,7 @@ public class B2BEdgeStrategy(bool serializeCompressed) public override string GetCsvHeader() { - return _mappings.GetCsvHeader(); + return Mappings.GetCsvHeader(); } public override string GetCsvRow(IGraphElement edge) @@ -26,7 +26,7 @@ public override string GetCsvRow(IGraphElement edge) public static string GetCsvRow(B2BEdge edge) { - return _mappings.GetCsv(edge); + return Mappings.GetCsv(edge); } public static B2BEdge Deserialize(BlockNode source, BlockNode target, IRelationship relationship) @@ -38,4 +38,14 @@ public override string GetQuery(string filename) { throw new NotImplementedException(); } + + public override string[] GetSeedingCommands() + { + return + [ + $"MATCH (target:Block), (source:Block) " + + $"WHERE target.{nameof(B2BEdge.BlockHeight)} + 1 = source.{nameof(B2BEdge.BlockHeight)} " + + $"MERGE (target)-[:{RelationType.Follows}]->(source)" + ]; + } } diff --git a/EBA/Graph/Bitcoin/Strategies/B2TEdgeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/B2TEdgeStrategy.cs index b87a2bf3..49b1700b 100644 --- a/EBA/Graph/Bitcoin/Strategies/B2TEdgeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/B2TEdgeStrategy.cs @@ -7,7 +7,7 @@ public class B2TEdgeStrategy(bool serializeCompressed) $"edges_{B2TEdge.Kind.Source}_{B2TEdge.Kind.Relation}_{B2TEdge.Kind.Target}", serializeCompressed) { - private static readonly PropertyMapping[] _mappings = + public static readonly PropertyMapping[] Mappings = [ Factory.SourceId(BlockNodeStrategy.IdSpace, e => e.Source.BlockMetadata.Height), Factory.TargetId(TxNodeStrategy.IdSpace, e => e.Target.Txid), @@ -18,7 +18,7 @@ public class B2TEdgeStrategy(bool serializeCompressed) public override string GetCsvHeader() { - return _mappings.GetCsvHeader(); + return Mappings.GetCsvHeader(); } public override string GetCsvRow(IGraphElement edge) @@ -28,7 +28,7 @@ public override string GetCsvRow(IGraphElement edge) public static string GetCsv(B2TEdge edge) { - return _mappings.GetCsv(edge); + return Mappings.GetCsv(edge); } public static B2TEdge Deserialize(BlockNode source, TxNode target, IRelationship relationship) @@ -37,8 +37,8 @@ public static B2TEdge Deserialize(BlockNode source, TxNode target, IRelationship source: source, target: target, timestamp: 0, - blockHeight: _mappings.Get(Factory.HeightProperty.Name).Deserialize(relationship.Properties), - value: _mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties)); + blockHeight: Mappings.Get(Factory.HeightProperty.Name).Deserialize(relationship.Properties), + value: Mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties)); } public override string GetQuery(string filename) diff --git a/EBA/Graph/Bitcoin/Strategies/BitcoinStrategyBase.cs b/EBA/Graph/Bitcoin/Strategies/BitcoinStrategyBase.cs index 4108031f..f1ab4fe6 100644 --- a/EBA/Graph/Bitcoin/Strategies/BitcoinStrategyBase.cs +++ b/EBA/Graph/Bitcoin/Strategies/BitcoinStrategyBase.cs @@ -2,6 +2,6 @@ namespace EBA.Graph.Bitcoin.Strategies; -public abstract class BitcoinStrategyBase(string defaultBaseFilename, bool serializeCompressed) +public abstract class BitcoinStrategyBase(string defaultBaseFilename, bool serializeCompressed) : StrategyBase(defaultBaseFilename, serializeCompressed) { } \ No newline at end of file diff --git a/EBA/Graph/Bitcoin/Strategies/BitcoinStrategyFactory.cs b/EBA/Graph/Bitcoin/Strategies/BitcoinStrategyFactory.cs index dce39eab..f9db5bd1 100644 --- a/EBA/Graph/Bitcoin/Strategies/BitcoinStrategyFactory.cs +++ b/EBA/Graph/Bitcoin/Strategies/BitcoinStrategyFactory.cs @@ -28,7 +28,8 @@ public BitcoinStrategyFactory(Options options) { T2TEdge.KindFee, new T2TEdgeStrategy(T2TEdge.KindFee, compressOutput) }, { S2TEdge.Kind, new S2TEdgeStrategy(compressOutput) }, { T2SEdge.Kind, new T2SEdgeStrategy(compressOutput) }, - { B2TEdge.Kind, new B2TEdgeStrategy(compressOutput) } + { B2TEdge.Kind, new B2TEdgeStrategy(compressOutput) }, + { B2BEdge.Kind, new B2BEdgeStrategy(compressOutput) } }; } diff --git a/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs index a3bb876c..7cef47e2 100644 --- a/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs @@ -10,7 +10,7 @@ public class BlockNodeStrategy(bool serializeCompressed) public static string IdSpace { get; } = BlockNode.Kind.ToString(); private const Block v = null!; - private static readonly PropertyMapping[] _mappings = + public static readonly PropertyMapping[] Mappings = [ new("", FieldType.String, x=>x.BlockMetadata.Height, p => p.GetIdFieldCsvHeader(IdSpace.ToString())), PropertyMappingFactory.Height(n => n.BlockMetadata.Height), @@ -68,12 +68,12 @@ .. PropertyMappingFactory.DictionaryToColumns( ]; private static readonly Dictionary> _mappingsDict = - _mappings.ToDictionary(m => m.Property.Name, m => m); + Mappings.ToDictionary(m => m.Property.Name, m => m); public override string GetCsvHeader() { - return _mappings.GetCsvHeader(); + return Mappings.GetCsvHeader(); } public override string GetCsvRow(IGraphElement component) @@ -83,7 +83,7 @@ public override string GetCsvRow(IGraphElement component) public static string GetCsv(BlockNode node) { - return _mappings.GetCsv(node); + return Mappings.GetCsv(node); } public static BlockNode Deserialize( @@ -188,4 +188,15 @@ public override string GetQuery(string filename) */ throw new NotImplementedException(); } + + public override string[] GetSchemaConfiguration() + { + var heightName = PropertyMappingFactory.HeightProperty.Name; + return + [ + $"CREATE CONSTRAINT {BlockNode.Kind}_{heightName}_Unique " + + $"IF NOT EXISTS " + + $"FOR (v:{BlockNode.Kind}) REQUIRE v.{heightName} IS UNIQUE" + ]; + } } \ No newline at end of file diff --git a/EBA/Graph/Bitcoin/Strategies/C2TEdgeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/C2TEdgeStrategy.cs index 1521d6c6..dbba754b 100644 --- a/EBA/Graph/Bitcoin/Strategies/C2TEdgeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/C2TEdgeStrategy.cs @@ -9,7 +9,7 @@ public class C2TEdgeStrategy(bool serializeCompressed) { public static string IdSpaceCoinbase { get; } = CoinbaseNode.Kind.ToString(); - public static readonly PropertyMapping[] _mappings = + public static readonly PropertyMapping[] Mappings = [ Factory.SourceId(IdSpaceCoinbase, _ => CoinbaseNode.Kind), Factory.TargetId(TxNodeStrategy.IdSpace, e => e.Target.Txid), @@ -20,7 +20,7 @@ public class C2TEdgeStrategy(bool serializeCompressed) public override string GetCsvHeader() { - return _mappings.GetCsvHeader(); + return Mappings.GetCsvHeader(); } public override string GetCsvRow(IGraphElement edge) @@ -30,16 +30,16 @@ public override string GetCsvRow(IGraphElement edge) public static string GetCsv(C2TEdge edge) { - return _mappings.GetCsv(edge); + return Mappings.GetCsv(edge); } public static C2TEdge Deserialize(TxNode target, IRelationship relationship) { return new C2TEdge( target: target, - value: _mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties), + value: Mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties), timestamp: 0, - blockHeight: _mappings.Get(Factory.HeightProperty.Name).Deserialize(relationship.Properties)); + blockHeight: Mappings.Get(Factory.HeightProperty.Name).Deserialize(relationship.Properties)); } public override string GetQuery(string csvFilename) diff --git a/EBA/Graph/Bitcoin/Strategies/S2TEdgeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/S2TEdgeStrategy.cs index 58783dd6..e9bb1a87 100644 --- a/EBA/Graph/Bitcoin/Strategies/S2TEdgeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/S2TEdgeStrategy.cs @@ -9,7 +9,7 @@ public class S2TEdgeStrategy(bool serializeCompressed) $"edges_{S2TEdge.Kind.Source}_{S2TEdge.Kind.Relation}_{S2TEdge.Kind.Target}", serializeCompressed) { - private static readonly PropertyMapping[] _mappings = + public static readonly PropertyMapping[] Mappings = [ Factory.SourceId(ScriptNodeStrategy.IdSpace, e => e.Source.Id), Factory.TargetId(TxNodeStrategy.IdSpace, e => e.Target.Txid), @@ -23,7 +23,7 @@ public class S2TEdgeStrategy(bool serializeCompressed) public override string GetCsvHeader() { - return _mappings.GetCsvHeader(); + return Mappings.GetCsvHeader(); } public override string GetCsvRow(IGraphElement edge) @@ -33,7 +33,7 @@ public override string GetCsvRow(IGraphElement edge) public static string GetCsvRow(S2TEdge edge) { - return _mappings.GetCsv(edge); + return Mappings.GetCsv(edge); } public static S2TEdge Deserialize(ScriptNode source, TxNode target, IRelationship relationship) @@ -42,11 +42,11 @@ public static S2TEdge Deserialize(ScriptNode source, TxNode target, IRelationshi source: source, target: target, timestamp: 0, - spentHeight: _mappings.Get(nameof(S2TEdge.SpentHeight)).Deserialize(relationship.Properties), - value: _mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties), - txid: _mappings.Get(nameof(S2TEdge.Txid)).Deserialize(relationship.Properties), - vout: _mappings.Get(nameof(S2TEdge.Vout)).Deserialize(relationship.Properties), - generated: _mappings.Get(nameof(S2TEdge.Generated)).Deserialize(relationship.Properties) + spentHeight: Mappings.Get(nameof(S2TEdge.SpentHeight)).Deserialize(relationship.Properties), + value: Mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties), + txid: Mappings.Get(nameof(S2TEdge.Txid)).Deserialize(relationship.Properties), + vout: Mappings.Get(nameof(S2TEdge.Vout)).Deserialize(relationship.Properties), + generated: Mappings.Get(nameof(S2TEdge.Generated)).Deserialize(relationship.Properties) ); } diff --git a/EBA/Graph/Bitcoin/Strategies/ScriptNodeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/ScriptNodeStrategy.cs index 5ba8cbf6..b6afad65 100644 --- a/EBA/Graph/Bitcoin/Strategies/ScriptNodeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/ScriptNodeStrategy.cs @@ -15,7 +15,7 @@ public class ScriptNodeStrategy(bool serializeCompressed) n => n.SHA256Hash, p => p.GetIdFieldCsvHeader(IdSpace)); - private readonly static PropertyMapping[] _mappings = + public readonly static PropertyMapping[] Mappings = [ _sha256Hash, new(nameof(v.Address), FieldType.String, n => n.Address), @@ -25,11 +25,11 @@ public class ScriptNodeStrategy(bool serializeCompressed) ]; private static readonly Dictionary> _mappingsDict = - _mappings.ToDictionary(m => m.Property.Name, m => m); + Mappings.ToDictionary(m => m.Property.Name, m => m); public override string GetCsvHeader() { - return _mappings.GetCsvHeader(); + return Mappings.GetCsvHeader(); } public override string GetCsvRow(IGraphElement component) @@ -39,7 +39,7 @@ public override string GetCsvRow(IGraphElement component) public static string GetCsv(ScriptNode node) { - return _mappings.GetCsv(node); + return Mappings.GetCsv(node); } public static ScriptNode Deserialize( @@ -104,4 +104,14 @@ public override string GetQuery(string filename) */ throw new NotImplementedException(); } + + public override string[] GetSchemaConfiguration() + { + return + [ + $"CREATE CONSTRAINT {ScriptNode.Kind}_{_sha256Hash.Property.Name}_Unique " + + $"IF NOT EXISTS " + + $"FOR (v:{ScriptNode.Kind}) REQUIRE v.{_sha256Hash.Property.Name} IS UNIQUE" + ]; + } } \ No newline at end of file diff --git a/EBA/Graph/Bitcoin/Strategies/T2SEdgeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/T2SEdgeStrategy.cs index 2c4594c3..595561cd 100644 --- a/EBA/Graph/Bitcoin/Strategies/T2SEdgeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/T2SEdgeStrategy.cs @@ -9,7 +9,7 @@ public class T2SEdgeStrategy(bool serializeCompressed) $"edges_{T2SEdge.Kind.Source}_{T2SEdge.Kind.Relation}_{T2SEdge.Kind.Target}", serializeCompressed) { - public static readonly PropertyMapping[] _mappings = + public static readonly PropertyMapping[] Mappings = [ Factory.SourceId(TxNodeStrategy.IdSpace, e => e.Source.Txid), Factory.TargetId(ScriptNodeStrategy.IdSpace, e => e.Target.Id), @@ -22,7 +22,7 @@ public class T2SEdgeStrategy(bool serializeCompressed) public override string GetCsvHeader() { - return _mappings.GetCsvHeader(); + return Mappings.GetCsvHeader(); } public override string GetCsvRow(IGraphElement edge) @@ -32,7 +32,7 @@ public override string GetCsvRow(IGraphElement edge) public static string GetCsv(T2SEdge edge) { - return _mappings.GetCsv(edge); + return Mappings.GetCsv(edge); } public static T2SEdge Deserialize(TxNode source, ScriptNode target, IRelationship relationship) @@ -41,14 +41,24 @@ public static T2SEdge Deserialize(TxNode source, ScriptNode target, IRelationshi source: source, target: target, timestamp: 0, - creationHeight: _mappings.Get(nameof(T2SEdge.CreationHeight)).Deserialize(relationship.Properties), - value: _mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties), - outputIndex: _mappings.Get(nameof(T2SEdge.Vout)).Deserialize(relationship.Properties), - spentHeight: _mappings.Get(nameof(T2SEdge.SpentHeight)).Deserialize(relationship.Properties)); + creationHeight: Mappings.Get(nameof(T2SEdge.CreationHeight)).Deserialize(relationship.Properties), + value: Mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties), + outputIndex: Mappings.Get(nameof(T2SEdge.Vout)).Deserialize(relationship.Properties), + spentHeight: Mappings.Get(nameof(T2SEdge.SpentHeight)).Deserialize(relationship.Properties)); } public override string GetQuery(string filename) { throw new NotImplementedException(); } + + public override string[] GetSchemaConfiguration() + { + return + [ + $"CREATE INDEX utxo_spending_idx IF NOT EXISTS " + + $"FOR ()-[r:{T2SEdge.Kind.Relation}]-() " + + $"ON (r.{nameof(T2SEdge.CreationHeight)}, r.{nameof(T2SEdge.SpentHeight)})" + ]; + } } \ No newline at end of file diff --git a/EBA/Graph/Bitcoin/Strategies/T2TEdgeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/T2TEdgeStrategy.cs index 562c9c2d..4ac1dc06 100644 --- a/EBA/Graph/Bitcoin/Strategies/T2TEdgeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/T2TEdgeStrategy.cs @@ -10,7 +10,7 @@ public class T2TEdgeStrategy(EdgeKind kind, bool serializeCompressed) $"{kind.Target}", serializeCompressed) { - private static readonly PropertyMapping[] _mappings = + public static readonly PropertyMapping[] Mappings = [ Factory.SourceId(TxNodeStrategy.IdSpace, e => e.Source.Txid), Factory.TargetId(TxNodeStrategy.IdSpace, e => e.Target.Txid), @@ -21,7 +21,7 @@ public class T2TEdgeStrategy(EdgeKind kind, bool serializeCompressed) public override string GetCsvHeader() { - return _mappings.GetCsvHeader(); + return Mappings.GetCsvHeader(); } public override string GetCsvRow(IGraphElement edge) @@ -31,7 +31,7 @@ public override string GetCsvRow(IGraphElement edge) public static string GetCsv(T2TEdge edge) { - return _mappings.GetCsv(edge); + return Mappings.GetCsv(edge); } public static T2TEdge Deserialize(TxNode source, TxNode target, IRelationship relationship) @@ -40,8 +40,8 @@ public static T2TEdge Deserialize(TxNode source, TxNode target, IRelationship re source: source, target: target, timestamp: 0, - blockHeight: _mappings.Get(Factory.HeightProperty.Name).Deserialize(relationship.Properties), - value: _mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties), + blockHeight: Mappings.Get(Factory.HeightProperty.Name).Deserialize(relationship.Properties), + value: Mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties), type: Enum.Parse(relationship.Type)); } diff --git a/EBA/Graph/Bitcoin/Strategies/TxNodeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/TxNodeStrategy.cs index e05f7df5..979b3129 100644 --- a/EBA/Graph/Bitcoin/Strategies/TxNodeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/TxNodeStrategy.cs @@ -14,7 +14,7 @@ public class TxNodeStrategy(bool serializeCompressed) public static string IdSpace { get; } = TxNode.Kind.ToString(); private const TxNode v = null!; - private static readonly PropertyMapping[] _mappings = + public static readonly PropertyMapping[] Mappings = [ PropertyMappingFactory.Txid(n => n.Txid, p => p.GetIdFieldCsvHeader(IdSpace)), new(nameof(v.Version), FieldType.Long, n => n.Version), @@ -27,11 +27,11 @@ public class TxNodeStrategy(bool serializeCompressed) ]; private static readonly Dictionary> _mappingsDict = - _mappings.ToDictionary(m => m.Property.Name, m => m); + Mappings.ToDictionary(m => m.Property.Name, m => m); public override string GetCsvHeader() { - return _mappings.GetCsvHeader(); + return Mappings.GetCsvHeader(); } public override string GetCsvRow(IGraphElement component) @@ -41,7 +41,7 @@ public override string GetCsvRow(IGraphElement component) public static string GetCsv(TxNode node) { - return _mappings.GetCsv(node); + return Mappings.GetCsv(node); } public static TxNode Deserialize( @@ -96,4 +96,18 @@ public override string GetQuery(string filename) return builder.ToString();*/ throw new NotImplementedException(); } + + public override string[] GetSchemaConfiguration() + { + var txidName = PropertyMappingFactory.Txid(n => n.Txid).Property.Name; + return + [ + $"CREATE CONSTRAINT {TxNode.Kind}_{txidName}_Unique " + + $"IF NOT EXISTS " + + $"FOR (v:{TxNode.Kind}) REQUIRE v.{txidName} IS UNIQUE", + + $"CREATE INDEX tx_txid_index IF NOT EXISTS " + + $"FOR (t:{TxNode.Kind}) ON (t.{nameof(TxNode.Txid)})" + ]; + } } \ No newline at end of file diff --git a/EBA/Graph/Db/Neo4jDb/StrategyBase.cs b/EBA/Graph/Db/Neo4jDb/StrategyBase.cs index 783725d5..cf59e35d 100644 --- a/EBA/Graph/Db/Neo4jDb/StrategyBase.cs +++ b/EBA/Graph/Db/Neo4jDb/StrategyBase.cs @@ -61,6 +61,16 @@ await GetStreamWriter(filename).WriteLineAsync( public abstract string GetQuery(string filename); + public virtual string[] GetSchemaConfiguration() + { + return []; + } + + public virtual string[] GetSeedingCommands() + { + return []; + } + public void Dispose() { Dispose(true); From 002473360979b3111fd344c2f5f32f7a5bf8fbe6 Mon Sep 17 00:00:00 2001 From: Vahid Date: Tue, 31 Mar 2026 21:24:41 -0400 Subject: [PATCH 06/30] Add a method to deserialize a list --- EBA/Graph/Bitcoin/Factories/NodeFactory.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/EBA/Graph/Bitcoin/Factories/NodeFactory.cs b/EBA/Graph/Bitcoin/Factories/NodeFactory.cs index 4f42c242..b0511295 100644 --- a/EBA/Graph/Bitcoin/Factories/NodeFactory.cs +++ b/EBA/Graph/Bitcoin/Factories/NodeFactory.cs @@ -61,4 +61,19 @@ public static bool TryCreate( $"Unexpected node type, labels: {string.Join(',', node.Labels)}"); } } + + public static bool TryCreate(List records, out List nodes, string nodeVar = "n") where T: Model.INode + { + nodes = []; + foreach (var record in records) + { + TryCreate(record[nodeVar].As(), out var createdNode); + if (createdNode is not T) + return false; + + nodes.Add((T)createdNode); + } + + return true; + } } From 153fdb5d8d813c714a9e76dbd324710831289993 Mon Sep 17 00:00:00 2001 From: Vahid Date: Tue, 31 Mar 2026 21:25:26 -0400 Subject: [PATCH 07/30] refactor --- EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs | 2 +- EBA/Graph/Bitcoin/Strategies/ScriptNodeStrategy.cs | 2 +- EBA/Graph/Bitcoin/Strategies/T2SEdgeStrategy.cs | 2 +- EBA/Graph/Bitcoin/Strategies/TxNodeStrategy.cs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs index 7cef47e2..e6f9d688 100644 --- a/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs @@ -189,7 +189,7 @@ public override string GetQuery(string filename) throw new NotImplementedException(); } - public override string[] GetSchemaConfiguration() + public override string[] GetSchemaConfigs() { var heightName = PropertyMappingFactory.HeightProperty.Name; return diff --git a/EBA/Graph/Bitcoin/Strategies/ScriptNodeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/ScriptNodeStrategy.cs index b6afad65..a2eef04e 100644 --- a/EBA/Graph/Bitcoin/Strategies/ScriptNodeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/ScriptNodeStrategy.cs @@ -105,7 +105,7 @@ public override string GetQuery(string filename) throw new NotImplementedException(); } - public override string[] GetSchemaConfiguration() + public override string[] GetSchemaConfigs() { return [ diff --git a/EBA/Graph/Bitcoin/Strategies/T2SEdgeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/T2SEdgeStrategy.cs index 595561cd..b0394061 100644 --- a/EBA/Graph/Bitcoin/Strategies/T2SEdgeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/T2SEdgeStrategy.cs @@ -52,7 +52,7 @@ public override string GetQuery(string filename) throw new NotImplementedException(); } - public override string[] GetSchemaConfiguration() + public override string[] GetSchemaConfigs() { return [ diff --git a/EBA/Graph/Bitcoin/Strategies/TxNodeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/TxNodeStrategy.cs index 979b3129..d72087e9 100644 --- a/EBA/Graph/Bitcoin/Strategies/TxNodeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/TxNodeStrategy.cs @@ -97,7 +97,7 @@ public override string GetQuery(string filename) throw new NotImplementedException(); } - public override string[] GetSchemaConfiguration() + public override string[] GetSchemaConfigs() { var txidName = PropertyMappingFactory.Txid(n => n.Txid).Property.Name; return From 5943e05b3e6f4818ffeea54bfc22e67b781f56c0 Mon Sep 17 00:00:00 2001 From: Vahid Date: Tue, 31 Mar 2026 21:26:12 -0400 Subject: [PATCH 08/30] Add bulk import finalizer --- EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs | 100 +++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs diff --git a/EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs b/EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs new file mode 100644 index 00000000..5b663f05 --- /dev/null +++ b/EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs @@ -0,0 +1,100 @@ +using EBA.Graph.Bitcoin.Factories; + +namespace EBA.Graph.Bitcoin; + +public class PostBulkImportFinalizer( + Options options, + IGraphDb graphDb, + ILogger logger) +{ + private readonly Options _options = options; + private readonly IGraphDb _graphDb = graphDb; + private readonly ILogger _logger = logger; + + public async Task Finalize(CancellationToken ct) + { + await AddSchemaAndSeeding(ct); + + await SetSupplyAmount(ct); + + await _graphDb.SetUTxOSpentHeight(ct); + } + + private async Task AddSchemaAndSeeding(CancellationToken ct) + { + var schemas = new List(); + var seedingCommands = new List(); + foreach (var nodeStrategy in _graphDb.StrategyFactory.NodeStrategies) + { + schemas.AddRange(nodeStrategy.Value.GetSchemaConfigs()); + seedingCommands.AddRange(nodeStrategy.Value.GetSeedingCommands()); + } + foreach (var edgeStrategy in _graphDb.StrategyFactory.EdgeStrategies) + { + schemas.AddRange(edgeStrategy.Value.GetSchemaConfigs()); + seedingCommands.AddRange(edgeStrategy.Value.GetSeedingCommands()); + } + + _logger.LogInformation("Executing {count:n0} schema configuration commands.", schemas.Count); + await _graphDb.ExecuteWriteQueryAsync(schemas, ct); + + _logger.LogInformation("Executing {count:n0} seeding commands.", seedingCommands.Count); + await _graphDb.ExecuteWriteQueryAsync(seedingCommands, ct); + } + + private async Task SetSupplyAmount(CancellationToken ct) + { + _logger.LogInformation("Fetching blocks from graph database."); + var nodeVar = "n"; + var records = await _graphDb.GetNodesAsync(NodeKind.Block, CancellationToken.None, nodeVariable: nodeVar); + _logger.LogInformation("Retrieved {count:n0} records from graph database. Creating block nodes.", records.Count); + + NodeFactory.TryCreate(records, out var blockNodes, nodeVar); + + var blocks = new SortedList(); + foreach (var block in blockNodes) + blocks.Add(block.BlockMetadata.Height, block); + + if (blocks.First().Value.BlockMetadata.Height != 0) + { + throw new InvalidOperationException( + $"The first block in the graph has height " + + $"{blocks.First().Value.BlockMetadata.Height:,}, " + + $"expected 0."); + } + + if (blocks.Last().Value.BlockMetadata.Height != blocks.Count - 1) + { + throw new InvalidOperationException( + $"This operation requires a continues set of blocks, there are missing blocks"); + } + + blocks[0].BlockMetadata.TotalSupply = blocks[0].BlockMetadata.MintedBitcoins; + blocks[0].BlockMetadata.TotalSupplyNominal = blocks[0].BlockMetadata.TotalSupply; + for (var i = 1; i < blocks.Count; i++) + { + blocks[i].BlockMetadata.TotalSupply = + blocks[i - 1].BlockMetadata.TotalSupply + + blocks[i].TripletTypeValueSum[C2TEdge.Kind] - + blocks[i].BlockMetadata.ProvablyUnspendableBitcoins; + + blocks[i].BlockMetadata.TotalSupplyNominal = + blocks[i - 1].BlockMetadata.TotalSupplyNominal + blocks[i].BlockMetadata.MintedBitcoins; + } + + var updates = blocks.Values.Select(b => new Dictionary + { + [nameof(BlockMetadata.Height)] = b.BlockMetadata.Height, + [nameof(BlockMetadata.TotalSupply)] = b.BlockMetadata.TotalSupply, + [nameof(BlockMetadata.TotalSupplyNominal)] = b.BlockMetadata.TotalSupplyNominal, + }).ToList(); + + _logger.LogInformation("Pushing {count:n0} block updates to graph database.", updates.Count); + await _graphDb.BulkUpdateNodePropertiesAsync( + NodeKind.Block, + nameof(BlockMetadata.Height), + updates, + ct); + _logger.LogInformation("Completed pushing block updates."); + } +} From 588ccaa9fa5715e6b8c24ab333b79129b30dc656 Mon Sep 17 00:00:00 2001 From: Vahid Date: Tue, 31 Mar 2026 21:27:24 -0400 Subject: [PATCH 09/30] remove postprocess method --- EBA/Graph/Bitcoin/PostProcess.cs | 76 -------------------------------- 1 file changed, 76 deletions(-) delete mode 100644 EBA/Graph/Bitcoin/PostProcess.cs diff --git a/EBA/Graph/Bitcoin/PostProcess.cs b/EBA/Graph/Bitcoin/PostProcess.cs deleted file mode 100644 index 78e04022..00000000 --- a/EBA/Graph/Bitcoin/PostProcess.cs +++ /dev/null @@ -1,76 +0,0 @@ -using EBA.Graph.Bitcoin.Factories; - -namespace EBA.Graph.Bitcoin; - -public class PostProcess -{ - private readonly Options _options; - private readonly IGraphDb _graphDb; - private readonly ILogger _logger; - - public PostProcess(Options options, IGraphDb graphDb, ILogger logger) - { - _options = options; - _graphDb = graphDb; - _logger = logger; - } - - public async Task Run() - { - var blocks = new SortedList(); - var nodeVar = "n"; - - _logger.LogInformation("Fetching blocks from graph database."); - var records = await _graphDb.GetNodesAsync(NodeKind.Block, CancellationToken.None, nodeVariable: nodeVar); - _logger.LogInformation("Retrieved {count:n0} records from graph database. Creating block nodes.", records.Count); - - foreach (var record in records) - { - NodeFactory.TryCreate(record[nodeVar].As(), out var blockNode); - var block = (BlockNode)blockNode; - blocks.Add(block.BlockMetadata.Height, block); - } - - if (blocks.First().Value.BlockMetadata.Height != 0) - { - throw new InvalidOperationException( - $"The first block in the graph has height " + - $"{blocks.First().Value.BlockMetadata.Height:,}, " + - $"expected 0."); - } - - if (blocks.Last().Value.BlockMetadata.Height != blocks.Count - 1) - { - throw new InvalidOperationException( - $"This operation requires a continues set of blocks, there are missing blocks"); - } - - blocks[0].BlockMetadata.TotalSupply = blocks[0].BlockMetadata.MintedBitcoins; - blocks[0].BlockMetadata.TotalSupplyNominal = blocks[0].BlockMetadata.TotalSupply; - for (var i = 1; i < blocks.Count; i++) - { - blocks[i].BlockMetadata.TotalSupply = - blocks[i - 1].BlockMetadata.TotalSupply + - blocks[i].TripletTypeValueSum[C2TEdge.Kind] - - blocks[i].BlockMetadata.ProvablyUnspendableBitcoins; - - blocks[i].BlockMetadata.TotalSupplyNominal = - blocks[i - 1].BlockMetadata.TotalSupplyNominal + blocks[i].BlockMetadata.MintedBitcoins; - } - - var updates = blocks.Values.Select(b => new Dictionary - { - [nameof(BlockMetadata.Height)] = b.BlockMetadata.Height, - [nameof(BlockMetadata.TotalSupply)] = b.BlockMetadata.TotalSupply, - [nameof(BlockMetadata.TotalSupplyNominal)] = b.BlockMetadata.TotalSupplyNominal, - }).ToList(); - - _logger.LogInformation("Pushing {count:n0} block updates to graph database.", updates.Count); - await _graphDb.BulkUpdateNodePropertiesAsync( - NodeKind.Block, - nameof(BlockMetadata.Height), - updates, - CancellationToken.None); - _logger.LogInformation("Completed pushing block updates."); - } -} From d39379e1d2476a9ab9ff7e2eba8dcf98ebcff93a Mon Sep 17 00:00:00 2001 From: Vahid Date: Tue, 31 Mar 2026 21:28:13 -0400 Subject: [PATCH 10/30] Add new methods to neo4jdb. --- EBA/Graph/Db/Neo4jDb/Neo4jDb.cs | 179 +++++++++++++++++++++++++-- EBA/Graph/Db/Neo4jDb/StrategyBase.cs | 2 +- EBA/Graph/IGraphAgent.cs | 2 +- 3 files changed, 168 insertions(+), 15 deletions(-) diff --git a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs index ec4ed524..35070319 100644 --- a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs +++ b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs @@ -1,9 +1,13 @@ -namespace EBA.Graph.Db.Neo4jDb; +using EBA.Utilities; + +namespace EBA.Graph.Db.Neo4jDb; public class Neo4jDb : IGraphDb where T : GraphBase { private readonly Options _options; private readonly IDriver _driver; + + public IStrategyFactory StrategyFactory { get { return _strategyFactory; } } private readonly IStrategyFactory _strategyFactory; private List _batches = []; @@ -223,6 +227,156 @@ private async Task> DeserializeBatchesAsync() _options.Neo4j.BatchesFilename); } + public async Task ExecuteWriteQueryAsync(List queries, CancellationToken ct) + { + await VerifyConnectivityAsync(ct); + + await using var session = _driver.AsyncSession( + x => x.WithDefaultAccessMode(AccessMode.Write)); + + foreach (var query in queries) + { + ct.ThrowIfCancellationRequested(); + + _logger.LogInformation("Executing write query: {schema}", query); + + var summary = await session.ExecuteWriteAsync(async tx => + { + var cursor = await tx.RunAsync(query); + return await cursor.ConsumeAsync(); + }); + + _logger.LogInformation("Finished executing write query; {counters}", summary.Counters); + } + } + + public async Task TEST_NUPL(BlockNode blockNode, Dictionary ohlcv) + { + await VerifyConnectivityAsync(CancellationToken.None); + await using var session = _driver.AsyncSession(x => x.WithDefaultAccessMode(AccessMode.Read)); + + var readTx = await session.BeginTransactionAsync(); + + var cursor = await readTx.RunAsync( + $"MATCH (tx)-[r:{T2SEdge.Kind.Relation}]->(script) " + + $"WHERE r.{nameof(T2SEdge.CreationHeight)} <= {blockNode.BlockMetadata.Height} " + + $"AND r.{nameof(T2SEdge.SpentHeight)} > {blockNode.BlockMetadata.Height} " + + $"AND script.{nameof(ScriptNode.ScriptType)} <> '{ScriptType.NullData}' " + + $"RETURN r"); + + decimal realizedCap = 0; + + while (await cursor.FetchAsync()) + { + var record = cursor.Current; + + var r = record["r"].As(); + var creationHeight = (long)r.Properties[nameof(T2SEdge.CreationHeight)]; + var value = (long)r.Properties[nameof(T2SEdge.Value)]; + + if (ohlcv.TryGetValue(creationHeight, out var blockOHLCV)) + { + realizedCap += blockOHLCV.GetFiatValue(value); + } + } + + blockNode.BlockMetadata.RealizedCap = realizedCap; + } + + // TODO: this method should not be here; + // it is specific to the Bitcoin graph model + // and this class should be kept agnostic to the graph model. + public async Task SetUTxOSpentHeight(CancellationToken ct) + { + _logger.LogInformation( + "Starting to find UTxO spending, and set their status to spent by setting {prop} on {r} edges.", + nameof(S2TEdge.SpentHeight), T2SEdge.Kind.Relation); + + await VerifyConnectivityAsync(ct); + + var batch = new List>(); + var batchIndex = 0; + long totalProcessed = 0; + + await using var readSession = _driver.AsyncSession( + x => x.WithDefaultAccessMode(AccessMode.Read)); + + var readTx = await readSession.BeginTransactionAsync(); + + var cursor = await readTx.RunAsync( + $"MATCH ()-[r:{S2TEdge.Kind.Relation}]->() " + + $"WHERE r.{nameof(S2TEdge.Generated)} = false " + + $"RETURN " + + $"r.{nameof(S2TEdge.Txid)} AS txid, " + + $"r.{nameof(S2TEdge.Vout)} AS vout, " + + $"r.{nameof(S2TEdge.SpentHeight)} AS height"); + + while (await cursor.FetchAsync()) + { + ct.ThrowIfCancellationRequested(); + + var record = cursor.Current; + var height = record["height"].As(); + + batch.Add(new Dictionary + { + ["txid"] = record["txid"].As(), + ["vout"] = record["vout"].As(), + ["height"] = height + }); + + if (batch.Count >= _options.Neo4j.MaxEntitiesPerBatch) + { + await CommitUTxOSpentHeight(batch, batchIndex++, ct); + totalProcessed += batch.Count; + batch = []; + } + } + + if (batch.Count > 0) + { + await CommitUTxOSpentHeight(batch, batchIndex++, ct); + totalProcessed += batch.Count; + } + + await readTx.CommitAsync(); + + _logger.LogInformation( + "Completed setting SpentHeight on Credits for {total:n0} Redeems edges.", + totalProcessed); + } + + private async Task CommitUTxOSpentHeight( + IReadOnlyList> batch, + int batchIndex, + CancellationToken ct) + { + ct.ThrowIfCancellationRequested(); + + _logger.LogInformation( + "Committing batch {batch} with {count:n0} records.", + batchIndex, batch.Count); + + await using var writeSession = _driver.AsyncSession( + x => x.WithDefaultAccessMode(AccessMode.Write)); + + var summary = await writeSession.ExecuteWriteAsync(async x => + { + var cursor = await x.RunAsync( + $"UNWIND $batch AS row " + + $"MATCH (t:{TxNode.Kind} {{{nameof(TxNode.Txid)}: row.txid}})-[c:{T2SEdge.Kind.Relation}]->() " + + $"WHERE c.{nameof(S2TEdge.Vout)} = row.vout " + + $"SET c.{nameof(S2TEdge.SpentHeight)} = row.height", + new Dictionary { ["batch"] = batch }); + + return await cursor.ConsumeAsync(); + }); + + _logger.LogInformation( + "Committing batch {batch} finished; set {props:n0} properties on {relation} edges.", + batchIndex, summary.Counters.PropertiesSet, T2SEdge.Kind.Relation); + } + public void Dispose() { Dispose(true); @@ -252,11 +406,10 @@ public async Task BulkUpdateNodePropertiesAsync( await VerifyConnectivityAsync(ct); + var sampleProps = updates[0].Keys.Where(k => k != idProperty); var setClause = string.Join( - ", ", - updates[0].Keys - .Where(k => k != idProperty) - .Select(k => $"n.{k} = row.{k}")); + ", ", + sampleProps.Select(k => $"n.{k} = row.{k}")); // using toString() on the id property to match the string-typed // :ID column created by neo4j-admin import. @@ -265,8 +418,8 @@ public async Task BulkUpdateNodePropertiesAsync( $"MATCH (n:{label} {{{idProperty}: toString(row.{idProperty})}}) " + $"SET {setClause}"; - var chunkIndex = 0; - foreach (var chunk in updates.Chunk(_options.Neo4j.MaxEntitiesPerBatch)) + var batchIndex = 0; + foreach (var batch in updates.Chunk(_options.Neo4j.MaxEntitiesPerBatch)) { ct.ThrowIfCancellationRequested(); @@ -275,20 +428,20 @@ public async Task BulkUpdateNodePropertiesAsync( { var cursor = await tx.RunAsync( query, - new Dictionary { ["batch"] = chunk }); + new Dictionary { ["batch"] = batch }); return await cursor.ConsumeAsync(); }); var counters = summary.Counters; if (counters.PropertiesSet == 0) - _logger.LogWarning("Chunk {chunk}: 0 properties set (MATCH may have found no nodes).", - chunkIndex); + _logger.LogWarning("Batch {batch}: 0 properties set (MATCH may have found no nodes).", + batchIndex); else - _logger.LogInformation("Chunk {chunk}: set {props:n0} properties on nodes.", - chunkIndex, counters.PropertiesSet); + _logger.LogInformation("Batch {batch}: set {props:n0} properties on nodes.", + batchIndex, counters.PropertiesSet); - chunkIndex++; + batchIndex++; } _logger.LogInformation( diff --git a/EBA/Graph/Db/Neo4jDb/StrategyBase.cs b/EBA/Graph/Db/Neo4jDb/StrategyBase.cs index cf59e35d..1e032906 100644 --- a/EBA/Graph/Db/Neo4jDb/StrategyBase.cs +++ b/EBA/Graph/Db/Neo4jDb/StrategyBase.cs @@ -61,7 +61,7 @@ await GetStreamWriter(filename).WriteLineAsync( public abstract string GetQuery(string filename); - public virtual string[] GetSchemaConfiguration() + public virtual string[] GetSchemaConfigs() { return []; } diff --git a/EBA/Graph/IGraphAgent.cs b/EBA/Graph/IGraphAgent.cs index ed3cd3c9..af448256 100644 --- a/EBA/Graph/IGraphAgent.cs +++ b/EBA/Graph/IGraphAgent.cs @@ -4,5 +4,5 @@ public interface IGraphAgent where T : GraphBase { public Task SampleAsync(CancellationToken ct); public Task SerializeAsync(T g, CancellationToken ct); - public Task PostProcessAsync(CancellationToken ct); + public Task PostBullkImportFinalizeAsync(CancellationToken ct); } \ No newline at end of file From f08d6524cdcd9ea97250bfe090151e8c3ecccd42 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sat, 4 Apr 2026 10:16:42 -0400 Subject: [PATCH 11/30] Draft adding off-chain info to blocks and edges --- .../Bitcoin/BitcoinOrchestrator.cs | 29 +++++++- .../Bitcoin/ChainModel/BlockMetadata.cs | 25 +++++++ EBA/Blockchains/Bitcoin/Context.cs | 9 +++ .../Bitcoin/GraphModel/BlockGraph.cs | 19 ++++-- EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs | 4 ++ EBA/CLI/CLI.cs | 58 +++++++++++++--- EBA/CLI/Config/BitcoinAugmentorOptions.cs | 6 ++ EBA/CLI/Config/BitcoinOptions.cs | 1 + EBA/CLI/Config/BitcoinTraverseOptions.cs | 2 + EBA/CLI/OptionsBinder.cs | 15 +++- EBA/Graph/Bitcoin/BitcoinGraphAgent.cs | 12 +++- EBA/Graph/Bitcoin/OffChain/Augmentor.cs | 49 +++++++++++++ EBA/Graph/Db/IGraphDb.cs | 13 +++- EBA/Graph/Db/Neo4jDb/Neo4jDb.cs | 68 +++++++++++++++++-- EBA/Orchestrator.cs | 15 +++- 15 files changed, 295 insertions(+), 30 deletions(-) create mode 100644 EBA/Blockchains/Bitcoin/Context.cs create mode 100644 EBA/CLI/Config/BitcoinAugmentorOptions.cs create mode 100644 EBA/Graph/Bitcoin/OffChain/Augmentor.cs diff --git a/EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs b/EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs index ec72d04d..b40f36e2 100644 --- a/EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs +++ b/EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs @@ -1,4 +1,5 @@ using EBA.Blockchains.Bitcoin.Utilities; +using EBA.Utilities; namespace EBA.Blockchains.Bitcoin; @@ -21,7 +22,7 @@ public BitcoinOrchestrator( } public async Task TraverseAsync( - Options options, + Options options, CancellationToken cT) { var chainInfo = await _agent.AssertChainAsync(cT); @@ -38,6 +39,30 @@ public async Task TraverseAsync( return; } + if (options.Bitcoin.Traverse.BlockMarketMappingFilename != null) + { + _logger.LogInformation( + "Parsing market mapping from file {f}.", + options.Bitcoin.Traverse.BlockMarketMappingFilename); + + if (OHLCV.TryParseFile(options.Bitcoin.Traverse.BlockMarketMappingFilename, out var mappings)) + { + BitcoinContext.OHLCVCache = new ConcurrentDictionary(mappings); + + _logger.LogInformation( + "Read market mapping for {n:n0} blocks from file {f}.", + new Dictionary().Count, + options.Bitcoin.Traverse.BlockMarketMappingFilename); + } + else + { + _logger.LogWarning( + "Failed to read market mapping from file {f}. " + + "Proceeding without market mapping.", + options.Bitcoin.Traverse.BlockMarketMappingFilename); + } + } + cT.ThrowIfCancellationRequested(); var stopwatch = new Stopwatch(); @@ -59,7 +84,7 @@ public async Task TraverseAsync( throw; } - } + } public async Task DeDupAsync( Options options, diff --git a/EBA/Blockchains/Bitcoin/ChainModel/BlockMetadata.cs b/EBA/Blockchains/Bitcoin/ChainModel/BlockMetadata.cs index e8d8b552..e8dbc38a 100644 --- a/EBA/Blockchains/Bitcoin/ChainModel/BlockMetadata.cs +++ b/EBA/Blockchains/Bitcoin/ChainModel/BlockMetadata.cs @@ -90,4 +90,29 @@ public long ProvablyUnspendableBitcoins public long? TotalSupply { set; get; } = null; public long? TotalSupplyNominal { set; get; } = null; + + public decimal? RealizedCap { set; get; } = null; + public decimal? MarketCap + { + get + { + if (TotalSupply == null || Ohlcv == null) + return null; + + return Ohlcv.GetFiatValue((long)TotalSupply); + } + } + + public decimal? NUPL + { + get + { + if (RealizedCap == null || MarketCap == null) + return null; + + return (MarketCap - RealizedCap) / MarketCap; + } + } + + public OHLCV? Ohlcv { set; get; } = null; } diff --git a/EBA/Blockchains/Bitcoin/Context.cs b/EBA/Blockchains/Bitcoin/Context.cs new file mode 100644 index 00000000..736a6262 --- /dev/null +++ b/EBA/Blockchains/Bitcoin/Context.cs @@ -0,0 +1,9 @@ +using EBA.Utilities; + +namespace EBA.Blockchains.Bitcoin +{ + public static class BitcoinContext + { + public static ConcurrentDictionary OHLCVCache = new(); + } +} diff --git a/EBA/Blockchains/Bitcoin/GraphModel/BlockGraph.cs b/EBA/Blockchains/Bitcoin/GraphModel/BlockGraph.cs index 8720efea..5a3ab4ab 100644 --- a/EBA/Blockchains/Bitcoin/GraphModel/BlockGraph.cs +++ b/EBA/Blockchains/Bitcoin/GraphModel/BlockGraph.cs @@ -1,4 +1,6 @@ -namespace EBA.Blockchains.Bitcoin.GraphModel; +using EBA.Utilities; + +namespace EBA.Blockchains.Bitcoin.GraphModel; public class BlockGraph : BitcoinGraph, IEquatable { @@ -74,6 +76,7 @@ public void BuildGraph(CancellationToken ct) var v = _coinbaseTxGraph.TxNode; var t = Timestamp; var h = Block.Height; + BitcoinContext.OHLCVCache.TryGetValue(h, out var ohlcv); Parallel.ForEach(_txGraphsQueue, #if (DEBUG) @@ -84,7 +87,7 @@ public void BuildGraph(CancellationToken ct) if (ct.IsCancellationRequested) { state.Stop(); return; } - AddTxGraphToBlockGraph(txGraph); + AddTxGraphToBlockGraph(txGraph, ohlcv); if (ct.IsCancellationRequested) { state.Stop(); return; } @@ -95,7 +98,9 @@ public void BuildGraph(CancellationToken ct) long miningReward = 0; foreach (var u in _coinbaseTxGraph.Outputs) { - TryGetOrAddEdge(new T2SEdge(v, new ScriptNode(u.ScriptPubKey), t, h, output: u), out var edge); + TryGetOrAddEdge( + new T2SEdge(v, new ScriptNode(u.ScriptPubKey), t, h, u, ohlcv?.GetFiatValue(u.Value)), + out T2SEdge _); Block.ProfileCreatedOutput(u); miningReward += u.Value; @@ -110,7 +115,7 @@ public void BuildGraph(CancellationToken ct) BlockNode.TripletTypeValueSum = _edgeLabelValueSum.ToDictionary(x => x.Key, x => x.Value); } - private void AddTxGraphToBlockGraph(TxGraph txGraph) + private void AddTxGraphToBlockGraph(TxGraph txGraph, OHLCV? ohlcv) { var v = txGraph.TxNode; var h = Block.Height; @@ -123,14 +128,16 @@ private void AddTxGraphToBlockGraph(TxGraph txGraph) { TryGetOrAddEdge( new S2TEdge(new ScriptNode(u.PrevOut.ScriptPubKey), v, t, h, u), - out var edge); + out S2TEdge _); Block.ProfileSpentOutput(u); } foreach (var u in txGraph.Outputs) { - TryGetOrAddEdge(new T2SEdge(v, new ScriptNode(u.ScriptPubKey), t, h, output: u), out var edge); + TryGetOrAddEdge( + new T2SEdge(v, new ScriptNode(u.ScriptPubKey), t, h, u, ohlcv?.GetFiatValue(u.Value)), + out T2SEdge _); Block.ProfileCreatedOutput(u); } diff --git a/EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs b/EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs index 894e058c..5ca0b80a 100644 --- a/EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs +++ b/EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs @@ -8,6 +8,8 @@ public class T2SEdge : Edge public long SpentHeight { get; } + public decimal? FiatValue { get; } = null; + public long CreationHeight => BlockHeight; public T2SEdge( @@ -16,6 +18,7 @@ public T2SEdge( uint timestamp, long creationHeight, Output output, + decimal? fiatValue = null, long spentHeight = long.MaxValue) : base( source: source, @@ -27,6 +30,7 @@ public T2SEdge( { Vout = output.N; SpentHeight = spentHeight; + FiatValue = fiatValue; } public T2SEdge( diff --git a/EBA/CLI/CLI.cs b/EBA/CLI/CLI.cs index e225f0ec..ad9be5dd 100644 --- a/EBA/CLI/CLI.cs +++ b/EBA/CLI/CLI.cs @@ -21,7 +21,8 @@ public Cli( Func bitcoinMapMarketHandlerAsync, Func bitcoinAddressStatsHandlerAsync, Func bitcoinImportCypherQueriesAsync, - Func bitcoinPostProcessGraphHandlerAsync, + Func bitcoinFinalizeImportHandlerAsync, + Func bitcoinAugmentHandlerAsync, Action exceptionHandler) { _exceptionHandler = exceptionHandler; @@ -89,7 +90,8 @@ e is NotSupportedException || mapMarketHandlerAsync: bitcoinMapMarketHandlerAsync, addressStatsHandlerAsync: bitcoinAddressStatsHandlerAsync, importCypherQueriesAsync: bitcoinImportCypherQueriesAsync, - postProcessGraphHandlerAsync: bitcoinPostProcessGraphHandlerAsync) + finalizeImportHandlerAsync: bitcoinFinalizeImportHandlerAsync, + bitcoinAugmentHandlerAsync: bitcoinAugmentHandlerAsync) }; for (int i = 0; i < _rootCmd.Options.Count; i++) @@ -125,7 +127,8 @@ private Command GetBitcoinCmd( Func mapMarketHandlerAsync, Func addressStatsHandlerAsync, Func importCypherQueriesAsync, - Func postProcessGraphHandlerAsync) + Func finalizeImportHandlerAsync, + Func bitcoinAugmentHandlerAsync) { var cmd = new Command( name: "bitcoin", @@ -138,7 +141,8 @@ private Command GetBitcoinCmd( GetBitcoinMapMarketCmd(defaultOptions, mapMarketHandlerAsync), GetBitcoinSampleCmd(defaultOptions, sampleHandlerAsync), GetBitcoinAddressStatsCmd(defaultOptions, addressStatsHandlerAsync), - GetPostProcessGraphCmd(defaultOptions, postProcessGraphHandlerAsync) + GetFinalizeImportCmd(defaultOptions, finalizeImportHandlerAsync), + GetBitcoinAugmentCmd(defaultOptions, bitcoinAugmentHandlerAsync) }; return cmd; } @@ -234,6 +238,11 @@ private Command GetBitcoinTraverseCmd(Options defaultOptions, Func("--map-block-market") + { + Required = false + }; + var cmd = new Command( name: "traverse", description: "Traverses the blockchain in the defined range collecting the set metrics.") @@ -247,7 +256,8 @@ private Command GetBitcoinTraverseCmd(Options defaultOptions, Func handlerAsync) + private Command GetFinalizeImportCmd(Options defaultOptions, Func handlerAsync) { - var cmd = new Command(name: "post-process-graph"); + var cmd = new Command(name: "finalize-import"); cmd.SetAction(async (parseResult, cancellationToken) => { var options = OptionsBinder.Build( @@ -681,4 +692,35 @@ private Command GetBitcoinAddressStatsCmd(Options defaultOptions, Func handlerAsync) + { + var ohlcvFilenameOption = new Option("--ohlcv-filename") + { + Description = "A TSV file containing block metadata (height, median time) mapped to aggregated OHLCV market data." + }; + var cmd = new Command( + name: "augment", + description: "Augments the graph with additional features, such as NUPL, using the output of the map-market command and other similar files.") + { + ohlcvFilenameOption + }; + cmd.SetAction(async (parseResult, cancellationToken) => + { + var options = OptionsBinder.Build( + parseResult, + workingDirOption: _workingDirOption, + statusFilenameOption: _statusFilenameOption, + augmentroOhlcvOption: ohlcvFilenameOption); + try + { + await handlerAsync(options); + } + catch (Exception e) + { + _exceptionHandler(e, parseResult); + } + }); + return cmd; + } } diff --git a/EBA/CLI/Config/BitcoinAugmentorOptions.cs b/EBA/CLI/Config/BitcoinAugmentorOptions.cs new file mode 100644 index 00000000..dd219784 --- /dev/null +++ b/EBA/CLI/Config/BitcoinAugmentorOptions.cs @@ -0,0 +1,6 @@ +namespace EBA.CLI.Config; + +public class BitcoinAugmentorOptions +{ + public string BlockOhlcvMappedFilename { init; get; } +} diff --git a/EBA/CLI/Config/BitcoinOptions.cs b/EBA/CLI/Config/BitcoinOptions.cs index 485e4ee3..a9672ed4 100644 --- a/EBA/CLI/Config/BitcoinOptions.cs +++ b/EBA/CLI/Config/BitcoinOptions.cs @@ -9,4 +9,5 @@ public BitcoinOptions() : this(DateTimeOffset.Now.ToUnixTimeSeconds()) { } public BitcoinDedupOptions Dedup { init; get; } = new(); public BitcoinGraphSampleOptions GraphSample { init; get; } = new(); public BitcoinMapMarketOptions MapMarket { init; get; } = new(); + public BitcoinAugmentorOptions Augmentor { init; get; } = new(); } diff --git a/EBA/CLI/Config/BitcoinTraverseOptions.cs b/EBA/CLI/Config/BitcoinTraverseOptions.cs index f0a982b5..f283bb8f 100644 --- a/EBA/CLI/Config/BitcoinTraverseOptions.cs +++ b/EBA/CLI/Config/BitcoinTraverseOptions.cs @@ -85,6 +85,8 @@ public int Granularity public bool SkipGraphSerialization { init; get; } = false; + public string? BlockMarketMappingFilename { init; get; } = null; + public ResilienceStrategyOptions HttpClientResilienceStrategy { init; get; } = new(); public ResilienceStrategyOptions BitcoinAgentResilienceStrategy { init; get; } = new() diff --git a/EBA/CLI/OptionsBinder.cs b/EBA/CLI/OptionsBinder.cs index 82db1e32..77ae3d2f 100644 --- a/EBA/CLI/OptionsBinder.cs +++ b/EBA/CLI/OptionsBinder.cs @@ -29,7 +29,9 @@ public static Options Build( Option? sortedTxNodeFilenameOption = null, Option? sortedScriptNodeFilenameOption = null, Option? marketDataFilenameOption = null, - Option? outputFilenameOption = null) + Option? outputFilenameOption = null, + Option? blockMarketMappingOption = null, + Option? augmentroOhlcvOption = null) { if (statusFilenameOption != null && c.GetResult(statusFilenameOption) is not null) { @@ -56,7 +58,8 @@ public static Options Build( MaxBlocksInBuffer = GetValue(defs.Bitcoin.Traverse.MaxBlocksInBuffer, maxBlocksInBufferOption, c), TxoFilename = GetValue(defs.Bitcoin.Traverse.TxoFilename, txoFilenameOption, c, (x) => { return Path.Join(wd, Path.GetFileName(x)); }), TrackTxo = GetValue(defs.Bitcoin.Traverse.TrackTxo, trackTxoOption, c), - SkipGraphSerialization = GetValue(defs.Bitcoin.Traverse.SkipGraphSerialization, skipGraphSerializationOption, c) + SkipGraphSerialization = GetValue(defs.Bitcoin.Traverse.SkipGraphSerialization, skipGraphSerializationOption, c), + BlockMarketMappingFilename = GetValue(defs.Bitcoin.Traverse.BlockMarketMappingFilename, blockMarketMappingOption, c, (x) => { return Path.Join(wd, Path.GetFileName(x)); }), }; var traversalAlgorithm = GetValue(defs.Bitcoin.GraphSample.TraversalAlgorithm, graphSampleMethodOption, c); @@ -103,12 +106,18 @@ public static Options Build( BlockOhlcvMappedFilename = GetValue(defs.Bitcoin.MapMarket.BlockOhlcvMappedFilename, outputFilenameOption, c, (x) => { return Path.Join(wd, Path.GetFileName(x)); }) }; + var bitcoinAugmentorOps = new BitcoinAugmentorOptions() + { + BlockOhlcvMappedFilename = GetValue(defs.Bitcoin.Augmentor.BlockOhlcvMappedFilename, augmentroOhlcvOption, c, (x) => { return Path.Join(wd, Path.GetFileName(x)); }), + }; + var bitcoinOps = new BitcoinOptions(defs.Timestamp) { Traverse = bitcoinTraverseOptions, Dedup = bitcoinDedupOps, GraphSample = gsample, - MapMarket = bitcoinMapMarketOps + MapMarket = bitcoinMapMarketOps, + Augmentor = bitcoinAugmentorOps, }; var options = new Options() diff --git a/EBA/Graph/Bitcoin/BitcoinGraphAgent.cs b/EBA/Graph/Bitcoin/BitcoinGraphAgent.cs index 70662327..dc16eb8d 100644 --- a/EBA/Graph/Bitcoin/BitcoinGraphAgent.cs +++ b/EBA/Graph/Bitcoin/BitcoinGraphAgent.cs @@ -46,10 +46,16 @@ public async Task SerializeAsync(BitcoinGraph g, CancellationToken ct) await _db.SerializeAsync(g, ct); } - public async Task PostProcessAsync(CancellationToken ct) + public async Task PostBullkImportFinalizeAsync(CancellationToken ct) { - var postProcess = new PostProcess(_options, _db, _logger); - await postProcess.Run(); + var finalizer = new PostBulkImportFinalizer(_options, _db, _logger); + await finalizer.Finalize(ct); + } + + public async Task AddMarketData(CancellationToken ct) + { + var augmentor = new OffChain.Augmentor(_options, _db, _logger); + await augmentor.SetRealizedCap(ct); } public void Dispose() diff --git a/EBA/Graph/Bitcoin/OffChain/Augmentor.cs b/EBA/Graph/Bitcoin/OffChain/Augmentor.cs new file mode 100644 index 00000000..d6a53423 --- /dev/null +++ b/EBA/Graph/Bitcoin/OffChain/Augmentor.cs @@ -0,0 +1,49 @@ +using EBA.Graph.Bitcoin.Factories; +using EBA.Utilities; + +namespace EBA.Graph.Bitcoin.OffChain; + +public class Augmentor(Options options, IGraphDb graphDb, ILogger logger) +{ + private readonly Options _options = options; + private readonly IGraphDb _graphDb = graphDb; + private readonly ILogger _logger = logger; + + public async Task SetRealizedCap(CancellationToken ct) + { + OHLCV.TryParseFile(_options.Bitcoin.Augmentor.BlockOhlcvMappedFilename, out var mappings); + + _logger.LogInformation("Fetching block nodes."); + var blockRecords = await _graphDb.GetNodesAsync(NodeKind.Block, ct, nodeVariable: "b"); + _logger.LogInformation("Fetched {count:n0} block nodes.", blockRecords.Count); + + var blocks = new SortedDictionary(); + if (NodeFactory.TryCreate(blockRecords, out var blockNodes, nodeVar: "b")) + { + foreach (var blockNode in blockNodes) + blocks.Add(blockNode.BlockMetadata.Height, blockNode); + } + else + { + throw new Exception("Invalid node types received from database"); + } + + _logger.LogInformation("Setting realized cap for {count:n0} block nodes.", blocks.Count); + await _graphDb.SetRealizedCap(blocks, mappings, CancellationToken.None); + + _logger.LogInformation("Saving realized cap for {count:n0} block nodes.", blocks.Count); + var updates = blocks.Values + .Select(b => new Dictionary + { + [nameof(BlockMetadata.Height)] = b.BlockMetadata.Height, + [nameof(BlockMetadata.RealizedCap)] = b.BlockMetadata.RealizedCap + }) + .ToList(); + + await _graphDb.BulkUpdateNodePropertiesAsync( + NodeKind.Block, + nameof(BlockMetadata.Height), + updates, + ct); + } +} diff --git a/EBA/Graph/Db/IGraphDb.cs b/EBA/Graph/Db/IGraphDb.cs index 79092ce4..314ddcb0 100644 --- a/EBA/Graph/Db/IGraphDb.cs +++ b/EBA/Graph/Db/IGraphDb.cs @@ -1,7 +1,12 @@ -namespace EBA.Graph.Db; +using EBA.Graph.Db.Neo4jDb; +using EBA.Utilities; + +namespace EBA.Graph.Db; public interface IGraphDb : IDisposable where T : GraphBase { + public IStrategyFactory StrategyFactory { get; } + public Task VerifyConnectivityAsync(CancellationToken ct); public Task SerializeAsync(T graph, CancellationToken ct); @@ -47,4 +52,10 @@ public Task> GetNodesAsync( CancellationToken ct, string nodeVariable = "n", int? count = null); + + public Task SetUTxOSpentHeight(CancellationToken ct); + + public Task ExecuteWriteQueryAsync(List schemas, CancellationToken ct); + + public Task SetRealizedCap(SortedDictionary blockNodes, Dictionary ohlcv, CancellationToken ct); } \ No newline at end of file diff --git a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs index 35070319..6e52edb9 100644 --- a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs +++ b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs @@ -250,7 +250,7 @@ public async Task ExecuteWriteQueryAsync(List queries, CancellationToken } } - public async Task TEST_NUPL(BlockNode blockNode, Dictionary ohlcv) + public async Task SetRealizedCap(BlockNode blockNode, Dictionary ohlcv, CancellationToken ct) { await VerifyConnectivityAsync(CancellationToken.None); await using var session = _driver.AsyncSession(x => x.WithDefaultAccessMode(AccessMode.Read)); @@ -268,9 +268,7 @@ public async Task TEST_NUPL(BlockNode blockNode, Dictionary ohlcv) while (await cursor.FetchAsync()) { - var record = cursor.Current; - - var r = record["r"].As(); + var r = cursor.Current["r"].As(); var creationHeight = (long)r.Properties[nameof(T2SEdge.CreationHeight)]; var value = (long)r.Properties[nameof(T2SEdge.Value)]; @@ -283,6 +281,68 @@ public async Task TEST_NUPL(BlockNode blockNode, Dictionary ohlcv) blockNode.BlockMetadata.RealizedCap = realizedCap; } + public async Task SetRealizedCap( + SortedDictionary blockNodes, + Dictionary ohlcv, + CancellationToken ct) + { + if (blockNodes.Count == 0) + return; + + var sortedHeights = blockNodes.Keys.ToArray(); + var minHeight = sortedHeights[0]; + var maxHeight = sortedHeights[^1]; + + foreach (var block in blockNodes.Values) + block.BlockMetadata.RealizedCap = 0; + + await VerifyConnectivityAsync(ct); + await using var session = _driver.AsyncSession(x => x.WithDefaultAccessMode(AccessMode.Read)); + + var readTx = await session.BeginTransactionAsync(); + + var cursor = await readTx.RunAsync( + $"MATCH (tx)-[r:{T2SEdge.Kind.Relation}]->(script) " + + $"WHERE r.{nameof(T2SEdge.CreationHeight)} <= {maxHeight} " + + $"AND r.{nameof(T2SEdge.SpentHeight)} > {minHeight} " + + $"AND script.{nameof(ScriptNode.ScriptType)} <> '{ScriptType.NullData}' " + + $"RETURN " + + $"r.{nameof(T2SEdge.CreationHeight)} AS creationHeight, " + + $"r.{nameof(T2SEdge.SpentHeight)} AS spentHeight, " + + $"r.{nameof(T2SEdge.Value)} AS value"); + + var counter = 0; + + while (await cursor.FetchAsync()) + { + ct.ThrowIfCancellationRequested(); + + var creationHeight = cursor.Current["creationHeight"].As(); + var spentHeight = cursor.Current["spentHeight"].As(); + var value = cursor.Current["value"].As(); + + if (!ohlcv.TryGetValue(creationHeight, out var blockOHLCV)) + continue; + + var fiatValue = blockOHLCV.GetFiatValue(value); + + foreach (var x in blockNodes) + { + if (x.Key < creationHeight) + continue; + + if (x.Key >= spentHeight) + break; + + x.Value.BlockMetadata.RealizedCap += fiatValue; + } + + counter++; + if (counter % 10000 == 0) + _logger.LogInformation("Processed {count:n0} edges for realized cap calculation.", counter); + } + } + // TODO: this method should not be here; // it is specific to the Bitcoin graph model // and this class should be kept agnostic to the graph model. diff --git a/EBA/Orchestrator.cs b/EBA/Orchestrator.cs index 5b7e31bb..67df0c46 100644 --- a/EBA/Orchestrator.cs +++ b/EBA/Orchestrator.cs @@ -24,7 +24,8 @@ public Orchestrator(CancellationToken cancelationToken) bitcoinAddressStatsHandlerAsync: BitcoinAddressStatsAsync, bitcoinImportCypherQueriesAsync: BitcoinImportCypherQueriesAsync, bitcoinMapMarketHandlerAsync: BitcoinMarketMapAsync, - bitcoinPostProcessGraphHandlerAsync: BitcoinPostProcessGraph, + bitcoinFinalizeImportHandlerAsync: BitcoinFinalizeImport, + bitcoinAugmentHandlerAsync: BitcoinAugmentGraphAsync, exceptionHandler: (e, _) => { if (_logger != null) @@ -88,13 +89,21 @@ private async Task BitcoinImportCypherQueriesAsync(Options options) graphDb.ReportQueries(); } - private async Task BitcoinPostProcessGraph(Options options) + private async Task BitcoinFinalizeImport(Options options) { var host = await SetupAndGetHostAsync(options); await JsonSerializer.SerializeAsync(options, options.StatusFile, _cT); var bitcoinGraphAgent = host.Services.GetRequiredService(); - await bitcoinGraphAgent.PostProcessAsync(_cT); + await bitcoinGraphAgent.PostBullkImportFinalizeAsync(_cT); + } + + private async Task BitcoinAugmentGraphAsync(Options options) + { + var host = await SetupAndGetHostAsync(options); + await JsonSerializer.SerializeAsync(options, options.StatusFile, _cT); + var bitcoinGraphAgent = host.Services.GetRequiredService(); + await bitcoinGraphAgent.AddMarketData(_cT); } private async Task BitcoinSampleGraphAsync(Options options) From 8e6e7d2328cb961b6dff872899494d2d15badee5 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sat, 4 Apr 2026 11:25:57 -0400 Subject: [PATCH 12/30] use an extension method to improve search performance. --- EBA/Graph/Bitcoin/BitcoinGraphAgent.cs | 2 +- EBA/Graph/Bitcoin/OffChain/Augmentor.cs | 21 ++++++-- EBA/Graph/Db/Neo4jDb/Neo4jDb.cs | 72 +++++++++++++------------ EBA/Utilities/Extensions.cs | 19 +++++++ 4 files changed, 77 insertions(+), 37 deletions(-) create mode 100644 EBA/Utilities/Extensions.cs diff --git a/EBA/Graph/Bitcoin/BitcoinGraphAgent.cs b/EBA/Graph/Bitcoin/BitcoinGraphAgent.cs index dc16eb8d..9179278a 100644 --- a/EBA/Graph/Bitcoin/BitcoinGraphAgent.cs +++ b/EBA/Graph/Bitcoin/BitcoinGraphAgent.cs @@ -55,7 +55,7 @@ public async Task PostBullkImportFinalizeAsync(CancellationToken ct) public async Task AddMarketData(CancellationToken ct) { var augmentor = new OffChain.Augmentor(_options, _db, _logger); - await augmentor.SetRealizedCap(ct); + await augmentor.AddMarketData(ct); } public void Dispose() diff --git a/EBA/Graph/Bitcoin/OffChain/Augmentor.cs b/EBA/Graph/Bitcoin/OffChain/Augmentor.cs index d6a53423..08b1fcbd 100644 --- a/EBA/Graph/Bitcoin/OffChain/Augmentor.cs +++ b/EBA/Graph/Bitcoin/OffChain/Augmentor.cs @@ -9,10 +9,17 @@ public class Augmentor(Options options, IGraphDb graphDb, ILogger< private readonly IGraphDb _graphDb = graphDb; private readonly ILogger _logger = logger; - public async Task SetRealizedCap(CancellationToken ct) + public async Task AddMarketData(CancellationToken ct) { - OHLCV.TryParseFile(_options.Bitcoin.Augmentor.BlockOhlcvMappedFilename, out var mappings); + OHLCV.TryParseFile(_options.Bitcoin.Augmentor.BlockOhlcvMappedFilename, out var blockOHLCVMapping); + var blockNodes = await GetBlockNodes(ct); + + await SetRealizedCap(blockNodes, blockOHLCVMapping, ct); + } + + private async Task> GetBlockNodes(CancellationToken ct) + { _logger.LogInformation("Fetching block nodes."); var blockRecords = await _graphDb.GetNodesAsync(NodeKind.Block, ct, nodeVariable: "b"); _logger.LogInformation("Fetched {count:n0} block nodes.", blockRecords.Count); @@ -28,8 +35,16 @@ public async Task SetRealizedCap(CancellationToken ct) throw new Exception("Invalid node types received from database"); } + return blocks; + } + + private async Task SetRealizedCap( + SortedDictionary blocks, + Dictionary blockOHLCVMapping, + CancellationToken ct) + { _logger.LogInformation("Setting realized cap for {count:n0} block nodes.", blocks.Count); - await _graphDb.SetRealizedCap(blocks, mappings, CancellationToken.None); + await _graphDb.SetRealizedCap(blocks, blockOHLCVMapping, CancellationToken.None); _logger.LogInformation("Saving realized cap for {count:n0} block nodes.", blocks.Count); var updates = blocks.Values diff --git a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs index 6e52edb9..98fb1af4 100644 --- a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs +++ b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs @@ -281,6 +281,9 @@ public async Task SetRealizedCap(BlockNode blockNode, Dictionary oh blockNode.BlockMetadata.RealizedCap = realizedCap; } + // TODO: this method should not be here; + // it is specific to the Bitcoin graph model + // and this class should be kept agnostic to the graph model. public async Task SetRealizedCap( SortedDictionary blockNodes, Dictionary ohlcv, @@ -299,48 +302,51 @@ public async Task SetRealizedCap( await VerifyConnectivityAsync(ct); await using var session = _driver.AsyncSession(x => x.WithDefaultAccessMode(AccessMode.Read)); - var readTx = await session.BeginTransactionAsync(); - - var cursor = await readTx.RunAsync( - $"MATCH (tx)-[r:{T2SEdge.Kind.Relation}]->(script) " + - $"WHERE r.{nameof(T2SEdge.CreationHeight)} <= {maxHeight} " + - $"AND r.{nameof(T2SEdge.SpentHeight)} > {minHeight} " + - $"AND script.{nameof(ScriptNode.ScriptType)} <> '{ScriptType.NullData}' " + - $"RETURN " + - $"r.{nameof(T2SEdge.CreationHeight)} AS creationHeight, " + - $"r.{nameof(T2SEdge.SpentHeight)} AS spentHeight, " + - $"r.{nameof(T2SEdge.Value)} AS value"); - - var counter = 0; + var processedEdgeCount = 0; + var skippedEdgeCounter = 0; - while (await cursor.FetchAsync()) + await session.ExecuteReadAsync(async tx => { - ct.ThrowIfCancellationRequested(); - - var creationHeight = cursor.Current["creationHeight"].As(); - var spentHeight = cursor.Current["spentHeight"].As(); - var value = cursor.Current["value"].As(); - - if (!ohlcv.TryGetValue(creationHeight, out var blockOHLCV)) - continue; + var cursor = await tx.RunAsync( + $"MATCH (tx)-[r:{T2SEdge.Kind.Relation}]->(script) " + + $"WHERE " + + $"r.{nameof(T2SEdge.CreationHeight)} <= {maxHeight} " + + $"AND " + + $"r.{nameof(T2SEdge.SpentHeight)} > {minHeight} " + + $"AND " + + $"script.{nameof(ScriptNode.ScriptType)} <> '{ScriptType.NullData}' " + + $"RETURN " + + $"r.{nameof(T2SEdge.CreationHeight)} AS creationHeight, " + + $"r.{nameof(T2SEdge.SpentHeight)} AS spentHeight, " + + $"r.{nameof(T2SEdge.Value)} AS value"); + + while (await cursor.FetchAsync()) + { + ct.ThrowIfCancellationRequested(); - var fiatValue = blockOHLCV.GetFiatValue(value); + var creationHeight = cursor.Current["creationHeight"].As(); + var spentHeight = cursor.Current["spentHeight"].As(); + var value = cursor.Current["value"].As(); - foreach (var x in blockNodes) - { - if (x.Key < creationHeight) + if (!ohlcv.TryGetValue(creationHeight, out var blockOHLCV)) + { + skippedEdgeCounter++; continue; + } - if (x.Key >= spentHeight) - break; + var fiatValue = blockOHLCV.GetFiatValue(value); + foreach (var h in sortedHeights.GetViewBetween(creationHeight, spentHeight)) + blockNodes[h].BlockMetadata.RealizedCap += fiatValue; - x.Value.BlockMetadata.RealizedCap += fiatValue; + processedEdgeCount++; + if (processedEdgeCount % 1000 == 0) + _logger.LogInformation("Processed {count:n0} edges for realized cap calculation.", processedEdgeCount); } + }); - counter++; - if (counter % 10000 == 0) - _logger.LogInformation("Processed {count:n0} edges for realized cap calculation.", counter); - } + _logger.LogInformation( + "Processed {p:n0} edges and skipped {s:n0} edges (due to missing OHLCV data at their creation height.)", + processedEdgeCount, skippedEdgeCounter); } // TODO: this method should not be here; diff --git a/EBA/Utilities/Extensions.cs b/EBA/Utilities/Extensions.cs new file mode 100644 index 00000000..f5cc4f73 --- /dev/null +++ b/EBA/Utilities/Extensions.cs @@ -0,0 +1,19 @@ +namespace EBA.Utilities; + +public static class Extensions +{ + public static IEnumerable GetViewBetween( + this long[] sortedArray, + long lowerBound, + long upperBound) + { + int idx = Array.BinarySearch(sortedArray, lowerBound); + if (idx < 0) idx = ~idx; + + for (int i = idx; i < sortedArray.Length; i++) + { + if (sortedArray[i] >= upperBound) break; + yield return sortedArray[i]; + } + } +} From 2a9a86f8a0047e8d07e3a72789ff64dbf8877d34 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sat, 4 Apr 2026 12:00:41 -0400 Subject: [PATCH 13/30] refactor method name --- EBA/Blockchains/Bitcoin/ChainModel/BlockMetadata.cs | 2 ++ EBA/Graph/Bitcoin/OffChain/Augmentor.cs | 13 ++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/EBA/Blockchains/Bitcoin/ChainModel/BlockMetadata.cs b/EBA/Blockchains/Bitcoin/ChainModel/BlockMetadata.cs index e8dbc38a..838ef0f7 100644 --- a/EBA/Blockchains/Bitcoin/ChainModel/BlockMetadata.cs +++ b/EBA/Blockchains/Bitcoin/ChainModel/BlockMetadata.cs @@ -115,4 +115,6 @@ public decimal? NUPL } public OHLCV? Ohlcv { set; get; } = null; + + // TODO: add MVRV, NVT, Thermocap } diff --git a/EBA/Graph/Bitcoin/OffChain/Augmentor.cs b/EBA/Graph/Bitcoin/OffChain/Augmentor.cs index 08b1fcbd..cc13d1af 100644 --- a/EBA/Graph/Bitcoin/OffChain/Augmentor.cs +++ b/EBA/Graph/Bitcoin/OffChain/Augmentor.cs @@ -15,7 +15,11 @@ public async Task AddMarketData(CancellationToken ct) var blockNodes = await GetBlockNodes(ct); - await SetRealizedCap(blockNodes, blockOHLCVMapping, ct); + foreach (var block in blockNodes) + if (blockOHLCVMapping.TryGetValue(block.Key, out var ohlcv)) + block.Value.BlockMetadata.Ohlcv = ohlcv; + + await ComputeBlockValuationMetrics(blockNodes, blockOHLCVMapping, ct); } private async Task> GetBlockNodes(CancellationToken ct) @@ -38,7 +42,7 @@ private async Task> GetBlockNodes(Cancellation return blocks; } - private async Task SetRealizedCap( + private async Task ComputeBlockValuationMetrics( SortedDictionary blocks, Dictionary blockOHLCVMapping, CancellationToken ct) @@ -51,7 +55,10 @@ private async Task SetRealizedCap( .Select(b => new Dictionary { [nameof(BlockMetadata.Height)] = b.BlockMetadata.Height, - [nameof(BlockMetadata.RealizedCap)] = b.BlockMetadata.RealizedCap + [nameof(BlockMetadata.RealizedCap)] = b.BlockMetadata.RealizedCap, + [nameof(BlockMetadata.MarketCap)] = b.BlockMetadata.MarketCap, + [nameof(BlockMetadata.NUPL)] = b.BlockMetadata.NUPL + // TODO: save ohlcv }) .ToList(); From 717910766e4d15604bebf9cce94f78a2cb189a70 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sat, 4 Apr 2026 12:13:35 -0400 Subject: [PATCH 14/30] refactor --- EBA/Graph/Bitcoin/BitcoinGraphAgent.cs | 4 ++-- .../Bitcoin/OffChain/{Augmentor.cs => EconomicAugmentor.cs} | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename EBA/Graph/Bitcoin/OffChain/{Augmentor.cs => EconomicAugmentor.cs} (93%) diff --git a/EBA/Graph/Bitcoin/BitcoinGraphAgent.cs b/EBA/Graph/Bitcoin/BitcoinGraphAgent.cs index 9179278a..b93ef3eb 100644 --- a/EBA/Graph/Bitcoin/BitcoinGraphAgent.cs +++ b/EBA/Graph/Bitcoin/BitcoinGraphAgent.cs @@ -54,8 +54,8 @@ public async Task PostBullkImportFinalizeAsync(CancellationToken ct) public async Task AddMarketData(CancellationToken ct) { - var augmentor = new OffChain.Augmentor(_options, _db, _logger); - await augmentor.AddMarketData(ct); + var augmentor = new OffChain.EconomicAugmentor(_options, _db, _logger); + await augmentor.SetBlockMarketIndicators(ct); } public void Dispose() diff --git a/EBA/Graph/Bitcoin/OffChain/Augmentor.cs b/EBA/Graph/Bitcoin/OffChain/EconomicAugmentor.cs similarity index 93% rename from EBA/Graph/Bitcoin/OffChain/Augmentor.cs rename to EBA/Graph/Bitcoin/OffChain/EconomicAugmentor.cs index cc13d1af..871b6c84 100644 --- a/EBA/Graph/Bitcoin/OffChain/Augmentor.cs +++ b/EBA/Graph/Bitcoin/OffChain/EconomicAugmentor.cs @@ -3,13 +3,13 @@ namespace EBA.Graph.Bitcoin.OffChain; -public class Augmentor(Options options, IGraphDb graphDb, ILogger logger) +public class EconomicAugmentor(Options options, IGraphDb graphDb, ILogger logger) { private readonly Options _options = options; private readonly IGraphDb _graphDb = graphDb; private readonly ILogger _logger = logger; - public async Task AddMarketData(CancellationToken ct) + public async Task SetBlockMarketIndicators(CancellationToken ct) { OHLCV.TryParseFile(_options.Bitcoin.Augmentor.BlockOhlcvMappedFilename, out var blockOHLCVMapping); From ab290650fa2cc8c2ddadbfe7783af315953d8db2 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sat, 4 Apr 2026 17:59:56 -0400 Subject: [PATCH 15/30] improve ohlcv serialization --- .../Bitcoin/OffChain/EconomicAugmentor.cs | 25 ++++++++----- .../Bitcoin/Strategies/BlockNodeStrategy.cs | 23 ++++++++++++ .../Strategies/PropertyMappingExtensions.cs | 10 ++++++ .../Strategies/PropertyMappingFactory.cs | 36 +++++++++++++++++++ EBA/Graph/Db/Neo4jDb/Neo4jDb.cs | 2 +- 5 files changed, 86 insertions(+), 10 deletions(-) diff --git a/EBA/Graph/Bitcoin/OffChain/EconomicAugmentor.cs b/EBA/Graph/Bitcoin/OffChain/EconomicAugmentor.cs index 871b6c84..c2a09b84 100644 --- a/EBA/Graph/Bitcoin/OffChain/EconomicAugmentor.cs +++ b/EBA/Graph/Bitcoin/OffChain/EconomicAugmentor.cs @@ -1,4 +1,5 @@ using EBA.Graph.Bitcoin.Factories; +using EBA.Graph.Bitcoin.Strategies; using EBA.Utilities; namespace EBA.Graph.Bitcoin.OffChain; @@ -11,7 +12,20 @@ public class EconomicAugmentor(Options options, IGraphDb graphDb, public async Task SetBlockMarketIndicators(CancellationToken ct) { - OHLCV.TryParseFile(_options.Bitcoin.Augmentor.BlockOhlcvMappedFilename, out var blockOHLCVMapping); + if (!OHLCV.TryParseFile(_options.Bitcoin.Augmentor.BlockOhlcvMappedFilename, out var blockOHLCVMapping)) + { + _logger.LogError( + "Failed to parse OHLCV data from file: {filename}", + _options.Bitcoin.Augmentor.BlockOhlcvMappedFilename); + return; + } + else + { + _logger.LogInformation( + "Successfully parsed OHLCV data for {count:n0} blocks from file: {filename}", + blockOHLCVMapping.Count, + _options.Bitcoin.Augmentor.BlockOhlcvMappedFilename); + } var blockNodes = await GetBlockNodes(ct); @@ -52,14 +66,7 @@ private async Task ComputeBlockValuationMetrics( _logger.LogInformation("Saving realized cap for {count:n0} block nodes.", blocks.Count); var updates = blocks.Values - .Select(b => new Dictionary - { - [nameof(BlockMetadata.Height)] = b.BlockMetadata.Height, - [nameof(BlockMetadata.RealizedCap)] = b.BlockMetadata.RealizedCap, - [nameof(BlockMetadata.MarketCap)] = b.BlockMetadata.MarketCap, - [nameof(BlockMetadata.NUPL)] = b.BlockMetadata.NUPL - // TODO: save ohlcv - }) + .Select(b => BlockNodeStrategy.EconomicMappings.ToDictionary(b)) .ToList(); await _graphDb.BulkUpdateNodePropertiesAsync( diff --git a/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs index e6f9d688..164cc407 100644 --- a/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs @@ -10,6 +10,27 @@ public class BlockNodeStrategy(bool serializeCompressed) public static string IdSpace { get; } = BlockNode.Kind.ToString(); private const Block v = null!; + + private static readonly PropertyMapping[] _economicMappings = + [ + new(nameof(v.RealizedCap), FieldType.Double, n => (double?)n.BlockMetadata.RealizedCap), + new(nameof(v.MarketCap), FieldType.Double, n => (double?)n.BlockMetadata.MarketCap), + new(nameof(v.NUPL), FieldType.Double, n => (double?)n.BlockMetadata.NUPL), + + .. PropertyMappingFactory.OHLCV(n => n.BlockMetadata.Ohlcv), + ]; + + /// + /// The subset of mappings used by + /// to persist valuation metrics back to the graph. + /// Includes as the match key. + /// + public static PropertyMapping[] EconomicMappings { get; } = + [ + PropertyMappingFactory.Height(n => n.BlockMetadata.Height), + .. _economicMappings, + ]; + public static readonly PropertyMapping[] Mappings = [ new("", FieldType.String, x=>x.BlockMetadata.Height, p => p.GetIdFieldCsvHeader(IdSpace.ToString())), @@ -64,6 +85,8 @@ .. PropertyMappingFactory.DictionaryToColumns( .. PropertyMappingFactory.DictionaryToColumns( nameof(BlockNode.TripletTypeValueSum), Schema.EdgeKinds, n => n.TripletTypeValueSum), + .. _economicMappings, + new(":LABEL", FieldType.String, _ => BlockNode.Kind, _ => ":LABEL"), ]; diff --git a/EBA/Graph/Bitcoin/Strategies/PropertyMappingExtensions.cs b/EBA/Graph/Bitcoin/Strategies/PropertyMappingExtensions.cs index 65ecb230..def589f0 100644 --- a/EBA/Graph/Bitcoin/Strategies/PropertyMappingExtensions.cs +++ b/EBA/Graph/Bitcoin/Strategies/PropertyMappingExtensions.cs @@ -27,4 +27,14 @@ public static PropertyMapping Get(this PropertyMapping[] mappings, stri throw new KeyNotFoundException($"No mapping found for property '{propertyName}'."); } + + public static Dictionary ToDictionary( + this PropertyMapping[] mappings, + T source) + { + var dict = new Dictionary(mappings.Length); + foreach (var m in mappings) + dict[m.Property.Name] = m.SerializeValue(source); + return dict; + } } diff --git a/EBA/Graph/Bitcoin/Strategies/PropertyMappingFactory.cs b/EBA/Graph/Bitcoin/Strategies/PropertyMappingFactory.cs index 02717cd9..a3236f4a 100644 --- a/EBA/Graph/Bitcoin/Strategies/PropertyMappingFactory.cs +++ b/EBA/Graph/Bitcoin/Strategies/PropertyMappingFactory.cs @@ -197,4 +197,40 @@ private static string EdgeKindToPropertyName(EdgeKind edgeKind) // Make sure to keep EdgeKind string representation compatible with neo4j header requirements. return $"{edgeKind.Source}_{edgeKind.Relation}_{edgeKind.Target}"; } + + + public static PropertyMapping[] OHLCV( + Func getOhlcv, + string prefix = "OHLCV") + { + OHLCV o = null!; + return + [ + new($"{prefix}.{nameof(o.Open)}", FieldType.Double, s => (double?)(getOhlcv(s)?.Open)), + new($"{prefix}.{nameof(o.High)}", FieldType.Double, s => (double?)(getOhlcv(s)?.High)), + new($"{prefix}.{nameof(o.Low)}", FieldType.Double, s => (double?)(getOhlcv(s)?.Low)), + new($"{prefix}.{nameof(o.Close)}", FieldType.Double, s => (double?)(getOhlcv(s)?.Close)), + new($"{prefix}.{nameof(o.VWAP)}", FieldType.Double, s => (double?)(getOhlcv(s)?.VWAP)), + new($"{prefix}.{nameof(o.OHLC4)}", FieldType.Double, s => (double?)(getOhlcv(s)?.OHLC4)), + new($"{prefix}.{nameof(o.Volume)}", FieldType.Long, s => getOhlcv(s)?.Volume), + ]; + } + + public static OHLCV? ReadOHLCV( + IReadOnlyDictionary properties, + string prefix = "OHLCV") + { + OHLCV o = null!; + if (!properties.TryGetValue($"{prefix}.{nameof(o.Open)}", out _)) + return null; + + return new OHLCV( + timestamp: 0, + open: (decimal)(double)properties[$"{prefix}.{nameof(o.Open)}"], + high: (decimal)(double)properties[$"{prefix}.{nameof(o.High)}"], + low: (decimal)(double)properties[$"{prefix}.{nameof(o.Low)}"], + close: (decimal)(double)properties[$"{prefix}.{nameof(o.Close)}"], + volume: (long)properties[$"{prefix}.{nameof(o.Volume)}"], + vwap: (decimal)(double)properties[$"{prefix}.{nameof(o.VWAP)}"]); + } } \ No newline at end of file diff --git a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs index 98fb1af4..6b4d4693 100644 --- a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs +++ b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs @@ -345,7 +345,7 @@ await session.ExecuteReadAsync(async tx => }); _logger.LogInformation( - "Processed {p:n0} edges and skipped {s:n0} edges (due to missing OHLCV data at their creation height.)", + "Processed {p:n0} edges and skipped {s:n0} edges due to missing OHLCV data at their creation height.", processedEdgeCount, skippedEdgeCounter); } From 4b19f54032407fb1dea5b5f1ca24052b2a994e80 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sat, 4 Apr 2026 18:08:49 -0400 Subject: [PATCH 16/30] fix deserializing block economic features --- EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs | 10 ++++------ EBA/Graph/Bitcoin/Strategies/PropertyMappingFactory.cs | 6 +++--- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs index 164cc407..a4882260 100644 --- a/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs @@ -20,11 +20,6 @@ public class BlockNodeStrategy(bool serializeCompressed) .. PropertyMappingFactory.OHLCV(n => n.BlockMetadata.Ohlcv), ]; - /// - /// The subset of mappings used by - /// to persist valuation metrics back to the graph. - /// Includes as the match key. - /// public static PropertyMapping[] EconomicMappings { get; } = [ PropertyMappingFactory.Height(n => n.BlockMetadata.Height), @@ -148,7 +143,10 @@ public static BlockNode Deserialize( InputScriptTypeCount = PropertyMappingFactory.ReadScriptTypeCounts("Inputs", props), OutputScriptTypeCount = PropertyMappingFactory.ReadScriptTypeCounts("Outputs", props), InputScriptTypeValue = PropertyMappingFactory.ReadScriptTypeCounts("Inputs", props), - OutputScriptTypeValue = PropertyMappingFactory.ReadScriptTypeCounts("Outputs", props) + OutputScriptTypeValue = PropertyMappingFactory.ReadScriptTypeCounts("Outputs", props), + + RealizedCap = (decimal?)_mappingsDict[nameof(v.RealizedCap)].Deserialize(props), + Ohlcv = PropertyMappingFactory.ReadOHLCV(props), }; var blockNode = new BlockNode( diff --git a/EBA/Graph/Bitcoin/Strategies/PropertyMappingFactory.cs b/EBA/Graph/Bitcoin/Strategies/PropertyMappingFactory.cs index a3236f4a..664b972c 100644 --- a/EBA/Graph/Bitcoin/Strategies/PropertyMappingFactory.cs +++ b/EBA/Graph/Bitcoin/Strategies/PropertyMappingFactory.cs @@ -210,9 +210,9 @@ public static PropertyMapping[] OHLCV( new($"{prefix}.{nameof(o.High)}", FieldType.Double, s => (double?)(getOhlcv(s)?.High)), new($"{prefix}.{nameof(o.Low)}", FieldType.Double, s => (double?)(getOhlcv(s)?.Low)), new($"{prefix}.{nameof(o.Close)}", FieldType.Double, s => (double?)(getOhlcv(s)?.Close)), - new($"{prefix}.{nameof(o.VWAP)}", FieldType.Double, s => (double?)(getOhlcv(s)?.VWAP)), - new($"{prefix}.{nameof(o.OHLC4)}", FieldType.Double, s => (double?)(getOhlcv(s)?.OHLC4)), new($"{prefix}.{nameof(o.Volume)}", FieldType.Long, s => getOhlcv(s)?.Volume), + new($"{prefix}.{nameof(o.VWAP)}", FieldType.Double, s => (double?)(getOhlcv(s)?.VWAP)), + new($"{prefix}.{nameof(o.OHLC4)}", FieldType.Double, s => (double?)(getOhlcv(s)?.OHLC4)) ]; } @@ -220,7 +220,7 @@ public static PropertyMapping[] OHLCV( IReadOnlyDictionary properties, string prefix = "OHLCV") { - OHLCV o = null!; + OHLCV o; if (!properties.TryGetValue($"{prefix}.{nameof(o.Open)}", out _)) return null; From 973a8d2226adfe48d222602678e90099e6f4657c Mon Sep 17 00:00:00 2001 From: Vahid Date: Sat, 4 Apr 2026 20:44:14 -0400 Subject: [PATCH 17/30] add backticks. --- EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs | 10 +++++++++- EBA/Graph/Db/Neo4jDb/Neo4jDb.cs | 10 +++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs b/EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs index 5b663f05..99cc5608 100644 --- a/EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs +++ b/EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs @@ -12,7 +12,7 @@ public class PostBulkImportFinalizer( private readonly ILogger _logger = logger; public async Task Finalize(CancellationToken ct) - { + { await AddSchemaAndSeeding(ct); await SetSupplyAmount(ct); @@ -22,6 +22,8 @@ public async Task Finalize(CancellationToken ct) private async Task AddSchemaAndSeeding(CancellationToken ct) { + _logger.LogInformation("Setting schema and seeding data."); + var schemas = new List(); var seedingCommands = new List(); foreach (var nodeStrategy in _graphDb.StrategyFactory.NodeStrategies) @@ -40,10 +42,14 @@ private async Task AddSchemaAndSeeding(CancellationToken ct) _logger.LogInformation("Executing {count:n0} seeding commands.", seedingCommands.Count); await _graphDb.ExecuteWriteQueryAsync(seedingCommands, ct); + + _logger.LogInformation("Completed setting schema and seeding data."); } private async Task SetSupplyAmount(CancellationToken ct) { + _logger.LogInformation("Setting total supply feature."); + _logger.LogInformation("Fetching blocks from graph database."); var nodeVar = "n"; var records = await _graphDb.GetNodesAsync(NodeKind.Block, CancellationToken.None, nodeVariable: nodeVar); @@ -96,5 +102,7 @@ await _graphDb.BulkUpdateNodePropertiesAsync( updates, ct); _logger.LogInformation("Completed pushing block updates."); + + _logger.LogInformation("Completed setting total supply feature."); } } diff --git a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs index 6b4d4693..07d7f0a8 100644 --- a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs +++ b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs @@ -289,6 +289,8 @@ public async Task SetRealizedCap( Dictionary ohlcv, CancellationToken ct) { + // TODO: add more logging info here + if (blockNodes.Count == 0) return; @@ -339,7 +341,7 @@ await session.ExecuteReadAsync(async tx => blockNodes[h].BlockMetadata.RealizedCap += fiatValue; processedEdgeCount++; - if (processedEdgeCount % 1000 == 0) + if (processedEdgeCount % 10000 == 0) _logger.LogInformation("Processed {count:n0} edges for realized cap calculation.", processedEdgeCount); } }); @@ -369,6 +371,8 @@ public async Task SetUTxOSpentHeight(CancellationToken ct) var readTx = await readSession.BeginTransactionAsync(); + // TODO: this should also report the total number of edges reported. + var cursor = await readTx.RunAsync( $"MATCH ()-[r:{S2TEdge.Kind.Relation}]->() " + $"WHERE r.{nameof(S2TEdge.Generated)} = false " + @@ -475,13 +479,13 @@ public async Task BulkUpdateNodePropertiesAsync( var sampleProps = updates[0].Keys.Where(k => k != idProperty); var setClause = string.Join( ", ", - sampleProps.Select(k => $"n.{k} = row.{k}")); + sampleProps.Select(k => $"n.`{k}` = row.`{k}`")); // using toString() on the id property to match the string-typed // :ID column created by neo4j-admin import. var query = "UNWIND $batch AS row " + - $"MATCH (n:{label} {{{idProperty}: toString(row.{idProperty})}}) " + + $"MATCH (n:{label} {{`{idProperty}`: toString(row.`{idProperty}`)}}) " + $"SET {setClause}"; var batchIndex = 0; From a718d6f02c0476196fa75527550305998a69bff1 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sat, 4 Apr 2026 21:01:00 -0400 Subject: [PATCH 18/30] undo fiat value adding at traverse time --- EBA/Blockchains/Bitcoin/GraphModel/BlockGraph.cs | 9 ++++----- EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs | 4 ---- EBA/Graph/Db/Neo4jDb/Neo4jDb.cs | 2 ++ 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/EBA/Blockchains/Bitcoin/GraphModel/BlockGraph.cs b/EBA/Blockchains/Bitcoin/GraphModel/BlockGraph.cs index 5a3ab4ab..1b8be2ac 100644 --- a/EBA/Blockchains/Bitcoin/GraphModel/BlockGraph.cs +++ b/EBA/Blockchains/Bitcoin/GraphModel/BlockGraph.cs @@ -76,7 +76,6 @@ public void BuildGraph(CancellationToken ct) var v = _coinbaseTxGraph.TxNode; var t = Timestamp; var h = Block.Height; - BitcoinContext.OHLCVCache.TryGetValue(h, out var ohlcv); Parallel.ForEach(_txGraphsQueue, #if (DEBUG) @@ -87,7 +86,7 @@ public void BuildGraph(CancellationToken ct) if (ct.IsCancellationRequested) { state.Stop(); return; } - AddTxGraphToBlockGraph(txGraph, ohlcv); + AddTxGraphToBlockGraph(txGraph); if (ct.IsCancellationRequested) { state.Stop(); return; } @@ -99,7 +98,7 @@ public void BuildGraph(CancellationToken ct) foreach (var u in _coinbaseTxGraph.Outputs) { TryGetOrAddEdge( - new T2SEdge(v, new ScriptNode(u.ScriptPubKey), t, h, u, ohlcv?.GetFiatValue(u.Value)), + new T2SEdge(v, new ScriptNode(u.ScriptPubKey), t, h, u), out T2SEdge _); Block.ProfileCreatedOutput(u); @@ -115,7 +114,7 @@ public void BuildGraph(CancellationToken ct) BlockNode.TripletTypeValueSum = _edgeLabelValueSum.ToDictionary(x => x.Key, x => x.Value); } - private void AddTxGraphToBlockGraph(TxGraph txGraph, OHLCV? ohlcv) + private void AddTxGraphToBlockGraph(TxGraph txGraph) { var v = txGraph.TxNode; var h = Block.Height; @@ -136,7 +135,7 @@ private void AddTxGraphToBlockGraph(TxGraph txGraph, OHLCV? ohlcv) foreach (var u in txGraph.Outputs) { TryGetOrAddEdge( - new T2SEdge(v, new ScriptNode(u.ScriptPubKey), t, h, u, ohlcv?.GetFiatValue(u.Value)), + new T2SEdge(v, new ScriptNode(u.ScriptPubKey), t, h, u), out T2SEdge _); Block.ProfileCreatedOutput(u); diff --git a/EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs b/EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs index 5ca0b80a..894e058c 100644 --- a/EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs +++ b/EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs @@ -8,8 +8,6 @@ public class T2SEdge : Edge public long SpentHeight { get; } - public decimal? FiatValue { get; } = null; - public long CreationHeight => BlockHeight; public T2SEdge( @@ -18,7 +16,6 @@ public T2SEdge( uint timestamp, long creationHeight, Output output, - decimal? fiatValue = null, long spentHeight = long.MaxValue) : base( source: source, @@ -30,7 +27,6 @@ public T2SEdge( { Vout = output.N; SpentHeight = spentHeight; - FiatValue = fiatValue; } public T2SEdge( diff --git a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs index 07d7f0a8..c5d238ef 100644 --- a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs +++ b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs @@ -340,6 +340,8 @@ await session.ExecuteReadAsync(async tx => foreach (var h in sortedHeights.GetViewBetween(creationHeight, spentHeight)) blockNodes[h].BlockMetadata.RealizedCap += fiatValue; + // TODO: also log the number of skipped edges + processedEdgeCount++; if (processedEdgeCount % 10000 == 0) _logger.LogInformation("Processed {count:n0} edges for realized cap calculation.", processedEdgeCount); From cb05462c2bc4ae00acefeba7078abee05e1080c1 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sat, 4 Apr 2026 21:05:02 -0400 Subject: [PATCH 19/30] undo other fiat setting at traverse time --- .../Bitcoin/BitcoinOrchestrator.cs | 25 ------------------- EBA/Blockchains/Bitcoin/Context.cs | 9 ------- .../Bitcoin/GraphModel/BlockGraph.cs | 4 +-- EBA/CLI/Config/BitcoinTraverseOptions.cs | 2 -- EBA/CLI/OptionsBinder.cs | 3 +-- 5 files changed, 2 insertions(+), 41 deletions(-) delete mode 100644 EBA/Blockchains/Bitcoin/Context.cs diff --git a/EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs b/EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs index b40f36e2..c856f7a3 100644 --- a/EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs +++ b/EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs @@ -1,5 +1,4 @@ using EBA.Blockchains.Bitcoin.Utilities; -using EBA.Utilities; namespace EBA.Blockchains.Bitcoin; @@ -39,30 +38,6 @@ public async Task TraverseAsync( return; } - if (options.Bitcoin.Traverse.BlockMarketMappingFilename != null) - { - _logger.LogInformation( - "Parsing market mapping from file {f}.", - options.Bitcoin.Traverse.BlockMarketMappingFilename); - - if (OHLCV.TryParseFile(options.Bitcoin.Traverse.BlockMarketMappingFilename, out var mappings)) - { - BitcoinContext.OHLCVCache = new ConcurrentDictionary(mappings); - - _logger.LogInformation( - "Read market mapping for {n:n0} blocks from file {f}.", - new Dictionary().Count, - options.Bitcoin.Traverse.BlockMarketMappingFilename); - } - else - { - _logger.LogWarning( - "Failed to read market mapping from file {f}. " + - "Proceeding without market mapping.", - options.Bitcoin.Traverse.BlockMarketMappingFilename); - } - } - cT.ThrowIfCancellationRequested(); var stopwatch = new Stopwatch(); diff --git a/EBA/Blockchains/Bitcoin/Context.cs b/EBA/Blockchains/Bitcoin/Context.cs deleted file mode 100644 index 736a6262..00000000 --- a/EBA/Blockchains/Bitcoin/Context.cs +++ /dev/null @@ -1,9 +0,0 @@ -using EBA.Utilities; - -namespace EBA.Blockchains.Bitcoin -{ - public static class BitcoinContext - { - public static ConcurrentDictionary OHLCVCache = new(); - } -} diff --git a/EBA/Blockchains/Bitcoin/GraphModel/BlockGraph.cs b/EBA/Blockchains/Bitcoin/GraphModel/BlockGraph.cs index 1b8be2ac..5bfd3bf3 100644 --- a/EBA/Blockchains/Bitcoin/GraphModel/BlockGraph.cs +++ b/EBA/Blockchains/Bitcoin/GraphModel/BlockGraph.cs @@ -1,6 +1,4 @@ -using EBA.Utilities; - -namespace EBA.Blockchains.Bitcoin.GraphModel; +namespace EBA.Blockchains.Bitcoin.GraphModel; public class BlockGraph : BitcoinGraph, IEquatable { diff --git a/EBA/CLI/Config/BitcoinTraverseOptions.cs b/EBA/CLI/Config/BitcoinTraverseOptions.cs index f283bb8f..f0a982b5 100644 --- a/EBA/CLI/Config/BitcoinTraverseOptions.cs +++ b/EBA/CLI/Config/BitcoinTraverseOptions.cs @@ -85,8 +85,6 @@ public int Granularity public bool SkipGraphSerialization { init; get; } = false; - public string? BlockMarketMappingFilename { init; get; } = null; - public ResilienceStrategyOptions HttpClientResilienceStrategy { init; get; } = new(); public ResilienceStrategyOptions BitcoinAgentResilienceStrategy { init; get; } = new() diff --git a/EBA/CLI/OptionsBinder.cs b/EBA/CLI/OptionsBinder.cs index 77ae3d2f..f3c4a390 100644 --- a/EBA/CLI/OptionsBinder.cs +++ b/EBA/CLI/OptionsBinder.cs @@ -58,8 +58,7 @@ public static Options Build( MaxBlocksInBuffer = GetValue(defs.Bitcoin.Traverse.MaxBlocksInBuffer, maxBlocksInBufferOption, c), TxoFilename = GetValue(defs.Bitcoin.Traverse.TxoFilename, txoFilenameOption, c, (x) => { return Path.Join(wd, Path.GetFileName(x)); }), TrackTxo = GetValue(defs.Bitcoin.Traverse.TrackTxo, trackTxoOption, c), - SkipGraphSerialization = GetValue(defs.Bitcoin.Traverse.SkipGraphSerialization, skipGraphSerializationOption, c), - BlockMarketMappingFilename = GetValue(defs.Bitcoin.Traverse.BlockMarketMappingFilename, blockMarketMappingOption, c, (x) => { return Path.Join(wd, Path.GetFileName(x)); }), + SkipGraphSerialization = GetValue(defs.Bitcoin.Traverse.SkipGraphSerialization, skipGraphSerializationOption, c) }; var traversalAlgorithm = GetValue(defs.Bitcoin.GraphSample.TraversalAlgorithm, graphSampleMethodOption, c); From 0ed5f7753c5dfd488e0f17249ed5ef8641bb7116 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sat, 4 Apr 2026 22:41:00 -0400 Subject: [PATCH 20/30] improve logging --- EBA/Graph/Db/Neo4jDb/Neo4jDb.cs | 47 ++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs index c5d238ef..de71226e 100644 --- a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs +++ b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs @@ -281,6 +281,20 @@ public async Task SetRealizedCap(BlockNode blockNode, Dictionary oh blockNode.BlockMetadata.RealizedCap = realizedCap; } + private async Task GetEdgeTypeCount(RelationType relationType, CancellationToken ct) + { + await using var session = _driver.AsyncSession(x => x.WithDefaultAccessMode(AccessMode.Read)); + + long edgeCount = await session.ExecuteReadAsync(async tx => + { + var cursor = await tx.RunAsync($"MATCH ()-[r:{relationType}]->() RETURN count(r)"); + var record = await cursor.SingleAsync(); + return record[0].As(); + }); + + return edgeCount; + } + // TODO: this method should not be here; // it is specific to the Bitcoin graph model // and this class should be kept agnostic to the graph model. @@ -289,8 +303,6 @@ public async Task SetRealizedCap( Dictionary ohlcv, CancellationToken ct) { - // TODO: add more logging info here - if (blockNodes.Count == 0) return; @@ -304,9 +316,13 @@ public async Task SetRealizedCap( await VerifyConnectivityAsync(ct); await using var session = _driver.AsyncSession(x => x.WithDefaultAccessMode(AccessMode.Read)); + var edgeCounter = 0; + var totalEdgeCount = await GetEdgeTypeCount(T2SEdge.Kind.Relation, ct); var processedEdgeCount = 0; var skippedEdgeCounter = 0; + _logger.LogInformation("Starting iteration through {b} edges.", T2SEdge.Kind.Relation); + await session.ExecuteReadAsync(async tx => { var cursor = await tx.RunAsync( @@ -325,6 +341,7 @@ await session.ExecuteReadAsync(async tx => while (await cursor.FetchAsync()) { ct.ThrowIfCancellationRequested(); + edgeCounter++; var creationHeight = cursor.Current["creationHeight"].As(); var spentHeight = cursor.Current["spentHeight"].As(); @@ -333,24 +350,28 @@ await session.ExecuteReadAsync(async tx => if (!ohlcv.TryGetValue(creationHeight, out var blockOHLCV)) { skippedEdgeCounter++; - continue; } + else + { + var fiatValue = blockOHLCV.GetFiatValue(value); + foreach (var h in sortedHeights.GetViewBetween(creationHeight, spentHeight)) + blockNodes[h].BlockMetadata.RealizedCap += fiatValue; - var fiatValue = blockOHLCV.GetFiatValue(value); - foreach (var h in sortedHeights.GetViewBetween(creationHeight, spentHeight)) - blockNodes[h].BlockMetadata.RealizedCap += fiatValue; - - // TODO: also log the number of skipped edges + processedEdgeCount++; + } - processedEdgeCount++; - if (processedEdgeCount % 10000 == 0) - _logger.LogInformation("Processed {count:n0} edges for realized cap calculation.", processedEdgeCount); + if (edgeCounter % 100000 == 0) + { + _logger.LogInformation( + "Traversed {e:n0} / {t:n0} edges. Processed {p:n0} edges and skipped {s:n0} edges due to missing related OHLCV data.", + edgeCounter, totalEdgeCount, processedEdgeCount, skippedEdgeCounter); + } } }); _logger.LogInformation( - "Processed {p:n0} edges and skipped {s:n0} edges due to missing OHLCV data at their creation height.", - processedEdgeCount, skippedEdgeCounter); + "Traversed {e:n0} / {t:n0} edges. Processed {p:n0} edges and skipped {s:n0} edges due to missing related OHLCV data.", + edgeCounter, totalEdgeCount, processedEdgeCount, skippedEdgeCounter); } // TODO: this method should not be here; From 4c1e5362d5ac954a498b1ae72e602a3689a57849 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sun, 5 Apr 2026 12:28:24 -0400 Subject: [PATCH 21/30] update logs & refactor subcommand. --- EBA/CLI/CLI.cs | 12 ++--- .../Bitcoin/Strategies/BlockNodeStrategy.cs | 8 +++- EBA/Graph/Db/Neo4jDb/Neo4jDb.cs | 44 +++++++++++++------ EBA/Orchestrator.cs | 4 +- 4 files changed, 46 insertions(+), 22 deletions(-) diff --git a/EBA/CLI/CLI.cs b/EBA/CLI/CLI.cs index ad9be5dd..6415d318 100644 --- a/EBA/CLI/CLI.cs +++ b/EBA/CLI/CLI.cs @@ -21,7 +21,7 @@ public Cli( Func bitcoinMapMarketHandlerAsync, Func bitcoinAddressStatsHandlerAsync, Func bitcoinImportCypherQueriesAsync, - Func bitcoinFinalizeImportHandlerAsync, + Func bitcoinPostProcessHandlerAsync, Func bitcoinAugmentHandlerAsync, Action exceptionHandler) { @@ -90,7 +90,7 @@ e is NotSupportedException || mapMarketHandlerAsync: bitcoinMapMarketHandlerAsync, addressStatsHandlerAsync: bitcoinAddressStatsHandlerAsync, importCypherQueriesAsync: bitcoinImportCypherQueriesAsync, - finalizeImportHandlerAsync: bitcoinFinalizeImportHandlerAsync, + postProcessHandlerAsync: bitcoinPostProcessHandlerAsync, bitcoinAugmentHandlerAsync: bitcoinAugmentHandlerAsync) }; @@ -127,7 +127,7 @@ private Command GetBitcoinCmd( Func mapMarketHandlerAsync, Func addressStatsHandlerAsync, Func importCypherQueriesAsync, - Func finalizeImportHandlerAsync, + Func postProcessHandlerAsync, Func bitcoinAugmentHandlerAsync) { var cmd = new Command( @@ -141,7 +141,7 @@ private Command GetBitcoinCmd( GetBitcoinMapMarketCmd(defaultOptions, mapMarketHandlerAsync), GetBitcoinSampleCmd(defaultOptions, sampleHandlerAsync), GetBitcoinAddressStatsCmd(defaultOptions, addressStatsHandlerAsync), - GetFinalizeImportCmd(defaultOptions, finalizeImportHandlerAsync), + GetPostProcessCmd(defaultOptions, postProcessHandlerAsync), GetBitcoinAugmentCmd(defaultOptions, bitcoinAugmentHandlerAsync) }; return cmd; @@ -452,9 +452,9 @@ private Command GetBitcoinMapMarketCmd(Options defaultOptions, Func handlerAsync) + private Command GetPostProcessCmd(Options defaultOptions, Func handlerAsync) { - var cmd = new Command(name: "finalize-import"); + var cmd = new Command(name: "post-process"); cmd.SetAction(async (parseResult, cancellationToken) => { var options = OptionsBinder.Build( diff --git a/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs index a4882260..29f0fc19 100644 --- a/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs @@ -80,6 +80,9 @@ .. PropertyMappingFactory.DictionaryToColumns( .. PropertyMappingFactory.DictionaryToColumns( nameof(BlockNode.TripletTypeValueSum), Schema.EdgeKinds, n => n.TripletTypeValueSum), + new(nameof(v.TotalSupply), FieldType.Long, n => n.BlockMetadata.TotalSupply), + new(nameof(v.TotalSupplyNominal), FieldType.Long, n => n.BlockMetadata.TotalSupplyNominal), + .. _economicMappings, new(":LABEL", FieldType.String, _ => BlockNode.Kind, _ => ":LABEL"), @@ -145,8 +148,11 @@ public static BlockNode Deserialize( InputScriptTypeValue = PropertyMappingFactory.ReadScriptTypeCounts("Inputs", props), OutputScriptTypeValue = PropertyMappingFactory.ReadScriptTypeCounts("Outputs", props), + TotalSupply = _mappingsDict[nameof(v.TotalSupply)].Deserialize(props), + TotalSupplyNominal = _mappingsDict[nameof(v.TotalSupplyNominal)].Deserialize(props), + RealizedCap = (decimal?)_mappingsDict[nameof(v.RealizedCap)].Deserialize(props), - Ohlcv = PropertyMappingFactory.ReadOHLCV(props), + Ohlcv = PropertyMappingFactory.ReadOHLCV(props) }; var blockNode = new BlockNode( diff --git a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs index de71226e..6862b61b 100644 --- a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs +++ b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs @@ -2,6 +2,9 @@ namespace EBA.Graph.Db.Neo4jDb; +// TODO: all the property name setting here should not nameof(), +// instead they should use the property mapping in the strategies. + public class Neo4jDb : IGraphDb where T : GraphBase { private readonly Options _options; @@ -317,7 +320,7 @@ public async Task SetRealizedCap( await using var session = _driver.AsyncSession(x => x.WithDefaultAccessMode(AccessMode.Read)); var edgeCounter = 0; - var totalEdgeCount = await GetEdgeTypeCount(T2SEdge.Kind.Relation, ct); + double totalEdgeCount = await GetEdgeTypeCount(T2SEdge.Kind.Relation, ct); var processedEdgeCount = 0; var skippedEdgeCounter = 0; @@ -388,14 +391,14 @@ public async Task SetUTxOSpentHeight(CancellationToken ct) var batch = new List>(); var batchIndex = 0; long totalProcessed = 0; + var totalEdgesToProcess = await GetEdgeTypeCount(S2TEdge.Kind.Relation, ct); + long processedEdgeCount = 0; await using var readSession = _driver.AsyncSession( x => x.WithDefaultAccessMode(AccessMode.Read)); var readTx = await readSession.BeginTransactionAsync(); - // TODO: this should also report the total number of edges reported. - var cursor = await readTx.RunAsync( $"MATCH ()-[r:{S2TEdge.Kind.Relation}]->() " + $"WHERE r.{nameof(S2TEdge.Generated)} = false " + @@ -418,9 +421,11 @@ public async Task SetUTxOSpentHeight(CancellationToken ct) ["height"] = height }); + processedEdgeCount++; + if (batch.Count >= _options.Neo4j.MaxEntitiesPerBatch) { - await CommitUTxOSpentHeight(batch, batchIndex++, ct); + await CommitUTxOSpentHeight(batch, totalEdgesToProcess, processedEdgeCount, batchIndex++, ct); totalProcessed += batch.Count; batch = []; } @@ -428,27 +433,33 @@ public async Task SetUTxOSpentHeight(CancellationToken ct) if (batch.Count > 0) { - await CommitUTxOSpentHeight(batch, batchIndex++, ct); + await CommitUTxOSpentHeight(batch, totalEdgesToProcess, processedEdgeCount, batchIndex++, ct); totalProcessed += batch.Count; } await readTx.CommitAsync(); _logger.LogInformation( - "Completed setting SpentHeight on Credits for {total:n0} Redeems edges.", - totalProcessed); + "Completed setting {p} on {e} for {total:n0} edges.", + nameof(T2SEdge.SpentHeight), T2SEdge.Kind.Relation, totalProcessed); } private async Task CommitUTxOSpentHeight( IReadOnlyList> batch, + long totalEdgesToProcess, + long processedEdgeCount, int batchIndex, CancellationToken ct) { ct.ThrowIfCancellationRequested(); _logger.LogInformation( - "Committing batch {batch} with {count:n0} records.", - batchIndex, batch.Count); + "Committing batch update of {p} property of {e} edges: " + + "Batch {i} containing {c:n0} edges.", + nameof(T2SEdge.SpentHeight), + T2SEdge.Kind.Relation, + batchIndex, + batch.Count); await using var writeSession = _driver.AsyncSession( x => x.WithDefaultAccessMode(AccessMode.Write)); @@ -458,16 +469,23 @@ private async Task CommitUTxOSpentHeight( var cursor = await x.RunAsync( $"UNWIND $batch AS row " + $"MATCH (t:{TxNode.Kind} {{{nameof(TxNode.Txid)}: row.txid}})-[c:{T2SEdge.Kind.Relation}]->() " + - $"WHERE c.{nameof(S2TEdge.Vout)} = row.vout " + - $"SET c.{nameof(S2TEdge.SpentHeight)} = row.height", + $"WHERE c.{nameof(T2SEdge.Vout)} = row.vout " + + $"SET c.{nameof(T2SEdge.SpentHeight)} = row.height", new Dictionary { ["batch"] = batch }); return await cursor.ConsumeAsync(); }); _logger.LogInformation( - "Committing batch {batch} finished; set {props:n0} properties on {relation} edges.", - batchIndex, summary.Counters.PropertiesSet, T2SEdge.Kind.Relation); + "Committing batch update of {p} property of {e} edges: " + + "Batch {i} containing {c:n0} edges succeeded. Processed {processed:n0}/{total:n0} edges, set {props:n0} properties.", + nameof(T2SEdge.SpentHeight), + T2SEdge.Kind.Relation, + batchIndex, + batch.Count, + processedEdgeCount, + totalEdgesToProcess, + summary.Counters.PropertiesSet); } public void Dispose() diff --git a/EBA/Orchestrator.cs b/EBA/Orchestrator.cs index 67df0c46..0cd8f986 100644 --- a/EBA/Orchestrator.cs +++ b/EBA/Orchestrator.cs @@ -24,7 +24,7 @@ public Orchestrator(CancellationToken cancelationToken) bitcoinAddressStatsHandlerAsync: BitcoinAddressStatsAsync, bitcoinImportCypherQueriesAsync: BitcoinImportCypherQueriesAsync, bitcoinMapMarketHandlerAsync: BitcoinMarketMapAsync, - bitcoinFinalizeImportHandlerAsync: BitcoinFinalizeImport, + bitcoinPostProcessHandlerAsync: BitcoinPostProcess, bitcoinAugmentHandlerAsync: BitcoinAugmentGraphAsync, exceptionHandler: (e, _) => { @@ -89,7 +89,7 @@ private async Task BitcoinImportCypherQueriesAsync(Options options) graphDb.ReportQueries(); } - private async Task BitcoinFinalizeImport(Options options) + private async Task BitcoinPostProcess(Options options) { var host = await SetupAndGetHostAsync(options); await JsonSerializer.SerializeAsync(options, options.StatusFile, _cT); From 7c8a7922a92443f371993e5d2c8725220d839bf4 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sun, 5 Apr 2026 16:46:48 -0400 Subject: [PATCH 22/30] bug fix in composing query & update logging --- EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs b/EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs index 99cc5608..1ec283f6 100644 --- a/EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs +++ b/EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs @@ -48,12 +48,12 @@ private async Task AddSchemaAndSeeding(CancellationToken ct) private async Task SetSupplyAmount(CancellationToken ct) { - _logger.LogInformation("Setting total supply feature."); + _logger.LogInformation("Setting {f} property.", nameof(BlockMetadata.TotalSupply)); - _logger.LogInformation("Fetching blocks from graph database."); + _logger.LogInformation("Fetching {n} nodes from graph database.", BlockNode.Kind); var nodeVar = "n"; var records = await _graphDb.GetNodesAsync(NodeKind.Block, CancellationToken.None, nodeVariable: nodeVar); - _logger.LogInformation("Retrieved {count:n0} records from graph database. Creating block nodes.", records.Count); + _logger.LogInformation("Fetching {n} nodes from graph database succeeded, retrieved {count:n0} nodes.", BlockNode.Kind, records.Count); NodeFactory.TryCreate(records, out var blockNodes, nodeVar); @@ -95,14 +95,15 @@ private async Task SetSupplyAmount(CancellationToken ct) [nameof(BlockMetadata.TotalSupplyNominal)] = b.BlockMetadata.TotalSupplyNominal, }).ToList(); - _logger.LogInformation("Pushing {count:n0} block updates to graph database.", updates.Count); + _logger.LogInformation("Pushing {count:n0} {n} node updates to graph database.", updates.Count, BlockNode.Kind); await _graphDb.BulkUpdateNodePropertiesAsync( NodeKind.Block, nameof(BlockMetadata.Height), updates, ct); - _logger.LogInformation("Completed pushing block updates."); - _logger.LogInformation("Completed setting total supply feature."); + _logger.LogInformation( + "Successfully updated {f} property of {n} nodes.", + nameof(BlockMetadata.TotalSupply), BlockNode.Kind); } } From 737ccc021ea832f21f78f47ac5df6fc1d6b8a101 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sun, 5 Apr 2026 16:47:07 -0400 Subject: [PATCH 23/30] bug fix --- EBA/Graph/Db/Neo4jDb/Neo4jDb.cs | 51 ++++++++++++++------------------- 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs index 6862b61b..d5090b90 100644 --- a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs +++ b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs @@ -71,10 +71,10 @@ public async Task> GetRandomNodesAsync( } public async Task> GetNodesAsync( - NodeKind label, - CancellationToken ct, + NodeKind label, + CancellationToken ct, string nodeVariable = "n", - int? count=null) + int? count = null) { await VerifyConnectivityAsync(ct); @@ -94,11 +94,11 @@ public async Task> GetNodesAsync( } public async Task> GetNeighborsAsync( - NodeKind rootNodeLabel, + NodeKind rootNodeLabel, string rootNodeIdProperty, string rootNodeId, - int queryLimit, - int maxLevel, + int queryLimit, + int maxLevel, bool useBFS, CancellationToken ct, string relationshipFilter = "") @@ -114,7 +114,7 @@ public async Task> GetNeighborsAsync( if (useBFS) qBuilder.Append($"bfs: true "); - else + else qBuilder.Append($"bfs: false "); //qBuilder.Append($", labelFilter: '{labelFilters}'"); @@ -176,7 +176,7 @@ public async Task SerializeConstantsAndConstraintsAsync(CancellationToken ct) await _strategyFactory.SerializeSchemasAsync(_options.WorkingDir, ct); } - public async Task SerializeAsync(T g,CancellationToken ct) + public async Task SerializeAsync(T g, CancellationToken ct) { var nodes = g.GetNodes(); var edges = g.GetEdges(); @@ -188,7 +188,7 @@ public async Task SerializeAsync(T g,CancellationToken ct) { batchInfo.Update(nodeGroup.Key, nodeGroup.Value.Count); var _strategy = _strategyFactory.GetStrategy(nodeGroup.Key); - if (_strategy == null) + if (_strategy == null) continue; tasks.Add(_strategy.ToCsvAsync(nodeGroup.Value, batchInfo.GetFilename(nodeGroup.Key))); @@ -456,9 +456,9 @@ private async Task CommitUTxOSpentHeight( _logger.LogInformation( "Committing batch update of {p} property of {e} edges: " + "Batch {i} containing {c:n0} edges.", - nameof(T2SEdge.SpentHeight), - T2SEdge.Kind.Relation, - batchIndex, + nameof(T2SEdge.SpentHeight), + T2SEdge.Kind.Relation, + batchIndex, batch.Count); await using var writeSession = _driver.AsyncSession( @@ -479,12 +479,12 @@ private async Task CommitUTxOSpentHeight( _logger.LogInformation( "Committing batch update of {p} property of {e} edges: " + "Batch {i} containing {c:n0} edges succeeded. Processed {processed:n0}/{total:n0} edges, set {props:n0} properties.", - nameof(T2SEdge.SpentHeight), - T2SEdge.Kind.Relation, - batchIndex, - batch.Count, - processedEdgeCount, - totalEdgesToProcess, + nameof(T2SEdge.SpentHeight), + T2SEdge.Kind.Relation, + batchIndex, + batch.Count, + processedEdgeCount, + totalEdgesToProcess, summary.Counters.PropertiesSet); } @@ -520,13 +520,11 @@ public async Task BulkUpdateNodePropertiesAsync( var sampleProps = updates[0].Keys.Where(k => k != idProperty); var setClause = string.Join( ", ", - sampleProps.Select(k => $"n.`{k}` = row.`{k}`")); + sampleProps.Select(k => $"n.{k} = row.{k}")); - // using toString() on the id property to match the string-typed - // :ID column created by neo4j-admin import. var query = "UNWIND $batch AS row " + - $"MATCH (n:{label} {{`{idProperty}`: toString(row.`{idProperty}`)}}) " + + $"MATCH (n:{label} {{{idProperty}: row.{idProperty}}}) " + $"SET {setClause}"; var batchIndex = 0; @@ -544,14 +542,7 @@ public async Task BulkUpdateNodePropertiesAsync( return await cursor.ConsumeAsync(); }); - var counters = summary.Counters; - if (counters.PropertiesSet == 0) - _logger.LogWarning("Batch {batch}: 0 properties set (MATCH may have found no nodes).", - batchIndex); - else - _logger.LogInformation("Batch {batch}: set {props:n0} properties on nodes.", - batchIndex, counters.PropertiesSet); - + _logger.LogInformation("Finished batch {batch}; summary: {s}", batchIndex, summary.Counters); batchIndex++; } From e5c491a78798887a87c4d981e87670d1ffc67cc2 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sun, 5 Apr 2026 16:47:26 -0400 Subject: [PATCH 24/30] bug fix dividing by zero. --- EBA/Blockchains/Bitcoin/ChainModel/BlockMetadata.cs | 2 +- EBA/Graph/Db/IGraphDb.cs | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/EBA/Blockchains/Bitcoin/ChainModel/BlockMetadata.cs b/EBA/Blockchains/Bitcoin/ChainModel/BlockMetadata.cs index 838ef0f7..797a5976 100644 --- a/EBA/Blockchains/Bitcoin/ChainModel/BlockMetadata.cs +++ b/EBA/Blockchains/Bitcoin/ChainModel/BlockMetadata.cs @@ -107,7 +107,7 @@ public decimal? NUPL { get { - if (RealizedCap == null || MarketCap == null) + if (RealizedCap == null || MarketCap == null || MarketCap == 0) return null; return (MarketCap - RealizedCap) / MarketCap; diff --git a/EBA/Graph/Db/IGraphDb.cs b/EBA/Graph/Db/IGraphDb.cs index 314ddcb0..ac8e6e3d 100644 --- a/EBA/Graph/Db/IGraphDb.cs +++ b/EBA/Graph/Db/IGraphDb.cs @@ -53,9 +53,15 @@ public Task> GetNodesAsync( string nodeVariable = "n", int? count = null); - public Task SetUTxOSpentHeight(CancellationToken ct); + public Task SetUTxOSpentHeight( + CancellationToken ct); - public Task ExecuteWriteQueryAsync(List schemas, CancellationToken ct); + public Task ExecuteWriteQueryAsync( + List schemas, + CancellationToken ct); - public Task SetRealizedCap(SortedDictionary blockNodes, Dictionary ohlcv, CancellationToken ct); + public Task SetRealizedCap( + SortedDictionary blockNodes, + Dictionary ohlcv, + CancellationToken ct); } \ No newline at end of file From be522838e6538c639bfaaabd4984e78d00f25752 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sun, 5 Apr 2026 18:10:20 -0400 Subject: [PATCH 25/30] Add docs. --- website/docs/bitcoin/etl/s6-db-config.md | 37 ++---------------- website/docs/bitcoin/etl/s7-off-chain.md | 49 ++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 34 deletions(-) create mode 100644 website/docs/bitcoin/etl/s7-off-chain.md diff --git a/website/docs/bitcoin/etl/s6-db-config.md b/website/docs/bitcoin/etl/s6-db-config.md index eb1482b3..41938f38 100644 --- a/website/docs/bitcoin/etl/s6-db-config.md +++ b/website/docs/bitcoin/etl/s6-db-config.md @@ -44,44 +44,13 @@ To improve query performance we define database constraints, such as uniqueness constraints, which index commonly used properties. - -EBA generates a `schema.cypher` file containing the queries required -to create these constraints and indexes. -EBA creates this file in the same directory where the Bitcoin graph was serialized. - - -The Cypher queries in the `schema.cypher` file are as follows: - -```javascript -// EBA Bitcoin Graph Schema - -// Uniqueness constraint for Script.Address property. -CREATE CONSTRAINT Script_Address_Unique -IF NOT EXISTS -FOR (v:Script) REQUIRE v.Address IS UNIQUE; - -// Uniqueness constraint for Tx.Txid property. -CREATE CONSTRAINT Tx_Txid_Unique -IF NOT EXISTS -FOR (v:Tx) REQUIRE v.Txid IS UNIQUE; - -// Uniqueness constraint for Block.Height property. -CREATE CONSTRAINT Block_Height_Unique -IF NOT EXISTS -FOR (v:Block) REQUIRE v.Height IS UNIQUE; -``` - - Take the following steps to apply these constraints to your database: + 1. Start the Neo4j database service. -2. Execute the schema.cypher file using cypher-shell: +2. Run the `post-process` subcommand. ```Shell - cypher-shell -u neo4j -p password -f schema.cypher + .\eba.exe bitcoin post-process ``` - - Please refer to - [this documentation](https://neo4j.com/docs/operations-manual/current/cypher-shell/) - for details on locating and running `cypher-shell` depending on your Neo4j deployment. diff --git a/website/docs/bitcoin/etl/s7-off-chain.md b/website/docs/bitcoin/etl/s7-off-chain.md new file mode 100644 index 00000000..29cb9767 --- /dev/null +++ b/website/docs/bitcoin/etl/s7-off-chain.md @@ -0,0 +1,49 @@ +--- +title: Augmentation +description: Step 7. Augment graph using off-chain data +sidebar_label: Augmentation +sidebar_position: 7 +slug: /bitcoin/etl/augment +--- + +EBA currently supports adding market data to the Bitcoin graph. +This enriches the graph with market-related features that enable quantitative analysis. + + +### Step 1: Get Market Data + +You may download Bitcoin market data from any source you prefer. +We recommend the following publicly accessible source: + +``` +https://www.kaggle.com/datasets/mczielinski/bitcoin-historical-data +``` + +### Step 2: Map Market Data to Blocks + +Bitcoin targets an average block interval of about 10 minutes +(check [this paper](https://arxiv.org/abs/2510.20028) for detailed plots). +For this step, EBA uses each block's median time (`mediantime`) +to align market data with blocks. +Because market data is typically sampled at shorter intervals +and is not synchronized with block times, +EBA maps candles to each block using the time window between +two consecutive blocks' median times. +For each block, candles with timestamps in that interval are +used to compute aggregated OHLCV values. + +You may run the following command for this step. + +```shell +.\eba.exe bitcoin map-market --ohlcv-source-filename btcusd_1-min_data.csv --block-market-output-filename mapped-block-ohlcv.tsv +``` + +### Step 3: Augment the Graph + +Run the following command to add market-related features to the graph. + +```shell +.\eba.exe bitcoin augment --ohlcv-filename mapped-block-ohlcv.tsv +``` + +Note that this process may take a considerable amount of time to complete. From 3514acc0bf1de1b80d9c56d5685c024d64436fb2 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sun, 5 Apr 2026 18:52:49 -0400 Subject: [PATCH 26/30] Add for properties because otherwise it causes issues with characters like dot in attribute keys. --- EBA/Graph/Db/Neo4jDb/Neo4jDb.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs index d5090b90..5afa6c32 100644 --- a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs +++ b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs @@ -520,11 +520,11 @@ public async Task BulkUpdateNodePropertiesAsync( var sampleProps = updates[0].Keys.Where(k => k != idProperty); var setClause = string.Join( ", ", - sampleProps.Select(k => $"n.{k} = row.{k}")); + sampleProps.Select(k => $"n.`{k}` = row.`{k}`")); var query = "UNWIND $batch AS row " + - $"MATCH (n:{label} {{{idProperty}: row.{idProperty}}}) " + + $"MATCH (n:{label} {{`{idProperty}`: row.`{idProperty}`}}) " + $"SET {setClause}"; var batchIndex = 0; From 439fcff9271075b5029d0a783019a12e0a5a3e2d Mon Sep 17 00:00:00 2001 From: Vahid Date: Sun, 5 Apr 2026 20:47:33 -0400 Subject: [PATCH 27/30] update docs --- website/docs/bitcoin/etl/s5a-import.md | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/website/docs/bitcoin/etl/s5a-import.md b/website/docs/bitcoin/etl/s5a-import.md index 04c76f69..abd6760a 100644 --- a/website/docs/bitcoin/etl/s5a-import.md +++ b/website/docs/bitcoin/etl/s5a-import.md @@ -139,22 +139,19 @@ meaning it does not support incremental updates to an existing graph. ```shell sudo -u neo4j HEAP_SIZE=4G neo4j-admin database import full \ --overwrite-destination neo4j \ - --nodes="$GDIR/BitcoinCoinbase.tsv.gz" \ - --nodes="$GDIR/BitcoinGraph_header.tsv.gz,$GDIR/0_BitcoinGraph.tsv.gz" \ - --nodes="$GDIR/BitcoinScriptNode_header.tsv.gz,$GDIR/unique_BitcoinScriptNode.tsv.gz" \ - --nodes="$GDIR/BitcoinTxNode_header.tsv.gz,$GDIR/unique_BitcoinTxNode.tsv.gz" \ - --relationships="$GDIR/BitcoinS2S_header.tsv.gz,$GDIR/.*BitcoinS2S.tsv.gz" \ - --relationships="$GDIR/BitcoinT2T_header.tsv.gz,$GDIR/.*BitcoinT2T.tsv.gz" \ - --relationships="$GDIR/BitcoinC2T_header.tsv.gz,$GDIR/.*BitcoinC2T.tsv.gz" \ - --relationships="$GDIR/BitcoinC2S_header.tsv.gz,$GDIR/.*BitcoinC2S.tsv.gz" \ - --relationships="$GDIR/BitcoinB2T_header.tsv.gz,$GDIR/.*BitcoinB2T.tsv.gz" \ - --relationships="$GDIR/BitcoinB2S_header.tsv.gz,$GDIR/.*BitcoinB2S.tsv.gz" \ - --relationships="$GDIR/BitcoinS2B_header.tsv.gz,$GDIR/.*BitcoinS2B.tsv.gz" \ - --relationships="$GDIR/BitcoinT2B_header.tsv.gz,$GDIR/.*BitcoinT2B.tsv.gz" \ + --nodes="$GDIR/Coinbase.csv.gz" \ + --nodes="$GDIR/header_nodes_Block.csv.gz.gz,$GDIR/[0-9].*_nodes_Block.csv.gz" \ + --nodes="$GDIR/header_nodes_Script.csv.gz,$GDIR/unique_nodes_Script.csv.gz" \ + --nodes="$GDIR/header_nodes_Tx.csv.gz,$GDIR/unique_nodes_Tx.tsv.gz" \ + --relationships="$GDIR/header_edges_Block_Confirms_Tx.csv.gz,$GDIR/[0-9].*_edges_Block_Confirms_Tx.csv.gz" \ + --relationships="$GDIR/header_edges_Coinbase_Mints_Tx.csv.gz,$GDIR/[0-9].*_edges_Coinbase_Mints_Tx.csv.gz" \ + --relationships="$GDIR/header_edges_Tx_Credits_Script.csv.gz,$GDIR/[0-9].*_edges_Tx_Credits_Script.csv.gz" \ + --relationships="$GDIR/header_edges_Script_Redeems_Tx.csv.gz,$GDIR/[0-9].*_edges_Script_Redeems_Tx.csv.gz" \ + --relationships="$GDIR/header_edges_Tx_Fee_Tx.csv.gz,$GDIR/[0-9].*_edges_Tx_Fee_Tx.csv.gz" \ + --relationships="$GDIR/header_edges_Tx_Transfers_Tx.csv.gz,$GDIR/[0-9].*_edges_Tx_Transfers_Tx.csv.gz" \ --delimiter="\t" \ --array-delimiter=";" \ - --verbose \ - --skip-duplicate-nodes + --verbose ``` From dca4f35c5116dfea9d111a5474fc1cef55aaef9c Mon Sep 17 00:00:00 2001 From: Vahid Date: Sun, 5 Apr 2026 22:26:09 -0400 Subject: [PATCH 28/30] bug fix --- EBA/Graph/Bitcoin/Strategies/PropertyMapping.cs | 2 ++ EBA/Graph/Bitcoin/Strategies/PropertyMappingExtensions.cs | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/EBA/Graph/Bitcoin/Strategies/PropertyMapping.cs b/EBA/Graph/Bitcoin/Strategies/PropertyMapping.cs index f635f766..8217cd84 100644 --- a/EBA/Graph/Bitcoin/Strategies/PropertyMapping.cs +++ b/EBA/Graph/Bitcoin/Strategies/PropertyMapping.cs @@ -36,6 +36,8 @@ public PropertyMapping( deserializer) { } + public object? GetValue(T source) => _propertySelector(source); + public string SerializeHeader() { return _headerOverride?.Invoke(Property) ?? Property.TypeAnnotatedCsvHeader; diff --git a/EBA/Graph/Bitcoin/Strategies/PropertyMappingExtensions.cs b/EBA/Graph/Bitcoin/Strategies/PropertyMappingExtensions.cs index def589f0..837afaad 100644 --- a/EBA/Graph/Bitcoin/Strategies/PropertyMappingExtensions.cs +++ b/EBA/Graph/Bitcoin/Strategies/PropertyMappingExtensions.cs @@ -34,7 +34,7 @@ public static PropertyMapping Get(this PropertyMapping[] mappings, stri { var dict = new Dictionary(mappings.Length); foreach (var m in mappings) - dict[m.Property.Name] = m.SerializeValue(source); + dict[m.Property.Name] = m.GetValue(source); return dict; } } From 4a06d4cfb91991ca7410ab18874c3668df607eb2 Mon Sep 17 00:00:00 2001 From: Vahid Date: Sat, 11 Apr 2026 21:44:13 -0400 Subject: [PATCH 29/30] fix bugs after merge. --- EBA/CLI/CLI.cs | 10 +++------- EBA/CLI/OptionsBinder.cs | 4 +--- EBA/Graph/Bitcoin/Strategies/S2TEdgeStrategy.cs | 7 +------ EBA/Orchestrator.cs | 1 - 4 files changed, 5 insertions(+), 17 deletions(-) diff --git a/EBA/CLI/CLI.cs b/EBA/CLI/CLI.cs index 4e9e30b1..6bf0f41f 100644 --- a/EBA/CLI/CLI.cs +++ b/EBA/CLI/CLI.cs @@ -23,7 +23,6 @@ public Cli( Func bitcoinImportCypherQueriesAsync, Func bitcoinPostProcessHandlerAsync, Func bitcoinAugmentHandlerAsync, - Func bitcoinPostProcessGraphHandlerAsync, Func bitcoinMapSpendsHandlerAsync, Action exceptionHandler) { @@ -93,8 +92,7 @@ e is NotSupportedException || addressStatsHandlerAsync: bitcoinAddressStatsHandlerAsync, importCypherQueriesAsync: bitcoinImportCypherQueriesAsync, postProcessHandlerAsync: bitcoinPostProcessHandlerAsync, - bitcoinAugmentHandlerAsync: bitcoinAugmentHandlerAsync) - postProcessGraphHandlerAsync: bitcoinPostProcessGraphHandlerAsync, + bitcoinAugmentHandlerAsync: bitcoinAugmentHandlerAsync, mapSpendsHandlerAsync: bitcoinMapSpendsHandlerAsync) }; @@ -132,8 +130,7 @@ private Command GetBitcoinCmd( Func addressStatsHandlerAsync, Func importCypherQueriesAsync, Func postProcessHandlerAsync, - Func bitcoinAugmentHandlerAsync) - Func postProcessGraphHandlerAsync, + Func bitcoinAugmentHandlerAsync, Func mapSpendsHandlerAsync) { var cmd = new Command( @@ -148,8 +145,7 @@ private Command GetBitcoinCmd( GetBitcoinSampleCmd(defaultOptions, sampleHandlerAsync), GetBitcoinAddressStatsCmd(defaultOptions, addressStatsHandlerAsync), GetPostProcessCmd(defaultOptions, postProcessHandlerAsync), - GetBitcoinAugmentCmd(defaultOptions, bitcoinAugmentHandlerAsync) - GetPostProcessGraphCmd(defaultOptions, postProcessGraphHandlerAsync), + GetBitcoinAugmentCmd(defaultOptions, bitcoinAugmentHandlerAsync), GetBitcoinMapSpendsCmd(defaultOptions, mapSpendsHandlerAsync) }; return cmd; diff --git a/EBA/CLI/OptionsBinder.cs b/EBA/CLI/OptionsBinder.cs index 93c3081e..34f1ff6d 100644 --- a/EBA/CLI/OptionsBinder.cs +++ b/EBA/CLI/OptionsBinder.cs @@ -31,8 +31,7 @@ public static Options Build( Option? marketDataFilenameOption = null, Option? outputFilenameOption = null, Option? blockMarketMappingOption = null, - Option? augmentroOhlcvOption = null) - Option? outputFilenameOption = null, + Option? augmentroOhlcvOption = null, Option? batchesFilenameOption = null) { if (statusFilenameOption != null && c.GetResult(statusFilenameOption) is not null) @@ -124,7 +123,6 @@ public static Options Build( GraphSample = gsample, MapMarket = bitcoinMapMarketOps, Augmentor = bitcoinAugmentorOps, - MapMarket = bitcoinMapMarketOps, MapSpends = bitcoinMapSpendsOps }; diff --git a/EBA/Graph/Bitcoin/Strategies/S2TEdgeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/S2TEdgeStrategy.cs index d442ac48..e48e576f 100644 --- a/EBA/Graph/Bitcoin/Strategies/S2TEdgeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/S2TEdgeStrategy.cs @@ -43,12 +43,7 @@ public static S2TEdge Deserialize(ScriptNode source, TxNode target, IRelationshi source: source, target: target, timestamp: 0, - spentHeight: _mappings.Get(nameof(S2TEdge.SpentHeight)).Deserialize(relationship.Properties), - value: _mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties), - txid: _mappings.Get(nameof(S2TEdge.Txid)).Deserialize(relationship.Properties), - vout: _mappings.Get(nameof(S2TEdge.Vout)).Deserialize(relationship.Properties), - generated: _mappings.Get(nameof(S2TEdge.Generated)).Deserialize(relationship.Properties), - creationHeight: _mappings.Get(nameof(S2TEdge.CreationHeight)).Deserialize(relationship.Properties) + creationHeight: Mappings.Get(nameof(S2TEdge.CreationHeight)).Deserialize(relationship.Properties), spentHeight: Mappings.Get(nameof(S2TEdge.SpentHeight)).Deserialize(relationship.Properties), value: Mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties), txid: Mappings.Get(nameof(S2TEdge.Txid)).Deserialize(relationship.Properties), diff --git a/EBA/Orchestrator.cs b/EBA/Orchestrator.cs index c21bd354..e4814f2a 100644 --- a/EBA/Orchestrator.cs +++ b/EBA/Orchestrator.cs @@ -27,7 +27,6 @@ public Orchestrator(CancellationToken cancelationToken) bitcoinMapMarketHandlerAsync: BitcoinMarketMapAsync, bitcoinPostProcessHandlerAsync: BitcoinPostProcess, bitcoinAugmentHandlerAsync: BitcoinAugmentGraphAsync, - bitcoinPostProcessGraphHandlerAsync: BitcoinPostProcessGraph, bitcoinMapSpendsHandlerAsync: BitcoinMapSpends, exceptionHandler: (e, _) => { From 8ae7c9964164aae4f49f7eea048b189809f8993f Mon Sep 17 00:00:00 2001 From: Vahid Date: Sun, 12 Apr 2026 10:24:48 -0400 Subject: [PATCH 30/30] Drop setting utxo spent height. it is implemented in https://github.com/B1AAB/EBA/pull/43 --- .../Bitcoin/OffChain/EconomicAugmentor.cs | 3 + EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs | 2 - EBA/Graph/Db/IGraphDb.cs | 3 - EBA/Graph/Db/Neo4jDb/Neo4jDb.cs | 111 ------------------ 4 files changed, 3 insertions(+), 116 deletions(-) diff --git a/EBA/Graph/Bitcoin/OffChain/EconomicAugmentor.cs b/EBA/Graph/Bitcoin/OffChain/EconomicAugmentor.cs index c2a09b84..2d9d5850 100644 --- a/EBA/Graph/Bitcoin/OffChain/EconomicAugmentor.cs +++ b/EBA/Graph/Bitcoin/OffChain/EconomicAugmentor.cs @@ -56,6 +56,9 @@ private async Task> GetBlockNodes(Cancellation return blocks; } + // TODO: this can be set as part of the pre-bulk import finalization step, + // instead of being computed and updated after the fact. + // This would require reordering the import steps to ensure that the OHLCV data is available before the block nodes are created. private async Task ComputeBlockValuationMetrics( SortedDictionary blocks, Dictionary blockOHLCVMapping, diff --git a/EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs b/EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs index 1ec283f6..dd149bc3 100644 --- a/EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs +++ b/EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs @@ -16,8 +16,6 @@ public async Task Finalize(CancellationToken ct) await AddSchemaAndSeeding(ct); await SetSupplyAmount(ct); - - await _graphDb.SetUTxOSpentHeight(ct); } private async Task AddSchemaAndSeeding(CancellationToken ct) diff --git a/EBA/Graph/Db/IGraphDb.cs b/EBA/Graph/Db/IGraphDb.cs index b5002fc3..3f326502 100644 --- a/EBA/Graph/Db/IGraphDb.cs +++ b/EBA/Graph/Db/IGraphDb.cs @@ -53,9 +53,6 @@ public Task> GetNodesAsync( string nodeVariable = "n", int? count = null); - public Task SetUTxOSpentHeight( - CancellationToken ct); - public Task ExecuteWriteQueryAsync( List schemas, CancellationToken ct); diff --git a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs index 49df4ce3..d10390f3 100644 --- a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs +++ b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs @@ -377,117 +377,6 @@ await session.ExecuteReadAsync(async tx => edgeCounter, totalEdgeCount, processedEdgeCount, skippedEdgeCounter); } - // TODO: this method should not be here; - // it is specific to the Bitcoin graph model - // and this class should be kept agnostic to the graph model. - public async Task SetUTxOSpentHeight(CancellationToken ct) - { - _logger.LogInformation( - "Starting to find UTxO spending, and set their status to spent by setting {prop} on {r} edges.", - nameof(S2TEdge.SpentHeight), T2SEdge.Kind.Relation); - - await VerifyConnectivityAsync(ct); - - var batch = new List>(); - var batchIndex = 0; - long totalProcessed = 0; - var totalEdgesToProcess = await GetEdgeTypeCount(S2TEdge.Kind.Relation, ct); - long processedEdgeCount = 0; - - await using var readSession = _driver.AsyncSession( - x => x.WithDefaultAccessMode(AccessMode.Read)); - - var readTx = await readSession.BeginTransactionAsync(); - - var cursor = await readTx.RunAsync( - $"MATCH ()-[r:{S2TEdge.Kind.Relation}]->() " + - $"WHERE r.{nameof(S2TEdge.Generated)} = false " + - $"RETURN " + - $"r.{nameof(S2TEdge.Txid)} AS txid, " + - $"r.{nameof(S2TEdge.Vout)} AS vout, " + - $"r.{nameof(S2TEdge.SpentHeight)} AS height"); - - while (await cursor.FetchAsync()) - { - ct.ThrowIfCancellationRequested(); - - var record = cursor.Current; - var height = record["height"].As(); - - batch.Add(new Dictionary - { - ["txid"] = record["txid"].As(), - ["vout"] = record["vout"].As(), - ["height"] = height - }); - - processedEdgeCount++; - - if (batch.Count >= _options.Neo4j.MaxEntitiesPerBatch) - { - await CommitUTxOSpentHeight(batch, totalEdgesToProcess, processedEdgeCount, batchIndex++, ct); - totalProcessed += batch.Count; - batch = []; - } - } - - if (batch.Count > 0) - { - await CommitUTxOSpentHeight(batch, totalEdgesToProcess, processedEdgeCount, batchIndex++, ct); - totalProcessed += batch.Count; - } - - await readTx.CommitAsync(); - - _logger.LogInformation( - "Completed setting {p} on {e} for {total:n0} edges.", - nameof(T2SEdge.SpentHeight), T2SEdge.Kind.Relation, totalProcessed); - } - - private async Task CommitUTxOSpentHeight( - IReadOnlyList> batch, - long totalEdgesToProcess, - long processedEdgeCount, - int batchIndex, - CancellationToken ct) - { - ct.ThrowIfCancellationRequested(); - - _logger.LogInformation( - "Committing batch update of {p} property of {e} edges: " + - "Batch {i} containing {c:n0} edges.", - nameof(T2SEdge.SpentHeight), - T2SEdge.Kind.Relation, - batchIndex, - batch.Count); - - await using var writeSession = _driver.AsyncSession( - x => x.WithDefaultAccessMode(AccessMode.Write)); - - var summary = await writeSession.ExecuteWriteAsync(async x => - { - var cursor = await x.RunAsync( - $"UNWIND $batch AS row " + - $"MATCH (t:{TxNode.Kind} {{{nameof(TxNode.Txid)}: row.txid}})-[c:{T2SEdge.Kind.Relation}]->() " + - $"WHERE c.{nameof(T2SEdge.Vout)} = row.vout " + - $"SET c.{nameof(T2SEdge.SpentHeight)} = row.height", - new Dictionary { ["batch"] = batch }); - - return await cursor.ConsumeAsync(); - }); - - _logger.LogInformation( - "Committing batch update of {p} property of {e} edges: " + - "Batch {i} containing {c:n0} edges succeeded. Processed {processed:n0}/{total:n0} edges, set {props:n0} properties.", - nameof(T2SEdge.SpentHeight), - T2SEdge.Kind.Relation, - batchIndex, - batch.Count, - processedEdgeCount, - totalEdgesToProcess, - summary.Counters.PropertiesSet); - } - public void Dispose() { Dispose(true);