diff --git a/EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs b/EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs index ec72d04d..c856f7a3 100644 --- a/EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs +++ b/EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs @@ -21,7 +21,7 @@ public BitcoinOrchestrator( } public async Task TraverseAsync( - Options options, + Options options, CancellationToken cT) { var chainInfo = await _agent.AssertChainAsync(cT); @@ -59,7 +59,7 @@ public async Task TraverseAsync( throw; } - } + } public async Task DeDupAsync( Options options, diff --git a/EBA/Blockchains/Bitcoin/ChainModel/BlockMetadata.cs b/EBA/Blockchains/Bitcoin/ChainModel/BlockMetadata.cs index e8d8b552..797a5976 100644 --- a/EBA/Blockchains/Bitcoin/ChainModel/BlockMetadata.cs +++ b/EBA/Blockchains/Bitcoin/ChainModel/BlockMetadata.cs @@ -90,4 +90,31 @@ public long ProvablyUnspendableBitcoins public long? TotalSupply { set; get; } = null; public long? TotalSupplyNominal { set; get; } = null; + + public decimal? RealizedCap { set; get; } = null; + public decimal? MarketCap + { + get + { + if (TotalSupply == null || Ohlcv == null) + return null; + + return Ohlcv.GetFiatValue((long)TotalSupply); + } + } + + public decimal? NUPL + { + get + { + if (RealizedCap == null || MarketCap == null || MarketCap == 0) + return null; + + return (MarketCap - RealizedCap) / MarketCap; + } + } + + public OHLCV? Ohlcv { set; get; } = null; + + // TODO: add MVRV, NVT, Thermocap } diff --git a/EBA/Blockchains/Bitcoin/GraphModel/BlockGraph.cs b/EBA/Blockchains/Bitcoin/GraphModel/BlockGraph.cs index 8720efea..5bfd3bf3 100644 --- a/EBA/Blockchains/Bitcoin/GraphModel/BlockGraph.cs +++ b/EBA/Blockchains/Bitcoin/GraphModel/BlockGraph.cs @@ -95,7 +95,9 @@ public void BuildGraph(CancellationToken ct) long miningReward = 0; foreach (var u in _coinbaseTxGraph.Outputs) { - TryGetOrAddEdge(new T2SEdge(v, new ScriptNode(u.ScriptPubKey), t, h, output: u), out var edge); + TryGetOrAddEdge( + new T2SEdge(v, new ScriptNode(u.ScriptPubKey), t, h, u), + out T2SEdge _); Block.ProfileCreatedOutput(u); miningReward += u.Value; @@ -123,14 +125,16 @@ private void AddTxGraphToBlockGraph(TxGraph txGraph) { TryGetOrAddEdge( new S2TEdge(new ScriptNode(u.PrevOut.ScriptPubKey), v, t, h, u), - out var edge); + out S2TEdge _); Block.ProfileSpentOutput(u); } foreach (var u in txGraph.Outputs) { - TryGetOrAddEdge(new T2SEdge(v, new ScriptNode(u.ScriptPubKey), t, h, output: u), out var edge); + TryGetOrAddEdge( + new T2SEdge(v, new ScriptNode(u.ScriptPubKey), t, h, u), + out T2SEdge _); Block.ProfileCreatedOutput(u); } diff --git a/EBA/CLI/CLI.cs b/EBA/CLI/CLI.cs index bb9cfec9..6bf0f41f 100644 --- a/EBA/CLI/CLI.cs +++ b/EBA/CLI/CLI.cs @@ -21,7 +21,8 @@ public Cli( Func bitcoinMapMarketHandlerAsync, Func bitcoinAddressStatsHandlerAsync, Func bitcoinImportCypherQueriesAsync, - Func bitcoinPostProcessGraphHandlerAsync, + Func bitcoinPostProcessHandlerAsync, + Func bitcoinAugmentHandlerAsync, Func bitcoinMapSpendsHandlerAsync, Action exceptionHandler) { @@ -90,7 +91,8 @@ e is NotSupportedException || mapMarketHandlerAsync: bitcoinMapMarketHandlerAsync, addressStatsHandlerAsync: bitcoinAddressStatsHandlerAsync, importCypherQueriesAsync: bitcoinImportCypherQueriesAsync, - postProcessGraphHandlerAsync: bitcoinPostProcessGraphHandlerAsync, + postProcessHandlerAsync: bitcoinPostProcessHandlerAsync, + bitcoinAugmentHandlerAsync: bitcoinAugmentHandlerAsync, mapSpendsHandlerAsync: bitcoinMapSpendsHandlerAsync) }; @@ -127,7 +129,8 @@ private Command GetBitcoinCmd( Func mapMarketHandlerAsync, Func addressStatsHandlerAsync, Func importCypherQueriesAsync, - Func postProcessGraphHandlerAsync, + Func postProcessHandlerAsync, + Func bitcoinAugmentHandlerAsync, Func mapSpendsHandlerAsync) { var cmd = new Command( @@ -141,7 +144,8 @@ private Command GetBitcoinCmd( GetBitcoinMapMarketCmd(defaultOptions, mapMarketHandlerAsync), GetBitcoinSampleCmd(defaultOptions, sampleHandlerAsync), GetBitcoinAddressStatsCmd(defaultOptions, addressStatsHandlerAsync), - GetPostProcessGraphCmd(defaultOptions, postProcessGraphHandlerAsync), + GetPostProcessCmd(defaultOptions, postProcessHandlerAsync), + GetBitcoinAugmentCmd(defaultOptions, bitcoinAugmentHandlerAsync), GetBitcoinMapSpendsCmd(defaultOptions, mapSpendsHandlerAsync) }; return cmd; @@ -238,6 +242,11 @@ private Command GetBitcoinTraverseCmd(Options defaultOptions, Func("--map-block-market") + { + Required = false + }; + var cmd = new Command( name: "traverse", description: "Traverses the blockchain in the defined range collecting the set metrics.") @@ -251,7 +260,8 @@ private Command GetBitcoinTraverseCmd(Options defaultOptions, Func handlerAsync) + private Command GetPostProcessCmd(Options defaultOptions, Func handlerAsync) { - var cmd = new Command(name: "post-process-graph"); + var cmd = new Command(name: "post-process"); cmd.SetAction(async (parseResult, cancellationToken) => { var options = OptionsBinder.Build( @@ -686,6 +697,37 @@ private Command GetBitcoinAddressStatsCmd(Options defaultOptions, Func handlerAsync) + { + var ohlcvFilenameOption = new Option("--ohlcv-filename") + { + Description = "A TSV file containing block metadata (height, median time) mapped to aggregated OHLCV market data." + }; + var cmd = new Command( + name: "augment", + description: "Augments the graph with additional features, such as NUPL, using the output of the map-market command and other similar files.") + { + ohlcvFilenameOption + }; + cmd.SetAction(async (parseResult, cancellationToken) => + { + var options = OptionsBinder.Build( + parseResult, + workingDirOption: _workingDirOption, + statusFilenameOption: _statusFilenameOption, + augmentroOhlcvOption: ohlcvFilenameOption); + try + { + await handlerAsync(options); + } + catch (Exception e) + { + _exceptionHandler(e, parseResult); + } + }); + return cmd; + } + private Command GetBitcoinMapSpendsCmd(Options defaultOptions, Func handlerAsync) { var batchesFilenameOption = new Option("--batches-filename") diff --git a/EBA/CLI/Config/BitcoinAugmentorOptions.cs b/EBA/CLI/Config/BitcoinAugmentorOptions.cs new file mode 100644 index 00000000..dd219784 --- /dev/null +++ b/EBA/CLI/Config/BitcoinAugmentorOptions.cs @@ -0,0 +1,6 @@ +namespace EBA.CLI.Config; + +public class BitcoinAugmentorOptions +{ + public string BlockOhlcvMappedFilename { init; get; } +} diff --git a/EBA/CLI/Config/BitcoinOptions.cs b/EBA/CLI/Config/BitcoinOptions.cs index 929805ac..2caebee0 100644 --- a/EBA/CLI/Config/BitcoinOptions.cs +++ b/EBA/CLI/Config/BitcoinOptions.cs @@ -9,5 +9,6 @@ public BitcoinOptions() : this(DateTimeOffset.Now.ToUnixTimeSeconds()) { } public BitcoinDedupOptions Dedup { init; get; } = new(); public BitcoinGraphSampleOptions GraphSample { init; get; } = new(); public BitcoinMapMarketOptions MapMarket { init; get; } = new(); + public BitcoinAugmentorOptions Augmentor { init; get; } = new(); public BitcoinMapSpendsOptions MapSpends { init; get; } = new(); } diff --git a/EBA/CLI/OptionsBinder.cs b/EBA/CLI/OptionsBinder.cs index 7a606988..34f1ff6d 100644 --- a/EBA/CLI/OptionsBinder.cs +++ b/EBA/CLI/OptionsBinder.cs @@ -30,6 +30,8 @@ public static Options Build( Option? sortedScriptNodeFilenameOption = null, Option? marketDataFilenameOption = null, Option? outputFilenameOption = null, + Option? blockMarketMappingOption = null, + Option? augmentroOhlcvOption = null, Option? batchesFilenameOption = null) { if (statusFilenameOption != null && c.GetResult(statusFilenameOption) is not null) @@ -104,6 +106,11 @@ public static Options Build( BlockOhlcvMappedFilename = GetValue(defs.Bitcoin.MapMarket.BlockOhlcvMappedFilename, outputFilenameOption, c, (x) => { return Path.Join(wd, Path.GetFileName(x)); }) }; + var bitcoinAugmentorOps = new BitcoinAugmentorOptions() + { + BlockOhlcvMappedFilename = GetValue(defs.Bitcoin.Augmentor.BlockOhlcvMappedFilename, augmentroOhlcvOption, c, (x) => { return Path.Join(wd, Path.GetFileName(x)); }), + }; + var bitcoinMapSpendsOps = new BitcoinMapSpendsOptions() { BatchesFilename = GetValue(defs.Bitcoin.MapSpends.BatchesFilename, batchesFilenameOption, c, (x) => { return Path.Join(wd, Path.GetFileName(x)); }) @@ -115,6 +122,7 @@ public static Options Build( Dedup = bitcoinDedupOps, GraphSample = gsample, MapMarket = bitcoinMapMarketOps, + Augmentor = bitcoinAugmentorOps, MapSpends = bitcoinMapSpendsOps }; diff --git a/EBA/Graph/Bitcoin/BitcoinGraphAgent.cs b/EBA/Graph/Bitcoin/BitcoinGraphAgent.cs index 70662327..b93ef3eb 100644 --- a/EBA/Graph/Bitcoin/BitcoinGraphAgent.cs +++ b/EBA/Graph/Bitcoin/BitcoinGraphAgent.cs @@ -46,10 +46,16 @@ public async Task SerializeAsync(BitcoinGraph g, CancellationToken ct) await _db.SerializeAsync(g, ct); } - public async Task PostProcessAsync(CancellationToken ct) + public async Task PostBullkImportFinalizeAsync(CancellationToken ct) { - var postProcess = new PostProcess(_options, _db, _logger); - await postProcess.Run(); + var finalizer = new PostBulkImportFinalizer(_options, _db, _logger); + await finalizer.Finalize(ct); + } + + public async Task AddMarketData(CancellationToken ct) + { + var augmentor = new OffChain.EconomicAugmentor(_options, _db, _logger); + await augmentor.SetBlockMarketIndicators(ct); } public void Dispose() diff --git a/EBA/Graph/Bitcoin/Factories/NodeFactory.cs b/EBA/Graph/Bitcoin/Factories/NodeFactory.cs index 4f42c242..b0511295 100644 --- a/EBA/Graph/Bitcoin/Factories/NodeFactory.cs +++ b/EBA/Graph/Bitcoin/Factories/NodeFactory.cs @@ -61,4 +61,19 @@ public static bool TryCreate( $"Unexpected node type, labels: {string.Join(',', node.Labels)}"); } } + + public static bool TryCreate(List records, out List nodes, string nodeVar = "n") where T: Model.INode + { + nodes = []; + foreach (var record in records) + { + TryCreate(record[nodeVar].As(), out var createdNode); + if (createdNode is not T) + return false; + + nodes.Add((T)createdNode); + } + + return true; + } } diff --git a/EBA/Graph/Bitcoin/OffChain/EconomicAugmentor.cs b/EBA/Graph/Bitcoin/OffChain/EconomicAugmentor.cs new file mode 100644 index 00000000..2d9d5850 --- /dev/null +++ b/EBA/Graph/Bitcoin/OffChain/EconomicAugmentor.cs @@ -0,0 +1,81 @@ +using EBA.Graph.Bitcoin.Factories; +using EBA.Graph.Bitcoin.Strategies; +using EBA.Utilities; + +namespace EBA.Graph.Bitcoin.OffChain; + +public class EconomicAugmentor(Options options, IGraphDb graphDb, ILogger logger) +{ + private readonly Options _options = options; + private readonly IGraphDb _graphDb = graphDb; + private readonly ILogger _logger = logger; + + public async Task SetBlockMarketIndicators(CancellationToken ct) + { + if (!OHLCV.TryParseFile(_options.Bitcoin.Augmentor.BlockOhlcvMappedFilename, out var blockOHLCVMapping)) + { + _logger.LogError( + "Failed to parse OHLCV data from file: {filename}", + _options.Bitcoin.Augmentor.BlockOhlcvMappedFilename); + return; + } + else + { + _logger.LogInformation( + "Successfully parsed OHLCV data for {count:n0} blocks from file: {filename}", + blockOHLCVMapping.Count, + _options.Bitcoin.Augmentor.BlockOhlcvMappedFilename); + } + + var blockNodes = await GetBlockNodes(ct); + + foreach (var block in blockNodes) + if (blockOHLCVMapping.TryGetValue(block.Key, out var ohlcv)) + block.Value.BlockMetadata.Ohlcv = ohlcv; + + await ComputeBlockValuationMetrics(blockNodes, blockOHLCVMapping, ct); + } + + private async Task> GetBlockNodes(CancellationToken ct) + { + _logger.LogInformation("Fetching block nodes."); + var blockRecords = await _graphDb.GetNodesAsync(NodeKind.Block, ct, nodeVariable: "b"); + _logger.LogInformation("Fetched {count:n0} block nodes.", blockRecords.Count); + + var blocks = new SortedDictionary(); + if (NodeFactory.TryCreate(blockRecords, out var blockNodes, nodeVar: "b")) + { + foreach (var blockNode in blockNodes) + blocks.Add(blockNode.BlockMetadata.Height, blockNode); + } + else + { + throw new Exception("Invalid node types received from database"); + } + + return blocks; + } + + // TODO: this can be set as part of the pre-bulk import finalization step, + // instead of being computed and updated after the fact. + // This would require reordering the import steps to ensure that the OHLCV data is available before the block nodes are created. + private async Task ComputeBlockValuationMetrics( + SortedDictionary blocks, + Dictionary blockOHLCVMapping, + CancellationToken ct) + { + _logger.LogInformation("Setting realized cap for {count:n0} block nodes.", blocks.Count); + await _graphDb.SetRealizedCap(blocks, blockOHLCVMapping, CancellationToken.None); + + _logger.LogInformation("Saving realized cap for {count:n0} block nodes.", blocks.Count); + var updates = blocks.Values + .Select(b => BlockNodeStrategy.EconomicMappings.ToDictionary(b)) + .ToList(); + + await _graphDb.BulkUpdateNodePropertiesAsync( + NodeKind.Block, + nameof(BlockMetadata.Height), + updates, + ct); + } +} diff --git a/EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs b/EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs new file mode 100644 index 00000000..dd149bc3 --- /dev/null +++ b/EBA/Graph/Bitcoin/PostBulkImportFinalizer.cs @@ -0,0 +1,107 @@ +using EBA.Graph.Bitcoin.Factories; + +namespace EBA.Graph.Bitcoin; + +public class PostBulkImportFinalizer( + Options options, + IGraphDb graphDb, + ILogger logger) +{ + private readonly Options _options = options; + private readonly IGraphDb _graphDb = graphDb; + private readonly ILogger _logger = logger; + + public async Task Finalize(CancellationToken ct) + { + await AddSchemaAndSeeding(ct); + + await SetSupplyAmount(ct); + } + + private async Task AddSchemaAndSeeding(CancellationToken ct) + { + _logger.LogInformation("Setting schema and seeding data."); + + var schemas = new List(); + var seedingCommands = new List(); + foreach (var nodeStrategy in _graphDb.StrategyFactory.NodeStrategies) + { + schemas.AddRange(nodeStrategy.Value.GetSchemaConfigs()); + seedingCommands.AddRange(nodeStrategy.Value.GetSeedingCommands()); + } + foreach (var edgeStrategy in _graphDb.StrategyFactory.EdgeStrategies) + { + schemas.AddRange(edgeStrategy.Value.GetSchemaConfigs()); + seedingCommands.AddRange(edgeStrategy.Value.GetSeedingCommands()); + } + + _logger.LogInformation("Executing {count:n0} schema configuration commands.", schemas.Count); + await _graphDb.ExecuteWriteQueryAsync(schemas, ct); + + _logger.LogInformation("Executing {count:n0} seeding commands.", seedingCommands.Count); + await _graphDb.ExecuteWriteQueryAsync(seedingCommands, ct); + + _logger.LogInformation("Completed setting schema and seeding data."); + } + + private async Task SetSupplyAmount(CancellationToken ct) + { + _logger.LogInformation("Setting {f} property.", nameof(BlockMetadata.TotalSupply)); + + _logger.LogInformation("Fetching {n} nodes from graph database.", BlockNode.Kind); + var nodeVar = "n"; + var records = await _graphDb.GetNodesAsync(NodeKind.Block, CancellationToken.None, nodeVariable: nodeVar); + _logger.LogInformation("Fetching {n} nodes from graph database succeeded, retrieved {count:n0} nodes.", BlockNode.Kind, records.Count); + + NodeFactory.TryCreate(records, out var blockNodes, nodeVar); + + var blocks = new SortedList(); + foreach (var block in blockNodes) + blocks.Add(block.BlockMetadata.Height, block); + + if (blocks.First().Value.BlockMetadata.Height != 0) + { + throw new InvalidOperationException( + $"The first block in the graph has height " + + $"{blocks.First().Value.BlockMetadata.Height:,}, " + + $"expected 0."); + } + + if (blocks.Last().Value.BlockMetadata.Height != blocks.Count - 1) + { + throw new InvalidOperationException( + $"This operation requires a continues set of blocks, there are missing blocks"); + } + + blocks[0].BlockMetadata.TotalSupply = blocks[0].BlockMetadata.MintedBitcoins; + blocks[0].BlockMetadata.TotalSupplyNominal = blocks[0].BlockMetadata.TotalSupply; + for (var i = 1; i < blocks.Count; i++) + { + blocks[i].BlockMetadata.TotalSupply = + blocks[i - 1].BlockMetadata.TotalSupply + + blocks[i].TripletTypeValueSum[C2TEdge.Kind] - + blocks[i].BlockMetadata.ProvablyUnspendableBitcoins; + + blocks[i].BlockMetadata.TotalSupplyNominal = + blocks[i - 1].BlockMetadata.TotalSupplyNominal + blocks[i].BlockMetadata.MintedBitcoins; + } + + var updates = blocks.Values.Select(b => new Dictionary + { + [nameof(BlockMetadata.Height)] = b.BlockMetadata.Height, + [nameof(BlockMetadata.TotalSupply)] = b.BlockMetadata.TotalSupply, + [nameof(BlockMetadata.TotalSupplyNominal)] = b.BlockMetadata.TotalSupplyNominal, + }).ToList(); + + _logger.LogInformation("Pushing {count:n0} {n} node updates to graph database.", updates.Count, BlockNode.Kind); + await _graphDb.BulkUpdateNodePropertiesAsync( + NodeKind.Block, + nameof(BlockMetadata.Height), + updates, + ct); + + _logger.LogInformation( + "Successfully updated {f} property of {n} nodes.", + nameof(BlockMetadata.TotalSupply), BlockNode.Kind); + } +} diff --git a/EBA/Graph/Bitcoin/PostProcess.cs b/EBA/Graph/Bitcoin/PostProcess.cs deleted file mode 100644 index 78e04022..00000000 --- a/EBA/Graph/Bitcoin/PostProcess.cs +++ /dev/null @@ -1,76 +0,0 @@ -using EBA.Graph.Bitcoin.Factories; - -namespace EBA.Graph.Bitcoin; - -public class PostProcess -{ - private readonly Options _options; - private readonly IGraphDb _graphDb; - private readonly ILogger _logger; - - public PostProcess(Options options, IGraphDb graphDb, ILogger logger) - { - _options = options; - _graphDb = graphDb; - _logger = logger; - } - - public async Task Run() - { - var blocks = new SortedList(); - var nodeVar = "n"; - - _logger.LogInformation("Fetching blocks from graph database."); - var records = await _graphDb.GetNodesAsync(NodeKind.Block, CancellationToken.None, nodeVariable: nodeVar); - _logger.LogInformation("Retrieved {count:n0} records from graph database. Creating block nodes.", records.Count); - - foreach (var record in records) - { - NodeFactory.TryCreate(record[nodeVar].As(), out var blockNode); - var block = (BlockNode)blockNode; - blocks.Add(block.BlockMetadata.Height, block); - } - - if (blocks.First().Value.BlockMetadata.Height != 0) - { - throw new InvalidOperationException( - $"The first block in the graph has height " + - $"{blocks.First().Value.BlockMetadata.Height:,}, " + - $"expected 0."); - } - - if (blocks.Last().Value.BlockMetadata.Height != blocks.Count - 1) - { - throw new InvalidOperationException( - $"This operation requires a continues set of blocks, there are missing blocks"); - } - - blocks[0].BlockMetadata.TotalSupply = blocks[0].BlockMetadata.MintedBitcoins; - blocks[0].BlockMetadata.TotalSupplyNominal = blocks[0].BlockMetadata.TotalSupply; - for (var i = 1; i < blocks.Count; i++) - { - blocks[i].BlockMetadata.TotalSupply = - blocks[i - 1].BlockMetadata.TotalSupply + - blocks[i].TripletTypeValueSum[C2TEdge.Kind] - - blocks[i].BlockMetadata.ProvablyUnspendableBitcoins; - - blocks[i].BlockMetadata.TotalSupplyNominal = - blocks[i - 1].BlockMetadata.TotalSupplyNominal + blocks[i].BlockMetadata.MintedBitcoins; - } - - var updates = blocks.Values.Select(b => new Dictionary - { - [nameof(BlockMetadata.Height)] = b.BlockMetadata.Height, - [nameof(BlockMetadata.TotalSupply)] = b.BlockMetadata.TotalSupply, - [nameof(BlockMetadata.TotalSupplyNominal)] = b.BlockMetadata.TotalSupplyNominal, - }).ToList(); - - _logger.LogInformation("Pushing {count:n0} block updates to graph database.", updates.Count); - await _graphDb.BulkUpdateNodePropertiesAsync( - NodeKind.Block, - nameof(BlockMetadata.Height), - updates, - CancellationToken.None); - _logger.LogInformation("Completed pushing block updates."); - } -} diff --git a/EBA/Graph/Bitcoin/Strategies/B2BEdgeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/B2BEdgeStrategy.cs index 3b829ffc..a1fc0850 100644 --- a/EBA/Graph/Bitcoin/Strategies/B2BEdgeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/B2BEdgeStrategy.cs @@ -7,7 +7,7 @@ public class B2BEdgeStrategy(bool serializeCompressed) $"edges_{B2BEdge.Kind.Source}_{B2BEdge.Kind.Relation}_{B2BEdge.Kind.Target}", serializeCompressed) { - private static readonly PropertyMapping[] _mappings = + public static readonly PropertyMapping[] Mappings = [ Factory.SourceId(BlockNodeStrategy.IdSpace, e => e.BlockHeight), Factory.TargetId(BlockNodeStrategy.IdSpace, e => e.BlockHeight), @@ -16,7 +16,7 @@ public class B2BEdgeStrategy(bool serializeCompressed) public override string GetCsvHeader() { - return _mappings.GetCsvHeader(); + return Mappings.GetCsvHeader(); } public override string GetCsvRow(IGraphElement edge) @@ -26,7 +26,7 @@ public override string GetCsvRow(IGraphElement edge) public static string GetCsvRow(B2BEdge edge) { - return _mappings.GetCsv(edge); + return Mappings.GetCsv(edge); } public static B2BEdge Deserialize(BlockNode source, BlockNode target, IRelationship relationship) @@ -38,4 +38,14 @@ public override string GetQuery(string filename) { throw new NotImplementedException(); } + + public override string[] GetSeedingCommands() + { + return + [ + $"MATCH (target:Block), (source:Block) " + + $"WHERE target.{nameof(B2BEdge.BlockHeight)} + 1 = source.{nameof(B2BEdge.BlockHeight)} " + + $"MERGE (target)-[:{RelationType.Follows}]->(source)" + ]; + } } diff --git a/EBA/Graph/Bitcoin/Strategies/B2TEdgeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/B2TEdgeStrategy.cs index b87a2bf3..49b1700b 100644 --- a/EBA/Graph/Bitcoin/Strategies/B2TEdgeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/B2TEdgeStrategy.cs @@ -7,7 +7,7 @@ public class B2TEdgeStrategy(bool serializeCompressed) $"edges_{B2TEdge.Kind.Source}_{B2TEdge.Kind.Relation}_{B2TEdge.Kind.Target}", serializeCompressed) { - private static readonly PropertyMapping[] _mappings = + public static readonly PropertyMapping[] Mappings = [ Factory.SourceId(BlockNodeStrategy.IdSpace, e => e.Source.BlockMetadata.Height), Factory.TargetId(TxNodeStrategy.IdSpace, e => e.Target.Txid), @@ -18,7 +18,7 @@ public class B2TEdgeStrategy(bool serializeCompressed) public override string GetCsvHeader() { - return _mappings.GetCsvHeader(); + return Mappings.GetCsvHeader(); } public override string GetCsvRow(IGraphElement edge) @@ -28,7 +28,7 @@ public override string GetCsvRow(IGraphElement edge) public static string GetCsv(B2TEdge edge) { - return _mappings.GetCsv(edge); + return Mappings.GetCsv(edge); } public static B2TEdge Deserialize(BlockNode source, TxNode target, IRelationship relationship) @@ -37,8 +37,8 @@ public static B2TEdge Deserialize(BlockNode source, TxNode target, IRelationship source: source, target: target, timestamp: 0, - blockHeight: _mappings.Get(Factory.HeightProperty.Name).Deserialize(relationship.Properties), - value: _mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties)); + blockHeight: Mappings.Get(Factory.HeightProperty.Name).Deserialize(relationship.Properties), + value: Mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties)); } public override string GetQuery(string filename) diff --git a/EBA/Graph/Bitcoin/Strategies/BitcoinStrategyBase.cs b/EBA/Graph/Bitcoin/Strategies/BitcoinStrategyBase.cs index 4108031f..f1ab4fe6 100644 --- a/EBA/Graph/Bitcoin/Strategies/BitcoinStrategyBase.cs +++ b/EBA/Graph/Bitcoin/Strategies/BitcoinStrategyBase.cs @@ -2,6 +2,6 @@ namespace EBA.Graph.Bitcoin.Strategies; -public abstract class BitcoinStrategyBase(string defaultBaseFilename, bool serializeCompressed) +public abstract class BitcoinStrategyBase(string defaultBaseFilename, bool serializeCompressed) : StrategyBase(defaultBaseFilename, serializeCompressed) { } \ No newline at end of file diff --git a/EBA/Graph/Bitcoin/Strategies/BitcoinStrategyFactory.cs b/EBA/Graph/Bitcoin/Strategies/BitcoinStrategyFactory.cs index dce39eab..f9db5bd1 100644 --- a/EBA/Graph/Bitcoin/Strategies/BitcoinStrategyFactory.cs +++ b/EBA/Graph/Bitcoin/Strategies/BitcoinStrategyFactory.cs @@ -28,7 +28,8 @@ public BitcoinStrategyFactory(Options options) { T2TEdge.KindFee, new T2TEdgeStrategy(T2TEdge.KindFee, compressOutput) }, { S2TEdge.Kind, new S2TEdgeStrategy(compressOutput) }, { T2SEdge.Kind, new T2SEdgeStrategy(compressOutput) }, - { B2TEdge.Kind, new B2TEdgeStrategy(compressOutput) } + { B2TEdge.Kind, new B2TEdgeStrategy(compressOutput) }, + { B2BEdge.Kind, new B2BEdgeStrategy(compressOutput) } }; } diff --git a/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs index 45c5c017..5ee99ee1 100644 --- a/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/BlockNodeStrategy.cs @@ -10,7 +10,23 @@ public class BlockNodeStrategy(bool serializeCompressed) public static string IdSpace { get; } = BlockNode.Kind.ToString(); private const Block v = null!; - private static readonly PropertyMapping[] _mappings = + + private static readonly PropertyMapping[] _economicMappings = + [ + new(nameof(v.RealizedCap), FieldType.Double, n => (double?)n.BlockMetadata.RealizedCap), + new(nameof(v.MarketCap), FieldType.Double, n => (double?)n.BlockMetadata.MarketCap), + new(nameof(v.NUPL), FieldType.Double, n => (double?)n.BlockMetadata.NUPL), + + .. PropertyMappingFactory.OHLCV(n => n.BlockMetadata.Ohlcv), + ]; + + public static PropertyMapping[] EconomicMappings { get; } = + [ + PropertyMappingFactory.Height(n => n.BlockMetadata.Height), + .. _economicMappings, + ]; + + public static readonly PropertyMapping[] Mappings = [ new("", FieldType.String, x=>x.BlockMetadata.Height, p => p.GetIdFieldCsvHeader(IdSpace.ToString())), PropertyMappingFactory.Height(n => n.BlockMetadata.Height), @@ -64,16 +80,21 @@ .. PropertyMappingFactory.DictionaryToColumns( .. PropertyMappingFactory.DictionaryToColumns( nameof(BlockNode.TripletTypeValueSum), Schema.EdgeKinds, n => n.TripletTypeValueSum), + new(nameof(v.TotalSupply), FieldType.Long, n => n.BlockMetadata.TotalSupply), + new(nameof(v.TotalSupplyNominal), FieldType.Long, n => n.BlockMetadata.TotalSupplyNominal), + + .. _economicMappings, + new(":LABEL", FieldType.String, _ => BlockNode.Kind, _ => ":LABEL"), ]; private static readonly Dictionary> _mappingsDict = - _mappings.ToDictionary(m => m.Property.Name, m => m); + Mappings.ToDictionary(m => m.Property.Name, m => m); public override string GetCsvHeader() { - return _mappings.GetCsvHeader(); + return Mappings.GetCsvHeader(); } public override string GetCsvRow(IGraphElement component) @@ -83,7 +104,7 @@ public override string GetCsvRow(IGraphElement component) public static string GetCsv(BlockNode node) { - return _mappings.GetCsv(node); + return Mappings.GetCsv(node); } // TODO: need a deserializer from string that returns only a given property, @@ -143,7 +164,13 @@ public static BlockNode Deserialize( InputScriptTypeCount = PropertyMappingFactory.ReadScriptTypeCounts("Inputs", props), OutputScriptTypeCount = PropertyMappingFactory.ReadScriptTypeCounts("Outputs", props), InputScriptTypeValue = PropertyMappingFactory.ReadScriptTypeCounts("Inputs", props), - OutputScriptTypeValue = PropertyMappingFactory.ReadScriptTypeCounts("Outputs", props) + OutputScriptTypeValue = PropertyMappingFactory.ReadScriptTypeCounts("Outputs", props), + + TotalSupply = _mappingsDict[nameof(v.TotalSupply)].Deserialize(props), + TotalSupplyNominal = _mappingsDict[nameof(v.TotalSupplyNominal)].Deserialize(props), + + RealizedCap = (decimal?)_mappingsDict[nameof(v.RealizedCap)].Deserialize(props), + Ohlcv = PropertyMappingFactory.ReadOHLCV(props) }; var blockNode = new BlockNode( @@ -206,4 +233,15 @@ public override string GetQuery(string filename) */ throw new NotImplementedException(); } + + public override string[] GetSchemaConfigs() + { + var heightName = PropertyMappingFactory.HeightProperty.Name; + return + [ + $"CREATE CONSTRAINT {BlockNode.Kind}_{heightName}_Unique " + + $"IF NOT EXISTS " + + $"FOR (v:{BlockNode.Kind}) REQUIRE v.{heightName} IS UNIQUE" + ]; + } } \ No newline at end of file diff --git a/EBA/Graph/Bitcoin/Strategies/C2TEdgeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/C2TEdgeStrategy.cs index 1521d6c6..dbba754b 100644 --- a/EBA/Graph/Bitcoin/Strategies/C2TEdgeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/C2TEdgeStrategy.cs @@ -9,7 +9,7 @@ public class C2TEdgeStrategy(bool serializeCompressed) { public static string IdSpaceCoinbase { get; } = CoinbaseNode.Kind.ToString(); - public static readonly PropertyMapping[] _mappings = + public static readonly PropertyMapping[] Mappings = [ Factory.SourceId(IdSpaceCoinbase, _ => CoinbaseNode.Kind), Factory.TargetId(TxNodeStrategy.IdSpace, e => e.Target.Txid), @@ -20,7 +20,7 @@ public class C2TEdgeStrategy(bool serializeCompressed) public override string GetCsvHeader() { - return _mappings.GetCsvHeader(); + return Mappings.GetCsvHeader(); } public override string GetCsvRow(IGraphElement edge) @@ -30,16 +30,16 @@ public override string GetCsvRow(IGraphElement edge) public static string GetCsv(C2TEdge edge) { - return _mappings.GetCsv(edge); + return Mappings.GetCsv(edge); } public static C2TEdge Deserialize(TxNode target, IRelationship relationship) { return new C2TEdge( target: target, - value: _mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties), + value: Mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties), timestamp: 0, - blockHeight: _mappings.Get(Factory.HeightProperty.Name).Deserialize(relationship.Properties)); + blockHeight: Mappings.Get(Factory.HeightProperty.Name).Deserialize(relationship.Properties)); } public override string GetQuery(string csvFilename) diff --git a/EBA/Graph/Bitcoin/Strategies/PropertyMapping.cs b/EBA/Graph/Bitcoin/Strategies/PropertyMapping.cs index f635f766..8217cd84 100644 --- a/EBA/Graph/Bitcoin/Strategies/PropertyMapping.cs +++ b/EBA/Graph/Bitcoin/Strategies/PropertyMapping.cs @@ -36,6 +36,8 @@ public PropertyMapping( deserializer) { } + public object? GetValue(T source) => _propertySelector(source); + public string SerializeHeader() { return _headerOverride?.Invoke(Property) ?? Property.TypeAnnotatedCsvHeader; diff --git a/EBA/Graph/Bitcoin/Strategies/PropertyMappingExtensions.cs b/EBA/Graph/Bitcoin/Strategies/PropertyMappingExtensions.cs index 65ecb230..837afaad 100644 --- a/EBA/Graph/Bitcoin/Strategies/PropertyMappingExtensions.cs +++ b/EBA/Graph/Bitcoin/Strategies/PropertyMappingExtensions.cs @@ -27,4 +27,14 @@ public static PropertyMapping Get(this PropertyMapping[] mappings, stri throw new KeyNotFoundException($"No mapping found for property '{propertyName}'."); } + + public static Dictionary ToDictionary( + this PropertyMapping[] mappings, + T source) + { + var dict = new Dictionary(mappings.Length); + foreach (var m in mappings) + dict[m.Property.Name] = m.GetValue(source); + return dict; + } } diff --git a/EBA/Graph/Bitcoin/Strategies/PropertyMappingFactory.cs b/EBA/Graph/Bitcoin/Strategies/PropertyMappingFactory.cs index 02717cd9..664b972c 100644 --- a/EBA/Graph/Bitcoin/Strategies/PropertyMappingFactory.cs +++ b/EBA/Graph/Bitcoin/Strategies/PropertyMappingFactory.cs @@ -197,4 +197,40 @@ private static string EdgeKindToPropertyName(EdgeKind edgeKind) // Make sure to keep EdgeKind string representation compatible with neo4j header requirements. return $"{edgeKind.Source}_{edgeKind.Relation}_{edgeKind.Target}"; } + + + public static PropertyMapping[] OHLCV( + Func getOhlcv, + string prefix = "OHLCV") + { + OHLCV o = null!; + return + [ + new($"{prefix}.{nameof(o.Open)}", FieldType.Double, s => (double?)(getOhlcv(s)?.Open)), + new($"{prefix}.{nameof(o.High)}", FieldType.Double, s => (double?)(getOhlcv(s)?.High)), + new($"{prefix}.{nameof(o.Low)}", FieldType.Double, s => (double?)(getOhlcv(s)?.Low)), + new($"{prefix}.{nameof(o.Close)}", FieldType.Double, s => (double?)(getOhlcv(s)?.Close)), + new($"{prefix}.{nameof(o.Volume)}", FieldType.Long, s => getOhlcv(s)?.Volume), + new($"{prefix}.{nameof(o.VWAP)}", FieldType.Double, s => (double?)(getOhlcv(s)?.VWAP)), + new($"{prefix}.{nameof(o.OHLC4)}", FieldType.Double, s => (double?)(getOhlcv(s)?.OHLC4)) + ]; + } + + public static OHLCV? ReadOHLCV( + IReadOnlyDictionary properties, + string prefix = "OHLCV") + { + OHLCV o; + if (!properties.TryGetValue($"{prefix}.{nameof(o.Open)}", out _)) + return null; + + return new OHLCV( + timestamp: 0, + open: (decimal)(double)properties[$"{prefix}.{nameof(o.Open)}"], + high: (decimal)(double)properties[$"{prefix}.{nameof(o.High)}"], + low: (decimal)(double)properties[$"{prefix}.{nameof(o.Low)}"], + close: (decimal)(double)properties[$"{prefix}.{nameof(o.Close)}"], + volume: (long)properties[$"{prefix}.{nameof(o.Volume)}"], + vwap: (decimal)(double)properties[$"{prefix}.{nameof(o.VWAP)}"]); + } } \ No newline at end of file diff --git a/EBA/Graph/Bitcoin/Strategies/S2TEdgeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/S2TEdgeStrategy.cs index 3a7e9e99..e48e576f 100644 --- a/EBA/Graph/Bitcoin/Strategies/S2TEdgeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/S2TEdgeStrategy.cs @@ -9,7 +9,7 @@ public class S2TEdgeStrategy(bool serializeCompressed) $"edges_{S2TEdge.Kind.Source}_{S2TEdge.Kind.Relation}_{S2TEdge.Kind.Target}", serializeCompressed) { - private static readonly PropertyMapping[] _mappings = + public static readonly PropertyMapping[] Mappings = [ Factory.SourceId(ScriptNodeStrategy.IdSpace, e => e.Source.Id), Factory.TargetId(TxNodeStrategy.IdSpace, e => e.Target.Txid), @@ -24,7 +24,7 @@ public class S2TEdgeStrategy(bool serializeCompressed) public override string GetCsvHeader() { - return _mappings.GetCsvHeader(); + return Mappings.GetCsvHeader(); } public override string GetCsvRow(IGraphElement edge) @@ -34,7 +34,7 @@ public override string GetCsvRow(IGraphElement edge) public static string GetCsvRow(S2TEdge edge) { - return _mappings.GetCsv(edge); + return Mappings.GetCsv(edge); } public static S2TEdge Deserialize(ScriptNode source, TxNode target, IRelationship relationship) @@ -43,12 +43,12 @@ public static S2TEdge Deserialize(ScriptNode source, TxNode target, IRelationshi source: source, target: target, timestamp: 0, - spentHeight: _mappings.Get(nameof(S2TEdge.SpentHeight)).Deserialize(relationship.Properties), - value: _mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties), - txid: _mappings.Get(nameof(S2TEdge.Txid)).Deserialize(relationship.Properties), - vout: _mappings.Get(nameof(S2TEdge.Vout)).Deserialize(relationship.Properties), - generated: _mappings.Get(nameof(S2TEdge.Generated)).Deserialize(relationship.Properties), - creationHeight: _mappings.Get(nameof(S2TEdge.CreationHeight)).Deserialize(relationship.Properties) + creationHeight: Mappings.Get(nameof(S2TEdge.CreationHeight)).Deserialize(relationship.Properties), + spentHeight: Mappings.Get(nameof(S2TEdge.SpentHeight)).Deserialize(relationship.Properties), + value: Mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties), + txid: Mappings.Get(nameof(S2TEdge.Txid)).Deserialize(relationship.Properties), + vout: Mappings.Get(nameof(S2TEdge.Vout)).Deserialize(relationship.Properties), + generated: Mappings.Get(nameof(S2TEdge.Generated)).Deserialize(relationship.Properties) ); } diff --git a/EBA/Graph/Bitcoin/Strategies/ScriptNodeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/ScriptNodeStrategy.cs index 5ba8cbf6..a2eef04e 100644 --- a/EBA/Graph/Bitcoin/Strategies/ScriptNodeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/ScriptNodeStrategy.cs @@ -15,7 +15,7 @@ public class ScriptNodeStrategy(bool serializeCompressed) n => n.SHA256Hash, p => p.GetIdFieldCsvHeader(IdSpace)); - private readonly static PropertyMapping[] _mappings = + public readonly static PropertyMapping[] Mappings = [ _sha256Hash, new(nameof(v.Address), FieldType.String, n => n.Address), @@ -25,11 +25,11 @@ public class ScriptNodeStrategy(bool serializeCompressed) ]; private static readonly Dictionary> _mappingsDict = - _mappings.ToDictionary(m => m.Property.Name, m => m); + Mappings.ToDictionary(m => m.Property.Name, m => m); public override string GetCsvHeader() { - return _mappings.GetCsvHeader(); + return Mappings.GetCsvHeader(); } public override string GetCsvRow(IGraphElement component) @@ -39,7 +39,7 @@ public override string GetCsvRow(IGraphElement component) public static string GetCsv(ScriptNode node) { - return _mappings.GetCsv(node); + return Mappings.GetCsv(node); } public static ScriptNode Deserialize( @@ -104,4 +104,14 @@ public override string GetQuery(string filename) */ throw new NotImplementedException(); } + + public override string[] GetSchemaConfigs() + { + return + [ + $"CREATE CONSTRAINT {ScriptNode.Kind}_{_sha256Hash.Property.Name}_Unique " + + $"IF NOT EXISTS " + + $"FOR (v:{ScriptNode.Kind}) REQUIRE v.{_sha256Hash.Property.Name} IS UNIQUE" + ]; + } } \ No newline at end of file diff --git a/EBA/Graph/Bitcoin/Strategies/T2SEdgeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/T2SEdgeStrategy.cs index 2c4594c3..b0394061 100644 --- a/EBA/Graph/Bitcoin/Strategies/T2SEdgeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/T2SEdgeStrategy.cs @@ -9,7 +9,7 @@ public class T2SEdgeStrategy(bool serializeCompressed) $"edges_{T2SEdge.Kind.Source}_{T2SEdge.Kind.Relation}_{T2SEdge.Kind.Target}", serializeCompressed) { - public static readonly PropertyMapping[] _mappings = + public static readonly PropertyMapping[] Mappings = [ Factory.SourceId(TxNodeStrategy.IdSpace, e => e.Source.Txid), Factory.TargetId(ScriptNodeStrategy.IdSpace, e => e.Target.Id), @@ -22,7 +22,7 @@ public class T2SEdgeStrategy(bool serializeCompressed) public override string GetCsvHeader() { - return _mappings.GetCsvHeader(); + return Mappings.GetCsvHeader(); } public override string GetCsvRow(IGraphElement edge) @@ -32,7 +32,7 @@ public override string GetCsvRow(IGraphElement edge) public static string GetCsv(T2SEdge edge) { - return _mappings.GetCsv(edge); + return Mappings.GetCsv(edge); } public static T2SEdge Deserialize(TxNode source, ScriptNode target, IRelationship relationship) @@ -41,14 +41,24 @@ public static T2SEdge Deserialize(TxNode source, ScriptNode target, IRelationshi source: source, target: target, timestamp: 0, - creationHeight: _mappings.Get(nameof(T2SEdge.CreationHeight)).Deserialize(relationship.Properties), - value: _mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties), - outputIndex: _mappings.Get(nameof(T2SEdge.Vout)).Deserialize(relationship.Properties), - spentHeight: _mappings.Get(nameof(T2SEdge.SpentHeight)).Deserialize(relationship.Properties)); + creationHeight: Mappings.Get(nameof(T2SEdge.CreationHeight)).Deserialize(relationship.Properties), + value: Mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties), + outputIndex: Mappings.Get(nameof(T2SEdge.Vout)).Deserialize(relationship.Properties), + spentHeight: Mappings.Get(nameof(T2SEdge.SpentHeight)).Deserialize(relationship.Properties)); } public override string GetQuery(string filename) { throw new NotImplementedException(); } + + public override string[] GetSchemaConfigs() + { + return + [ + $"CREATE INDEX utxo_spending_idx IF NOT EXISTS " + + $"FOR ()-[r:{T2SEdge.Kind.Relation}]-() " + + $"ON (r.{nameof(T2SEdge.CreationHeight)}, r.{nameof(T2SEdge.SpentHeight)})" + ]; + } } \ No newline at end of file diff --git a/EBA/Graph/Bitcoin/Strategies/T2TEdgeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/T2TEdgeStrategy.cs index 562c9c2d..4ac1dc06 100644 --- a/EBA/Graph/Bitcoin/Strategies/T2TEdgeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/T2TEdgeStrategy.cs @@ -10,7 +10,7 @@ public class T2TEdgeStrategy(EdgeKind kind, bool serializeCompressed) $"{kind.Target}", serializeCompressed) { - private static readonly PropertyMapping[] _mappings = + public static readonly PropertyMapping[] Mappings = [ Factory.SourceId(TxNodeStrategy.IdSpace, e => e.Source.Txid), Factory.TargetId(TxNodeStrategy.IdSpace, e => e.Target.Txid), @@ -21,7 +21,7 @@ public class T2TEdgeStrategy(EdgeKind kind, bool serializeCompressed) public override string GetCsvHeader() { - return _mappings.GetCsvHeader(); + return Mappings.GetCsvHeader(); } public override string GetCsvRow(IGraphElement edge) @@ -31,7 +31,7 @@ public override string GetCsvRow(IGraphElement edge) public static string GetCsv(T2TEdge edge) { - return _mappings.GetCsv(edge); + return Mappings.GetCsv(edge); } public static T2TEdge Deserialize(TxNode source, TxNode target, IRelationship relationship) @@ -40,8 +40,8 @@ public static T2TEdge Deserialize(TxNode source, TxNode target, IRelationship re source: source, target: target, timestamp: 0, - blockHeight: _mappings.Get(Factory.HeightProperty.Name).Deserialize(relationship.Properties), - value: _mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties), + blockHeight: Mappings.Get(Factory.HeightProperty.Name).Deserialize(relationship.Properties), + value: Mappings.Get(Factory.ValueProperty.Name).Deserialize(relationship.Properties), type: Enum.Parse(relationship.Type)); } diff --git a/EBA/Graph/Bitcoin/Strategies/TxNodeStrategy.cs b/EBA/Graph/Bitcoin/Strategies/TxNodeStrategy.cs index e05f7df5..d72087e9 100644 --- a/EBA/Graph/Bitcoin/Strategies/TxNodeStrategy.cs +++ b/EBA/Graph/Bitcoin/Strategies/TxNodeStrategy.cs @@ -14,7 +14,7 @@ public class TxNodeStrategy(bool serializeCompressed) public static string IdSpace { get; } = TxNode.Kind.ToString(); private const TxNode v = null!; - private static readonly PropertyMapping[] _mappings = + public static readonly PropertyMapping[] Mappings = [ PropertyMappingFactory.Txid(n => n.Txid, p => p.GetIdFieldCsvHeader(IdSpace)), new(nameof(v.Version), FieldType.Long, n => n.Version), @@ -27,11 +27,11 @@ public class TxNodeStrategy(bool serializeCompressed) ]; private static readonly Dictionary> _mappingsDict = - _mappings.ToDictionary(m => m.Property.Name, m => m); + Mappings.ToDictionary(m => m.Property.Name, m => m); public override string GetCsvHeader() { - return _mappings.GetCsvHeader(); + return Mappings.GetCsvHeader(); } public override string GetCsvRow(IGraphElement component) @@ -41,7 +41,7 @@ public override string GetCsvRow(IGraphElement component) public static string GetCsv(TxNode node) { - return _mappings.GetCsv(node); + return Mappings.GetCsv(node); } public static TxNode Deserialize( @@ -96,4 +96,18 @@ public override string GetQuery(string filename) return builder.ToString();*/ throw new NotImplementedException(); } + + public override string[] GetSchemaConfigs() + { + var txidName = PropertyMappingFactory.Txid(n => n.Txid).Property.Name; + return + [ + $"CREATE CONSTRAINT {TxNode.Kind}_{txidName}_Unique " + + $"IF NOT EXISTS " + + $"FOR (v:{TxNode.Kind}) REQUIRE v.{txidName} IS UNIQUE", + + $"CREATE INDEX tx_txid_index IF NOT EXISTS " + + $"FOR (t:{TxNode.Kind}) ON (t.{nameof(TxNode.Txid)})" + ]; + } } \ No newline at end of file diff --git a/EBA/Graph/Db/IGraphDb.cs b/EBA/Graph/Db/IGraphDb.cs index 6745ebac..3f326502 100644 --- a/EBA/Graph/Db/IGraphDb.cs +++ b/EBA/Graph/Db/IGraphDb.cs @@ -1,7 +1,12 @@ -namespace EBA.Graph.Db; +using EBA.Graph.Db.Neo4jDb; +using EBA.Utilities; + +namespace EBA.Graph.Db; public interface IGraphDb : IDisposable, IAsyncDisposable where T : GraphBase { + public IStrategyFactory StrategyFactory { get; } + public Task VerifyConnectivityAsync(CancellationToken ct); public Task SerializeAsync(T graph, CancellationToken ct); @@ -47,4 +52,13 @@ public Task> GetNodesAsync( CancellationToken ct, string nodeVariable = "n", int? count = null); + + public Task ExecuteWriteQueryAsync( + List schemas, + CancellationToken ct); + + public Task SetRealizedCap( + SortedDictionary blockNodes, + Dictionary ohlcv, + CancellationToken ct); } \ No newline at end of file diff --git a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs index eefc47b2..d10390f3 100644 --- a/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs +++ b/EBA/Graph/Db/Neo4jDb/Neo4jDb.cs @@ -1,9 +1,16 @@ -namespace EBA.Graph.Db.Neo4jDb; +using EBA.Utilities; + +namespace EBA.Graph.Db.Neo4jDb; + +// TODO: all the property name setting here should not nameof(), +// instead they should use the property mapping in the strategies. public class Neo4jDb : IGraphDb where T : GraphBase { private readonly Options _options; private readonly IDriver _driver; + + public IStrategyFactory StrategyFactory { get { return _strategyFactory; } } private readonly IStrategyFactory _strategyFactory; private List _batches = []; @@ -64,10 +71,10 @@ public async Task> GetRandomNodesAsync( } public async Task> GetNodesAsync( - NodeKind label, - CancellationToken ct, + NodeKind label, + CancellationToken ct, string nodeVariable = "n", - int? count=null) + int? count = null) { await VerifyConnectivityAsync(ct); @@ -87,11 +94,11 @@ public async Task> GetNodesAsync( } public async Task> GetNeighborsAsync( - NodeKind rootNodeLabel, + NodeKind rootNodeLabel, string rootNodeIdProperty, string rootNodeId, - int queryLimit, - int maxLevel, + int queryLimit, + int maxLevel, bool useBFS, CancellationToken ct, string relationshipFilter = "") @@ -107,7 +114,7 @@ public async Task> GetNeighborsAsync( if (useBFS) qBuilder.Append($"bfs: true "); - else + else qBuilder.Append($"bfs: false "); //qBuilder.Append($", labelFilter: '{labelFilters}'"); @@ -169,7 +176,7 @@ public async Task SerializeConstantsAndConstraintsAsync(CancellationToken ct) await _strategyFactory.SerializeSchemasAsync(_options.WorkingDir, ct); } - public async Task SerializeAsync(T g,CancellationToken ct) + public async Task SerializeAsync(T g, CancellationToken ct) { var nodes = g.GetNodes(); var edges = g.GetEdges(); @@ -181,7 +188,7 @@ public async Task SerializeAsync(T g,CancellationToken ct) { batchInfo.Update(nodeGroup.Key, nodeGroup.Value.Count); var _strategy = _strategyFactory.GetStrategy(nodeGroup.Key); - if (_strategy == null) + if (_strategy == null) continue; tasks.Add(_strategy.ToCsvAsync(nodeGroup.Value, batchInfo.GetFilename(nodeGroup.Key))); @@ -217,6 +224,159 @@ private async Task GetBatchAsync() return _batches[^1]; } + private async Task> DeserializeBatchesAsync() + { + return await JsonSerializer>.DeserializeAsync( + _options.Neo4j.BatchesFilename); + } + + public async Task ExecuteWriteQueryAsync(List queries, CancellationToken ct) + { + await VerifyConnectivityAsync(ct); + + await using var session = _driver.AsyncSession( + x => x.WithDefaultAccessMode(AccessMode.Write)); + + foreach (var query in queries) + { + ct.ThrowIfCancellationRequested(); + + _logger.LogInformation("Executing write query: {schema}", query); + + var summary = await session.ExecuteWriteAsync(async tx => + { + var cursor = await tx.RunAsync(query); + return await cursor.ConsumeAsync(); + }); + + _logger.LogInformation("Finished executing write query; {counters}", summary.Counters); + } + } + + public async Task SetRealizedCap(BlockNode blockNode, Dictionary ohlcv, CancellationToken ct) + { + await VerifyConnectivityAsync(CancellationToken.None); + await using var session = _driver.AsyncSession(x => x.WithDefaultAccessMode(AccessMode.Read)); + + var readTx = await session.BeginTransactionAsync(); + + var cursor = await readTx.RunAsync( + $"MATCH (tx)-[r:{T2SEdge.Kind.Relation}]->(script) " + + $"WHERE r.{nameof(T2SEdge.CreationHeight)} <= {blockNode.BlockMetadata.Height} " + + $"AND r.{nameof(T2SEdge.SpentHeight)} > {blockNode.BlockMetadata.Height} " + + $"AND script.{nameof(ScriptNode.ScriptType)} <> '{ScriptType.NullData}' " + + $"RETURN r"); + + decimal realizedCap = 0; + + while (await cursor.FetchAsync()) + { + var r = cursor.Current["r"].As(); + var creationHeight = (long)r.Properties[nameof(T2SEdge.CreationHeight)]; + var value = (long)r.Properties[nameof(T2SEdge.Value)]; + + if (ohlcv.TryGetValue(creationHeight, out var blockOHLCV)) + { + realizedCap += blockOHLCV.GetFiatValue(value); + } + } + + blockNode.BlockMetadata.RealizedCap = realizedCap; + } + + private async Task GetEdgeTypeCount(RelationType relationType, CancellationToken ct) + { + await using var session = _driver.AsyncSession(x => x.WithDefaultAccessMode(AccessMode.Read)); + + long edgeCount = await session.ExecuteReadAsync(async tx => + { + var cursor = await tx.RunAsync($"MATCH ()-[r:{relationType}]->() RETURN count(r)"); + var record = await cursor.SingleAsync(); + return record[0].As(); + }); + + return edgeCount; + } + + // TODO: this method should not be here; + // it is specific to the Bitcoin graph model + // and this class should be kept agnostic to the graph model. + public async Task SetRealizedCap( + SortedDictionary blockNodes, + Dictionary ohlcv, + CancellationToken ct) + { + if (blockNodes.Count == 0) + return; + + var sortedHeights = blockNodes.Keys.ToArray(); + var minHeight = sortedHeights[0]; + var maxHeight = sortedHeights[^1]; + + foreach (var block in blockNodes.Values) + block.BlockMetadata.RealizedCap = 0; + + await VerifyConnectivityAsync(ct); + await using var session = _driver.AsyncSession(x => x.WithDefaultAccessMode(AccessMode.Read)); + + var edgeCounter = 0; + double totalEdgeCount = await GetEdgeTypeCount(T2SEdge.Kind.Relation, ct); + var processedEdgeCount = 0; + var skippedEdgeCounter = 0; + + _logger.LogInformation("Starting iteration through {b} edges.", T2SEdge.Kind.Relation); + + await session.ExecuteReadAsync(async tx => + { + var cursor = await tx.RunAsync( + $"MATCH (tx)-[r:{T2SEdge.Kind.Relation}]->(script) " + + $"WHERE " + + $"r.{nameof(T2SEdge.CreationHeight)} <= {maxHeight} " + + $"AND " + + $"r.{nameof(T2SEdge.SpentHeight)} > {minHeight} " + + $"AND " + + $"script.{nameof(ScriptNode.ScriptType)} <> '{ScriptType.NullData}' " + + $"RETURN " + + $"r.{nameof(T2SEdge.CreationHeight)} AS creationHeight, " + + $"r.{nameof(T2SEdge.SpentHeight)} AS spentHeight, " + + $"r.{nameof(T2SEdge.Value)} AS value"); + + while (await cursor.FetchAsync()) + { + ct.ThrowIfCancellationRequested(); + edgeCounter++; + + var creationHeight = cursor.Current["creationHeight"].As(); + var spentHeight = cursor.Current["spentHeight"].As(); + var value = cursor.Current["value"].As(); + + if (!ohlcv.TryGetValue(creationHeight, out var blockOHLCV)) + { + skippedEdgeCounter++; + } + else + { + var fiatValue = blockOHLCV.GetFiatValue(value); + foreach (var h in sortedHeights.GetViewBetween(creationHeight, spentHeight)) + blockNodes[h].BlockMetadata.RealizedCap += fiatValue; + + processedEdgeCount++; + } + + if (edgeCounter % 100000 == 0) + { + _logger.LogInformation( + "Traversed {e:n0} / {t:n0} edges. Processed {p:n0} edges and skipped {s:n0} edges due to missing related OHLCV data.", + edgeCounter, totalEdgeCount, processedEdgeCount, skippedEdgeCounter); + } + } + }); + + _logger.LogInformation( + "Traversed {e:n0} / {t:n0} edges. Processed {p:n0} edges and skipped {s:n0} edges due to missing related OHLCV data.", + edgeCounter, totalEdgeCount, processedEdgeCount, skippedEdgeCounter); + } + public void Dispose() { Dispose(true); @@ -264,21 +424,18 @@ public async Task BulkUpdateNodePropertiesAsync( await VerifyConnectivityAsync(ct); + var sampleProps = updates[0].Keys.Where(k => k != idProperty); var setClause = string.Join( - ", ", - updates[0].Keys - .Where(k => k != idProperty) - .Select(k => $"n.{k} = row.{k}")); + ", ", + sampleProps.Select(k => $"n.`{k}` = row.`{k}`")); - // using toString() on the id property to match the string-typed - // :ID column created by neo4j-admin import. var query = "UNWIND $batch AS row " + - $"MATCH (n:{label} {{{idProperty}: toString(row.{idProperty})}}) " + + $"MATCH (n:{label} {{`{idProperty}`: row.`{idProperty}`}}) " + $"SET {setClause}"; - var chunkIndex = 0; - foreach (var chunk in updates.Chunk(_options.Neo4j.MaxEntitiesPerBatch)) + var batchIndex = 0; + foreach (var batch in updates.Chunk(_options.Neo4j.MaxEntitiesPerBatch)) { ct.ThrowIfCancellationRequested(); @@ -287,20 +444,13 @@ public async Task BulkUpdateNodePropertiesAsync( { var cursor = await tx.RunAsync( query, - new Dictionary { ["batch"] = chunk }); + new Dictionary { ["batch"] = batch }); return await cursor.ConsumeAsync(); }); - var counters = summary.Counters; - if (counters.PropertiesSet == 0) - _logger.LogWarning("Chunk {chunk}: 0 properties set (MATCH may have found no nodes).", - chunkIndex); - else - _logger.LogInformation("Chunk {chunk}: set {props:n0} properties on nodes.", - chunkIndex, counters.PropertiesSet); - - chunkIndex++; + _logger.LogInformation("Finished batch {batch}; summary: {s}", batchIndex, summary.Counters); + batchIndex++; } _logger.LogInformation( diff --git a/EBA/Graph/Db/Neo4jDb/StrategyBase.cs b/EBA/Graph/Db/Neo4jDb/StrategyBase.cs index 783725d5..1e032906 100644 --- a/EBA/Graph/Db/Neo4jDb/StrategyBase.cs +++ b/EBA/Graph/Db/Neo4jDb/StrategyBase.cs @@ -61,6 +61,16 @@ await GetStreamWriter(filename).WriteLineAsync( public abstract string GetQuery(string filename); + public virtual string[] GetSchemaConfigs() + { + return []; + } + + public virtual string[] GetSeedingCommands() + { + return []; + } + public void Dispose() { Dispose(true); diff --git a/EBA/Graph/IGraphAgent.cs b/EBA/Graph/IGraphAgent.cs index ed3cd3c9..af448256 100644 --- a/EBA/Graph/IGraphAgent.cs +++ b/EBA/Graph/IGraphAgent.cs @@ -4,5 +4,5 @@ public interface IGraphAgent where T : GraphBase { public Task SampleAsync(CancellationToken ct); public Task SerializeAsync(T g, CancellationToken ct); - public Task PostProcessAsync(CancellationToken ct); + public Task PostBullkImportFinalizeAsync(CancellationToken ct); } \ No newline at end of file diff --git a/EBA/Orchestrator.cs b/EBA/Orchestrator.cs index 4a7be01b..e4814f2a 100644 --- a/EBA/Orchestrator.cs +++ b/EBA/Orchestrator.cs @@ -25,7 +25,8 @@ public Orchestrator(CancellationToken cancelationToken) bitcoinAddressStatsHandlerAsync: BitcoinAddressStatsAsync, bitcoinImportCypherQueriesAsync: BitcoinImportCypherQueriesAsync, bitcoinMapMarketHandlerAsync: BitcoinMarketMapAsync, - bitcoinPostProcessGraphHandlerAsync: BitcoinPostProcessGraph, + bitcoinPostProcessHandlerAsync: BitcoinPostProcess, + bitcoinAugmentHandlerAsync: BitcoinAugmentGraphAsync, bitcoinMapSpendsHandlerAsync: BitcoinMapSpends, exceptionHandler: (e, _) => { @@ -96,13 +97,21 @@ private async Task BitcoinImportCypherQueriesAsync(Options options) graphDb.ReportQueries(); } - private async Task BitcoinPostProcessGraph(Options options) + private async Task BitcoinPostProcess(Options options) { var host = await SetupAndGetHostAsync(options); await JsonSerializer.SerializeAsync(options, options.StatusFile, _cT); var bitcoinGraphAgent = host.Services.GetRequiredService(); - await bitcoinGraphAgent.PostProcessAsync(_cT); + await bitcoinGraphAgent.PostBullkImportFinalizeAsync(_cT); + } + + private async Task BitcoinAugmentGraphAsync(Options options) + { + var host = await SetupAndGetHostAsync(options); + await JsonSerializer.SerializeAsync(options, options.StatusFile, _cT); + var bitcoinGraphAgent = host.Services.GetRequiredService(); + await bitcoinGraphAgent.AddMarketData(_cT); } private async Task BitcoinSampleGraphAsync(Options options) diff --git a/EBA/Utilities/Extensions.cs b/EBA/Utilities/Extensions.cs new file mode 100644 index 00000000..f5cc4f73 --- /dev/null +++ b/EBA/Utilities/Extensions.cs @@ -0,0 +1,19 @@ +namespace EBA.Utilities; + +public static class Extensions +{ + public static IEnumerable GetViewBetween( + this long[] sortedArray, + long lowerBound, + long upperBound) + { + int idx = Array.BinarySearch(sortedArray, lowerBound); + if (idx < 0) idx = ~idx; + + for (int i = idx; i < sortedArray.Length; i++) + { + if (sortedArray[i] >= upperBound) break; + yield return sortedArray[i]; + } + } +} diff --git a/website/docs/bitcoin/etl/s5a-import.md b/website/docs/bitcoin/etl/s5a-import.md index 04c76f69..abd6760a 100644 --- a/website/docs/bitcoin/etl/s5a-import.md +++ b/website/docs/bitcoin/etl/s5a-import.md @@ -139,22 +139,19 @@ meaning it does not support incremental updates to an existing graph. ```shell sudo -u neo4j HEAP_SIZE=4G neo4j-admin database import full \ --overwrite-destination neo4j \ - --nodes="$GDIR/BitcoinCoinbase.tsv.gz" \ - --nodes="$GDIR/BitcoinGraph_header.tsv.gz,$GDIR/0_BitcoinGraph.tsv.gz" \ - --nodes="$GDIR/BitcoinScriptNode_header.tsv.gz,$GDIR/unique_BitcoinScriptNode.tsv.gz" \ - --nodes="$GDIR/BitcoinTxNode_header.tsv.gz,$GDIR/unique_BitcoinTxNode.tsv.gz" \ - --relationships="$GDIR/BitcoinS2S_header.tsv.gz,$GDIR/.*BitcoinS2S.tsv.gz" \ - --relationships="$GDIR/BitcoinT2T_header.tsv.gz,$GDIR/.*BitcoinT2T.tsv.gz" \ - --relationships="$GDIR/BitcoinC2T_header.tsv.gz,$GDIR/.*BitcoinC2T.tsv.gz" \ - --relationships="$GDIR/BitcoinC2S_header.tsv.gz,$GDIR/.*BitcoinC2S.tsv.gz" \ - --relationships="$GDIR/BitcoinB2T_header.tsv.gz,$GDIR/.*BitcoinB2T.tsv.gz" \ - --relationships="$GDIR/BitcoinB2S_header.tsv.gz,$GDIR/.*BitcoinB2S.tsv.gz" \ - --relationships="$GDIR/BitcoinS2B_header.tsv.gz,$GDIR/.*BitcoinS2B.tsv.gz" \ - --relationships="$GDIR/BitcoinT2B_header.tsv.gz,$GDIR/.*BitcoinT2B.tsv.gz" \ + --nodes="$GDIR/Coinbase.csv.gz" \ + --nodes="$GDIR/header_nodes_Block.csv.gz.gz,$GDIR/[0-9].*_nodes_Block.csv.gz" \ + --nodes="$GDIR/header_nodes_Script.csv.gz,$GDIR/unique_nodes_Script.csv.gz" \ + --nodes="$GDIR/header_nodes_Tx.csv.gz,$GDIR/unique_nodes_Tx.tsv.gz" \ + --relationships="$GDIR/header_edges_Block_Confirms_Tx.csv.gz,$GDIR/[0-9].*_edges_Block_Confirms_Tx.csv.gz" \ + --relationships="$GDIR/header_edges_Coinbase_Mints_Tx.csv.gz,$GDIR/[0-9].*_edges_Coinbase_Mints_Tx.csv.gz" \ + --relationships="$GDIR/header_edges_Tx_Credits_Script.csv.gz,$GDIR/[0-9].*_edges_Tx_Credits_Script.csv.gz" \ + --relationships="$GDIR/header_edges_Script_Redeems_Tx.csv.gz,$GDIR/[0-9].*_edges_Script_Redeems_Tx.csv.gz" \ + --relationships="$GDIR/header_edges_Tx_Fee_Tx.csv.gz,$GDIR/[0-9].*_edges_Tx_Fee_Tx.csv.gz" \ + --relationships="$GDIR/header_edges_Tx_Transfers_Tx.csv.gz,$GDIR/[0-9].*_edges_Tx_Transfers_Tx.csv.gz" \ --delimiter="\t" \ --array-delimiter=";" \ - --verbose \ - --skip-duplicate-nodes + --verbose ``` diff --git a/website/docs/bitcoin/etl/s6-db-config.md b/website/docs/bitcoin/etl/s6-db-config.md index eb1482b3..41938f38 100644 --- a/website/docs/bitcoin/etl/s6-db-config.md +++ b/website/docs/bitcoin/etl/s6-db-config.md @@ -44,44 +44,13 @@ To improve query performance we define database constraints, such as uniqueness constraints, which index commonly used properties. - -EBA generates a `schema.cypher` file containing the queries required -to create these constraints and indexes. -EBA creates this file in the same directory where the Bitcoin graph was serialized. - - -The Cypher queries in the `schema.cypher` file are as follows: - -```javascript -// EBA Bitcoin Graph Schema - -// Uniqueness constraint for Script.Address property. -CREATE CONSTRAINT Script_Address_Unique -IF NOT EXISTS -FOR (v:Script) REQUIRE v.Address IS UNIQUE; - -// Uniqueness constraint for Tx.Txid property. -CREATE CONSTRAINT Tx_Txid_Unique -IF NOT EXISTS -FOR (v:Tx) REQUIRE v.Txid IS UNIQUE; - -// Uniqueness constraint for Block.Height property. -CREATE CONSTRAINT Block_Height_Unique -IF NOT EXISTS -FOR (v:Block) REQUIRE v.Height IS UNIQUE; -``` - - Take the following steps to apply these constraints to your database: + 1. Start the Neo4j database service. -2. Execute the schema.cypher file using cypher-shell: +2. Run the `post-process` subcommand. ```Shell - cypher-shell -u neo4j -p password -f schema.cypher + .\eba.exe bitcoin post-process ``` - - Please refer to - [this documentation](https://neo4j.com/docs/operations-manual/current/cypher-shell/) - for details on locating and running `cypher-shell` depending on your Neo4j deployment. diff --git a/website/docs/bitcoin/etl/s7-off-chain.md b/website/docs/bitcoin/etl/s7-off-chain.md new file mode 100644 index 00000000..29cb9767 --- /dev/null +++ b/website/docs/bitcoin/etl/s7-off-chain.md @@ -0,0 +1,49 @@ +--- +title: Augmentation +description: Step 7. Augment graph using off-chain data +sidebar_label: Augmentation +sidebar_position: 7 +slug: /bitcoin/etl/augment +--- + +EBA currently supports adding market data to the Bitcoin graph. +This enriches the graph with market-related features that enable quantitative analysis. + + +### Step 1: Get Market Data + +You may download Bitcoin market data from any source you prefer. +We recommend the following publicly accessible source: + +``` +https://www.kaggle.com/datasets/mczielinski/bitcoin-historical-data +``` + +### Step 2: Map Market Data to Blocks + +Bitcoin targets an average block interval of about 10 minutes +(check [this paper](https://arxiv.org/abs/2510.20028) for detailed plots). +For this step, EBA uses each block's median time (`mediantime`) +to align market data with blocks. +Because market data is typically sampled at shorter intervals +and is not synchronized with block times, +EBA maps candles to each block using the time window between +two consecutive blocks' median times. +For each block, candles with timestamps in that interval are +used to compute aggregated OHLCV values. + +You may run the following command for this step. + +```shell +.\eba.exe bitcoin map-market --ohlcv-source-filename btcusd_1-min_data.csv --block-market-output-filename mapped-block-ohlcv.tsv +``` + +### Step 3: Augment the Graph + +Run the following command to add market-related features to the graph. + +```shell +.\eba.exe bitcoin augment --ohlcv-filename mapped-block-ohlcv.tsv +``` + +Note that this process may take a considerable amount of time to complete.