From b8ad2de7e5718bc7d4afdc43ae8fd095279b9a2b Mon Sep 17 00:00:00 2001 From: Vahid Date: Tue, 9 Jun 2026 22:19:23 -0400 Subject: [PATCH 1/2] Deprecate the option to track utxo lifecycle as part of the traverse. --- .../Blockchains/Bitcoin/BitcoinChainAgent.cs | 55 ------------------- .../Bitcoin/BitcoinOrchestrator.cs | 2 - .../Blockchains/Bitcoin/ChainModel/Block.cs | 2 - src/AAB.EBA/CLI/CLI.cs | 12 ---- src/AAB.EBA/CLI/OptionsBinder.cs | 2 - .../PersistentObject/PersistentGraphBuffer.cs | 9 --- 6 files changed, 82 deletions(-) diff --git a/src/AAB.EBA/Blockchains/Bitcoin/BitcoinChainAgent.cs b/src/AAB.EBA/Blockchains/Bitcoin/BitcoinChainAgent.cs index b4b2df54..7fa1c09e 100644 --- a/src/AAB.EBA/Blockchains/Bitcoin/BitcoinChainAgent.cs +++ b/src/AAB.EBA/Blockchains/Bitcoin/BitcoinChainAgent.cs @@ -265,23 +265,8 @@ private async Task ProcessBlockAsync( var mintingTxGraph = new TxGraph(coinbaseTx); foreach (var output in coinbaseTx.Outputs) - { mintingTxGraph.AddOutput(output); - if (options.Traverse.TrackTxo) - { - output.TryGetAddress(out string? address); - var utxo = new Utxo( - output.ScriptPubKey, - address, - output.Value, - output.ScriptPubKey.ScriptType, - isGenerated: true, - createdInHeight: block.Height); - g.Block.TxoLifecycle.TryAdd(utxo.Id, utxo); - } - } - g.SetCoinbaseTx(mintingTxGraph); cT.ThrowIfCancellationRequested(); @@ -317,53 +302,13 @@ private static async Task ProcessTx(BlockGraph g, Tx tx, BitcoinOptions options, foreach (var input in tx.Inputs) { cT.ThrowIfCancellationRequested(); - txGraph.AddInput(input); - - if (options.Traverse.TrackTxo) - { - var prevOut = input.PrevOut.ConstructedOutput; - prevOut.TryGetAddress(out string? address); - - var utxo = new Utxo( - input.PrevOut.ScriptPubKey, - address: address, - value: input.PrevOut.Value, - isGenerated: input.PrevOut.Generated, - scriptType: input.PrevOut.ConstructedOutput.ScriptPubKey.ScriptType, - createdInHeight: input.PrevOut.Height, - spentInHeight: g.Block.Height); - - g.Block.TxoLifecycle.AddOrUpdate(utxo.Id, utxo, (_, oldValue) => - { - oldValue.SpentInHeight = g.Block.Height; - return oldValue; - }); - } } foreach (var output in tx.Outputs) { cT.ThrowIfCancellationRequested(); - txGraph.AddOutput(output); - - if (output.ScriptPubKey.ScriptType != ScriptType.NullData && - output.ScriptPubKey.ScriptType != ScriptType.nonstandard && - options.Traverse.TrackTxo) - { - output.TryGetAddress(out string? address); - - var utxo = new Utxo( - id: Utxo.GetId(txGraph.TxNode.Txid, output.N), - address: address, - value: output.Value, - scriptType: output.ScriptPubKey.ScriptType, - isGenerated: false, - createdInHeight: g.Block.Height); - - g.Block.TxoLifecycle.TryAdd(utxo.Id, utxo); - } } g.Enqueue(txGraph); diff --git a/src/AAB.EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs b/src/AAB.EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs index 2ead60d1..7de441c3 100644 --- a/src/AAB.EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs +++ b/src/AAB.EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs @@ -222,8 +222,6 @@ void RegisterFailed(long h) using var gBuffer = new PersistentGraphBuffer( graphOrchestrator: _host.Services.GetRequiredService(), logger: _host.Services.GetRequiredService>(), - pTxoLifeCyccleLogger: options.Bitcoin.Traverse.TrackTxo ?_host.Services.GetRequiredService>() : null, - txoLifeCycleFilename: options.Bitcoin.Traverse.TrackTxo ? options.Bitcoin.Traverse.TxoFilename : null, maxTxoPerFile: options.Bitcoin.Traverse.MaxTxoPerFile, options: options, semaphore: pgbSemaphore, diff --git a/src/AAB.EBA/Blockchains/Bitcoin/ChainModel/Block.cs b/src/AAB.EBA/Blockchains/Bitcoin/ChainModel/Block.cs index 654e3a76..f37ad3f4 100644 --- a/src/AAB.EBA/Blockchains/Bitcoin/ChainModel/Block.cs +++ b/src/AAB.EBA/Blockchains/Bitcoin/ChainModel/Block.cs @@ -7,8 +7,6 @@ public class Block : BlockMetadata [JsonPropertyName("tx")] public List Transactions { init; get; } = []; - public ConcurrentDictionary TxoLifecycle { init; get; } = []; - public override DescriptiveStatistics InputCountsStats { get { return new DescriptiveStatistics([.. _inputsCounts]); } } private readonly ConcurrentBag _inputsCounts = []; diff --git a/src/AAB.EBA/CLI/CLI.cs b/src/AAB.EBA/CLI/CLI.cs index 4ec47ff9..b3617178 100644 --- a/src/AAB.EBA/CLI/CLI.cs +++ b/src/AAB.EBA/CLI/CLI.cs @@ -190,16 +190,6 @@ private Command GetBitcoinTraverseCmd(Options defaultOptions, Func("--track-txo") - { - Description = - "If set, writes the list of txo it sees to a text file, this file will need to further processed" + - "and it will also add to storage requirements. " + - "Enabling this will slow down the traverse (e.g., from 7h to 11h for the first 500k blocks), and additional storage " + - "requirements (e.g., ~140GB for the first 500k blocks) that needs post-traverse processing. Aggregated stats about Txo " + - "are recoded in block stats, so set this flag only if you need the complete list of spent and unspent Tx outputs." - }; - var txoFilenameOption = new Option("--txo-filename") { Description = @@ -250,7 +240,6 @@ private Command GetBitcoinTraverseCmd(Options defaultOptions, Func? batchFilenameOption = null, Option? maxBlocksInBufferOption = null, Option? txoFilenameOption = null, - Option? trackTxoOption = null, Option? skipGraphSerializationOption = null, Option? maxEntriesPerBatch = null, Option? sortedTxNodeFilenameOption = null, @@ -56,7 +55,6 @@ public static Options Build( BlocksFailedToProcessListFilename = Path.Join(wd, defs.Bitcoin.Traverse.BlocksFailedToProcessListFilename), MaxBlocksInBuffer = GetValue(defs.Bitcoin.Traverse.MaxBlocksInBuffer, maxBlocksInBufferOption, c), TxoFilename = GetValue(defs.Bitcoin.Traverse.TxoFilename, txoFilenameOption, c, (x) => { return Path.Join(wd, Path.GetFileName(x)); }), - TrackTxo = GetValue(defs.Bitcoin.Traverse.TrackTxo, trackTxoOption, c), SkipGraphSerialization = GetValue(defs.Bitcoin.Traverse.SkipGraphSerialization, skipGraphSerializationOption, c) }; diff --git a/src/AAB.EBA/PersistentObject/PersistentGraphBuffer.cs b/src/AAB.EBA/PersistentObject/PersistentGraphBuffer.cs index 0506377f..57d1dc7b 100644 --- a/src/AAB.EBA/PersistentObject/PersistentGraphBuffer.cs +++ b/src/AAB.EBA/PersistentObject/PersistentGraphBuffer.cs @@ -6,7 +6,6 @@ public class PersistentGraphBuffer : PersistentObjectBase, IDisposab { private readonly Graph.Bitcoin.BitcoinGraphOrchestrator? _graphOrchestrator; private readonly ILogger _logger; - private readonly PersistentTxoLifeCycleBuffer? _pTxoLifeCycleBuffer = null; private readonly SemaphoreSlim _semaphore; private bool _disposed = false; @@ -22,8 +21,6 @@ public ReadOnlyCollection BlocksHeightInBuffer public PersistentGraphBuffer( Graph.Bitcoin.BitcoinGraphOrchestrator? graphOrchestrator, ILogger logger, - ILogger? pTxoLifeCyccleLogger, - string? txoLifeCycleFilename, int maxTxoPerFile, SemaphoreSlim semaphore, Options options, @@ -33,9 +30,6 @@ public PersistentGraphBuffer( _graphOrchestrator = graphOrchestrator; _logger = logger; - if (txoLifeCycleFilename != null && pTxoLifeCyccleLogger != null) - _pTxoLifeCycleBuffer = new(txoLifeCycleFilename, maxTxoPerFile, pTxoLifeCyccleLogger, ct); - _semaphore = semaphore; } @@ -62,9 +56,6 @@ public override async Task SerializeAsync( if (_graphOrchestrator != null) tasks.Add(_graphOrchestrator.SerializeAsync(obj, default)); - if (_pTxoLifeCycleBuffer != null) - tasks.Add(_pTxoLifeCycleBuffer.SerializeAsync(obj.Block.TxoLifecycle.Values, default)); - await Task.WhenAll(tasks); _blocksHeightsInBuffer.TryRemove(obj.Block.Height, out byte _); From 96bcc16548b8b3d416c5470efe0b9571933db293 Mon Sep 17 00:00:00 2001 From: Vahid Date: Tue, 9 Jun 2026 23:39:35 -0400 Subject: [PATCH 2/2] Update docs --- .../docs/bitcoin/etl/s2-traverse-bitcoin.md | 21 ++++++++++++++++++- website/docs/bitcoin/etl/s3-off-chain.md | 10 +++++++-- website/docs/bitcoin/etl/s4a-import.md | 4 ++-- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/website/docs/bitcoin/etl/s2-traverse-bitcoin.md b/website/docs/bitcoin/etl/s2-traverse-bitcoin.md index c04cf63f..73a2b44e 100644 --- a/website/docs/bitcoin/etl/s2-traverse-bitcoin.md +++ b/website/docs/bitcoin/etl/s2-traverse-bitcoin.md @@ -165,4 +165,23 @@ node files. ```shell .\aab.eba.exe bitcoin post-traverse --batches-filename batches.json -``` \ No newline at end of file +``` + +For each `*_nodes_Block` and `*_edges_Tx_Credits_Script` file, +the above command creates a new file that contains the updated information. +To ensure the downstream steps of the pipeline correctly refer to the files +containing the updated information, we can run the following script, +which creates a backup folder named `original` and moves the unmodified data files into it. +It then takes the newly updated files and removes the extra text from their filenames. + + +```shell +mkdir -p original && \ +mv *_nodes_Block.csv.gz *_edges_Tx_Credits_Script.csv.gz original/ 2>/dev/null; \ +for f in *_edges_Tx_Credits_Script_with_txo_spent_height_set.csv.gz; do \ + [ -e "$f" ] && mv "$f" "${f/_with_txo_spent_height_set/}"; \ +done; \ +for f in *_nodes_Block_supply_updated.csv.gz; do \ + [ -e "$f" ] && mv "$f" "${f/_supply_updated/}"; \ +done +``` diff --git a/website/docs/bitcoin/etl/s3-off-chain.md b/website/docs/bitcoin/etl/s3-off-chain.md index 964247d3..ec4360f4 100644 --- a/website/docs/bitcoin/etl/s3-off-chain.md +++ b/website/docs/bitcoin/etl/s3-off-chain.md @@ -43,7 +43,13 @@ You may run the following command for this step. Run the following command to add market-related features to the graph. ```shell -.\aab.eba.exe bitcoin augment --ohlcv-filename mapped-block-ohlcv.tsv +.\aab.eba.exe bitcoin augment --ohlcv-filename mapped-block-ohlcv.tsv --batches-filename batches.json ``` -Note that this process may take a considerable amount of time to complete. +```shell +mkdir -p pre_augment && \ +mv *_nodes_Block.csv.gz pre_augment/ 2>/dev/null; \ +for f in *_nodes_Block_with_economic_indicators.csv.gz; do \ + [ -e "$f" ] && mv "$f" "${f/_with_economic_indicators/}"; \ +done +``` diff --git a/website/docs/bitcoin/etl/s4a-import.md b/website/docs/bitcoin/etl/s4a-import.md index 3cc60586..5214ed31 100644 --- a/website/docs/bitcoin/etl/s4a-import.md +++ b/website/docs/bitcoin/etl/s4a-import.md @@ -225,12 +225,12 @@ meaning it does not support incremental updates to an existing graph. & "$env:NDIR\bin\neo4j-admin.bat" database import full ` --overwrite-destination neo4j ` --nodes="$env:GDIR\Coinbase.csv.gz" ` - --nodes="$env:GDIR\header_nodes_Block.csv.gz,$env:GDIR\[0-9].*_nodes_Block_supply_updated.csv.gz" ` + --nodes="$env:GDIR\header_nodes_Block.csv.gz,$env:GDIR\[0-9].*_nodes_Block.csv.gz" ` --nodes="$env:GDIR\header_nodes_Script.csv.gz,$env:GDIR\unique_nodes_Script.csv.gz" ` --nodes="$env:GDIR\header_nodes_Tx.csv.gz,$env:GDIR\unique_nodes_Tx.csv.gz" ` --relationships="$env:GDIR\header_edges_Block_Confirms_Tx.csv.gz,$env:GDIR\[0-9].*_edges_Block_Confirms_Tx.csv.gz" ` --relationships="$env:GDIR\header_edges_Coinbase_Mints_Tx.csv.gz,$env:GDIR\[0-9].*_edges_Coinbase_Mints_Tx.csv.gz" ` - --relationships="$env:GDIR\header_edges_Tx_Credits_Script.csv.gz,$env:GDIR\[0-9].*_edges_Tx_Credits_Script_with_txo_spent_height_set.csv.gz" ` + --relationships="$env:GDIR\header_edges_Tx_Credits_Script.csv.gz,$env:GDIR\[0-9].*_edges_Tx_Credits_Script.csv.gz" ` --relationships="$env:GDIR\header_edges_Script_Redeems_Tx.csv.gz,$env:GDIR\[0-9].*_edges_Script_Redeems_Tx.csv.gz" ` --relationships="$env:GDIR\header_edges_Tx_Fee_Tx.csv.gz,$env:GDIR\[0-9].*_edges_Tx_Fee_Tx.csv.gz" ` --relationships="$env:GDIR\header_edges_Tx_Transfers_Tx.csv.gz,$env:GDIR\[0-9].*_edges_Tx_Transfers_Tx.csv.gz" `