Misc bug fix, refactoring, & docs update

EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs

@@ -215,7 +215,7 @@ void RegisterFailed(long h)
         // TODO: refactor the following so that only options is passed to the buffer
 
         using var gBuffer = new PersistentGraphBuffer(
-            graphAgent: _host.Services.GetRequiredService<Graph.Bitcoin.BitcoinGraphAgent>(),
+            graphOrchestrator: _host.Services.GetRequiredService<Graph.Bitcoin.BitcoinGraphOrchestrator>(),
             logger: _host.Services.GetRequiredService<ILogger<PersistentGraphBuffer>>(),
             pTxoLifeCyccleLogger: options.Bitcoin.Traverse.TrackTxo ?_host.Services.GetRequiredService<ILogger<PersistentTxoLifeCycleBuffer>>() : null,
             txoLifeCycleFilename: options.Bitcoin.Traverse.TrackTxo ? options.Bitcoin.Traverse.TxoFilename : null,

EBA/Blockchains/Bitcoin/Utilities/TraverseFinalizer.cs

@@ -20,16 +20,16 @@ public async Task UpdatePostTraverse(CancellationToken ct)
         var batches = await Batch.DeserializeBatchesAsync(_options.Bitcoin.MapSpends.BatchesFilename);
         _logger.LogInformation("Deserialized {n} batches from {filename}.", batches.Count, _options.Bitcoin.MapSpends.BatchesFilename);
 
-        _processStep = "[1/4] Block-to-batch mapping: ";
+        _processStep = "[1/4] Block-to-batch mapping:";
         GetBlockHeightToBatchMapping(_options, batches, out var blockToBatch, out var blockNodes);
 
-        _processStep = "[2/4] Collecting Txo spending: ";
+        _processStep = "[2/4] Collecting Txo spending:";
         await CreatePerBatchSpentUtxo(batches, blockToBatch, ct);
 
-        _processStep = "[3/4] Setting UTxO spending: ";
+        _processStep = "[3/4] Setting UTxO spending:";
         await SetTxoSpentHeight(batches, ct);
 
-        _processStep = "[4/4] Setting supply amount: ";
+        _processStep = "[4/4] Setting supply amount:";
         await SetSupplyAmount(batches, blockNodes, ct);
     }
 
@@ -211,10 +211,10 @@ private async Task SetSupplyAmount(List<Batch> batches, SortedDictionary<long, B
         var totalSupplyIndex = BlockNodeDescriptor.StaticMapper.GetPropertyCsvIndex(x => x.BlockMetadata.TotalSupply);
         var totalSupplyNominalIndex = BlockNodeDescriptor.StaticMapper.GetPropertyCsvIndex(x => x.BlockMetadata.TotalSupplyNominal);
 
+        _logger.LogInformation("{s} Updating block node files with total supply information.", _processStep);
+        int counter = 0;
         foreach (var batch in batches)
         {
-            _logger.LogInformation("{s} Updating block node files with total supply information...", _processStep);
-
             var blockNodesFilename = batch.GetFilename(BlockNode.Kind);
             using var writer = new StreamWriter(new GZipStream(
                 File.Create(Helpers.AddPostfixToFilename(blockNodesFilename, "_supply_updated")),
@@ -235,9 +235,17 @@ private async Task SetSupplyAmount(List<Batch> batches, SortedDictionary<long, B
             finally
             {
                 writer.Close();
-            }
 
-            _logger.LogInformation("{s} Finished updating block node files with total supply information.", _processStep);
+                counter++;
+                if (counter % 10 == 0)
+                    _logger.LogInformation(
+                        "{s} Finished updating block node files with total supply information for {n}/{total} batches.", 
+                        _processStep, counter, batches.Count);
+            }
         }
+
+        _logger.LogInformation(
+            "{s} Finished updating block node files with total supply information for {n}/{total} batches.",
+            _processStep, counter, batches.Count);
     }
 }

EBA/Graph/Bitcoin/BitcoinGraphAgent.cs → EBA/Graph/Bitcoin/BitcoinGraphOrchestrator.cs

@@ -2,20 +2,20 @@
 
 namespace EBA.Graph.Bitcoin;
 
-public class BitcoinGraphAgent : IGraphAgent<BitcoinGraph>, IDisposable
+public class BitcoinGraphOrchestrator : IGraphOrchestrator<BitcoinGraph>, IDisposable
 {
     private readonly Options _options;
     private readonly IGraphDb<BitcoinGraph> _db;
-    private readonly ILogger<BitcoinGraphAgent> _logger;
+    private readonly ILogger<BitcoinGraphOrchestrator> _logger;
 
     private bool _hasSerializedConstants = false;
 
     private bool _disposed = false;
 
-    public BitcoinGraphAgent(
+    public BitcoinGraphOrchestrator(
         Options options,
         IGraphDb<BitcoinGraph> graphDb,
-        ILogger<BitcoinGraphAgent> logger)
+        ILogger<BitcoinGraphOrchestrator> logger)
     {
         _options = options;
         _db = graphDb;

EBA/Graph/Bitcoin/Descriptors/BlockNodeDescriptor.cs

@@ -10,8 +10,12 @@ public class BlockNodeDescriptor : IElementDescriptor<BlockNode>
     public static ElementMapper<BlockNode> StaticMapper => _mapper;
     private static readonly ElementMapper<BlockNode> _mapper = new(
         new MappingBuilder<BlockNode>()
-            .MapSourceId(_idSpace, n => n.BlockMetadata.Height)
+            // using empty string as propName in header override since height property is already defined.
+            // Height is defined twice because id is treated string, but we want a long height as property,
+            // so only option currently is double-prop definition.
+            .Map(n => n.BlockMetadata.Height).WithCsvHeader(p => p.GetIdFieldCsvHeader("", _idSpace))
             .Map(n => n.BlockMetadata.Height)
+
             .Map(n => n.BlockMetadata.Hash)
             .Map(n => n.BlockMetadata.Confirmations)
             .Map(n => n.BlockMetadata.Version)
@@ -68,12 +72,6 @@ public class BlockNodeDescriptor : IElementDescriptor<BlockNode>
             .MapRange(PropertyMappingFactory.ToMappings<BlockNode, long>(
                     nameof(BlockNode.TripletTypeValueSum), n => n.TripletTypeValueSum))
 
-            .MapRange(PropertyMappingFactory.ToMappings(
-                nameof(BlockNode.TripletTypeCount), (BlockNode n) => n.TripletTypeCount))
-
-            .MapRange(PropertyMappingFactory.ToMappings(
-                nameof(BlockNode.TripletTypeValueSum), (BlockNode n) => n.TripletTypeValueSum))
-
             .Map(n => n.BlockMetadata.TotalSupply)
             .Map(n => n.BlockMetadata.TotalSupplyNominal)
 

EBA/Graph/Bitcoin/Descriptors/C2TEdgeDescriptor.cs

@@ -9,7 +9,7 @@ public class C2TEdgeDescriptor : IElementDescriptor<C2TEdge>
     public static ElementMapper<C2TEdge> StaticMapper => _mapper;
     private static readonly ElementMapper<C2TEdge> _mapper = new(
         new MappingBuilder<C2TEdge>()
-            .MapSourceId(_idSpace, _ => CoinbaseNode.Kind)
+            .MapSourceId(CoinbaseNode.Kind.ToString(), _ => CoinbaseNode.Kind)
             .MapTargetId(TxNodeDescriptor.IdSpace, e => e.Target.Txid)
             .Map(e => e.Value)
             .Map(e => e.BlockHeight)

EBA/Graph/Bitcoin/OffChain/EconomicAugmentor.cs

@@ -2,11 +2,11 @@
 
 namespace EBA.Graph.Bitcoin.OffChain;
 
-public class EconomicAugmentor(Options options, IGraphDb<BitcoinGraph> graphDb, ILogger<BitcoinGraphAgent> logger)
+public class EconomicAugmentor(Options options, IGraphDb<BitcoinGraph> graphDb, ILogger<BitcoinGraphOrchestrator> logger)
 {
     private readonly Options _options = options;
     private readonly IGraphDb<BitcoinGraph> _graphDb = graphDb;
-    private readonly ILogger<BitcoinGraphAgent> _logger = logger;
+    private readonly ILogger<BitcoinGraphOrchestrator> _logger = logger;
 
     public async Task SetBlockMarketIndicators(CancellationToken ct)
     {

EBA/Graph/Bitcoin/TraversalAlgorithms/ForestFire.cs

@@ -4,11 +4,11 @@ public class ForestFire : ITraversalAlgorithm
 {
     private readonly Options _options;
     private readonly IGraphDb<BitcoinGraph> _graphDb;
-    private readonly ILogger<BitcoinGraphAgent> _logger;
+    private readonly ILogger<BitcoinGraphOrchestrator> _logger;
 
     private int _maxHopReached = 0;
 
-    public ForestFire(Options options, IGraphDb<BitcoinGraph> graphDb, ILogger<BitcoinGraphAgent> logger)
+    public ForestFire(Options options, IGraphDb<BitcoinGraph> graphDb, ILogger<BitcoinGraphOrchestrator> logger)
     {
         _options = options;
         _graphDb = graphDb;

EBA/Graph/IGraphAgent.cs → EBA/Graph/IGraphOrchestrator.cs

@@ -1,6 +1,6 @@
 namespace EBA.Graph;
 
-public interface IGraphAgent<T> where T : GraphBase
+public interface IGraphOrchestrator<T> where T : GraphBase
 {
     public Task SampleAsync(CancellationToken ct);
     public Task SerializeAsync(T g, CancellationToken ct);

EBA/Graph/Model/Property.cs

@@ -50,6 +50,11 @@ public string GetIdFieldCsvHeader(string idGroup)
         return $"{Name}:ID({idGroup})";
     }
 
+    public string GetIdFieldCsvHeader(string propName, string idGroup)
+    {
+        return $"{propName}:ID({idGroup})";
+    }
+
     public string GetSetter()
     {
         return $"{Name}:{GetReader()}";

EBA/Infrastructure/StartupSolutions/Startup.cs

@@ -72,7 +72,7 @@ private static void ConfigureServices(IServiceCollection services, Options optio
         services.AddSingleton(options);
         services.AddSingleton<IGraphDb<BitcoinGraph>, BitcoinNeo4jDb>();
         services.AddSingleton<BitcoinOrchestrator>();
-        services.AddSingleton<Graph.Bitcoin.BitcoinGraphAgent>();
+        services.AddSingleton<Graph.Bitcoin.BitcoinGraphOrchestrator>();
 
         // Passing BitcoinAgent type as the generic argument
         // to AddHttpClient will cause registering it 

EBA/Orchestrator.cs

@@ -100,17 +100,17 @@ private async Task BitcoinAugmentGraphAsync(Options options)
     {
         var host = await SetupAndGetHostAsync(options);
         await JsonSerializer<Options>.SerializeAsync(options, options.StatusFile, _cT);
-        var bitcoinGraphAgent = host.Services.GetRequiredService<BitcoinGraphAgent>();
-        await bitcoinGraphAgent.AddMarketData(_cT);
+        var bitcoinGraphOrchestrator = host.Services.GetRequiredService<BitcoinGraphOrchestrator>();
+        await bitcoinGraphOrchestrator.AddMarketData(_cT);
     }
 
     private async Task BitcoinSampleGraphAsync(Options options)
     {
         var host = await SetupAndGetHostAsync(options);
         await JsonSerializer<Options>.SerializeAsync(options, options.StatusFile, _cT);
 
-        var bitcoinGraphAgent = host.Services.GetRequiredService<BitcoinGraphAgent>();
-        await bitcoinGraphAgent.SampleAsync(_cT);
+        var bitcoinGraphOrchestrator = host.Services.GetRequiredService<BitcoinGraphOrchestrator>();
+        await bitcoinGraphOrchestrator.SampleAsync(_cT);
     }
 
     private async Task BitcoinAddressStatsAsync(Options options)

EBA/PersistentObject/PersistentGraphBuffer.cs

@@ -4,7 +4,7 @@ namespace EBA.PersistentObject;
 
 public class PersistentGraphBuffer : PersistentObjectBase<BlockGraph>, IDisposable, IAsyncDisposable
 {
-    private readonly Graph.Bitcoin.BitcoinGraphAgent? _graphAgent;
+    private readonly Graph.Bitcoin.BitcoinGraphOrchestrator? _graphOrchestrator;
     private readonly ILogger<PersistentGraphBuffer> _logger;
     private readonly PersistentTxoLifeCycleBuffer? _pTxoLifeCycleBuffer = null;
     private readonly SemaphoreSlim _semaphore;
@@ -20,7 +20,7 @@ public ReadOnlyCollection<long> BlocksHeightInBuffer
     private readonly ConcurrentDictionary<long, byte> _blocksHeightsInBuffer = new();
 
     public PersistentGraphBuffer(
-        Graph.Bitcoin.BitcoinGraphAgent? graphAgent,
+        Graph.Bitcoin.BitcoinGraphOrchestrator? graphOrchestrator,
         ILogger<PersistentGraphBuffer> logger,
         ILogger<PersistentTxoLifeCycleBuffer>? pTxoLifeCyccleLogger,
         string? txoLifeCycleFilename,
@@ -30,7 +30,7 @@ public PersistentGraphBuffer(
         CancellationToken ct) :
         base(logger, ct)
     {
-        _graphAgent = graphAgent;
+        _graphOrchestrator = graphOrchestrator;
         _logger = logger;
 
         if (txoLifeCycleFilename != null && pTxoLifeCyccleLogger != null)
@@ -59,8 +59,8 @@ public override async Task SerializeAsync(
         // on cancellation and recovery, but that can add additional complexities.
         var tasks = new List<Task> { };
 
-        if (_graphAgent != null)
-            tasks.Add(_graphAgent.SerializeAsync(obj, default));
+        if (_graphOrchestrator != null)
+            tasks.Add(_graphOrchestrator.SerializeAsync(obj, default));
 
         if (_pTxoLifeCycleBuffer != null)
             tasks.Add(_pTxoLifeCycleBuffer.SerializeAsync(obj.Block.TxoLifecycle.Values, default));
@@ -107,7 +107,7 @@ public async Task WaitForBufferToEmptyAsync()
 
             if (disposing)
             {
-                _graphAgent?.Dispose();
+                _graphOrchestrator?.Dispose();
             }
 
             _disposed = true;
@@ -120,7 +120,7 @@ public async Task WaitForBufferToEmptyAsync()
         {
             base.Dispose(true);
 
-            if (_graphAgent is IAsyncDisposable asyncDisposable)
+            if (_graphOrchestrator is IAsyncDisposable asyncDisposable)
                 await asyncDisposable.DisposeAsync();
 
             _disposed = true;

website/docs/bitcoin/etl/s1-sync-bitcoin.mdx

@@ -72,8 +72,3 @@ EBA does **NOT** try to access Wallets defined in `Bitcoin Core`.
 Regardless, there are some known privacy concerns when running `Bitcoin Core` 
 with a defined wallet while the REST API is enabled. 
 See [Bitcoin docs](https://github.com/bitcoin/bitcoin/blob/master/doc/REST-interface.md#risks).
-
-
-
-
-

website/docs/bitcoin/etl/s2-traverse-bitcoin.md

@@ -33,33 +33,35 @@ For this task, you may take the following steps.
     .\eba.exe bitcoin traverse --help
     ```
 
+<details> 
+    <summary>Optimizations and Scaling Potential</summary>
+
+    Traversing Bitcoin blocks can take a considerable amount of time. 
+    To accelerate this, 
+    EBA heavily leverages multi-threading, 
+    and all time-consuming operations are implemented to be non-blocking. 
+    It also minimizes the latency between submitting API calls and 
+    processing the returned data, 
+    which allows data to be handled in parallel threads, 
+    so it doesn't wait to encode and persist a block's graph elements before processing the next block.
+    However, there is a limit to how many concurrent requests EBA and Bitcoin Core can process optimally. 
+    Therefore, despite these optimizations, 
+    if both applications are running on the same machine, 
+    their performance is ultimately bound by its I/O limits, 
+    primarily the random read/write performance of the storage.
+
+
+    Since EBA processes each block independently, 
+    one potential improvement is to deploy the application on a Kubernetes (k8s) cluster 
+    (requires dockerizing both EBA and Bitcoin Core).
+    In this setup, each instance of EBA service could process a subset of blocks 
+    while a load balancer directs its API calls to replicas of the Bitcoin Core services. 
+    This horizontal scaling would significantly improve performance; 
+    however, because this requires a k8s cluster and cloud or on-premises HPC resources 
+    that may not be [widely accessible](/docs/gs/accessibility), 
+    the specifics of such a deployment are not currently covered.
+</details>
 
-### Performance and Scalability
-
-Traversing Bitcoin blocks can take a considerable amount of time. 
-To accelerate this, 
-EBA heavily leverages multi-threading, 
-and all time-consuming operations are implemented to be non-blocking. 
-It also minimizes the latency between submitting API calls and 
-processing the returned data, 
-which allows data to be handled in parallel threads, 
-so it doesn't wait to encode and persist a block's graph elements before processing the next block.
-However, there is a limit to how many concurrent requests EBA and Bitcoin Core can process optimally. 
-Therefore, despite these optimizations, 
-if both applications are running on the same machine, 
-their performance is ultimately bound by its I/O limits, 
-primarily the random read/write performance of the storage.
-
-
-Since EBA processes each block independently, 
-one potential improvement is to deploy the application on a Kubernetes (k8s) cluster 
-(requires dockerizing both EBA and Bitcoin Core).
-In this setup, each instance of EBA service could process a subset of blocks 
-while a load balancer directs its API calls to replicas of the Bitcoin Core services. 
-This horizontal scaling would significantly improve performance; 
-however, because this requires a k8s cluster and cloud or on-premises HPC resources 
-that may not be [widely accessible](/docs/gs/accessibility), 
-the specifics of such a deployment are not currently covered.
 
 
 
@@ -149,3 +151,11 @@ node files.
     ```shell
     .\eba.exe bitcoin dedup --sorted-script-nodes-file sorted_nodes_Script.csv --sorted-tx-nodes-file sorted_nodes_Tx.csv
     ```
+
+
+## Post-Traverse
+
+
+```shell
+.\eba.exe bitcoin post-traverse --batches-filename batches.json
+```