diff --git a/EBA/Blockchains/Bitcoin/BitcoinChainAgent.cs b/EBA/Blockchains/Bitcoin/BitcoinChainAgent.cs index 75e3067a..331e9a28 100644 --- a/EBA/Blockchains/Bitcoin/BitcoinChainAgent.cs +++ b/EBA/Blockchains/Bitcoin/BitcoinChainAgent.cs @@ -153,18 +153,18 @@ await JsonSerializer.DeserializeAsync(stream, cancellationToken: cT) } public async Task> GetBlockMetadataAsync( - long startBlockHeight, - long endBlockHeight, + long startHeight, + long endHeight, CancellationToken cT) { _logger.LogInformation( "Getting block metadata for blocks {start:n0} to {end:n0}.", - startBlockHeight, endBlockHeight); + startHeight, endHeight); var pageSize = 2000; var pages = new List(); - for (var i = startBlockHeight; i <= endBlockHeight; i += pageSize) - pages.Add([i, i + pageSize < endBlockHeight ? pageSize : endBlockHeight - i + 1]); + for (var i = startHeight; i <= endHeight; i += pageSize) + pages.Add([i, i + pageSize < endHeight ? pageSize : endHeight - i + 1]); var blocksMetadata = new ConcurrentBag(); @@ -191,7 +191,7 @@ await Parallel.ForEachAsync( }); _logger.LogInformation("Finished Getting block metadata for blocks {start:n0} to {end:n0}.", - startBlockHeight, endBlockHeight); + startHeight, endHeight); return [.. blocksMetadata]; } @@ -216,7 +216,7 @@ await strategy.ExecuteAsync( }, new Context() .SetLogger(_logger) - .SetBlockHeight(height), + .SetHeight(height), cT); } catch (Polly.CircuitBreaker.BrokenCircuitException e) @@ -336,7 +336,7 @@ private static async Task ProcessTx(BlockGraph g, Tx tx, BitcoinOptions options, g.Block.TxoLifecycle.AddOrUpdate(utxo.Id, utxo, (_, oldValue) => { - oldValue.SpentInBlockHeight = g.Block.Height; + oldValue.SpentInHeight = g.Block.Height; return oldValue; }); } @@ -360,7 +360,7 @@ private static async Task ProcessTx(BlockGraph g, Tx tx, BitcoinOptions options, value: output.Value, scriptType: output.ScriptPubKey.ScriptType, isGenerated: false, - createdInBlockHeight: g.Block.Height); + createdInHeight: g.Block.Height); g.Block.TxoLifecycle.TryAdd(utxo.Id, utxo); } diff --git a/EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs b/EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs index 099abf1c..9d1a825c 100644 --- a/EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs +++ b/EBA/Blockchains/Bitcoin/BitcoinOrchestrator.cs @@ -28,11 +28,11 @@ public async Task TraverseAsync( _logger.LogInformation("Head of the chain is at block {block:n0}.", chainInfo.Blocks); options.Bitcoin.Traverse.To ??= chainInfo.Blocks; - SetupPersistedQueues(options, out var blockHeightQueue, out var failedBlocksQueue); + SetupPersistedQueues(options, out var heightQueue, out var failedBlocksQueue); await JsonSerializer.SerializeAsync(options, options.StatusFile, cT); - if (blockHeightQueue.Count == 0) + if (heightQueue.Count == 0) { _logger.LogInformation("No blocks to process."); return; @@ -44,7 +44,7 @@ public async Task TraverseAsync( try { stopwatch.Start(); - await TraverseBlocksAsync(options, blockHeightQueue, failedBlocksQueue, cT); + await TraverseBlocksAsync(options, heightQueue, failedBlocksQueue, cT); stopwatch.Stop(); _logger.LogInformation("Successfully finished traverse in {et}.", stopwatch.Elapsed); diff --git a/EBA/Blockchains/Bitcoin/ChainModel/Utxo.cs b/EBA/Blockchains/Bitcoin/ChainModel/Utxo.cs index becc9090..522a3bf2 100644 --- a/EBA/Blockchains/Bitcoin/ChainModel/Utxo.cs +++ b/EBA/Blockchains/Bitcoin/ChainModel/Utxo.cs @@ -14,22 +14,22 @@ public class Utxo public bool IsGenerated { set; get; } - public long CreatedInBlockHeight { get; } + public long CreatedInHeight { get; } - public long? SpentInBlockHeight { set; get; } + public long? SpentInHeight { set; get; } public Utxo( string id, string? address, long value, ScriptType scriptType, bool isGenerated, - long createdInBlockHeight, - long? spentInBlockHeight = null) + long createdInHeight, + long? spentInHeight = null) { Id = id; Address = address ?? Id; Value = value; ScriptType = scriptType; IsGenerated = isGenerated; - CreatedInBlockHeight = createdInBlockHeight; - SpentInBlockHeight = spentInBlockHeight; + CreatedInHeight = createdInHeight; + SpentInHeight = spentInHeight; } public Utxo( @@ -58,18 +58,4 @@ public static string GetTxid(string id) { return id.Split('-')[1]; } - - public static string GetHeader() - { - return string.Join( - '\t', - "Id", - "Value", - "CreatedInBlockHeights", - "CreatedInBlockHeightsCount", - "SpentInBlockHeights", - "SpentInBlockHeightsCount", - "ScriptType", - "IsGenerated(0=No,1=Yes)"); - } } diff --git a/EBA/Blockchains/Bitcoin/GraphModel/B2BEdge.cs b/EBA/Blockchains/Bitcoin/GraphModel/B2BEdge.cs index bdc6b33d..03f3f443 100644 --- a/EBA/Blockchains/Bitcoin/GraphModel/B2BEdge.cs +++ b/EBA/Blockchains/Bitcoin/GraphModel/B2BEdge.cs @@ -9,7 +9,7 @@ public class B2BEdge( value: 0, relation: Kind.Relation, timestamp: 0, - blockHeight: 0) + height: 0) { public static new EdgeKind Kind => new(BlockNode.Kind, BlockNode.Kind, RelationType.Follows); diff --git a/EBA/Blockchains/Bitcoin/GraphModel/B2TEdge.cs b/EBA/Blockchains/Bitcoin/GraphModel/B2TEdge.cs index f08bd8a9..fe9cf803 100644 --- a/EBA/Blockchains/Bitcoin/GraphModel/B2TEdge.cs +++ b/EBA/Blockchains/Bitcoin/GraphModel/B2TEdge.cs @@ -7,18 +7,18 @@ public class B2TEdge : Edge public B2TEdge( BlockNode source, TxNode target, long value, - uint timestamp, long blockHeight) : + uint timestamp, long height) : base( source: source, target: target, value: value, relation: Kind.Relation, timestamp: timestamp, - blockHeight: blockHeight) + height: height) { } public B2TEdge Update(long value) { - return new B2TEdge(Source, Target, Value + value, Timestamp, BlockHeight); + return new B2TEdge(Source, Target, Value + value, Timestamp, Height); } } diff --git a/EBA/Blockchains/Bitcoin/GraphModel/C2TEdge.cs b/EBA/Blockchains/Bitcoin/GraphModel/C2TEdge.cs index f9053168..b999c1e7 100644 --- a/EBA/Blockchains/Bitcoin/GraphModel/C2TEdge.cs +++ b/EBA/Blockchains/Bitcoin/GraphModel/C2TEdge.cs @@ -4,19 +4,19 @@ public class C2TEdge( TxNode target, long value, uint timestamp, - long blockHeight) + long height) : Edge( source: new CoinbaseNode(), target: target, value: value, relation: RelationType.Mints, timestamp: timestamp, - blockHeight: blockHeight) + height: height) { public new static EdgeKind Kind => new(CoinbaseNode.Kind, TxNode.Kind, RelationType.Mints); public C2TEdge Update(long value) { - return new C2TEdge(Target, Value + value, Timestamp, BlockHeight); + return new C2TEdge(Target, Value + value, Timestamp, Height); } } diff --git a/EBA/Blockchains/Bitcoin/GraphModel/S2TEdge.cs b/EBA/Blockchains/Bitcoin/GraphModel/S2TEdge.cs index a0a565ed..1416b446 100644 --- a/EBA/Blockchains/Bitcoin/GraphModel/S2TEdge.cs +++ b/EBA/Blockchains/Bitcoin/GraphModel/S2TEdge.cs @@ -9,7 +9,7 @@ public class S2TEdge : Edge public bool Generated { get; } public long CreationHeight { get; } - public long SpentHeight { get { return BlockHeight; } } + public long SpentHeight { get { return Height; } } public S2TEdge( ScriptNode source, @@ -45,7 +45,7 @@ public S2TEdge( value: value, relation: Kind.Relation, timestamp: timestamp, - blockHeight: spentHeight) + height: spentHeight) { Txid = txid; Vout = vout; diff --git a/EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs b/EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs index 894e058c..c13f173d 100644 --- a/EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs +++ b/EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs @@ -8,7 +8,7 @@ public class T2SEdge : Edge public long SpentHeight { get; } - public long CreationHeight => BlockHeight; + public long CreationHeight => Height; public T2SEdge( TxNode source, @@ -23,7 +23,7 @@ public T2SEdge( relation: Kind.Relation, value: output.Value, timestamp: timestamp, - blockHeight: creationHeight) + height: creationHeight) { Vout = output.N; SpentHeight = spentHeight; @@ -43,7 +43,7 @@ public T2SEdge( relation: Kind.Relation, value: value, timestamp: timestamp, - blockHeight: creationHeight) + height: creationHeight) { Vout = outputIndex; SpentHeight = spentHeight; diff --git a/EBA/Blockchains/Bitcoin/GraphModel/T2TEdge.cs b/EBA/Blockchains/Bitcoin/GraphModel/T2TEdge.cs index e9e47420..2372c938 100644 --- a/EBA/Blockchains/Bitcoin/GraphModel/T2TEdge.cs +++ b/EBA/Blockchains/Bitcoin/GraphModel/T2TEdge.cs @@ -4,14 +4,14 @@ public class T2TEdge : Edge { public T2TEdge( TxNode source, TxNode target, - long value, RelationType type, uint timestamp, long blockHeight) : + long value, RelationType type, uint timestamp, long height) : base( source: source, target: target, value: value, relation: type, timestamp: timestamp, - blockHeight: blockHeight) + height: height) { } public static EdgeKind KindTransfers => new(TxNode.Kind, TxNode.Kind, RelationType.Transfers); @@ -40,6 +40,6 @@ public static T2TEdge Update(T2TEdge oldEdge, T2TEdge newEdge) newEdge.Value, newEdge.Relation, newEdge.Timestamp, - newEdge.BlockHeight); + newEdge.Height); } } diff --git a/EBA/Blockchains/Bitcoin/Utilities/TraverseFinalizer.cs b/EBA/Blockchains/Bitcoin/Utilities/TraverseFinalizer.cs index 124755d5..659ff640 100644 --- a/EBA/Blockchains/Bitcoin/Utilities/TraverseFinalizer.cs +++ b/EBA/Blockchains/Bitcoin/Utilities/TraverseFinalizer.cs @@ -21,7 +21,7 @@ public async Task UpdatePostTraverse(CancellationToken ct) _logger.LogInformation("Deserialized {n} batches from {filename}.", batches.Count, _options.Bitcoin.MapSpends.BatchesFilename); _processStep = "[1/4] Block-to-batch mapping:"; - GetBlockHeightToBatchMapping(_options, batches, out var blockToBatch, out var blockNodes); + GetHeightToBatchMapping(_options, batches, out var blockToBatch, out var blockNodes); _processStep = "[2/4] Collecting Txo spending:"; await CreatePerBatchSpentUtxo(batches, blockToBatch, ct); @@ -33,7 +33,7 @@ public async Task UpdatePostTraverse(CancellationToken ct) await SetSupplyAmount(batches, blockNodes, ct); } - private void GetBlockHeightToBatchMapping( + private void GetHeightToBatchMapping( Options options, List batches, out Dictionary blockToBatch, @@ -73,7 +73,7 @@ private static string GetSpentTxoFilename(Batch batch) batch.FilenamePrefix + "_spent_utxos.csv"); } - private async Task CreatePerBatchSpentUtxo(List batches, Dictionary blockHeightToBatchMapping, CancellationToken ct) + private async Task CreatePerBatchSpentUtxo(List batches, Dictionary heightToBatchMapping, CancellationToken ct) { _logger.LogInformation("{s} Creating per-batch spent Txo files; these are temporary files and can be deleted after process ends.", _processStep); @@ -95,7 +95,7 @@ private async Task CreatePerBatchSpentUtxo(List batches, Dictionary public static ElementMapper StaticMapper => _mapper; private static readonly ElementMapper _mapper = new( new MappingBuilder() - .MapSourceId(BlockNodeDescriptor.IdSpace, e => e.BlockHeight) - .MapTargetId(BlockNodeDescriptor.IdSpace, e => e.BlockHeight) + .MapSourceId(BlockNodeDescriptor.IdSpace, e => e.Height) + .MapTargetId(BlockNodeDescriptor.IdSpace, e => e.Height) .MapEdgeType(e => e.Relation) .ToArray()); @@ -25,7 +25,7 @@ public string[] Neo4jSeedingOverride return [ $"MATCH (target:Block), (source:Block) " + - $"\r\nWHERE target.{nameof(B2BEdge.BlockHeight)} + 1 = source.{nameof(B2BEdge.BlockHeight)} " + + $"\r\nWHERE target.{nameof(B2BEdge.Height)} + 1 = source.{nameof(B2BEdge.Height)} " + $"\r\nMERGE (target)-[:{RelationType.Follows}]->(source)" ]; } diff --git a/EBA/Graph/Bitcoin/Descriptors/B2TEdgeDescriptor.cs b/EBA/Graph/Bitcoin/Descriptors/B2TEdgeDescriptor.cs index 23276727..890a1e54 100644 --- a/EBA/Graph/Bitcoin/Descriptors/B2TEdgeDescriptor.cs +++ b/EBA/Graph/Bitcoin/Descriptors/B2TEdgeDescriptor.cs @@ -11,7 +11,7 @@ public class B2TEdgeDescriptor : IElementDescriptor .MapSourceId(BlockNodeDescriptor.IdSpace, e => e.Source.BlockMetadata.Height) .MapTargetId(TxNodeDescriptor.IdSpace, e => e.Target.Txid) .Map(e => e.Value) - .Map(e => e.BlockHeight) + .Map(e => e.Height) .MapEdgeType(e => e.Relation) .ToArray()); @@ -24,7 +24,7 @@ public static B2TEdge Deserialize( source: source, target: target, timestamp: 0, - blockHeight: _mapper.GetValue(e => e.BlockHeight, props), + height: _mapper.GetValue(e => e.Height, props), value: _mapper.GetValue(e => e.Value, props)); } } \ No newline at end of file diff --git a/EBA/Graph/Bitcoin/Descriptors/C2TEdgeDescriptor.cs b/EBA/Graph/Bitcoin/Descriptors/C2TEdgeDescriptor.cs index dd162ef1..b887063e 100644 --- a/EBA/Graph/Bitcoin/Descriptors/C2TEdgeDescriptor.cs +++ b/EBA/Graph/Bitcoin/Descriptors/C2TEdgeDescriptor.cs @@ -12,7 +12,7 @@ public class C2TEdgeDescriptor : IElementDescriptor .MapSourceId(CoinbaseNode.Kind.ToString(), _ => CoinbaseNode.Kind) .MapTargetId(TxNodeDescriptor.IdSpace, e => e.Target.Txid) .Map(e => e.Value) - .Map(e => e.BlockHeight) + .Map(e => e.Height) .MapEdgeType(e => e.Relation) .ToArray()); @@ -24,6 +24,6 @@ public static C2TEdge Deserialize( target: target, value: _mapper.GetValue(e => e.Value, props), timestamp: 0, - blockHeight: _mapper.GetValue(e => e.BlockHeight, props)); + height: _mapper.GetValue(e => e.Height, props)); } } \ No newline at end of file diff --git a/EBA/Graph/Bitcoin/Descriptors/T2TEdgeDescriptor.cs b/EBA/Graph/Bitcoin/Descriptors/T2TEdgeDescriptor.cs index 89846ad3..4c79ebe3 100644 --- a/EBA/Graph/Bitcoin/Descriptors/T2TEdgeDescriptor.cs +++ b/EBA/Graph/Bitcoin/Descriptors/T2TEdgeDescriptor.cs @@ -11,7 +11,7 @@ public class T2TEdgeDescriptor : IElementDescriptor .MapSourceId(TxNodeDescriptor.IdSpace, e => e.Source.Txid) .MapTargetId(TxNodeDescriptor.IdSpace, e => e.Target.Txid) .Map(e => e.Value) - .Map(e => e.BlockHeight) + .Map(e => e.Height) .MapEdgeType(e => e.Relation) .ToArray()); @@ -25,7 +25,7 @@ public static T2TEdge Deserialize( source: source, target: target, timestamp: 0, - blockHeight: _mapper.GetValue(e => e.BlockHeight, props), + height: _mapper.GetValue(e => e.Height, props), value: _mapper.GetValue(e => e.Value, props), type: Enum.Parse(relationType, ignoreCase: true)); } diff --git a/EBA/Graph/Model/Edge.cs b/EBA/Graph/Model/Edge.cs index aef3b547..e7770640 100644 --- a/EBA/Graph/Model/Edge.cs +++ b/EBA/Graph/Model/Edge.cs @@ -1,7 +1,4 @@ -using EBA.Graph.Bitcoin.Descriptors; -using EBA.Utilities; - -namespace EBA.Graph.Model; +namespace EBA.Graph.Model; public class Edge : IEdge, IEquatable> where TSource : notnull, INode @@ -16,23 +13,7 @@ public class Edge : IEdge, IEquatable _actions = new(); - - /// - /// Sets the number of most recent messeges that should - /// be flushed to console before the process exits on - /// cancellelation signal. - /// - private const int _nLastItems = 5; - - static AsyncConsole() - { - var thread = new Thread(() => - { - while (true) - { - try - { - _actions.Take(CancellationToken)(); - } - catch (OperationCanceledException) - { - // Note that when the cancellelation signal - // is received, this process is not necessarily - // current, there could be a big backlog of messages - // to write to console. Therefore, waiting for - // all of them to be sent to console before exiting - // may take a considerable amount of time; hence - // only the _n_ recent items are sent to console. - // Flush latest messages before exiting. - var mostRecentActions = _actions.TakeLast(_nLastItems); - - // Set to a new instance so all the post-cancellation - // messages are still queued up for display. - _actions = new BlockingCollection(); - - // Show the last _n_ messeges before the - // cancellation signal. - foreach (var action in mostRecentActions) - action(); - - // Still consuming produced message, hence - // it can display any post cancellation messages. - // Note that this loop only ends when the - // application exits. - while (true) - _actions.Take()(); - } - } - }) - { - IsBackground = true - }; - thread.Start(); - } - - public static void Write(string value, ConsoleColor color, int lineOffset = 1) - { - _actions.Add(() => - { - // Write to Block Traversal Progress Line. - bool w2BTL = false; - try - { - if (Console.CursorTop != BookmarkedLine + 1) - w2BTL = true; - } - catch (IOException) - { - // This exception is thrown when tested under - // xunit, because xunit does not have a console - // attached. - w2BTL = false; - } - - if (w2BTL) - { - var (Left, Top) = Console.GetCursorPosition(); - Console.SetCursorPosition(0, BookmarkedLine + lineOffset); - Console.ForegroundColor = color; - Console.Write(value); - Console.ResetColor(); - Console.SetCursorPosition(Left, Top); - } - else - { - Console.ForegroundColor = color; - Console.Write(value); - Console.ResetColor(); - } - }); - } - - public static void WriteLine(string value) - { - _actions.Add(() => - { - Console.WriteLine(value); - }); - } - - public static void WriteLine(string value, ConsoleColor color) - { - _actions.Add(() => - { - Console.ForegroundColor = color; - Console.WriteLine(value); - Console.ResetColor(); - }); - } - - public static void WriteLines(string[] msgs, ConsoleColor[] colors, int cursorTopOffset = 1) - { - _actions.Add(() => - { - var (currentLeft, currentTop) = Console.GetCursorPosition(); - Console.SetCursorPosition(0, BookmarkedLine + cursorTopOffset); - for (int i = 0; i < msgs.Length; i++) - { - Console.ForegroundColor = colors[i]; - if (msgs[i].Length - 1 >= Console.WindowWidth) - { - Console.WriteLine(string.Concat( - msgs[i].AsSpan(0, Console.WindowWidth - 6), "... ")); - } - else - { - Console.Write(msgs[i]); - Console.WriteLine(new string( - ' ', Console.WindowWidth - Console.CursorLeft - 1)); - } - } - Console.ResetColor(); - Console.SetCursorPosition(currentLeft, currentTop); - }); - } - - public static void WriteErrorLine(string value, ConsoleColor color = ConsoleColor.Red) - { - _actions.Add(() => - { - Console.ForegroundColor = color; - Console.Error.WriteLine(value); - Console.ResetColor(); - }); - } - - public static void BookmarkCurrentLine() - { - _actions.Add(() => - { - // The exception is thrown with the message 'The handle is invalid.' - // only when running the tests, because Xunit does not have a console. - try { BookmarkedLine = Console.CursorTop; } - catch (IOException) { } - }); - } - - public static void WaitUntilBufferEmpty() - { - while (true) - { - if (_actions.Count == 0) return; - Thread.Sleep(500); - } - } -} diff --git a/EBA/Infrastructure/Logging/BlockTraversalLoggingBase.cs b/EBA/Infrastructure/Logging/BlockTraversalLoggingBase.cs deleted file mode 100644 index fde69440..00000000 --- a/EBA/Infrastructure/Logging/BlockTraversalLoggingBase.cs +++ /dev/null @@ -1,88 +0,0 @@ -namespace EBA.Logging; - -public abstract class BlockTraversalLoggingBase -{ - public int MovingAvgWindowSize { set; get; } = 10; - - public int FromInclusive { get; } - public int ToExclusive { get; } - public int BlocksCount { get; } - public double Total { get; } - public double Percentage { get { return (Completed / Total) * 100; } } - - public int Completed { get { return _completed; } } - private int _completed; - - public string ActiveBlocks - { - get - { - return - $"{_activeBlocks.Keys.Count,9}: " + - $"{string.Join(", ", _activeBlocks.Keys)}"; - } - } - private readonly ConcurrentDictionary _activeBlocks = new(); - public int ActiveBlocksCount { get { return _activeBlocks.Count; } } - - public MovingAverage BlockRuntimeMovingAvg { get; } - - private const string _cancelling = "Cancelling ... do not turn off your computer."; - - public BlockTraversalLoggingBase( - int fromInclusive, - int toExclusive, - int blocksCount, - int templateLinesCount) - { - FromInclusive = fromInclusive; - ToExclusive = toExclusive; - Total = blocksCount; //ToExclusive - FromInclusive; - - BlockRuntimeMovingAvg = new MovingAverage(MovingAvgWindowSize); - - // The exception is thrown with the message 'The handle is invalid.' - // only when running the tests, because Xunit does not have a console. - try { Console.CursorVisible = false; } - catch (IOException) { } - - AsyncConsole.BookmarkCurrentLine(); - AsyncConsole.WriteLine(""); - for (int i = 0; i <= templateLinesCount; i++) - AsyncConsole.WriteLine(""); - } - - public virtual string Log(int height) - { - _activeBlocks.TryAdd(height, true); - ToConsole(); - return $"Started processing block {height}."; - } - - public virtual string Log( - int height, - double runtime) - { - _activeBlocks.TryRemove(height, out bool _); - - Interlocked.Increment(ref _completed); - - ToConsole(); - - return - $"Active:{ActiveBlocksCount} [{string.Join(";", _activeBlocks.Keys)}]; " + - $"Completed:{height} ({Completed}/{Total})."; - /* - return - $"Active:{ActiveBlocksCount};" + - $"Completed:{Completed}/{Total}.";*/ - } - - protected abstract void ToConsole(); - - public virtual string LogCancelling() - { - AsyncConsole.WriteLine(_cancelling, ConsoleColor.Yellow); - return _cancelling; - } -} diff --git a/EBA/Infrastructure/Logging/BlockTraversalLoggingComplete.cs b/EBA/Infrastructure/Logging/BlockTraversalLoggingComplete.cs deleted file mode 100644 index 08fd31ad..00000000 --- a/EBA/Infrastructure/Logging/BlockTraversalLoggingComplete.cs +++ /dev/null @@ -1,35 +0,0 @@ -namespace EBA.Logging; - -public class BlockTraversalLoggingComplete : BlockTraversalLoggingBase -{ - private static readonly ConsoleColor[] _colors = new[] - { - ConsoleColor.Cyan, - ConsoleColor.Blue, - ConsoleColor.Blue, - ConsoleColor.Blue, - ConsoleColor.Blue, - ConsoleColor.Blue, - }; - - public BlockTraversalLoggingComplete(int fromInclusive, int toExclusive, int blocksCount) : - base(fromInclusive, toExclusive, blocksCount, 6) - { } - - protected override void ToConsole() - { - // Do not use tab (\t) since the length of each string - // is used to determine how many blank spaces to add or - // when to truncate the line w.r.t console window width. - string[] msgs = new[] - { - $"\r Active Blocks: {ActiveBlocks}", - $"\r Completed: {Completed,9:n0}/{Total:n0} ({Percentage:f2}%)", - $"\r Block Rate: {BlockRuntimeMovingAvg.Speed,9} blocks/sec", - //$"\r Edge Rate: {EdgeRuntimeMovingAvg.Speed,9} edges/sec", - //$"\r Nodes: {NodesCount,9:n0}", - //$"\r Edges: {EdgesCount,9:n0}" - }; - AsyncConsole.WriteLines(msgs, _colors); - } -} diff --git a/EBA/Infrastructure/Logging/BlockTraversalLoggingMinimal.cs b/EBA/Infrastructure/Logging/BlockTraversalLoggingMinimal.cs deleted file mode 100644 index d7d9059a..00000000 --- a/EBA/Infrastructure/Logging/BlockTraversalLoggingMinimal.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace EBA.Logging; - -internal class BlockTraversalLoggingMinimal : BlockTraversalLoggingBase -{ - public BlockTraversalLoggingMinimal(int fromInclusive, int toExclusive, int blocksCount) : - base(fromInclusive, toExclusive, blocksCount, 0) - { } - - protected override void ToConsole() - { - AsyncConsole.Write( - $"\r Completed:{$"{Completed:n0}",12}/{$"{Total:n0}"} ({$"{Percentage:f2}%",2})", - ConsoleColor.Cyan); - } -} diff --git a/EBA/Infrastructure/Logging/ConsoleLoggingInterface.cs b/EBA/Infrastructure/Logging/ConsoleLoggingInterface.cs deleted file mode 100644 index 8cc9719d..00000000 --- a/EBA/Infrastructure/Logging/ConsoleLoggingInterface.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace EBA.Logging; - -public enum ConsoleLoggingInterface -{ - Minimal, - Complete -} diff --git a/EBA/Infrastructure/Logging/Logger.cs b/EBA/Infrastructure/Logging/Logger.cs deleted file mode 100644 index e7311f59..00000000 --- a/EBA/Infrastructure/Logging/Logger.cs +++ /dev/null @@ -1,172 +0,0 @@ -/* -using log4net; -using log4net.Appender; -using log4net.Core; -using log4net.Layout; -using log4net.Repository.Hierarchy; - -namespace EBA.Logging -{ - public class Logger : IDisposable - { - public string MaxLogFileSize { get; } = "2GB"; - - public ConsoleLoggingInterface ConsoleLoggingInterface { get; } - - private readonly ILog log; - private readonly string _name; - private readonly string _repository; - - private BlockTraversalLoggingBase _consoleLogging; - - private bool disposed = false; - - public Logger( - string logFilename, string repository, - string name, string exportPath, - string maxLogFileSize) - { - MaxLogFileSize = maxLogFileSize; - ConsoleLoggingInterface = ConsoleLoggingInterface.Minimal; - - _name = name; - _repository = repository; - LogManager.CreateRepository(_repository); - var hierarchy = (Hierarchy)LogManager.GetRepository(_repository); - - var patternLayout = new PatternLayout - { - ConversionPattern = "%date\t[%thread]\t%-5level\t%message%newline" - }; - patternLayout.ActivateOptions(); - - var roller = new RollingFileAppender - { - AppendToFile = true, - File = logFilename, - Layout = patternLayout, - MaxSizeRollBackups = 5, - MaximumFileSize = MaxLogFileSize, - RollingStyle = RollingFileAppender.RollingMode.Size, - StaticLogFileName = true, - LockingModel = new FileAppender.MinimalLock() - }; - roller.ActivateOptions(); - hierarchy.Root.AddAppender(roller); - - hierarchy.Root.Level = Level.Info; - hierarchy.Configured = true; - log = LogManager.GetLogger(_repository, _name); - - log.Info("NOTE THAT THE LOG PATTERN IS: <#Thread> "); - Log($"Export Directory: {exportPath}", true, ConsoleColor.DarkGray); - } - - public void InitBlocksTraverse(int from, int to, int blocksToProcess) - { - switch(ConsoleLoggingInterface) - { - case ConsoleLoggingInterface.Minimal: - _consoleLogging = new BlockTraversalLoggingMinimal(from, to, blocksToProcess); - break; - case ConsoleLoggingInterface.Complete: - _consoleLogging = new BlockTraversalLoggingComplete(from, to, blocksToProcess); - break; - } - } - - public void Log(string message) - { - log.Info(message); - } - - public void Log(string message, bool writeLine, ConsoleColor color) - { - if (writeLine) - AsyncConsole.WriteLine(message, color); - else - AsyncConsole.Write(message, color); - - log.Info(message); - } - - public void LogStartProcessingBlock(int blockHeight) - { - log.Info(_consoleLogging.Log(blockHeight)); - } - - public void LogFinishProcessingBlock(int height, double runtime) - { - log.Info(_consoleLogging.Log(height, runtime)); - } - - public void LogCancelling() - { - log.Info(_consoleLogging.LogCancelling()); - } - - public void LogException(Exception e) - { - LogException(e.Message); - } - - public void LogException(string message) - { - LogExceptionStatic(message); - log.Error(message); - - // TODO: - /* - log.Info(_linkToDocumentation); - log.Info(_linkToIssuesPage); - log.Info(HintHelpMessage); - log.Info(_cannotContinue);*/ - /*} - - public static void LogExceptionStatic(string message) - { - AsyncConsole.WriteErrorLine($"Error: {message}"); - // TODO: - // Console.WriteLine(HintHelpMessage); - // Console.WriteLine(_cannotContinue); - } - - public void LogWarning(string message) - { - log.Warn(message); - LogWarningStatic(message); - } - - public static void LogWarningStatic(string message) - { - AsyncConsole.WriteLine($"Warning: {message}", ConsoleColor.DarkMagenta); - } - - // The IDisposable interface is implemented following .NET docs: - // https://docs.microsoft.com/en-us/dotnet/api/system.idisposable?view=net-6.0 - public void Dispose() - { - Dispose(disposing: true); - GC.SuppressFinalize(this); - } - protected virtual void Dispose(bool disposing) - { - if (!disposed) - { - if (disposing) - { - LogManager.Flush(5000); - - var l = (log4net.Repository.Hierarchy.Logger)LogManager - .GetLogger(_repository, _name).Logger; - l.RemoveAllAppenders(); - - LogManager.GetLogger(_repository, _name) - .Logger.Repository.Shutdown(); - } - - disposed = true; - } - } - } -}*/ diff --git a/EBA/Infrastructure/Logging/MovingAverage.cs b/EBA/Infrastructure/Logging/MovingAverage.cs deleted file mode 100644 index dd72c99d..00000000 --- a/EBA/Infrastructure/Logging/MovingAverage.cs +++ /dev/null @@ -1,31 +0,0 @@ -namespace EBA.Logging; - -public class MovingAverage -{ - private double _average = 1; - public double Average { get { return _average; } } - public double Speed - { - get { return Math.Round(1.0 / _average, 3); } - } - - private readonly int _windowSize; - private readonly ConcurrentQueue _queue = new(); - private static readonly object _locker = new(); - - public MovingAverage(int windowSize) - { - _windowSize = windowSize; - } - - public void Add(double runtime) - { - lock (_locker) - { - if (_queue.Count == _windowSize) - _queue.TryDequeue(out double _); - _queue.Enqueue(runtime); - _average = _queue.Average(); - } - } -} diff --git a/EBA/Infrastructure/StartupSolutions/ContextExtension.cs b/EBA/Infrastructure/StartupSolutions/ContextExtension.cs index d219586b..9da3c4a2 100644 --- a/EBA/Infrastructure/StartupSolutions/ContextExtension.cs +++ b/EBA/Infrastructure/StartupSolutions/ContextExtension.cs @@ -5,17 +5,17 @@ namespace EBA.Infrastructure.StartupSolutions; internal static class ContextExtension { private static readonly string _loggerKey = "ILogger"; - private static readonly string _blockHeightKey = "BlockHeight"; + private static readonly string _heightKey = "Height"; - public static Context SetBlockHeight(this Context context, long height) + public static Context SetHeight(this Context context, long height) { - context[_blockHeightKey] = height; + context[_heightKey] = height; return context; } - public static long? GetBlockHeight(this Context context) + public static long? GetHeight(this Context context) { - if (context.TryGetValue(_blockHeightKey, out var h)) + if (context.TryGetValue(_heightKey, out var h)) return (long)h; return null; diff --git a/EBA/Infrastructure/StartupSolutions/ResilienceStrategyFactory.cs b/EBA/Infrastructure/StartupSolutions/ResilienceStrategyFactory.cs index 88e4bb5a..b1adaa00 100644 --- a/EBA/Infrastructure/StartupSolutions/ResilienceStrategyFactory.cs +++ b/EBA/Infrastructure/StartupSolutions/ResilienceStrategyFactory.cs @@ -116,12 +116,12 @@ public static IAsyncPolicy GetGraphStrategy(ResilienceStrategyOptions options) "Retry: {message} Waiting for {timespan} " + "seconds before {retryAttempt} retry. Block height: {h:n0}", exception.Message, timeSpan.TotalSeconds, retryAttempt, - context.GetBlockHeight()); + context.GetHeight()); else Console.Error.WriteLine( $"Retry: {exception.Message} Waiting for " + $"{timeSpan.TotalSeconds} second before {retryAttempt} retry." + - $"Block height: {context.GetBlockHeight():n0}"); + $"Block height: {context.GetHeight():n0}"); }); var circuitBreaker = Policy @@ -139,11 +139,11 @@ public static IAsyncPolicy GetGraphStrategy(ResilienceStrategyOptions options) logger.LogWarning( "CircuitBreaker: Circuit on break; height {h:n0}; " + "exception message: {exMsg}; timespan: {timeSpan}.", - context.GetBlockHeight(), exception.Message, timeSpan); + context.GetHeight(), exception.Message, timeSpan); else Console.Error.WriteLine( $"CircuitBreaker: Circuit on break; height " + - $"{context.GetBlockHeight():n0}; " + + $"{context.GetHeight():n0}; " + $"exception message: {exception.Message}; " + $"timespan: {timeSpan}."); }, @@ -152,10 +152,10 @@ public static IAsyncPolicy GetGraphStrategy(ResilienceStrategyOptions options) var logger = context.GetLogger(); if (logger != null) logger.LogWarning( - "CircuitBreaker: Reset. Height: {h:n0}", context.GetBlockHeight()); + "CircuitBreaker: Reset. Height: {h:n0}", context.GetHeight()); else Console.Error.WriteLine( - $"CircuitBreaker: Reset. Height: {context.GetBlockHeight():n0}"); + $"CircuitBreaker: Reset. Height: {context.GetHeight():n0}"); }, onHalfOpen: () => { @@ -170,10 +170,10 @@ public static IAsyncPolicy GetGraphStrategy(ResilienceStrategyOptions options) if (logger != null) logger.LogError( "Timeout getting graph block height {h:n0} after {timespan} seconds. {context}", - context.GetBlockHeight(), timespan.TotalSeconds, context); + context.GetHeight(), timespan.TotalSeconds, context); else Console.Error.WriteLine( - $"Timeout getting graph block height {context.GetBlockHeight():n0} " + + $"Timeout getting graph block height {context.GetHeight():n0} " + $"after {timespan.TotalSeconds} seconds. {context}"); }); diff --git a/EBA/Utilities/NewAddressCounter.cs b/EBA/Utilities/NewAddressCounter.cs index ef77e8c2..8fc67b7c 100644 --- a/EBA/Utilities/NewAddressCounter.cs +++ b/EBA/Utilities/NewAddressCounter.cs @@ -8,7 +8,7 @@ internal class NewAddressCounter(ILogger logger) private class Stats { - public int BlockHeight { set; get; } + public int Height { set; get; } public int AddressesInBlockCount { get; set; } public int UniqueAddressesInBlockCount { set; get; } public int UniqueAddressesCount { set; get; } @@ -34,18 +34,18 @@ public static Stats Parse(string line) var cols = line.Split(_delimiter); return new Stats() { - BlockHeight = int.Parse(cols[0]), + Height = int.Parse(cols[0]), AddressesInBlockCount = int.Parse(cols[1]), UniqueAddressesInBlockCount = int.Parse(cols[2]), UniqueAddressesCount = int.Parse(cols[3]) }; } - public static string ToString(string blockHeight, int addressInBlockCount, int uniqueAddressesInBlockCount, int uniqueAddressCount) + public static string ToString(string height, int addressInBlockCount, int uniqueAddressesInBlockCount, int uniqueAddressCount) { return string.Join( _delimiter, - blockHeight, addressInBlockCount, uniqueAddressesInBlockCount, uniqueAddressCount); + height, addressInBlockCount, uniqueAddressesInBlockCount, uniqueAddressCount); } public string ToStringWithoutHeight() @@ -59,7 +59,7 @@ public override string ToString() { return string.Join( _delimiter, - BlockHeight, AddressesInBlockCount, UniqueAddressesInBlockCount, UniqueAddressesCount); + Height, AddressesInBlockCount, UniqueAddressesInBlockCount, UniqueAddressesCount); } } @@ -108,7 +108,7 @@ private void ExtractAddressStats(string addressesFilename, string outFilename, C _logger.LogInformation("Read {counter} Lines.", lineCounter.ToString("N0")); var cols = line.Split('\t'); - var blockHeight = cols[0]; + var height = cols[0]; var blockAddresses = cols[1].Split(';'); var newAddressesCounter = 0; @@ -153,11 +153,11 @@ private void ExtractAddressStats(string addressesFilename, string outFilename, C } */ // the following is a simplified alternative to the above. - if (!blocks.Add(blockHeight)) + if (!blocks.Add(height)) { _logger.LogWarning( "Duplicate block {b} found, manually check the files if the duplicated " + - "entries have the same addresses. This process considers the addresses in the first entry.", blockHeight); + "entries have the same addresses. This process considers the addresses in the first entry.", height); continue; } @@ -177,7 +177,7 @@ private void ExtractAddressStats(string addressesFilename, string outFilename, C if (ct.IsCancellationRequested) return; - streamWriter.WriteLine(Stats.ToString(blockHeight, blockAddresses.Length, blockSpecificAddresses.Count, newAddressesCounter)); + streamWriter.WriteLine(Stats.ToString(height, blockAddresses.Length, blockSpecificAddresses.Count, newAddressesCounter)); } _logger.LogInformation("Finished extracting address stats, and persisting them in a temp file ({tmpFilename}).", outFilename); @@ -201,7 +201,7 @@ private void AddExtractedAddressStatsToStatsFile(string workingDir, string stats while ((line = addressesStreamReader.ReadLine()) != null) { var stat = Stats.Parse(line); - stats.Add(stat.BlockHeight, stat); + stats.Add(stat.Height, stat); } if (ct.IsCancellationRequested) @@ -222,8 +222,8 @@ private void AddExtractedAddressStatsToStatsFile(string workingDir, string stats line = line.TrimEnd('\r', '\n', '\t'); - var blockHeight = int.Parse(line.Split('\t')[0]); - var stat = stats[blockHeight]; + var height = int.Parse(line.Split('\t')[0]); + var stat = stats[height]; streamWriter.WriteLine($"{line}\t{stat.ToStringWithoutHeight()}"); } diff --git a/README.md b/README.md index 51e050a9..050a6a51 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@

-**EBA addresses a long-standing issue that has hindered -the ML community from developing applications for Bitcoin: -a lack of ML-first data.** +**EBA eliminates the steepest barrier to building machine +learning applications for Bitcoin: +the lack of structured, ML-native data.** EBA interfaces with the Bitcoin network and creates a graph of the full history of transactions recorded on-chain. On this graph, the nodes are Bitcoin _scripts_ (aka _addresses_), @@ -28,19 +28,19 @@ by traversing these paths. -This graph is built for machine learning, -particularly Graph Neural Networks (GNNs), -allowing a graph-based model to -aggregate information from neighbors through message passing and -learn the topology of fund flows for various applications -across the vibrant cryptocurrency ecosystem, including: +> [!IMPORTANT] +> [See the graph in action!](https://demo.b1aab.ai) 🚀 +> Trace the real-world transaction topology of the mysterious _Individual X_ in [our live interactive demo](https://demo.b1aab.ai); a sneak peek into the upcoming **EBA v2**! +[(Context slides)](https://docs.google.com/presentation/d/1BWJutm-5GMxFCUB59og05l6nmS2xhJ7yJ6xZMG7ufro/edit?usp=sharing) -* Exploring economic evolution and temporal behaviors. -* Analyzing network dynamics and trading patterns. -* Identifying suspicious or illicit activities. -* Benchmarking large-scale, graph-based machine learning models. +This graph is built natively for machine learning, +particularly Graph Neural Networks (GNNs). +It enables models to aggregate information from neighbors +through message passing and learn the deep topology of +fund flows across the cryptocurrency ecosystem. +

| 101 Quick Start Jupyter Notebook @@ -69,7 +69,6 @@ We share the complete ETL pipeline and all the data it generates.

- To simplify working with the pipeline and its resources, we have split them into separate repositories. The following resource map will help you navigate to the resources that suit your application. diff --git a/website/docs/bitcoin/etl/overview.mdx b/website/docs/bitcoin/etl/overview.mdx index 9a614754..db01b1eb 100644 --- a/website/docs/bitcoin/etl/overview.mdx +++ b/website/docs/bitcoin/etl/overview.mdx @@ -111,7 +111,6 @@ based on your specific goals and available resources. * Noe4j format files on: * AWS: `s3://bitcoin-graph/v1/data_to_import_neo4j/` - * [Google Drive](https://drive.google.com/drive/folders/11X6QiVvWSOzxvDIAD0OWu3g2Sa0as3UQ?usp=sharing) * [Block summary statistics](https://www.kaggle.com/datasets/aab/bitcoin-block-metadata) @@ -213,7 +212,6 @@ based on your specific goals and available resources. * Neo4j database dump on: * AWS: `s3://bitcoin-graph/v1/neo4j_db_dump/` - * [Google Drive](https://drive.google.com/drive/folders/1bAsjgVaIQrG2TDGkMtIEDPX8xKoiHJUf?usp=sharing) diff --git a/website/releases/data-releases/v1.md b/website/releases/data-releases/v1.md index 71141862..62069bf9 100644 --- a/website/releases/data-releases/v1.md +++ b/website/releases/data-releases/v1.md @@ -48,11 +48,9 @@ The release contains the following resources: * Download Neo4j format (`1.17TB`) from: - * [Google Drive](https://drive.google.com/drive/folders/11X6QiVvWSOzxvDIAD0OWu3g2Sa0as3UQ?usp=drive_link) * AWS: `s3://bitcoin-graph/v1/data_to_import_neo4j/` ([Walkthrough](/docs/bitcoin/etl/import)) * Download Neo4j database dump (`731GB`) from: - * [Google Drive](https://drive.google.com/drive/folders/1bAsjgVaIQrG2TDGkMtIEDPX8xKoiHJUf?usp=drive_link) * AWS: `s3://bitcoin-graph/v1/neo4j_db_dump/` ([Walkthrough](/docs/bitcoin/etl/restore)) diff --git a/website/versioned_docs/version-v1.0/bitcoin/etl/overview.mdx b/website/versioned_docs/version-v1.0/bitcoin/etl/overview.mdx index 9a614754..db01b1eb 100644 --- a/website/versioned_docs/version-v1.0/bitcoin/etl/overview.mdx +++ b/website/versioned_docs/version-v1.0/bitcoin/etl/overview.mdx @@ -111,7 +111,6 @@ based on your specific goals and available resources. * Noe4j format files on: * AWS: `s3://bitcoin-graph/v1/data_to_import_neo4j/` - * [Google Drive](https://drive.google.com/drive/folders/11X6QiVvWSOzxvDIAD0OWu3g2Sa0as3UQ?usp=sharing) * [Block summary statistics](https://www.kaggle.com/datasets/aab/bitcoin-block-metadata) @@ -213,7 +212,6 @@ based on your specific goals and available resources. * Neo4j database dump on: * AWS: `s3://bitcoin-graph/v1/neo4j_db_dump/` - * [Google Drive](https://drive.google.com/drive/folders/1bAsjgVaIQrG2TDGkMtIEDPX8xKoiHJUf?usp=sharing)