From 0324b1c47c77094858205145295b6de4aa589c53 Mon Sep 17 00:00:00 2001 From: Vahid Date: Fri, 29 May 2026 21:04:18 -0400 Subject: [PATCH 1/5] Extend graph elemen features serialized. --- .../Bitcoin/GraphModel/BlockNode.cs | 24 +++++++++++++++ .../Blockchains/Bitcoin/GraphModel/S2TEdge.cs | 22 ++++++++++++++ .../Blockchains/Bitcoin/GraphModel/T2SEdge.cs | 20 +++++++++++++ src/AAB.EBA/Utilities/OHLCV.cs | 30 +++++++++++++------ 4 files changed, 87 insertions(+), 9 deletions(-) diff --git a/src/AAB.EBA/Blockchains/Bitcoin/GraphModel/BlockNode.cs b/src/AAB.EBA/Blockchains/Bitcoin/GraphModel/BlockNode.cs index c16b7a8d..d2e1e63a 100644 --- a/src/AAB.EBA/Blockchains/Bitcoin/GraphModel/BlockNode.cs +++ b/src/AAB.EBA/Blockchains/Bitcoin/GraphModel/BlockNode.cs @@ -57,12 +57,24 @@ public override string GetIdPropertyName() nameof(BlockMetadata.Weight), nameof(BlockMetadata.CoinbaseOutputsCount), nameof(BlockMetadata.MintedBitcoins), + nameof(BlockMetadata.TotalSupply), + nameof(BlockMetadata.TotalSupplyNominal), + nameof(BlockMetadata.RealizedCap), + nameof(BlockMetadata.UnrealizedLoss), + nameof(BlockMetadata.UnrealizedProfit), + nameof(BlockMetadata.MarketCap), + nameof(BlockMetadata.NUL), + nameof(BlockMetadata.NUP), + nameof(BlockMetadata.NUPL), + nameof(BlockMetadata.MVRV), + nameof(BlockMetadata.Thermocap), .. DescriptiveStatistics.GetFeaturesName(nameof(BlockMetadata.InputCountsStats)), .. DescriptiveStatistics.GetFeaturesName(nameof(BlockMetadata.OutputCountsStats)), .. DescriptiveStatistics.GetFeaturesName(nameof(BlockMetadata.InputValuesStats)), .. DescriptiveStatistics.GetFeaturesName(nameof(BlockMetadata.OutputValuesStats)), .. DescriptiveStatistics.GetFeaturesName(nameof(BlockMetadata.SpentOutputAgeStats)), .. DescriptiveStatistics.GetFeaturesName(nameof(BlockMetadata.FeesStats)), + .. OHLCV.GetFeaturesName(), .. Node.GetFeaturesName() ]; } @@ -80,12 +92,24 @@ public override string[] GetFeatures() BlockMetadata.Weight.ToString(), BlockMetadata.CoinbaseOutputsCount.ToString(), BlockMetadata.MintedBitcoins.ToString(), + BlockMetadata.TotalSupply?.ToString() ?? "NA", + BlockMetadata.TotalSupplyNominal?.ToString() ?? "NA", + BlockMetadata.RealizedCap?.ToString() ?? "NA", + BlockMetadata.UnrealizedLoss?.ToString() ?? "NA", + BlockMetadata.UnrealizedProfit?.ToString() ?? "NA", + BlockMetadata.MarketCap?.ToString() ?? "NA", + BlockMetadata.NUL?.ToString() ?? "NA", + BlockMetadata.NUP?.ToString() ?? "NA", + BlockMetadata.NUPL?.ToString() ?? "NA", + BlockMetadata.MVRV?.ToString() ?? "NA", + BlockMetadata.Thermocap?.ToString() ?? "NA", .. BlockMetadata.InputCountsStats.GetFeatures(), .. BlockMetadata.OutputCountsStats.GetFeatures(), .. BlockMetadata.InputValuesStats.GetFeatures(), .. BlockMetadata.OutputValuesStats.GetFeatures(), .. BlockMetadata.SpentOutputAgeStats.GetFeatures(), .. BlockMetadata.FeesStats.GetFeatures(), + .. BlockMetadata.Ohlcv?.GetFeatures() ?? OHLCV.GetEmptyFeatures(), .. base.GetFeatures() ]; } diff --git a/src/AAB.EBA/Blockchains/Bitcoin/GraphModel/S2TEdge.cs b/src/AAB.EBA/Blockchains/Bitcoin/GraphModel/S2TEdge.cs index 40e81dbc..1742989d 100644 --- a/src/AAB.EBA/Blockchains/Bitcoin/GraphModel/S2TEdge.cs +++ b/src/AAB.EBA/Blockchains/Bitcoin/GraphModel/S2TEdge.cs @@ -48,4 +48,26 @@ public S2TEdge( Generated = generated; CreationHeight = creationHeight; } + + public static new string[] GetFeaturesName() + { + return + [ + .. Edge.GetFeaturesName(), + nameof(Generated), + nameof(CreationHeight), + nameof(SpentHeight) + ]; + } + + public override double[] GetFeatures() + { + return + [ + .. base.GetFeatures(), + Generated ? 1.0 : 0.0, + CreationHeight, + SpentHeight, + ]; + } } diff --git a/src/AAB.EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs b/src/AAB.EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs index 76b1ef76..41ce7a67 100644 --- a/src/AAB.EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs +++ b/src/AAB.EBA/Blockchains/Bitcoin/GraphModel/T2SEdge.cs @@ -45,6 +45,26 @@ public T2SEdge( SpentHeight = spentHeight; } + public static new string[] GetFeaturesName() + { + return + [ + .. Edge.GetFeaturesName(), + nameof(CreationHeight), + nameof(SpentHeight) + ]; + } + + public override double[] GetFeatures() + { + return + [ + .. base.GetFeatures(), + CreationHeight, + SpentHeight, + ]; + } + public override int GetHashCode() { return HashCode.Combine(base.GetHashCode(), Vout); diff --git a/src/AAB.EBA/Utilities/OHLCV.cs b/src/AAB.EBA/Utilities/OHLCV.cs index da91ce4d..7639f617 100644 --- a/src/AAB.EBA/Utilities/OHLCV.cs +++ b/src/AAB.EBA/Utilities/OHLCV.cs @@ -110,14 +110,13 @@ public static string[] GetFeaturesName() { return [ - nameof(Timestamp), - nameof(Open), - nameof(High), - nameof(Low), - nameof(Close), - nameof(VWAP), - nameof(OHLC4), - $"{nameof(Volume)}(Satoshi)" + $"OHLCV.{nameof(Open)}", + $"OHLCV.{nameof(High)}", + $"OHLCV.{nameof(Low)}", + $"OHLCV.{nameof(Close)}", + $"OHLCV.{nameof(VWAP)}", + $"OHLCV.{nameof(OHLC4)}", + $"OHLCV.{nameof(Volume)}(Satoshi)" ]; } @@ -125,7 +124,6 @@ public string[] GetFeatures() { return [ - Timestamp.ToString(CultureInfo.InvariantCulture), Open.ToString(CultureInfo.InvariantCulture), High.ToString(CultureInfo.InvariantCulture), Low.ToString(CultureInfo.InvariantCulture), @@ -135,4 +133,18 @@ public string[] GetFeatures() Volume.ToString(CultureInfo.InvariantCulture) ]; } + + public static string[] GetEmptyFeatures() + { + return + [ + "NA", // Open + "NA", // High + "NA", // Low + "NA", // Close + "NA", // VWAP + "NA", // OHLC4 + "NA" // Volume + ]; + } } From 0d8a55a9e4a188c222437c38008de855af61be30 Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 1 Jun 2026 20:09:49 -0400 Subject: [PATCH 2/5] Bug trying to get block and tx node types if they don't exist. --- .../Bitcoin/TraversalAlgorithms/Panorama.cs | 78 ++++++++++--------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/src/AAB.EBA/Graph/Bitcoin/TraversalAlgorithms/Panorama.cs b/src/AAB.EBA/Graph/Bitcoin/TraversalAlgorithms/Panorama.cs index 62ca8903..f46e1bf3 100644 --- a/src/AAB.EBA/Graph/Bitcoin/TraversalAlgorithms/Panorama.cs +++ b/src/AAB.EBA/Graph/Bitcoin/TraversalAlgorithms/Panorama.cs @@ -334,47 +334,55 @@ private async Task GetNeighborsAsync( private async Task EnsureB2T(BitcoinGraph g, CancellationToken ct) { - foreach (var txNode in g.NodesByType[NodeKind.Tx]) + // there could be many reasons as to why a graph may not contain any Tx nodes; + // e.g., if the graph is empty, the traversal was so small that did not include any Tx nodes. + if (g.NodesByType.TryGetValue(NodeKind.Tx, out var txNodes)) { - var v = (TxNode)txNode; - var found = false; - foreach (var edge in g.EdgesByType[B2TEdge.Kind]) + foreach (var txNode in txNodes) { - var t = (B2TEdge)edge; - if (t.Target.Txid == v.Txid) + var v = (TxNode)txNode; + var found = false; + if (g.EdgesByType.TryGetValue(B2TEdge.Kind, out var b2tEdges)) { - found = true; - break; + foreach (var edge in b2tEdges) + { + var t = (B2TEdge)edge; + if (t.Target.Txid == v.Txid) + { + found = true; + break; + } + } } - } - - if (!found) - { - var records = await _graphDb.GetEdgesAsync( - edgeKind: B2TEdge.Kind, - sourceNodeVariable: "block", - targetNodeVariable: "tx", - relationshipVariable: "b2t", - targetNodeIdProperty: _txidMapping.Property.Name, - targetNodeId: _txidMapping.GetValue(v), - ct: ct); - var record = records[0]; - double txNodeOutHops = v.OutHopsFromRoot ?? 0; - - TryUnpackNodeDict( - record["block"].As().As>(), - txNodeOutHops + 1, - out var builtBlockNode); - - if (builtBlockNode != null) + if (!found) { - var blockNode = g.GetOrAddNode(builtBlockNode); - - var candidateEdge = _graphDb.StrategyFactory.CreateEdge(blockNode, v, record["b2t"].As()); - - if (candidateEdge != null) - g.TryGetOrAddEdge(candidateEdge, out IEdge _); + var records = await _graphDb.GetEdgesAsync( + edgeKind: B2TEdge.Kind, + sourceNodeVariable: "block", + targetNodeVariable: "tx", + relationshipVariable: "b2t", + targetNodeIdProperty: _txidMapping.Property.Name, + targetNodeId: _txidMapping.GetValue(v), + ct: ct); + + var record = records[0]; + double txNodeOutHops = v.OutHopsFromRoot ?? 0; + + TryUnpackNodeDict( + record["block"].As().As>(), + txNodeOutHops + 1, + out var builtBlockNode); + + if (builtBlockNode != null) + { + var blockNode = g.GetOrAddNode(builtBlockNode); + + var candidateEdge = _graphDb.StrategyFactory.CreateEdge(blockNode, v, record["b2t"].As()); + + if (candidateEdge != null) + g.TryGetOrAddEdge(candidateEdge, out IEdge _); + } } } } From f7f3e8994c8fcfb8996c9438ad910d31d27d1637 Mon Sep 17 00:00:00 2001 From: Vahid Date: Mon, 15 Jun 2026 20:20:07 -0400 Subject: [PATCH 3/5] bug fix after merging remote. --- .../Bitcoin/TraversalAlgorithms/Panorama.cs | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/AAB.EBA/Graph/Bitcoin/TraversalAlgorithms/Panorama.cs b/src/AAB.EBA/Graph/Bitcoin/TraversalAlgorithms/Panorama.cs index dea7599f..f46e1bf3 100644 --- a/src/AAB.EBA/Graph/Bitcoin/TraversalAlgorithms/Panorama.cs +++ b/src/AAB.EBA/Graph/Bitcoin/TraversalAlgorithms/Panorama.cs @@ -338,17 +338,22 @@ private async Task EnsureB2T(BitcoinGraph g, CancellationToken ct) // e.g., if the graph is empty, the traversal was so small that did not include any Tx nodes. if (g.NodesByType.TryGetValue(NodeKind.Tx, out var txNodes)) { - var v = (TxNode)txNode; - var found = false; - foreach (var edge in g.EdgesByType[B2TEdge.Kind]) + foreach (var txNode in txNodes) { - var t = (B2TEdge)edge; - if (t.Target.Txid == v.Txid) + var v = (TxNode)txNode; + var found = false; + if (g.EdgesByType.TryGetValue(B2TEdge.Kind, out var b2tEdges)) { - found = true; - break; + foreach (var edge in b2tEdges) + { + var t = (B2TEdge)edge; + if (t.Target.Txid == v.Txid) + { + found = true; + break; + } + } } - } if (!found) { From 872aa0aa3638264924084d4dab17e99a363616c8 Mon Sep 17 00:00:00 2001 From: Vahid Date: Thu, 18 Jun 2026 00:55:07 -0400 Subject: [PATCH 4/5] Extend the sampling algorithm --- ...BitcoinPanoramaSamplingAlgorithmOptions.cs | 47 ++++- .../Bitcoin/TraversalAlgorithms/Panorama.cs | 166 +++++++++++++----- src/AAB.EBA/Graph/Db/IGraphDb.cs | 2 +- src/AAB.EBA/Graph/Db/Neo4jDb/Neo4jDb.cs | 2 +- 4 files changed, 161 insertions(+), 56 deletions(-) diff --git a/src/AAB.EBA/CLI/Config/BitcoinPanoramaSamplingAlgorithmOptions.cs b/src/AAB.EBA/CLI/Config/BitcoinPanoramaSamplingAlgorithmOptions.cs index 965b9746..1eeeac03 100644 --- a/src/AAB.EBA/CLI/Config/BitcoinPanoramaSamplingAlgorithmOptions.cs +++ b/src/AAB.EBA/CLI/Config/BitcoinPanoramaSamplingAlgorithmOptions.cs @@ -8,16 +8,34 @@ public class BitcoinPanoramaSamplingAlgorithmOptions [JsonPropertyName("queryLimit")] public int QueryLimit { init; get; } = 1000; - [JsonPropertyName("reductionFactor")] - public double NodeCountReductionFactorByHop { init; get; } = 4.0; + [JsonPropertyName("neighborhoodSamplePercentagePerHop")] + public double NeighborhoodSamplePercentagePerHop { init; get; } = 10.0; /// - /// This could be used to exclude high degree nodes, - /// such as mixer nodes or nodes belonging to exchanges; - /// since two nodes connected such nodes cannot necessarily indicate they are related. + /// Sets the percentage of the immediate neighbors of the root node + /// to be included in the sampled community. + /// The value should be between 0 and 100, + /// where 100 means all immediate neighbors of the root node will be included. /// - [JsonPropertyName("nodeCountAtRoot")] - public int NodeSamplingCountAtRoot { init; get; } = 100; + [JsonPropertyName("nodeSamplingPercentageAtRoot")] + public double NodeSamplingPercentageAtRoot + { + init + { + if (value < 0 || value > 100) + { + throw new ArgumentOutOfRangeException( + nameof(NodeSamplingPercentageAtRoot), + "Value must be between 0 and 100."); + } + _nodeSamplingPercentageAtOtherHops = value; + } + get + { + return _nodeSamplingPercentageAtOtherHops; + } + } + private double _nodeSamplingPercentageAtOtherHops = 100; /// /// This could be used to exclude high degree nodes, @@ -30,6 +48,9 @@ public class BitcoinPanoramaSamplingAlgorithmOptions [JsonPropertyName("maxOriginalOutDegree")] public int MaxOriginalOutDegree { init; get; } = 1000; + [JsonPropertyName("maxNeighborsPerNode")] + public int MaxNeighborsPerNode { init; get; } = 100; + /// /// This ensures a block context for each transaction is /// also included in the sampled communities. @@ -37,4 +58,16 @@ public class BitcoinPanoramaSamplingAlgorithmOptions /// [JsonPropertyName("includeBlockTxEdges")] public bool IncludeB2TEdges { init; get; } = false; + + /// + /// As part of the random expansion, coinbase node gets + /// the same selection probability as other nodes; + /// hence, it may or may not be included in the sampled community. + /// If this property is set to true, + /// if the coinbase node is included in the traversed neighborhood, + /// the algorithm will include it in the pseudo-randomly sampled subset + /// of the traversed neighborhood. + /// + [JsonPropertyName("forceIncludeCoinbaseNode")] + public bool ForceIncludeCoinbaseNode { init; get; } = true; } diff --git a/src/AAB.EBA/Graph/Bitcoin/TraversalAlgorithms/Panorama.cs b/src/AAB.EBA/Graph/Bitcoin/TraversalAlgorithms/Panorama.cs index f46e1bf3..3e2529e1 100644 --- a/src/AAB.EBA/Graph/Bitcoin/TraversalAlgorithms/Panorama.cs +++ b/src/AAB.EBA/Graph/Bitcoin/TraversalAlgorithms/Panorama.cs @@ -55,10 +55,6 @@ public async Task SampleAsync(CancellationToken ct) rootNodeLabel: ScriptNode.Kind, rootNodeIdProperty: rootNode.GetIdPropertyName(), rootNodeId: rootNode.Id, - nodeSamplingCountAtRoot: _options.Bitcoin.GraphSample.PanoramaOptions.NodeSamplingCountAtRoot, // can set this to max int - maxHops: _options.Bitcoin.GraphSample.PanoramaOptions.MaxHops, - queryLimit: _options.Bitcoin.GraphSample.PanoramaOptions.QueryLimit, - nodeCountReductionFactorByHop: _options.Bitcoin.GraphSample.PanoramaOptions.NodeCountReductionFactorByHop, ct: ct); var perBatchLabelsFilename = Path.Join(_options.WorkingDir, "labels.tsv"); @@ -110,11 +106,9 @@ public async Task SampleAsync(CancellationToken ct) /// This method ensures that only unique nodes and edges not already present in the graph are added. /// The number of nodes sampled at each hop decreases linearly based on the specified reduction factor. /// - private List ProcessQueriedNeighborhood( + private List ProcessNeighborhood( List samplingResult, int hop, - int nodeSamplingCountAtRoot, - double nodeCountReductionFactorByHop, BitcoinGraph g, CancellationToken ct) { @@ -127,15 +121,16 @@ public async Task SampleAsync(CancellationToken ct) var edges = new List(); var rootList = samplingResult[0]["root"].As>(); - if (!TryUnpackNodeDict(rootList[0].As>(), hop, out var builtRootNode) || builtRootNode == null) + if (!TryUnpackNodeDict(rootList[0].As>(), hop, out var builtParentNode) || builtParentNode == null) return nodesAddedToGraph; - Model.INode rootNode = g.GetOrAddNode(builtRootNode); + Model.INode parentNode = g.GetOrAddNode(builtParentNode); if (hop == 0) - g.AddLabel("RootNodeId", rootNode.Id); + g.AddLabel("RootNodeId", parentNode.Id); - for (int i = 1; i < samplingResult.Count; i++) + var coinbaseNodeIdInGraphDb = ""; + for (var i = 1; i < samplingResult.Count; i++) { var r = samplingResult[i]; foreach (var nodeObject in r["nodes"].As>()) @@ -146,9 +141,16 @@ public async Task SampleAsync(CancellationToken ct) continue; if (g.TryGetNode(node.Id, out var nodeInG)) + { nodesInThisHopAlreadyInGraph.TryAdd(node.IdInGraphDb, nodeInG); + } else + { nodesUniqueToThisHop.TryAdd(node.IdInGraphDb, node); + + if (node.NodeKind == NodeKind.Coinbase) + coinbaseNodeIdInGraphDb = node.IdInGraphDb; + } } foreach (var edge in r.Values["relationships"].As>()) @@ -158,10 +160,27 @@ public async Task SampleAsync(CancellationToken ct) ct.ThrowIfCancellationRequested(); var rnd = new Random(31); - var nodesToKeep = nodesUniqueToThisHop.Keys.OrderBy( - x => rnd.Next()).Take( - (int)Math.Floor(nodeSamplingCountAtRoot - hop * nodeCountReductionFactorByHop)) - .ToHashSet(); + + var parentNodeDegree = (parentNode.OriginalInDegree ?? 0) + (parentNode.OriginalOutDegree ?? 0); + var samplingPercentage = + _options.Bitcoin.GraphSample.PanoramaOptions.NodeSamplingPercentageAtRoot - + (hop * _options.Bitcoin.GraphSample.PanoramaOptions.NeighborhoodSamplePercentagePerHop); + var neighborsToKeep = (int)Math.Floor((parentNodeDegree * samplingPercentage) / 100.0); + var count = Math.Min( + _options.Bitcoin.GraphSample.PanoramaOptions.MaxNeighborsPerNode, + neighborsToKeep - nodesInThisHopAlreadyInGraph.Count); + + var nodesToKeep = nodesUniqueToThisHop.Keys.OrderBy(x => rnd.Next()).Take(count).ToHashSet(); + + if (_options.Bitcoin.GraphSample.PanoramaOptions.ForceIncludeCoinbaseNode && + !string.IsNullOrEmpty(coinbaseNodeIdInGraphDb)) + { + if (!nodesToKeep.Contains(coinbaseNodeIdInGraphDb)) + { + nodesToKeep.Remove(nodesToKeep.First()); + nodesToKeep.Add(coinbaseNodeIdInGraphDb); + } + } foreach (var edge in edges) { @@ -173,9 +192,9 @@ public async Task SampleAsync(CancellationToken ct) } string subjectNodeGraphDbId; - if (edge.StartNodeElementId == rootNode.IdInGraphDb) + if (edge.StartNodeElementId == parentNode.IdInGraphDb) subjectNodeGraphDbId = edge.EndNodeElementId; - else if (edge.EndNodeElementId == rootNode.IdInGraphDb) + else if (edge.EndNodeElementId == parentNode.IdInGraphDb) subjectNodeGraphDbId = edge.StartNodeElementId; else continue; // edge is not connected to rootNode @@ -196,46 +215,111 @@ public async Task SampleAsync(CancellationToken ct) } IEdge candidateEdge = - edge.StartNodeElementId == rootNode.IdInGraphDb ? - _graphDb.StrategyFactory.CreateEdge(rootNode, subjectNode, edge) : - _graphDb.StrategyFactory.CreateEdge(subjectNode, rootNode, edge); + edge.StartNodeElementId == parentNode.IdInGraphDb ? + _graphDb.StrategyFactory.CreateEdge(parentNode, subjectNode, edge) : + _graphDb.StrategyFactory.CreateEdge(subjectNode, parentNode, edge); g.TryGetOrAddEdge(candidateEdge, out candidateEdge); } return nodesAddedToGraph; } - private async Task ProcessHops( + private async Task ProcessHopsBFS( + NodeKind rootNodeLabel, + string rootNodeIdProperty, + string rootNodeId, + int initialHop, + BitcoinGraph g, + CancellationToken ct) + { + var queue = new Queue<(NodeKind Label, string IdProperty, string Id, int Hop)>(); + queue.Enqueue((rootNodeLabel, rootNodeIdProperty, rootNodeId, initialHop)); + + var expandedNodes = new HashSet { rootNodeId }; + + while (queue.Count > 0) + { + ct.ThrowIfCancellationRequested(); + + var currentNode = queue.Dequeue(); + _maxHopReached = Math.Max(_maxHopReached, currentNode.Hop); + + var rawNeighborhood = await _graphDb.GetNeighborhoodAsync( + currentNode.Label, + currentNode.IdProperty, + currentNode.Id, + _options.Bitcoin.GraphSample.PanoramaOptions.QueryLimit, + 1, + true, + ct: ct); + + var nodesAddedToGraph = ProcessNeighborhood( + rawNeighborhood, + currentNode.Hop, + g: g, + ct: ct); + + if (g.NodeCount >= _options.Bitcoin.GraphSample.MaxNodeCount || + g.EdgeCount >= _options.Bitcoin.GraphSample.MaxEdgeCount) + { + return false; + } + + if (currentNode.Hop < _options.Bitcoin.GraphSample.PanoramaOptions.MaxHops) + { + foreach (var node in nodesAddedToGraph) + { + // Note that the `node` is already in the graph, we are just skipping expanding to its neighbors + if (node.NodeKind == NodeKind.Block || + node.NodeKind == NodeKind.Coinbase) + { + continue; + } + + // Note that the `node` is already in the graph, we are just skipping expanding to its neighbors + if ((node.OriginalInDegree != null && node.OriginalInDegree > _options.Bitcoin.GraphSample.PanoramaOptions.MaxOriginalInDegree) || + (node.OriginalOutDegree != null && node.OriginalOutDegree > _options.Bitcoin.GraphSample.PanoramaOptions.MaxOriginalOutDegree)) + { + continue; + } + + // prevent expanding the same node twice + if (expandedNodes.Add(node.Id)) + { + queue.Enqueue((node.NodeKind, node.GetIdPropertyName(), node.Id, currentNode.Hop + 1)); + } + } + } + } + + return true; + } + + private async Task ProcessHopsDFS( NodeKind rootNodeLabel, string rootNodeIdProperty, string rootNodeId, - int maxHops, int hop, - int queryLimit, - int nodeSamplingCountAtRoot, - double nodeCountReductionFactorByHop, BitcoinGraph g, CancellationToken ct) { ct.ThrowIfCancellationRequested(); - var samplingResult = await _graphDb.GetNeighborsAsync( + var samplingResult = await _graphDb.GetNeighborhoodAsync( rootNodeLabel, rootNodeIdProperty, rootNodeId, - queryLimit, + _options.Bitcoin.GraphSample.PanoramaOptions.QueryLimit, 1, true, ct: ct); - var selectedNodes = ProcessQueriedNeighborhood( + var selectedNodes = ProcessNeighborhood( samplingResult, hop, - nodeSamplingCountAtRoot: nodeSamplingCountAtRoot, - nodeCountReductionFactorByHop: nodeCountReductionFactorByHop, g: g, ct: ct); - if (hop < maxHops) + if (hop < _options.Bitcoin.GraphSample.PanoramaOptions.MaxHops) { _maxHopReached = Math.Max(_maxHopReached, hop); @@ -257,15 +341,11 @@ private async Task ProcessHops( continue; } - await ProcessHops( + await ProcessHopsDFS( rootNodeLabel: node.NodeKind, rootNodeIdProperty: node.GetIdPropertyName(), rootNodeId: node.Id, hop: hop + 1, - maxHops: maxHops, - queryLimit: queryLimit, - nodeSamplingCountAtRoot: nodeSamplingCountAtRoot, - nodeCountReductionFactorByHop: nodeCountReductionFactorByHop, g: g, ct: ct); } @@ -278,10 +358,6 @@ private async Task GetNeighborsAsync( NodeKind rootNodeLabel, string rootNodeIdProperty, string rootNodeId, - int nodeSamplingCountAtRoot, - int maxHops, - int queryLimit, - double nodeCountReductionFactorByHop, CancellationToken ct) { ct.ThrowIfCancellationRequested(); @@ -292,18 +368,14 @@ private async Task GetNeighborsAsync( _logger.LogInformation( "Retrieving neighbors for root node {rootNodeTag} up to {MaxHops} hop(s). ", rootNodeLabelInLogs, - maxHops); + _options.Bitcoin.GraphSample.PanoramaOptions.MaxHops); _maxHopReached = 0; - var completedWalk = await ProcessHops( + var completedWalk = await ProcessHopsBFS( rootNodeLabel: rootNodeLabel, rootNodeIdProperty: rootNodeIdProperty, rootNodeId: rootNodeId, - maxHops: maxHops, - hop: 0, - queryLimit: queryLimit, - nodeSamplingCountAtRoot: nodeSamplingCountAtRoot, - nodeCountReductionFactorByHop: nodeCountReductionFactorByHop, + initialHop: 0, g: g, ct: ct); diff --git a/src/AAB.EBA/Graph/Db/IGraphDb.cs b/src/AAB.EBA/Graph/Db/IGraphDb.cs index fd2ed1a5..6cca1dee 100644 --- a/src/AAB.EBA/Graph/Db/IGraphDb.cs +++ b/src/AAB.EBA/Graph/Db/IGraphDb.cs @@ -37,7 +37,7 @@ public Task> GetRandomNodesAsync( double rootNodeSelectProbability = 0.1, string nodeVariable = "randomNode"); - public Task> GetNeighborsAsync( + public Task> GetNeighborhoodAsync( NodeKind rootNodeLabel, string rootNodeIdProperty, string rootNodeId, diff --git a/src/AAB.EBA/Graph/Db/Neo4jDb/Neo4jDb.cs b/src/AAB.EBA/Graph/Db/Neo4jDb/Neo4jDb.cs index 31cd38d3..4f7c2d18 100644 --- a/src/AAB.EBA/Graph/Db/Neo4jDb/Neo4jDb.cs +++ b/src/AAB.EBA/Graph/Db/Neo4jDb/Neo4jDb.cs @@ -95,7 +95,7 @@ public async Task> GetNodesAsync( return rndRecords; } - public async Task> GetNeighborsAsync( + public async Task> GetNeighborhoodAsync( NodeKind rootNodeLabel, string rootNodeIdProperty, string rootNodeId, From 570827b05ada9987c64b25487e1b0d6c5e8916a1 Mon Sep 17 00:00:00 2001 From: Vahid Date: Thu, 18 Jun 2026 23:27:24 -0400 Subject: [PATCH 5/5] Add FeaturesSchema class that allows creating a schema.json file with every serialized community. --- .../CLI/Config/BitcoinGraphSampleOptions.cs | 15 ---- .../Graph/Bitcoin/BitcoinGraphOrchestrator.cs | 2 - src/AAB.EBA/Graph/Bitcoin/FeaturesSchema.cs | 68 +++++++++++++++++++ .../Bitcoin/TraversalAlgorithms/Panorama.cs | 2 + src/AAB.EBA/Graph/Model/GraphBase.cs | 19 +++--- src/AAB.EBA/Graph/Model/GraphFeatures.cs | 20 ++---- 6 files changed, 87 insertions(+), 39 deletions(-) create mode 100644 src/AAB.EBA/Graph/Bitcoin/FeaturesSchema.cs diff --git a/src/AAB.EBA/CLI/Config/BitcoinGraphSampleOptions.cs b/src/AAB.EBA/CLI/Config/BitcoinGraphSampleOptions.cs index 7ceb4673..2f199f43 100644 --- a/src/AAB.EBA/CLI/Config/BitcoinGraphSampleOptions.cs +++ b/src/AAB.EBA/CLI/Config/BitcoinGraphSampleOptions.cs @@ -2,21 +2,6 @@ namespace AAB.EBA.CLI.Config; public enum GraphTraversal { - // path search algorithm - // traverse the graph using the given algorithm - // deterministic sampling algorithm - // stops when a criteria is met (e.g., max number of nodes or edges sampled) - // Breadth-first Search - //BFS, - - // path search algorithm - // traverse the graph using the given algorithm - // deterministic sampling algorithm - // stops when a criteria is met (e.g., max number of nodes or edges sampled) - // Depth-first Search - //DFS, - - // sampling algorithm // non-deterministic sampling algorithm Panorama } diff --git a/src/AAB.EBA/Graph/Bitcoin/BitcoinGraphOrchestrator.cs b/src/AAB.EBA/Graph/Bitcoin/BitcoinGraphOrchestrator.cs index 67055a5c..9802975e 100644 --- a/src/AAB.EBA/Graph/Bitcoin/BitcoinGraphOrchestrator.cs +++ b/src/AAB.EBA/Graph/Bitcoin/BitcoinGraphOrchestrator.cs @@ -27,8 +27,6 @@ public async Task SampleAsync(CancellationToken ct) ITraversalAlgorithm sampler = _options.Bitcoin.GraphSample.TraversalAlgorithm switch { GraphTraversal.Panorama => new Panorama(_options, _db, _logger), - //GraphTraversal.BFS => throw new NotImplementedException(), - //GraphTraversal.DFS => throw new NotImplementedException(), _ => throw new NotImplementedException("Unsupported graph traversal algorithm."), }; diff --git a/src/AAB.EBA/Graph/Bitcoin/FeaturesSchema.cs b/src/AAB.EBA/Graph/Bitcoin/FeaturesSchema.cs new file mode 100644 index 00000000..b0aa7cd7 --- /dev/null +++ b/src/AAB.EBA/Graph/Bitcoin/FeaturesSchema.cs @@ -0,0 +1,68 @@ +namespace AAB.EBA.Graph.Bitcoin; + +public class FeaturesSchema +{ + public class Metadata + { + [JsonPropertyName("filename")] + public string Filename { get; } + + [JsonPropertyName("kind")] + public string Kind { get; } + + [JsonPropertyName("features")] + public List Features { get; } + + public Metadata(NodeKind kind, List features) + { + Filename = GetElementFilename(kind); + Kind = kind.ToString(); + Features = features; + } + + public Metadata(EdgeKind kind, List features) + { + Filename = GetElementFilename(kind); + Kind = kind.ToString(); + Features = features; + } + } + + [JsonPropertyName("nodeTypes")] + public Dictionary NodeTypes => + StaticNodeTypes.ToDictionary(k => k.Key.ToString(), v => v.Value); + + [JsonPropertyName("edgeTypes")] + public Dictionary EdgeTypes => + StaticEdgeTypes.ToDictionary(k => k.Key.ToString(), v => v.Value); + + public static Dictionary StaticNodeTypes { get; } = new() + { + { BlockNode.Kind, new Metadata(BlockNode.Kind, ["Index", .. BlockNode.GetFeaturesName()]) }, + { TxNode.Kind, new Metadata(TxNode.Kind, ["Index", .. TxNode.GetFeaturesName()]) }, + { ScriptNode.Kind, new Metadata(ScriptNode.Kind, ["Index", .. ScriptNode.GetFeaturesName()]) }, + { CoinbaseNode.Kind, new Metadata(CoinbaseNode.Kind, ["Index", .. CoinbaseNode.GetFeaturesName()]) } + }; + + private static readonly string[] SourceAndTarget = ["Source", "Target"]; + public static Dictionary StaticEdgeTypes { get; } = new() + { + { C2TEdge.Kind, new Metadata(C2TEdge.Kind, [.. SourceAndTarget, .. C2TEdge.GetFeaturesName()]) }, + { B2TEdge.Kind, new Metadata(B2TEdge.Kind, [.. SourceAndTarget, .. B2TEdge.GetFeaturesName()]) }, + { S2TEdge.Kind, new Metadata(S2TEdge.Kind, [.. SourceAndTarget, .. S2TEdge.GetFeaturesName()]) }, + { T2SEdge.Kind, new Metadata(T2SEdge.Kind, [.. SourceAndTarget, .. T2SEdge.GetFeaturesName()]) }, + { T2TEdge.KindFee, new Metadata(T2TEdge.KindFee, [.. SourceAndTarget, .. T2TEdge.GetFeaturesName()]) }, + { T2TEdge.KindTransfers, new Metadata(T2TEdge.KindTransfers, [.. SourceAndTarget, .. T2TEdge.GetFeaturesName()]) }, + { B2BEdge.Kind, new Metadata(B2BEdge.Kind, [.. SourceAndTarget, .. B2BEdge.GetFeaturesName()]) } + }; + + public static string GetElementFilename(NodeKind nodeKind) + { + return $"node_features_{nodeKind.ToString().ToLower().Replace('-', '_')}.tsv"; + } + + public static string GetElementFilename(EdgeKind edgeKind) + { + return $"edge_features_{edgeKind.ToString().ToLower().Replace('-', '_')}.tsv"; + } +} diff --git a/src/AAB.EBA/Graph/Bitcoin/TraversalAlgorithms/Panorama.cs b/src/AAB.EBA/Graph/Bitcoin/TraversalAlgorithms/Panorama.cs index 3e2529e1..e6aa65c1 100644 --- a/src/AAB.EBA/Graph/Bitcoin/TraversalAlgorithms/Panorama.cs +++ b/src/AAB.EBA/Graph/Bitcoin/TraversalAlgorithms/Panorama.cs @@ -26,6 +26,8 @@ public async Task SampleAsync(CancellationToken ct) _logger.LogInformation("Sampling {n:N0} graphs.", _options.Bitcoin.GraphSample.Count - sampledSubGraphsCount); + JsonSerializer.Serialize(new FeaturesSchema(), Path.Combine(_options.WorkingDir, "schema.json")); + while ( sampledSubGraphsCount < _options.Bitcoin.GraphSample.Count && ++attempts <= _options.Bitcoin.GraphSample.MaxAttempts) diff --git a/src/AAB.EBA/Graph/Model/GraphBase.cs b/src/AAB.EBA/Graph/Model/GraphBase.cs index 9382a7e3..07583b13 100644 --- a/src/AAB.EBA/Graph/Model/GraphBase.cs +++ b/src/AAB.EBA/Graph/Model/GraphBase.cs @@ -1,4 +1,5 @@ -using AAB.EBA.Utilities; +using AAB.EBA.Graph.Bitcoin; +using AAB.EBA.Utilities; using System.Collections.Immutable; @@ -189,7 +190,7 @@ public GraphFeatures GetFeatures() } public void SerializeFeatures( - string workingDir, + string graphDir, string perBatchLabelsFilename, string perGraphLabelsFilename = "metadata.tsv") { @@ -202,7 +203,7 @@ public void SerializeFeatures( Helpers.CsvSerialize( nodeType.Value, - Path.Join(workingDir, $"node_features_{nodeType.Key.ToString().ToLower().Replace('-', '_')}.tsv"), + Path.Join(graphDir, FeaturesSchema.GetElementFilename(nodeType.Key)), gFeatures.NodeFeaturesHeader[nodeType.Key]); } @@ -213,20 +214,20 @@ public void SerializeFeatures( Helpers.CsvSerialize( edgeType.Value, - Path.Join(workingDir, $"edge_features_{edgeType.Key.ToString().ToLower().Replace('-', '_')}.tsv"), + Path.Join(graphDir, FeaturesSchema.GetElementFilename(edgeType.Key)), gFeatures.EdgeFeaturesHeader[edgeType.Key]); } Helpers.CsvSerialize( [gFeatures.Labels.ToArray()], - Path.Combine(workingDir, perBatchLabelsFilename), - gFeatures.LabelsHeader, - append: true); + Path.Combine(graphDir, perGraphLabelsFilename), + gFeatures.LabelsHeader); Helpers.CsvSerialize( [gFeatures.Labels.ToArray()], - Path.Combine(workingDir, perGraphLabelsFilename), - gFeatures.LabelsHeader); + perBatchLabelsFilename, + gFeatures.LabelsHeader, + append: true); } public bool Equals(GraphBase? other) diff --git a/src/AAB.EBA/Graph/Model/GraphFeatures.cs b/src/AAB.EBA/Graph/Model/GraphFeatures.cs index 112f4d6b..66bf12e2 100644 --- a/src/AAB.EBA/Graph/Model/GraphFeatures.cs +++ b/src/AAB.EBA/Graph/Model/GraphFeatures.cs @@ -1,4 +1,6 @@ -namespace AAB.EBA.Graph.Model; +using AAB.EBA.Graph.Bitcoin; + +namespace AAB.EBA.Graph.Model; public class GraphFeatures { @@ -25,20 +27,12 @@ public GraphFeatures(GraphBase graph) LabelsHeader = new ReadOnlyCollection(["GraphID", "RootNodeId", "RootNodeIdx", "NodeCount", "EdgeCount"]); NodeFeaturesHeader = []; - NodeFeaturesHeader.Add(BlockNode.Kind, ["Index", .. BlockNode.GetFeaturesName()]); - NodeFeaturesHeader.Add(TxNode.Kind, ["Index", .. TxNode.GetFeaturesName()]); - NodeFeaturesHeader.Add(ScriptNode.Kind, ["Index", .. ScriptNode.GetFeaturesName()]); - NodeFeaturesHeader.Add(CoinbaseNode.Kind, ["Index", .. CoinbaseNode.GetFeaturesName()]); + foreach(var nodeType in FeaturesSchema.StaticNodeTypes) + NodeFeaturesHeader.Add(nodeType.Key, [.. nodeType.Value.Features]); - var sourceAndTarget = new[] { "Source", "Target" }; EdgeFeaturesHeader = []; - EdgeFeaturesHeader.Add(C2TEdge.Kind, [.. sourceAndTarget, .. C2TEdge.GetFeaturesName()]); - EdgeFeaturesHeader.Add(B2TEdge.Kind, [.. sourceAndTarget, .. B2TEdge.GetFeaturesName()]); - EdgeFeaturesHeader.Add(S2TEdge.Kind, [.. sourceAndTarget, .. S2TEdge.GetFeaturesName()]); - EdgeFeaturesHeader.Add(T2SEdge.Kind, [.. sourceAndTarget, .. T2SEdge.GetFeaturesName()]); - EdgeFeaturesHeader.Add(T2TEdge.KindFee, [.. sourceAndTarget, .. T2TEdge.GetFeaturesName()]); - EdgeFeaturesHeader.Add(T2TEdge.KindTransfers, [.. sourceAndTarget, .. T2TEdge.GetFeaturesName()]); - EdgeFeaturesHeader.Add(B2BEdge.Kind, [.. sourceAndTarget, .. B2BEdge.GetFeaturesName()]); + foreach(var edgeType in FeaturesSchema.StaticEdgeTypes) + EdgeFeaturesHeader.Add(edgeType.Key, [.. edgeType.Value.Features]); var nodeFeatures = new Dictionary>(); var nodeIdToIdx = new Dictionary>();