From f2e3540f02b1a2a243e1d9f40d8f529f2da41ecf Mon Sep 17 00:00:00 2001 From: Code-haeler Date: Thu, 28 May 2026 05:18:06 +0200 Subject: [PATCH] feat: add JSON output for trace comparison reports --- docs/doc/compare.md | 172 --------------------------------------- src/cli/args.rs | 4 + src/cli/commands.rs | 21 ++++- src/compare/engine.rs | 53 +++++++++--- src/inspector/storage.rs | 24 ++++++ src/runtime/executor.rs | 8 +- 6 files changed, 97 insertions(+), 185 deletions(-) diff --git a/docs/doc/compare.md b/docs/doc/compare.md index aa5c7c00..e69de29b 100644 --- a/docs/doc/compare.md +++ b/docs/doc/compare.md @@ -1,172 +0,0 @@ -# Comparing Execution Traces - -The `compare` subcommand lets you diff two execution trace JSON files -side-by-side. It is designed for **regression testing** — run your -contract, save the trace, make changes, run again, then compare the two -traces to spot any unintended differences. - -## Quick-start - -```bash -soroban-debug compare examples/trace_a.json examples/trace_b.json -``` - -Save the report to a file instead of stdout: - -```bash -soroban-debug compare examples/trace_a.json examples/trace_b.json --output report.txt -``` - -Ignore noisy fields or paths directly in the compare step: - -```bash -soroban-debug compare baseline.json new.json \ - --ignore-field timestamp \ - --ignore-field seq \ - --ignore-path /storage/ledger_seq \ - --ignore-path /return_value/meta/debug -``` - -## What is compared? - -| Dimension | Details | -|-------------------|-------------------------------------------------------| -| **Storage** | Keys added, removed, and modified with old/new values | -| **Budget** | CPU instructions and memory deltas (absolute + %) | -| **Return values** | Equality check with full value display | -| **Execution flow**| LCS-based unified diff of the call sequence | -| **Events** | Side-by-side comparison of emitted events | - -## Ignore filters - -- `--ignore-field ` removes that object field name everywhere in the trace before diffing. -- `--ignore-path ` removes one slash-delimited subtree before diffing. The match applies to the exact path and everything below it. -- Paths are rooted at the trace object. Common examples: - - `/storage/fee_pool` - - `/storage/balance:Alice/timestamp` - - `/return_value/meta/timestamp` - - `/events/0/data` - -These filters affect storage, budget, return values, call sequences, and events in the rendered report. - -## Trace JSON format - -A trace file is a JSON object with the following fields (all optional -except where noted): - -```json -{ - "label": "Human-readable name for the trace", - "contract": "token.wasm", - "function": "transfer", - "args": "[\"Alice\", \"Bob\", 100]", - "storage": { - "balance:Alice": 900, - "balance:Bob": 100, - "total_supply": 1000 - }, - "budget": { - "cpu_instructions": 45000, - "memory_bytes": 15360, - "cpu_limit": 100000, - "memory_limit": 40960 - }, - "return_value": { "status": "ok" }, - "call_sequence": [ - { "function": "transfer", "depth": 0 }, - { "function": "get_balance", "args": "Alice", "depth": 1 }, - { "function": "set_balance", "args": "Alice, 900", "depth": 1 } - ], - "events": [ - { - "contract_id": "CA7QYN...", - "topics": ["transfer"], - "data": "Alice→Bob 100" - } - ] -} -``` - -### Field reference - -| Field | Type | Description | -|------------------|-----------------|-------------------------------------------------| -| `label` | `string?` | Friendly name shown in the report header | -| `contract` | `string?` | Contract WASM path or ID | -| `function` | `string?` | Invoked function name | -| `args` | `string?` | Function arguments (JSON-encoded) | -| `storage` | `object` | Post-execution storage key→value map | -| `budget` | `object?` | CPU and memory usage | -| `return_value` | `any?` | Return value (arbitrary JSON) | -| `call_sequence` | `array` | Ordered list of function calls | -| `events` | `array` | Events emitted during execution | - -## Regression testing workflow - -1. **Capture baseline trace** — run your contract and save the execution - output as `baseline.json`. - -2. **Make contract changes** — e.g., optimize gas usage, add fee logic, etc. - -3. **Capture new trace** — run the modified contract and save as `new.json`. - -4. **Compare** — - ```bash - soroban-debug compare baseline.json new.json - ``` - -5. **Review the report** — look for: - - Unexpected storage modifications (regressions) - - Budget increases (performance regressions) - - Changed return values (behavioural regressions) - - New or missing function calls in the execution flow - -### Example: Detecting a fee regression - -Suppose `v1.0` of your token contract transfers the full amount, and -`v1.1` introduces a fee. The compare output will clearly show: - -``` -───────────────── Storage Changes ───────────────── - - Keys only in B (1): - + fee_pool = 5 - - Modified keys (1): - ~ balance:Alice - A: 900 - B: 895 - -───────────────── Budget Usage ──────────────────── - - A B Delta - CPU instructions 45000 38000 -7000 - - CPU change: -15.56% - Memory change: -8.85% - -───────────────── Return Values ─────────────────── - - A: {"status":"ok"} - B: {"fee_charged":5,"status":"ok"} - -───────────────── Execution Flow ────────────────── - - Unified diff (- = only in A, + = only in B): - - transfer() - + check_allowance(Alice) - get_balance(Alice) - + compute_fee(100) - ... -``` - -## Tips - -- Keep trace files in version control alongside your contract code - so you can compare across Git commits. -- Use `--output` to save the report, then `diff` two reports over time. -- Use `--ignore-field` for volatile metadata such as timestamps, nonces, or sequence numbers. -- Use `--ignore-path` for selected storage keys or specific nested JSON branches that are expected to vary. -- Combine with CI: generate traces in your test suite and run - `soroban-debug compare` as a CI step to catch regressions automatically. diff --git a/src/cli/args.rs b/src/cli/args.rs index 4025fc9d..1b897fd8 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -1062,6 +1062,10 @@ pub struct CompareArgs { /// Repeatable. Useful for timestamps, sequence numbers, and similar metadata. #[arg(long, value_name = "FIELD")] pub ignore_field: Vec, + + /// Output format for the comparison report (pretty or json) + #[arg(long, value_enum, default_value_t = OutputFormat::Pretty)] + pub format: OutputFormat, } /// Arguments for the TUI dashboard subcommand diff --git a/src/cli/commands.rs b/src/cli/commands.rs index cbee9755..28d73bf8 100644 --- a/src/cli/commands.rs +++ b/src/cli/commands.rs @@ -1058,11 +1058,22 @@ pub fn run(args: RunArgs, verbosity: Verbosity) -> Result<()> { let mut pauses = Vec::new(); let hit_entry_breakpoint = args.breakpoint.iter().any(|bp| bp == function); if engine.is_paused() && hit_entry_breakpoint { + let storage_mutation = if storage_diff.is_empty() { + None + } else { + let mutated_keys = storage_diff.mutated_keys(); + Some(crate::debugger::timeline::StorageMutationMarker { + affected_key_count: mutated_keys.len(), + mutated_keys, + }) + }; + pauses.push(TimelinePausePoint { index: 0, reason: "breakpoint".to_string(), location: None, call_stack: stack_summary.clone(), + storage_mutation, }); } @@ -1910,7 +1921,15 @@ pub fn compare(args: CompareArgs) -> Result<()> { args.ignore_field.clone(), )?; let report = crate::compare::CompareEngine::compare_with_filters(&trace_a, &trace_b, &filters); - let rendered = crate::compare::CompareEngine::render_report(&report); + let rendered = match args.format { + OutputFormat::Json => { + let envelope = crate::output::VersionedOutput::success("compare", &report); + serde_json::to_string_pretty(&envelope).map_err(|e| { + DebuggerError::FileError(format!("Failed to serialize comparison report: {}", e)) + })? + } + OutputFormat::Pretty => crate::compare::CompareEngine::render_report(&report), + }; if let Some(output_path) = &args.output { fs::write(output_path, &rendered).map_err(|e| { diff --git a/src/compare/engine.rs b/src/compare/engine.rs index 7af6686f..c7bda8e0 100644 --- a/src/compare/engine.rs +++ b/src/compare/engine.rs @@ -3,15 +3,17 @@ //! execution flow differences. use super::trace::{BudgetTrace, CallEntry, EventEntry, ExecutionTrace}; +use serde::Serialize; use std::collections::{BTreeMap, BTreeSet}; // ─── Diff types ────────────────────────────────────────────────────── /// Overall comparison report returned by [`CompareEngine::compare`]. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize)] pub struct ComparisonReport { pub label_a: String, pub label_b: String, + pub filters: CompareFiltersReport, pub storage_diff: StorageDiff, pub budget_diff: BudgetDiff, pub return_value_diff: ReturnValueDiff, @@ -19,8 +21,14 @@ pub struct ComparisonReport { pub event_diff: EventDiff, } +#[derive(Debug, Clone, Serialize)] +pub struct CompareFiltersReport { + pub ignore_paths: Vec, + pub ignore_fields: Vec, +} + /// Storage key-level differences. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize)] pub struct StorageDiff { /// Keys present only in trace A pub only_in_a: BTreeMap, @@ -33,7 +41,7 @@ pub struct StorageDiff { } /// Numeric deltas for resource budgets. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize)] pub struct BudgetDiff { pub a: Option, pub b: Option, @@ -43,7 +51,7 @@ pub struct BudgetDiff { } /// Return value comparison. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize)] pub struct ReturnValueDiff { pub a: Option, pub b: Option, @@ -51,7 +59,7 @@ pub struct ReturnValueDiff { } /// Call-sequence comparison. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize)] pub struct FlowDiff { pub a_calls: Vec, pub b_calls: Vec, @@ -63,7 +71,7 @@ pub struct FlowDiff { } /// Event comparison. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize)] pub struct EventDiff { pub a_events: Vec, pub b_events: Vec, @@ -73,7 +81,8 @@ pub struct EventDiff { } /// A single line in a unified-style diff. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +#[serde(tag = "type", content = "value")] pub enum DiffLine { /// Present in both traces at the same position. Same(String), @@ -88,18 +97,22 @@ pub enum DiffLine { pub struct CompareFilters { ignore_paths: Vec>, ignore_fields: BTreeSet, + pub raw_ignore_paths: Vec, + pub raw_ignore_fields: Vec, } impl CompareFilters { pub fn new(ignore_paths: Vec, ignore_fields: Vec) -> crate::Result { let mut parsed_paths = Vec::with_capacity(ignore_paths.len()); - for path in ignore_paths { - parsed_paths.push(Self::parse_path(&path)?); + for path in &ignore_paths { + parsed_paths.push(Self::parse_path(path)?); } Ok(Self { ignore_paths: parsed_paths, - ignore_fields: ignore_fields.into_iter().collect(), + ignore_fields: ignore_fields.iter().cloned().collect(), + raw_ignore_paths: ignore_paths, + raw_ignore_fields: ignore_fields, }) } @@ -170,6 +183,10 @@ impl CompareEngine { ComparisonReport { label_a, label_b, + filters: CompareFiltersReport { + ignore_paths: filters.raw_ignore_paths.clone(), + ignore_fields: filters.raw_ignore_fields.clone(), + }, storage_diff: Self::diff_storage(&trace_a.storage, &trace_b.storage, filters), budget_diff: Self::diff_budget(&trace_a.budget, &trace_b.budget, filters), return_value_diff: Self::diff_return_value( @@ -954,4 +971,20 @@ mod tests { assert!(report.flow_diff.identical); assert_eq!(report.flow_diff.filtered_a_calls, vec!["transfer()"]); } + + #[test] + fn test_report_json_serialization() { + let a = make_trace_a(); + let b = make_trace_b(); + let filters = filters(&["/storage/fee_pool"], &["timestamp"]); + let report = CompareEngine::compare_with_filters(&a, &b, &filters); + + let json = serde_json::to_string(&report).expect("should serialize to JSON"); + assert!(json.contains("\"balance:Bob\"")); + assert!(json.contains("\"/storage/fee_pool\"")); + assert!(json.contains("\"timestamp\"")); + assert!(json.contains("\"only_in_b\"")); + assert!(json.contains("\"ignore_paths\"")); + assert!(json.contains("\"ignore_fields\"")); + } } diff --git a/src/inspector/storage.rs b/src/inspector/storage.rs index 6f5b86bd..45e183ea 100644 --- a/src/inspector/storage.rs +++ b/src/inspector/storage.rs @@ -401,6 +401,13 @@ impl StorageInspector { *self.writes.entry(key.to_string()).or_insert(0) += 1; } + /// Get a sorted list of all keys that have been written to + pub fn mutated_keys(&self) -> Vec { + let mut keys: Vec = self.writes.keys().cloned().collect(); + keys.sort(); + keys + } + /// Analyze access patterns pub fn analyze_access_patterns(&self) -> AccessPatternReport { let mut stats: HashMap = HashMap::new(); @@ -706,6 +713,17 @@ impl StorageDiff { pub fn is_empty(&self) -> bool { self.added.is_empty() && self.modified.is_empty() && self.deleted.is_empty() } + + /// Extract a sorted list of all keys that were mutated (added, modified, or deleted) + pub fn mutated_keys(&self) -> Vec { + let mut keys: Vec = self.added.keys().cloned() + .chain(self.modified.keys().cloned()) + .chain(self.deleted.iter().cloned()) + .collect(); + keys.sort(); + keys.dedup(); + keys + } } /// Statistics for a single storage access key @@ -1152,6 +1170,12 @@ mod tests { &("old".to_string(), "new".to_string()) ); assert!(diff.deleted.contains(&"key_76".to_string())); + + let mutated = diff.mutated_keys(); + assert_eq!(mutated.len(), 75); + assert!(mutated.contains(&"key_101".to_string())); + assert!(mutated.contains(&"key_51".to_string())); + assert!(mutated.contains(&"key_76".to_string())); } #[test] diff --git a/src/runtime/executor.rs b/src/runtime/executor.rs index b8d72c73..5ee40876 100644 --- a/src/runtime/executor.rs +++ b/src/runtime/executor.rs @@ -12,6 +12,7 @@ use crate::inspector::budget::MemorySummary; use crate::output::InvocationReason; use crate::runtime::env::DebugEnv; use crate::runtime::mocking::{MockCallLogEntry, MockContractDispatcher, MockRegistry}; +use crate::debugger::timeline::StorageMutationMarker; use crate::server::protocol::{DynamicTraceEvent, DynamicTraceEventKind}; use crate::utils::arguments::ArgumentParser; use crate::{DebuggerError, Result}; @@ -152,7 +153,7 @@ impl ContractExecutor { // Track storage changes as accesses let storage_after = &record.storage_after; - self.track_storage_changes(&storage_before, storage_after); + let _mutation_marker = self.track_storage_changes(&storage_before, storage_after); // Record completed function call let result_str = display.clone(); @@ -178,15 +179,18 @@ impl ContractExecutor { &mut self, storage_before: &HashMap, storage_after: &HashMap, - ) { + ) -> Option { + let mut mutated_keys = Vec::new(); // Track writes (new or modified entries) for (key, value) in storage_after { if !storage_before.contains_key(key) { // New write self.debug_env.track_storage_write(key, value); + mutated_keys.push(key.clone()); } else if storage_before.get(key) != Some(value) { // Modified write self.debug_env.track_storage_write(key, value); + mutated_keys.push(key.clone()); } }