diff --git a/docs/doc/compare.md b/docs/doc/compare.md index aa5c7c00..e69de29b 100644 --- a/docs/doc/compare.md +++ b/docs/doc/compare.md @@ -1,172 +0,0 @@ -# Comparing Execution Traces - -The `compare` subcommand lets you diff two execution trace JSON files -side-by-side. It is designed for **regression testing** — run your -contract, save the trace, make changes, run again, then compare the two -traces to spot any unintended differences. - -## Quick-start - -```bash -soroban-debug compare examples/trace_a.json examples/trace_b.json -``` - -Save the report to a file instead of stdout: - -```bash -soroban-debug compare examples/trace_a.json examples/trace_b.json --output report.txt -``` - -Ignore noisy fields or paths directly in the compare step: - -```bash -soroban-debug compare baseline.json new.json \ - --ignore-field timestamp \ - --ignore-field seq \ - --ignore-path /storage/ledger_seq \ - --ignore-path /return_value/meta/debug -``` - -## What is compared? - -| Dimension | Details | -|-------------------|-------------------------------------------------------| -| **Storage** | Keys added, removed, and modified with old/new values | -| **Budget** | CPU instructions and memory deltas (absolute + %) | -| **Return values** | Equality check with full value display | -| **Execution flow**| LCS-based unified diff of the call sequence | -| **Events** | Side-by-side comparison of emitted events | - -## Ignore filters - -- `--ignore-field ` removes that object field name everywhere in the trace before diffing. -- `--ignore-path ` removes one slash-delimited subtree before diffing. The match applies to the exact path and everything below it. -- Paths are rooted at the trace object. Common examples: - - `/storage/fee_pool` - - `/storage/balance:Alice/timestamp` - - `/return_value/meta/timestamp` - - `/events/0/data` - -These filters affect storage, budget, return values, call sequences, and events in the rendered report. - -## Trace JSON format - -A trace file is a JSON object with the following fields (all optional -except where noted): - -```json -{ - "label": "Human-readable name for the trace", - "contract": "token.wasm", - "function": "transfer", - "args": "[\"Alice\", \"Bob\", 100]", - "storage": { - "balance:Alice": 900, - "balance:Bob": 100, - "total_supply": 1000 - }, - "budget": { - "cpu_instructions": 45000, - "memory_bytes": 15360, - "cpu_limit": 100000, - "memory_limit": 40960 - }, - "return_value": { "status": "ok" }, - "call_sequence": [ - { "function": "transfer", "depth": 0 }, - { "function": "get_balance", "args": "Alice", "depth": 1 }, - { "function": "set_balance", "args": "Alice, 900", "depth": 1 } - ], - "events": [ - { - "contract_id": "CA7QYN...", - "topics": ["transfer"], - "data": "Alice→Bob 100" - } - ] -} -``` - -### Field reference - -| Field | Type | Description | -|------------------|-----------------|-------------------------------------------------| -| `label` | `string?` | Friendly name shown in the report header | -| `contract` | `string?` | Contract WASM path or ID | -| `function` | `string?` | Invoked function name | -| `args` | `string?` | Function arguments (JSON-encoded) | -| `storage` | `object` | Post-execution storage key→value map | -| `budget` | `object?` | CPU and memory usage | -| `return_value` | `any?` | Return value (arbitrary JSON) | -| `call_sequence` | `array` | Ordered list of function calls | -| `events` | `array` | Events emitted during execution | - -## Regression testing workflow - -1. **Capture baseline trace** — run your contract and save the execution - output as `baseline.json`. - -2. **Make contract changes** — e.g., optimize gas usage, add fee logic, etc. - -3. **Capture new trace** — run the modified contract and save as `new.json`. - -4. **Compare** — - ```bash - soroban-debug compare baseline.json new.json - ``` - -5. **Review the report** — look for: - - Unexpected storage modifications (regressions) - - Budget increases (performance regressions) - - Changed return values (behavioural regressions) - - New or missing function calls in the execution flow - -### Example: Detecting a fee regression - -Suppose `v1.0` of your token contract transfers the full amount, and -`v1.1` introduces a fee. The compare output will clearly show: - -``` -───────────────── Storage Changes ───────────────── - - Keys only in B (1): - + fee_pool = 5 - - Modified keys (1): - ~ balance:Alice - A: 900 - B: 895 - -───────────────── Budget Usage ──────────────────── - - A B Delta - CPU instructions 45000 38000 -7000 - - CPU change: -15.56% - Memory change: -8.85% - -───────────────── Return Values ─────────────────── - - A: {"status":"ok"} - B: {"fee_charged":5,"status":"ok"} - -───────────────── Execution Flow ────────────────── - - Unified diff (- = only in A, + = only in B): - - transfer() - + check_allowance(Alice) - get_balance(Alice) - + compute_fee(100) - ... -``` - -## Tips - -- Keep trace files in version control alongside your contract code - so you can compare across Git commits. -- Use `--output` to save the report, then `diff` two reports over time. -- Use `--ignore-field` for volatile metadata such as timestamps, nonces, or sequence numbers. -- Use `--ignore-path` for selected storage keys or specific nested JSON branches that are expected to vary. -- Combine with CI: generate traces in your test suite and run - `soroban-debug compare` as a CI step to catch regressions automatically. diff --git a/src/cli/args.rs b/src/cli/args.rs index a3d88b7e..898eb150 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -1075,6 +1075,10 @@ pub struct CompareArgs { /// Repeatable. Useful for timestamps, sequence numbers, and similar metadata. #[arg(long, value_name = "FIELD")] pub ignore_field: Vec, + + /// Output format for the comparison report (pretty or json) + #[arg(long, value_enum, default_value_t = OutputFormat::Pretty)] + pub format: OutputFormat, } /// Arguments for the TUI dashboard subcommand diff --git a/src/cli/commands.rs b/src/cli/commands.rs index f8b4714b..e0c02054 100644 --- a/src/cli/commands.rs +++ b/src/cli/commands.rs @@ -2095,7 +2095,15 @@ pub fn compare(args: CompareArgs) -> Result<()> { args.ignore_field.clone(), )?; let report = crate::compare::CompareEngine::compare_with_filters(&trace_a, &trace_b, &filters); - let rendered = crate::compare::CompareEngine::render_report(&report); + let rendered = match args.format { + OutputFormat::Json => { + let envelope = crate::output::VersionedOutput::success("compare", &report); + serde_json::to_string_pretty(&envelope).map_err(|e| { + DebuggerError::FileError(format!("Failed to serialize comparison report: {}", e)) + })? + } + OutputFormat::Pretty => crate::compare::CompareEngine::render_report(&report), + }; if let Some(output_path) = &args.output { fs::write(output_path, &rendered).map_err(|e| { diff --git a/src/compare/engine.rs b/src/compare/engine.rs index b345dff9..6c677bae 100644 --- a/src/compare/engine.rs +++ b/src/compare/engine.rs @@ -3,6 +3,7 @@ //! execution flow differences. use super::trace::{BudgetTrace, CallEntry, EventEntry, ExecutionTrace}; +use serde::Serialize; use std::collections::{BTreeMap, BTreeSet}; // ─── Diff types ────────────────────────────────────────────────────── @@ -12,6 +13,7 @@ use std::collections::{BTreeMap, BTreeSet}; pub struct ComparisonReport { pub label_a: String, pub label_b: String, + pub filters: CompareFiltersReport, pub storage_diff: StorageDiff, pub budget_diff: BudgetDiff, pub return_value_diff: ReturnValueDiff, @@ -19,6 +21,12 @@ pub struct ComparisonReport { pub event_diff: EventDiff, } +#[derive(Debug, Clone, Serialize)] +pub struct CompareFiltersReport { + pub ignore_paths: Vec, + pub ignore_fields: Vec, +} + /// Storage key-level differences. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] pub struct StorageDiff { @@ -89,18 +97,22 @@ pub enum DiffLine { pub struct CompareFilters { ignore_paths: Vec>, ignore_fields: BTreeSet, + pub raw_ignore_paths: Vec, + pub raw_ignore_fields: Vec, } impl CompareFilters { pub fn new(ignore_paths: Vec, ignore_fields: Vec) -> crate::Result { let mut parsed_paths = Vec::with_capacity(ignore_paths.len()); - for path in ignore_paths { - parsed_paths.push(Self::parse_path(&path)?); + for path in &ignore_paths { + parsed_paths.push(Self::parse_path(path)?); } Ok(Self { ignore_paths: parsed_paths, - ignore_fields: ignore_fields.into_iter().collect(), + ignore_fields: ignore_fields.iter().cloned().collect(), + raw_ignore_paths: ignore_paths, + raw_ignore_fields: ignore_fields, }) } @@ -171,6 +183,10 @@ impl CompareEngine { ComparisonReport { label_a, label_b, + filters: CompareFiltersReport { + ignore_paths: filters.raw_ignore_paths.clone(), + ignore_fields: filters.raw_ignore_fields.clone(), + }, storage_diff: Self::diff_storage(&trace_a.storage, &trace_b.storage, filters), budget_diff: Self::diff_budget(&trace_a.budget, &trace_b.budget, filters), return_value_diff: Self::diff_return_value( @@ -955,4 +971,20 @@ mod tests { assert!(report.flow_diff.identical); assert_eq!(report.flow_diff.filtered_a_calls, vec!["transfer()"]); } + + #[test] + fn test_report_json_serialization() { + let a = make_trace_a(); + let b = make_trace_b(); + let filters = filters(&["/storage/fee_pool"], &["timestamp"]); + let report = CompareEngine::compare_with_filters(&a, &b, &filters); + + let json = serde_json::to_string(&report).expect("should serialize to JSON"); + assert!(json.contains("\"balance:Bob\"")); + assert!(json.contains("\"/storage/fee_pool\"")); + assert!(json.contains("\"timestamp\"")); + assert!(json.contains("\"only_in_b\"")); + assert!(json.contains("\"ignore_paths\"")); + assert!(json.contains("\"ignore_fields\"")); + } }