diff --git a/docs/superpowers/plans/2026-06-05-reporting.md b/docs/superpowers/plans/2026-06-05-reporting.md new file mode 100644 index 0000000..9fc5359 --- /dev/null +++ b/docs/superpowers/plans/2026-06-05-reporting.md @@ -0,0 +1,493 @@ +# Reporting Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a `Reporter` class to `pcapprocessor` that reads metric CSVs and generates one PNG/SVG figure per requested metric, with protocols overlaid and confidence interval error bars. + +**Architecture:** `Reporter` follows the existing stateful-class pattern. `_load_csv` reads one tab-separated CSV into column arrays. `_plot_metric` draws one figure for all protocols and saves it. `plot()` orchestrates both. A `__main__` block wires up the CLI. + +**Tech Stack:** Python 3.10+, matplotlib, seaborn (both already in venv/requirements). No new dependencies. + +--- + +## File Structure + +| File | Action | Responsibility | +|---|---|---| +| `pcapprocessor/report.py` | Create | `Reporter` class + `__main__` CLI | +| `tests/test_report.py` | Create | Unit tests (headless Agg backend) | +| `pcapprocessor/__init__.py` | Modify | Export `Reporter` | + +--- + +### Task 1: `_load_csv` — read one CSV into column arrays + +**Files:** +- Create: `pcapprocessor/report.py` +- Create: `tests/test_report.py` + +- [ ] **Step 1: Write the failing test** + +```python +# tests/test_report.py +import csv +import os + +import matplotlib +matplotlib.use("Agg") + +import numpy as np +import pytest + +from pcapprocessor.report import Reporter + + +def _write_csv(path, headers, rows): + """Write tab-separated rows to path.""" + with open(path, "w", newline="") as f: + writer = csv.writer(f, delimiter="\t") + writer.writerow(headers) + writer.writerows(rows) + + +def test_load_csv_returns_column_arrays(tmp_path): + csv_path = tmp_path / "tcp0.csv" + _write_csv(csv_path, + ["x-scale", "avg_throughput", "confInt_throughput"], + [[1.0, 10.5, 0.4], [2.0, 20.1, 0.6]]) + r = Reporter(csvs=[str(csv_path)], metrics=["throughput"], + output_dir=str(tmp_path / "plots")) + data = r._load_csv(str(csv_path)) + assert list(data.keys()) == ["x-scale", "avg_throughput", "confInt_throughput"] + np.testing.assert_array_almost_equal(data["avg_throughput"], [10.5, 20.1]) + + +def test_load_csv_raises_for_missing_file(tmp_path): + r = Reporter(csvs=[], metrics=[], output_dir=str(tmp_path / "plots")) + with pytest.raises(FileNotFoundError): + r._load_csv(str(tmp_path / "nonexistent.csv")) +``` + +- [ ] **Step 2: Run to confirm it fails** + +```bash +pytest tests/test_report.py::test_load_csv_returns_column_arrays tests/test_report.py::test_load_csv_raises_for_missing_file -v +``` + +Expected: `ERROR` — `ModuleNotFoundError: No module named 'pcapprocessor.report'` + +- [ ] **Step 3: Create `pcapprocessor/report.py` with `Reporter.__init__` and `_load_csv`** + +```python +# pcapprocessor/report.py +import csv +import os + +import numpy as np + + +class Reporter: + def __init__( + self, + csvs: list, + metrics: list, + output_dir: str = "plots", + fmt: str = "png", + ): + self.csvs = csvs + self.metrics = metrics + self.output_dir = output_dir + self.fmt = fmt + + def plot(self) -> list: + raise NotImplementedError + + def _load_csv(self, path: str) -> dict: + with open(path, newline="") as f: + reader = csv.reader(f, delimiter="\t") + headers = next(reader) + rows = list(reader) + data = np.array(rows, dtype=float) + return {headers[i]: data[:, i] for i in range(len(headers))} + + def _plot_metric(self, metric: str, protocol_data: dict) -> str: + raise NotImplementedError +``` + +Note: `open()` raises `FileNotFoundError` natively — no explicit check needed. + +- [ ] **Step 4: Run tests — expect both to pass** + +```bash +pytest tests/test_report.py::test_load_csv_returns_column_arrays tests/test_report.py::test_load_csv_raises_for_missing_file -v +``` + +Expected: `2 passed` + +- [ ] **Step 5: Commit** + +```bash +git add pcapprocessor/report.py tests/test_report.py +git commit -m "feat: add Reporter._load_csv with tab-separated CSV parsing" +``` + +--- + +### Task 2: `_plot_metric` — draw one figure for all protocols + +**Files:** +- Modify: `pcapprocessor/report.py` +- Modify: `tests/test_report.py` + +- [ ] **Step 1: Write the failing tests** + +Add to `tests/test_report.py` (after the existing `_write_csv` helper and existing tests): + +```python +def _make_protocol_data(tmp_path, name="tcp0", + x=(1.0, 2.0), avg=(10.0, 20.0), ci=(0.5, 0.8)): + csv_path = tmp_path / f"{name}.csv" + _write_csv(csv_path, + ["x-scale", "avg_throughput", "confInt_throughput"], + [[x[0], avg[0], ci[0]], [x[1], avg[1], ci[1]]]) + r = Reporter(csvs=[str(csv_path)], metrics=["throughput"], + output_dir=str(tmp_path / "plots")) + return str(csv_path), {name: r._load_csv(str(csv_path))} + + +def test_plot_metric_creates_file(tmp_path): + _, protocol_data = _make_protocol_data(tmp_path) + out = str(tmp_path / "plots") + os.makedirs(out, exist_ok=True) + r = Reporter(csvs=[], metrics=[], output_dir=out) + path = r._plot_metric("throughput", protocol_data) + assert os.path.exists(path) + assert path.endswith(".png") + + +def test_plot_metric_raises_for_unknown_metric(tmp_path): + _, protocol_data = _make_protocol_data(tmp_path) + out = str(tmp_path / "plots") + os.makedirs(out, exist_ok=True) + r = Reporter(csvs=[], metrics=[], output_dir=out) + with pytest.raises(ValueError, match="delay"): + r._plot_metric("delay", protocol_data) + + +def test_plot_metric_filename_uses_metric_name(tmp_path): + _, protocol_data = _make_protocol_data(tmp_path) + out = str(tmp_path / "plots") + os.makedirs(out, exist_ok=True) + r = Reporter(csvs=[], metrics=[], output_dir=out, fmt="svg") + path = r._plot_metric("throughput", protocol_data) + assert path == os.path.join(out, "throughput.svg") +``` + +- [ ] **Step 2: Run to confirm they fail** + +```bash +pytest tests/test_report.py::test_plot_metric_creates_file tests/test_report.py::test_plot_metric_raises_for_unknown_metric tests/test_report.py::test_plot_metric_filename_uses_metric_name -v +``` + +Expected: `FAILED` — `NotImplementedError` + +- [ ] **Step 3: Implement `_plot_metric` in `pcapprocessor/report.py`** + +Replace the `_plot_metric` stub: + +```python + def _plot_metric(self, metric: str, protocol_data: dict) -> str: + import matplotlib.pyplot as plt + import seaborn as sns + + avg_col = f"avg_{metric}" + ci_col = f"confInt_{metric}" + + for proto, cols in protocol_data.items(): + if avg_col not in cols or ci_col not in cols: + available = [k[4:] for k in cols if k.startswith("avg_")] + raise ValueError( + f"unknown metric {metric!r} for protocol {proto!r}. " + f"Available: {available}" + ) + + sns.set_theme(style="whitegrid") + palette = sns.color_palette() + fig, ax = plt.subplots() + + for idx, (proto, cols) in enumerate(protocol_data.items()): + ax.errorbar( + cols["x-scale"], + cols[avg_col], + yerr=cols[ci_col] / 2, + label=proto, + capsize=4, + color=palette[idx % len(palette)], + marker="o", + ) + + ax.set_xlabel("x-scale") + ax.set_ylabel(metric) + ax.set_title(metric) + ax.legend() + + out_path = os.path.join(self.output_dir, f"{metric}.{self.fmt}") + fig.savefig(out_path, bbox_inches="tight") + plt.close(fig) + return out_path +``` + +- [ ] **Step 4: Run all tests so far — expect 5 to pass** + +```bash +pytest tests/test_report.py -v +``` + +Expected: `5 passed` + +- [ ] **Step 5: Commit** + +```bash +git add pcapprocessor/report.py tests/test_report.py +git commit -m "feat: implement Reporter._plot_metric with seaborn styling and CI error bars" +``` + +--- + +### Task 3: `plot()`, CLI, and `__init__.py` export + +**Files:** +- Modify: `pcapprocessor/report.py` +- Modify: `pcapprocessor/__init__.py` +- Modify: `tests/test_report.py` + +- [ ] **Step 1: Write the failing tests** + +Add to `tests/test_report.py`: + +```python +def test_plot_creates_one_file_per_metric(tmp_path): + for name in ("tcp0.csv", "udp0.csv"): + _write_csv( + tmp_path / name, + ["x-scale", "avg_throughput", "confInt_throughput", + "avg_delay", "confInt_delay"], + [[1.0, 10.0, 0.5, 5.0, 0.1], [2.0, 20.0, 0.8, 4.5, 0.2]], + ) + out = str(tmp_path / "plots") + r = Reporter( + csvs=[str(tmp_path / "tcp0.csv"), str(tmp_path / "udp0.csv")], + metrics=["throughput", "delay"], + output_dir=out, + ) + paths = r.plot() + assert len(paths) == 2 + assert all(os.path.exists(p) for p in paths) + assert any("throughput" in p for p in paths) + assert any("delay" in p for p in paths) + + +def test_plot_creates_output_dir(tmp_path): + _write_csv( + tmp_path / "tcp0.csv", + ["x-scale", "avg_throughput", "confInt_throughput"], + [[1.0, 10.0, 0.5]], + ) + out = str(tmp_path / "new_dir" / "plots") + r = Reporter(csvs=[str(tmp_path / "tcp0.csv")], + metrics=["throughput"], output_dir=out) + r.plot() + assert os.path.isdir(out) + + +def test_reporter_importable_from_package(): + from pcapprocessor import Reporter as R + assert R is not None + + +def test_cli_exits_zero_on_success(tmp_path): + import subprocess + import sys + _write_csv( + tmp_path / "tcp0.csv", + ["x-scale", "avg_throughput", "confInt_throughput"], + [[10.0, 100.0, 5.0], [20.0, 200.0, 8.0]], + ) + result = subprocess.run( + [sys.executable, "-m", "pcapprocessor.report", + "--csvs", str(tmp_path / "tcp0.csv"), + "--metrics", "throughput", + "--output", str(tmp_path / "plots")], + capture_output=True, text=True, + ) + assert result.returncode == 0 + assert "throughput.png" in result.stdout + + +def test_cli_exits_nonzero_on_missing_metric(tmp_path): + import subprocess + import sys + _write_csv( + tmp_path / "tcp0.csv", + ["x-scale", "avg_throughput", "confInt_throughput"], + [[10.0, 100.0, 5.0]], + ) + result = subprocess.run( + [sys.executable, "-m", "pcapprocessor.report", + "--csvs", str(tmp_path / "tcp0.csv"), + "--metrics", "nonexistent", + "--output", str(tmp_path / "plots")], + capture_output=True, text=True, + ) + assert result.returncode == 1 + assert "Error" in result.stderr +``` + +- [ ] **Step 2: Run to confirm they fail** + +```bash +pytest tests/test_report.py::test_plot_creates_one_file_per_metric tests/test_report.py::test_plot_creates_output_dir tests/test_report.py::test_reporter_importable_from_package tests/test_report.py::test_cli_exits_zero_on_success tests/test_report.py::test_cli_exits_nonzero_on_missing_metric -v +``` + +Expected: `FAILED` — `NotImplementedError` (plot), `ImportError` (Reporter not in package) + +- [ ] **Step 3: Implement `plot()` in `pcapprocessor/report.py`** + +Replace the `plot` stub: + +```python + def plot(self) -> list: + os.makedirs(self.output_dir, exist_ok=True) + protocol_data = { + os.path.splitext(os.path.basename(path))[0]: self._load_csv(path) + for path in self.csvs + } + return [self._plot_metric(metric, protocol_data) for metric in self.metrics] +``` + +- [ ] **Step 4: Add `__main__` CLI block to the bottom of `pcapprocessor/report.py`** + +```python +if __name__ == "__main__": + import argparse + import sys + + def _main() -> int: + parser = argparse.ArgumentParser( + description="Generate metric figures from pcapprocessor CSV output." + ) + parser.add_argument("--csvs", nargs="+", required=True, + help="Paths to metric CSV files (one per protocol)") + parser.add_argument("--metrics", nargs="+", required=True, + help="Metric short names to plot (e.g. throughput delay)") + parser.add_argument("--output", default="plots", + help="Output directory (default: plots)") + parser.add_argument("--format", choices=["png", "svg"], default="png", + dest="fmt", help="Output format (default: png)") + args = parser.parse_args() + + try: + paths = Reporter( + csvs=args.csvs, + metrics=args.metrics, + output_dir=args.output, + fmt=args.fmt, + ).plot() + for p in paths: + print(p) + return 0 + except (FileNotFoundError, ValueError) as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + + sys.exit(_main()) +``` + +- [ ] **Step 5: Export `Reporter` from `pcapprocessor/__init__.py`** + +Current content of `pcapprocessor/__init__.py`: +```python +from pcapprocessor.processor import PcapProcessor +from pcapprocessor.stats import ConfidenceInterval, MetricStats, XScaleArray +from pcapprocessor.simulation import QueueSizeCalculator, WafCommandBuilder +from pcapprocessor.trace import TraceProcessor +from pcapprocessor.metrics import MetricsWriter +from pcapprocessor.runner import BfsRunner, SimulationRunner + +__all__ = [ + "PcapProcessor", + "ConfidenceInterval", + "MetricStats", + "XScaleArray", + "QueueSizeCalculator", + "WafCommandBuilder", + "TraceProcessor", + "MetricsWriter", + "BfsRunner", + "SimulationRunner", +] +``` + +Replace with: +```python +from pcapprocessor.processor import PcapProcessor +from pcapprocessor.stats import ConfidenceInterval, MetricStats, XScaleArray +from pcapprocessor.simulation import QueueSizeCalculator, WafCommandBuilder +from pcapprocessor.trace import TraceProcessor +from pcapprocessor.metrics import MetricsWriter +from pcapprocessor.runner import BfsRunner, SimulationRunner +from pcapprocessor.report import Reporter + +__all__ = [ + "PcapProcessor", + "ConfidenceInterval", + "MetricStats", + "XScaleArray", + "QueueSizeCalculator", + "WafCommandBuilder", + "TraceProcessor", + "MetricsWriter", + "BfsRunner", + "SimulationRunner", + "Reporter", +] +``` + +- [ ] **Step 6: Run the full test suite** + +```bash +pytest tests/test_report.py -v +``` + +Expected: `10 passed` + +- [ ] **Step 7: Smoke-test the CLI end-to-end** + +```bash +printf "x-scale\tavg_throughput\tconfInt_throughput\tavg_delay\tconfInt_delay\n10\t5.2\t0.3\t12.1\t0.5\n20\t9.8\t0.4\t10.3\t0.4\n30\t14.1\t0.6\t9.7\t0.3\n" > /tmp/tcp0.csv + +python -m pcapprocessor.report \ + --csvs /tmp/tcp0.csv \ + --metrics throughput delay \ + --output /tmp/report_out \ + --format png + +ls -lh /tmp/report_out/ +``` + +Expected: two files — `throughput.png` and `delay.png` — both non-empty. + +- [ ] **Step 8: Run the full project test suite to check for regressions** + +```bash +pytest --cov=pcapprocessor --cov-report=term-missing -q +``` + +Expected: all tests pass, coverage ≥ 90%. + +- [ ] **Step 9: Commit** + +```bash +git add pcapprocessor/report.py pcapprocessor/__init__.py tests/test_report.py +git commit -m "feat: add Reporter.plot(), CLI entry point, and package export" +``` diff --git a/docs/superpowers/specs/2026-06-05-reporting-design.md b/docs/superpowers/specs/2026-06-05-reporting-design.md new file mode 100644 index 0000000..a2f25d2 --- /dev/null +++ b/docs/superpowers/specs/2026-06-05-reporting-design.md @@ -0,0 +1,149 @@ +# Reporting Design + +**Date:** 2026-06-05 +**Scope:** Static figure generation from pcapprocessor metric CSVs + +--- + +## Goal + +Generate publication-quality PNG or SVG figures from the tab-separated CSV files produced by `MetricsWriter`. One figure per metric, all protocols overlaid on the same axes, confidence interval error bars. + +--- + +## Architecture + +One new file: `pcapprocessor/report.py`. Follows the existing stateful-class pattern (inputs in `__init__`, work in a named method). + +``` +pcapprocessor/ +└── report.py ← Reporter class + CLI entry point +tests/ +└── test_report.py ← unit tests +``` + +No new dependencies. Uses `matplotlib` and `seaborn`, both already present in the venv. + +--- + +## Class Design + +```python +class Reporter: + def __init__( + self, + csvs: list[str], + metrics: list[str], + output_dir: str = "plots", + fmt: str = "png", + ): ... + + def plot(self) -> list[str]: + """Generate one figure per metric. Returns list of written file paths.""" + + def _load_csv(self, path: str) -> dict: + """Read one tab-separated CSV. Returns dict of column_name → np.ndarray.""" + + def _plot_metric(self, metric: str, protocol_data: dict[str, dict]) -> str: + """Draw one figure (all protocols overlaid), save, return output path.""" +``` + +### `plot()` + +1. Creates `output_dir` if it does not exist. +2. Calls `_load_csv` for each path in `self.csvs`. +3. Derives protocol name from the CSV filename stem (e.g. `results/tcp0.csv` → `"tcp0"`). +4. For each metric in `self.metrics`, calls `_plot_metric` and collects the returned path. +5. Returns the list of written paths. + +### `_load_csv(path)` + +- Reads the tab-separated file with `numpy.genfromtxt` (or `csv` module) using the first row as column headers. +- Returns `{column_name: np.ndarray}`. +- Raises `FileNotFoundError` if the path does not exist. + +### `_plot_metric(metric, protocol_data)` + +`protocol_data` is `{protocol_name: column_dict}` for all loaded CSVs. + +- Validates that `avg_{metric}` and `confInt_{metric}` exist in every protocol's column dict. Raises `ValueError` if not, naming the missing metric and the CSV it came from. +- Calls `seaborn.set_theme(style="whitegrid")`. +- Plots one line per protocol: x = `x-scale` column, y = `avg_{metric}`, `yerr = confInt_{metric} / 2` (half-width, since `confInt` stores the full CI width from `2 × t × SEM`). +- Labels: x-axis = `"x-scale"`, y-axis = metric name, legend = protocol names. +- Saves to `{output_dir}/{metric}.{fmt}` with `bbox_inches="tight"`. +- Closes the figure after saving to free memory. +- Returns the saved path. + +--- + +## CLI + +Runnable as a module: + +```bash +python -m pcapprocessor.report \ + --csvs results/tcp0.csv results/udp0.csv \ + --metrics throughput utilization delay \ + --output plots/ \ + --format png +``` + +| Flag | Required | Default | Description | +|---|---|---|---| +| `--csvs` | yes | — | One or more CSV paths | +| `--metrics` | yes | — | Metric short names (`throughput`, `delay`, …) | +| `--output` | no | `./plots` | Output directory (created if missing) | +| `--format` | no | `png` | `png` or `svg` | + +On success: prints each written path to stdout. +On error: exits with code 1 and a descriptive message (unknown metric, missing column, missing file). + +--- + +## Metric Name Resolution + +User-facing short names map to CSV columns as follows: + +| Short name | avg column | confInt column | +|---|---|---| +| `throughput` | `avg_throughput` | `confInt_throughput` | +| `delay` | `avg_delay` | `confInt_delay` | +| `utilization` | `avg_utilization` | `confInt_utilization` | +| *(any metric)* | `avg_{metric}` | `confInt_{metric}` | + +Valid short names are any string that appears between `avg_` and the end of a column header. The reporter does not hardcode the list — it validates against actual column names present in the CSV. + +--- + +## Figure Style + +- `seaborn.set_theme(style="whitegrid")` +- One line per protocol, colours from seaborn's default palette +- Error bars: `plt.errorbar` with `yerr=confInt/2`, `capsize=4` +- Legend shows protocol names +- Title: metric short name +- Output: `{output_dir}/{metric}.{fmt}`, `bbox_inches="tight"` + +--- + +## Testing + +`tests/test_report.py` covers: + +- `_load_csv` returns correct column arrays from a temp CSV +- `_load_csv` raises `FileNotFoundError` for missing file +- `_plot_metric` raises `ValueError` for unknown metric name +- `plot()` creates the output directory and writes one file per metric +- `plot()` returns the correct list of paths +- CLI (`__main__`) exits non-zero on missing `--csvs` / `--metrics` + +Figures are generated with `matplotlib.use("Agg")` (non-interactive backend) so tests run headlessly without a display. + +--- + +## Out of Scope + +- Interactive dashboards +- Automatic metric selection (user always specifies `--metrics`) +- Multi-scenario comparison across different config runs +- PDF or PowerPoint export diff --git a/pcapprocessor/__init__.py b/pcapprocessor/__init__.py index 3c97333..78ac4e5 100644 --- a/pcapprocessor/__init__.py +++ b/pcapprocessor/__init__.py @@ -4,6 +4,7 @@ from pcapprocessor.trace import TraceProcessor from pcapprocessor.metrics import MetricsWriter from pcapprocessor.runner import BfsRunner, SimulationRunner +from pcapprocessor.report import Reporter __all__ = [ "PcapProcessor", @@ -16,4 +17,5 @@ "MetricsWriter", "BfsRunner", "SimulationRunner", + "Reporter", ] diff --git a/pcapprocessor/report.py b/pcapprocessor/report.py new file mode 100644 index 0000000..2370be0 --- /dev/null +++ b/pcapprocessor/report.py @@ -0,0 +1,112 @@ +import csv +import os + +import numpy as np + + +class Reporter: + def __init__( + self, + csvs: list, + metrics: list, + output_dir: str = "plots", + fmt: str = "png", + ): + self.csvs = csvs + self.metrics = metrics + self.output_dir = output_dir + self.fmt = fmt + + def plot(self) -> list: + os.makedirs(self.output_dir, exist_ok=True) + protocol_data = { + os.path.splitext(os.path.basename(path))[0]: self._load_csv(path) + for path in self.csvs + } + return [self._plot_metric(metric, protocol_data) for metric in self.metrics] + + def _load_csv(self, path: str) -> dict: + with open(path, newline="") as f: + reader = csv.reader(f, delimiter="\t") + headers = next(reader) + rows = list(reader) + data = np.array(rows, dtype=float) + return {headers[i]: data[:, i] for i in range(len(headers))} + + def _plot_metric(self, metric: str, protocol_data: dict) -> str: + import matplotlib.pyplot as plt + import seaborn as sns + + # Validate that metric columns exist for all protocols + for protocol_name, cols in protocol_data.items(): + if f"avg_{metric}" not in cols or f"confInt_{metric}" not in cols: + raise ValueError(f"unknown metric {metric!r} for protocol {protocol_name}") + + # Set seaborn theme + sns.set_theme(style="whitegrid") + + # Create figure + fig, ax = plt.subplots() + + # Get color palette + palette = sns.color_palette() + + # Plot each protocol + for idx, (proto, cols) in enumerate(protocol_data.items()): + ax.errorbar( + cols["x-scale"], + cols[f"avg_{metric}"], + yerr=cols[f"confInt_{metric}"] / 2, + label=proto, + capsize=4, + color=palette[idx % len(palette)], + marker="o" + ) + + # Set labels + ax.set_xlabel("x-scale") + ax.set_ylabel(metric) + ax.set_title(metric) + ax.legend() + + # Save figure + out_path = os.path.join(self.output_dir, f"{metric}.{self.fmt}") + try: + fig.savefig(out_path, bbox_inches="tight") + finally: + plt.close(fig) + return out_path + +if __name__ == "__main__": + import argparse + import sys + + def _main() -> int: + parser = argparse.ArgumentParser( + description="Generate metric figures from pcapprocessor CSV output." + ) + parser.add_argument("--csvs", nargs="+", required=True, + help="Paths to metric CSV files (one per protocol)") + parser.add_argument("--metrics", nargs="+", required=True, + help="Metric short names to plot (e.g. throughput delay)") + parser.add_argument("--output", default="plots", + help="Output directory (default: plots)") + parser.add_argument("--format", choices=["png", "svg"], default="png", + dest="fmt", help="Output format (default: png)") + args = parser.parse_args() + + try: + paths = Reporter( + csvs=args.csvs, + metrics=args.metrics, + output_dir=args.output, + fmt=args.fmt, + ).plot() + for p in paths: + print(p) + return 0 + except (FileNotFoundError, ValueError) as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + + sys.exit(_main()) diff --git a/pcapprocessor/trace.py b/pcapprocessor/trace.py index 7e2f301..99c3576 100644 --- a/pcapprocessor/trace.py +++ b/pcapprocessor/trace.py @@ -1,10 +1,43 @@ import re -import shlex -import datetime +from collections import defaultdict import numpy as np - -from pcapprocessor import exe_comm +import pyshark + + +class _FlowAccumulator: + """Accumulates per-packet stats for one TCP flow direction.""" + + def __init__(self): + self.tx_packets = 0 + self.unique_bytes = 0 + self.rexmt_packets = 0 + self.first_ts = None + self.last_ts = None + self.rtt_samples = [] + + def add_packet(self, ts: float, payload_len: int, is_retrans: bool, rtt_ms) -> None: + if self.first_ts is None: + self.first_ts = ts + self.last_ts = ts + if payload_len > 0: + self.tx_packets += 1 + if is_retrans: + self.rexmt_packets += 1 + else: + self.unique_bytes += payload_len + if rtt_ms is not None: + self.rtt_samples.append(rtt_ms) + + @property + def tx_time(self) -> float: + if self.first_ts is None: + return 0.0 + return max(self.last_ts - self.first_ts, 0.0) + + @property + def avg_rtt_ms(self) -> float: + return float(np.mean(self.rtt_samples)) if self.rtt_samples else 0.0 class TraceProcessor: @@ -30,84 +63,66 @@ def process(self) -> list: fact_by = self._unit_factor() bn_speed = self._bottleneck_speed(fact_by) - print("Processing ascii trace output") pkt_size = int(self.config.get(self.scenario, "pktSize")) - with open(self.ascii_trace_file, "r") as fl: - lines = list(fl) - axis_y1 = [int(line.strip().split(",")[1]) for line in lines] + with open(self.ascii_trace_file) as fl: + axis_y1 = [int(ln.strip().split(",")[1]) for ln in fl] a = np.array(axis_y1) - queue_mean = a.mean() - queue_variance = a.var() - - trace_cmd = "tcptrace -l -r -n -W --csv " + self.pcap_file - print("Executing tcptrace command. it may take few seconds") - result = exe_comm.exe_comm(shlex.split(trace_cmd)) - print("Processing trace output") - - pcap_lines = result.split("\n") - regex_con = re.compile(r"#([0-9]*) TCP connection traced:") - matches = [ - pcap_lines.index(ln) - for ln in pcap_lines - if re.match(regex_con, ln) - ] + queue_mean = float(a.mean()) + queue_variance = float(a.var()) - if not matches: + flow = self._dominant_flow() + if flow is None: raise ValueError("No TCP connections found in pcap file") - connections = [ - pcap_lines[matches[j]: matches[j + 1]] - for j in range(len(matches) - 1) + tx_time = flow.tx_time + throughput = flow.unique_bytes / tx_time if tx_time > 0 else 0.0 + utilization = throughput * 100.0 / bn_speed if bn_speed > 0 else 0.0 + + return [ + flow.tx_packets, + self.PACKET_OVERHEAD * flow.tx_packets, + round(throughput / fact_by, 3), + round(flow.avg_rtt_ms / 2, 3), + round(throughput * 8 / fact_by, 3), + flow.unique_bytes, + flow.rexmt_packets, + round(utilization, 3), + round(queue_mean, 3), + round(queue_variance, 3), + round(queue_mean * 100.0 / (self.buf_size / pkt_size), 3), + round(tx_time * 1000, 3), ] - connections.append(pcap_lines[matches[-1]:]) - - result_str = "\n" - for i, item in enumerate(connections): - flow_cmp_time = 0 - try: - time_stamp = pcap_lines[matches[i] - 2].split()[-1] - t = datetime.datetime.strptime(time_stamp, "%H:%M:%S.%f") - flow_cmp_time = ( - t.time().hour * 3600 + t.time().minute * 60 + t.time().second - ) * 1000 + t.time().microsecond / 1000 - except Exception: - print("Couldn't parse the flow completion time") - - labels = item[1].split(",") - values = item[3].split(",") - - conn_suffix = "a2b" - if int(values[labels.index("unique_bytes_sent_a2b")]) <= 0: - conn_suffix = "b2a" - - tx_packets = int(values[labels.index("total_packets_" + conn_suffix)]) - rexmt_packets = int(values[labels.index("rexmt_data_pkts_" + conn_suffix)]) - overhead = self.PACKET_OVERHEAD * tx_packets - goodput = 8 * int(values[labels.index("throughput_" + conn_suffix)]) - unique_bytes = int(values[labels.index("unique_bytes_sent_" + conn_suffix)]) - tx_time = (1.0 * unique_bytes) / goodput - throughput = ( - int(values[labels.index("actual_data_bytes_" + conn_suffix)]) / tx_time - ) - rtt = float(values[labels.index("RTT_avg_" + conn_suffix)]) / 2 - utilization = throughput * 100.0 / bn_speed - - result_str = [ - tx_packets, - overhead, - round(throughput / fact_by, 3), - round(rtt, 3), - round(goodput / fact_by, 3), - unique_bytes, - rexmt_packets, - round(utilization, 3), - round(queue_mean, 3), - round(queue_variance, 3), - round(queue_mean * 100.0 / (self.buf_size / pkt_size), 3), - round(flow_cmp_time, 3), - ] - - return result_str + + def _dominant_flow(self): + """Return the flow accumulator with the most unique bytes, or None.""" + flows = defaultdict(_FlowAccumulator) + cap = pyshark.FileCapture( + self.pcap_file, + display_filter="tcp", + keep_packets=False, + ) + try: + for pkt in cap: + self._process_packet(pkt, flows) + finally: + cap.close() + if not flows: + return None + best = max(flows.values(), key=lambda f: f.unique_bytes) + return best if best.unique_bytes > 0 else None + + @staticmethod + def _process_packet(pkt, flows) -> None: + try: + tcp = pkt.tcp + key = (int(tcp.stream), pkt.ip.src, tcp.srcport, pkt.ip.dst, tcp.dstport) + ts = float(pkt.sniff_timestamp) + payload_len = int(tcp.len) if hasattr(tcp, "len") else 0 + is_retrans = hasattr(tcp, "analysis_retransmission") + rtt_ms = float(tcp.analysis_ack_rtt) * 1000 if hasattr(tcp, "analysis_ack_rtt") else None + flows[key].add_packet(ts, payload_len, is_retrans, rtt_ms) + except AttributeError: + pass def _unit_factor(self) -> float: first, second = list(self.unit) diff --git a/tests/test_report.py b/tests/test_report.py new file mode 100644 index 0000000..43e690f --- /dev/null +++ b/tests/test_report.py @@ -0,0 +1,152 @@ +import csv +import os + +import matplotlib +matplotlib.use("Agg") + +import numpy as np +import pytest + +from pcapprocessor.report import Reporter + + +def _write_csv(path, headers, rows): + """Write tab-separated rows to path.""" + with open(path, "w", newline="") as f: + writer = csv.writer(f, delimiter="\t") + writer.writerow(headers) + writer.writerows(rows) + + +def test_load_csv_returns_column_arrays(tmp_path): + csv_path = tmp_path / "tcp0.csv" + _write_csv(csv_path, + ["x-scale", "avg_throughput", "confInt_throughput"], + [[1.0, 10.5, 0.4], [2.0, 20.1, 0.6]]) + r = Reporter(csvs=[str(csv_path)], metrics=["throughput"], + output_dir=str(tmp_path / "plots")) + data = r._load_csv(str(csv_path)) + assert list(data.keys()) == ["x-scale", "avg_throughput", "confInt_throughput"] + np.testing.assert_array_almost_equal(data["avg_throughput"], [10.5, 20.1]) + + +def test_load_csv_raises_for_missing_file(tmp_path): + r = Reporter(csvs=[], metrics=[], output_dir=str(tmp_path / "plots")) + with pytest.raises(FileNotFoundError): + r._load_csv(str(tmp_path / "nonexistent.csv")) + + +def _make_protocol_data(tmp_path, name="tcp0", + x=(1.0, 2.0), avg=(10.0, 20.0), ci=(0.5, 0.8)): + csv_path = tmp_path / f"{name}.csv" + _write_csv(csv_path, + ["x-scale", "avg_throughput", "confInt_throughput"], + [[x[0], avg[0], ci[0]], [x[1], avg[1], ci[1]]]) + r = Reporter(csvs=[str(csv_path)], metrics=["throughput"], + output_dir=str(tmp_path / "plots")) + return str(csv_path), {name: r._load_csv(str(csv_path))} + + +def test_plot_metric_creates_file(tmp_path): + _, protocol_data = _make_protocol_data(tmp_path) + out = str(tmp_path / "plots") + os.makedirs(out, exist_ok=True) + r = Reporter(csvs=[], metrics=[], output_dir=out) + path = r._plot_metric("throughput", protocol_data) + assert os.path.exists(path) + assert path.endswith(".png") + + +def test_plot_metric_raises_for_unknown_metric(tmp_path): + _, protocol_data = _make_protocol_data(tmp_path) + out = str(tmp_path / "plots") + os.makedirs(out, exist_ok=True) + r = Reporter(csvs=[], metrics=[], output_dir=out) + with pytest.raises(ValueError, match="delay"): + r._plot_metric("delay", protocol_data) + + +def test_plot_metric_filename_uses_metric_name(tmp_path): + _, protocol_data = _make_protocol_data(tmp_path) + out = str(tmp_path / "plots") + os.makedirs(out, exist_ok=True) + r = Reporter(csvs=[], metrics=[], output_dir=out, fmt="svg") + path = r._plot_metric("throughput", protocol_data) + assert path == os.path.join(out, "throughput.svg") + + +def test_plot_creates_one_file_per_metric(tmp_path): + for name in ("tcp0.csv", "udp0.csv"): + _write_csv( + tmp_path / name, + ["x-scale", "avg_throughput", "confInt_throughput", + "avg_delay", "confInt_delay"], + [[1.0, 10.0, 0.5, 5.0, 0.1], [2.0, 20.0, 0.8, 4.5, 0.2]], + ) + out = str(tmp_path / "plots") + r = Reporter( + csvs=[str(tmp_path / "tcp0.csv"), str(tmp_path / "udp0.csv")], + metrics=["throughput", "delay"], + output_dir=out, + ) + paths = r.plot() + assert len(paths) == 2 + assert all(os.path.exists(p) for p in paths) + assert any("throughput" in p for p in paths) + assert any("delay" in p for p in paths) + + +def test_plot_creates_output_dir(tmp_path): + _write_csv( + tmp_path / "tcp0.csv", + ["x-scale", "avg_throughput", "confInt_throughput"], + [[1.0, 10.0, 0.5]], + ) + out = str(tmp_path / "new_dir" / "plots") + r = Reporter(csvs=[str(tmp_path / "tcp0.csv")], + metrics=["throughput"], output_dir=out) + r.plot() + assert os.path.isdir(out) + + +def test_reporter_importable_from_package(): + from pcapprocessor import Reporter as R + assert R is not None + + +def test_cli_exits_zero_on_success(tmp_path): + import subprocess + import sys as _sys + _write_csv( + tmp_path / "tcp0.csv", + ["x-scale", "avg_throughput", "confInt_throughput"], + [[10.0, 100.0, 5.0], [20.0, 200.0, 8.0]], + ) + result = subprocess.run( + [_sys.executable, "-m", "pcapprocessor.report", + "--csvs", str(tmp_path / "tcp0.csv"), + "--metrics", "throughput", + "--output", str(tmp_path / "plots")], + capture_output=True, text=True, + ) + assert result.returncode == 0 + assert "throughput.png" in result.stdout + + +def test_cli_exits_nonzero_on_missing_metric(tmp_path): + import subprocess + import sys as _sys + _write_csv( + tmp_path / "tcp0.csv", + ["x-scale", "avg_throughput", "confInt_throughput"], + [[10.0, 100.0, 5.0]], + ) + result = subprocess.run( + [_sys.executable, "-m", "pcapprocessor.report", + "--csvs", str(tmp_path / "tcp0.csv"), + "--metrics", "nonexistent", + "--output", str(tmp_path / "plots")], + capture_output=True, text=True, + ) + assert result.returncode == 1 + assert "Error" in result.stderr diff --git a/tests/test_trace.py b/tests/test_trace.py index d97d91b..0fbe32a 100644 --- a/tests/test_trace.py +++ b/tests/test_trace.py @@ -1,31 +1,9 @@ import pytest from configparser import ConfigParser -from unittest.mock import patch - -from pcapprocessor.trace import TraceProcessor - - -_A2B_OUTPUT = "\n".join([ - "timestamp line 00:01:00.000000", - "filler line", - "#1 TCP connection traced:", - "unique_bytes_sent_a2b,total_packets_a2b,rexmt_data_pkts_a2b," - "throughput_a2b,actual_data_bytes_a2b,RTT_avg_a2b", - "extra line", - "1000,10,0,1000,500,5.0", - "", -]) - -_B2A_OUTPUT = "\n".join([ - "timestamp line 00:01:00.000000", - "filler line", - "#1 TCP connection traced:", - "unique_bytes_sent_a2b,total_packets_b2a,rexmt_data_pkts_b2a," - "throughput_b2a,unique_bytes_sent_b2a,actual_data_bytes_b2a,RTT_avg_b2a", - "extra line", - "0,10,0,1000,1000,500,5.0", - "", -]) +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from pcapprocessor.trace import TraceProcessor, _FlowAccumulator def _make_config(scenario="myScenario"): @@ -47,76 +25,170 @@ def _make_processor(tmp_path, scenario="myScenario", unit="MB", buf_size=1000): ) +def _fake_pkt(stream="0", src="1.1.1.1", srcport="1234", + dst="2.2.2.2", dstport="80", + payload_len=1000, ts=0.0, + is_retrans=False, rtt_ms=None): + """SimpleNamespace fake pyshark packet — hasattr behaves naturally.""" + tcp = SimpleNamespace(stream=stream, srcport=srcport, dstport=dstport, len=str(payload_len)) + if is_retrans: + tcp.analysis_retransmission = "1" + if rtt_ms is not None: + tcp.analysis_ack_rtt = str(rtt_ms / 1000) + ip = SimpleNamespace(src=src, dst=dst) + pkt = SimpleNamespace(tcp=tcp, ip=ip) + pkt.sniff_timestamp = str(ts) + return pkt + + +def _mock_cap(packets): + cap = MagicMock() + cap.__iter__ = MagicMock(return_value=iter(packets)) + return cap + + +# --- _FlowAccumulator --- + +def test_flow_accumulator_tracks_unique_bytes(): + acc = _FlowAccumulator() + acc.add_packet(0.0, 500, False, None) + acc.add_packet(1.0, 300, False, 10.0) + assert acc.unique_bytes == 800 + assert acc.tx_packets == 2 + + +def test_flow_accumulator_separates_retransmissions(): + acc = _FlowAccumulator() + acc.add_packet(0.0, 1000, False, None) + acc.add_packet(0.5, 500, True, None) + assert acc.unique_bytes == 1000 + assert acc.rexmt_packets == 1 + assert acc.tx_packets == 2 + + +def test_flow_accumulator_tx_time(): + acc = _FlowAccumulator() + acc.add_packet(1.0, 100, False, None) + acc.add_packet(3.5, 100, False, None) + assert acc.tx_time == pytest.approx(2.5) + + +def test_flow_accumulator_avg_rtt_ms(): + acc = _FlowAccumulator() + acc.add_packet(0.0, 100, False, 10.0) + acc.add_packet(1.0, 100, False, 20.0) + assert acc.avg_rtt_ms == pytest.approx(15.0) + + +def test_flow_accumulator_avg_rtt_ms_empty(): + assert _FlowAccumulator().avg_rtt_ms == 0.0 + + +def test_flow_accumulator_tx_time_no_packets(): + assert _FlowAccumulator().tx_time == 0.0 + + +# --- _unit_factor --- + def test_unit_factor_megabytes(): - proc = TraceProcessor("f.pcap", "MB", None, "s", "t.txt", 100) - assert proc._unit_factor() == 8_000_000 + assert TraceProcessor("f.pcap", "MB", None, "s", "t.txt", 100)._unit_factor() == 8_000_000 def test_unit_factor_megabits(): - proc = TraceProcessor("f.pcap", "Mb", None, "s", "t.txt", 100) - assert proc._unit_factor() == 1_000_000 + assert TraceProcessor("f.pcap", "Mb", None, "s", "t.txt", 100)._unit_factor() == 1_000_000 def test_unit_factor_kilobytes(): - proc = TraceProcessor("f.pcap", "KB", None, "s", "t.txt", 100) - assert proc._unit_factor() == 8_000 + assert TraceProcessor("f.pcap", "KB", None, "s", "t.txt", 100)._unit_factor() == 8_000 def test_unit_factor_gigabits(): - proc = TraceProcessor("f.pcap", "Gb", None, "s", "t.txt", 100) - assert proc._unit_factor() == 1_000_000_000 + assert TraceProcessor("f.pcap", "Gb", None, "s", "t.txt", 100)._unit_factor() == 1_000_000_000 +# --- _bottleneck_speed --- + def test_bottleneck_speed_extracts_numeric(tmp_path): proc = _make_processor(tmp_path) - # bottleneckSpeed="10Mbps", fact_by=1 → 10*1 = 10 assert proc._bottleneck_speed(1.0) == 10.0 +# --- _dominant_flow --- + +def test_dominant_flow_returns_none_when_no_packets(tmp_path): + proc = _make_processor(tmp_path) + with patch("pcapprocessor.trace.pyshark.FileCapture", return_value=_mock_cap([])): + assert proc._dominant_flow() is None + + +def test_dominant_flow_selects_flow_with_most_bytes(tmp_path): + proc = _make_processor(tmp_path) + pkts = [ + _fake_pkt(src="1.1.1.1", srcport="1234", dst="2.2.2.2", dstport="80", + payload_len=1000, ts=0.0), + _fake_pkt(src="2.2.2.2", srcport="80", dst="1.1.1.1", dstport="1234", + payload_len=50, ts=0.1), + ] + with patch("pcapprocessor.trace.pyshark.FileCapture", return_value=_mock_cap(pkts)): + flow = proc._dominant_flow() + assert flow.unique_bytes == 1000 + + +def test_dominant_flow_counts_retransmissions(tmp_path): + proc = _make_processor(tmp_path) + pkts = [ + _fake_pkt(payload_len=1000, ts=0.0), + _fake_pkt(payload_len=500, ts=0.1, is_retrans=True), + ] + with patch("pcapprocessor.trace.pyshark.FileCapture", return_value=_mock_cap(pkts)): + flow = proc._dominant_flow() + assert flow.rexmt_packets == 1 + assert flow.unique_bytes == 1000 + + +def test_dominant_flow_collects_rtt_samples(tmp_path): + proc = _make_processor(tmp_path) + pkts = [ + _fake_pkt(payload_len=1000, ts=0.0, rtt_ms=10.0), + _fake_pkt(payload_len=500, ts=0.5, rtt_ms=20.0), + ] + with patch("pcapprocessor.trace.pyshark.FileCapture", return_value=_mock_cap(pkts)): + flow = proc._dominant_flow() + assert flow.avg_rtt_ms == pytest.approx(15.0) + + +# --- process --- + def test_process_returns_12_metrics(tmp_path): proc = _make_processor(tmp_path) - with patch("pcapprocessor.exe_comm.exe_comm", return_value=_A2B_OUTPUT): + pkts = [ + _fake_pkt(payload_len=1000, ts=0.0, rtt_ms=10.0), + _fake_pkt(payload_len=1000, ts=1.0), + ] + with patch("pcapprocessor.trace.pyshark.FileCapture", return_value=_mock_cap(pkts)): result = proc.process() assert isinstance(result, list) assert len(result) == 12 -def test_process_a2b_tx_packets(tmp_path): +def test_process_overhead_is_packet_overhead_times_tx_packets(tmp_path): proc = _make_processor(tmp_path) - with patch("pcapprocessor.exe_comm.exe_comm", return_value=_A2B_OUTPUT): + pkts = [_fake_pkt(payload_len=500, ts=0.0), _fake_pkt(payload_len=500, ts=1.0)] + with patch("pcapprocessor.trace.pyshark.FileCapture", return_value=_mock_cap(pkts)): result = proc.process() - assert result[0] == 10 # tx_packets - assert result[1] == 320 # overhead = PACKET_OVERHEAD * tx_packets + assert result[1] == result[0] * TraceProcessor.PACKET_OVERHEAD -def test_process_b2a_path_returns_12_metrics(tmp_path): +def test_process_flow_completion_time_in_ms(tmp_path): proc = _make_processor(tmp_path) - with patch("pcapprocessor.exe_comm.exe_comm", return_value=_B2A_OUTPUT): + pkts = [_fake_pkt(payload_len=1000, ts=0.0), _fake_pkt(payload_len=500, ts=1.0)] + with patch("pcapprocessor.trace.pyshark.FileCapture", return_value=_mock_cap(pkts)): result = proc.process() - assert isinstance(result, list) - assert len(result) == 12 + assert result[-1] == pytest.approx(1000.0) def test_process_raises_on_no_connections(tmp_path): proc = _make_processor(tmp_path) - with patch("pcapprocessor.exe_comm.exe_comm", return_value="no tcp data here\n"): + with patch("pcapprocessor.trace.pyshark.FileCapture", return_value=_mock_cap([])): with pytest.raises(ValueError, match="No TCP connections found"): proc.process() - - -def test_process_flow_cmp_time_fallback_on_bad_timestamp(tmp_path): - # Timestamp that can't be parsed → flow_cmp_time stays 0 - bad_ts_output = "\n".join([ - "no timestamp here", - "filler line", - "#1 TCP connection traced:", - "unique_bytes_sent_a2b,total_packets_a2b,rexmt_data_pkts_a2b," - "throughput_a2b,actual_data_bytes_a2b,RTT_avg_a2b", - "extra line", - "1000,10,0,1000,500,5.0", - "", - ]) - proc = _make_processor(tmp_path) - with patch("pcapprocessor.exe_comm.exe_comm", return_value=bad_ts_output): - result = proc.process() - assert result[-1] == 0 # flow_cmp_time defaults to 0