diff --git a/.deepsource.toml b/.deepsource.toml deleted file mode 100644 index c9e3b50..0000000 --- a/.deepsource.toml +++ /dev/null @@ -1,7 +0,0 @@ -version = 1 - -[[analyzers]] -name = "python" - - [analyzers.meta] - runtime_version = "3.x.x" diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index f3d4fca..8ee19db 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -26,7 +26,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest + pip install flake8 pytest pytest-cov if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Lint with flake8 run: | @@ -36,4 +36,4 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest run: | - pytest + pytest --cov=pcapprocessor --cov-report=term-missing --cov-fail-under=90 diff --git a/.gitignore b/.gitignore index d2e560b..c78c09e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ tmp/ __pycache__/ *.py[cod] data/ +.coverage diff --git a/pcapprocessor.py b/pcapprocessor.py index 28d1c37..4c3ee1f 100644 --- a/pcapprocessor.py +++ b/pcapprocessor.py @@ -3,7 +3,7 @@ __all__ = ["PcapProcessor"] -if __name__ == "__main__": +def _main() -> int: import sys from configparser import ConfigParser @@ -12,7 +12,7 @@ "Usage: python pcapprocessor.py " " " ) - sys.exit(1) + return 1 cfg = ConfigParser() cfg.read(sys.argv[3]) @@ -25,5 +25,10 @@ ascii_trace_file=sys.argv[5], buf_size=int(sys.argv[6]), ) - result = proc.process() - print(result) + print(proc.process()) + return 0 + + +if __name__ == "__main__": + import sys + sys.exit(_main()) diff --git a/pcapprocessor/trace.py b/pcapprocessor/trace.py index 7b31163..7e2f301 100644 --- a/pcapprocessor/trace.py +++ b/pcapprocessor/trace.py @@ -1,5 +1,4 @@ import re -import sys import shlex import datetime @@ -54,8 +53,7 @@ def process(self) -> list: ] if not matches: - print("No TCP connections found") - sys.exit() + raise ValueError("No TCP connections found in pcap file") connections = [ pcap_lines[matches[j]: matches[j + 1]] diff --git a/requirements.txt b/requirements.txt index c97c5ed..7a5591c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,4 @@ pyshark setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability pydantic dash +pytest-cov diff --git a/tests/test_dce_runner.py b/tests/test_dce_runner.py new file mode 100644 index 0000000..cc630d0 --- /dev/null +++ b/tests/test_dce_runner.py @@ -0,0 +1,33 @@ +from unittest.mock import patch + +from pcapprocessor.pbs.dce_runner import DceRunner + + +def test_run_with_scenario_delegates_to_bfs_runner(tmp_path): + config_file = str(tmp_path / "config.cfg") + with patch("pcapprocessor.pbs.dce_runner.BfsRunner") as mock_bfs: + DceRunner(config_file, scenario="myScenario").run() + mock_bfs.assert_called_once_with(config_file, "myScenario") + mock_bfs.return_value.run.assert_called_once() + + +def test_run_without_scenario_delegates_to_pbs_writer(tmp_path): + config_file = str(tmp_path / "config.cfg") + with patch("pcapprocessor.pbs.dce_runner.PbsJobWriter") as mock_writer: + DceRunner(config_file).run() + mock_writer.assert_called_once_with(config_file, jobs_dir="initial-test") + mock_writer.return_value.write.assert_called_once() + + +def test_custom_jobs_dir_passed_to_pbs_writer(tmp_path): + config_file = str(tmp_path / "config.cfg") + with patch("pcapprocessor.pbs.dce_runner.PbsJobWriter") as mock_writer: + DceRunner(config_file, jobs_dir="custom-jobs").run() + mock_writer.assert_called_once_with(config_file, jobs_dir="custom-jobs") + + +def test_init_stores_params(): + runner = DceRunner("cfg.cfg", scenario="s", jobs_dir="jobs") + assert runner.config_file == "cfg.cfg" + assert runner.scenario == "s" + assert runner.jobs_dir == "jobs" diff --git a/tests/test_exe_comm.py b/tests/test_exe_comm.py new file mode 100644 index 0000000..c064138 --- /dev/null +++ b/tests/test_exe_comm.py @@ -0,0 +1,33 @@ +import subprocess +from unittest.mock import MagicMock, patch + +from pcapprocessor.exe_comm import exe_comm + + +def test_capture_true_returns_stdout(): + mock_result = MagicMock() + mock_result.stdout = b"hello world" + with patch("subprocess.run", return_value=mock_result) as mock_run: + result = exe_comm(["echo", "hello"]) + assert result == "hello world" + mock_run.assert_called_once_with( + ["echo", "hello"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + + +def test_capture_false_returns_empty_string(): + with patch("subprocess.run") as mock_run: + result = exe_comm(["echo", "hello"], capture=False) + assert result == "" + mock_run.assert_called_once_with(["echo", "hello"], check=True) + + +def test_non_ascii_output_decoded_with_replace(): + mock_result = MagicMock() + mock_result.stdout = "héllo".encode("utf-8") + with patch("subprocess.run", return_value=mock_result): + result = exe_comm(["cmd"]) + assert result == "héllo" diff --git a/tests/test_processor.py b/tests/test_processor.py new file mode 100644 index 0000000..f470c55 --- /dev/null +++ b/tests/test_processor.py @@ -0,0 +1,49 @@ +from configparser import ConfigParser +from unittest.mock import patch + +import numpy as np + +from pcapprocessor.processor import PcapProcessor + + +def _make_config(): + config = ConfigParser() + config["myScenario"] = {"transProt": "tcp", "pcapFile": "/tmp/t", "csvName": ""} + return config + + +def test_init_stores_all_params(): + config = _make_config() + proc = PcapProcessor("/tmp/t.pcap", "MB", config, "myScenario", "trace.txt", 1000) + assert proc.pcap_file_path == "/tmp/t.pcap" + assert proc.unit == "MB" + assert proc.scenario == "myScenario" + assert proc.ascii_trace_file == "trace.txt" + assert proc.buf_size == 1000 + + +def test_process_delegates_to_trace_processor(): + config = _make_config() + proc = PcapProcessor("/tmp/t.pcap", "MB", config, "myScenario", "trace.txt", 1000) + fake = [1, 2, 3] + with patch("pcapprocessor.processor.TraceProcessor") as mock_tp: + mock_tp.return_value.process.return_value = fake + result = proc.process() + assert result == fake + mock_tp.assert_called_once_with("/tmp/t.pcap", "MB", config, "myScenario", "trace.txt", 1000) + + +def test_process_and_write_chains_trace_and_metrics(): + config = _make_config() + proc = PcapProcessor("/tmp/t.pcap", "MB", config, "myScenario", "trace.txt", 1000) + x_array = np.array([1.0, 2.0]) + with ( + patch("pcapprocessor.processor.TraceProcessor") as mock_tp, + patch("pcapprocessor.processor.MetricsWriter") as mock_mw, + ): + mock_tp.return_value.process.return_value = [0] * 12 + proc.process_and_write(x_array, "test.pcap") + mock_mw.return_value.write.assert_called_once() + mock_mw.assert_called_once_with( + [0] * 12, x_array, "myScenario", config, "test.pcap" + ) diff --git a/tests/test_runner.py b/tests/test_runner.py new file mode 100644 index 0000000..0c5f037 --- /dev/null +++ b/tests/test_runner.py @@ -0,0 +1,144 @@ +from configparser import ConfigParser +from unittest.mock import patch + +import numpy as np +import pytest + +from pcapprocessor.runner import BfsRunner, SimulationRunner + + +def _make_config(section="myScenario", **kwargs): + config = ConfigParser() + config[section] = { + "runs": "1", + "pcapFile": "test", + "outputFactor": "MB", + "transProt": "tcp", + "qSizeFileName": "qsize.txt", + **kwargs, + } + return config + + +def test_simulation_runner_run_returns_correct_shape(tmp_path): + config = _make_config() + runner = SimulationRunner("10", 12, "myScenario", config) + fake_pcap = str(tmp_path / "test-tcp0-0.pcap") + fake_ascii = str(tmp_path / "qsize.txt") + + with ( + patch("pcapprocessor.runner.WafCommandBuilder") as mock_waf, + patch("pcapprocessor.runner.exe_comm.exe_comm"), + patch("pcapprocessor.runner.glob") as mock_glob, + patch("pcapprocessor.runner.TraceProcessor") as mock_trace, + patch("pcapprocessor.runner.os.remove"), + ): + mock_waf.return_value.build.return_value = ("./waf --x=10", 100) + mock_glob.side_effect = [[fake_pcap], [fake_ascii]] + mock_trace.return_value.process.return_value = list(np.ones(12)) + + run_stats, pcap_files = runner.run() + + assert run_stats.shape == (1, 1, 12) + assert fake_pcap in pcap_files + + +def test_simulation_runner_removes_pcap_after_trace(tmp_path): + config = _make_config() + runner = SimulationRunner("10", 12, "myScenario", config) + fake_pcap = str(tmp_path / "test-tcp0-0.pcap") + + with ( + patch("pcapprocessor.runner.WafCommandBuilder") as mock_waf, + patch("pcapprocessor.runner.exe_comm.exe_comm"), + patch("pcapprocessor.runner.glob") as mock_glob, + patch("pcapprocessor.runner.TraceProcessor") as mock_trace, + patch("pcapprocessor.runner.os.remove") as mock_remove, + ): + mock_waf.return_value.build.return_value = ("./waf", 100) + mock_glob.side_effect = [[fake_pcap], [str(tmp_path / "qsize.txt")]] + mock_trace.return_value.process.return_value = list(np.ones(12)) + + runner.run() + + mock_remove.assert_called_once_with(fake_pcap) + + +def test_simulation_runner_raises_on_missing_ascii_file(): + config = _make_config() + runner = SimulationRunner("10", 12, "myScenario", config) + + with ( + patch("pcapprocessor.runner.WafCommandBuilder") as mock_waf, + patch("pcapprocessor.runner.exe_comm.exe_comm"), + patch("pcapprocessor.runner.glob") as mock_glob, + ): + mock_waf.return_value.build.return_value = ("./waf", 100) + mock_glob.side_effect = [["test.pcap"], []] + + with pytest.raises(ValueError, match="No ASCII trace files"): + runner.run() + + +def test_bfs_runner_iterates_xscale(tmp_path): + config_file = tmp_path / "config.cfg" + config_file.write_text( + "[myScenario]\n" + "myScenario = 1,2\n" + "transProt = tcp\n" + "runs = 1\n" + "pcapFile = test\n" + "csvName = out\n" + "bdpQsz = 1.0\n" + "bottleneckDelay = 10ms\n" + "bottleneckSpeed = 10Mbps\n" + "outputFactor = MB\n" + "qSizeFileName = qsize.txt\n" + "cmd = run\n" + ) + runner = BfsRunner(str(config_file), "myScenario") + fake_stats = np.zeros((1, 1, 12)) + fake_pcaps = ["test-tcp0-0.pcap"] + + with ( + patch("pcapprocessor.runner.SimulationRunner") as mock_sim, + patch("pcapprocessor.runner.MetricsWriter") as mock_writer, + ): + mock_sim.return_value.run.return_value = (fake_stats, fake_pcaps) + runner.run() + + # one SimulationRunner call per xscale value (1 and 2) + assert mock_sim.call_count == 2 + mock_writer.return_value.write.assert_called() + + +def test_bfs_runner_writes_metrics_per_flow(tmp_path): + config_file = tmp_path / "config.cfg" + config_file.write_text( + "[myScenario]\n" + "myScenario = 5\n" + "transProt = tcp,udp\n" + "runs = 1\n" + "pcapFile = test\n" + "csvName = out\n" + "bdpQsz = 1.0\n" + "bottleneckDelay = 10ms\n" + "bottleneckSpeed = 10Mbps\n" + "outputFactor = MB\n" + "qSizeFileName = qsize.txt\n" + "cmd = run\n" + ) + runner = BfsRunner(str(config_file), "myScenario") + # 2 flows + fake_stats = np.zeros((2, 1, 12)) + fake_pcaps = ["test-tcp0-0.pcap", "test-udp0-0.pcap"] + + with ( + patch("pcapprocessor.runner.SimulationRunner") as mock_sim, + patch("pcapprocessor.runner.MetricsWriter") as mock_writer, + ): + mock_sim.return_value.run.return_value = (fake_stats, fake_pcaps) + runner.run() + + # one write per flow + assert mock_writer.return_value.write.call_count == 2 diff --git a/tests/test_simulation.py b/tests/test_simulation.py index b40e121..d6e8f72 100644 --- a/tests/test_simulation.py +++ b/tests/test_simulation.py @@ -67,3 +67,30 @@ def test_waf_command_builder_q_monitoring_is_1_when_single_run(): }) cmd, _ = WafCommandBuilder("1", "10", "bottleneckDelay", config).build() assert "qm=1" in cmd + + +def test_queue_size_else_branch(): + # scenario is neither "bottleneckDelay" nor "bottleneckSpeed" → else branch + config = _make_config("changingDelay", { + "bdpQsz": "1.0", + "changingDelay": "5,10", # xscale (key == scenario name) + "bottleneckDelay": "10ms", + "bottleneckSpeed": "10Mbps", + }) + # delay = int("10ms"[:-2]) = 10, speed_numeric = 10*1_000_000 = 10_000_000 + # queue_size = int(1.0 * 10_000_000 * 2 * 0.001 * 10 / 8) = 25000 + result = QueueSizeCalculator("changingDelay", "5", config).calculate() + assert result == "25000" + + +def test_waf_command_builder_appends_mbps_for_bottleneck_speed(): + config = _make_config("bottleneckSpeed", { + "bdpQsz": "1.0", + "bottleneckSpeed": "10,20", # xscale + "bottleneckDelay": "10ms", + "speedUnit": "Mbps", + "runs": "1", + "cmd": "run --x=%(x)s --runNo=%(runNo)s --qs=%(queue_size)s --qm=%(qMonitoring)s", + }) + cmd, _ = WafCommandBuilder("1", "10", "bottleneckSpeed", config).build() + assert "10Mbps" in cmd diff --git a/tests/test_trace.py b/tests/test_trace.py new file mode 100644 index 0000000..d97d91b --- /dev/null +++ b/tests/test_trace.py @@ -0,0 +1,122 @@ +import pytest +from configparser import ConfigParser +from unittest.mock import patch + +from pcapprocessor.trace import TraceProcessor + + +_A2B_OUTPUT = "\n".join([ + "timestamp line 00:01:00.000000", + "filler line", + "#1 TCP connection traced:", + "unique_bytes_sent_a2b,total_packets_a2b,rexmt_data_pkts_a2b," + "throughput_a2b,actual_data_bytes_a2b,RTT_avg_a2b", + "extra line", + "1000,10,0,1000,500,5.0", + "", +]) + +_B2A_OUTPUT = "\n".join([ + "timestamp line 00:01:00.000000", + "filler line", + "#1 TCP connection traced:", + "unique_bytes_sent_a2b,total_packets_b2a,rexmt_data_pkts_b2a," + "throughput_b2a,unique_bytes_sent_b2a,actual_data_bytes_b2a,RTT_avg_b2a", + "extra line", + "0,10,0,1000,1000,500,5.0", + "", +]) + + +def _make_config(scenario="myScenario"): + config = ConfigParser() + config[scenario] = {"pktSize": "100", "bottleneckSpeed": "10Mbps"} + return config + + +def _make_processor(tmp_path, scenario="myScenario", unit="MB", buf_size=1000): + ascii_file = tmp_path / "qsize.txt" + ascii_file.write_text("0,100\n0,200\n") + return TraceProcessor( + pcap_file=str(tmp_path / "test.pcap"), + unit=unit, + config=_make_config(scenario), + scenario=scenario, + ascii_trace_file=str(ascii_file), + buf_size=buf_size, + ) + + +def test_unit_factor_megabytes(): + proc = TraceProcessor("f.pcap", "MB", None, "s", "t.txt", 100) + assert proc._unit_factor() == 8_000_000 + + +def test_unit_factor_megabits(): + proc = TraceProcessor("f.pcap", "Mb", None, "s", "t.txt", 100) + assert proc._unit_factor() == 1_000_000 + + +def test_unit_factor_kilobytes(): + proc = TraceProcessor("f.pcap", "KB", None, "s", "t.txt", 100) + assert proc._unit_factor() == 8_000 + + +def test_unit_factor_gigabits(): + proc = TraceProcessor("f.pcap", "Gb", None, "s", "t.txt", 100) + assert proc._unit_factor() == 1_000_000_000 + + +def test_bottleneck_speed_extracts_numeric(tmp_path): + proc = _make_processor(tmp_path) + # bottleneckSpeed="10Mbps", fact_by=1 → 10*1 = 10 + assert proc._bottleneck_speed(1.0) == 10.0 + + +def test_process_returns_12_metrics(tmp_path): + proc = _make_processor(tmp_path) + with patch("pcapprocessor.exe_comm.exe_comm", return_value=_A2B_OUTPUT): + result = proc.process() + assert isinstance(result, list) + assert len(result) == 12 + + +def test_process_a2b_tx_packets(tmp_path): + proc = _make_processor(tmp_path) + with patch("pcapprocessor.exe_comm.exe_comm", return_value=_A2B_OUTPUT): + result = proc.process() + assert result[0] == 10 # tx_packets + assert result[1] == 320 # overhead = PACKET_OVERHEAD * tx_packets + + +def test_process_b2a_path_returns_12_metrics(tmp_path): + proc = _make_processor(tmp_path) + with patch("pcapprocessor.exe_comm.exe_comm", return_value=_B2A_OUTPUT): + result = proc.process() + assert isinstance(result, list) + assert len(result) == 12 + + +def test_process_raises_on_no_connections(tmp_path): + proc = _make_processor(tmp_path) + with patch("pcapprocessor.exe_comm.exe_comm", return_value="no tcp data here\n"): + with pytest.raises(ValueError, match="No TCP connections found"): + proc.process() + + +def test_process_flow_cmp_time_fallback_on_bad_timestamp(tmp_path): + # Timestamp that can't be parsed → flow_cmp_time stays 0 + bad_ts_output = "\n".join([ + "no timestamp here", + "filler line", + "#1 TCP connection traced:", + "unique_bytes_sent_a2b,total_packets_a2b,rexmt_data_pkts_a2b," + "throughput_a2b,actual_data_bytes_a2b,RTT_avg_a2b", + "extra line", + "1000,10,0,1000,500,5.0", + "", + ]) + proc = _make_processor(tmp_path) + with patch("pcapprocessor.exe_comm.exe_comm", return_value=bad_ts_output): + result = proc.process() + assert result[-1] == 0 # flow_cmp_time defaults to 0