diff --git a/Dockerfile.backend b/Dockerfile.backend index 9623eed..cdfe2a4 100644 --- a/Dockerfile.backend +++ b/Dockerfile.backend @@ -3,6 +3,18 @@ FROM python:3.12-slim WORKDIR /app +# tshark is used by PcapDecoder.get_packet_details() to decode previously +# captured .pcap files (via `tshark -T json`) — the actual packet capture +# itself happens remotely over SSH on each device, so no extra Linux +# capabilities (NET_RAW/NET_ADMIN) are needed on this container, just the +# tshark binary. DEBIAN_FRONTEND=noninteractive + the debconf preseed below +# stop the wireshark-common postinst from prompting for the "allow +# non-superusers to capture packets" question during the build. +RUN apt-get update && \ + echo "wireshark-common wireshark-common/install-setuid boolean false" | debconf-set-selections && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tshark && \ + rm -rf /var/lib/apt/lists/* + # Install dependencies first (layer caching) COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt gunicorn @@ -32,4 +44,4 @@ CMD gunicorn \ --timeout 120 \ --access-logfile - \ --error-logfile - \ - backend.app:app + backend.app:app \ No newline at end of file diff --git a/backend/app.py b/backend/app.py index a7448c8..26fe1d2 100644 --- a/backend/app.py +++ b/backend/app.py @@ -9,22 +9,27 @@ import uuid from pathlib import Path -from flask import Flask, jsonify, request +import psutil +from flask import Flask, jsonify, request, send_file from flask_cors import CORS from paramiko_expect import SSHClientInteraction from backend.models.device import Device from backend.models.device_config import DeviceConfig -from backend.utils.config_helper import ConfigurationHelper +from backend.utils.log_snapshots_helper import LogSnapshotsHelper from backend.utils.device_config_loader import DeviceConfigLoader +from backend.utils.pcap_decoder import DissectorRegistry, PcapDecoder +from backend.utils.fabric_connection import build_nested_connection as _build_nested_connection -SETTINGS_FILE = Path("settings.json") -FRONTEND_BASE = os.getenv("FRONTEND_BASE", "http://localhost:8100") - +SETTINGS_FILE = Path("settings.json") +FRONTEND_BASE = os.getenv("FRONTEND_BASE", "http://localhost:8100") +PROJECT_ROOT = Path(__file__).resolve().parent.parent +DISSECTORS_DIR = PROJECT_ROOT / "data" / "dissectors" app = Flask(__name__) -CORS(app) # allow the React dev-server / built bundle to call the API +CORS(app) +dissector_registry = DissectorRegistry(DISSECTORS_DIR) # ── helpers ─────────────────────────────────────────────────────────────────── @@ -41,6 +46,9 @@ def get_current_devices() -> list[Device]: def get_target_device(device_id: str) -> Device | None: """Return the 'Device' whose config ID matches device_id. + Calls get_current_devices() once and searches the result, avoiding a + second filesystem scan inside a single request. + Args: device_id (str): The device config ID to look up. @@ -153,6 +161,57 @@ def _save_settings(settings: dict) -> None: SETTINGS_FILE.write_text(json.dumps(settings, indent=2)) +# ── system ──────────────────────────────────────────────────────────────────── + +@app.get("/api/system/stats") +def system_stats(): + """Return current CPU, RAM and storage usage for the backend host. + + GET '/api/system/stats' + + Returns: + 200 OK: + JSON object containing: + + - cpuPercent (float) - Current CPU utilisation, 0-100. + - ramPercent (float) - Current RAM utilisation, 0-100. + - ramUsedGb (float) - RAM currently in use, in GB. + - ramTotalGb (float) - Total installed RAM, in GB. + - diskPercent (float) - Utilisation of the filesystem hosting the + app, 0-100. + - diskUsedGb (float) - Disk space currently in use, in GB. + - diskTotalGb (float) - Total disk capacity, in GB. + + Example:: + + { + "cpuPercent": 12.5, + "ramPercent": 43.2, + "ramUsedGb": 6.9, + "ramTotalGb": 16.0, + "diskPercent": 58.1, + "diskUsedGb": 232.4, + "diskTotalGb": 400.0 + } + """ + # interval=0.1 blocks briefly to get a real (non-zero) reading instead of + # the meaningless 0.0 psutil returns on the very first call in a process. + cpu_percent = psutil.cpu_percent(interval=0.1) + mem = psutil.virtual_memory() + disk = psutil.disk_usage(str(PROJECT_ROOT)) + + gb = 1024 ** 3 + return jsonify({ + "cpuPercent": cpu_percent, + "ramPercent": mem.percent, + "ramUsedGb": round(mem.used / gb, 1), + "ramTotalGb": round(mem.total / gb, 1), + "diskPercent": disk.percent, + "diskUsedGb": round(disk.used / gb, 1), + "diskTotalGb": round(disk.total / gb, 1), + }) + + # ── devices ─────────────────────────────────────────────────────────────────── @app.get("/api/devices") @@ -294,6 +353,64 @@ def get_device(device_id: str): return jsonify(device_to_dict(device)) +@app.get("/api/devices//errors") +def get_device_errors(device_id: str): + """Return the error log for a single device. + + Reads the ``errors.feather`` file written by the device watchdog whenever + a command execution fails or an SSH exception is recorded. + + GET '/api/devices//errors' + + Path parameters: + - device_id (str) - The config ID of the device. + + Returns: + 200 OK: + JSON object with an ``errors`` list. Each entry contains: + + - time (str) - ISO-formatted timestamp of the error. + - error_info (str) - Human-readable error description. + + Example:: + + { + "errors": [ + { + "time": "2024-01-01 10:03:12.456789", + "error_info": "cmd 'journalctl -b -n 200 --no-pager' failed with -> timed out" + } + ] + } + + 404 Not Found: + '{ "error": "not_found" }' - No device with the given ID exists. + """ + device = get_target_device(device_id) + if not device: + return _bad("not_found", 404) + + errors_path = Path("data") / device_id / "errors.feather" + if not errors_path.exists(): + return jsonify({"errors": []}) + + try: + import pandas as pd # lazy import — pandas is a heavy dep; keep it out of module scope + df = pd.read_feather(str(errors_path)) + # Ensure consistent column presence even if the file is empty + if df.empty or "time" not in df.columns: + return jsonify({"errors": []}) + rows = df[["time", "error_info"]].copy() + rows["time"] = rows["time"].astype(str) + # Most-recent errors first + errors = rows.iloc[::-1].to_dict(orient="records") + return jsonify({"errors": errors}) + except ImportError: + return _bad("pandas is not installed on the server", 500) + except Exception as exc: + return _bad(f"failed to read error log: {exc}", 500) + + # ── log snapshots ───────────────────────────────────────────────────────────── @app.get("/api/snapshots") @@ -370,11 +487,11 @@ def list_snapshots(): devices = get_current_devices() if search_param and search_value: - snapshots = ConfigurationHelper.get_filtered_log_snapshots_list( + snapshots = LogSnapshotsHelper.get_filtered_log_snapshots_list( devices, search_param, search_value, is_chart ) else: - snapshots = ConfigurationHelper.get_log_snapshots_list(devices, is_chart) + snapshots = LogSnapshotsHelper.get_log_snapshots_list(devices, is_chart) total = len(snapshots) total_pages = max(1, -(-total // page_size)) # ceiling division @@ -417,16 +534,119 @@ def get_snapshot_content(snapshot_id: str): """ is_chart = request.args.get("log_type", "text") == "chart" devices = get_current_devices() - snapshots = ConfigurationHelper.get_log_snapshots_list(devices, is_chart) + snapshots = LogSnapshotsHelper.get_log_snapshots_list(devices, is_chart) target = next((s for s in snapshots if s.id == snapshot_id), None) if not target: return _bad("not_found", 404) - rows = ConfigurationHelper.get_log_content_for_selected_snapshots([target]).to_dict(orient="records") + rows = LogSnapshotsHelper.get_log_content_for_selected_snapshots([target]).to_dict(orient="records") return jsonify({"rows": rows}) +@app.get("/api/snapshots//packets/") +def get_packet_details(snapshot_id: str, packet_number: int): + """Return the full decoded tshark field detail for a single packet. + + Only valid for "packet_capture" snapshots (produced by + PcapDecoder.to_log_snapshot / DeviceWatchdog.save_log_snapshots). + Locates the snapshot's saved '.pcap' file via the + snapshot's device_config_id/session_id and decodes the requested frame + with PcapDecoder.get_session_packet_details. + + GET '/api/snapshots//packets/' + + Path parameters: + - snapshot_id (str) - ID of the packet_capture snapshot. + - packet_number (int) - 1-based tshark frame number, i.e. the + leading field of PacketInfo.to_content_str() shown at the start + of each packet line's "content". + + Returns: + 200 OK: + '{ "packet_number": N, "details": { "": { "": "", … }, … } }' + + 404 Not Found: + '{ "error": "not_found" }' - No snapshot with the given ID + exists, it isn't a "packet_capture" snapshot, or no packet + with that frame number exists in the capture. + + 500 Internal Server Error: + '{ "error": "tshark not available: …" }' - tshark is not + installed on the server, or the session's pcap file is + missing on disk. + """ + devices = get_current_devices() + snapshots = LogSnapshotsHelper.get_log_snapshots_list(devices, log_type_chart=False) + + target = next((s for s in snapshots if s.id == snapshot_id), None) + if not target or getattr(target, "log_name", "") != "network capture": + return _bad("not_found", 404) + + device_data_dir = os.path.join("data", target.device_id) + try: + details = PcapDecoder.get_session_packet_details( + device_data_dir, target.session_id, packet_number, + dissectors_dir=dissector_registry, + ) + except FileNotFoundError as exc: + return _bad(f"tshark not available: {exc}", 500) + + if not details: + return _bad("not_found", 404) + + return jsonify({"packet_number": packet_number, "details": details}) + + +@app.get("/api/snapshots//pcap") +def download_snapshot_pcap(snapshot_id: str): + """Download the full raw pcap file backing a "network capture" snapshot. + + Only valid for "packet_capture" snapshots (see get_packet_details for + how the underlying '.pcap' file is produced/located). + Unlike the decoded/paginated view used by the log content and packet + detail endpoints, this streams the untouched pcap file as-is so it can + be opened directly in Wireshark or any other pcap tool. + + GET '/api/snapshots//pcap' + + Path parameters: + - snapshot_id (str) - ID of the "network capture" snapshot. + + Returns: + 200 OK: + The raw pcap file, streamed as an attachment + ('application/vnd.tcpdump.pcap'). + + 404 Not Found: + '{ "error": "not_found" }' - No snapshot with the given ID + exists, it isn't a "network capture" snapshot, or its pcap + file is missing on disk. + """ + devices = get_current_devices() + snapshots = LogSnapshotsHelper.get_log_snapshots_list(devices, log_type_chart=False) + + target = next((s for s in snapshots if s.id == snapshot_id), None) + if not target or getattr(target, "log_name", "") != "network capture": + return _bad("not_found", 404) + + pcap_path = os.path.join(PROJECT_ROOT, "data", target.device_id, f"{target.session_id}.pcap") + print(pcap_path) + if not os.path.exists(pcap_path): + return _bad("not_found", 404) + + device_name = getattr(target, "device_name", "device") + safe_name = "_".join(device_name.split()) + download_name = f"{safe_name}_{target.session_id}.pcap" + + return send_file( + pcap_path, + mimetype="application/vnd.tcpdump.pcap", + as_attachment=True, + download_name=download_name, + ) + + @app.delete("/api/snapshots") def remove_snapshots(): """Remove one or more log snapshots and delete their underlying files. @@ -462,7 +682,7 @@ def remove_snapshots(): return _bad("snapshot_ids must be a non-empty list") devices = get_current_devices() - snapshots = ConfigurationHelper.get_log_snapshots_list(devices, is_chart) + snapshots = LogSnapshotsHelper.get_log_snapshots_list(devices, is_chart) by_id = {s.id: s for s in snapshots} removed = [] @@ -503,7 +723,7 @@ def start_logs_collection(): 400 Bad Request: '{ "error": "selected_devices must be a list" }' - or '{ "error": "session_scenario must be a string" }' + or '{ "error": "session_scenario is required and must be a non-empty string" }' """ body = request.get_json(force=True) selected_devices = body.get("selected_devices", []) @@ -511,7 +731,8 @@ def start_logs_collection(): if not isinstance(selected_devices, list): return _bad("selected_devices must be a list") - if not isinstance(session_scenario, str): + + if not isinstance(session_scenario, str) or not session_scenario.strip(): return _bad("session_scenario is required and must be a non-empty string") session_id = uuid.uuid1().hex[:12] @@ -695,17 +916,6 @@ def set_auto_collection(): return jsonify({"status": "ok", "devices": updated}) -# ── device config builder helpers ───────────────────────────────────────────── -# -# Connection building now lives in backend.utils.fabric_connection, shared -# with device_watchdog.py, so that a connection (including ssh_key_string of -# any supported type, with or without a passphrase, and either gateway -# shape) which passes Test Connection here behaves identically once the -# watchdog picks up the saved config. See that module's docstring for the -# accepted field/gateway shapes. -from backend.utils.fabric_connection import build_nested_connection as _build_nested_connection - - @app.post("/api/devices/test-connection") def test_device_connection(): """Test SSH connectivity to a device using provided credentials. @@ -740,11 +950,6 @@ def test_device_connection(): 400 Bad Request: '{ "error": "missing required fields" }' """ - try: - from fabric import Connection # noqa: F401 (validate import) - except ImportError: - return jsonify({"success": False, "message": "fabric not installed on server"}), 200 - body = request.get_json(force=True) ip = body.get("ip_address", "").strip() port = int(body.get("port", 22)) @@ -813,16 +1018,17 @@ def exec_device_command(): conn = _build_nested_connection(body) if custom_shell_prompt: conn.open() - client = conn.client - interact = SSHClientInteraction(client, timeout=20, display=False) - interact.expect(custom_shell_prompt) - cmd_output = "" - for single_cmd in command.split(";"): - interact.send(single_cmd) + try: + client = conn.client + interact = SSHClientInteraction(client, timeout=20, display=False) interact.expect(custom_shell_prompt) - cmd_output = cmd_output + interact.current_output_clean - - conn.close() + cmd_output = "" + for single_cmd in command.split(";"): + interact.send(single_cmd) + interact.expect(custom_shell_prompt) + cmd_output = cmd_output + interact.current_output_clean + finally: + conn.close() return jsonify({"stdout": cmd_output, "stderr": "", "exit_code": 0}) # ── Standard exec path ──────────────────────────────────────────────── @@ -877,3 +1083,177 @@ def change_password(): _save_settings(settings) return jsonify({"status": "ok"}) + + + +# ── dissectors ──────────────────────────────────────────────────────────────── + +@app.get("/api/settings/dissectors") +def list_dissectors(): + """Return the list of custom dissector files installed on the server. + + GET '/api/settings/dissectors' + + Returns: + 200 OK: + JSON array of dissector objects. Each element contains: + + - name (str) - Bare filename (e.g. ``"my_proto.lua"``). + - size_bytes (int) - File size in bytes. + - extension (str) - File extension, e.g. ``".lua"``. + + Example:: + + [ + {"name": "my_proto.lua", "size_bytes": 1024, "extension": ".lua"} + ] + """ + return jsonify(dissector_registry.list_dissectors()) + + +@app.post("/api/settings/dissectors") +def upload_dissector(): + """Upload a custom tshark dissector file (Lua script or native plugin). + + The file is saved to the server's dissectors directory and automatically + loaded by tshark on every subsequent packet-detail request. + + POST '/api/settings/dissectors' + + Request body (multipart/form-data): + - file (file) - The dissector file to upload. + Accepted extensions: ``.lua``, ``.so``, ``.dll``. + + Returns: + 201 Created: + JSON object describing the saved file: + + - name (str) - Saved filename. + - size_bytes (int) - File size in bytes. + - extension (str) - File extension. + + Example:: + + {"name": "my_proto.lua", "size_bytes": 1024, "extension": ".lua"} + + 400 Bad Request: + ``{"error": "no file provided"}`` — request contained no file part. + + ``{"error": "filename is required"}`` — file part had an empty name. + + ``{"error": "unsupported dissector extension …"}`` — the extension is + not in the allowed set (``.lua``, ``.so``, ``.dll``). + """ + if "file" not in request.files: + return _bad("no file provided") + + f = request.files["file"] + if not f.filename: + return _bad("filename is required") + + try: + data = f.read() + saved = dissector_registry.save(f.filename, data) + except ValueError as exc: + return _bad(str(exc)) + + return jsonify({ + "name": saved.name, + "size_bytes": saved.stat().st_size, + "extension": saved.suffix.lower(), + }), 201 + + +@app.delete("/api/settings/dissectors/") +def delete_dissector(filename: str): + """Remove a custom dissector file from the server. + + DELETE '/api/settings/dissectors/' + + Path parameters: + - filename (str) - Bare filename of the dissector to remove + (e.g. ``"my_proto.lua"``). Path separators are stripped server-side + to prevent directory-traversal attacks. + + Returns: + 200 OK: + ``{"deleted": true}`` — file existed and was removed. + + ``{"deleted": false}`` — no file with that name was found + (treated as a no-op rather than a 404 so repeated DELETE calls + are idempotent). + """ + deleted = dissector_registry.delete(filename) + return jsonify({"deleted": deleted}) + + +# ── login ───────────────────────────────────────────────────────────────────── + +@app.post("/api/auth/login") +def login(): + """Verify credentials and return an auth token for the session. + + Compares the SHA-256 hash of the submitted password against the hash + stored in 'settings.json'. Falls back to the default password + ('logoctopus') if no hash has been persisted yet. + + POST '/api/auth/login' + + Request body (JSON): + - username (str) - Must match the configured admin username. + - password (str) - Plain-text password; compared via SHA-256 hash. + + Returns: + 200 OK: + '{ "status": "ok", "token": "<32-char hex>" }' + + 401 Unauthorized: + '{ "error": "invalid credentials" }' + """ + body = request.get_json(force=True) + username = body.get("username", "").strip() + password = body.get("password", "") + + admin_user = os.getenv("ADMIN_USER", "admin") + if username != admin_user: + return _bad("invalid credentials", 401) + + settings = _load_settings() + default_pw_hash = hashlib.sha256(b"logoctopus").hexdigest() + stored_hash = settings.get("admin_password_hash", default_pw_hash) + submitted_hash = hashlib.sha256(password.encode()).hexdigest() + + if submitted_hash != stored_hash: + return _bad("invalid credentials", 401) + + token = uuid.uuid4().hex + settings.setdefault("auth_tokens", []) + settings["auth_tokens"] = (settings["auth_tokens"] + [token])[-10:] + _save_settings(settings) + + return jsonify({"status": "ok", "token": token}) + + +@app.post("/api/auth/logout") +def logout(): + """Invalidate the current auth token. + + POST '/api/auth/logout' + + Request body (JSON): + - token (str) - The token to revoke. + + Returns: + 200 OK: '{ "status": "ok" }' + """ + body = request.get_json(force=True) + token = body.get("token", "") + + settings = _load_settings() + tokens = settings.get("auth_tokens", []) + if token in tokens: + tokens.remove(token) + settings["auth_tokens"] = tokens + _save_settings(settings) + + return jsonify({"status": "ok"}) diff --git a/backend/models/device_config.py b/backend/models/device_config.py index f239b9d..a2e9ea7 100644 --- a/backend/models/device_config.py +++ b/backend/models/device_config.py @@ -25,6 +25,15 @@ def __init__(self, file_content_str): self.device_config = None def save_config_file(self, file_content_str): + """ + Save JSON config file to '/tmp/' directory under generated device config ID + + Args: + file_content_str (str): Config file in raw str format. + + Returns: + str: Unique device config ID. + """ decoded = base64.b64decode(file_content_str) device_config_id = self.get_device_config_id(json.loads(decoded)) with open(f"/tmp/{device_config_id}.json", "wb") as f: @@ -78,6 +87,13 @@ def remove_device_config(self): logging.error("Configuraiton file -> '%s' not exists ", self.device_config_path) def update_runtime_parameter(self, key, value): + """ + Update runtime parmater in target device config. + + Args: + key (str): Runtime paramter key. + value (str): Runtime paramter value. + """ config_path = self.device_config_path with open(config_path, "r", encoding="utf-8") as f: data = json.load(f) @@ -86,6 +102,15 @@ def update_runtime_parameter(self, key, value): json.dump(data, f, indent=2) def get_device_config_id(self, device_config): + """ + Generate unique device config ID based on config content. + + Args: + device_config (dict): Full device config in dict format. + + Returns: + str: Unique device config ID. + """ for not_const_key in list(self.watchdog_data.keys()): if not_const_key in device_config.keys(): device_config.pop(not_const_key) diff --git a/backend/models/log_snapshot.py b/backend/models/log_snapshot.py index 471052e..708661c 100644 --- a/backend/models/log_snapshot.py +++ b/backend/models/log_snapshot.py @@ -11,10 +11,11 @@ class LogSnapshot: """ A class to perform basic operations on collected logs. """ - def __init__(self, device_id, device_name, log_name, session_id, session_scenario, data_unit, log_type, collected_data, loaded_from_file=False): + def __init__(self, device_id, device_name, log_name, log_description, session_id, session_scenario, data_unit, log_type, collected_data, loaded_from_file=False): self.device_id = device_id self.device_name = device_name self.log_name = log_name + self.log_description = log_description self.id = hashlib.md5(f"{device_id}_{log_name}_{session_id}".encode()).hexdigest()[:16] self.collected_data = collected_data self.creation_time =datetime.now() @@ -93,6 +94,7 @@ def create_parquet_data_file(self): "data_unit": self.data_unit, "log_type": self.log_type, "device_name": self.device_name, + "log_description": self.log_description } collected_data_table = pa.Table.from_pandas(self.collected_data) existing_metadata = collected_data_table.schema.metadata or {} diff --git a/backend/services/device_watchdog.py b/backend/services/device_watchdog.py index 247fbe5..b40ec73 100644 --- a/backend/services/device_watchdog.py +++ b/backend/services/device_watchdog.py @@ -2,6 +2,8 @@ from concurrent.futures import ThreadPoolExecutor from backend.models.log_snapshot import LogSnapshot from backend.utils.fabric_connection import build_fabric_connection +from backend.utils.ssh_network_capture import SshNetworkCapture +from backend.utils.pcap_decoder import PcapDecoder import pandas as pd from paramiko_expect import SSHClientInteraction from datetime import datetime @@ -10,6 +12,7 @@ import uuid import time import threading +import os from time import sleep import argparse import json @@ -20,30 +23,19 @@ class DeviceWatchdog: A class used to collect logs for a target device using a defined configuration. """ - # Maximum rows kept per log before old entries are trimmed (memory guard) MAX_LOG_ROWS = 50_000 def __init__(self, device_config, device_config_id): - """ - Initializes a DeviceWatchdog instance. - - Args: - device_config (dict): Connection and logging configuration for the target device. - device_config_id (str): Unique identifier for this device config. - """ self.device_config = device_config self.device_config_id = device_config_id - - # SSH channels keyed by log_name; guarded by a per-channel lock + self.device_data_dir = os.path.join("data", str(device_config_id)) + os.makedirs(self.device_data_dir, exist_ok=True) self.ssh_channels: dict[str, Connection] = {} self._channel_lock = threading.Lock() - - # Log data stored as lists of dicts; converted to DataFrames on demand self.collected_data: dict[str, list[dict]] = {} - - # Errors collected as a plain list to avoid repeated pd.concat overhead + self._data_locks: dict[str, threading.Lock] = {} self._error_list: list[dict] = [] - + self._error_lock = threading.Lock() self.collection_ongoing = False self.thread: threading.Thread | None = None self.collection_stop_event: threading.Event | None = None @@ -51,55 +43,49 @@ def __init__(self, device_config, device_config_id): self.log_snapshots: list[LogSnapshot] = [] self.connection_status = False self.log_access = False - - # Pre-build lookup structures from config for O(1) access - self._log_config_map: dict[str, dict] = { - lc["log_name"]: lc for lc in device_config["log_file_configs"] - } - # Pre-compile per-log regexes once + self._log_config_map: dict[str, dict] = {lc["log_name"]: lc for lc in device_config["log_file_configs"]} self._log_regex_map: dict[str, re.Pattern] = { lc["log_name"]: re.compile(lc["data_extraction_regex"]) for lc in device_config["log_file_configs"] } + self._executor = ThreadPoolExecutor(max_workers=len(device_config["log_file_configs"])) + self.network_capture = None + self.packets_capture_file: str | None = None - # Persistent thread pool sized to the number of log sources - self._executor = ThreadPoolExecutor( - max_workers=len(device_config["log_file_configs"]) - ) + def _get_or_create_channel(self, ssh_channel_id: str) -> Connection: + """ + Return an existing SSH channel or create one, thread-safely. - # ------------------------------------------------------------------ - # SSH / command execution - # ------------------------------------------------------------------ + Args: + ssh_channel_id (str): SSH channel ID. - def _get_or_create_channel(self, ssh_channel_id: str) -> Connection: - """Return an existing SSH channel or create one, thread-safely.""" - if ssh_channel_id in self.ssh_channels: - return self.ssh_channels[ssh_channel_id] + Returns: + (Connection): SSH Fabric connection object. + """ with self._channel_lock: - # Double-checked locking if ssh_channel_id not in self.ssh_channels: conn = self.create_device_connection() conn.open() self.ssh_channels[ssh_channel_id] = conn - return self.ssh_channels[ssh_channel_id] + return self.ssh_channels[ssh_channel_id] def execute_cmd(self, cmd: str | None, ssh_channel_id: str, custom_shell_prompt: str | None = None) -> str | None: """ Execute a command via SSH on the target device. Args: - cmd: Command string to run. - ssh_channel_id: SSH channel identifier (usually log_name). - custom_shell_prompt: Shell prompt to expect when using a custom shell. + cmd (str): Command string to run. + ssh_channel_id (str): SSH channel identifier (usually log_name). + custom_shell_prompt (str): Shell prompt to expect when using a custom shell. Returns: - Full command stdout on success, or None on failure. + (str): Full command stdout on success, or None on failure. """ if cmd is None: return None try: channel = self._get_or_create_channel(ssh_channel_id) - + self.ssh_channels[ssh_channel_id].open() if custom_shell_prompt: interact = SSHClientInteraction(channel.client, timeout=20, display=False) interact.expect(custom_shell_prompt) @@ -128,40 +114,94 @@ def execute_cmd(self, cmd: str | None, ssh_channel_id: str, custom_shell_prompt: return None def _record_error(self, error_info: str) -> None: - """Append an error entry cheaply (list-based, no pd.concat).""" - self._error_list.append({"time": datetime.now(), "error_info": error_info}) + """ + Append an error entry thread-safely. + + Args: + error_info (str): Error info string. + """ + with self._error_lock: + self._error_list.append({"time": datetime.now(), "error_info": error_info}) @property def errors(self) -> pd.DataFrame: - """Return errors as a DataFrame (built lazily on access).""" - return pd.DataFrame(self._error_list) if self._error_list else pd.DataFrame({"time": [], "error_info": []}) + """ + Return errors as a DataFrame. - # ------------------------------------------------------------------ - # Log collector lifecycle - # ------------------------------------------------------------------ + Returns: + (pd.DataFrame): DataFrame object with full error info list. + """ + with self._error_lock: + snapshot = list(self._error_list) + return pd.DataFrame(snapshot) if snapshot else pd.DataFrame({"time": [], "error_info": []}) def initialize_log_collectors(self) -> None: - """Initialize log collectors for all defined log file configs.""" + """ + Initialize log collectors for all defined log file configs. + """ for log_config in self.device_config["log_file_configs"]: + log_name = log_config["log_name"] self.execute_cmd( log_config.get("log_activation_cmd"), - log_config["log_name"], + log_name, log_config.get("custom_shell_prompt"), ) - self.collected_data[log_config["log_name"]] = [] + self.collected_data[log_name] = [] + self._data_locks[log_name] = threading.Lock() + if self.device_config.get("packets_capture_config", None): + packets_capture_channel = self._get_or_create_channel("packet_capture") + capture_config = self.device_config["packets_capture_config"] + capture_start_cmd = capture_config["capture_start_cmd"] + capture_stop_cmd = capture_config["capture_stop_cmd"] + packets_capture_file = os.path.join(self.device_data_dir, "capture.pcap") + self.packets_capture_file = packets_capture_file + max_pcap_size_mb = capture_config.get("max_pcap_size_mb") + max_file_size_bytes = ( + int(max_pcap_size_mb * 1024 * 1024) if max_pcap_size_mb else None + ) + self.network_capture = SshNetworkCapture( + connection=packets_capture_channel, + capture_start_cmd=capture_start_cmd, + capture_stop_cmd=capture_stop_cmd, + local_file=packets_capture_file, + max_file_size_bytes=max_file_size_bytes, + on_limit_reached=lambda: self._record_error( + f"network capture stopped automatically: reached max " + f"pcap size of {max_pcap_size_mb} MB" + ), + ) + self.network_capture.start() def teardown_log_collectors(self) -> None: - """Teardown log collectors for all defined log file configs.""" + """ + Teardown log collectors and close all open SSH channels. Sends any configured deactivation + command on each channel before closing it, then clears the channel registry so that future + calls will open fresh connections. + """ for log_config in self.device_config["log_file_configs"]: self.execute_cmd( log_config.get("log_deactivation_cmd"), log_config["log_name"], log_config.get("custom_shell_prompt"), ) + if self.device_config.get("packets_capture_config", None): + self.network_capture.stop() + with self._channel_lock: + for conn in self.ssh_channels.values(): + try: + conn.close() + except Exception: + pass + self.ssh_channels.clear() + + def close(self) -> None: + """ + Shut down the thread-pool executor. - # ------------------------------------------------------------------ - # Log collection - # ------------------------------------------------------------------ + Call this when the DeviceWatchdog is no longer needed (e.g. when + a device is removed) to release the underlying worker threads. + """ + self._executor.shutdown(wait=False) def get_log_file_content(self, log_config: dict) -> None: """ @@ -169,9 +209,11 @@ def get_log_file_content(self, log_config: dict) -> None: Uses a pre-compiled regex and tracks the last-seen timestamp per log to avoid scanning the full history for duplicates on every poll. + Each log's list is protected by its own lock so concurrent pollers + do not race on reads or appends. Args: - log_config: Log collector configuration dict. + log_config (dict): Log collector configuration dict. """ log_name = log_config["log_name"] pattern = self._log_regex_map[log_name] @@ -184,36 +226,37 @@ def get_log_file_content(self, log_config: dict) -> None: if not raw: return - existing: list[dict] = self.collected_data[log_name] - - # Track the latest timestamp we already have so we can skip old lines - last_ts: datetime | None = existing[-1]["time"] if existing else None - - new_entries: list[dict] = [] - for line in raw.splitlines(): - m = pattern.search(line) - if not m: - continue - ts = parser.parse(m.group("TIME")) - if last_ts is None or ts > last_ts: - new_entries.append({"time": ts, "content": m.group("ENTRY")}) + lock = self._data_locks.get(log_name) + if lock is None: + return - if new_entries: - existing.extend(new_entries) - # Memory guard: trim to the most recent MAX_LOG_ROWS rows - if len(existing) > self.MAX_LOG_ROWS: - self.collected_data[log_name] = existing[-self.MAX_LOG_ROWS:] + with lock: + existing: list[dict] = self.collected_data[log_name] + last_ts: datetime | None = existing[-1]["time"] if existing else None + new_entries: list[dict] = [] + for line in raw.splitlines(): + m = pattern.search(line) + if not m: + continue + ts = parser.parse(m.group("TIME")) + if last_ts is None or ts > last_ts: + new_entries.append({"time": ts, "content": m.group("ENTRY")}) + + if new_entries: + existing.extend(new_entries) + if len(existing) > self.MAX_LOG_ROWS: + self.collected_data[log_name] = existing[-self.MAX_LOG_ROWS:] def get_all_log_files_content(self) -> None: - """Fetch all logs concurrently using the persistent thread pool.""" + """ + Fetch all logs concurrently using the persistent thread pool. + """ list(self._executor.map(self.get_log_file_content, self.device_config["log_file_configs"])) - # ------------------------------------------------------------------ - # Collection loop - # ------------------------------------------------------------------ - def start_logs_collection(self) -> None: - """Start the background log-collection thread.""" + """ + Start the background log-collection thread. + """ self.collection_ongoing = True self.collection_stop_event = threading.Event() self.cutoff_time = pd.Timestamp.now() @@ -225,10 +268,15 @@ def start_logs_collection(self) -> None: self.thread.start() def stop_logs_collection(self) -> None: - """Signal the collection thread to stop and wait for it to finish.""" + """ + Signal the collection thread to stop and wait for it to finish. + """ + if not self.collection_ongoing or self.collection_stop_event is None: + return self.collection_stop_event.set() self.collection_ongoing = False - self.thread.join() + if self.thread is not None: + self.thread.join() self.remove_all_outdated_entries() self.teardown_log_collectors() @@ -236,31 +284,40 @@ def logs_collection_loop(self, interval: int) -> None: """ Background loop: collect logs, wait for the interval, repeat. - Checks the stop event *before* each collection so a stop request is - honoured without waiting for the next network round-trip. - Args: - interval: Seconds between log fetches. + interval (int): Seconds between log fetches. """ while not self.collection_stop_event.is_set(): self.get_all_log_files_content() - # Wait for interval or until stop is requested self.collection_stop_event.wait(timeout=interval) - # ------------------------------------------------------------------ - # Post-collection processing - # ------------------------------------------------------------------ - def remove_all_outdated_entries(self) -> None: - """Drop log entries older than the collection start time and sort by time.""" + """ + Drop log entries older than the collection start time and sort by time. + """ cutoff = self.cutoff_time - for log_name, entries in self.collected_data.items(): - filtered = [e for e in entries if pd.Timestamp(e["time"]) >= cutoff] - self.collected_data[log_name] = sorted(filtered, key=lambda e: e["time"]) + for log_name, lock in self._data_locks.items(): + with lock: + entries = self.collected_data.get(log_name, []) + filtered = [e for e in entries if e["time"] >= cutoff.to_pydatetime()] + self.collected_data[log_name] = sorted(filtered, key=lambda e: e["time"]) def _entries_to_dataframe(self, log_name: str) -> pd.DataFrame: - """Convert the internal list-of-dicts for a log into a DataFrame.""" - entries = self.collected_data.get(log_name, []) + """ + Convert the internal list-of-dicts for a log into a DataFrame. + + Args: + log_name (str): Target log name. + + Returns: + (pd.DataFrame): DataFrame object with full log content. + """ + lock = self._data_locks.get(log_name) + if lock is not None: + with lock: + entries = list(self.collected_data.get(log_name, [])) + else: + entries = self.collected_data.get(log_name, []) return pd.DataFrame(entries) if entries else pd.DataFrame({"time": [], "content": []}) def save_log_snapshots(self, session_id: str, session_scenario: str) -> None: @@ -268,21 +325,23 @@ def save_log_snapshots(self, session_id: str, session_scenario: str) -> None: Persist collected logs as LogSnapshot objects. Args: - session_id: Unique logs collection session ID. - session_scenario: Scenario ID for this session. + session_id (str): Unique logs collection session ID. + session_scenario (str): Scenario ID for this session. """ for log_name, entries in self.collected_data.items(): if not entries: continue - log_config = self._log_config_map[log_name] # O(1) lookup + log_config = self._log_config_map[log_name] log_type = log_config.get("log_type", "") data_unit = log_config.get("data_unit", "") + log_description = log_config.get("log_description", "") log_content = self._entries_to_dataframe(log_name) self.log_snapshots.append( LogSnapshot( self.device_config_id, self.device_config["device_name"], log_name, + log_description, session_id, session_scenario, data_unit, @@ -290,32 +349,57 @@ def save_log_snapshots(self, session_id: str, session_scenario: str) -> None: log_content, ) ) + if (self.packets_capture_file and os.path.exists(self.packets_capture_file) and os.path.getsize(self.packets_capture_file) > 0): + session_pcap_path = os.path.join(self.device_data_dir, f"{session_id}.pcap") + try: + os.replace(self.packets_capture_file, session_pcap_path) + self.packets_capture_file = session_pcap_path + except OSError as exc: + self._record_error(f"failed to rename pcap file -> {exc}") + try: + decoder = PcapDecoder(self.packets_capture_file) + self.log_snapshots.append( + decoder.to_log_snapshot( + device_config_id=self.device_config_id, + device_name=self.device_config["device_name"], + session_id=session_id, + session_scenario=session_scenario, + log_description=self.device_config["packets_capture_config"]["capture_description"] + ) + ) + except FileNotFoundError as exc: + self._record_error(f"pcap decode failed -> {exc}") def get_target_log_param_based_on_log_name(self, log_name: str, log_param: str) -> str: """ Get a log config parameter by log name in O(1) via the pre-built map. Args: - log_name: Log name to look up. - log_param: Config key to retrieve. + log_name (str): Log name to look up. + log_param (str): Config key to retrieve. Returns: - Parameter value, or empty string if not found. + (str): Parameter value, or empty string if not found. """ return self._log_config_map.get(log_name, {}).get(log_param, "") - # ------------------------------------------------------------------ - # Status checks - # ------------------------------------------------------------------ - def get_connection_status(self) -> None: - """Update connection_status based on the first SSH channel.""" - first_log_name = self.device_config["log_file_configs"][0]["log_name"] - channel = self.ssh_channels.get(first_log_name) - self.connection_status = bool(channel and channel.is_connected) + """ + Update 'connection_status' based on all SSH channels. + """ + with self._channel_lock: + channels = dict(self.ssh_channels) + + if not channels: + self.connection_status = False + return + + self.connection_status = all(bool(ch and ch.is_connected) for ch in channels.values()) def test_log_files_access(self) -> None: - """Check whether the first configured log file is accessible via SSH.""" + """ + Check whether the first configured log file is accessible via SSH. + """ log_file_config = self.device_config["log_file_configs"][0] result = self.execute_cmd( log_file_config["log_file_cmd"], @@ -324,43 +408,25 @@ def test_log_files_access(self) -> None: ) self.log_access = bool(result) - # ------------------------------------------------------------------ - # Connection factories - # ------------------------------------------------------------------ - def create_device_connection(self) -> Connection: """ Create a Fabric SSH Connection from the device config. - Uses the shared connection builder (backend.utils.fabric_connection), - the same one used by the config builder's Test Connection / Exec - Command endpoints in app.py. This guarantees that any connection - which passed "Test Connection" in the UI — including the SSH key - type detection (RSA / Ed25519 / ECDSA / DSS) and passphrase - handling — behaves identically here. Accepts either the nested - ``gateway: {...}`` shape (as saved by the config builder) or a flat - ``gateways: [...]`` list. - Returns: - Configured Fabric Connection object. + (Connection): Configured Fabric Connection object. """ return build_fabric_connection(self.device_config) - -# --------------------------------------------------------------------------- -# Config file helpers -# --------------------------------------------------------------------------- - def get_current_device_config(path_to_config_file: str) -> dict: """ Load the device JSON configuration file. Args: - path_to_config_file: Path to the JSON config file. + path_to_config_file (str): Path to the JSON config file. Returns: - Parsed configuration dict. + (dict): Parsed configuration dict. """ with open(path_to_config_file, "r", encoding="utf-8") as f: return json.load(f) @@ -368,12 +434,11 @@ def get_current_device_config(path_to_config_file: str) -> dict: def update_device_config_parameters(path_to_config_file: str, updates: dict) -> None: """ - Apply multiple key/value updates to the device config file in a single - read–write cycle (reduces disk I/O compared to one write per parameter). + Apply multiple key/value updates to the device config file in a single read-write cycle. Args: - path_to_config_file: Path to the JSON config file. - updates: Dict of {key: value} pairs to apply. + path_to_config_file (str): Path to the JSON config file. + updates (dict): Dict of {key: value} pairs to apply. """ with open(path_to_config_file, "r", encoding="utf-8") as f: data = json.load(f) @@ -382,15 +447,6 @@ def update_device_config_parameters(path_to_config_file: str, updates: dict) -> json.dump(data, f, indent=2) -# Keep the single-key variant for backward compatibility -def update_device_config_parameter(path_to_config_file: str, key: str, value) -> None: - update_device_config_parameters(path_to_config_file, {key: value}) - - -# --------------------------------------------------------------------------- -# Entry point -# --------------------------------------------------------------------------- - if __name__ == "__main__": arg_parser = argparse.ArgumentParser(description="Device watchdog") arg_parser.add_argument("device_config_file_path", help="Path to target device config file") @@ -398,17 +454,18 @@ def update_device_config_parameter(path_to_config_file: str, key: str, value) -> init_device_config = get_current_device_config(args.device_config_file_path) device_watchdog = DeviceWatchdog(init_device_config, args.device_config_file_path.split("/")[1]) + auto_collection_timer = 0.0 + auto_collection_armed = False + last_errors_len = 0 while True: current_device_config = get_current_device_config(args.device_config_file_path) - # --- Start collection when requested --- if current_device_config["logs_collection"] and not device_watchdog.collection_ongoing: device_watchdog.initialize_log_collectors() device_watchdog.start_logs_collection() - # --- Stop collection when requested --- if not current_device_config["logs_collection"] and device_watchdog.collection_ongoing: device_watchdog.stop_logs_collection() device_watchdog.save_log_snapshots( @@ -419,11 +476,12 @@ def update_device_config_parameter(path_to_config_file: str, key: str, value) -> args.device_config_file_path, {"current_session_id": "no_active_session"}, ) + auto_collection_armed = False sleep(2) - # --- Auto-collection: arm --- - if current_device_config["auto_collection_enabled"] and not device_watchdog.collection_ongoing: + if (current_device_config["auto_collection_enabled"] and not device_watchdog.collection_ongoing and not auto_collection_armed): auto_collection_timer = time.time() + auto_collection_armed = True update_device_config_parameters( args.device_config_file_path, { @@ -432,18 +490,12 @@ def update_device_config_parameter(path_to_config_file: str, key: str, value) -> }, ) - # --- Auto-collection: disarm after interval --- - if ( - current_device_config["auto_collection_enabled"] - and time.time() - auto_collection_timer - > current_device_config["auto_collection_interval"] * 3600 - ): + if (current_device_config["auto_collection_enabled"] and auto_collection_armed and time.time() - auto_collection_timer > current_device_config["auto_collection_interval"] * 3600): update_device_config_parameters( args.device_config_file_path, {"logs_collection": False}, ) - # --- Status probe & single-write config update --- device_watchdog.test_log_files_access() device_watchdog.get_connection_status() @@ -455,8 +507,10 @@ def update_device_config_parameter(path_to_config_file: str, key: str, value) -> }, ) - # --- Persist errors --- - errors_file_path = f"data/{device_watchdog.device_config_id}/errors.feather" - device_watchdog.errors.to_feather(errors_file_path) + current_errors_len = len(device_watchdog._error_list) + if current_errors_len != last_errors_len: + errors_file_path = os.path.join(device_watchdog.device_data_dir, "errors.feather") + device_watchdog.errors.to_feather(errors_file_path) + last_errors_len = current_errors_len sleep(5) diff --git a/backend/utils/config_helper.py b/backend/utils/log_snapshots_helper.py similarity index 97% rename from backend/utils/config_helper.py rename to backend/utils/log_snapshots_helper.py index 62b4c74..8cbb43f 100644 --- a/backend/utils/config_helper.py +++ b/backend/utils/log_snapshots_helper.py @@ -1,6 +1,9 @@ import pandas as pd -class ConfigurationHelper: +class LogSnapshotsHelper: + """ + A class to perform basic operations on device confiu. + """ @staticmethod def get_log_content_for_selected_snapshots(selected_log_snapshots): diff --git a/backend/utils/log_snapshots_loader.py b/backend/utils/log_snapshots_loader.py index 7db2252..48da9d9 100644 --- a/backend/utils/log_snapshots_loader.py +++ b/backend/utils/log_snapshots_loader.py @@ -25,8 +25,16 @@ def load_log_snapshots_from_file(self, log_snapshot_path): k.decode(): v.decode() for k, v in raw_meta.items() } - return LogSnapshot(self.device_config_id, log_metadata["device_name"], log_metadata["log_name"], log_metadata["session_id"], log_metadata["session_scenario"], log_metadata["data_unit"], log_metadata["log_type"], pyarrow_table.to_pandas(), True) - + return LogSnapshot(self.device_config_id, + log_metadata["device_name"], + log_metadata["log_name"], + log_metadata["log_description"], + log_metadata["session_id"], + log_metadata["session_scenario"], + log_metadata["data_unit"], + log_metadata["log_type"], + pyarrow_table.to_pandas(), + True) def load_all_log_snapshots(self): """ diff --git a/backend/utils/pcap_decoder.py b/backend/utils/pcap_decoder.py new file mode 100644 index 0000000..8bcd771 --- /dev/null +++ b/backend/utils/pcap_decoder.py @@ -0,0 +1,413 @@ +from __future__ import annotations +import json +import logging +import os +import shutil +import subprocess +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +import pandas as pd +from backend.models.log_snapshot import LogSnapshot + +logger = logging.getLogger(__name__) + +_TSHARK_FIELDS = [ + "frame.time_epoch", + "frame.number", + "frame.len", + "frame.protocols", + "ip.src", + "ip.dst", + "ipv6.src", + "ipv6.dst", + "tcp.srcport", + "tcp.dstport", + "udp.srcport", + "udp.dstport", + "_ws.col.Info", +] + +DISSECTOR_EXTENSIONS = {".lua", ".so", ".dll"} + +@dataclass +class PacketInfo: + """Decoded representation of a single captured packet.""" + + number: int + time: datetime + length: int + protocols: str + src_ip: str + dst_ip: str + src_port: str + dst_port: str + info: str + + def to_content_str(self) -> str: + """ + Convert PacketInfo object to single string with all params divided by '|'. + + Returns: + (str): String with all PacketInfo params divided by '|' + """ + endpoints = f"{self.src_ip}:{self.src_port} -> {self.dst_ip}:{self.dst_port}" + parts = [ + str(self.number), + self.protocols, + endpoints, + f"len={self.length}", + self.info, + ] + return " | ".join(p for p in parts if p) + + +class DissectorRegistry: + """ + Manages a directory of custom tshark dissector files (Lua plugins, shared + libraries) that the user uploads through the settings UI. + """ + + def __init__(self, dissectors_dir: str | os.PathLike): + self.dissectors_dir = Path(dissectors_dir) + self.dissectors_dir.mkdir(parents=True, exist_ok=True) + + def save(self, filename: str, data: bytes) -> Path: + """ + Persist a dissector file. + + Args: + filename (str): Bare filename of dissector (e.g. ``"my_proto.lua"``). + data (bytes): Raw dissector file bytes. + + Returns: + (str): Path to the saved file. + + Raises: + ValueError: The filename has an unsupported extension. + """ + safe_name = Path(filename).name # strip any path components + if Path(safe_name).suffix.lower() not in DISSECTOR_EXTENSIONS: + raise ValueError( + f"Unsupported dissector extension '{Path(safe_name).suffix}'. " + f"Allowed: {', '.join(sorted(DISSECTOR_EXTENSIONS))}" + ) + dest = self.dissectors_dir / safe_name + dest.write_bytes(data) + logger.info("Dissector saved: %s (%d bytes)", dest, len(data)) + return dest + + def delete(self, filename: str) -> bool: + """ + Remove a dissector file. + + Args: + filename (str): Bare dissector filename. + + Returns: + (bool): True if the file existed and was deleted, False otherwise. + """ + target = self.dissectors_dir / Path(filename).name + if target.exists(): + target.unlink() + logger.info("Dissector deleted: %s", target) + return True + return False + + def list_dissectors(self) -> list[dict]: + """ + List all dissector files in the registry. + + Returns: + (list): List of dicts with ``name``, ``size_bytes``, and ``extension`` keys, sorted alphabetically by name. + """ + entries = [] + for p in sorted(self.dissectors_dir.iterdir()): + if p.is_file() and p.suffix.lower() in DISSECTOR_EXTENSIONS: + entries.append( + { + "name": p.name, + "size_bytes": p.stat().st_size, + "extension": p.suffix.lower(), + } + ) + return entries + + def get_plugin_args(self) -> list[str]: + """ + Build the tshark command-line arguments that load all registered dissector files. + Lua scripts are loaded individually via ``-X lua_script:``. + Shared libraries (.so/.dll) are added via ``--plugin-path`` pointing + at the dissectors directory (tshark scans the directory for plugins). + + Returns: + (list): List of extra arguments to append to a tshark command. + """ + args: list[str] = [] + has_native = False + + for p in sorted(self.dissectors_dir.iterdir()): + if not p.is_file(): + continue + ext = p.suffix.lower() + if ext == ".lua": + args += ["-X", f"lua_script:{p}"] + elif ext in {".so", ".dll"}: + has_native = True + + if has_native: + args += ["--plugin-path", str(self.dissectors_dir)] + + return args + + +class PcapDecoder: + """ + Decodes a pcap file using tshark and exposes the packets both as + PacketInfo objects and as a "time" + "content" DataFrame. Custom dissectors + will be used to support extra protocol fields. + """ + + TSHARK_BIN = "tshark" + + def __init__(self, pcap_file: str, dissectors_dir: str | os.PathLike | DissectorRegistry | None = None): + self.pcap_file = pcap_file + self.packets: list[PacketInfo] = [] + if isinstance(dissectors_dir, DissectorRegistry): + self._registry: DissectorRegistry | None = dissectors_dir + elif dissectors_dir is not None: + self._registry = DissectorRegistry(dissectors_dir) + else: + self._registry = None + + @classmethod + def for_session(cls, device_data_dir: str, session_id: str, dissectors_dir: str | os.PathLike | DissectorRegistry | None = None) -> PcapDecoder: + """ + Build a PcapDecoder for a session's saved pcap file, i.e. + /.pcap -- the naming convention used by + DeviceWatchdog.save_log_snapshots once it renames the raw capture. + + Args: + device_data_dir (str): Per-device data directory. + session_id (str): Session whose pcap should be decoded. + dissectors_dir (str): Optional custom dissectors directory or registry. + + Returns: + (PcapDecoder): PcapDecoder class object. + """ + return cls(os.path.join(device_data_dir, f"{session_id}.pcap"), dissectors_dir=dissectors_dir) + + @classmethod + def get_session_packet_details(cls, device_data_dir: str, session_id: str, packet_number: int, dissectors_dir: str | os.PathLike | DissectorRegistry | None = None) -> dict: + """ + Convenience one-shot: resolve a session's pcap path and decode a + single packet's full field detail from it. + + Args: + device_data_dir (str): Per-device data directory. + session_id (str): Session whose pcap to decode. + packet_number (int): 1-based frame number. + dissectors_dir (str): Optional custom dissectors directory or registry. + + Returns: + (dict): Single packet detaile extraced with tshark in dict format. + + Raises: + FileNotFoundError: tshark is not installed / not on PATH. + """ + decoder = cls.for_session(device_data_dir, session_id, dissectors_dir=dissectors_dir) + if not os.path.exists(decoder.pcap_file): + return {} + return decoder.get_packet_details(packet_number) + + + def _plugin_args(self) -> list[str]: + """ + Return extra tshark args for custom dissectors (empty list if none). + + Returns: + (list): Extra tshark args for custom dissectors. + """ + if self._registry is None: + return [] + return self._registry.get_plugin_args() + + def _check_tshark(self) -> None: + """ + Raise FileNotFoundError if tshark is not on PATH + """ + if shutil.which(self.TSHARK_BIN) is None: + raise FileNotFoundError( + f"'{self.TSHARK_BIN}' not found on PATH; install Wireshark/tshark to decode pcaps." + ) + + def decode(self) -> list[PacketInfo]: + """ + Run tshark over the pcap file and populate self.packets. + + Parses whatever tshark writes to stdout on a best-effort basis; + a non-zero tshark exit status is not treated as fatal (tshark can + exit non-zero while still having emitted usable lines, e.g. for + warnings on link-layer types or a still-growing capture file). + + Custom dissectors registered in ``dissectors_dir`` are loaded + automatically; they do *not* affect the flat summary fields decoded + here but do enrich ``get_packet_details()`` output. + + Returns: + (list): The decoded list of PacketInfo objects (also stored on self.packets). + + Raises: + FileNotFoundError: tshark is not installed / not on PATH, or no pcap_file path was given. + """ + if not self.pcap_file: + raise FileNotFoundError("PcapDecoder was given no pcap_file path to decode.") + + self._check_tshark() + + cmd = [ + self.TSHARK_BIN, + "-r", self.pcap_file, + "-T", "fields", + "-E", "separator=\t", + "-E", "quote=n", + "-E", "occurrence=f", + ] + self._plugin_args() + + for f in _TSHARK_FIELDS: + cmd += ["-e", f] + + result = subprocess.run(cmd, capture_output=True, text=True) + + self.packets = [ + pkt + for pkt in (self._parse_line(line) for line in result.stdout.splitlines() if line.strip()) + if pkt is not None + ] + return self.packets + + def _parse_line(self, line: str) -> PacketInfo | None: + """ + Parse a single tab-separated tshark output line into a PacketInfo. + + Args: + line (str): Single tab-separated tshark output line. + + Returns: + (PacketInfo): PacketInfo object. + """ + fields = line.split("\t") + # Pad in case trailing empty fields were stripped by tshark. + fields += [""] * (len(_TSHARK_FIELDS) - len(fields)) + ( + time_epoch, number, length, protocols, + ip_src, ip_dst, ipv6_src, ipv6_dst, + tcp_sport, tcp_dport, udp_sport, udp_dport, + info, + ) = fields[: len(_TSHARK_FIELDS)] + + if not time_epoch: + return None + + try: + pkt_time = datetime.fromtimestamp(float(time_epoch)) + except ValueError: + return None + + return PacketInfo( + number=int(number) if number.isdigit() else 0, + time=pkt_time, + length=int(length) if length.isdigit() else 0, + protocols=protocols, + src_ip=ip_src or ipv6_src, + dst_ip=ip_dst or ipv6_dst, + src_port=tcp_sport or udp_sport, + dst_port=tcp_dport or udp_dport, + info=info, + ) + + def get_packet_details(self, packet_number: int) -> dict: + """ + Decode a single packet's full field detail into a nested dict. + + When custom dissectors are registered via ``dissectors_dir``, tshark + loads them before parsing so their protocol trees appear alongside the + standard layers in the returned dict. + + Args: + packet_number (int): Frame number ("packet index" shown in the "content" column). + + Returns: + (dict): Nested dict of every layer/field tshark parsed for that packet. + + Raises: + FileNotFoundError: tshark is not installed / not on PATH, or the pcap file doesn't exist. + """ + if not self.pcap_file or not os.path.exists(self.pcap_file): + raise FileNotFoundError(f"pcap file not found: {self.pcap_file}") + + self._check_tshark() + + cmd = [ + self.TSHARK_BIN, + "-r", self.pcap_file, + "-Y", f"frame.number=={packet_number}", + "-T", "json", + ] + self._plugin_args() + + result = subprocess.run(cmd, capture_output=True, text=True) + + stdout = result.stdout.strip() + if not stdout: + return {} + + try: + parsed = json.loads(stdout) + except json.JSONDecodeError: + return {} + + if not parsed: + return {} + + return parsed[0].get("_source", {}).get("layers", {}) + + def to_dataframe(self) -> pd.DataFrame: + """ + Build a "time" / "content" DataFrame from the decoded packets. + + Returns: + (pd.DataFrame): DataFrame object generated decoded packets. + """ + if not self.packets: + self.decode() + + if not self.packets: + return pd.DataFrame({"time": [], "content": []}) + + rows = [{"time": pkt.time, "content": pkt.to_content_str()} for pkt in self.packets] + return pd.DataFrame(rows) + + def to_log_snapshot(self, + device_config_id: str, + device_name: str, + session_id: str, + session_scenario: str, + log_name: str = "network capture", + log_description: str = "Decoded network packet capture", + data_unit: str = "", + log_type: str = "text") -> LogSnapshot: + """ + Wrap the decoded packets in a LogSnapshot, mirroring DeviceWatchdog.save_log_snapshots. + """ + log_content = self.to_dataframe() + return LogSnapshot( + device_config_id, + device_name, + log_name, + log_description, + session_id, + session_scenario, + data_unit, + log_type, + log_content, + ) diff --git a/backend/utils/ssh_network_capture.py b/backend/utils/ssh_network_capture.py new file mode 100644 index 0000000..94d300a --- /dev/null +++ b/backend/utils/ssh_network_capture.py @@ -0,0 +1,321 @@ +import threading +import time +import socket +import logging + +logger = logging.getLogger(__name__) + + +class SshNetworkCapture: + """ + A class used to start and stop network capture on targe remote device. + """ + def __init__(self, + connection, + capture_start_cmd, + capture_stop_cmd, + local_file="capture.pcap", + max_reconnect_attempts=5, + reconnect_backoff_base=1.0, + reconnect_backoff_max=30.0, + recv_size=65536, + max_file_size_bytes=None, + on_limit_reached=None): + """ + Initializes a SshNetworkCapture instance. + + Args: + connection (Connection): Connection to trigger SSH network capture on remote device. + capture_start_cmd (str): Command to start network capture on remote device. + capture_stop_cmd (str): Command to stop network capture on remote device. + local_file (str): Name for local file where pcap will be collected. + max_reconnect_attempts (int): Max number of reconnect attempts. + reconnect_backoff_base (float): Initial time for device to wait before next attempt to reconnect. + reconnect_backoff_max (float): Max time for device to wait before next attempt to reconnect. + recv_size (int): Size of buffer during network capture collection. + max_file_size_bytes (int): Max size of network capture file. + on_limit_reached (function): Function to trigger when max capture file limit is reached. + """ + self.conn = connection + self.capture_start_cmd = capture_start_cmd + self.capture_stop_cmd = capture_stop_cmd + self.local_file_base = local_file + self.session = None + self.thread = None + self.running = False + self.local_file = None + self._segment_index = 1 + self._lock = threading.RLock() + self.max_reconnect_attempts = max_reconnect_attempts + self.reconnect_backoff_base = reconnect_backoff_base + self.reconnect_backoff_max = reconnect_backoff_max + self.recv_size = recv_size + self.max_file_size_bytes = max_file_size_bytes + self.on_limit_reached = on_limit_reached + self._total_bytes_written = 0 + self.limit_reached = False + self._stopped_event = threading.Event() + + def start(self): + """ + Start network capture on remote device. + """ + with self._lock: + if self.running: + logger.warning("capture already running") + return + + self._segment_index = 1 + self.running = True + self._total_bytes_written = 0 + self.limit_reached = False + self._stopped_event.clear() + + self._open_segment(self.local_file_base) + self._open_session() + + self.thread = threading.Thread(target=self._reader_loop, daemon=True) + self.thread.start() + + logger.info("Network capture started -> %s (cmd=%r)", self.local_file_base, self.capture_start_cmd) + + def stop(self, timeout=10): + """ + Stop network capture on remote device. + + Args: + timeout (int): Number of seconds to wait for gracfull network capture stop. + """ + with self._lock: + if not self.running: + return + self.running = False + + self._signal_remote_tcpdump() + + if self.session is not None: + try: + self.session.close() + except Exception: + logger.exception("error closing capture session") + + if self.thread is not None: + self.thread.join(timeout=timeout) + if self.thread.is_alive(): + logger.warning("reader thread did not exit within %ss", timeout) + + if self.local_file is not None: + try: + self.local_file.flush() + self.local_file.close() + except Exception: + logger.exception("error closing local capture file") + self.local_file = None + + logger.info("capture stopped") + + def _open_segment(self, path): + """ + (Re)open the local file that capture bytes are written to. + + Args: + path (str): Path of segment local file. + """ + if self.local_file is not None: + try: + self.local_file.flush() + self.local_file.close() + except Exception: + logger.exception("error closing previous capture segment") + self.local_file = open(path, "wb") + + def _next_segment_path(self): + """ + Get next segment file path. + + Returns: + str: Path of next segment file. + """ + self._segment_index += 1 + return f"{self.local_file_base}.part{self._segment_index}" + + def _get_transport(self): + """ + Get paramiko transport object if connection is active. + + Returns: + object: Paramiko SSH transport object. + """ + if not getattr(self.conn, "is_connected", False): + self.conn.open() + + transport = self.conn.client.get_transport() + if transport is None or not transport.is_active(): + raise ConnectionError("SSH transport is not active") + return transport + + def _open_session(self): + """ + Open SSH capturing session. + """ + cmd = f"{self.capture_start_cmd} -w -" + transport = self._get_transport() + channel = transport.open_session() + channel.exec_command(cmd) + channel.settimeout(1.0) + self.session = channel + + def _signal_remote_tcpdump(self): + """ + Send target cmd to stop ongoing network capture. + """ + if not self.capture_stop_cmd: + logger.info("No stop command configured for this capture command; relying on channel close to terminate the remote process") + return + + try: + transport = self.conn.client.get_transport() + if transport is None or not transport.is_active(): + return + kill_cmd = self.capture_stop_cmd + channel = transport.open_session() + channel.exec_command(kill_cmd) + channel.settimeout(5) + try: + channel.recv_exit_status() + except Exception: + pass + channel.close() + except Exception: + logger.exception("failed to signal remote capture process to stop") + + def _stop_due_to_limit(self): + """ + Stop ongoing network capture when capture file size limit is reached. + """ + with self._lock: + if not self.running: + return + self.running = False + + logger.warning( + "pcap size limit reached (%s bytes >= %s byte limit); stopping capture", + self._total_bytes_written, self.max_file_size_bytes, + ) + + self._signal_remote_tcpdump() + + if self.session is not None: + try: + self.session.close() + except Exception: + logger.exception("error closing capture session") + + if self.local_file is not None: + try: + self.local_file.flush() + self.local_file.close() + except Exception: + logger.exception("error closing local capture file") + self.local_file = None + + self.limit_reached = True + if self.on_limit_reached is not None: + try: + self.on_limit_reached() + except Exception: + logger.exception("on_limit_reached callback failed") + + def _reader_loop(self): + """ + Main loop for SSH network capture colllector. + """ + consecutive_errors = 0 + + while True: + with self._lock: + if not self.running: + break + + try: + data = self.session.recv(self.recv_size) + except socket.timeout: + continue + except Exception: + logger.exception("recv() failed on capture session") + data = None + + with self._lock: + if not self.running: + break + + if data: + consecutive_errors = 0 + try: + self.local_file.write(data) + except Exception: + logger.exception("failed writing capture data to disk") + with self._lock: + self.running = False + break + + self._total_bytes_written += len(data) + if (self.max_file_size_bytes is not None and self._total_bytes_written >= self.max_file_size_bytes): + self._stop_due_to_limit() + break + continue + + consecutive_errors += 1 + if not self._attempt_reconnect(consecutive_errors): + logger.error("giving up after %s failed reconnect attempts", + consecutive_errors) + with self._lock: + self.running = False + break + + self._stopped_event.set() + + def _attempt_reconnect(self, attempt_number): + """ + Try to re-establish network capture collection with split new session to seperate file. + + Args: + attempt_number (int): Number of re-establish network capture collection attempt. + """ + if attempt_number > self.max_reconnect_attempts: + return False + + delay = min(self.reconnect_backoff_base * (2 ** (attempt_number - 1)), + self.reconnect_backoff_max) + logger.warning("capture connection lost, reconnect attempt %s/%s " + "in %.1fs", attempt_number, self.max_reconnect_attempts, + delay) + time.sleep(delay) + + with self._lock: + if not self.running: + return False + + try: + try: + self.session.close() + except Exception: + pass + + try: + self._get_transport() + except Exception: + try: + self.conn.close() + except Exception: + pass + self.conn.open() + + new_path = self._next_segment_path() + self._open_segment(new_path) + self._open_session() + logger.info("capture resumed -> new segment %s", new_path) + return True + except Exception: + logger.exception("reconnect attempt %s failed", attempt_number) + return False diff --git a/frontend/src/LogOctopus.jsx b/frontend/src/LogOctopus.jsx index d8adfd2..f206d0e 100644 --- a/frontend/src/LogOctopus.jsx +++ b/frontend/src/LogOctopus.jsx @@ -1,4 +1,4 @@ -import { useState, useEffect, useCallback, useRef } from "react"; +import { useState, useEffect, useCallback, useRef, useMemo } from "react"; // ── CONFIG ──────────────────────────────────────────────────────────────────── const API_BASE = (import.meta.env.VITE_API_BASE) || "http://localhost:8050" @@ -22,41 +22,76 @@ async function apiFetch(path, options = {}) { } // ── AUTH CONTEXT ────────────────────────────────────────────────────────────── -// Simple client-side auth gate. In production, back this with a real session/JWT. -// Default credentials come from env vars; admin password can be changed at runtime -// and is persisted in localStorage so it survives page refreshes. -const ADMIN_USER_DEFAULT = import.meta?.env?.VITE_ADMIN_USER || "admin"; -const ADMIN_PASS_DEFAULT = import.meta?.env?.VITE_ADMIN_PASS || "logoctopus"; +// Server-side auth: credentials are validated by POST /api/auth/login which +// compares a SHA-256 hash against the one stored in settings.json. The server +// returns a random token that is kept in sessionStorage (tab-scoped, never +// persisted to disk by the browser). +// +// FIX: the previous implementation stored the admin password in plain-text in +// localStorage and performed all comparison client-side, meaning any XSS or +// browser extension could trivially extract the password. Auth is now backed +// by the backend's /api/auth/login and /api/auth/logout endpoints. function useAuth() { - const [role, setRole] = useState(() => sessionStorage.getItem("lo_role") || "guest"); - // Password persisted in localStorage so changes survive refreshes. - const [adminPass, setAdminPassState] = useState( - () => localStorage.getItem("lo_admin_pass") || ADMIN_PASS_DEFAULT - ); + const [role, setRole] = useState(() => sessionStorage.getItem("lo_role") || "guest"); + const [token, setToken] = useState(() => sessionStorage.getItem("lo_token") || ""); - const login = (user, pass) => { - if (user === ADMIN_USER_DEFAULT && pass === adminPass) { - sessionStorage.setItem("lo_role", "admin"); - setRole("admin"); - return true; - } - return false; + // login: calls the backend; returns true on success, false on bad credentials, + // throws on network error so the caller can show a toast. + const login = async (user, pass) => { + const res = await fetch(`${API_BASE}/api/auth/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ username: user, password: pass }), + }); + if (res.status === 401) return false; + if (!res.ok) throw new Error(`Login failed (${res.status})`); + const { token: tok } = await res.json(); + sessionStorage.setItem("lo_role", "admin"); + sessionStorage.setItem("lo_token", tok); + setRole("admin"); + setToken(tok); + return true; }; const logout = () => { + const tok = sessionStorage.getItem("lo_token") || ""; sessionStorage.removeItem("lo_role"); + sessionStorage.removeItem("lo_token"); setRole("guest"); + setToken(""); + // Best-effort server-side token revocation + if (tok) { + fetch(`${API_BASE}/api/auth/logout`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ token: tok }), + }).catch(() => {}); + } }; - const changePassword = (currentPass, newPass) => { - if (currentPass !== adminPass) return false; - localStorage.setItem("lo_admin_pass", newPass); - setAdminPassState(newPass); + // changePassword: validates the current password server-side, then updates + // the hash via the existing /api/settings/change-password endpoint. + // Returns true on success, false if currentPass is wrong, throws on error. + const changePassword = async (currentPass, newPass) => { + // Re-authenticate to verify the current password before allowing a change. + const adminUser = import.meta?.env?.VITE_ADMIN_USER || "admin"; + const checkRes = await fetch(`${API_BASE}/api/auth/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ username: adminUser, password: currentPass }), + }); + if (checkRes.status === 401) return false; + if (!checkRes.ok) throw new Error(`Verification failed (${checkRes.status})`); + + await apiFetch("/api/settings/change-password", { + method: "POST", + body: JSON.stringify({ new_password: newPass }), + }); return true; }; - return { role, isAdmin: role === "admin", login, logout, changePassword }; + return { role, isAdmin: role === "admin", token, login, logout, changePassword }; } // ── PLOTLY CHART PANEL ──────────────────────────────────────────────────────── @@ -418,6 +453,46 @@ function buildPairColorMap(rows) { return map; } +// ── PACKET-CAPTURE GLYPH BUTTON ─────────────────────────────────────────────── +// Singleton + + {/* Concentric rings + logo */} +
+ + + + + + +
+ + + + {[0, 45, 90, 135, 180, 225, 270, 315].map((a, i) => { + const rad = (a * Math.PI) / 180; + return ; + })} + +
+
+ + {/* Title */} +
+ Loading Logs +
+ + {/* Progress bar */} +
+
+ {pct !== null ? ( +
+ ) : ( +
+ )} +
+ + {/* Counts row */} +
+ + {done > 0 ? ( + <>{done} + {total > 0 ? ` / ${total} snapshot${total !== 1 ? "s" : ""} loaded` : ` snapshot${done !== 1 ? "s" : ""} loaded`} + ) : ( + "Waiting for response…" + )} + + {pct !== null && ( + + {pct}% + + )} +
+
+ + {/* Cycling status message */} +
+ {LOG_LOAD_MESSAGES[msgIdx]} +
+ + {/* Bouncing dots */} +
+ {[0, 1, 2].map(i => ( +
+ ))} +
+
+ ); +} + // ── MAIN APP ────────────────────────────────────────────────────────────────── const PULSE_KF = ` @keyframes pulse { 0%,100%{opacity:1;transform:scale(1)} 50%{opacity:0.4;transform:scale(1.6)} } @@ -3804,6 +4655,7 @@ export default function App() { const [devices, setDevices] = useState([]); const [devicesLoading, setDevicesLoading] = useState(true); + const [systemStats, setSystemStats] = useState(null); const [selectedDevices, setSelectedDevices] = useState([]); // Device groups: { id, name, deviceIds[] } const [groups, setGroups] = useState(() => { @@ -3838,7 +4690,15 @@ export default function App() { const [logRows, setLogRows] = useState([]); const [chartGroups, setChartGroups] = useState([]); // [{ snapInfo, rows }] const [logRowsLoading, setLogRowsLoading] = useState(false); + const [viewingSnaps, setViewingSnaps] = useState([]); // snapshots currently open in the log modal + const [logLoadProgress, setLogLoadProgress] = useState({ done: 0, total: 0 }); const [colorMode, setColorMode] = useState(false); + + // packet_capture "view packet details" modal + const [packetModal, setPacketModal] = useState(false); + const [packetModalData, setPacketModalData] = useState(null); // { packet_number, details } + const [packetModalLoading, setPacketModalLoading] = useState(false); + const [packetModalError, setPacketModalError] = useState(""); const [deviceModal, setDeviceModal] = useState(null); const [sessionModal, setSessionModal] = useState(null); const [apiModal, setApiModal] = useState(false); @@ -3872,6 +4732,15 @@ export default function App() { } }, [addToast]); + const fetchSystemStats = useCallback(async () => { + try { + setSystemStats(await apiFetch("/api/system/stats")); + } catch { + // Non-critical: leave the last known stats (or null) in place rather + // than spamming a toast every poll interval. + } + }, []); + const fetchSnapshots = useCallback(async (param, value, chart, page = 1, pSize) => { setSnapsLoading(true); try { @@ -3983,18 +4852,24 @@ export default function App() { useEffect(() => { fetchDevices(); }, [fetchDevices]); useEffect(() => { fetchSnapshots("", "", false); }, [fetchSnapshots]); + useEffect(() => { fetchSystemStats(); }, [fetchSystemStats]); useEffect(() => { const id = setInterval(fetchDevices, 10000); return () => clearInterval(id); }, [fetchDevices]); - const prevIsChart = useRef(isChart); useEffect(() => { - if (prevIsChart.current === isChart) return; - prevIsChart.current = isChart; + const id = setInterval(fetchSystemStats, 5000); + return () => clearInterval(id); + }, [fetchSystemStats]); + + // FIX: this effect previously had no dependency array, causing it to run + // after every render and rely on a manual ref comparison to detect changes. + // Using [isChart] as the dependency array is correct and idiomatic. + useEffect(() => { fetchSnapshots(filterActive ? searchParam : "", filterActive ? searchValue : "", isChart); - }); + }, [isChart]); // eslint-disable-line react-hooks/exhaustive-deps useEffect(() => { const p = new URLSearchParams(window.location.search); @@ -4211,14 +5086,18 @@ export default function App() { setLogRowsLoading(true); setLogRows([]); setChartGroups([]); + setViewingSnaps(snapsToView); + setLogLoadProgress({ done: 0, total: snapsToView.length }); try { + let done = 0; const results = await Promise.all( snapsToView.map((s) => - apiFetch(`/api/snapshots/${s.id}/content?log_type=${isChart ? "chart" : "text"}`).then((r) => ({ - snapInfo: s, - rows: r.rows, - })) + apiFetch(`/api/snapshots/${s.id}/content?log_type=${isChart ? "chart" : "text"}`).then((r) => { + done += 1; + setLogLoadProgress({ done, total: snapsToView.length }); + return { snapInfo: s, rows: r.rows }; + }) ) ); @@ -4226,7 +5105,14 @@ export default function App() { setChartGroups(results); } else { const merged = results.flatMap((r) => - r.rows.map((row) => ({ ...row, device_name: r.snapInfo.deviceName ?? r.snapInfo.device_name ?? "" })) + r.rows.map((row) => ({ + ...row, + device_name: r.snapInfo.deviceName ?? r.snapInfo.device_name ?? "", + // Needed so packet_capture rows can call back to + // /api/snapshots//packets/ for that specific + // snapshot's pcap file. + snapshotId: r.snapInfo.id, + })) ); // Sort all text entries by timestamp ascending merged.sort((a, b) => { @@ -4244,6 +5130,34 @@ export default function App() { } }; + /** + * Opens the "packet details" modal for a single packet_capture row. + * row.content's leading field (before the first " | ") is the tshark + * frame number produced by PacketInfo.to_content_str(); row.snapshotId + * identifies which snapshot's pcap file to decode it from. + */ + const openPacketDetails = async (row) => { + const packetNumber = parsePacketNumber(row?.content); + if (!row?.snapshotId || packetNumber === null) { + addToast("Couldn't determine which packet to look up.", "error"); + return; + } + + setPacketModal(true); + setPacketModalLoading(true); + setPacketModalError(""); + setPacketModalData(null); + + try { + const res = await apiFetch(`/api/snapshots/${row.snapshotId}/packets/${packetNumber}`); + setPacketModalData(res); + } catch (e) { + setPacketModalError(e.message || "Failed to load packet details."); + } finally { + setPacketModalLoading(false); + } + }; + const applyFilter = () => { setFilterActive(true); fetchSnapshots(searchParam, searchValue, isChart); @@ -4349,9 +5263,12 @@ ${chartSections} `; const blob = new Blob([html], { type: "text/html" }); const a = document.createElement("a"); - a.href = URL.createObjectURL(blob); + const chartBlobUrl = URL.createObjectURL(blob); + a.href = chartBlobUrl; a.download = "charts.html"; a.click(); + // FIX: revoke to release blob memory. + URL.revokeObjectURL(chartBlobUrl); addToast(`${chartGroups.length} chart(s) exported as HTML.`, "success"); } return; @@ -4447,10 +5364,15 @@ ${rows} filename = "logs.html"; } + const blobUrl = URL.createObjectURL(blob); const a = document.createElement("a"); - a.href = URL.createObjectURL(blob); + a.href = blobUrl; a.download = filename; a.click(); + // FIX: revoke the object URL immediately after triggering the download so + // the browser can release the underlying memory. Previously the URL was + // never revoked, causing a blob memory leak on every export. + URL.revokeObjectURL(blobUrl); addToast(`Logs exported as ${format === "html-color" ? "HTML" : format.toUpperCase()}.`, "success"); }; @@ -4551,9 +5473,12 @@ ${chartSections} `; const blob = new Blob([html], { type: "text/html" }); const a = document.createElement("a"); - a.href = URL.createObjectURL(blob); + const selChartBlobUrl = URL.createObjectURL(blob); + a.href = selChartBlobUrl; a.download = "charts.html"; a.click(); + // FIX: revoke to release blob memory. + URL.revokeObjectURL(selChartBlobUrl); addToast(`${results.length} chart(s) exported as HTML.`, "success"); } return; @@ -4643,10 +5568,13 @@ ${rowsHtml} filename = "logs.html"; } + const blobUrl2 = URL.createObjectURL(blob); const a = document.createElement("a"); - a.href = URL.createObjectURL(blob); + a.href = blobUrl2; a.download = filename; a.click(); + // FIX: revoke object URL to release blob memory. + URL.revokeObjectURL(blobUrl2); addToast(`${merged.length} log rows exported as ${format === "html-color" ? "HTML" : format.toUpperCase()}.`, "success"); } catch (e) { addToast(`Download failed: ${e.message}`); @@ -4655,11 +5583,38 @@ ${rowsHtml} } }; + // Downloads the untouched raw .pcap file for a "network capture" snapshot + // (as opposed to downloadLogs, which exports the *decoded* packet rows + // shown in the log content view). + const downloadRawPcap = async (snap) => { + try { + const res = await fetch(`${API_BASE}/api/snapshots/${snap.id}/pcap`); + if (!res.ok) { + const err = await res.json().catch(() => ({})); + throw new Error(err.error || res.statusText); + } + const blob = await res.blob(); + const a = document.createElement("a"); + const blobUrl = URL.createObjectURL(blob); + a.href = blobUrl; + a.download = `${snap.deviceName}_${snap.sessionId}.pcap`.replace(/\s+/g, "_"); + a.click(); + URL.revokeObjectURL(blobUrl); + addToast("Raw pcap downloaded.", "success"); + } catch (e) { + addToast(`PCAP download failed: ${e.message}`); + } + }; + // Modal title with chart count info const logModalTitle = isChart && chartGroups.length > 0 ? `Chart Data — ${chartGroups.length} snapshot${chartGroups.length > 1 ? "s" : ""}` : "Logs Content"; + // Snapshots open in the log modal that are network captures — used to + // show a "Download Raw PCAP" button per capture in the footer. + const networkCaptureSnaps = viewingSnaps.filter((s) => s.logName === "network capture"); + // ── Inject SVG favicon matching the header logo color ────────────────────── useEffect(() => { const svgFavicon = ` @@ -4752,9 +5707,21 @@ ${rowsHtml}
-
- - LIVE +
+ {systemStats ? ( + <> + CPU {systemStats.cpuPercent.toFixed(0)}% + · + RAM {systemStats.ramPercent.toFixed(0)}% + · + DISK {systemStats.diskPercent.toFixed(0)}% + + ) : ( + … + )}
{/* Auth controls */} {auth.isAdmin ? ( @@ -5080,13 +6047,18 @@ ${rowsHtml} footer={ <> {!isChart && } + {networkCaptureSnaps.map((s) => ( + downloadRawPcap(s)}> + ⬇ Raw PCAP{networkCaptureSnaps.length > 1 ? `: ${s.deviceName}` : ""} + + ))} setLogModal(false)}>Close } > {logRowsLoading ? ( - + ) : (
)} + {/* Packet details modal — opened from the 🔎 glyph next to packet_capture rows */} + setPacketModal(false)} + title={packetModalData ? `Packet #${packetModalData.packet_number}` : "Packet Details"} + size="lg" + footer={ setPacketModal(false)}>Close} + > + {packetModalLoading ? ( + + ) : packetModalError ? ( +

+ ⚠ {packetModalError} +

+ ) : packetModalData?.details && Object.keys(packetModalData.details).length > 0 ? ( + + ) : ( +

No packet detail available.

+ )} + + setDeviceModal(null)} @@ -5125,4 +6119,4 @@ ${rowsHtml} ); -} +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 7a25b14..abd27e6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ python-dateutil pytest fastapi pyarrow -flask_cors \ No newline at end of file +flask_cors +psutil \ No newline at end of file diff --git a/tests/test_rest_api.py b/tests/test_rest_api.py index 00d0ec8..47c56b2 100644 --- a/tests/test_rest_api.py +++ b/tests/test_rest_api.py @@ -239,7 +239,7 @@ def test_returns_all_snapshots_by_default(self, client): with ( patch("backend.app.get_current_devices", return_value=[]), patch( - "backend.app.ConfigurationHelper.get_log_snapshots_list", + "backend.app.LogSnapshotsHelper.get_log_snapshots_list", return_value=[snap], ), ): @@ -259,7 +259,7 @@ def test_uses_filtered_list_when_search_params_provided(self, client): with ( patch("backend.app.get_current_devices", return_value=[]), patch( - "backend.app.ConfigurationHelper.get_filtered_log_snapshots_list", + "backend.app.LogSnapshotsHelper.get_filtered_log_snapshots_list", return_value=[snap], ) as mock_filtered, ): @@ -272,7 +272,7 @@ def test_chart_log_type_sets_is_chart_true(self, client): with ( patch("backend.app.get_current_devices", return_value=[]), patch( - "backend.app.ConfigurationHelper.get_log_snapshots_list", + "backend.app.LogSnapshotsHelper.get_log_snapshots_list", return_value=[], ) as mock_list, ): @@ -283,7 +283,7 @@ def test_chart_log_type_sets_is_chart_true(self, client): def test_returns_empty_list_when_no_snapshots(self, client): with ( patch("backend.app.get_current_devices", return_value=[]), - patch("backend.app.ConfigurationHelper.get_log_snapshots_list", return_value=[]), + patch("backend.app.LogSnapshotsHelper.get_log_snapshots_list", return_value=[]), ): resp = client.get("/api/snapshots") data = resp.get_json() @@ -301,9 +301,9 @@ def test_returns_content_rows_for_valid_snapshot(self, client): with ( patch("backend.app.get_current_devices", return_value=[]), - patch("backend.app.ConfigurationHelper.get_log_snapshots_list", return_value=[snap]), + patch("backend.app.LogSnapshotsHelper.get_log_snapshots_list", return_value=[snap]), patch( - "backend.app.ConfigurationHelper.get_log_content_for_selected_snapshots", + "backend.app.LogSnapshotsHelper.get_log_content_for_selected_snapshots", return_value=mock_df, ), ): @@ -317,7 +317,7 @@ def test_returns_content_rows_for_valid_snapshot(self, client): def test_returns_404_for_unknown_snapshot(self, client): with ( patch("backend.app.get_current_devices", return_value=[]), - patch("backend.app.ConfigurationHelper.get_log_snapshots_list", return_value=[]), + patch("backend.app.LogSnapshotsHelper.get_log_snapshots_list", return_value=[]), ): resp = client.get("/api/snapshots/does-not-exist/content") assert resp.status_code == 404 @@ -360,7 +360,7 @@ def test_returns_400_when_selected_devices_is_not_list(self, client): def test_defaults_to_empty_list_when_key_missing(self, client): with patch("backend.app.get_current_devices", return_value=[]): - resp = self._post(client, {}) + resp = self._post(client, {"session_scenario": "test_1"}) assert resp.status_code == 200