diff --git a/.gitattributes b/.gitattributes index d9bd16b..0354f3c 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,3 @@ *.py text eol=lf + +panoptic/data/*.csv whitespace=cr-at-eol diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..87440a0 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,27 @@ +#!/bin/bash +# Pre-commit hook for Panoptic +# +# This hook delegates to the pre-commit framework for consistent tool configuration. +# The canonical check definitions are in .pre-commit-config.yaml. +# +# Setup options: +# Option A (recommended): Use pre-commit framework directly +# pip install pre-commit && pre-commit install +# +# Option B: Use this script via git config +# git config core.hooksPath .githooks +# +# Option C: Copy to .git/hooks +# cp .githooks/pre-commit .git/hooks/pre-commit && chmod +x .git/hooks/pre-commit + +# All setup options require the pre-commit package. Keeping one execution path +# ensures the pinned hooks and arguments in .pre-commit-config.yaml are always +# used, including Ruff fixes, formatting, strict mypy, Markdown linting, and the +# project dependency audit. +if ! command -v pre-commit >/dev/null 2>&1; then + echo "Error: pre-commit is required to run the Panoptic checks." >&2 + echo "Install it with: python -m pip install pre-commit" >&2 + exit 1 +fi + +exec pre-commit run diff --git a/.github/constraints-min.txt b/.github/constraints-min.txt new file mode 100644 index 0000000..09e1513 --- /dev/null +++ b/.github/constraints-min.txt @@ -0,0 +1,7 @@ +# Lower bounds of the declared runtime dependencies (see pyproject.toml). +# Used by the "minimum-versions" CI job to prove Panoptic works against the +# oldest dependency releases it claims to support, not just the newest. +httpx==0.28.1 +rich==15.0.0 +rich-argparse==1.4 +tomli==2.0; python_version < "3.11" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index dc3cf7d..d8d8758 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,14 +1,22 @@ version: 2 updates: - - package-ecosystem: "pip" - directory: "/" + - package-ecosystem: pip + directory: / schedule: - interval: "weekly" + interval: weekly commit-message: - prefix: "deps" - - package-ecosystem: "github-actions" - directory: "/" + prefix: deps + groups: + python-dependencies: + patterns: + - "*" + - package-ecosystem: github-actions + directory: / schedule: - interval: "weekly" + interval: weekly commit-message: - prefix: "ci" + prefix: ci + groups: + github-actions: + patterns: + - "*" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..8f65149 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,68 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + tests: + name: Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - run: python -m pip install --upgrade pip + - run: python -m pip install -e ".[dev]" + - run: pytest -q + + minimum-versions: + name: Minimum dependency versions (Python 3.10) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.10" + cache: pip + - run: python -m pip install --upgrade pip + # Install the package pinned to the lowest declared runtime dependency + # versions. Install only the test dependencies separately so unrelated + # development tools cannot raise those runtime lower bounds. + - run: python -m pip install -e . -c .github/constraints-min.txt + - run: python -m pip install pytest pytest-asyncio pytest-httpx -c .github/constraints-min.txt + - run: python -m pip check + - run: pytest -q + + quality: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.13" + cache: pip + - run: python -m pip install --upgrade pip + - run: python -m pip install -e ".[dev]" + - run: ruff check . + - run: ruff format --check . + - run: mypy panoptic + - run: python -m build + - run: python -m twine check dist/* + - run: pip-audit . diff --git a/.gitignore b/.gitignore index 3ebfcd2..69ccf55 100644 --- a/.gitignore +++ b/.gitignore @@ -4,10 +4,15 @@ output/ .worktrees/ .venv/ panoptic.egg-info/ +build/ +dist/ +.coverage +htmlcov/ .mcp.json CLAUDE.md .claude/ .claire/ .codex-review/ .coderabbit-results/ -docs/ \ No newline at end of file +.fathom/ +docs/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8e1e817..5201703 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.15.6 + rev: v0.16.0 hooks: - id: ruff args: [--fix] - id: ruff-format - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.16.0 + rev: v2.3.0 hooks: - id: mypy exclude: ^(panoptic\.py|tests/|\.claire/) @@ -25,10 +25,10 @@ repos: exclude: ^(docs/superpowers/|\.claire/) - repo: https://github.com/pypa/pip-audit - rev: v2.10.0 + rev: v2.10.1 hooks: - id: pip-audit - # Audit the project's declared dependencies (pyproject.toml), not the - # ambient hook environment — the latter reports CVEs in pip-audit's own - # transitive tooling, which are irrelevant to Panoptic. - args: [".", "--desc"] + # Audit the project's declared dependencies rather than pip-audit's + # ambient hook environment. + args: [--desc=on, .] + pass_filenames: false diff --git a/README.md b/README.md index 9a0fd7b..9758f4e 100644 --- a/README.md +++ b/README.md @@ -38,9 +38,12 @@ path traversal vulnerabilities. * TOML config files for persistent settings (`--config`) * Multiple traversal bypass techniques: prefixes, postfixes, multiplier, slash replacement, double encoding -* HTTP/HTTPS and SOCKS4/SOCKS5 proxy support with validation +* HTTP/HTTPS and SOCKS5 proxy support with validation * Random or custom User-Agent, cookie, and header support -* Credential redaction in banner and log output +* Credential redaction in banner, log, and machine-readable output + (including numeric and boolean JSON body values) +* Sensitive artifacts hardened with owner-only `0600` permissions on + POSIX and final-component symlink protection where the OS supports it * Self-update with remote URL verification (`--update`) ## Requirements @@ -197,28 +200,180 @@ panoptic --url "http://target/include.php?file=test.txt" \ --config ~/.config/panoptic/config.toml ``` -Default config location: `~/.config/panoptic/config.toml` +Default config location: `~/.config/panoptic/config.toml` (loaded +automatically when present, even without `--config`). ```toml [defaults] +# Any long option name (with dashes as underscores) is accepted here, +# including the target and output destinations. +url = "http://target/include.php?file=test.txt" concurrency = 8 verbose = true automatic = true +all_versions = true +output_format = "json" +output_file = "results.json" +log_file = "scan.log" +resume_file = "scan.checkpoint" [proxy] url = "socks5://127.0.0.1:9050" [headers] user_agent = "Mozilla/5.0" +cookie = "sid=foobar; auth=1" +values = ["X-Forwarded-For: 127.0.0.1"] ``` -Priority: CLI args > config file > built-in defaults. +Priority: **CLI args > config file > built-in defaults.** + +### Config-supported target and output options + +The `[defaults]` table accepts any scan option, not just performance +tuning. In particular you can persist: + +* `url` — the default target (override per-run with `--url`) +* `output_format`, `output_file` — where machine-readable results go +* `log_file` — mirror console output to a file +* `resume_file` — checkpoint location for resumable scans + +Sensitive artifacts written by Panoptic — the log file, the results/list +output file, and any files saved with `--write-files` — are forced to +owner-only `0600` permissions on POSIX. Platforms exposing `O_NOFOLLOW` +also atomically refuse a pre-existing symlink at the final path component. +On platforms without `O_NOFOLLOW`, Panoptic performs a best-effort +pre-open symlink/junction check, but the check cannot eliminate a race. +Windows mode bits do not configure NTFS ACLs, so use an appropriately +restricted directory when artifacts may contain sensitive data. + +### Overriding config booleans on the command line + +Every boolean flag has a `--no-` counterpart, so a value set to `true` +in the config file can be turned off for a single run without editing +the file: + +```bash +# config.toml sets verbose = true and automatic = true +panoptic --url "http://target/x.php?file=test.txt" --no-verbose --no-auto +``` + +Because an omitted boolean flag is left unset (rather than defaulting to +`false`), the config value is used unless you pass the flag or its +`--no-` form explicitly. + +## Proxying + +Route traffic through an HTTP(S) or SOCKS proxy: + +```bash +panoptic --url "https://target/x.php?file=test.txt" \ + --proxy "socks5://127.0.0.1:9050" +``` + +* Accepted schemes: `http://`, `https://`, `socks5://`, `socks5h://` + (`socks5h` resolves DNS through the proxy — useful for Tor and to + avoid local DNS leaks). SOCKS4 is not supported by the underlying + HTTP client. The scheme and host are validated before the scan starts. +* SOCKS support comes from the `httpx[socks]` extra, which is installed + by default. +* By default Panoptic honours the standard `HTTP_PROXY` / `HTTPS_PROXY` / + `NO_PROXY` environment variables. Pass `--ignore-proxy` to bypass them + and connect directly (this also disables any env-configured proxy). +* Combine with `--invalid-ssl` when intercepting HTTPS through a proxy + with its own CA. This disables certificate verification and is unsafe + on untrusted networks. + +## Version Expansion (`--all-versions`) + +Some bundled paths are version templates (for example JBoss release +directories written as `[JBOSS]`). By default these template rows are +skipped, because a literal `[JBOSS]` path can never match a real file. +Passing `--all-versions` expands each template against the bundled +version list, adding one concrete path per known version — a much larger +but far more thorough scan: + +```bash +panoptic --url "http://target/x.php?file=test.txt" --auto --all-versions +``` + +## Response Comparison + +Panoptic does not classify a path from the target-controlled +`Content-Length` header alone. It compares the complete normalized body +against the invalid-path baseline. A cheap similarity upper bound avoids +the full comparison only when it can already prove the bodies differ; +ambiguous responses use the exact ratio to avoid reordered-body false +negatives. + +## Resume / Checkpoints + +`--resume-file PATH` records completed cases so an interrupted scan can +continue where it left off: + +```bash +panoptic --url "http://target/x.php?file=test.txt" \ + --resume-file scan.checkpoint +# ... interrupt with Ctrl-C, then re-run the same command to resume +``` + +The checkpoint stores only completed case IDs plus a SHA-256 +**fingerprint** of the scan definition (URL, injection parameters, +detection options, headers, cookie, User-Agent, and the exact case set). +It never stores credentials, headers, or URLs in plaintext. + +On resume, the fingerprint is recomputed and compared. If anything that +would change the meaning of the scan differs — a different target, +parameter, header, cookie, filter set, or `--all-versions` toggle — the +checkpoint is rejected with a warning and the scan starts fresh, so a +stale checkpoint can never silently skip cases from a different scan. +Legacy bare-list checkpoints are accepted with a warning and restricted +to case IDs present in the current scan; because that old format has no +fingerprint, users should replace it with the newly written format. +Failed (network-error) requests are deliberately **not** checkpointed so +they are retried on resume. + +## Version and Update + +```bash +panoptic --version # print the installed version and exit +panoptic --update # fast-forward update from the official GitHub repo +``` + +`--update` only works from a git checkout whose `origin` remote uses +HTTPS or SSH and matches the official upstream (verified before pulling, +to resist insecure or tampered remote configuration). It fast-forwards +the checked-out `main` branch from the explicit upstream `main` ref; for +pip installs it prints the correct +`pip install -U` command instead. When run from a checkout, the banner +also shows the current short git revision. + +## Exit Codes + +Panoptic uses conventional process exit codes so it can be scripted: + +* `0` — Success: the scan (or `--list` / `--update`) completed. Individual + requests that exhaust their retries are warned about and remain eligible + for a resumed scan, but do not fail an otherwise successful run. +* `1` — Invalid usage: missing or invalid arguments, or no injectable + parameter could be found. +* `2` — Operational failure: cannot connect, no matching test cases, all + queued requests failed, or a worker, checkpoint, output write, or + configuration failed. +* `130` — Interrupted by the user (Ctrl-C / SIGINT). ## Contributing Contributions are welcome! Please open issues or pull requests on [GitHub](https://github.com/lightos/Panoptic). +## Testbed + +For safe, reproducible end-to-end testing, see the +[Panoptic LFI Testbed](https://github.com/lightos/panoptic-lfi-testbed). +It provides deliberately vulnerable endpoints covering Panoptic's supported +injection methods and traversal transformations. + ## License This project is licensed under the MIT License - see the diff --git a/panoptic/__main__.py b/panoptic/__main__.py index d12fb49..f41f06a 100644 --- a/panoptic/__main__.py +++ b/panoptic/__main__.py @@ -9,7 +9,7 @@ def main() -> None: from panoptic.cli import run try: - asyncio.run(run()) + sys.exit(asyncio.run(run())) except KeyboardInterrupt: sys.exit(130) diff --git a/panoptic/cases.py b/panoptic/cases.py index 25ba0e6..b701399 100644 --- a/panoptic/cases.py +++ b/panoptic/cases.py @@ -8,12 +8,17 @@ from urllib.parse import urlsplit from panoptic.models import Case, FileType, ScanConfig -from panoptic.utils import load_data_file +from panoptic.utils import load_data_file, normalize_os_name, normalize_os_names, os_matches_restriction REQUIRED_COLUMNS = frozenset({"path", "os", "software", "category", "type"}) VALID_TYPES = frozenset(t.value for t in FileType) +def _expand_os_values(raw: str) -> tuple[str, ...]: + """Expand comma-separated OS metadata and normalize known aliases.""" + return normalize_os_names(raw) + + def _validate_csv(rows: list[dict[str, str]]) -> None: """Validate CSV data on first load. Raises ValueError on bad data.""" for i, row in enumerate(rows, start=2): # line 1 is header @@ -65,7 +70,9 @@ def parse_cases(config: ScanConfig) -> list[Case]: # Build replacements dict for placeholder expansion replacements: dict[str, str] = {} if config.url: - replacements["HOST"] = urlsplit(config.url).netloc + hostname = urlsplit(config.url).hostname or "" + replacements["HOST"] = hostname + replacements["DOMAIN"] = hostname # Load versions for expansion versions = load_versions() if config.all_versions else {} @@ -73,14 +80,19 @@ def parse_cases(config: ScanConfig) -> list[Case]: cases: list[Case] = [] for row in rows: - os_val = row["os"] + os_values = _expand_os_values(row["os"]) software_val = row["software"] category_val = row["category"] file_type = FileType(row["type"]) location = row["path"] - # Apply filters - if config.os_filter and os_val.lower() != config.os_filter.lower(): + # Apply filters. The OS prefilter uses the same Unix-family hierarchy as + # the runtime restriction (os_matches_restriction) so a "*NIX" filter + # keeps FreeBSD/OS X/... cases and a specific Unix filter still keeps the + # generic "*NIX" cases — otherwise the list would silently drop cases the + # scan would actually test. + normalized_filter = normalize_os_name(config.os_filter) + if normalized_filter and not any(os_matches_restriction(os_val, normalized_filter) for os_val in os_values): continue if config.software_filter and software_val.lower() != config.software_filter.lower(): continue @@ -97,13 +109,22 @@ def parse_cases(config: ScanConfig) -> list[Case]: # Version expansion ([SECTION] patterns) match = re.search(r"\[([^\]]+)\]", location) - if match and config.all_versions and match.group(1) in versions: + if match and config.all_versions: + if match.group(1) not in versions: + raise ValueError(f"cases.csv references unknown version section '{match.group(1)}'") locations = [location.replace(match.group(0), v) for v in versions[match.group(1)]] + elif match: + # Version templates are meaningful only when explicitly expanded. + # Sending the raw "[SECTION]" path can never find a real file. + continue else: locations = [location] + # A comma-separated OS field describes one case compatible with any of + # those systems; it must not fan out into duplicate requests. + case_os = ", ".join(os_values) for loc in locations: cases.append( - Case(location=loc, os=os_val, category=category_val, software=software_val, file_type=file_type) + Case(location=loc, os=case_os, category=category_val, software=software_val, file_type=file_type) ) return cases @@ -133,9 +154,13 @@ def list_values(group: str, config: ScanConfig | None = None) -> set[str]: if config is not None: cases = parse_cases(config) + if group == "os": + return {os_value for case in cases for os_value in normalize_os_names(case.os)} return {getattr(c, group) for c in cases if getattr(c, group)} rows = _load_csv() + if group == "os": + return {os_val for row in rows for os_val in _expand_os_values(row[group])} return {row[group] for row in rows if row.get(group)} diff --git a/panoptic/cli.py b/panoptic/cli.py index b71b338..c17cfb9 100644 --- a/panoptic/cli.py +++ b/panoptic/cli.py @@ -6,36 +6,23 @@ from __future__ import annotations import argparse +import csv import json -import re import sys +from pathlib import Path from typing import Any -from urllib.parse import urlsplit +from urllib.parse import parse_qsl, urlsplit from rich_argparse import RawDescriptionRichHelpFormatter -from panoptic.utils import normalize_url, parse_status_codes, validate_header, validate_url_scheme - -# Boolean/value defaults from argparse that should be stripped before config merge, -# so they don't override config file values. ScanConfig defaults apply when absent -# from both CLI and file config. -_ARGPARSE_DEFAULTS: dict[str, object] = { - "path_based": False, - "write_files": False, - "skip_parsing": False, - "invalid_ssl": False, - "automatic": False, - "all_versions": False, - "random_agent": False, - "ignore_proxy": False, - "verbose": False, - "follow_redirects": False, - "quiet": False, - "base64_encode": False, - "prefix": "", - "postfix": "", - "multiplier": 1, -} +from panoptic.utils import ( + has_parameter, + normalize_url, + open_secure_write, + parse_status_codes, + validate_header, + validate_url_scheme, +) EXAMPLES = """ Examples: @@ -61,8 +48,19 @@ def parse_args(argv: list[str] | None = None) -> dict[str, Any]: conn = parser.add_argument_group("Connection / Proxy") conn.add_argument("-u", "--url", help="Target URL vulnerable to path traversal") conn.add_argument("--proxy", help="Route requests through proxy (e.g. 'socks5://127.0.0.1:9050')") - conn.add_argument("--ignore-proxy", action="store_true", help="Bypass system proxy settings") - conn.add_argument("--random-agent", action="store_true", dest="random_agent", help="Choose random User-Agent") + conn.add_argument( + "--ignore-proxy", + action=argparse.BooleanOptionalAction, + default=None, + help="Bypass system proxy settings", + ) + conn.add_argument( + "--random-agent", + action=argparse.BooleanOptionalAction, + default=None, + dest="random_agent", + help="Choose random User-Agent", + ) conn.add_argument( "--header", dest="headers", @@ -76,7 +74,8 @@ def parse_args(argv: list[str] | None = None) -> dict[str, Any]: conn.add_argument( "--follow-redirects", dest="follow_redirects", - action="store_true", + action=argparse.BooleanOptionalAction, + default=None, help="Follow HTTP redirects (default: don't follow)", ) @@ -100,14 +99,15 @@ def parse_args(argv: list[str] | None = None) -> dict[str, Any]: "-P", "--path-based", dest="path_based", - action="store_true", + action=argparse.BooleanOptionalAction, + default=None, help="Target file paths directly instead of query parameters", ) scan.add_argument("-d", "--data", help="Send parameters via POST instead of GET") scan.add_argument("-t", "--type", dest="type_filter", help="Filter by type ('conf', 'log', 'other')") - scan.add_argument("--prefix", default="", help="Add prefix to file paths (e.g. '../')") - scan.add_argument("--postfix", default="", help="Add suffix to file paths (e.g. '%%00')") - scan.add_argument("--multiplier", type=int, default=1, help="Repeat prefix N times (default: 1)") + scan.add_argument("--prefix", default=None, help="Add prefix to file paths (e.g. '../')") + scan.add_argument("--postfix", default=None, help="Add suffix to file paths (e.g. '%%00')") + scan.add_argument("--multiplier", type=int, default=None, help="Repeat prefix N times (default: 1)") scan.add_argument("--bad-string", dest="bad_string", help="Skip paths if this string appears in response") scan.add_argument( "--match-string", dest="match_string", help="Only report findings containing this string in response" @@ -126,11 +126,18 @@ def parse_args(argv: list[str] | None = None) -> dict[str, Any]: scan.add_argument( "--base64", dest="base64_encode", - action="store_true", + action=argparse.BooleanOptionalAction, + default=None, help="Base64-encode file paths before injection (for endpoints that decode)", ) scan.add_argument("--ext-param", dest="ext_param", help="Name of parameter containing file extension") - scan.add_argument("--all-versions", dest="all_versions", action="store_true", help="Test all versioned file paths") + scan.add_argument( + "--all-versions", + dest="all_versions", + action=argparse.BooleanOptionalAction, + default=None, + help="Test all versioned file paths", + ) # Performance perf = parser.add_argument_group("Performance") @@ -147,28 +154,51 @@ def parse_args(argv: list[str] | None = None) -> dict[str, Any]: # Output out = parser.add_argument_group("Output") - out.add_argument("-v", "--verbose", action="store_true", help="Show detailed information") + out.add_argument( + "-v", + "--verbose", + action=argparse.BooleanOptionalAction, + default=None, + help="Show detailed information", + ) out.add_argument( "-q", "--quiet", - action="store_true", + action=argparse.BooleanOptionalAction, + default=None, help="Suppress banner, info, and progress — only show findings and warnings", ) out.add_argument( "-w", "--write-files", dest="write_files", - action="store_true", + action=argparse.BooleanOptionalAction, + default=None, help="Save discovered files to local output directory", ) out.add_argument( - "-x", "--skip-parsing", dest="skip_parsing", action="store_true", help="Don't extract users from passwd files" + "-x", + "--skip-parsing", + dest="skip_parsing", + action=argparse.BooleanOptionalAction, + default=None, + help="Don't extract users from passwd files", ) out.add_argument( - "-i", "--invalid-ssl", dest="invalid_ssl", action="store_true", help="Ignore SSL certificate validation errors" + "-i", + "--invalid-ssl", + dest="invalid_ssl", + action=argparse.BooleanOptionalAction, + default=None, + help="Ignore SSL certificate validation errors", ) out.add_argument( - "-a", "--auto", dest="automatic", action="store_true", help="Avoid user interaction by using default options" + "-a", + "--auto", + dest="automatic", + action=argparse.BooleanOptionalAction, + default=None, + help="Avoid user interaction by using default options", ) out.add_argument( "--output-format", @@ -185,6 +215,9 @@ def parse_args(argv: list[str] | None = None) -> dict[str, Any]: parser.add_argument("--load", dest="list_file", help="Test custom file list from FILE") parser.add_argument("--config", dest="config_file", help="Path to TOML config file") parser.add_argument("--update", action="store_true", help="Update from GitHub repository") + from panoptic import __version__ + + parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}") parser.add_argument( "--list-all-files", dest="list_all_files", @@ -195,18 +228,10 @@ def parse_args(argv: list[str] | None = None) -> dict[str, Any]: parsed = parser.parse_args(argv) result = vars(parsed) - for key, default in _ARGPARSE_DEFAULTS.items(): - if result.get(key) == default: - result.pop(key, None) - # Normalize URL if result.get("url") and not result["url"].lower().startswith(("http://", "https://")): result["url"] = normalize_url(result["url"]) - # Apply prefix multiplier - if result.get("prefix"): - result["prefix"] = result["prefix"] * result.get("multiplier", 1) - # Parse --random-delay "MIN-MAX" into tuple if result.get("random_delay") and isinstance(result["random_delay"], str): try: @@ -259,6 +284,12 @@ def validate_args(args: dict[str, Any]) -> None: if args.get("concurrency") is not None and args["concurrency"] < 1: print("[!] --concurrency must be at least 1", file=sys.stderr) sys.exit(1) + if args.get("retries") is not None and args["retries"] < 0: + print("[!] --retries must be non-negative", file=sys.stderr) + sys.exit(1) + if args.get("multiplier") is not None and args["multiplier"] < 1: + print("[!] --multiplier must be at least 1", file=sys.stderr) + sys.exit(1) if args.get("timeout") is not None and args["timeout"] <= 0: print("[!] --timeout must be greater than 0", file=sys.stderr) @@ -269,112 +300,187 @@ def validate_args(args: dict[str, Any]) -> None: # Proxy scheme validation if args.get("proxy"): - allowed_proxy_schemes = ("http://", "https://", "socks4://", "socks5://") + # SOCKS4 is intentionally excluded: httpx (via socksio) only supports + # SOCKS5, so accepting socks4:// here would fail later at connection time. + allowed_proxy_schemes = ("http://", "https://", "socks5://", "socks5h://") if not args["proxy"].lower().startswith(allowed_proxy_schemes): - print("[!] Invalid proxy scheme. Use http://, https://, socks4://, or socks5://", file=sys.stderr) + print( + "[!] Invalid proxy scheme. Use http://, https://, socks5://, or socks5h://", + file=sys.stderr, + ) sys.exit(1) + parsed_proxy = urlsplit(args["proxy"]) + if not parsed_proxy.hostname: + print("[!] Proxy URL must include a hostname", file=sys.stderr) + sys.exit(1) + try: + proxy_port = parsed_proxy.port + except ValueError as exc: + print(f"[!] Invalid proxy port: {exc}", file=sys.stderr) + sys.exit(1) + del proxy_port # Header CRLF validation for hdr in args.get("headers") or []: try: - validate_header(hdr) + validate_header(hdr, warn_deprecated=False) except ValueError as e: print(f"[!] Invalid header: {e}", file=sys.stderr) sys.exit(1) # Cookie CRLF validation - if args.get("cookie") and any(c in args["cookie"] for c in "\r\n"): - print("[!] Cookie contains CRLF characters (possible header injection)", file=sys.stderr) + if args.get("cookie") and any(c in args["cookie"] for c in "\r\n\x00"): + print("[!] Cookie contains CRLF/NUL characters (possible header injection)", file=sys.stderr) sys.exit(1) # User-Agent CRLF validation - if args.get("user_agent") and any(c in args["user_agent"] for c in "\r\n"): - print("[!] User-Agent contains CRLF characters (possible header injection)", file=sys.stderr) + if args.get("user_agent") and any(c in args["user_agent"] for c in "\r\n\x00"): + print("[!] User-Agent contains CRLF/NUL characters (possible header injection)", file=sys.stderr) + sys.exit(1) + + if args.get("quiet") and args.get("verbose"): + print("[!] --quiet and --verbose cannot be used together", file=sys.stderr) sys.exit(1) + if args.get("base64_encode") and args.get("ext_param"): + print("[!] --base64 and --ext-param cannot be combined", file=sys.stderr) + sys.exit(1) -async def run(argv: list[str] | None = None) -> None: + match_codes = set(args.get("match_codes") or []) + filter_codes = set(args.get("filter_codes") or []) + overlap = sorted(match_codes & filter_codes) + if overlap: + print(f"[!] Status codes cannot be both matched and filtered: {overlap}", file=sys.stderr) + sys.exit(1) + + output_paths = [args.get(key) for key in ("output_file", "log_file", "resume_file")] + normalized_paths = [str(Path(path).expanduser().resolve()) for path in output_paths if path] + if len(normalized_paths) != len(set(normalized_paths)): + print("[!] --output-file, --log-file, and --resume-file must use different paths", file=sys.stderr) + sys.exit(1) + + +def _write_list_output(values: list[str], header: str, fmt: str, output_file: str | None) -> int: + """Write list command output using format-aware serialization.""" + stream = sys.stdout + file_stream = None + try: + if output_file: + file_stream = open_secure_write(output_file, newline="") + stream = file_stream + if fmt == "json": + json.dump(values, stream, indent=2) + stream.write("\n") + elif fmt == "csv": + writer = csv.writer(stream, lineterminator="\n") + writer.writerow([header]) + writer.writerows([value] for value in values) + else: + for value in values: + stream.write(f"{value}\n") + except OSError as exc: + print(f"[!] Cannot write output: {exc}", file=sys.stderr) + return 2 + finally: + if file_stream: + file_stream.close() + return 0 + + +async def run(argv: list[str] | None = None) -> int: """Main async entry point — parse args, load config, dispatch.""" args = parse_args(argv) - validate_args(args) # Handle non-scan commands if args.get("update"): from panoptic.update import do_update - do_update() - return + validate_args(args) + return do_update() - if args.get("list_all_files"): - from panoptic.cases import list_all_files + # Lazy import — only needed for scan and --list modes + from panoptic.config import load_config, merge_config - paths = list_all_files() - fmt = args.get("output_format") or "text" + config_file = args.pop("config_file", None) + file_config = load_config(config_file) - if fmt == "json": - print(json.dumps(paths, indent=2)) - elif fmt == "csv": - print("path") - for path in paths: - print(path) - else: - for path in paths: - print(path) - return + if args.get("list_all_files"): + from panoptic.cases import list_all_files - # Lazy import — only needed for scan and --list modes - from panoptic.config import load_config, merge_config + validate_args(args) + try: + list_config = merge_config(args, file_config) + except (TypeError, ValueError) as exc: + print(f"[!] Invalid configuration: {exc}", file=sys.stderr) + return 2 + return _write_list_output( + list_all_files(), + "path", + list_config.output_format.value, + list_config.output_file, + ) if args.get("list"): from panoptic.cases import list_values - _file_config = load_config(args.get("config_file")) - _config = merge_config(args, _file_config) + validate_args(args) + try: + _config = merge_config(args, file_config) + except (TypeError, ValueError) as exc: + print(f"[!] Invalid configuration: {exc}", file=sys.stderr) + return 2 values = list_values(args["list"], config=_config) - fmt = args.get("output_format") or "text" - - if fmt == "json": - print(json.dumps(sorted(values), indent=2)) - elif fmt == "csv": - print(args["list"]) - for val in sorted(values): - print(val) - else: + fmt = _config.output_format.value + sorted_values = sorted(values) + if fmt == "text" and not _config.output_file: print(f"Available {args['list']} values ({len(values)}):") - for val in sorted(values): - print(f" {val}") - return + sorted_values = [f" {value}" for value in sorted_values] + return _write_list_output(sorted_values, args["list"], fmt, _config.output_file) - file_config = load_config(args.pop("config_file", None)) - config = merge_config(args, file_config) + try: + config = merge_config(args, file_config) + except (TypeError, ValueError) as exc: + print(f"[!] Invalid configuration: {exc}", file=sys.stderr) + return 2 + + validate_args(vars(config)) # Shared URL parsing for param detection and ext-param validation parsed = urlsplit(config.url) params = config.data if config.data else parsed.query - # Check if FUZZ marker is used in any injectable position - has_fuzz = "FUZZ" in (config.data or "") or any("FUZZ" in h for h in (config.headers or [])) + # Check if FUZZ is used in an injectable body or header value. A header name + # containing the literal text "FUZZ" is not an injection point. + has_fuzz = "FUZZ" in (config.data or "") or any( + "FUZZ" in validate_header(header, warn_deprecated=False)[1] for header in (config.headers or []) + ) # Auto-detect vulnerable parameter if not specified (ported from original) if not config.path_based and not config.param and not has_fuzz: - match = re.match(r"(?P[^=&]+)=(?P[^&]+)", params) - if match: - config = config.replace(param=match.group("param")) + detected_param = next( + (name for name, value in parse_qsl(params, keep_blank_values=True) if name and value), + None, + ) + if detected_param: + config = config.replace(param=detected_param) else: print("[!] No usable GET/POST parameters found.", file=sys.stderr) print("[!] If this is a path-based URL, use --path-based", file=sys.stderr) - sys.exit(1) + return 1 - # Validate --ext-param exists in query/data (anchored to prevent substring matches) - if config.ext_param and not re.search( - rf"(?:^|(?<=&)){re.escape(config.ext_param)}=(?P[^&]*)", - params, - ): + if not config.path_based and config.param and not has_fuzz and not has_parameter(params, config.param): + print(f"[!] Parameter '{config.param}' not found in query/POST form data.", file=sys.stderr) + if config.data: + print("[!] For JSON or opaque POST bodies, place FUZZ at the injection point.", file=sys.stderr) + return 1 + + # Validate --ext-param against decoded or raw query/form parameter names. + if config.ext_param and not has_parameter(params, config.ext_param): print(f"[!] Extension parameter '{config.ext_param}' not found.", file=sys.stderr) - sys.exit(1) + return 1 # Dispatch to scanner from panoptic.core import Scanner scanner = Scanner(config) - await scanner.run() + return await scanner.run() diff --git a/panoptic/config.py b/panoptic/config.py index b051ea6..0e3b0e1 100644 --- a/panoptic/config.py +++ b/panoptic/config.py @@ -11,7 +11,7 @@ from typing import Any from panoptic.models import OutputFormat, ScanConfig -from panoptic.utils import parse_status_codes +from panoptic.utils import normalize_url, parse_status_codes if sys.version_info >= (3, 11): import tomllib @@ -24,6 +24,65 @@ DEFAULT_CONFIG_PATH = Path.home() / ".config" / "panoptic" / "config.toml" +_STRING_FIELDS = { + "url", + "param", + "data", + "prefix", + "postfix", + "ext_param", + "bad_string", + "match_string", + "replace_slash", + "output_file", + "log_file", + "proxy", + "user_agent", + "cookie", + "os_filter", + "software_filter", + "category_filter", + "type_filter", + "list_file", + "resume_file", +} +_BOOLEAN_FIELDS = { + "path_based", + "base64_encode", + "write_files", + "skip_parsing", + "automatic", + "invalid_ssl", + "all_versions", + "follow_redirects", + "verbose", + "quiet", + "ignore_proxy", + "random_agent", +} +_INTEGER_FIELDS = {"concurrency", "retries", "multiplier"} +_NUMBER_FIELDS = {"timeout", "delay", "heuristic_ratio"} + + +def _validate_merged_types(merged: dict[str, Any]) -> None: + """Reject TOML values that would otherwise fail later with raw type errors.""" + for field in sorted(_STRING_FIELDS): + value = merged.get(field) + if value is not None and not isinstance(value, str): + raise ValueError(f"{field} must be a string") + for field in sorted(_BOOLEAN_FIELDS): + value = merged.get(field) + if value is not None and not isinstance(value, bool): + raise ValueError(f"{field} must be a boolean") + for field in sorted(_INTEGER_FIELDS): + value = merged.get(field) + if value is not None and (not isinstance(value, int) or isinstance(value, bool)): + raise ValueError(f"{field} must be an integer") + for field in sorted(_NUMBER_FIELDS): + value = merged.get(field) + if value is not None and (not isinstance(value, int | float) or isinstance(value, bool)): + raise ValueError(f"{field} must be a number") + def load_config(path: str | None = None) -> dict[str, Any]: """Load a TOML config file. @@ -33,6 +92,8 @@ def load_config(path: str | None = None) -> dict[str, Any]: config_path = Path(path) if path else DEFAULT_CONFIG_PATH if not config_path.exists(): + if path is not None: + print(f"[!] Warning: config file '{config_path}' does not exist", file=sys.stderr) return {} if tomllib is None: @@ -51,32 +112,78 @@ def merge_config(cli_args: dict[str, Any], file_config: dict[str, Any]) -> ScanC Priority: CLI args > file config > ScanConfig defaults. """ + valid_fields = set(ScanConfig.__dataclass_fields__) + known_sections = {"defaults", "proxy", "headers"} + for section in sorted(set(file_config) - known_sections): + print(f"[!] Warning: unknown config section '{section}'", file=sys.stderr) + + defaults = file_config.get("defaults", {}) + if not isinstance(defaults, dict): + raise ValueError("[defaults] must be a TOML table") + for key in sorted(set(defaults) - valid_fields - {"header"}): + print(f"[!] Warning: unknown config option 'defaults.{key}'", file=sys.stderr) + # Start with file config defaults - merged: dict[str, Any] = dict(file_config.get("defaults", {})) + merged: dict[str, Any] = {key: value for key, value in defaults.items() if key in valid_fields or key == "header"} + + # Backward compatibility: normalize singular "header" before merging the + # dedicated [headers] table and CLI repetitions. + if "header" in merged: + legacy_headers = merged.pop("header") + if isinstance(legacy_headers, str): + merged["headers"] = [legacy_headers] + elif isinstance(legacy_headers, list): + merged["headers"] = legacy_headers + else: + raise ValueError("defaults.header must be a string or a list of strings") # Map proxy and header sections to flat ScanConfig fields - proxy_url = file_config.get("proxy", {}).get("url") + proxy_section = file_config.get("proxy", {}) + if not isinstance(proxy_section, dict): + raise ValueError("[proxy] must be a TOML table") + for key in sorted(set(proxy_section) - {"url"}): + print(f"[!] Warning: unknown config option 'proxy.{key}'", file=sys.stderr) + proxy_url = proxy_section.get("url") if proxy_url: merged["proxy"] = proxy_url headers_section = file_config.get("headers", {}) - if isinstance(headers_section, dict): - header_ua = headers_section.get("user_agent") - if header_ua: - merged["user_agent"] = header_ua - - # Apply CLI args (override file config) + if not isinstance(headers_section, dict): + raise ValueError("[headers] must be a TOML table") + for key in sorted(set(headers_section) - {"user_agent", "cookie", "values"}): + print(f"[!] Warning: unknown config option 'headers.{key}'", file=sys.stderr) + header_ua = headers_section.get("user_agent") + if header_ua: + merged["user_agent"] = header_ua + header_cookie = headers_section.get("cookie") + if header_cookie: + merged["cookie"] = header_cookie + header_values = headers_section.get("values") + if header_values: + if not isinstance(header_values, list) or not all(isinstance(header, str) for header in header_values): + raise ValueError("headers.values must be a list of strings") + merged["headers"] = [*(merged.get("headers") or []), *header_values] + + # Scalar CLI arguments override file values. Repeatable headers are + # additive, with CLI values last so duplicate header names retain CLI + # precedence when the request header mapping is built. for key, value in cli_args.items(): if value is not None: - merged[key] = value + if key == "headers": + if not isinstance(value, list): + raise ValueError("headers must be a list of strings") + merged[key] = [*(merged.get(key) or []), *value] + else: + merged[key] = value # Handle output_format enum conversion - if "output_format" in merged and isinstance(merged["output_format"], str): + if "output_format" in merged and not isinstance(merged["output_format"], OutputFormat): + if not isinstance(merged["output_format"], str): + raise ValueError("output_format must be a string") try: merged["output_format"] = OutputFormat(merged["output_format"]) except ValueError as e: - print(f"[!] Invalid output_format in config: {e}", file=sys.stderr) - merged.pop("output_format") + raise ValueError(f"invalid output_format: {e}") from e # Normalize match_codes / filter_codes from TOML (may be string or list) for code_key in ("match_codes", "filter_codes"): @@ -85,20 +192,33 @@ def merge_config(cli_args: dict[str, Any], file_config: dict[str, Any]) -> ScanC try: merged[code_key] = parse_status_codes(val) except (ValueError, TypeError) as e: - print(f"[!] Invalid {code_key} in config: {e}", file=sys.stderr) - merged.pop(code_key) - - # Backward compatibility: normalize singular "header" to plural "headers" - if "header" in merged and "headers" not in merged: - val = merged.pop("header") - if isinstance(val, str): - merged["headers"] = [val] - elif isinstance(val, list): - merged["headers"] = val - elif "header" in merged: - merged.pop("header") + raise ValueError(f"invalid {code_key}: {e}") from e + + random_delay = merged.get("random_delay") + if isinstance(random_delay, str): + parts = random_delay.split("-") + if len(parts) != 2: + raise ValueError("random_delay must use MIN-MAX format") + try: + merged["random_delay"] = (float(parts[0]), float(parts[1])) + except ValueError as exc: + raise ValueError("random_delay must contain numeric MIN-MAX values") from exc + elif isinstance(random_delay, list): + if len(random_delay) != 2: + raise ValueError("random_delay must contain exactly two values") + merged["random_delay"] = (float(random_delay[0]), float(random_delay[1])) + + headers = merged.get("headers") + if headers is not None and ( + not isinstance(headers, list) or not all(isinstance(header, str) for header in headers) + ): + raise ValueError("headers must be a list of strings") + + _validate_merged_types(merged) # Ensure url is present url = merged.pop("url", "") + if url and not url.lower().startswith(("http://", "https://")): + url = normalize_url(url) - return ScanConfig(url=url, **{k: v for k, v in merged.items() if k in ScanConfig.__dataclass_fields__}) + return ScanConfig(url=url, **{k: v for k, v in merged.items() if k in valid_fields}) diff --git a/panoptic/core.py b/panoptic/core.py index 045cc35..79fb655 100644 --- a/panoptic/core.py +++ b/panoptic/core.py @@ -9,13 +9,14 @@ import asyncio import base64 import contextlib +import hashlib import json import os import random -import re import sys import tempfile import time +from dataclasses import fields as dataclass_fields from pathlib import Path from typing import TextIO from urllib.parse import quote as url_quote @@ -33,12 +34,7 @@ ) from panoptic.cases import load_custom_list, parse_cases -from panoptic.heuristic import ( - SKIP_RETRIEVE_THRESHOLD, - clean_response, - filter_content, - is_match, -) +from panoptic.heuristic import clean_response, filter_content, is_match from panoptic.models import Case, OutputFormat, ScanConfig, ScanResult from panoptic.network import NetworkClient from panoptic.output import CsvFormatter, JsonFormatter, TeeWriter, TextFormatter @@ -47,13 +43,36 @@ from panoptic.utils import ( generate_invalid_filename, get_random_agent, + normalize_os_name, + open_secure_write, + os_matches_restriction, redact_url, + replace_parameter_value, sanitize_filename, validate_header, ) PASSWD_FILES = frozenset({"/etc/passwd", "/etc/security/passwd"}) FUZZ_MARKER = "FUZZ" +CHECKPOINT_VERSION = 1 +_CHECKPOINT_CONFIG_EXCLUSIONS = frozenset( + { + # These options change execution mechanics or presentation, not which + # cases and responses constitute the scan. + "concurrency", + "timeout", + "retries", + "delay", + "random_delay", + "write_files", + "output_format", + "output_file", + "log_file", + "verbose", + "quiet", + "resume_file", + } +) def process_path(config: ScanConfig, location: str) -> str: @@ -61,7 +80,7 @@ def process_path(config: ScanConfig, location: str) -> str: if config.replace_slash: location = location.replace("/", config.replace_slash) - prefix = config.prefix + prefix = config.prefix * config.multiplier if prefix.endswith("/") and location.startswith("/"): location = location.lstrip("/") full_path = f"{prefix}{location}{config.postfix}" @@ -78,12 +97,44 @@ def _encode_param_value(value: str, *, is_post: bool = False) -> str: Safe chars (not encoded): / : LFI path traversal separators must stay literal % : Preserve pre-encoded sequences (e.g., --replace-slash "%2F") - GET-specific safe: = and + (base64 compat, PHP $_GET handles natively) - POST-specific: + MUST be encoded as %2B (decoded as space by form parsers) + The plus sign is always encoded because standard query and form parsers + decode a literal '+' as a space, corrupting some Base64 payloads. """ - if is_post: - return url_quote(value, safe="=/%") - return url_quote(value, safe="=+/%") + return url_quote(value, safe="/%") + + +def _replace_fuzz_in_json(obj: object, replacement: str) -> object: + """Recursively replace the FUZZ marker in every JSON string key and value.""" + if isinstance(obj, dict): + return { + (k.replace(FUZZ_MARKER, replacement) if isinstance(k, str) else k): _replace_fuzz_in_json(v, replacement) + for k, v in obj.items() + } + if isinstance(obj, list): + return [_replace_fuzz_in_json(v, replacement) for v in obj] + if isinstance(obj, str): + return obj.replace(FUZZ_MARKER, replacement) + return obj + + +def _substitute_fuzz(template: str, replacement: str, *, is_post: bool) -> str: + """Replace the FUZZ marker while keeping the surrounding body well-formed. + + A JSON body is parsed and re-serialized so injected Windows paths, backslashes, + and quotes stay valid JSON. A form-encoded body/query has the replacement + percent-encoded so plus signs and delimiters (``&``, ``=``) cannot corrupt the + structure. Any other (opaque) body keeps the raw, user-controlled substitution. + """ + if template.lstrip().startswith(("{", "[")): + try: + obj = json.loads(template) + except (json.JSONDecodeError, ValueError): + pass + else: + return json.dumps(_replace_fuzz_in_json(obj, replacement), separators=(",", ":")) + if "=" in template: + return template.replace(FUZZ_MARKER, _encode_param_value(replacement, is_post=is_post)) + return template.replace(FUZZ_MARKER, replacement) def build_payload(config: ScanConfig, location: str, request_params: str) -> str: @@ -98,48 +149,62 @@ def build_payload(config: ScanConfig, location: str, request_params: str) -> str last_slash = path.rfind("/") if last_slash >= 0: base_path = path[:last_slash] - return f"{parsed.scheme}://{parsed.netloc}{base_path}/{full_path}{query_suffix}" - return f"{parsed.scheme}://{parsed.netloc}/{full_path}{query_suffix}" + return f"{parsed.scheme}://{parsed.netloc}{base_path}/{full_path.lstrip('/')}{query_suffix}" + return f"{parsed.scheme}://{parsed.netloc}/{full_path.lstrip('/')}{query_suffix}" is_post = bool(config.data) result = request_params if FUZZ_MARKER in result: - result = result.replace(FUZZ_MARKER, full_path) + result = _substitute_fuzz(result, full_path, is_post=is_post) elif config.ext_param and config.param and "." in full_path: # When ext_param is set, split path into base and extension path_without_ext, ext = full_path.rsplit(".", 1) encoded_path = _encode_param_value(path_without_ext, is_post=is_post) encoded_ext = _encode_param_value(ext, is_post=is_post) - result = re.sub( - rf"(?:^|(?<=&)){re.escape(config.param)}=(?P[^&]*)", - rf"{config.param}={encoded_path}", - result, - ) - result = re.sub( - rf"(?:^|(?<=&)){re.escape(config.ext_param)}=(?P[^&]*)", - rf"{config.ext_param}={encoded_ext}", - result, - ) + result = replace_parameter_value(result, config.param, encoded_path) + result = replace_parameter_value(result, config.ext_param, encoded_ext) elif config.param: encoded_full_path = _encode_param_value(full_path, is_post=is_post) - result = re.sub( - rf"(?:^|(?<=&)){re.escape(config.param)}=(?P[^&]*)", - rf"{config.param}={encoded_full_path}", - result, - ) + result = replace_parameter_value(result, config.param, encoded_full_path) if config.data: return result return f"{parsed.scheme}://{parsed.netloc}{parsed.path}?{result}" -def save_checkpoint(filepath: str, completed_ids: set[str]) -> None: +def checkpoint_fingerprint(config: ScanConfig, cases: list[Case]) -> str: + """Hash all scan-defining inputs without storing credentials in the checkpoint.""" + # Include new ScanConfig fields by default. A future option cannot silently + # become resume-unsafe merely because this function was not updated. + scan_definition = { + field.name: getattr(config, field.name) + for field in dataclass_fields(config) + if field.name not in _CHECKPOINT_CONFIG_EXCLUSIONS + } + # Status code order does not affect scan semantics. + for code_field in ("match_codes", "filter_codes"): + if scan_definition[code_field] is not None: + scan_definition[code_field] = sorted(set(scan_definition[code_field])) + scan_definition["os_filter"] = normalize_os_name(scan_definition["os_filter"]) + scan_definition["case_ids"] = sorted(case.case_id for case in cases) + serialized = json.dumps(scan_definition, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(serialized.encode()).hexdigest() + + +def save_checkpoint(filepath: str, completed_ids: set[str], fingerprint: str = "") -> None: """Save completed case IDs to a checkpoint file atomically.""" dir_name = os.path.dirname(filepath) or "." fd, tmp_path = tempfile.mkstemp(dir=dir_name, suffix=".tmp") try: with os.fdopen(fd, "w", encoding="utf-8") as f: - json.dump(sorted(completed_ids), f) + json.dump( + { + "version": CHECKPOINT_VERSION, + "fingerprint": fingerprint, + "completed_ids": sorted(completed_ids), + }, + f, + ) os.replace(tmp_path, filepath) except BaseException: with contextlib.suppress(OSError): @@ -147,15 +212,43 @@ def save_checkpoint(filepath: str, completed_ids: set[str]) -> None: raise -def load_checkpoint(filepath: str) -> set[str]: - """Load completed case IDs from a checkpoint file.""" +def _load_checkpoint_data( + filepath: str, + expected_fingerprint: str | None = None, +) -> tuple[set[str], bool]: + """Load checkpoint IDs and report whether the file uses the legacy format.""" if not os.path.exists(filepath): - return set() - try: - with open(filepath, encoding="utf-8") as f: - return set(json.load(f)) - except (json.JSONDecodeError, OSError): - return set() + return set(), False + with open(filepath, encoding="utf-8") as f: + try: + data = json.load(f) + except json.JSONDecodeError as exc: + raise ValueError("checkpoint is not valid JSON") from exc + + if isinstance(data, list): + if not all(isinstance(case_id, str) for case_id in data): + raise ValueError("legacy checkpoint must be a list of strings") + return set(data), True + if not isinstance(data, dict): + raise ValueError("checkpoint must be an object or legacy list") + if data.get("version") != CHECKPOINT_VERSION: + raise ValueError(f"unsupported checkpoint version: {data.get('version')!r}") + + fingerprint = data.get("fingerprint") + if not isinstance(fingerprint, str): + raise ValueError("checkpoint fingerprint is missing or invalid") + if expected_fingerprint is not None and fingerprint != expected_fingerprint: + raise ValueError("checkpoint belongs to a different scan configuration") + + completed_ids = data.get("completed_ids") + if not isinstance(completed_ids, list) or not all(isinstance(case_id, str) for case_id in completed_ids): + raise ValueError("checkpoint completed_ids must be a list of strings") + return set(completed_ids), False + + +def load_checkpoint(filepath: str, expected_fingerprint: str | None = None) -> set[str]: + """Load completed case IDs from a current or legacy checkpoint file.""" + return _load_checkpoint_data(filepath, expected_fingerprint)[0] class Scanner: @@ -170,19 +263,26 @@ def __init__(self, config: ScanConfig) -> None: self.invalid_response: str = "" self.invalid_status_code: int = 0 self.invalid_filename: str = "" - self.restrict_os: str | None = config.os_filter + # Canonicalize OS aliases (e.g. "OSX" -> "OS X") so the runtime restriction + # compares against the same canonical OS labels parse_cases assigns to cases; + # otherwise an aliased --os would load cases and then skip every one of them. + self.restrict_os = normalize_os_name(config.os_filter) self.first_found = False self._first_found_lock = asyncio.Lock() self.completed_ids: set[str] = set() self.enqueued_ids: set[str] = set() self.total_queued = 0 self.total_processed = 0 + self.total_failed = 0 + self.operational_errors: list[str] = [] self._checkpoint_dirty = False self._last_checkpoint_time = 0.0 self._checkpoint_lock = asyncio.Lock() + self._checkpoint_fingerprint = "" + self._checkpoint_disabled = False self._pause_lock = asyncio.Lock() - async def run(self) -> None: + async def run(self) -> int: """Execute the full scan workflow.""" from panoptic import __version__ @@ -190,16 +290,27 @@ async def run(self) -> None: log_fp = None stderr_stream: TextIO = sys.stderr if self.config.log_file: - log_fp = open(self.config.log_file, "w", encoding="utf-8") # noqa: SIM115 + try: + log_fp = open_secure_write(self.config.log_file) + except OSError as exc: + print(f"[!] Cannot open log file '{self.config.log_file}': {exc}", file=sys.stderr) + self._write_output([], TextFormatter(sys.stderr, quiet=self.config.quiet)) + return 2 stderr_stream = TeeWriter(sys.stderr, log_fp) # type: ignore[assignment] try: - await self._run_scan(stderr_stream, __version__) + try: + return await self._run_scan(stderr_stream, __version__) + except (OSError, ValueError, httpx.HTTPError) as exc: + text_out = TextFormatter(stderr_stream, quiet=self.config.quiet) + text_out.write_warning(f"Scan failed: {exc}") + self._write_output([], text_out) + return 2 finally: if log_fp: log_fp.close() - async def _run_scan(self, stderr_stream: TextIO, version: str) -> None: + async def _run_scan(self, stderr_stream: TextIO, version: str) -> int: """Execute the scan with the given output stream.""" text_out = TextFormatter(stderr_stream, quiet=self.config.quiet) @@ -208,10 +319,6 @@ async def _run_scan(self, stderr_stream: TextIO, version: str) -> None: banner_version = f"{version}-{rev}" if rev else version text_out.write_banner(banner_version, redact_url(self.config.url), self.config) - if self.config.random_agent and not self.config.user_agent: - self.config = self.config.replace(user_agent=get_random_agent()) - text_out.write_info(f"Using random User-Agent: {self.config.user_agent}") - if self.config.invalid_ssl: text_out.write_warning("SSL certificate verification is disabled. Traffic is vulnerable to MITM.") @@ -219,10 +326,32 @@ async def _run_scan(self, stderr_stream: TextIO, version: str) -> None: if not cases: text_out.write_warning("No available test cases with the specified attributes.") - return + self._write_output([], text_out) + return 2 + + if self.config.random_agent and not self.config.user_agent: + self.config = self.config.replace(user_agent=get_random_agent()) + text_out.write_info(f"Using random User-Agent: {self.config.user_agent}") + + self._checkpoint_fingerprint = checkpoint_fingerprint(self.config, cases) if self.config.resume_file: - self.completed_ids = load_checkpoint(self.config.resume_file) + try: + self.completed_ids, legacy_checkpoint = _load_checkpoint_data( + self.config.resume_file, + self._checkpoint_fingerprint, + ) + except (OSError, ValueError) as exc: + text_out.write_warning(f"Ignoring resume checkpoint: {exc}") + self.completed_ids = set() + else: + valid_case_ids = {case.case_id for case in cases} + self.completed_ids.intersection_update(valid_case_ids) + if legacy_checkpoint: + text_out.write_warning( + "Legacy checkpoint has no scan fingerprint; " + "only IDs matching the current case set were accepted" + ) if self.completed_ids: text_out.write_info(f"Resuming: {len(self.completed_ids)} cases already completed") @@ -235,7 +364,8 @@ async def _run_scan(self, stderr_stream: TextIO, version: str) -> None: orig_resp = await self._fetch(client, self._base_url, self.config.data or self.config.url) if orig_resp is None: text_out.write_warning("Cannot connect to target. Check connection settings.") - return + self._write_output([], text_out) + return 2 self.original_response = orig_resp.text self.invalid_filename = generate_invalid_filename() @@ -245,7 +375,8 @@ async def _run_scan(self, stderr_stream: TextIO, version: str) -> None: if inv_resp is None: text_out.write_warning("Cannot retrieve invalid response baseline.") - return + self._write_output([], text_out) + return 2 self.invalid_response = inv_resp.text self.invalid_status_code = inv_resp.status_code @@ -296,15 +427,33 @@ async def worker() -> None: try: async with self._pause_lock: pass # Block while interactive prompt is active - await self._process_case(case, client, request_params, queue, scan_out, progress_ctx) - self.total_processed += 1 - # Update progress bar total when dynamic cases are added + processed = await self._process_case( + case, + client, + request_params, + queue, + scan_out, + progress_ctx, + ) + if processed: + self.total_processed += 1 + else: + self.total_failed += 1 + except Exception as exc: + self.total_failed += 1 + message = f"Case '{case.location}' failed: {exc}" + self.operational_errors.append(message) + scan_out.write_warning(message) + finally: + queue.task_done() + # Update progress even for failed cases so the queue's + # completion state remains visible and accurate. if progress_ctx is not None and progress_task_id is not None: progress_ctx.update( - progress_task_id, total=self.total_queued, completed=self.total_processed + progress_task_id, + total=self.total_queued, + completed=self.total_processed + self.total_failed, ) - finally: - queue.task_done() worker_tasks = [asyncio.create_task(worker()) for _ in range(self.config.concurrency)] @@ -314,7 +463,10 @@ async def worker() -> None: finally: await self._flush_checkpoint() stop_event.set() - await asyncio.gather(*worker_tasks, return_exceptions=True) + worker_results = await asyncio.gather(*worker_tasks, return_exceptions=True) + for result in worker_results: + if isinstance(result, BaseException): + self.operational_errors.append(f"Worker terminated unexpectedly: {result}") finally: if progress_ctx is not None: progress_ctx.stop() @@ -331,22 +483,40 @@ async def worker() -> None: text_out.write_info(f"Completed in {elapsed:.1f}s ({rps:.0f} req/s)") text_out.write_info(f"Finishing scan at: {time.strftime('%X')}") - if self.config.output_format != OutputFormat.TEXT or self.config.output_file: - - def _write_output(stream: TextIO) -> None: - match self.config.output_format: - case OutputFormat.JSON: - JsonFormatter(stream).write_results(found_results) - case OutputFormat.CSV: - CsvFormatter(stream).write_results(found_results) - case OutputFormat.TEXT: - TextFormatter(stream).write_summary(found_results, self.total_processed) + if self.total_failed: + text_out.write_warning(f"{self.total_failed} requests failed and were not checkpointed") + for error in self.operational_errors: + if not error.startswith("Case '"): + text_out.write_warning(error) + + output_ok = self._write_output(found_results, text_out) + all_requests_failed = self.total_failed > 0 and self.total_processed == 0 + return 0 if output_ok and not all_requests_failed and not self.operational_errors else 2 + + def _write_output(self, found_results: list[ScanResult], text_out: TextFormatter) -> bool: + """Write configured machine/file output, including valid empty results on failure.""" + if self.config.output_format == OutputFormat.TEXT and not self.config.output_file: + return True + + def write_to(stream: TextIO) -> None: + match self.config.output_format: + case OutputFormat.JSON: + JsonFormatter(stream).write_results(found_results) + case OutputFormat.CSV: + CsvFormatter(stream).write_results(found_results) + case OutputFormat.TEXT: + TextFormatter(stream).write_results(found_results, self.total_processed) + try: if self.config.output_file: - with open(self.config.output_file, "w", encoding="utf-8") as stream: - _write_output(stream) + with open_secure_write(self.config.output_file, newline="") as stream: + write_to(stream) else: - _write_output(sys.stdout) + write_to(sys.stdout) + except OSError as exc: + text_out.write_warning(f"Cannot write output: {exc}") + return False + return True async def _process_case( self, @@ -356,7 +526,7 @@ async def _process_case( queue: asyncio.Queue[Case], text_out: TextFormatter, progress: Progress | None = None, - ) -> None: + ) -> bool: """Process a single case: fetch, compare, record result.""" if self.config.random_delay: delay = random.uniform(*self.config.random_delay) @@ -364,10 +534,10 @@ async def _process_case( elif self.config.delay > 0: await asyncio.sleep(self.config.delay) - if self.restrict_os and case.os and case.os != self.restrict_os: + if not os_matches_restriction(case.os, self.restrict_os): # OS-filtered cases are marked complete so they are not retried on resume await self._mark_completed(case) - return + return True payload_str = build_payload(self.config, case.location, request_params) @@ -379,57 +549,35 @@ async def _process_case( if response is None: # Network failure: do NOT checkpoint so the case is retried on resume - return - - # Skip responses where the server returned a different error class than - # the baseline. E.g. baseline returns 200 (PHP warning) but this path - # gets a 403/404 from the web server before the app even runs — that's - # not a found file, it's a different error page. - if response.status_code // 100 != self.invalid_status_code // 100 and response.status_code >= 400: - await self._mark_completed(case) - return + return False # User-specified status code filtering if self.config.filter_codes and response.status_code in self.config.filter_codes: await self._mark_completed(case) - return + return True if self.config.match_codes and response.status_code not in self.config.match_codes: await self._mark_completed(case) - return + return True - # Content-Length skip optimization: if response is much larger than baseline - # and we're not writing files, classify as found by status alone - raw_length = response.headers.get("content-length", "0") - content_length = int(raw_length) if raw_length.isdigit() else 0 - baseline_length = max(len(self.original_response), len(self.invalid_response)) + # Unless the user explicitly selected status codes, skip responses where + # the server returned a different error class than the invalid baseline. if ( - not self.config.write_files - and not self.config.match_string - and not self.config.bad_string - and content_length > 0 - and content_length - baseline_length > SKIP_RETRIEVE_THRESHOLD + not self.config.match_codes + and response.status_code // 100 != self.invalid_status_code // 100 + and response.status_code >= 400 ): - result = ScanResult( - case=case, - found=True, - url=payload_str, - status_code=response.status_code, - content_length=content_length, - ) - self.results.append(result) - text_out.write_found(result) await self._mark_completed(case) - return + return True html = response.text if self.config.bad_string and self.config.bad_string in html: await self._mark_completed(case) - return + return True if self.config.match_string and self.config.match_string not in html: await self._mark_completed(case) - return + return True cleaned_html = clean_response(html, case.location) cleaned_invalid = clean_response(self.invalid_response, self.invalid_filename) @@ -459,10 +607,15 @@ async def _process_case( if progress is not None: progress.stop() try: - answer = await asyncio.to_thread( - input, - f"[?] Restrict further scans to '{case.os}'? [Y/n] ", - ) + try: + answer = await asyncio.to_thread( + input, + f"[?] Restrict further scans to '{case.os}'? [Y/n] ", + ) + except EOFError: + # EOF means there is no interactive user + # available to approve narrowing the scan. + answer = "n" if answer.strip().lower() in ("", "y", "yes"): self.restrict_os = case.os finally: @@ -479,6 +632,7 @@ async def _process_case( await self._enqueue_new_cases(extract_binlog_cases(html, case), queue) await self._mark_completed(case) + return True def _fuzz_headers(self, location: str) -> dict[str, str] | None: """Build per-request headers with FUZZ replaced, or None if no FUZZ in headers.""" @@ -488,14 +642,15 @@ def _fuzz_headers(self, location: str) -> dict[str, str] | None: fuzz_hdrs: dict[str, str] = {} processed = process_path(self.config, location) for hdr in self.config.headers: - if FUZZ_MARKER in hdr: - name, value = validate_header(hdr) - substituted = value.replace(FUZZ_MARKER, processed) - # Re-validate after substitution: case locations from custom - # lists could inject control characters via the FUZZ marker. - if any(c in substituted for c in "\r\n\x00"): - continue - fuzz_hdrs[name] = substituted + name, value = validate_header(hdr, warn_deprecated=False) + if FUZZ_MARKER not in value: + continue + substituted = value.replace(FUZZ_MARKER, processed) + # Re-validate after substitution: case locations from custom + # lists could inject control characters via the FUZZ marker. + if any(c in substituted for c in "\r\n\x00"): + continue + fuzz_hdrs[name] = substituted return fuzz_hdrs or None async def _fetch( @@ -513,7 +668,7 @@ async def _fetch( async def _mark_completed(self, case: Case) -> None: """Record a case as completed for resume/checkpoint support.""" self.completed_ids.add(case.case_id) - if self.config.resume_file: + if self.config.resume_file and not self._checkpoint_disabled: self._checkpoint_dirty = True now = time.monotonic() if now - self._last_checkpoint_time >= 5.0: @@ -523,14 +678,29 @@ async def _mark_completed(self, case: Case) -> None: async def _flush_checkpoint(self) -> None: """Flush checkpoint to disk if dirty.""" - if self._checkpoint_dirty and self.config.resume_file: - await asyncio.to_thread( - save_checkpoint, - self.config.resume_file, - self.completed_ids.copy(), - ) - self._checkpoint_dirty = False - self._last_checkpoint_time = time.monotonic() + if self._checkpoint_dirty and self.config.resume_file and not self._checkpoint_disabled: + # Snapshot exactly what is being persisted. completed_ids grows while + # the blocking save runs in a worker thread; if a case completes during + # the save the snapshot will not contain it, so the dirty flag must stay + # set to guarantee the newer ID is flushed on the next write. + snapshot = self.completed_ids.copy() + try: + await asyncio.to_thread( + save_checkpoint, + self.config.resume_file, + snapshot, + self._checkpoint_fingerprint, + ) + except OSError as exc: + self._checkpoint_disabled = True + self._checkpoint_dirty = False + self.operational_errors.append(f"Checkpoint disabled after write failure: {exc}") + else: + self._last_checkpoint_time = time.monotonic() + # completed_ids only ever grows, so a size change means new IDs + # landed during the save and still need to be written. + if len(self.completed_ids) == len(snapshot): + self._checkpoint_dirty = False async def _enqueue_new_cases(self, cases: list[Case], queue: asyncio.Queue[Case]) -> None: """Add newly discovered cases to the queue, skipping duplicates.""" @@ -545,10 +715,14 @@ def _write_file(self, case: Case, html: str) -> None: """Write discovered file content to local output directory.""" try: base = (Path.cwd() / "output").resolve() - output_dir = (base / self._parsed_url.netloc.replace(":", "_")).resolve() - if not str(output_dir).startswith(str(base)): + host = self._parsed_url.hostname or "unknown-host" + if self._parsed_url.port: + host = f"{host}_{self._parsed_url.port}" + output_dir = (base / sanitize_filename(host)).resolve() + if not output_dir.is_relative_to(base): raise ValueError(f"Unsafe output directory: {output_dir}") - output_dir.mkdir(parents=True, exist_ok=True) + output_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + output_dir.chmod(0o700) sanitized = sanitize_filename(case.location) # Always include case_id suffix to prevent collisions from paths that @@ -563,6 +737,9 @@ def _write_file(self, case: Case, html: str) -> None: content = filter_content(html, self.original_response) if self.original_response else html - filepath.write_text(content, encoding="utf-8") - except OSError as exc: + # open_secure_write uses the strongest final-component symlink and + # permission hardening exposed by the current platform. + with open_secure_write(str(filepath)) as stream: + stream.write(content) + except (OSError, ValueError) as exc: print(f"[!] Warning: could not write file for '{case.location}': {exc}", file=sys.stderr) diff --git a/panoptic/data/cases.csv b/panoptic/data/cases.csv index d88388e..4e19dd2 100644 --- a/panoptic/data/cases.csv +++ b/panoptic/data/cases.csv @@ -94,13 +94,13 @@ path,os,software,category,type /var/log/mysqlderror.log,*NIX,MySQL,Databases,log /var/mysql-bin.index,*NIX,MySQL,Databases,log /var/mysql.log,*NIX,MySQL,Databases,log -/usr/local/mysql/data/mysql-bin.index,OSX,MySQL,Databases,log -/usr/local/mysql/data/mysql-bin.log,OSX,MySQL,Databases,log -/usr/local/mysql/data/mysql-slow.log,OSX,MySQL,Databases,log -/usr/local/mysql/data/mysql.err,OSX,MySQL,Databases,log -/usr/local/mysql/data/mysql.log,OSX,MySQL,Databases,log -/usr/local/mysql/data/mysqlderror.log,OSX,MySQL,Databases,log -/usr/local/mysql/data/{HOST}.err,OSX,MySQL,Databases,log +/usr/local/mysql/data/mysql-bin.index,OS X,MySQL,Databases,log +/usr/local/mysql/data/mysql-bin.log,OS X,MySQL,Databases,log +/usr/local/mysql/data/mysql-slow.log,OS X,MySQL,Databases,log +/usr/local/mysql/data/mysql.err,OS X,MySQL,Databases,log +/usr/local/mysql/data/mysql.log,OS X,MySQL,Databases,log +/usr/local/mysql/data/mysqlderror.log,OS X,MySQL,Databases,log +/usr/local/mysql/data/{HOST}.err,OS X,MySQL,Databases,log /MySQL/data/mysql-bin.index,Windows,MySQL,Databases,log /MySQL/data/mysql-bin.log,Windows,MySQL,Databases,log /MySQL/data/mysql.err,Windows,MySQL,Databases,log @@ -427,7 +427,7 @@ path,os,software,category,type /usr/local/php5/lib/php.ini,*NIX,PHP,Programming,conf /var/local/www/conf/php.ini,*NIX,PHP,Programming,conf /web/conf/php.ini,*NIX,PHP,Programming,conf -/Volumes/Macintosh_HD1/usr/local/php/lib/php.ini,OSX,PHP,Programming,conf +/Volumes/Macintosh_HD1/usr/local/php/lib/php.ini,OS X,PHP,Programming,conf /NetServer/bin/stable/apache/php.ini,Windows,PHP,Programming,conf /PHP/php.ini,Windows,PHP,Programming,conf /WINDOWS/php.ini,Windows,PHP,Programming,conf @@ -970,32 +970,32 @@ path,os,software,category,type /var/log/webmin/miniserv.log,*NIX,Webmin,Web Servers,log /usr/local/zeus/web/global.cfg,*NIX,Zeus Web Server (ZWS),Web Servers,conf /usr/local/zeus/web/log/errors,*NIX,Zeus Web Server (ZWS),Web Servers,log -/etc/lighttpd/lighthttpd.conf,*NIX,lighthttpd,Web Servers,conf -/home/user/lighttpd/lighttpd.conf,*NIX,lighthttpd,Web Servers,conf -/usr/home/user/lighttpd/lighttpd.conf,*NIX,lighthttpd,Web Servers,conf -/usr/local/etc/lighttpd.conf,*NIX,lighthttpd,Web Servers,conf -/usr/local/etc/lighttpd.conf.new,*NIX,lighthttpd,Web Servers,conf -/usr/local/lighttpd/conf/lighttpd.conf,*NIX,lighthttpd,Web Servers,conf -/var/www/.lighttpdpassword,*NIX,lighthttpd,Web Servers,conf -/usr/home/user/var/log/apache.log,*NIX,lighthttpd,Web Servers,log -/usr/home/user/var/log/lighttpd.error.log,*NIX,lighthttpd,Web Servers,log -/usr/local/apache/logs/lighttpd.error.log,*NIX,lighthttpd,Web Servers,log -/usr/local/apache/logs/lighttpd.log,*NIX,lighthttpd,Web Servers,log -/usr/local/apache2/logs/lighttpd.error.log,*NIX,lighthttpd,Web Servers,log -/usr/local/apache2/logs/lighttpd.log,*NIX,lighthttpd,Web Servers,log -/usr/local/lighttpd/log/access.log,*NIX,lighthttpd,Web Servers,log -/usr/local/lighttpd/log/lighttpd.error.log,*NIX,lighthttpd,Web Servers,log -/var/lighttpd.log,*NIX,lighthttpd,Web Servers,log -/var/log/lighttpd.access.log,*NIX,lighthttpd,Web Servers,log -/var/log/lighttpd.error.log,*NIX,lighthttpd,Web Servers,log -/var/log/lighttpd/,*NIX,lighthttpd,Web Servers,log -/var/log/lighttpd/access.log,*NIX,lighthttpd,Web Servers,log -/var/log/lighttpd/access.www.log,*NIX,lighthttpd,Web Servers,log -/var/log/lighttpd/error.log,*NIX,lighthttpd,Web Servers,log -/var/log/lighttpd/error.www.log,*NIX,lighthttpd,Web Servers,log -/var/log/lighttpd/{DOMAIN}/access.log,*NIX,lighthttpd,Web Servers,log -/var/log/lighttpd/{DOMAIN}/error.log,*NIX,lighthttpd,Web Servers,log -/var/logs/access.log,*NIX,lighthttpd,Web Servers,log +/etc/lighttpd/lighthttpd.conf,*NIX,lighttpd,Web Servers,conf +/home/user/lighttpd/lighttpd.conf,*NIX,lighttpd,Web Servers,conf +/usr/home/user/lighttpd/lighttpd.conf,*NIX,lighttpd,Web Servers,conf +/usr/local/etc/lighttpd.conf,*NIX,lighttpd,Web Servers,conf +/usr/local/etc/lighttpd.conf.new,*NIX,lighttpd,Web Servers,conf +/usr/local/lighttpd/conf/lighttpd.conf,*NIX,lighttpd,Web Servers,conf +/var/www/.lighttpdpassword,*NIX,lighttpd,Web Servers,conf +/usr/home/user/var/log/apache.log,*NIX,lighttpd,Web Servers,log +/usr/home/user/var/log/lighttpd.error.log,*NIX,lighttpd,Web Servers,log +/usr/local/apache/logs/lighttpd.error.log,*NIX,lighttpd,Web Servers,log +/usr/local/apache/logs/lighttpd.log,*NIX,lighttpd,Web Servers,log +/usr/local/apache2/logs/lighttpd.error.log,*NIX,lighttpd,Web Servers,log +/usr/local/apache2/logs/lighttpd.log,*NIX,lighttpd,Web Servers,log +/usr/local/lighttpd/log/access.log,*NIX,lighttpd,Web Servers,log +/usr/local/lighttpd/log/lighttpd.error.log,*NIX,lighttpd,Web Servers,log +/var/lighttpd.log,*NIX,lighttpd,Web Servers,log +/var/log/lighttpd.access.log,*NIX,lighttpd,Web Servers,log +/var/log/lighttpd.error.log,*NIX,lighttpd,Web Servers,log +/var/log/lighttpd/,*NIX,lighttpd,Web Servers,log +/var/log/lighttpd/access.log,*NIX,lighttpd,Web Servers,log +/var/log/lighttpd/access.www.log,*NIX,lighttpd,Web Servers,log +/var/log/lighttpd/error.log,*NIX,lighttpd,Web Servers,log +/var/log/lighttpd/error.www.log,*NIX,lighttpd,Web Servers,log +/var/log/lighttpd/{DOMAIN}/access.log,*NIX,lighttpd,Web Servers,log +/var/log/lighttpd/{DOMAIN}/error.log,*NIX,lighttpd,Web Servers,log +/var/logs/access.log,*NIX,lighttpd,Web Servers,log /etc/nginx/nginx.conf,*NIX,nginx,Web Servers,conf /usr/local/etc/nginx/nginx.conf,*NIX,nginx,Web Servers,conf /usr/local/nginx/conf/nginx.conf,*NIX,nginx,Web Servers,conf diff --git a/panoptic/heuristic.py b/panoptic/heuristic.py index 86085ba..48eae58 100644 --- a/panoptic/heuristic.py +++ b/panoptic/heuristic.py @@ -11,9 +11,6 @@ # Default similarity ratio above which responses are considered "the same" DEFAULT_HEURISTIC_RATIO = 0.9 -# Content-Length threshold for skipping full body retrieval -SKIP_RETRIEVE_THRESHOLD = 1000 - def clean_response(response: str, filepath: str) -> str: """Remove occurrences of a filepath from a response string. @@ -58,8 +55,17 @@ def is_match( if html is None or invalid_response is None: return False + if html == invalid_response: + return False + + # quick_ratio() is an upper bound: when it is already below the threshold, + # the full ratio must also be below it and can safely be skipped. A high + # quick ratio is not conclusive (reordered bodies can score 1.0), so those + # responses still require the full comparison. matcher = difflib.SequenceMatcher(None, html, invalid_response) - return matcher.quick_ratio() < ratio + if matcher.quick_ratio() < ratio: + return True + return matcher.ratio() < ratio def filter_content(html: str, original_response: str) -> str: diff --git a/panoptic/models.py b/panoptic/models.py index a9a5101..a487866 100644 --- a/panoptic/models.py +++ b/panoptic/models.py @@ -67,6 +67,10 @@ class ScanConfig: def __post_init__(self) -> None: if self.concurrency < 1: raise ValueError(f"concurrency must be >= 1, got {self.concurrency}") + if self.retries < 0: + raise ValueError(f"retries must be >= 0, got {self.retries}") + if self.multiplier < 1: + raise ValueError(f"multiplier must be >= 1, got {self.multiplier}") if not (0.0 < self.heuristic_ratio < 1.0): raise ValueError(f"heuristic_ratio must be between 0 and 1 (exclusive), got {self.heuristic_ratio}") if self.timeout <= 0: diff --git a/panoptic/network.py b/panoptic/network.py index 14bfb31..55a2c65 100644 --- a/panoptic/network.py +++ b/panoptic/network.py @@ -5,7 +5,6 @@ from __future__ import annotations -import ssl import sys from types import TracebackType @@ -28,29 +27,19 @@ def __init__(self, config: ScanConfig) -> None: self._client: httpx.AsyncClient | None = None async def __aenter__(self) -> NetworkClient: - timeout = httpx.Timeout(self.config.timeout, connect=5.0) - - proxy = self.config.proxy - - # SSL verification - ssl_verify: bool | ssl.SSLContext = True - if self.config.invalid_ssl: - ctx = ssl.create_default_context() - ctx.check_hostname = False - ctx.verify_mode = ssl.CERT_NONE - ssl_verify = ctx - - # Transport with retry - transport = httpx.AsyncHTTPTransport(retries=self.config.retries) + timeout = httpx.Timeout(self.config.timeout) # Build default headers headers = self._build_headers() + # Do not pass an explicit transport here. In HTTPX, doing so makes + # client-level verify/trust_env settings inapplicable to direct requests + # and prevents environment proxy mounts from being created. Retries are + # handled uniformly in fetch() so they also work with proxy transports. self._client = httpx.AsyncClient( timeout=timeout, - proxy=proxy, - verify=ssl_verify, - transport=transport, + proxy=self.config.proxy, + verify=not self.config.invalid_ssl, headers=headers, follow_redirects=self.config.follow_redirects, trust_env=not self.config.ignore_proxy, @@ -81,27 +70,31 @@ async def fetch( if self._client is None: raise RuntimeError("NetworkClient must be used as async context manager") - try: - if data is not None: - # Infer content type from payload: JSON bodies get application/json, - # everything else defaults to form-encoded. - content_type = "application/x-www-form-urlencoded" - stripped = data.lstrip() - if stripped.startswith(("{", "[")): - content_type = "application/json" - post_headers = {"Content-Type": content_type} - if headers: - post_headers.update(headers) - response = await self._client.post( - url, - content=data.encode("utf-8"), - headers=post_headers, - ) - else: - response = await self._client.get(url, headers=headers) - return response - except httpx.HTTPError: - return None + for attempt in range(self.config.retries + 1): + try: + if data is not None: + # Infer content type from payload: JSON bodies get application/json, + # everything else defaults to form-encoded. + content_type = "application/x-www-form-urlencoded" + stripped = data.lstrip() + if stripped.startswith(("{", "[")): + content_type = "application/json" + post_headers = {"Content-Type": content_type} + if headers: + post_headers.update(headers) + return await self._client.post( + url, + content=data.encode("utf-8"), + headers=post_headers, + ) + return await self._client.get(url, headers=headers) + except (httpx.ConnectError, httpx.ConnectTimeout): + if attempt >= self.config.retries: + return None + except httpx.HTTPError: + return None + + return None def _build_headers(self) -> dict[str, str]: """Build default headers from config, with validation.""" diff --git a/panoptic/output.py b/panoptic/output.py index 1dc25a9..2ff6e05 100644 --- a/panoptic/output.py +++ b/panoptic/output.py @@ -14,7 +14,7 @@ from rich.markup import escape as rich_escape from panoptic.models import ScanConfig, ScanResult -from panoptic.utils import _PARAM_VALUE_RE, redact_url +from panoptic.utils import redact_parameter_values, redact_url, validate_header class TeeWriter: @@ -37,29 +37,42 @@ def fileno(self) -> int: def _redact_json_values(obj: object) -> object: - """Recursively replace all string values in a JSON structure with '***'.""" + """Recursively redact every scalar leaf in a JSON structure. + + Strings, numbers, and booleans are all replaced with '***' — a credential + can be a numeric PIN or a bare token just as easily as a string, so redacting + only strings would leak scalar secrets. ``null`` is preserved because it + carries no value and keeps the structure faithful. + """ if isinstance(obj, dict): return {k: _redact_json_values(v) for k, v in obj.items()} if isinstance(obj, list): return [_redact_json_values(v) for v in obj] - if isinstance(obj, str): - return "***" - return obj + if obj is None: + return None + return "***" def _redact_field(url: str) -> str: """Redact a URL or POST body for safe serialization.""" - if url.startswith(("http://", "https://")): + if url.lower().startswith(("http://", "https://")): return redact_url(url) - # JSON body: redact all string values by parsing then recursively replacing - if url.lstrip().startswith("{"): - try: - obj = json.loads(url) + # JSON body: redact every leaf value, preserving container structure and keys. + try: + obj = json.loads(url) + except (json.JSONDecodeError, ValueError): + pass + else: + if isinstance(obj, (dict, list)): return json.dumps(_redact_json_values(obj)) - except (json.JSONDecodeError, ValueError): - pass - # POST body: redact form-encoded values (key=VALUE → key=***) - return _PARAM_VALUE_RE.sub("***", url) + # Bare JSON scalar (number, quoted string, bool, null): nothing structural + # to keep, so redact the whole field. + return "***" + # Form-encoded body: redact values while keeping keys for context (key=VALUE → key=***). + if "=" in url: + return redact_parameter_values(url) + # Opaque body with no recognizable structure (e.g. a raw token): redact entirely. + return "***" if url else url def _result_to_dict(r: ScanResult) -> dict[str, object]: @@ -86,7 +99,13 @@ def _has_fuzz(config: ScanConfig) -> bool: if config.data and _FUZZ_MARKER in config.data: return True if config.headers: - return any(_FUZZ_MARKER in h for h in config.headers) + for header in config.headers: + try: + _name, value = validate_header(header, warn_deprecated=False) + except ValueError: + continue + if _FUZZ_MARKER in value: + return True return False @@ -143,19 +162,21 @@ def write_banner( pupil = _scan_mode_pupil(config) if config else "()" mode = _scan_mode_label(config) if config else "" mode_str = f" [dim]·[/dim] [cyan]{mode}[/cyan]" if mode else "" + # The URL is attacker-influenced (target, redacted userinfo, IPv6 brackets) + # and must be escaped so it cannot inject or break Rich markup. self._console.print( - f"[bold cyan] .-',--.`-.[/] [bold]Panoptic[/] {version}\n" - f"[bold cyan]<_ | {pupil} | _>[/] [dim]{url}[/dim]\n" + f"[bold cyan] .-',--.`-.[/] [bold]Panoptic[/] {rich_escape(version)}\n" + f"[bold cyan]<_ | {pupil} | _>[/] [dim]{rich_escape(url)}[/dim]\n" f"[bold cyan] `-`=='-'[/] {mode_str}\n" ) def write_info(self, message: str) -> None: if self._quiet: return - self._console.print(f"[blue][i][/blue] {message}") + self._console.print(f"[blue][i][/blue] {rich_escape(message)}") def write_warning(self, message: str) -> None: - self._console.print(f"[red][!][/red] {message}") + self._console.print(f"[red][!][/red] {rich_escape(message)}") def write_found(self, result: ScanResult) -> None: case = result.case @@ -167,7 +188,7 @@ def write_found(self, result: ScanResult) -> None: def write_verbose(self, message: str) -> None: if self._quiet: return - self._console.print(f"[dim][*] {message}[/dim]") + self._console.print(f"[dim][*] {rich_escape(message)}[/dim]") def write_summary(self, found: list[ScanResult], total_cases: int) -> None: if self._quiet: @@ -176,6 +197,23 @@ def write_summary(self, found: list[ScanResult], total_cases: int) -> None: self._console.print(f" Cases tested: {total_cases}") self._console.print(f" Files found: [green]{len(found)}[/green]") + def write_results(self, found: list[ScanResult], total_cases: int) -> None: + """Write complete plain-text results for files and redirected output.""" + if self._quiet: + return + if found: + self._console.print("[bold]Findings[/bold]") + for result in found: + status = result.status_code if result.status_code is not None else "-" + length = result.content_length if result.content_length is not None else "-" + self._console.print( + f" {rich_escape(result.case.location)} [dim](status={status}, length={length})[/dim]", + soft_wrap=True, + ) + else: + self._console.print("No files found.") + self.write_summary(found, total_cases) + class JsonFormatter: """JSON output for pipeline integration.""" diff --git a/panoptic/parsers.py b/panoptic/parsers.py index 5a12877..c256b91 100644 --- a/panoptic/parsers.py +++ b/panoptic/parsers.py @@ -74,8 +74,13 @@ def extract_binlog_cases( if not index_content: return [] - # Match entries like '.\mysql-bin.000001' (dot then backslash prefix). - binlogs = re.findall(r"\.\\(?Pmysql-bin\.\d{1,6})", index_content) + # Support Windows and Unix relative entries plus the modern MySQL 8 + # default "binlog" basename. + binlogs = re.findall( + r"(?:^|[\r\n])(?:\.[\\/])?(?P(?:mysql-bin|binlog)\.\d{1,9})(?:\r?$)", + index_content, + flags=re.MULTILINE, + ) # Extract directory from parent case location last_slash = parent_case.location.rfind("/") diff --git a/panoptic/update.py b/panoptic/update.py index 30cdfef..5bdae18 100644 --- a/panoptic/update.py +++ b/panoptic/update.py @@ -9,29 +9,50 @@ import os import re import subprocess +from urllib.parse import urlsplit GIT_REPOSITORY_URL = "https://github.com/lightos/Panoptic.git" +GIT_UPSTREAM_BRANCH = "main" +GIT_UPSTREAM_REF = f"refs/heads/{GIT_UPSTREAM_BRANCH}" _PACKAGE_DIR = os.path.dirname(os.path.abspath(__file__)) _PROJECT_ROOT = os.path.dirname(_PACKAGE_DIR) def _normalise_git_url(url: str) -> str: - """Normalise a git remote URL for comparison (strip trailing .git and slash).""" - url = url.rstrip("/") - if url.endswith(".git"): - url = url[:-4] - return url.lower() + """Normalize HTTPS, SSH URL, and scp-style git remotes for comparison.""" + url = url.strip().rstrip("/") + if "://" not in url: + scp_match = re.fullmatch(r"(?:[^@]+@)?(?P[^:]+):(?P.+)", url) + if scp_match: + host = scp_match.group("host") + path = scp_match.group("path") + else: + return url.removesuffix(".git").lower() + else: + parsed = urlsplit(url) + host = parsed.hostname or "" + path = parsed.path.lstrip("/") + return f"{host}/{path.removesuffix('.git')}".lower() + +def _uses_secure_git_transport(url: str) -> bool: + """Return whether a remote uses authenticated HTTPS or SSH transport.""" + stripped = url.strip() + if "://" not in stripped: + # Git's scp-like syntax (git@github.com:owner/repo.git) uses SSH. + return re.fullmatch(r"(?:[^@]+@)?[^:]+:.+", stripped) is not None + return urlsplit(stripped).scheme.lower() in {"https", "ssh"} -def do_update() -> None: + +def do_update() -> int: """Perform self-update from git or print pip guidance.""" git_dir = os.path.join(_PROJECT_ROOT, ".git") if not os.path.exists(git_dir): print("[i] Panoptic appears to be installed via pip.") print("[i] To update, run: pip install -U panoptic") - return + return 0 print("[i] Checking for updates...") @@ -42,32 +63,54 @@ def do_update() -> None: ["git", "remote", "get-url", "origin"], capture_output=True, cwd=_PROJECT_ROOT, + timeout=30, ) except FileNotFoundError: print("[!] 'git' is not installed or not in PATH.") print("[i] Please install git or update via: pip install -U panoptic") - return + return 2 + except subprocess.TimeoutExpired: + print("[!] Timed out while checking the git remote.") + return 2 if remote_check.returncode != 0: print("[!] Cannot determine remote URL. Please verify your git configuration.") - return + return 2 remote_url = remote_check.stdout.decode("utf-8", errors="replace").strip() - if _normalise_git_url(remote_url) != _normalise_git_url(GIT_REPOSITORY_URL): - print(f"[!] Remote 'origin' ({remote_url}) does not match expected upstream.") + if not _uses_secure_git_transport(remote_url) or _normalise_git_url(remote_url) != _normalise_git_url( + GIT_REPOSITORY_URL + ): + print(f"[!] Remote 'origin' ({remote_url}) is not a trusted upstream URL.") print(f"[i] Expected: {GIT_REPOSITORY_URL}") + print("[i] HTTPS and SSH remotes for the official repository are accepted.") print("[i] Aborting update for safety. Please verify your git remotes.") - return + return 2 try: + branch_check = subprocess.run( + ["git", "symbolic-ref", "--quiet", "--short", "HEAD"], + capture_output=True, + cwd=_PROJECT_ROOT, + timeout=30, + ) + current_branch = branch_check.stdout.decode("utf-8", errors="replace").strip() + if branch_check.returncode != 0 or current_branch != GIT_UPSTREAM_BRANCH: + print(f"[!] Self-update requires the '{GIT_UPSTREAM_BRANCH}' branch to be checked out.") + return 2 + result = subprocess.run( - ["git", "pull", "--ff-only", "origin"], + ["git", "pull", "--ff-only", "origin", GIT_UPSTREAM_REF], capture_output=True, cwd=_PROJECT_ROOT, + timeout=300, ) except FileNotFoundError: print("[!] 'git' is not installed or not in PATH.") - return + return 2 + except subprocess.TimeoutExpired: + print("[!] Update timed out.") + return 2 if result.returncode == 0: stdout = result.stdout.decode("utf-8", errors="replace") @@ -77,10 +120,12 @@ def do_update() -> None: print(f"[i] Updated to revision '{revision}'.") else: print(f"[i] Already at the latest revision '{revision}'.") + return 0 else: stderr = result.stderr.decode("utf-8", errors="replace").strip() print(f"[!] Update failed: {stderr}") print("[i] Please make sure 'git' is installed and accessible.") + return 2 def get_revision() -> str | None: @@ -90,11 +135,12 @@ def get_revision() -> str | None: ["git", "rev-parse", "--verify", "HEAD"], capture_output=True, cwd=_PROJECT_ROOT, + timeout=30, ) if result.returncode == 0: stdout = result.stdout.decode("utf-8", errors="replace").strip() if re.match(r"[0-9a-f]{40}", stdout): return stdout[:7] - except FileNotFoundError: + except (FileNotFoundError, subprocess.TimeoutExpired): pass return None diff --git a/panoptic/utils.py b/panoptic/utils.py index 2fc446a..750206e 100644 --- a/panoptic/utils.py +++ b/panoptic/utils.py @@ -2,16 +2,75 @@ from __future__ import annotations +import errno +import os import random import re import secrets +import stat import sys from importlib.resources import files -from typing import Any -from urllib.parse import urlsplit - -# Regex to redact values in key=value&... encoded strings (query params, form bodies) -_PARAM_VALUE_RE = re.compile(r"(?<==)[^&]*") +from typing import Any, TextIO +from urllib.parse import unquote_plus, urlsplit + +_HEADER_NAME_RE = re.compile(r"^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$") +_OS_ALIASES = { + "osx": "OS X", + "os x": "OS X", + "dragonflybsd": "DragonFly BSD", + "dragonfly bsd": "DragonFly BSD", +} + + +def normalize_os_name(value: str | None) -> str | None: + """Normalize known OS aliases case-insensitively, preserving unknown names.""" + if value is None: + return None + stripped = value.strip() + return _OS_ALIASES.get(stripped.casefold(), stripped) + + +def normalize_os_names(value: str | None) -> tuple[str, ...]: + """Split and normalize comma-separated OS metadata.""" + if not value: + return () + return tuple(normalized for part in value.split(",") if (normalized := normalize_os_name(part))) + + +def _single_os_matches(case_os: str, restriction: str) -> bool: + """Compare two already-normalized, non-empty OS labels.""" + case_folded = case_os.casefold() + restriction_folded = restriction.casefold() + if restriction_folded == "*nix": + return case_folded != "windows" + if case_folded == "*nix": + return restriction_folded != "windows" + return case_folded == restriction_folded + + +def os_matches_restriction(case_os: str | None, restriction: str | None) -> bool: + """Return True if a case's OS is compatible with an OS restriction/filter. + + Applies a hierarchical Unix-family rule so the case-list prefilter + (``parse_cases``) and the runtime scan restriction stay in lockstep: + + * an empty case OS or empty restriction always matches; + * a ``*NIX`` restriction includes every non-Windows OS (FreeBSD, OS X, ...); + * a specific Unix restriction (e.g. FreeBSD) still includes the generic + ``*NIX`` cases; + * otherwise the comparison is exact (case-insensitive). + """ + if not case_os or not restriction: + return True + case_values = normalize_os_names(case_os) + restriction_values = normalize_os_names(restriction) + if not case_values or not restriction_values: + return True + return any( + _single_os_matches(case_value, restriction_value) + for case_value in case_values + for restriction_value in restriction_values + ) def validate_url_scheme(url: str) -> None: @@ -20,11 +79,18 @@ def validate_url_scheme(url: str) -> None: Raises ValueError if the scheme is invalid (prevents SSRF via file://, ftp://, etc.). """ parsed = urlsplit(url) - if parsed.scheme not in ("http", "https"): + if parsed.scheme.lower() not in ("http", "https"): raise ValueError(f"Only http:// and https:// URLs are supported, got '{parsed.scheme}://'") + if not parsed.hostname: + raise ValueError("URL must include a hostname") + try: + port = parsed.port + except ValueError as exc: + raise ValueError(f"Invalid URL port: {exc}") from exc + del port -def validate_header(header: str) -> tuple[str, str]: +def validate_header(header: str, *, warn_deprecated: bool = True) -> tuple[str, str]: """Parse and validate a custom HTTP header string. Expected format: 'Name: Value' (standard HTTP header format). @@ -35,10 +101,11 @@ def validate_header(header: str) -> tuple[str, str]: if ":" not in header: # Backward compatibility: original used Name=Value format if "=" in header: - print( - "[!] Warning: header format 'Name=Value' is deprecated, use 'Name: Value'", - file=sys.stderr, - ) + if warn_deprecated: + print( + "[!] Warning: header format 'Name=Value' is deprecated, use 'Name: Value'", + file=sys.stderr, + ) name, _, value = header.partition("=") else: raise ValueError("Header must contain a colon separator (format: 'Name: Value')") @@ -46,18 +113,41 @@ def validate_header(header: str) -> tuple[str, str]: name, _, value = header.partition(":") # Check for CRLF BEFORE stripping — strip() would silently remove injection chars - if any(c in name + value for c in "\r\n"): - raise ValueError("Header contains CRLF characters (possible header injection)") + if any(c in name + value for c in "\r\n\x00"): + raise ValueError("Header contains CRLF/NUL characters (possible header injection)") name = name.strip() value = value.strip() if not name: raise ValueError("Header name cannot be empty") + if not _HEADER_NAME_RE.fullmatch(name): + raise ValueError(f"Invalid HTTP header name: {name!r}") return name, value +def has_parameter(parameters: str, expected_name: str) -> bool: + """Return whether form/query syntax contains a decoded or raw parameter name.""" + for segment in parameters.split("&"): + raw_name, separator, _value = segment.partition("=") + if separator and (raw_name == expected_name or unquote_plus(raw_name) == expected_name): + return True + return False + + +def replace_parameter_value(parameters: str, expected_name: str, value: str) -> str: + """Replace matching form/query values while preserving each raw parameter name.""" + replaced: list[str] = [] + for segment in parameters.split("&"): + raw_name, separator, _old_value = segment.partition("=") + if separator and (raw_name == expected_name or unquote_plus(raw_name) == expected_name): + replaced.append(f"{raw_name}={value}") + else: + replaced.append(segment) + return "&".join(replaced) + + def sanitize_filename(path: str) -> str: """Sanitize a file path for use as a local output filename. @@ -74,6 +164,54 @@ def sanitize_filename(path: str) -> str: return sanitized or "unnamed" +def open_secure_write(path: str, *, newline: str | None = None) -> TextIO: + """Open a file for writing with platform-appropriate hardening. + + Sensitive scan artifacts (log files, result/list output, discovered file + content) may echo target paths, redacted URLs, or retrieved file bodies. + On POSIX, the file is forced to owner-only mode 0o600 before truncation. + Symlink protection depends on the primitives exposed by the platform: + + * With ``O_NOFOLLOW``, ``os.open`` atomically refuses a symlink at the final + path component. + * Without ``O_NOFOLLOW``, an ``lstat`` check rejects an already-present + symlink (and Windows junction where detectable), but cannot eliminate a + check/open race. + + The file is opened *without* ``O_TRUNC`` and only truncated after + permissions are confirmed, so a permission-hardening failure fails closed + on POSIX instead of destroying a pre-existing file's contents. Windows mode + bits do not configure NTFS ACLs, so owner-only access is not guaranteed there. + Raises OSError on failure. + """ + nofollow = getattr(os, "O_NOFOLLOW", 0) + if not nofollow: + try: + target_stat = os.lstat(path) + except FileNotFoundError: + pass + else: + is_junction = getattr(os.path, "isjunction", lambda _path: False) + if stat.S_ISLNK(target_stat.st_mode) or is_junction(path): + raise OSError(errno.ELOOP, "refusing to follow symlink or junction", path) + + flags = os.O_WRONLY | os.O_CREAT | nofollow | getattr(os, "O_BINARY", 0) + fd = os.open(path, flags, 0o600) + try: + # Tighten permissions BEFORE truncating. If we cannot guarantee 0600 we + # must fail closed rather than proceed — and because we have not yet + # truncated, a pre-existing file's contents are left intact. + if hasattr(os, "fchmod"): + os.fchmod(fd, 0o600) + elif os.name == "posix": + raise OSError(errno.ENOTSUP, "fchmod is unavailable; cannot enforce mode 0o600") + os.ftruncate(fd, 0) + return os.fdopen(fd, "w", encoding="utf-8", newline=newline) + except BaseException: + os.close(fd) + raise + + def load_data_file(filename: str) -> str: """Load a data file from the panoptic.data package. @@ -100,6 +238,20 @@ def generate_invalid_filename() -> str: return secrets.token_hex(8) +def redact_parameter_values(value: str) -> str: + """Redact values in query/form syntax, including bare non-key segments.""" + redacted: list[str] = [] + for segment in value.split("&"): + if "=" in segment: + key, _, _parameter_value = segment.partition("=") + redacted.append(f"{key}=***") + elif segment: + redacted.append("***") + else: + redacted.append("") + return "&".join(redacted) + + def redact_url(url: str) -> str: """Redact sensitive parts of a URL for safe display. @@ -109,11 +261,17 @@ def redact_url(url: str) -> str: parsed = urlsplit(url) # Rebuild netloc without userinfo host = parsed.hostname or "" - if parsed.port: - host = f"{host}:{parsed.port}" + if ":" in host: + host = f"[{host}]" + try: + port = parsed.port + except ValueError: + port = None + if port: + host = f"{host}:{port}" # Redact query parameter values but keep keys for context if parsed.query: - redacted_query = _PARAM_VALUE_RE.sub("***", parsed.query) + redacted_query = redact_parameter_values(parsed.query) return f"{parsed.scheme}://{host}{parsed.path}?{redacted_query}" return f"{parsed.scheme}://{host}{parsed.path}" diff --git a/pyproject.toml b/pyproject.toml index a364b90..d00ed21 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=68.0"] +requires = ["setuptools>=77.0"] build-backend = "setuptools.build_meta" [project] @@ -7,28 +7,35 @@ name = "panoptic" version = "2.0.0" description = "Search and retrieve content of common log and config files via path traversal vulnerability" readme = "README.md" -license = {text = "MIT"} +license = "MIT" requires-python = ">=3.10" authors = [ {name = "Roberto Christopher Salgado Bjerre"}, {name = "Miroslav Stampar"}, ] dependencies = [ - "httpx[socks]>=0.27", - "rich>=13.0", + "httpx[socks]>=0.28.1", + "rich>=15.0.0", "rich-argparse>=1.4", "tomli>=2.0; python_version < '3.11'", ] +[project.urls] +Homepage = "https://github.com/lightos/Panoptic" +Issues = "https://github.com/lightos/Panoptic/issues" +Source = "https://github.com/lightos/Panoptic" + [project.optional-dependencies] dev = [ "pytest>=8.0", - "pytest-asyncio>=0.23", + "pytest-asyncio>=1.4.0", "pytest-httpx>=0.30", "ruff>=0.4", "mypy>=1.10", - "pre-commit>=4.0", - "pip-audit>=2.7", + "pre-commit>=4.6.0", + "pip-audit>=2.10.1", + "build>=1.2", + "twine>=5.0", ] [tool.setuptools.packages.find] diff --git a/tests/test_cases.py b/tests/test_cases.py index e299796..c4125df 100644 --- a/tests/test_cases.py +++ b/tests/test_cases.py @@ -22,10 +22,32 @@ def test_cases_are_frozen(self) -> None: with pytest.raises(AttributeError): cases[0].location = "modified" # type: ignore[misc] - def test_filter_by_os(self) -> None: + def test_filter_by_os_nix_includes_unix_family(self) -> None: + """A *NIX filter excludes Windows but keeps the whole Unix family.""" config = ScanConfig(url="http://example.com", os_filter="*NIX") cases = parse_cases(config) - assert all(c.os == "*NIX" for c in cases if c.os is not None) + os_values = {c.os for c in cases if c.os is not None} + assert "Windows" not in os_values + assert "*NIX" in os_values + assert "FreeBSD" in os_values # specific Unix OS retained under *NIX + + def test_filter_by_specific_unix_includes_generic_nix(self) -> None: + """A specific Unix filter still includes the generic *NIX cases (matches runtime).""" + config = ScanConfig(url="http://example.com", os_filter="FreeBSD") + cases = parse_cases(config) + os_values = {c.os for c in cases if c.os is not None} + assert "FreeBSD" in os_values + assert "*NIX" in os_values + assert "Windows" not in os_values + + def test_filter_by_lowercase_os_alias_keeps_os_x_cases(self) -> None: + config = ScanConfig(url="http://example.com", os_filter="osx") + cases = parse_cases(config) + os_values = {case.os for case in cases} + assert "OS X" in os_values + assert "*NIX" in os_values + assert "Windows" not in os_values + assert any(case.location == "/usr/local/mysql/data/mysql.log" for case in cases) def test_filter_by_software(self) -> None: config = ScanConfig(url="http://example.com", software_filter="PHP") @@ -44,6 +66,37 @@ def test_host_placeholder_expansion(self) -> None: for case in cases: assert "{HOST}" not in case.location + def test_domain_placeholder_expansion(self) -> None: + config = ScanConfig(url="http://target.example.com:8080/test.php?f=x") + cases = parse_cases(config) + domain_cases = [case for case in cases if "/var/log/lighttpd/target.example.com/" in case.location] + assert len(domain_cases) == 2 + assert all("{" not in case.location for case in cases) + + def test_version_templates_are_skipped_unless_requested(self) -> None: + default_cases = parse_cases(ScanConfig(url="http://example.com")) + versioned_cases = parse_cases(ScanConfig(url="http://example.com", all_versions=True)) + assert all("[" not in case.location for case in default_cases) + assert len(versioned_cases) > len(default_cases) + assert any("JBoss-6.0.0.Final" in case.location for case in versioned_cases) + + def test_unknown_version_template_fails_fast(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "panoptic.cases._load_csv", + lambda: [ + { + "path": "/opt/[UNKNOWN]/config", + "os": "*NIX", + "software": "Example", + "category": "Other", + "type": "conf", + } + ], + ) + monkeypatch.setattr("panoptic.cases.load_versions", lambda: {"KNOWN": ["1.0"]}) + with pytest.raises(ValueError, match="unknown version section 'UNKNOWN'"): + parse_cases(ScanConfig(url="http://example.com", all_versions=True)) + class TestLoadVersions: def test_returns_dict(self) -> None: @@ -82,7 +135,19 @@ def test_parse_cases_returns_expected_count(self) -> None: """CSV parser returns expected number of cases.""" config = ScanConfig(url="http://example.com") cases = parse_cases(config) - assert len(cases) == 1024 + assert len(cases) == 958 + + def test_composite_os_rows_emit_one_case_per_location(self) -> None: + cases = parse_cases(ScanConfig(url="http://example.com")) + locations = { + "/usr/ports/ftp/pure-ftpd/pure-ftpd.conf", + "/usr/ports/ftp/pure-ftpd/pureftpd.passwd", + "/usr/ports/ftp/pure-ftpd/pureftpd.pdb", + } + composite_cases = [case for case in cases if case.location in locations] + assert len(composite_cases) == len(locations) + assert {case.location for case in composite_cases} == locations + assert all(case.os == "DragonFly BSD, FreeBSD" for case in composite_cases) def test_all_cases_have_metadata(self) -> None: """Every case from CSV has all metadata fields populated.""" @@ -109,6 +174,15 @@ def test_list_values_os(self) -> None: values = list_values("os") assert "*NIX" in values assert "Windows" in values + assert "OS X" in values + assert "OSX" not in values + assert "DragonFly BSD" in values + assert "DragonflyBSD, FreeBSD" not in values + + filtered_values = list_values("os", ScanConfig(url="http://example.com")) + assert "DragonFly BSD" in filtered_values + assert "FreeBSD" in filtered_values + assert "DragonFly BSD, FreeBSD" not in filtered_values def test_list_values_software(self) -> None: from panoptic.cases import list_values diff --git a/tests/test_cli.py b/tests/test_cli.py index a23bc60..4e7a18a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,5 +1,7 @@ """Tests for panoptic.cli — argument parsing and validation.""" +import os +from pathlib import Path from unittest.mock import AsyncMock, patch import pytest @@ -48,7 +50,12 @@ def test_new_flags(self) -> None: def test_prefix_multiplier(self) -> None: args = parse_args(["-u", "http://example.com", "--prefix", "../", "--multiplier", "3"]) - assert args["prefix"] == "../../../" + assert args["prefix"] == "../" + assert args["multiplier"] == 3 + + def test_boolean_flag_can_override_config_with_false(self) -> None: + args = parse_args(["-u", "http://example.com", "--no-auto"]) + assert args["automatic"] is False def test_list_command(self) -> None: args = parse_args(["--list", "software"]) @@ -141,6 +148,46 @@ def test_rejects_negative_delay(self) -> None: } ) + def test_rejects_url_without_hostname(self) -> None: + with pytest.raises(SystemExit): + validate_args( + { + "url": "http:///test.php?file=x", + "list": None, + "update": False, + "list_all_files": False, + "headers": None, + } + ) + + def test_rejects_socks4_proxy(self) -> None: + """httpx (socksio) only supports SOCKS5, so socks4:// must be rejected up front.""" + with pytest.raises(SystemExit): + validate_args( + { + "url": "http://example.com/test.php?file=x", + "list": None, + "update": False, + "list_all_files": False, + "headers": None, + "proxy": "socks4://127.0.0.1:9050", + } + ) + + def test_accepts_socks5_proxy(self) -> None: + # Should not raise for supported SOCKS5 variants. + for scheme in ("socks5://127.0.0.1:9050", "socks5h://127.0.0.1:9050"): + validate_args( + { + "url": "http://example.com/test.php?file=x", + "list": None, + "update": False, + "list_all_files": False, + "headers": None, + "proxy": scheme, + } + ) + class TestListFormatting: def test_list_json_output(self) -> None: @@ -176,6 +223,24 @@ def test_list_all_files_csv_output(self) -> None: assert lines[0] == "path" assert len(lines) > 1 + async def test_list_csv_quotes_values_with_commas(self, capsys: pytest.CaptureFixture[str]) -> None: + import csv + import io + + exit_code = await run(["--list", "os", "--output-format", "csv"]) + rows = list(csv.reader(io.StringIO(capsys.readouterr().out))) + assert exit_code == 0 + assert all(len(row) == 1 for row in rows) + assert ["DragonFly BSD"] in rows + + @pytest.mark.skipif(os.name != "posix", reason="requires POSIX file permissions") + async def test_list_output_file_is_owner_only(self, tmp_path: Path) -> None: + out_path = tmp_path / "os-list.txt" + exit_code = await run(["--list", "os", "--output-file", str(out_path)]) + assert exit_code == 0 + assert out_path.exists() + assert os.stat(out_path).st_mode & 0o777 == 0o600 + class TestParamAutodetection: async def test_base64_param_autodetect(self) -> None: @@ -186,3 +251,60 @@ async def test_base64_param_autodetect(self) -> None: await run(["--url", "http://example.com/test.php?file=dGVzdC50eHQ=&id=1", "--auto"]) config = mock_scanner_cls.call_args[0][0] assert config.param == "file" + + async def test_skips_empty_parameter_during_autodetect(self) -> None: + with patch("panoptic.core.Scanner") as mock_scanner_cls: + mock_scanner = AsyncMock() + mock_scanner.run.return_value = 0 + mock_scanner_cls.return_value = mock_scanner + exit_code = await run(["--url", "http://example.com/test.php?empty=&file=test", "--auto"]) + config = mock_scanner_cls.call_args[0][0] + assert exit_code == 0 + assert config.param == "file" + + async def test_url_can_come_from_config(self, tmp_path: Path) -> None: + config_file = tmp_path / "config.toml" + config_file.write_text('[defaults]\nurl = "example.com/test.php?file=x"\nautomatic = true\n') + with patch("panoptic.core.Scanner") as mock_scanner_cls: + mock_scanner = AsyncMock() + mock_scanner.run.return_value = 0 + mock_scanner_cls.return_value = mock_scanner + exit_code = await run(["--config", str(config_file)]) + config = mock_scanner_cls.call_args[0][0] + assert exit_code == 0 + assert config.url == "http://example.com/test.php?file=x" + + async def test_invalid_config_type_returns_operational_error( + self, tmp_path: Path, capsys: pytest.CaptureFixture[str] + ) -> None: + config_file = tmp_path / "config.toml" + config_file.write_text('[defaults]\nurl = "example.com/?file=x"\nos_filter = 123\n') + exit_code = await run(["--config", str(config_file), "--auto"]) + assert exit_code == 2 + assert "os_filter must be a string" in capsys.readouterr().err + + async def test_explicit_missing_parameter_is_rejected(self, capsys: pytest.CaptureFixture[str]) -> None: + with patch("panoptic.core.Scanner") as mock_scanner_cls: + exit_code = await run(["--url", "http://example.com/?file=x", "--param", "typo"]) + assert exit_code == 1 + assert "Parameter 'typo' not found" in capsys.readouterr().err + mock_scanner_cls.assert_not_called() + + async def test_json_post_requires_fuzz_injection_marker(self, capsys: pytest.CaptureFixture[str]) -> None: + body = '{"file":"old"}' + with patch("panoptic.core.Scanner") as mock_scanner_cls: + exit_code = await run(["--url", "http://example.com/api", "--data", body, "--param", "file"]) + assert exit_code == 1 + assert "place FUZZ at the injection point" in capsys.readouterr().err + mock_scanner_cls.assert_not_called() + + async def test_json_post_with_fuzz_is_accepted(self) -> None: + body = '{"file":"FUZZ"}' + with patch("panoptic.core.Scanner") as mock_scanner_cls: + mock_scanner = AsyncMock() + mock_scanner.run.return_value = 0 + mock_scanner_cls.return_value = mock_scanner + exit_code = await run(["--url", "http://example.com/api", "--data", body]) + assert exit_code == 0 + config = mock_scanner_cls.call_args.args[0] + assert config.data == body diff --git a/tests/test_config.py b/tests/test_config.py index 6d1b7f2..c01e037 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -2,7 +2,10 @@ from pathlib import Path +import pytest + from panoptic.config import load_config, merge_config +from panoptic.core import build_payload from panoptic.models import OutputFormat @@ -13,9 +16,10 @@ def test_returns_dict(self, tmp_path: Path) -> None: result = load_config(str(config_file)) assert result["defaults"]["concurrency"] == 16 - def test_missing_file_returns_empty(self) -> None: + def test_missing_file_returns_empty(self, capsys: pytest.CaptureFixture[str]) -> None: result = load_config("/nonexistent/config.toml") assert result == {} + assert "does not exist" in capsys.readouterr().err def test_invalid_toml_returns_empty(self, tmp_path: Path) -> None: config_file = tmp_path / "bad.toml" @@ -55,7 +59,59 @@ def test_headers_from_file(self) -> None: config = merge_config(cli_args, file_config) assert config.user_agent == "CustomBot/1.0" + def test_cli_headers_extend_config_headers(self) -> None: + config = merge_config( + {"url": "http://example.com", "headers": ["X-CLI: cli"]}, + {"headers": {"values": ["X-Config: config"]}}, + ) + assert config.headers == ["X-Config: config", "X-CLI: cli"] + def test_output_format_enum(self) -> None: cli_args = {"url": "http://example.com", "output_format": "json"} config = merge_config(cli_args, {}) assert config.output_format == OutputFormat.JSON + + enum_config = merge_config({"url": "http://example.com", "output_format": OutputFormat.CSV}, {}) + assert enum_config.output_format == OutputFormat.CSV + + def test_cli_false_overrides_true_boolean(self) -> None: + config = merge_config( + {"url": "http://example.com", "automatic": False}, + {"defaults": {"automatic": True}}, + ) + assert config.automatic is False + + def test_config_multiplier_applies_to_payload(self) -> None: + config = merge_config( + {"url": "http://example.com/?file=x"}, + {"defaults": {"param": "file", "prefix": "../", "multiplier": 3}}, + ) + assert "../../../etc/passwd" in build_payload(config, "/etc/passwd", "file=x") + + def test_config_url_is_normalized(self) -> None: + config = merge_config({}, {"defaults": {"url": "example.com/?file=x"}}) + assert config.url == "http://example.com/?file=x" + + def test_invalid_output_format_raises(self) -> None: + with pytest.raises(ValueError, match="output_format"): + merge_config({"url": "http://example.com"}, {"defaults": {"output_format": "xml"}}) + + def test_unknown_option_warns(self, capsys: pytest.CaptureFixture[str]) -> None: + merge_config({"url": "http://example.com"}, {"defaults": {"concurreny": 8}}) + assert "concurreny" in capsys.readouterr().err + + @pytest.mark.parametrize( + ("file_config", "message"), + [ + ({"proxy": {"url": 123}}, "proxy must be a string"), + ({"headers": {"cookie": 123}}, "cookie must be a string"), + ({"defaults": {"os_filter": 123}}, "os_filter must be a string"), + ({"defaults": {"automatic": "yes"}}, "automatic must be a boolean"), + ({"defaults": {"concurrency": 4.5}}, "concurrency must be an integer"), + ({"defaults": {"header": 123}}, "defaults.header must be a string or a list of strings"), + ({"defaults": {"output_format": 123}}, "output_format must be a string"), + ], + ) + def test_invalid_config_types_raise(self, file_config: dict[str, object], message: str) -> None: + with pytest.raises(ValueError, match=message): + merge_config({"url": "http://example.com"}, file_config) diff --git a/tests/test_core.py b/tests/test_core.py index 53b0930..3c8136f 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,13 +1,26 @@ """Tests for panoptic.core — async scanner orchestrator.""" import asyncio +import io import json +import os +import threading import time from pathlib import Path from unittest.mock import AsyncMock, patch -from panoptic.core import Scanner, build_payload, load_checkpoint, save_checkpoint -from panoptic.models import Case, ScanConfig +import pytest + +from panoptic.core import ( + Scanner, + build_payload, + checkpoint_fingerprint, + load_checkpoint, + save_checkpoint, +) +from panoptic.models import Case, OutputFormat, ScanConfig +from panoptic.output import TextFormatter +from panoptic.utils import os_matches_restriction class TestBuildPayload: @@ -28,7 +41,7 @@ def test_prefix_postfix(self) -> None: def test_path_based(self) -> None: config = ScanConfig(url="http://example.com/files/view/test.txt", path_based=True) url = build_payload(config, "/etc/passwd", "") - assert "/etc/passwd" in url + assert url == "http://example.com/files/view/etc/passwd" def test_replace_slash(self) -> None: config = ScanConfig(url="http://example.com/test.php?file=test.txt", param="file", replace_slash="/./") @@ -68,11 +81,21 @@ def test_post_payload_encodes_plus_as_percent2b(self) -> None: payload = build_payload(config, "/etc/foo+bar", "file=test.txt") assert "%2B" in payload or "foo%2Bbar" in payload - def test_get_payload_preserves_plus(self) -> None: - """GET payloads should keep + unencoded for base64 compat with PHP $_GET.""" + def test_get_payload_encodes_plus(self) -> None: + """GET payloads must encode + because query parsers decode it as a space.""" config = ScanConfig(url="http://example.com/test.php?file=test.txt", param="file") payload = build_payload(config, "/etc/foo+bar", "file=test.txt") - assert "foo+bar" in payload + assert "foo%2Bbar" in payload + + def test_multiplier_applies_during_payload_construction(self) -> None: + config = ScanConfig( + url="http://example.com/test.php?file=test.txt", + param="file", + prefix="../", + multiplier=3, + ) + payload = build_payload(config, "/etc/passwd", "file=test.txt") + assert "../../../etc/passwd" in payload def test_replace_slash_percent_not_double_encoded(self) -> None: """--replace-slash with pre-encoded value like %2F must not double-encode.""" @@ -91,6 +114,62 @@ def test_fuzz_mode_no_encoding(self) -> None: payload = build_payload(config, "/etc/passwd", '{"file":"FUZZ"}') assert "/etc/passwd" in payload + def test_fuzz_json_body_preserves_valid_json_for_windows_path(self) -> None: + """A Windows path with backslashes/quotes must stay valid JSON after FUZZ substitution.""" + body = '{"file":"FUZZ"}' + config = ScanConfig(url="http://example.com/api", data=body) + payload = build_payload(config, r"C:\Windows\win.ini", body) + # Must round-trip as valid JSON with the raw path recovered intact. + assert json.loads(payload) == {"file": r"C:\Windows\win.ini"} + + def test_fuzz_json_body_escapes_embedded_quote(self) -> None: + body = '{"path":"FUZZ"}' + config = ScanConfig(url="http://example.com/api", data=body) + payload = build_payload(config, 'a"b', body) + assert json.loads(payload) == {"path": 'a"b'} + + def test_fuzz_json_marker_in_key_is_replaced(self) -> None: + body = '{"FUZZ":"x"}' + config = ScanConfig(url="http://example.com/api", data=body) + payload = build_payload(config, "/etc/passwd", body) + assert json.loads(payload) == {"/etc/passwd": "x"} + + def test_fuzz_form_body_percent_encodes_delimiters(self) -> None: + """Form-encoded FUZZ bodies must encode +, &, = so the structure survives.""" + body = "file=FUZZ&id=1" + config = ScanConfig(url="http://example.com/api", data=body) + payload = build_payload(config, "/etc/foo+bar&baz=qux", body) + assert "foo%2Bbar%26baz%3Dqux" in payload + assert payload.endswith("&id=1") # trailing structure preserved + + def test_fuzz_opaque_body_keeps_raw_substitution(self) -> None: + """A non-JSON, non-form body leaves the substitution untouched.""" + body = "prefix-FUZZ-suffix" + config = ScanConfig(url="http://example.com/api", data=body) + payload = build_payload(config, "/etc/passwd", body) + assert payload == "prefix-/etc/passwd-suffix" + + def test_encoded_parameter_name_is_replaced(self) -> None: + params = "file%5B0%5D=old&id=1" + config = ScanConfig(url=f"http://example.com/?{params}", param="file[0]") + payload = build_payload(config, "/etc/passwd", params) + assert payload == "http://example.com/?file%5B0%5D=/etc/passwd&id=1" + + def test_parameter_name_backslash_is_not_a_regex_replacement(self) -> None: + params = r"x\1=old&id=1" + config = ScanConfig(url=f"http://example.com/?{params}", param=r"x\1") + payload = build_payload(config, "/etc/passwd", params) + assert payload.endswith(r"?x\1=/etc/passwd&id=1") + + def test_base64_padding_is_percent_encoded(self) -> None: + config = ScanConfig( + url="http://example.com/?file=old", + param="file", + base64_encode=True, + ) + payload = build_payload(config, "/etc/passwd", "file=old") + assert payload.endswith("file=L2V0Yy9wYXNzd2Q%3D") + class TestScanner: async def test_scanner_initializes(self) -> None: @@ -126,6 +205,21 @@ async def test_checkpoint_state(self, tmp_path: Path) -> None: async def test_load_empty_checkpoint(self, tmp_path: Path) -> None: assert load_checkpoint(str(tmp_path / "nonexistent.json")) == set() + def test_generic_unix_restriction_keeps_specific_unix_cases(self) -> None: + assert os_matches_restriction("FreeBSD", "*NIX") is True + assert os_matches_restriction("OS X", "*NIX") is True + assert os_matches_restriction("Windows", "*NIX") is False + assert os_matches_restriction("*NIX", "FreeBSD") is True + assert os_matches_restriction("DragonFly BSD, FreeBSD", "FreeBSD") is True + + def test_runtime_restriction_canonicalizes_os_alias(self) -> None: + """An aliased --os (e.g. OSX) must match the canonical case OS at runtime.""" + for alias in ("OSX", "osx", "OsX"): + scanner = Scanner(ScanConfig(url="http://example.com", os_filter=alias)) + assert scanner.restrict_os == "OS X" + # A canonicalized OS X case must now pass the runtime restriction. + assert os_matches_restriction("OS X", scanner.restrict_os) is True + class TestWriteFile: def test_no_filename_collision(self, tmp_path: Path) -> None: @@ -149,6 +243,80 @@ def test_no_filename_collision(self, tmp_path: Path) -> None: output_dir = tmp_path / "output" / "example.com" files = list(output_dir.iterdir()) assert len(files) == 2, f"Expected 2 files, got {len(files)}: {files}" + if os.name == "posix": + assert all(os.stat(path).st_mode & 0o777 == 0o600 for path in files) + + @pytest.mark.skipif(os.name != "posix", reason="requires POSIX symlinks") + def test_discovered_file_write_refuses_symlink(self, tmp_path: Path) -> None: + """A symlink pre-planted at the discovered-file path must not be clobbered.""" + from panoptic.utils import sanitize_filename + + config = ScanConfig(url="http://example.com/test.php?file=x", param="file", write_files=True) + scanner = Scanner(config) + scanner.original_response = "original" + case = Case(location="/etc/passwd", os="*NIX") + + secret = tmp_path / "secret.txt" + secret.write_text("do-not-clobber", encoding="utf-8") + + with patch("panoptic.core.Path.cwd", return_value=tmp_path): + output_dir = tmp_path / "output" / "example.com" + output_dir.mkdir(parents=True) + filename = f"{sanitize_filename(case.location)}_{case.case_id[:8]}.txt" + (output_dir / filename).symlink_to(secret) + # _write_file swallows OSError (logs a warning) rather than raising. + scanner._write_file(case, "malicious content") + + assert secret.read_text(encoding="utf-8") == "do-not-clobber" + + +class TestScanOutputFilePermissions: + @pytest.mark.skipif(os.name != "posix", reason="requires POSIX file permissions") + async def test_output_file_is_owner_only(self, tmp_path: Path) -> None: + out_path = tmp_path / "results.json" + config = ScanConfig( + url="http://example.com/test.php?file=x", + param="file", + quiet=True, + output_format=OutputFormat.JSON, + output_file=str(out_path), + ) + scanner = Scanner(config) + with ( + patch("panoptic.core.parse_cases", return_value=[Case(location="/etc/passwd")]), + patch("panoptic.core.NetworkClient") as client_cls, + patch("panoptic.core.get_revision", return_value=None), + patch.object(scanner, "_fetch", new=AsyncMock(return_value=None)), + ): + client_cls.return_value.__aenter__ = AsyncMock(return_value=AsyncMock()) + client_cls.return_value.__aexit__ = AsyncMock(return_value=None) + await scanner._run_scan(io.StringIO(), "test") + + assert out_path.exists() + assert os.stat(out_path).st_mode & 0o777 == 0o600 + + @pytest.mark.skipif(os.name != "posix", reason="requires POSIX file permissions") + async def test_log_file_is_owner_only(self, tmp_path: Path) -> None: + log_path = tmp_path / "scan.log" + config = ScanConfig( + url="http://example.com/test.php?file=x", + param="file", + quiet=True, + log_file=str(log_path), + ) + scanner = Scanner(config) + with ( + patch("panoptic.core.parse_cases", return_value=[Case(location="/etc/passwd")]), + patch("panoptic.core.NetworkClient") as client_cls, + patch("panoptic.core.get_revision", return_value=None), + patch.object(scanner, "_fetch", new=AsyncMock(return_value=None)), + ): + client_cls.return_value.__aenter__ = AsyncMock(return_value=AsyncMock()) + client_cls.return_value.__aexit__ = AsyncMock(return_value=None) + await scanner.run() + + assert log_path.exists() + assert os.stat(log_path).st_mode & 0o777 == 0o600 class TestFirstFoundRace: @@ -196,10 +364,12 @@ class TestAtomicCheckpoint: def test_save_checkpoint_atomic(self, tmp_path: Path) -> None: """Checkpoint writes must be atomic (temp file + rename).""" filepath = str(tmp_path / "checkpoint.json") - save_checkpoint(filepath, {"id1", "id2"}) + save_checkpoint(filepath, {"id1", "id2"}, "fingerprint") with open(filepath) as f: data = json.load(f) - assert set(data) == {"id1", "id2"} + assert data["version"] == 1 + assert data["fingerprint"] == "fingerprint" + assert set(data["completed_ids"]) == {"id1", "id2"} def test_save_checkpoint_no_partial_writes(self, tmp_path: Path) -> None: """If process dies mid-write, old checkpoint must survive.""" @@ -208,7 +378,45 @@ def test_save_checkpoint_no_partial_writes(self, tmp_path: Path) -> None: save_checkpoint(filepath, {"id1", "id2", "id3"}) with open(filepath) as f: data = json.load(f) - assert len(data) == 3 + assert len(data["completed_ids"]) == 3 + + def test_checkpoint_rejects_different_scan(self, tmp_path: Path) -> None: + filepath = str(tmp_path / "checkpoint.json") + save_checkpoint(filepath, {"id1"}, "scan-a") + with pytest.raises(ValueError, match="different scan"): + load_checkpoint(filepath, "scan-b") + + def test_checkpoint_fingerprint_changes_with_target(self) -> None: + cases = [Case(location="/etc/passwd", os="*NIX")] + first = checkpoint_fingerprint(ScanConfig(url="http://one.example/?f=x"), cases) + second = checkpoint_fingerprint(ScanConfig(url="http://two.example/?f=x"), cases) + assert first != second + + def test_checkpoint_fingerprint_normalizes_unordered_status_codes(self) -> None: + cases = [Case(location="/etc/passwd", os="*NIX")] + first = checkpoint_fingerprint( + ScanConfig(url="http://example.com/?f=x", match_codes=[200, 404]), + cases, + ) + second = checkpoint_fingerprint( + ScanConfig(url="http://example.com/?f=x", match_codes=[404, 200]), + cases, + ) + assert first == second + + def test_checkpoint_fingerprint_ignores_output_destination(self) -> None: + cases = [Case(location="/etc/passwd", os="*NIX")] + first = checkpoint_fingerprint(ScanConfig(url="http://example.com/?f=x"), cases) + second = checkpoint_fingerprint( + ScanConfig(url="http://example.com/?f=x", output_file="results.json"), + cases, + ) + assert first == second + + def test_legacy_checkpoint_is_loaded(self, tmp_path: Path) -> None: + filepath = tmp_path / "checkpoint.json" + filepath.write_text(json.dumps(["id1", "id2"]), encoding="utf-8") + assert load_checkpoint(str(filepath), "fingerprint-not-present-in-legacy") == {"id1", "id2"} class TestMatchString: @@ -304,6 +512,65 @@ async def test_match_string_disables_content_length_fast_path(self) -> None: assert len(scanner.results) == 0 + async def test_large_identical_response_is_not_found(self) -> None: + """A large Content-Length must not bypass body comparison.""" + config = ScanConfig( + url="http://example.com/test.php?file=test.txt", + param="file", + automatic=True, + ) + scanner = Scanner(config) + scanner.invalid_response = "not found" * 1000 + scanner.original_response = scanner.invalid_response + scanner.invalid_status_code = 200 + scanner.invalid_filename = "nonexistent" + + mock_response = AsyncMock() + mock_response.status_code = 200 + mock_response.text = scanner.invalid_response + mock_response.headers = {"content-length": "50000"} + + queue: asyncio.Queue[Case] = asyncio.Queue() + case = Case(location="/etc/passwd", os="*NIX") + text_out = AsyncMock() + with patch.object(scanner, "_fetch", return_value=mock_response): + processed = await scanner._process_case(case, AsyncMock(), "file=test.txt", queue, text_out) + + assert processed is True + assert scanner.results == [] + + +class TestInteractiveRestriction: + async def test_eof_does_not_approve_os_restriction(self) -> None: + scanner = Scanner( + ScanConfig( + url="http://example.com/test.php?file=x", + param="file", + skip_parsing=True, + ) + ) + scanner.invalid_response = "not found" + scanner.invalid_status_code = 200 + scanner.invalid_filename = "missing" + response = AsyncMock() + response.status_code = 200 + response.text = "found contents" + + with ( + patch.object(scanner, "_fetch", return_value=response), + patch("panoptic.core.is_match", return_value=True), + patch("builtins.input", side_effect=EOFError), + ): + await scanner._process_case( + Case(location="/a/file", os="FreeBSD"), + AsyncMock(), + "file=x", + asyncio.Queue(), + TextFormatter(io.StringIO()), + ) + + assert scanner.restrict_os is None + class TestCheckpointThrottling: async def test_rapid_marks_throttle_writes(self, tmp_path: Path) -> None: @@ -316,10 +583,10 @@ async def test_rapid_marks_throttle_writes(self, tmp_path: Path) -> None: write_count = 0 original_save = save_checkpoint - def counting_save(filepath: str, completed_ids: set[str]) -> None: + def counting_save(filepath: str, completed_ids: set[str], fingerprint: str = "") -> None: nonlocal write_count write_count += 1 - original_save(filepath, completed_ids) + original_save(filepath, completed_ids, fingerprint) with patch("panoptic.core.save_checkpoint", side_effect=counting_save): for i in range(20): @@ -328,6 +595,50 @@ def counting_save(filepath: str, completed_ids: set[str]) -> None: assert write_count <= 1, f"Expected <=1 writes during throttle window, got {write_count}" + async def test_flush_retains_dirty_during_real_event_loop_concurrency(self, tmp_path: Path) -> None: + """An event-loop task completing a case mid-save must remain dirty.""" + checkpoint_file = str(tmp_path / "checkpoint.json") + config = ScanConfig(url="http://example.com", resume_file=checkpoint_file) + scanner = Scanner(config) + scanner._checkpoint_fingerprint = "fp" + first = Case(location="/first") + second = Case(location="/second") + scanner.completed_ids = {first.case_id} + scanner._checkpoint_dirty = True + # Keep _mark_completed from starting its own throttled flush; this test + # exercises a second event-loop task mutating state while the first flush + # is blocked inside asyncio.to_thread. + scanner._last_checkpoint_time = time.monotonic() + + original_save = save_checkpoint + save_started = threading.Event() + allow_save = threading.Event() + + def blocking_save(filepath: str, completed_ids: set[str], fingerprint: str = "") -> None: + save_started.set() + if not allow_save.wait(timeout=2): + raise TimeoutError("test did not release checkpoint save") + original_save(filepath, completed_ids, fingerprint) + + with patch("panoptic.core.save_checkpoint", side_effect=blocking_save): + flush_task = asyncio.create_task(scanner._flush_checkpoint()) + save_observed = await asyncio.wait_for(asyncio.to_thread(save_started.wait, 1), timeout=2) + assert save_observed, "checkpoint save did not start" + await scanner._mark_completed(second) + allow_save.set() + await asyncio.wait_for(flush_task, timeout=2) + + # The first snapshot contains only the case completed before the save. + with open(checkpoint_file) as f: + data = json.load(f) + assert data["completed_ids"] == [first.case_id] + # The second case arrived on the event loop while the worker-thread save + # was blocked, so dirty must remain set and a subsequent flush must persist it. + assert scanner._checkpoint_dirty is True + await scanner._flush_checkpoint() + assert load_checkpoint(checkpoint_file, "fp") == {first.case_id, second.case_id} + assert scanner._checkpoint_dirty is False + async def test_flush_on_shutdown(self, tmp_path: Path) -> None: """_flush_checkpoint must write dirty state even without time threshold.""" checkpoint_file = str(tmp_path / "checkpoint.json") @@ -340,5 +651,121 @@ async def test_flush_on_shutdown(self, tmp_path: Path) -> None: with open(checkpoint_file) as f: data = json.load(f) - assert set(data) == {"id1", "id2", "id3"} + assert set(data["completed_ids"]) == {"id1", "id2", "id3"} assert scanner._checkpoint_dirty is False + + +class TestWorkerSupervision: + async def test_case_exception_does_not_deadlock_queue(self) -> None: + config = ScanConfig( + url="http://example.com/test.php?file=x", + param="file", + concurrency=1, + automatic=True, + quiet=True, + ) + scanner = Scanner(config) + response = AsyncMock() + response.text = "baseline" + response.status_code = 200 + + cases = [Case(location="/first"), Case(location="/second")] + with ( + patch("panoptic.core.parse_cases", return_value=cases), + patch("panoptic.core.NetworkClient") as client_cls, + patch.object(scanner, "_fetch", new=AsyncMock(side_effect=[response, response])), + patch.object(scanner, "_process_case", new=AsyncMock(side_effect=[RuntimeError("boom"), True])), + ): + client_cls.return_value.__aenter__ = AsyncMock(return_value=AsyncMock()) + client_cls.return_value.__aexit__ = AsyncMock(return_value=None) + exit_code = await asyncio.wait_for(scanner._run_scan(io.StringIO(), "test"), timeout=1) + + assert exit_code == 2 + assert scanner.total_failed == 1 + assert scanner.total_processed == 1 + + async def test_partial_request_failure_does_not_fail_completed_scan(self) -> None: + config = ScanConfig( + url="http://example.com/test.php?file=x", + param="file", + concurrency=1, + automatic=True, + quiet=True, + ) + scanner = Scanner(config) + response = AsyncMock() + response.text = "baseline" + response.status_code = 200 + + with ( + patch("panoptic.core.parse_cases", return_value=[Case(location="/first"), Case(location="/second")]), + patch("panoptic.core.NetworkClient") as client_cls, + patch.object(scanner, "_fetch", new=AsyncMock(side_effect=[response, response])), + patch.object(scanner, "_process_case", new=AsyncMock(side_effect=[False, True])), + ): + client_cls.return_value.__aenter__ = AsyncMock(return_value=AsyncMock()) + client_cls.return_value.__aexit__ = AsyncMock(return_value=None) + exit_code = await scanner._run_scan(io.StringIO(), "test") + + assert exit_code == 0 + assert scanner.total_failed == 1 + assert scanner.total_processed == 1 + + async def test_all_request_failures_are_operational_failure(self) -> None: + config = ScanConfig( + url="http://example.com/test.php?file=x", + param="file", + concurrency=1, + automatic=True, + quiet=True, + ) + scanner = Scanner(config) + response = AsyncMock() + response.text = "baseline" + response.status_code = 200 + + with ( + patch("panoptic.core.parse_cases", return_value=[Case(location="/only")]), + patch("panoptic.core.NetworkClient") as client_cls, + patch.object(scanner, "_fetch", new=AsyncMock(side_effect=[response, response])), + patch.object(scanner, "_process_case", new=AsyncMock(return_value=False)), + ): + client_cls.return_value.__aenter__ = AsyncMock(return_value=AsyncMock()) + client_cls.return_value.__aexit__ = AsyncMock(return_value=None) + exit_code = await scanner._run_scan(io.StringIO(), "test") + + assert exit_code == 2 + assert scanner.total_failed == 1 + assert scanner.total_processed == 0 + + async def test_connection_failure_emits_valid_empty_json(self) -> None: + config = ScanConfig( + url="http://example.com/test.php?file=x", + param="file", + quiet=True, + output_format=OutputFormat.JSON, + ) + scanner = Scanner(config) + output = io.StringIO() + with ( + patch("panoptic.core.parse_cases", return_value=[Case(location="/etc/passwd")]), + patch("panoptic.core.NetworkClient") as client_cls, + patch("panoptic.core.get_revision", return_value=None), + patch.object(scanner, "_fetch", new=AsyncMock(return_value=None)), + patch("panoptic.core.sys.stdout", output), + ): + client_cls.return_value.__aenter__ = AsyncMock(return_value=AsyncMock()) + client_cls.return_value.__aexit__ = AsyncMock(return_value=None) + exit_code = await scanner._run_scan(io.StringIO(), "test") + + assert exit_code == 2 + assert json.loads(output.getvalue()) == [] + + async def test_checkpoint_write_failure_is_operational_error(self, tmp_path: Path) -> None: + checkpoint = tmp_path / "missing" / "checkpoint.json" + scanner = Scanner(ScanConfig(url="http://example.com", resume_file=str(checkpoint))) + scanner.completed_ids = {"id1"} + scanner._checkpoint_dirty = True + await scanner._flush_checkpoint() + assert scanner._checkpoint_disabled is True + assert any("Checkpoint disabled" in error for error in scanner.operational_errors) diff --git a/tests/test_heuristic.py b/tests/test_heuristic.py index aca809c..0093b3b 100644 --- a/tests/test_heuristic.py +++ b/tests/test_heuristic.py @@ -63,6 +63,12 @@ def test_none_invalid_returns_false(self) -> None: """Guard: if invalid_response is None, return False.""" assert is_match("some html", None) is False + def test_reordered_body_uses_full_similarity_ratio(self) -> None: + """Bodies with identical character counts but different structure must differ.""" + found = "A" * 5000 + "B" * 5000 + invalid = "AB" * 5000 + assert is_match(found, invalid) is True + class TestFilterContent: def test_strips_common_prefix_suffix(self) -> None: diff --git a/tests/test_integration.py b/tests/test_integration.py index b0122e7..891f9ba 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -2,6 +2,7 @@ from panoptic.cli import parse_args, validate_args from panoptic.config import merge_config +from panoptic.core import build_payload from panoptic.models import ScanConfig @@ -32,7 +33,9 @@ def test_cli_to_config_pipeline(self) -> None: assert config.url == "http://target.test/vuln.php?file=test.txt" assert config.param == "file" - assert config.prefix == "../../../" + assert config.prefix == "../" + assert config.multiplier == 3 + assert "../../../etc/passwd" in build_payload(config, "/etc/passwd", "file=test.txt") assert config.timeout == 5.0 assert config.concurrency == 2 assert config.automatic is True @@ -66,6 +69,7 @@ def test_cases_load_with_filters(self) -> None: assert len(cases) > 0 for case in cases: if case.os: - assert case.os == "*NIX" + # *NIX filter keeps the Unix family (FreeBSD, OS X, ...) but excludes Windows. + assert case.os != "Windows" if case.file_type: assert case.file_type.value == "conf" diff --git a/tests/test_models.py b/tests/test_models.py index b8f89b0..e329cee 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -98,6 +98,14 @@ def test_valid_random_delay_accepted(self) -> None: config = ScanConfig(url="http://example.com", random_delay=(0.5, 2.0)) assert config.random_delay == (0.5, 2.0) + def test_negative_retries_rejected(self) -> None: + with pytest.raises(ValueError, match="retries must be >= 0"): + ScanConfig(url="http://example.com", retries=-1) + + def test_zero_multiplier_rejected(self) -> None: + with pytest.raises(ValueError, match="multiplier must be >= 1"): + ScanConfig(url="http://example.com", multiplier=0) + class TestEnums: def test_file_type_values(self) -> None: diff --git a/tests/test_network.py b/tests/test_network.py index 78425ee..2704e68 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -1,5 +1,7 @@ """Tests for panoptic.network — async HTTP client.""" +from unittest.mock import AsyncMock, patch + import httpx import pytest from pytest_httpx import HTTPXMock @@ -44,10 +46,43 @@ async def test_fetch_timeout_returns_none(self, config: ScanConfig, httpx_mock: async def test_fetch_connection_error_returns_none(self, config: ScanConfig, httpx_mock: HTTPXMock) -> None: httpx_mock.add_exception(httpx.ConnectError("refused")) + httpx_mock.add_exception(httpx.ConnectError("refused again")) async with NetworkClient(config) as client: response = await client.fetch("http://example.com/down") assert response is None + async def test_connect_error_is_retried(self, config: ScanConfig, httpx_mock: HTTPXMock) -> None: + httpx_mock.add_exception(httpx.ConnectError("temporary")) + httpx_mock.add_response(url="http://example.com/retry", text="recovered") + async with NetworkClient(config) as client: + response = await client.fetch("http://example.com/retry") + assert response is not None + assert response.text == "recovered" + + async def test_ssl_and_environment_proxy_options_reach_httpx(self) -> None: + config = ScanConfig( + url="https://example.com", + invalid_ssl=True, + ignore_proxy=False, + ) + mock_client = AsyncMock() + with patch("panoptic.network.httpx.AsyncClient", return_value=mock_client) as client_cls: + async with NetworkClient(config): + pass + + kwargs = client_cls.call_args.kwargs + assert kwargs["verify"] is False + assert kwargs["trust_env"] is True + assert "transport" not in kwargs + + async def test_ignore_proxy_disables_environment_proxies(self) -> None: + config = ScanConfig(url="https://example.com", ignore_proxy=True) + mock_client = AsyncMock() + with patch("panoptic.network.httpx.AsyncClient", return_value=mock_client) as client_cls: + async with NetworkClient(config): + pass + assert client_cls.call_args.kwargs["trust_env"] is False + async def test_custom_user_agent(self, httpx_mock: HTTPXMock) -> None: config = ScanConfig(url="http://example.com", user_agent="CustomBot/1.0") httpx_mock.add_response() diff --git a/tests/test_output.py b/tests/test_output.py index 0c412e6..c2b99f1 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -6,8 +6,8 @@ import pytest -from panoptic.models import Case, FileType, ScanResult -from panoptic.output import CsvFormatter, JsonFormatter, TextFormatter +from panoptic.models import Case, FileType, ScanConfig, ScanResult +from panoptic.output import CsvFormatter, JsonFormatter, TextFormatter, _scan_mode_label @pytest.fixture @@ -120,6 +120,96 @@ def test_csv_output_redacts_url(self) -> None: content = buf.read() assert "secret123" not in content + def test_json_array_body_is_redacted(self) -> None: + result = ScanResult( + case=Case(location="/etc/passwd"), + found=True, + url='[{"file":"/etc/passwd","token":"supersecret"}]', + status_code=200, + ) + buf = io.StringIO() + JsonFormatter(buf).write_results([result]) + data = json.loads(buf.getvalue()) + assert "supersecret" not in data[0]["url"] + assert "***" in data[0]["url"] + + def test_json_numeric_and_bool_scalars_are_redacted(self) -> None: + """Non-string JSON leaves (a numeric PIN, a boolean flag) must not leak.""" + result = ScanResult( + case=Case(location="/etc/passwd"), + found=True, + url='{"file":"/etc/passwd","pin":90210,"admin":true}', + status_code=200, + ) + buf = io.StringIO() + JsonFormatter(buf).write_results([result]) + redacted = json.loads(buf.getvalue())[0]["url"] + assert "90210" not in redacted + assert "true" not in redacted + # Keys are preserved for context, values are not. + assert '"pin"' in redacted and '"admin"' in redacted + + def test_top_level_scalar_json_body_is_redacted(self) -> None: + """A bare scalar body (e.g. --data '31337') must not be echoed verbatim.""" + result = ScanResult( + case=Case(location="/etc/passwd"), + found=True, + url="31337", + status_code=200, + ) + buf = io.StringIO() + JsonFormatter(buf).write_results([result]) + assert json.loads(buf.getvalue())[0]["url"] == "***" + + def test_opaque_body_without_structure_is_fully_redacted(self) -> None: + """A non-URL, non-JSON, non-form body (a raw token) is redacted entirely.""" + result = ScanResult( + case=Case(location="/etc/passwd"), + found=True, + url="raw-session-token-abc123", + status_code=200, + ) + buf = io.StringIO() + JsonFormatter(buf).write_results([result]) + assert json.loads(buf.getvalue())[0]["url"] == "***" + + def test_form_body_keeps_keys_redacts_values(self) -> None: + result = ScanResult( + case=Case(location="/etc/passwd"), + found=True, + url="file=/etc/passwd&token=secret123", + status_code=200, + ) + buf = io.StringIO() + JsonFormatter(buf).write_results([result]) + redacted = json.loads(buf.getvalue())[0]["url"] + assert "secret123" not in redacted + assert redacted == "file=***&token=***" + + def test_uppercase_url_scheme_is_still_redacted(self) -> None: + result = ScanResult( + case=Case(location="/etc/passwd"), + found=True, + url="HTTPS://user:secret@example.com/path?token=secret123", + status_code=200, + ) + buf = io.StringIO() + JsonFormatter(buf).write_results([result]) + redacted = json.loads(buf.getvalue())[0]["url"] + assert "secret" not in redacted + assert redacted == "https://example.com/path?token=***" or redacted == "HTTPS://example.com/path?token=***" + + def test_bare_form_segment_is_redacted(self) -> None: + result = ScanResult( + case=Case(location="/etc/passwd"), + found=True, + url="file=/etc/passwd&bare-secret-token", + status_code=200, + ) + buf = io.StringIO() + JsonFormatter(buf).write_results([result]) + assert json.loads(buf.getvalue())[0]["url"] == "file=***&***" + class TestTextFormatter: def test_found_message(self, sample_results: list[ScanResult]) -> None: @@ -138,6 +228,41 @@ def test_summary(self, sample_results: list[ScanResult]) -> None: output = buf.read() assert "2" in output # 2 found + def test_file_results_include_discovered_paths(self, sample_results: list[ScanResult]) -> None: + buf = io.StringIO() + formatter = TextFormatter(buf) + formatter.write_results(sample_results, total_cases=100) + output = buf.getvalue() + assert "/etc/passwd" in output + assert "/var/log/syslog" in output + + def test_file_results_do_not_hard_wrap_long_paths(self) -> None: + location = f"/var/lib/{'very-long-directory/' * 8}configuration.conf" + result = ScanResult( + case=Case(location=location), + found=True, + url=f"http://example.com/?file={location}", + status_code=200, + content_length=123, + ) + buf = io.StringIO() + TextFormatter(buf).write_results([result], total_cases=1) + assert location in buf.getvalue() + + def test_fuzz_in_header_name_is_not_reported_as_injection_mode(self) -> None: + config = ScanConfig( + url="http://example.com/?file=x", + headers=["X-FUZZ-Label: fixed"], + ) + assert _scan_mode_label(config) == "GET" + + def test_fuzz_in_header_value_is_reported_as_injection_mode(self) -> None: + config = ScanConfig( + url="http://example.com/", + headers=["X-File: FUZZ"], + ) + assert _scan_mode_label(config) == "FUZZ" + class TestQuietMode: def test_quiet_suppresses_banner(self) -> None: @@ -180,3 +305,29 @@ def test_quiet_shows_warning(self) -> None: formatter.write_warning("SSL disabled") buf.seek(0) assert "SSL disabled" in buf.read() + + +class TestMarkupEscaping: + """Untrusted content (URLs, case locations, config values) must not inject Rich markup.""" + + def test_warning_escapes_markup(self) -> None: + buf = io.StringIO() + TextFormatter(buf).write_warning("Case '/etc/[red]passwd[/red]' failed") + assert "[red]passwd[/red]" in buf.getvalue() + + def test_info_escapes_markup(self) -> None: + buf = io.StringIO() + TextFormatter(buf).write_info("Using random User-Agent: [bold]evil[/bold]") + assert "[bold]evil[/bold]" in buf.getvalue() + + def test_verbose_escapes_markup(self) -> None: + buf = io.StringIO() + TextFormatter(buf).write_verbose("Trying '[green]/etc/passwd[/green]'") + assert "[green]/etc/passwd[/green]" in buf.getvalue() + + def test_banner_escapes_markup_in_url(self) -> None: + buf = io.StringIO() + # An injected style tag in the (redacted) URL must render literally, not + # be interpreted — an unescaped invalid tag would also raise at print time. + TextFormatter(buf).write_banner("1.0", "http://host/[red]x[/red]") + assert "[red]x[/red]" in buf.getvalue() diff --git a/tests/test_parsers.py b/tests/test_parsers.py index d635b2b..167a2f3 100644 --- a/tests/test_parsers.py +++ b/tests/test_parsers.py @@ -60,6 +60,15 @@ def test_preserves_directory(self) -> None: cases = extract_binlog_cases(content, parent_case) assert cases[0].location.startswith("/var/lib/mysql/") + def test_extracts_unix_and_modern_binlog_entries(self) -> None: + content = "./mysql-bin.000001\n./binlog.000002\n/absolute/binlog.000003\n" + parent_case = Case(location="/var/lib/mysql/mysql-bin.index", os="*NIX") + cases = extract_binlog_cases(content, parent_case) + assert [case.location for case in cases] == [ + "/var/lib/mysql/mysql-bin.000001", + "/var/lib/mysql/binlog.000002", + ] + def test_empty_content_returns_empty(self) -> None: parent_case = Case(location="/var/lib/mysql/mysql-bin.index") assert extract_binlog_cases("", parent_case) == [] diff --git a/tests/test_update.py b/tests/test_update.py index 5bd497b..4e64663 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -5,26 +5,63 @@ import pytest -from panoptic.update import do_update, get_revision +from panoptic.update import ( + GIT_UPSTREAM_REF, + _normalise_git_url, + _uses_secure_git_transport, + do_update, + get_revision, +) class TestDoUpdate: @patch("panoptic.update.subprocess.run") @patch("panoptic.update.os.path.exists", return_value=True) def test_git_checkout_runs_git_pull(self, mock_exists: Any, mock_run: Any) -> None: - mock_run.return_value = MagicMock(returncode=0, stdout=b"Already up to date.\n") - do_update() - mock_run.assert_called() + mock_run.side_effect = [ + MagicMock(returncode=0, stdout=b"git@github.com:lightos/Panoptic.git\n"), + MagicMock(returncode=0, stdout=b"main\n"), + MagicMock(returncode=0, stdout=b"Already up to date.\n"), + MagicMock(returncode=0, stdout=b"abc1234567890abcdef1234567890abcdef123456\n"), + ] + assert do_update() == 0 + assert mock_run.call_count == 4 # Should use list args, not shell=True - args = mock_run.call_args - assert isinstance(args[0][0], list) + pull_args = mock_run.call_args_list[2] + assert pull_args[0][0] == ["git", "pull", "--ff-only", "origin", GIT_UPSTREAM_REF] + + @patch("panoptic.update.subprocess.run") + @patch("panoptic.update.os.path.exists", return_value=True) + def test_git_checkout_rejects_non_main_branch(self, mock_exists: Any, mock_run: Any) -> None: + mock_run.side_effect = [ + MagicMock(returncode=0, stdout=b"git@github.com:lightos/Panoptic.git\n"), + MagicMock(returncode=0, stdout=b"feature\n"), + ] + assert do_update() == 2 + assert mock_run.call_count == 2 @patch("panoptic.update.os.path.exists", return_value=False) def test_pip_installed_prints_guidance(self, mock_exists: Any, capsys: pytest.CaptureFixture[str]) -> None: - do_update() + assert do_update() == 0 captured = capsys.readouterr() assert "pip" in captured.out.lower() + def test_ssh_and_https_remotes_are_equivalent(self) -> None: + assert _normalise_git_url("git@github.com:lightos/Panoptic.git") == _normalise_git_url( + "https://github.com/lightos/Panoptic.git" + ) + + @pytest.mark.parametrize( + "url", + [ + "http://github.com/lightos/Panoptic.git", + "git://github.com/lightos/Panoptic.git", + "file://github.com/lightos/Panoptic.git", + ], + ) + def test_insecure_remote_transports_are_rejected(self, url: str) -> None: + assert _uses_secure_git_transport(url) is False + class TestGetRevision: @patch("panoptic.update.subprocess.run") diff --git a/tests/test_utils.py b/tests/test_utils.py index 0b952f4..2983200 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,15 +1,23 @@ """Tests for panoptic.utils.""" +import os +from pathlib import Path + import pytest from panoptic.utils import ( get_random_agent, load_data_file, + normalize_os_name, + open_secure_write, + redact_url, sanitize_filename, validate_header, validate_url_scheme, ) +posix_only = pytest.mark.skipif(os.name != "posix", reason="requires POSIX file permissions/symlinks") + class TestValidateUrlScheme: def test_http_valid(self) -> None: @@ -30,6 +38,19 @@ def test_no_scheme_rejected(self) -> None: with pytest.raises(ValueError, match="Only http:// and https://"): validate_url_scheme("example.com/test") + def test_missing_hostname_rejected(self) -> None: + with pytest.raises(ValueError, match="hostname"): + validate_url_scheme("http:///path") + + +class TestNormalizeOsName: + @pytest.mark.parametrize("value", ["OSX", "osx", "OsX", "OS X", "os x"]) + def test_os_x_aliases_are_case_insensitive(self, value: str) -> None: + assert normalize_os_name(value) == "OS X" + + def test_unknown_os_preserves_display_name(self) -> None: + assert normalize_os_name(" FreeBSD ") == "FreeBSD" + class TestValidateHeader: def test_valid_header(self) -> None: @@ -55,6 +76,17 @@ def test_value_with_equals(self) -> None: assert name == "Authorization" assert value == "Bearer abc=def" + def test_invalid_header_name_rejected(self) -> None: + with pytest.raises(ValueError, match="header name"): + validate_header("Bad Header: value") + + def test_nul_rejected(self) -> None: + with pytest.raises(ValueError, match="NUL"): + validate_header("X-Test: value\x00injected") + + def test_ipv6_redaction_preserves_brackets(self) -> None: + assert redact_url("http://user:pass@[::1]:8080/path?token=x") == "http://[::1]:8080/path?token=***" + class TestSanitizeFilename: def test_slashes_replaced(self) -> None: @@ -92,3 +124,84 @@ def test_returns_different_values(self) -> None: random.seed(42) agents = {get_random_agent() for _ in range(20)} assert len(agents) > 1 + + +class TestOpenSecureWrite: + def test_creates_file_with_content(self, tmp_path: Path) -> None: + target = tmp_path / "out.txt" + with open_secure_write(str(target)) as fh: + fh.write("hello") + assert target.read_text(encoding="utf-8") == "hello" + + @posix_only + def test_new_file_is_owner_only(self, tmp_path: Path) -> None: + target = tmp_path / "out.txt" + with open_secure_write(str(target)) as fh: + fh.write("x") + assert os.stat(target).st_mode & 0o777 == 0o600 + + @posix_only + def test_tightens_permissions_on_existing_file(self, tmp_path: Path) -> None: + target = tmp_path / "out.txt" + target.write_text("old", encoding="utf-8") + os.chmod(target, 0o644) + with open_secure_write(str(target)) as fh: + fh.write("new") + assert os.stat(target).st_mode & 0o777 == 0o600 + assert target.read_text(encoding="utf-8") == "new" + + def test_truncates_existing_content(self, tmp_path: Path) -> None: + target = tmp_path / "out.txt" + target.write_text("a very long previous value", encoding="utf-8") + with open_secure_write(str(target)) as fh: + fh.write("short") + assert target.read_text(encoding="utf-8") == "short" + + def test_newline_is_not_translated(self, tmp_path: Path) -> None: + target = tmp_path / "out.csv" + with open_secure_write(str(target), newline="") as fh: + fh.write("a,b\r\n") + assert target.read_bytes() == b"a,b\r\n" + + @posix_only + def test_refuses_to_follow_symlink(self, tmp_path: Path) -> None: + """A pre-planted symlink at the target must not be written through.""" + secret = tmp_path / "secret.txt" + secret.write_text("do-not-clobber", encoding="utf-8") + link = tmp_path / "link.txt" + link.symlink_to(secret) + with pytest.raises(OSError): + open_secure_write(str(link)) + # The symlink target must be untouched. + assert secret.read_text(encoding="utf-8") == "do-not-clobber" + + @posix_only + def test_fallback_rejects_existing_symlink_without_nofollow( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Platforms lacking O_NOFOLLOW still reject an already-present symlink.""" + secret = tmp_path / "secret.txt" + secret.write_text("do-not-clobber", encoding="utf-8") + link = tmp_path / "link.txt" + link.symlink_to(secret) + monkeypatch.delattr(os, "O_NOFOLLOW", raising=False) + with pytest.raises(OSError): + open_secure_write(str(link)) + assert secret.read_text(encoding="utf-8") == "do-not-clobber" + + @posix_only + def test_fchmod_failure_fails_closed_without_truncating( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """If permissions can't be enforced, fail closed and preserve prior content.""" + target = tmp_path / "out.txt" + target.write_text("original content", encoding="utf-8") + + def boom(fd: int, mode: int) -> None: + raise PermissionError("cannot chmod") + + monkeypatch.setattr(os, "fchmod", boom) + with pytest.raises(OSError): + open_secure_write(str(target)) + # The pre-existing content must not have been truncated. + assert target.read_text(encoding="utf-8") == "original content"