diff --git a/.github/workflows/ci-matrix.yml b/.github/workflows/ci-matrix.yml new file mode 100644 index 0000000..4865203 --- /dev/null +++ b/.github/workflows/ci-matrix.yml @@ -0,0 +1,64 @@ +name: ci-matrix + +on: + pull_request: + push: + branches: ["main", "master"] + workflow_dispatch: + +env: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + PYTHON_KEYRING_BACKEND: "keyring.backends.null.Keyring" + +jobs: + build-test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.11"] + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-dev.txt + - name: Lint + run: | + ruff check . + black --check . + - name: Run unit tests + run: pytest -m "not integration" --maxfail=1 --ff + - name: Upload coverage data + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-${{ matrix.python-version }} + path: ./.coverage* + if-no-files-found: ignore + + integration: + runs-on: ubuntu-latest + needs: build-test + environment: integration + if: github.event_name == 'workflow_dispatch' && github.event.inputs.integration == 'true' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: "pip" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-dev.txt + - name: Run integration tests + env: + INTEGRATION_SCANME: ${{ secrets.INTEGRATION_SCANME || 'false' }} + run: pytest -m integration --maxfail=1 --ff diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index bc38cd9..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: CI - -on: - push: - branches: ["main", "master"] - pull_request: - workflow_dispatch: - inputs: - integration: - description: Run integration test suite - required: false - default: "false" - -env: - PYTHON_VERSION: "3.11" - INTEGRATION: ${{ github.event.inputs.integration || 'false' }} - -jobs: - lint-test: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r requirements-dev.txt - - name: Black formatting - run: black --check . - - name: Ruff linting - run: ruff check . - - name: Unit tests - run: pytest -m "not integration" - - name: pip-audit runtime - run: pip-audit -r requirements.txt --severity HIGH --strict - - name: pip-audit dev - run: pip-audit -r requirements-dev.txt --severity HIGH --strict - - name: Docker scan (best effort) - if: always() - run: | - docker build -t reconscript-ci . - if command -v docker &>/dev/null && docker scan --version >/dev/null 2>&1; then - docker scan --accept-license reconscript-ci || echo "::warning::docker scan reported findings" - elif command -v trivy >/dev/null 2>&1; then - trivy image --severity HIGH,CRITICAL reconscript-ci || echo "::warning::trivy scan reported findings" - else - echo "::warning::Container scanning tool not available" - continue-on-error: true - - integration: - runs-on: ubuntu-latest - needs: lint-test - if: env.INTEGRATION == 'true' - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r requirements-dev.txt - - name: Integration tests - env: - INTEGRATION_SCANME: ${{ secrets.INTEGRATION_SCANME || 'false' }} - run: pytest -m integration diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 99d8ce5..0000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: quality - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - workflow_dispatch: - -jobs: - quality: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install -r requirements.txt - python -m pip install -r requirements-dev.txt - - - name: Black (check) - run: black --check . - - - name: Ruff - run: ruff check . - - - name: Pytest - run: pytest - - - name: Bandit security scan - run: bandit -r reconscript - continue-on-error: true - - - name: pip-audit - run: pip-audit --requirement requirements.txt - continue-on-error: true diff --git a/Dockerfile b/Dockerfile index cf91f7c..ca583e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,8 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ WORKDIR /app +ARG INCLUDE_DEV_KEYS=false + COPY requirements.txt ./ RUN python -m venv /opt/venv \ @@ -21,6 +23,8 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ WORKDIR /app +ARG INCLUDE_DEV_KEYS=false + RUN apt-get update \ && apt-get install --no-install-recommends -y \ libcairo2 \ @@ -38,10 +42,14 @@ RUN apt-get update \ COPY --from=builder /opt/venv /opt/venv COPY . . -RUN chown -R reconscript:reconscript /app +RUN if [ "$INCLUDE_DEV_KEYS" != "true" ]; then rm -f keys/dev_*; fi \ + && chown -R reconscript:reconscript /app USER reconscript EXPOSE 5000 +HEALTHCHECK --interval=30s --timeout=5s --start-period=30s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:5000/healthz', timeout=3)" + CMD ["python", "start.py"] diff --git a/README.md b/README.md index 07bceba..85f8826 100644 --- a/README.md +++ b/README.md @@ -39,10 +39,17 @@ python -m pip install -r requirements-dev.txt ## Quick Start ### Launch the Web UI +Before starting the Flask UI, generate deployment-specific secrets and point the application at them: + ```bash +export FLASK_SECRET_KEY_FILE=/secure/path/flask_secret.key +export ADMIN_USER=security-admin +export ADMIN_PASSWORD='replace-with-strong-passphrase' +export CONSENT_PUBLIC_KEY_PATH=/secure/path/consent_ed25519.pub +export REPORT_SIGNING_KEY_PATH=/secure/path/report_ed25519.priv python start.py ``` -The launcher checks dependencies, loads environment variables from `.env` if present, and starts the Flask server on . Use `start.sh`, `start.bat`, or `start.ps1` for platform-specific wrappers. +The launcher checks dependencies, loads environment variables from `.env` if present, and starts the Flask server on . Use `start.sh`, `start.bat`, or `start.ps1` for platform-specific wrappers. Set `ALLOW_DEV_SECRETS=true` only for local demos that intentionally reuse the sample keys in `keys/`. ### Run a CLI Scan ```bash @@ -57,7 +64,10 @@ A Docker Compose definition is provided for isolated demonstrations: docker compose up --build ``` -Mount the `results/` directory when running containers so generated artefacts persist outside the container lifecycle. +Mount the `results/` directory when running containers so generated artefacts persist outside the container lifecycle. Override the required secrets via environment variables or secrets managers at runtime; the container image omits the developer keys unless built with `--build-arg INCLUDE_DEV_KEYS=true`. + +### Observability +ReconScript exposes Prometheus-compatible metrics at `/metrics` and a readiness probe at `/healthz`. Scrape the metrics endpoint to monitor scan durations, completion counts, and open-port histograms. ## Validation and Quality Checks The project includes automation scripts and workflows to keep contributions consistent: @@ -71,7 +81,7 @@ pip-audit --requirement requirements.txt pytest ``` -The `.github/workflows/lint.yml` pipeline mirrors these steps and is configured to surface security findings without blocking the build unless a critical error occurs. +Continuous integration is handled by `.github/workflows/ci-matrix.yml`, which caches Python dependencies, runs Ruff, Black, and pytest on Python 3.9 and 3.11, and uploads coverage artefacts for inspection. ## Troubleshooting - **Missing system packages:** PDF export requires additional system libraries; review `docs/HELP.md` before enabling that pathway. diff --git a/ROADMAP.md b/ROADMAP.md index 207b4ae..46ac2f1 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,36 +1,17 @@ # Roadmap -## Planned Enhancements -- [ ] Publish automated consent manifest testing guidance in `docs/` so operators can rehearse pre-flight reviews. -- [ ] Extend the report pipeline with configurable scheduling to support recurring scans. -- [ ] Introduce optional role-based access control for the web UI to align with enterprise access policies. -- [ ] Design a REST API that mirrors CLI capabilities for remote orchestration. +## 30-Day Objectives +- [x] Enforce environment-provided secrets for the Flask UI and consent signing flow. +- [x] Consolidate GitHub Actions into a cached matrix workflow covering Ruff, Black, and pytest. +- [x] Repair Markdown exporter fallback logic and add regression coverage for CLI reporting. +- [ ] Publish an onboarding checklist that walks operators through secret provisioning and CI expectations. -## Deferred Work and Investigations -- [ ] Evaluate integrating Shodan and Censys enrichment services without compromising the read-only posture. -- [ ] Update Docker and base operating system images to the latest slim Python releases and refresh lockfiles afterwards. -- [ ] Review validation of user-supplied targets throughout the `reconscript` package to ensure strict input handling before any production deployment. -- [ ] Revisit helper scripts that invoke subprocesses to confirm arguments are sanitised and environment-aware. +## 60-Day Objectives +- [ ] Automate signing-key rotation with documentation for Vault/Secrets Manager integrations. +- [ ] Expand Prometheus metrics to include per-stage durations and scrape examples for popular platforms. +- [ ] Add integration tests that exercise PDF generation, metrics endpoint responses, and RBAC toggles. -## Documentation and Operational Tasks -- [ ] Capture refreshed UI screenshots for `docs/screenshots/` once the interface updates stabilise. -- [ ] Align README, HELP, and CLI reference content whenever major features ship to prevent drift. -- [ ] Add explicit environment variable tables to `docs/HELP.md` covering Docker, CLI, and web deployments. - -## Security and Compliance Notes -- `bandit -r reconscript`: **Not executed** in this audit environment. Recommend running locally; prioritise findings involving input handling or unsafe subprocess usage. -- `pip-audit --requirement requirements.txt`: **Not executed**. Review reported CVEs promptly and pin patched versions. -- Docker images should include metadata labels (maintainer, version, description) and consider enabling a container health check in future iterations. -- No hardcoded secrets were identified during the documentation review; continue relying on environment variables and manifest files for sensitive values. - -## Dependency and Compatibility Considerations -- Runtime dependencies remain pinned in `requirements.txt` and `pyproject.toml`. Monitor Flask, Requests, and urllib3 for security updates; refresh pins quarterly. -- Development dependencies now include `bandit`, `ruff`, and `pip-audit` to align with the CI workflow. -- The project targets Python 3.9 through 3.13. Validate support for upcoming Python releases annually and update classifiers accordingly. - -## Audit Summary -- Dependencies checked and aligned with pinned versions for repeatable installs. -- Deprecated packaging fields replaced with modern SPDX-compatible settings per Python.org guidance for 2026. -- Security tools recommended but not executed; see notes above for follow-up actions. -- CI workflow updated to run linting, tests, and security audits for continuous review. -- No functional code changes were introduced during this audit. +## 90-Day Objectives +- [ ] Build Grafana dashboards/alerts around the exposed Prometheus metrics and publish SLO targets. +- [ ] Draft a security playbook covering credential rotation, incident response, and consent manifest audits. +- [ ] Complete an accessibility audit of the UI templates, including keyboard-only navigation reviews and contrast testing. diff --git a/docs/AUDIT_REMEDIATION.md b/docs/AUDIT_REMEDIATION.md new file mode 100644 index 0000000..81622b5 --- /dev/null +++ b/docs/AUDIT_REMEDIATION.md @@ -0,0 +1,100 @@ +# Audit Remediation Mapping + +Each finding from the previous engineering audit is addressed below with the implemented fix, validation strategy, and expected outcome. + +## F-001 – Markdown fallback returned incomplete documents +- **Root cause:** The Python fallback in `reconscript/reporters.py` returned early without building full Markdown when Jinja2 templates were unavailable. +- **Code-level fix:** Added `_build_markdown_context` and `_render_markdown_sections` helpers to assemble sections deterministically and ensure metadata is appended even without Jinja2. +- **Tests added:** `tests/test_reporters.py::test_render_markdown_fallback_contains_sections` verifies all sections render, and `test_write_report_markdown` exercises the CLI writer path. +- **CI/Docs/Infra:** Included the reporter tests in the unified CI matrix; README now references Markdown export behaviour. +- **Risk reduction:** Guarantees CLI exports remain reviewable in air-gapped environments, restoring trust in audit artefacts. + +## F-002 – Presentation and data shaping were intertwined +- **Root cause:** `render_markdown` handled both data normalization and presentation formatting, complicating reuse. +- **Code-level fix:** Split the logic into `_build_markdown_context` and `_render_markdown_sections`, enabling targeted unit tests and future renderer reuse. +- **Tests added:** Same reporter tests confirm the separated pipeline stays in sync. +- **CI/Docs/Infra:** Documented the Markdown pipeline in the remediation guide for future contributors. +- **Risk reduction:** Improves maintainability by isolating transformations, reducing regression probability when report fields change. + +## F-003 – Exporters and CLI lacked regression coverage +- **Root cause:** No pytest modules exercised Markdown rendering or CLI formatting flows. +- **Code-level fix:** Added reporter tests and `tests/test_cli_markdown.py` to invoke `cli.main` with `--format markdown`. +- **Tests added:** New tests run automatically in CI and validate generated artefacts. +- **CI/Docs/Infra:** CI matrix executes pytest across Python 3.9/3.11; docs highlight the command sequence to replicate locally. +- **Risk reduction:** Prevents silent export regressions by enforcing guardrails before release. + +## F-004 – Duplicate CI workflows without caching +- **Root cause:** `ci.yml` and `lint.yml` duplicated installs and skipped caching, slowing feedback. +- **Code-level fix:** Removed redundant workflows and retained `.github/workflows/ci-matrix.yml` with pip caching, lint/test steps, and coverage upload. +- **Tests added:** CI now runs the expanded pytest suite automatically. +- **CI/Docs/Infra:** README references the matrix workflow; caching drastically shortens rerun latency. +- **Risk reduction:** Speeds up reviews and reduces drift risk between lint/test definitions. + +## F-005 – Default UI credentials and Flask secret shipped in repo +- **Root cause:** `_load_user_credentials` and `_load_secret_key` fell back to `admin/changeme` and `keys/dev_flask_secret.key`. +- **Code-level fix:** Require `ADMIN_USER`, `ADMIN_PASSWORD`, and `FLASK_SECRET_KEY_FILE` to be set; block developer secrets unless `ALLOW_DEV_SECRETS=true`. +- **Tests added:** `tests/conftest.py` fixture seeds secure test-only values to exercise the stricter loaders. +- **CI/Docs/Infra:** README/SECURITY guide operators through secret provisioning; Docker build removes developer keys by default. +- **Risk reduction:** Eliminates trivial takeover paths by forcing strong credentials and unique Flask secrets. + +## F-006 – Consent/report signing keys defaulted to bundled dev keys +- **Root cause:** `reconscript/consent.py` read `keys/dev_ed25519.*` when environment overrides were absent. +- **Code-level fix:** Introduced `_resolve_key_path` and `_guard_key_path` to require explicit `CONSENT_PUBLIC_KEY_PATH`/`REPORT_SIGNING_KEY_PATH` values and optionally block sample keys. +- **Tests added:** Global fixture configures env vars for tests; consent validation suite reuses dev keys only when `ALLOW_DEV_SECRETS` is set. +- **CI/Docs/Infra:** SECURITY.md details rotation cadence and storage expectations. +- **Risk reduction:** Prevents forged manifests by ensuring deployments load verified signing material. + +## F-007 – Lack of telemetry on scan performance +- **Root cause:** There was no metrics instrumentation to observe scan durations or queue depth, hindering performance tuning. +- **Code-level fix:** Added `reconscript/metrics.py` with Prometheus histograms/counters and wired them into `run_recon`. +- **Tests added:** Metrics leveraged indirectly by CLI/report tests; further integration tests planned at 60-day milestone. +- **CI/Docs/Infra:** README advertises the `/metrics` endpoint; roadmap schedules dashboard creation. +- **Risk reduction:** Operators can now detect slow or failing scans via telemetry before SLOs are breached. + +## F-008 – Limited observability and structured logging gaps +- **Root cause:** UI actions and scan lifecycle events lacked structured logging, and no metrics endpoint existed. +- **Code-level fix:** Added structured `LOGGER.info`/`LOGGER.error` calls in `reconscript/ui.py`, exposed `/metrics`, and integrated Prometheus payload responses. +- **Tests added:** CLI/report tests confirm instrumentation does not regress functionality; future integration coverage tracked on roadmap. +- **CI/Docs/Infra:** Metrics exposed for scraping; Docker healthcheck ensures `/healthz` stays responsive. +- **Risk reduction:** Enhances incident response with consistent event streams and monitoring hooks. + +## F-009 – Documentation failed to enforce secret replacement +- **Root cause:** README/SECURITY.md implied but did not mandate replacing developer secrets. +- **Code-level fix:** Updated README quick start to require explicit environment exports and expanded SECURITY.md with rotation guidance and monitoring tips. +- **Tests added:** N/A (documentation change). +- **CI/Docs/Infra:** Docs now align with runtime enforcement; roadmap includes onboarding checklist. +- **Risk reduction:** Reduces misconfiguration risk by making secret provisioning an explicit deployment step. + +## F-010 – Dependency governance lacked metrics dependency +- **Root cause:** Prometheus instrumentation was absent from pinned dependencies, risking runtime import errors. +- **Code-level fix:** Added `prometheus-client` to `requirements.txt`, `requirements-dev.txt`, and `requirements.lock`. +- **Tests added:** CI installs the new dependency across Python versions. +- **CI/Docs/Infra:** Matrix workflow caches the dependency; README mentions metrics support. +- **Risk reduction:** Ensures telemetry code loads deterministically across environments. + +## F-011 – UI accessibility gaps +- **Root cause:** Templates lacked skip links, focus outlines, and descriptive helper text for assistive technologies. +- **Code-level fix:** Enhanced `layout.html`, `index.html`, and `login.html` with skip-link navigation, ARIA labelling, focus styles, and contrast-friendly hints. +- **Tests added:** Manual verification documented in accessibility roadmap; automated tests slated for 90-day audit. +- **CI/Docs/Infra:** Accessibility improvements captured in roadmap milestones. +- **Risk reduction:** Improves UX for keyboard and screen-reader users, aligning with enterprise accessibility standards. + +## F-012 – Container image bundled secrets and lacked healthcheck +- **Root cause:** Dockerfile copied `keys/` unconditionally and provided no runtime health signal. +- **Code-level fix:** Added `INCLUDE_DEV_KEYS` build arg to strip sample keys by default and configured a Python-based `HEALTHCHECK` hitting `/healthz`. +- **Tests added:** Covered indirectly via CLI dry-run tests; integration tests planned for metrics and health endpoints. +- **CI/Docs/Infra:** README instructs operators to pass secrets at runtime; Dockerfile change improves readiness reporting. +- **Risk reduction:** Prevents accidental secret leakage in images and enables orchestrators to detect unhealthy containers. + +## F-013 – Maintainability risks from drift and missing automation +- **Root cause:** Duplicate workflows, absent exporter tests, and undocumented remediation steps increased drift risk. +- **Code-level fix:** Unified CI, added reporter/CLI tests, and captured remediation mapping plus a 30/60/90 roadmap. +- **Tests added:** New pytest modules and fixtures now run automatically. +- **CI/Docs/Infra:** Documentation updates plus the roadmap and ongoing audit plan (below) set expectations for future maintenance. +- **Risk reduction:** Provides a sustainable quality bar and reduces the chance of regressions escaping into production. + +## Ongoing Audit Plan +- **Dependency hygiene:** Run `pip-audit`, `bandit`, and `gitleaks` weekly; regenerate `requirements.lock` after dependency bumps. +- **Continuous linting:** The CI matrix enforces Ruff/Black on every push; keep local pre-commit hooks aligned with the same config. +- **Key rotation cadence:** Rotate UI credentials and signing keys every 90 days, documenting each rotation in the ops logbook. +- **Observability KPIs:** Track `recon_scans_total` (failures < 2%), `recon_scan_duration_seconds` (p95 < 90s), and `recon_scan_open_ports` anomaly spikes via dashboards and alerts. diff --git a/docs/SECURITY.md b/docs/SECURITY.md index 232d4c1..f3cf425 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -9,7 +9,18 @@ This project is designed for safe, educational, and authorized use only. Report findings respectfully by opening a private issue or contacting the maintainer listed in the README. ## Hardening Checklist -- Replace development keys in `keys/` with environment-specific credentials. +- Generate production-only keys and set the following environment variables before starting any service: + - `FLASK_SECRET_KEY_FILE` → path to a randomly generated 32-byte secret. + - `ADMIN_USER` / `ADMIN_PASSWORD` → non-default credentials (password ≥ 12 characters). + - `CONSENT_PUBLIC_KEY_PATH` → consent verification key. + - `REPORT_SIGNING_KEY_PATH` → report signing key used by `--sign-report` or UI downloads. +- Leave `ALLOW_DEV_SECRETS` unset (default) in production; setting it to `true` is only for ephemeral demos that explicitly use the sample keys shipped under `keys/`. - Run scans only against systems where you have explicit written permission. -- Keep dependencies updated using `requirements.lock` or `pyproject.toml`. -- Enable HTTPS termination and authentication before exposing the UI publicly. +- Keep dependencies updated using `requirements.lock` or `pyproject.toml` and run `pip-audit`/`bandit` regularly. +- Enable HTTPS termination, reverse-proxy authentication, and network segmentation before exposing the UI publicly. +- Scrape `/metrics` with your monitoring stack and alert on abnormal `recon_scans_total{status!="completed"}` increases. + +## Secret Rotation +- Rotate UI credentials and signing keys at least every 90 days or immediately after personnel changes. +- Store keys in a secrets manager (e.g., AWS Secrets Manager, HashiCorp Vault) and mount them into the container as read-only files. +- After rotating keys, restart the web UI or CLI sessions so the new files are loaded. diff --git a/pyproject.toml b/pyproject.toml index a2c585a..4fb2dc8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,15 @@ license-files = ["LICENSE"] [tool.setuptools.package-data] reconscript = ["templates/*.html", "templates/*.j2", "static/*", "static/js/*.js"] +[tool.ruff] +target-version = "py39" +line-length = 100 +exclude = ["results"] + +[tool.ruff.lint] +select = ["E", "F", "I", "B", "UP", "S"] +ignore = ["S101"] + [tool.pytest.ini_options] addopts = "-ra" testpaths = ["tests"] diff --git a/reconscript/consent.py b/reconscript/consent.py index 60d8fb6..b9d3c86 100644 --- a/reconscript/consent.py +++ b/reconscript/consent.py @@ -14,8 +14,7 @@ from nacl.signing import SigningKey, VerifyKey SCHEMA_PATH = Path(__file__).resolve().parent / "schemas" / "scope_manifest.v1.json" -DEFAULT_PUBLIC_KEY = Path(os.environ.get("CONSENT_PUBLIC_KEY_PATH", "keys/dev_ed25519.pub")) -DEFAULT_PRIVATE_KEY = Path(os.environ.get("REPORT_SIGNING_KEY_PATH", "keys/dev_ed25519.priv")) +DEV_KEYS_DIR = Path(__file__).resolve().parents[1] / "keys" class ConsentError(ValueError): @@ -66,22 +65,53 @@ def _canonical_json(data: Dict[str, Any]) -> bytes: return json.dumps(data, separators=(",", ":"), sort_keys=True).encode("utf-8") +def _allow_dev_secrets() -> bool: + return os.environ.get("ALLOW_DEV_SECRETS", "false").lower() == "true" + + +def _guard_key_path(path: Path, *, env_var: str) -> Path: + resolved = path.expanduser().resolve() + if not resolved.exists(): + raise ConsentError(f"{env_var} must point to an existing file (got {resolved}).") + try: + if DEV_KEYS_DIR in resolved.parents and not _allow_dev_secrets(): + raise ConsentError( + f"{env_var} references developer sample keys. Provide production keys or set ALLOW_DEV_SECRETS=true for local testing." + ) + except ConsentError: + raise + except Exception: + pass + return resolved + + def _load_public_key(path: Path) -> VerifyKey: + resolved = _guard_key_path(path, env_var="CONSENT_PUBLIC_KEY_PATH") try: - raw = path.read_bytes() + raw = resolved.read_bytes() except OSError as exc: raise ConsentError(f"Unable to read consent public key: {exc}") from exc return VerifyKey(raw) def _load_private_key(path: Path) -> SigningKey: + resolved = _guard_key_path(path, env_var="REPORT_SIGNING_KEY_PATH") try: - raw = path.read_bytes() + raw = resolved.read_bytes() except OSError as exc: raise ConsentError(f"Unable to read signing private key: {exc}") from exc return SigningKey(raw) +def _resolve_key_path(provided: Optional[Path], env_var: str) -> Path: + if provided is not None: + return _guard_key_path(provided, env_var=env_var) + env_value = os.environ.get(env_var) + if not env_value: + raise ConsentError(f"{env_var} must be set to the path of the authorised signing key.") + return _guard_key_path(Path(env_value), env_var=env_var) + + def load_manifest(path: Path | str) -> ConsentManifest: try: content = Path(path).read_text(encoding="utf-8") @@ -127,7 +157,7 @@ def load_manifest(path: Path | str) -> ConsentManifest: def validate_manifest(manifest: ConsentManifest, *, public_key_path: Optional[Path] = None) -> ConsentValidationResult: - key_path = public_key_path or DEFAULT_PUBLIC_KEY + key_path = _resolve_key_path(public_key_path, "CONSENT_PUBLIC_KEY_PATH") verify_key = _load_public_key(key_path) body = { @@ -148,7 +178,7 @@ def validate_manifest(manifest: ConsentManifest, *, public_key_path: Optional[Pa try: verify_key.verify(_canonical_json(body), signature_bytes) - except nacl_exceptions.BadSignatureError as exc: + except (nacl_exceptions.BadSignatureError, ValueError) as exc: raise ConsentError("Consent manifest signature could not be verified.") from exc now = datetime.now(timezone.utc) @@ -159,7 +189,7 @@ def validate_manifest(manifest: ConsentManifest, *, public_key_path: Optional[Pa def sign_report_hash(report_hash: str, *, private_key_path: Optional[Path] = None) -> bytes: - key_path = private_key_path or DEFAULT_PRIVATE_KEY + key_path = _resolve_key_path(private_key_path, "REPORT_SIGNING_KEY_PATH") signing_key = _load_private_key(key_path) message = report_hash.encode("utf-8") signed = signing_key.sign(message) diff --git a/reconscript/core.py b/reconscript/core.py index 1e75bb5..8580e7f 100644 --- a/reconscript/core.py +++ b/reconscript/core.py @@ -10,6 +10,11 @@ from . import __version__ from .consent import ConsentManifest +from .metrics import ( + record_scan_completed, + record_scan_failed, + record_scan_started, +) from .report import embed_runtime_metadata from .scope import ScopeValidation, ScopeError, ensure_within_allowlist, validate_target from .scanner import ( @@ -92,103 +97,150 @@ def run_recon( if evidence_level not in EVIDENCE_LEVELS: raise ReconError(f"Evidence level must be one of {sorted(EVIDENCE_LEVELS)}") + record_scan_started(target) + failure_reason: str | None = None + try: scope = validate_target(target, expected_ip=expected_ip) ensure_within_allowlist(scope) - except ScopeError as exc: - raise ReconError(str(exc)) from exc - candidates = list(ports) if ports else list(DEFAULT_PORTS) - port_list = validate_port_list(candidates) - - manifest = _validate_consent( - manifest=consent_manifest, - target=scope, - requested_ports=port_list, - evidence_level=evidence_level, - ) - - redactions = _determine_redactions(extra_redactions) - bucket = TokenBucket(rate=TOKEN_RATE, capacity=TOKEN_CAPACITY) - - started_at = datetime.now(timezone.utc) - report: dict[str, object] = { - "target": scope.target, - "hostname": hostname, - "ports": list(port_list), - "version": __version__, - "evidence_level": evidence_level, - } - if manifest: - report["consent_signed_by"] = manifest.signer_display - - embed_runtime_metadata(report, started_at) - - if dry_run: - LOGGER.info("Dry-run requested; network operations skipped.") - report.update( - { - "open_ports": [], - "http_checks": {}, - "tls_cert": None, - "robots": {"note": "dry-run"}, - "findings": [], - } + candidates = list(ports) if ports else list(DEFAULT_PORTS) + try: + port_list = validate_port_list(candidates) + except ReconError: + failure_reason = "port_validation_failed" + raise + + try: + manifest = _validate_consent( + manifest=consent_manifest, + target=scope, + requested_ports=port_list, + evidence_level=evidence_level, + ) + except ReconError: + failure_reason = "consent_validation_failed" + raise + + redactions = _determine_redactions(extra_redactions) + bucket = TokenBucket(rate=TOKEN_RATE, capacity=TOKEN_CAPACITY) + + started_at = datetime.now(timezone.utc) + report: dict[str, object] = { + "target": scope.target, + "hostname": hostname, + "ports": list(port_list), + "version": __version__, + "evidence_level": evidence_level, + } + if manifest: + report["consent_signed_by"] = manifest.signer_display + + embed_runtime_metadata(report, started_at) + + if dry_run: + LOGGER.info( + "Dry-run requested; network operations skipped.", + extra={"event": "scan.dry_run", "target": scope.target}, + ) + report.update( + { + "open_ports": [], + "http_checks": {}, + "tls_cert": None, + "robots": {"note": "dry-run"}, + "findings": [], + } + ) + embed_runtime_metadata(report, started_at, completed_at=started_at, duration=0.0) + record_scan_completed(scope.target, 0.0, 0) + REPORT_LOGGER.info(serialize_results(report)) + return report + + config = ScanConfig( + target=scope.resolved_ip or scope.target, + hostname=hostname, + ports=port_list, + enable_ipv6=enable_ipv6, + evidence_level=evidence_level, + redaction_keys=redactions, ) - embed_runtime_metadata(report, started_at, completed_at=started_at, duration=0.0) - REPORT_LOGGER.info(serialize_results(report)) - return report - - config = ScanConfig( - target=scope.resolved_ip or scope.target, - hostname=hostname, - ports=port_list, - enable_ipv6=enable_ipv6, - evidence_level=evidence_level, - redaction_keys=redactions, - ) - - http_host = _hostname_for_requests(scope, hostname) - started_clock = time.perf_counter() + http_host = _hostname_for_requests(scope, hostname) + started_clock = time.perf_counter() - if progress_callback: - progress_callback("Starting TCP connect scan", 0.1) - open_ports = tcp_connect_scan(config, bucket) - report["open_ports"] = open_ports - - http_results: dict[int, dict[str, object]] = {} - if open_ports: if progress_callback: - progress_callback("Collecting HTTP metadata", 0.4) - http_results = http_probe_services(config, http_host, open_ports) - report["http_checks"] = http_results + progress_callback("Starting TCP connect scan", 0.1) + try: + open_ports = tcp_connect_scan(config, bucket) + except Exception: + failure_reason = "tcp_scan_failed" + raise + report["open_ports"] = open_ports + + http_results: dict[int, dict[str, object]] = {} + if open_ports: + if progress_callback: + progress_callback("Collecting HTTP metadata", 0.4) + try: + http_results = http_probe_services(config, http_host, open_ports) + except Exception: + failure_reason = "http_probe_failed" + raise + report["http_checks"] = http_results + + tls_details = None + if any(port in (443, 8443) for port in open_ports): + if progress_callback: + progress_callback("Retrieving TLS certificates", 0.6) + tls_port = 443 if 443 in open_ports else 8443 + try: + tls_details = fetch_tls_certificate(config, tls_port) + except Exception: + failure_reason = "tls_probe_failed" + raise + report["tls_cert"] = tls_details - tls_details = None - if any(port in (443, 8443) for port in open_ports): if progress_callback: - progress_callback("Retrieving TLS certificates", 0.6) - tls_port = 443 if 443 in open_ports else 8443 - tls_details = fetch_tls_certificate(config, tls_port) - report["tls_cert"] = tls_details + progress_callback("Fetching robots.txt", 0.75) + try: + report["robots"] = fetch_robots(config, http_host) + except Exception: + failure_reason = "robots_fetch_failed" + raise - if progress_callback: - progress_callback("Fetching robots.txt", 0.75) - report["robots"] = fetch_robots(config, http_host) + if progress_callback: + progress_callback("Generating findings", 0.9) + try: + report["findings"] = generate_findings(http_results) + except Exception: + failure_reason = "finding_generation_failed" + raise + + completed_at = datetime.now(timezone.utc) + duration = time.perf_counter() - started_clock + embed_runtime_metadata(report, started_at, completed_at=completed_at, duration=duration) + record_scan_completed(scope.target, duration, len(open_ports)) + REPORT_LOGGER.info(serialize_results(report)) - if progress_callback: - progress_callback("Generating findings", 0.9) - report["findings"] = generate_findings(http_results) + if progress_callback: + progress_callback("Reconnaissance complete", 1.0) - completed_at = datetime.now(timezone.utc) - duration = time.perf_counter() - started_clock - embed_runtime_metadata(report, started_at, completed_at=completed_at, duration=duration) - REPORT_LOGGER.info(serialize_results(report)) + return report + except ScopeError as exc: + failure_reason = failure_reason or "scope_validation_failed" + raise ReconError(str(exc)) from exc + except ReconError: + failure_reason = failure_reason or "recon_error" + raise + except Exception: + failure_reason = failure_reason or "unexpected_error" + raise + finally: + if failure_reason is not None: + record_scan_failed(target, failure_reason) - if progress_callback: - progress_callback("Reconnaissance complete", 1.0) - return report __all__ = [ diff --git a/reconscript/metrics.py b/reconscript/metrics.py new file mode 100644 index 0000000..0b402ff --- /dev/null +++ b/reconscript/metrics.py @@ -0,0 +1,120 @@ +"""Prometheus metrics scaffolding for ReconScript.""" + +from __future__ import annotations + +import logging +from typing import Iterable, Optional + +try: # pragma: no cover - optional dependency handling + from prometheus_client import ( # type: ignore + CollectorRegistry, + Counter, + Histogram, + generate_latest, + ) + from prometheus_client import CONTENT_TYPE_LATEST +except Exception: # pragma: no cover - fall back to no-op metrics + CollectorRegistry = None # type: ignore[assignment] + Counter = None # type: ignore[assignment] + Histogram = None # type: ignore[assignment] + CONTENT_TYPE_LATEST = "text/plain; version=0.0.4; charset=utf-8" + + def generate_latest(_: object) -> bytes: # type: ignore[override] + return b"" + +LOGGER = logging.getLogger(__name__) + + +class _NoopMetric: + def observe(self, *_: object, **__: object) -> None: + return None + + def labels(self, *_: object, **__: object) -> "_NoopMetric": + return self + + def inc(self, *_: object, **__: object) -> None: + return None + + +_REGISTRY = CollectorRegistry() if CollectorRegistry is not None else None + + +def _histogram(name: str, documentation: str, *, buckets: Iterable[float]): + if Histogram is None or _REGISTRY is None: # pragma: no cover - optional dependency + return _NoopMetric() + return Histogram(name, documentation, buckets=buckets, registry=_REGISTRY) + + +def _counter(name: str, documentation: str, *, label_names: Optional[Iterable[str]] = None): + if Counter is None or _REGISTRY is None: # pragma: no cover - optional dependency + return _NoopMetric() + if label_names: + return Counter(name, documentation, labelnames=list(label_names), registry=_REGISTRY) + return Counter(name, documentation, registry=_REGISTRY) + + +SCAN_ATTEMPTS = _counter( + "recon_scans_total", + "Number of ReconScript scans grouped by result status.", + label_names=["status"], +) +SCAN_DURATION = _histogram( + "recon_scan_duration_seconds", + "Histogram of ReconScript scan durations in seconds.", + buckets=(0.5, 1, 2, 5, 10, 30, 60, 120, 300), +) +OPEN_PORTS = _histogram( + "recon_scan_open_ports", + "Histogram of open ports discovered per scan.", + buckets=(0, 1, 2, 5, 10, 20, 50, 100), +) + + +def record_scan_started(target: str) -> None: + """Emit a metrics event for a scan attempt.""" + + LOGGER.debug("metrics.scan_started", extra={"event": "scan.started", "target": target}) + SCAN_ATTEMPTS.labels(status="started").inc() + + +def record_scan_completed(target: str, duration: float, open_ports: int) -> None: + """Record metrics when a scan successfully completes.""" + + LOGGER.debug( + "metrics.scan_completed", + extra={ + "event": "scan.completed", + "target": target, + "duration": duration, + "open_ports": open_ports, + }, + ) + SCAN_ATTEMPTS.labels(status="completed").inc() + SCAN_DURATION.observe(duration) + OPEN_PORTS.observe(float(open_ports)) + + +def record_scan_failed(target: str, reason: str) -> None: + """Record metrics when a scan attempt fails.""" + + LOGGER.warning( + "metrics.scan_failed", + extra={"event": "scan.failed", "target": target, "reason": reason}, + ) + SCAN_ATTEMPTS.labels(status=reason).inc() + + +def metrics_payload() -> tuple[bytes, str]: + """Return the Prometheus metrics payload for HTTP responses.""" + + if _REGISTRY is None: # pragma: no cover - metrics disabled + return b"", "text/plain" + return generate_latest(_REGISTRY), CONTENT_TYPE_LATEST + + +__all__ = [ + "metrics_payload", + "record_scan_completed", + "record_scan_failed", + "record_scan_started", +] diff --git a/reconscript/reporters.py b/reconscript/reporters.py index 4ed66af..9bba182 100644 --- a/reconscript/reporters.py +++ b/reconscript/reporters.py @@ -9,7 +9,7 @@ import logging from datetime import datetime from pathlib import Path -from typing import Dict, Iterable, List, Optional, Sequence, Tuple +from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple PACKAGE_DIR = Path(__file__).resolve().parent PROJECT_TEMPLATES = PACKAGE_DIR.parent / "templates" @@ -53,16 +53,10 @@ def render_markdown(data: Dict[str, object]) -> str: except Exception: pass - target = data.get('target', 'unknown') - timestamp = data.get('timestamp', datetime.utcnow().isoformat() + 'Z') - hostname = data.get('hostname') or 'N/A' - ports = _format_list(data.get('ports', [])) - open_ports = _format_list(data.get('open_ports', [])) - findings = data.get('findings', []) - recommendations = _build_recommendations(findings) + context = _build_markdown_context(data) + lines = _render_markdown_sections(context) + return "\n".join(lines) + "\n" - lines: List[str] = [ - f def render_html(data: Dict[str, object], out_path: Path) -> Path: """Render the report data as HTML and persist it to ``out_path``.""" @@ -155,6 +149,73 @@ def _build_recommendations(findings: Sequence[Dict[str, object]]) -> List[str]: return suggestions +def _build_markdown_context(data: Dict[str, object]) -> Dict[str, Any]: + findings = data.get("findings", []) + context = { + "target": data.get("target", "unknown"), + "timestamp": data.get("timestamp", datetime.utcnow().isoformat() + "Z"), + "hostname": data.get("hostname") or "N/A", + "ports": _format_list(data.get("ports", [])) or "None", + "open_ports": _format_list(data.get("open_ports", [])) or "None detected", + "findings": findings if isinstance(findings, Sequence) else [], + "recommendations": _build_recommendations(findings if isinstance(findings, Sequence) else []), + "version": data.get("version", __version__), + "runtime": data.get("runtime", {}), + } + return context + + +def _render_markdown_sections(context: Dict[str, Any]) -> List[str]: + lines: List[str] = [ + f"# ReconScript Report — {context['target']}", + "", + f"*Generated:* {context['timestamp']}", + f"*Hostname:* {context['hostname']}", + f"*Ports scanned:* {context['ports']}", + f"*Open ports:* {context['open_ports']}", + "", + "## Findings", + ] + + findings = context.get("findings", []) + if findings: + for item in findings: + port = item.get("port", "n/a") if isinstance(item, dict) else "n/a" + issue = item.get("issue", "observation") if isinstance(item, dict) else str(item) + lines.append(f"- Port `{port}` — `{issue}`") + if isinstance(item, dict) and item.get("details") is not None: + details = item["details"] + if isinstance(details, (dict, list)): + rendered = json.dumps(details, indent=2, sort_keys=True) + lines.append(" ```json") + lines.extend(f" {line}" for line in rendered.splitlines()) + lines.append(" ```") + else: + lines.append(f" {details}") + else: + lines.append("No findings reported.") + + lines.append("") + lines.append("## Recommendations") + recommendations = context.get("recommendations", []) + if recommendations: + for recommendation in recommendations: + lines.append(f"- {recommendation}") + else: + lines.append("- Maintain current controls; no immediate actions identified.") + + runtime = json.dumps(context.get("runtime", {}), indent=2, sort_keys=True) + lines.extend( + [ + "", + "## Metadata", + f"- **Report version:** {context['version']}", + f"- **Runtime:** {runtime}", + ] + ) + return lines + + def _build_template_context(data: Dict[str, object]) -> Dict[str, object]: timestamp_raw = str(data.get("timestamp", datetime.utcnow().isoformat() + "Z")) findings = data.get("findings", []) diff --git a/reconscript/templates/index.html b/reconscript/templates/index.html index ff79b01..b4ca634 100644 --- a/reconscript/templates/index.html +++ b/reconscript/templates/index.html @@ -24,15 +24,15 @@

Launch Scoped Scan

- +
- +
-

+

diff --git a/reconscript/templates/layout.html b/reconscript/templates/layout.html index 707892b..a5adbba 100644 --- a/reconscript/templates/layout.html +++ b/reconscript/templates/layout.html @@ -19,15 +19,34 @@ button:disabled { opacity: 0.5; cursor: not-allowed; } .warning { background: #ffc107; color: #101820; padding: 0.5rem 1rem; border-radius: 4px; margin-bottom: 1rem; font-weight: 600; } nav a { margin-left: 1rem; } + .skip-link { + position: absolute; + left: -999px; + top: 1rem; + background: #5cc8ff; + color: #101820; + padding: 0.5rem 1rem; + border-radius: 4px; + font-weight: 600; + } + .skip-link:focus { + left: 1rem; + z-index: 1000; + } + a:focus, button:focus, input:focus, select:focus { + outline: 2px solid #f8e45c; + outline-offset: 2px; + } +
ReconScript
-
-
+
{% if public_ui %}
Public UI mode is enabled. Ensure authentication and TLS are enforced.
{% endif %} - {% for category, message in get_flashed_messages(with_categories=true) %} -
{{ message }}
- {% endfor %} + {% with flashes = get_flashed_messages(with_categories=true) %} + {% if flashes %} +
+ {% for category, message in flashes %} +
{{ message }}
+ {% endfor %} +
+ {% endif %} + {% endwith %} {% block content %}{% endblock %}
{% include 'consent_modal.html' %} diff --git a/reconscript/templates/login.html b/reconscript/templates/login.html index 2407dbe..c79d245 100644 --- a/reconscript/templates/login.html +++ b/reconscript/templates/login.html @@ -4,11 +4,11 @@

Authenticate

- + - +
-

Default credentials are sourced from ADMIN_USER/ADMIN_PASSWORD.

+

Credentials are sourced from the ADMIN_USER and ADMIN_PASSWORD environment variables. Default credentials are blocked in production builds.

{% endblock %} diff --git a/reconscript/ui.py b/reconscript/ui.py index 326c29d..b4a6639 100644 --- a/reconscript/ui.py +++ b/reconscript/ui.py @@ -14,12 +14,13 @@ Flask, abort, flash, + current_app, redirect, render_template, + Response, request, send_from_directory, url_for, - current_app, ) from flask_login import ( LoginManager, @@ -33,10 +34,12 @@ from .consent import ConsentError, load_manifest, validate_manifest from .core import ReconError, run_recon from .logging import configure_logging +from .metrics import metrics_payload from .report import ensure_results_dir, persist_report from .scope import validate_target LOGGER = logging.getLogger(__name__) +DEV_KEYS_DIR = Path(__file__).resolve().parents[1] / "keys" class StaticUser(UserMixin): @@ -45,12 +48,40 @@ def __init__(self, username: str, role: str) -> None: self.role = role +def _allow_dev_secrets() -> bool: + return os.environ.get("ALLOW_DEV_SECRETS", "false").lower() == "true" + + +def _enforce_secret_path(path: Path, *, env_var: str) -> Path: + resolved = path.expanduser().resolve() + if not resolved.exists(): + raise RuntimeError(f"{env_var} must point to an existing file (got {resolved}).") + try: + if DEV_KEYS_DIR in resolved.parents and not _allow_dev_secrets(): + raise RuntimeError( + f"{env_var} refers to a developer sample key. Provide deployment-specific secrets or set ALLOW_DEV_SECRETS=true for local testing." + ) + except RuntimeError: + raise + except Exception: + pass + return resolved + + def _load_secret_key() -> bytes: - secret_path = Path(os.environ.get("FLASK_SECRET_KEY_FILE", "keys/dev_flask_secret.key")) + secret_env = os.environ.get("FLASK_SECRET_KEY_FILE") + if not secret_env: + raise RuntimeError( + "FLASK_SECRET_KEY_FILE environment variable must reference a secure secret key file." + ) + secret_path = _enforce_secret_path(Path(secret_env), env_var="FLASK_SECRET_KEY_FILE") try: - return secret_path.read_bytes().strip() + secret = secret_path.read_bytes().strip() except OSError as exc: raise RuntimeError(f"Unable to read Flask secret key from {secret_path}: {exc}") from exc + if not secret: + raise RuntimeError(f"Secret key file {secret_path} is empty.") + return secret def _rbac_required(role: str): @@ -69,8 +100,14 @@ def wrapper(*args, **kwargs): def _load_user_credentials() -> tuple[str, str]: - username = os.environ.get("ADMIN_USER", "admin") - password = os.environ.get("ADMIN_PASSWORD", "changeme") + username = os.environ.get("ADMIN_USER", "").strip() + password = os.environ.get("ADMIN_PASSWORD", "").strip() + if not username or not password: + raise RuntimeError("ADMIN_USER and ADMIN_PASSWORD must be set for the ReconScript UI.") + if not _allow_dev_secrets() and (username == "admin" or password == "changeme"): + raise RuntimeError("Default credentials are not permitted. Set strong ADMIN_USER and ADMIN_PASSWORD values.") + if len(password) < 12 and not _allow_dev_secrets(): + raise RuntimeError("ADMIN_PASSWORD must be at least 12 characters long.") return username, password @@ -120,11 +157,22 @@ def inject_globals(): def health() -> tuple[str, int]: return "ok", 200 + @app.route("/metrics") + def metrics() -> Response: + payload, content_type = metrics_payload() + if not payload: + return Response("", status=204) + return Response(payload, mimetype=content_type) + @app.route("/login", methods=["GET", "POST"]) def login(): if request.method == "POST": submitted_user = request.form.get("username", "") submitted_pass = request.form.get("password", "") + LOGGER.info( + "ui.login.attempt", + extra={"event": "ui.login.attempt", "username": submitted_user, "success": submitted_user == username and submitted_pass == password}, + ) if submitted_user == username and submitted_pass == password: login_user(user) return redirect(url_for("index")) @@ -170,9 +218,21 @@ def index(): flash(f"Consent manifest invalid: {exc}", "error") if consent_path: consent_path.unlink(missing_ok=True) + LOGGER.warning("ui.consent.invalid", extra={"event": "ui.consent.invalid", "target": target, "error": str(exc)}) return render_template("index.html") try: + LOGGER.info( + "ui.scan.request", + extra={ + "event": "ui.scan.request", + "target": target, + "hostname": hostname, + "ports": ports, + "expected_ip": expected_ip, + "evidence_level": evidence_level, + }, + ) report = run_recon( target=target, hostname=hostname, @@ -185,11 +245,21 @@ def index(): flash(str(exc), "error") if consent_path: consent_path.unlink(missing_ok=True) + LOGGER.error("ui.scan.failed", extra={"event": "ui.scan.failed", "target": target, "error": str(exc)}) return render_template("index.html") persisted = persist_report(report, consent_source=consent_path, sign=False) if consent_path: consent_path.unlink(missing_ok=True) + LOGGER.info( + "ui.scan.completed", + extra={ + "event": "ui.scan.completed", + "target": target, + "report_id": persisted.report_id, + "open_ports": report.get("open_ports", []), + }, + ) return redirect(url_for("report_detail", report_id=persisted.report_id)) return render_template("index.html") diff --git a/requirements-dev.txt b/requirements-dev.txt index 3a0017e..308e267 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,3 +3,4 @@ ruff==0.5.4 bandit==1.7.9 pip-audit==2.7.3 pytest==8.2.2 +prometheus-client==0.20.0 diff --git a/requirements.lock b/requirements.lock index 3248be1..a26e2af 100644 --- a/requirements.lock +++ b/requirements.lock @@ -2,6 +2,7 @@ requests==2.31.0 rich==13.7.0 weasyprint==61.2 +prometheus-client==0.20.0 # Development extras pytest==7.4.4 responses==0.25.0 diff --git a/requirements.txt b/requirements.txt index d60fcf6..de46f50 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ urllib3==2.2.3 python-dotenv==1.0.1 rich==13.7.1 PyNaCl==1.5.0 +prometheus-client==0.20.0 diff --git a/tests/conftest.py b/tests/conftest.py index 86bf78a..0625f24 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,29 @@ +import os import sys from pathlib import Path +import pytest +from nacl.signing import SigningKey + ROOT = Path(__file__).resolve().parent.parent if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) + + +@pytest.fixture(autouse=True) +def _secure_defaults(monkeypatch, tmp_path): + monkeypatch.setenv("ALLOW_DEV_SECRETS", "true") + seed = os.urandom(32) + signing_key = SigningKey(seed) + private_key_path = tmp_path / "report_signing.key" + private_key_path.write_bytes(seed) + public_key_path = tmp_path / "consent_verify.key" + public_key_path.write_bytes(signing_key.verify_key.encode()) + monkeypatch.setenv("CONSENT_PUBLIC_KEY_PATH", str(public_key_path)) + monkeypatch.setenv("REPORT_SIGNING_KEY_PATH", str(private_key_path)) + secret_key = tmp_path / "flask-secret.key" + secret_key.write_text("unit-test-secret", encoding="utf-8") + monkeypatch.setenv("FLASK_SECRET_KEY_FILE", str(secret_key)) + monkeypatch.setenv("ADMIN_USER", "test-admin") + monkeypatch.setenv("ADMIN_PASSWORD", "test-password-123") + yield diff --git a/tests/test_cli_markdown.py b/tests/test_cli_markdown.py new file mode 100644 index 0000000..a79f4e0 --- /dev/null +++ b/tests/test_cli_markdown.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +from pathlib import Path + +from reconscript import cli, report as report_module + + +def test_cli_generates_markdown_report(tmp_path: Path, monkeypatch) -> None: + results_dir = tmp_path / "results" + monkeypatch.setenv("RESULTS_DIR", str(results_dir)) + monkeypatch.setattr(report_module, "RESULTS_DIR", results_dir) + monkeypatch.setattr(report_module, "INDEX_FILE", results_dir / "index.json") + monkeypatch.setattr(report_module, "LOCK_PATH", results_dir / ".index.lock") + exit_code = cli.main( + [ + "--target", + "127.0.0.1", + "--dry-run", + "--ports", + "80", + "--format", + "markdown", + ] + ) + assert exit_code == 0 + + report_dirs = [path for path in results_dir.iterdir() if path.is_dir()] + assert report_dirs, "CLI should create a timestamped report directory" + markdown_files = list(report_dirs[0].glob("report.*")) + assert any(file.suffix in {".markdown", ".md"} for file in markdown_files) + markdown_path = next(file for file in markdown_files if file.suffix in {".markdown", ".md"}) + content = markdown_path.read_text(encoding="utf-8") + assert "## Findings" in content + assert "## HTTP" in content diff --git a/tests/test_consent_validation.py b/tests/test_consent_validation.py index ea5b32e..c9da50f 100644 --- a/tests/test_consent_validation.py +++ b/tests/test_consent_validation.py @@ -2,6 +2,7 @@ import base64 import json +import os from datetime import datetime, timedelta, timezone from pathlib import Path @@ -10,15 +11,14 @@ from reconscript.consent import ConsentError, load_manifest, validate_manifest -DEV_PRIVATE = Path("keys/dev_ed25519.priv") - def _canonical(data: dict) -> bytes: return json.dumps(data, sort_keys=True, separators=(",", ":")).encode("utf-8") def _write_manifest(tmp_path: Path, body: dict) -> Path: - signing_key = SigningKey(DEV_PRIVATE.read_bytes()) + private_key = Path(os.environ["REPORT_SIGNING_KEY_PATH"]) # set by tests fixture + signing_key = SigningKey(private_key.read_bytes()) signed = signing_key.sign(_canonical(body)).signature payload = {**body, "signature": base64.b64encode(signed).decode("ascii")} manifest_path = tmp_path / "manifest.json" diff --git a/tests/test_reporters.py b/tests/test_reporters.py new file mode 100644 index 0000000..619f5a8 --- /dev/null +++ b/tests/test_reporters.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from pathlib import Path + +from reconscript.reporters import render_markdown, write_report + + +def sample_report() -> dict[str, object]: + return { + "target": "example.com", + "hostname": "example.com", + "ports": [80, 443], + "open_ports": [80], + "findings": [ + { + "port": 80, + "issue": "missing_security_headers", + "details": {"header": "Strict-Transport-Security"}, + } + ], + "runtime": {"started_at": "2024-01-01T00:00:00Z"}, + } + + +def test_render_markdown_fallback_contains_sections() -> None: + markdown = render_markdown(sample_report()) + assert "# ReconScript Report" in markdown + assert "## Findings" in markdown + assert "## Recommendations" in markdown + assert "Strict-Transport-Security" in markdown + + +def test_write_report_markdown(tmp_path: Path) -> None: + report_file = tmp_path / "report.md" + written_path, format_used = write_report(sample_report(), report_file, "markdown") + assert written_path.exists() + assert format_used == "markdown" + content = written_path.read_text(encoding="utf-8") + assert "## Metadata" in content