Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 83 additions & 1 deletion analysis/checks/policy_compliance.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,8 @@
is reported as proved, and what could not be reached is reported separately.
"""

from typing import Any, Dict, List, Tuple
import re
from typing import Any, Dict, List, Set, Tuple

from pybatfish.client.session import Session
from pybatfish.datamodel.flow import HeaderConstraints
Expand Down Expand Up @@ -302,6 +303,47 @@ def _search(
return [frame.iloc[i] for i in range(len(frame))]


# A `filter` containing any of these is a Batfish specifier (a regex, an
# interface's filter, a list), not one name, so there is no single name to look
# for. Such rules are checked exactly as before #430.
_FILTER_SPECIFIER = re.compile(r"[/@(,]")


def _applied_filters(bf: Session) -> Dict[str, Set[str]]:
"""{node: filter names applied to any of its interfaces, either direction}.

WHY (#430). searchFilters proves what a filter WOULD do to traffic passing
through it, not that any traffic passes through it. Measured on
rtr-us5-secure with `ip access-group acl_in in` removed: every rule held,
PC-000 gave a green tick, and traceroute delivered telnet from the LAN to
the server. So a rule may only pass if its filter is in the path.

Names are casefolded and stripped, because Batfish resolves `ACL_IN` and
`acl_in ` to `acl_in` (#387 review). Exceptions are NOT caught: run()
turns them into "could not check", because not knowing whether the filter
is applied is not knowing whether the rule holds.
"""
frame = (
bf.q.interfaceProperties(properties="Incoming_Filter_Name,Outgoing_Filter_Name")
.answer()
.frame()
)
applied: Dict[str, Set[str]] = {}
for _, row in frame.iterrows():
names = applied.setdefault(row["Interface"].hostname, set())
for column in ("Incoming_Filter_Name", "Outgoing_Filter_Name"):
if row[column]:
names.add(str(row[column]).strip().casefold())
return applied


def _not_in_path(node: str, filter_name: str, applied: Dict[str, Set[str]]) -> bool:
"""True when Batfish's model applies `filter_name` to no interface of `node`."""
if _FILTER_SPECIFIER.search(filter_name):
return False
return filter_name.strip().casefold() not in applied.get(node, set())


def _describe(rule: Dict[str, Any], hits: List[Any]) -> str:
"""Turn matched rows into the evidence string, in Batfish's own words.

Expand Down Expand Up @@ -601,11 +643,51 @@ def run(bf: Session) -> List[Dict[str, Any]]:
)
)

# Is each rule's filter in the path at all? (#430). Asked once, not per rule.
applied: Dict[str, Set[str]] = {}
lookup_error = None
if applicable:
try:
applied = _applied_filters(bf)
except Exception as error:
lookup_error = error

for rule in applicable:
node = rule["node"]
filter_name = rule["filter"]
action = ACTION_BY_KIND[rule["kind"]]

# A filter in no interface's path decides nothing, so whatever
# searchFilters says about it, the rule is unchecked rather than
# held or broken. The wording says "Batfish's model", not "your
# config": on vendor-asa the config does apply its ACL, in a line
# Batfish did not parse, and claiming otherwise would be false.
# A failed lookup leaves `applied` empty, so every plainly named filter
# lands here too; a specifier filter is checked as before either way.
if _not_in_path(node, filter_name, applied):
if lookup_error is not None:
why = ("We could not find out which filters are applied to "
f"{node}'s interfaces, so we cannot tell whether "
f"{filter_name!r} is in the path of any traffic. "
+ findings.describe_error(lookup_error))
else:
why = (f"In Batfish's model of {node}, filter {filter_name!r} "
"is not applied to any interface, so no traffic passes "
"through it and what it permits or denies is not what "
"the device does. If your config does apply it, Batfish "
"did not understand that line; check the parse warnings.")
results.append(
findings.error_finding(
check=CHECK_NAME,
device=node,
summary=f"Could not check policy rule: {rule['description'].lower()}",
detail=_with_provenance(why, policy_label),
source=f"{node}:{filter_name}",
number=rule["number"] + ERROR_NUMBER_OFFSET,
)
)
continue

# Each arm is isolated. The arms describe ONE space between them, so a
# hit in any of them violates the rule -- but a failure in one must not
# discard what another one proved. See issue #22 and the PARTIAL CHECKS
Expand Down
16 changes: 13 additions & 3 deletions docs/policy-rules.md
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,17 @@ can surface as `success`. **Neither bucket means what a policy check needs it
to mean.**

`searchFilters` reasons at the filter level and needs no routing, so it is
correct on these fixtures and on any config where policy intent lives in ACLs
— and it cannot be fooled in either direction.
correct on these fixtures and on any config where policy intent lives in ACLs.

**It can be fooled in one way: a filter that is in no traffic's path (#430).**
With `ip access-group acl_in in` removed from `rtr-us5-secure`, every rule held
and `PC-000` gave a green tick, while `traceroute` delivered telnet from the LAN
to the server. The proof was right about the filter and wrong about the device.
So before a rule can pass, the check confirms its filter is some interface's
incoming or outgoing filter on that node (`interfaceProperties`). If it is not,
or that cannot be found out, the rule reports **could not check** (`PC-05n`).
A `filter` written as a specifier (`/regex/`, `@in(...)`, a list) names no
single filter to look for, and is checked as before.

> Found by @ARSH871-bot re-running this section rather than reading it, after
> it was quoted in a team message. The measurement here was right and the
Expand All @@ -122,7 +131,8 @@ name **raises `BatfishException`** rather than returning an empty frame.

So the check must let that exception surface as a `status="error"` finding and
must never wrap these calls in a bare `except: pass`. Empty then means exactly
one thing: the policy holds.
one thing: the policy holds **for that filter**, and the applied-filter check
above is what makes the filter's answer the device's answer.

### Do not use `invertSearch` to express "everything except X"

Expand Down
3 changes: 3 additions & 0 deletions tests/test_device_scoping.py
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,9 @@ def test_a_missing_device_never_becomes_a_clean_result(monkeypatch):
def _run_policy_with_devices(monkeypatch, present):
"""Run policy_compliance with a known device set and Batfish stubbed out."""
monkeypatch.setattr(policy_compliance.snapshot, "device_names", lambda bf: present)
# The filter is in the path (#430); that guard has its own tests.
monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False)
monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {})
# No rule may actually reach Batfish: every one that runs returns "holds".
monkeypatch.setattr(policy_compliance, "_search", lambda *a, **k: [])
return policy_compliance.run(_FakeSession())
Expand Down
3 changes: 3 additions & 0 deletions tests/test_found_status_is_reported.py
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,9 @@ def test_policy_compliance_reports_a_violated_rule_as_found(monkeypatch):
monkeypatch.setattr(policy_compliance, "POLICY_RULES", [rule])
monkeypatch.setattr(policy_compliance.snapshot, "device_names",
lambda bf: {"rtr-us5"})
# The filter is in the path (#430); that guard has its own tests.
monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False)
monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {})
# A non-empty hit list means the rule is violated. The shape is what
# `_describe()` actually reads, so this exercises the real evidence text
# rather than a stand-in that would break the moment it was used.
Expand Down
210 changes: 210 additions & 0 deletions tests/test_policy_filter_applied.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
"""
Netwise -- a policy rule only passes if its filter is in the path (#430).

WHY THIS FILE EXISTS
searchFilters proves what a filter WOULD do to traffic passing through it.
It does not prove any traffic passes through it. Measured while researching
#423, on rtr-us5-secure with `ip access-group acl_in in` removed:

PC-000 none No issues found by policy compliance
traceroute LAN -> 10.20.0.5 TCP/23 DELIVERED_TO_SUBNET

A green tick while forbidden traffic is delivered. A rule whose filter is
applied to no interface now reports "could not check" instead.

Most tests here fake the session. The last one runs the measurement above
against real Batfish, because the claim is about what Batfish's model
says, and a fake can only repeat what the test told it.

RUN
pytest tests/test_policy_filter_applied.py -v
"""

from pathlib import Path

import pytest

from analysis import pipeline
from analysis.checks import policy_compliance
from conftest import needs_batfish

RULE = {
"number": 2,
"description": "The internal server is reachable only over HTTPS",
"kind": "prohibition",
"node": "rtr-us5",
"filter": "acl_in",
"violation_severity": "high",
"violation_summary": "The internal server accepts traffic other than HTTPS",
"queries": [{"srcIps": "10.10.10.0/24", "dstIps": "10.20.0.5"}],
}


# --- which filters count as applied -----------------------------------------


class _Interface:
def __init__(self, hostname):
self.hostname = hostname


class _Frame:
"""Just enough of a pandas frame for _applied_filters to iterate."""

def __init__(self, rows):
self._rows = rows

def iterrows(self):
return enumerate(self._rows)


class _Session:
def __init__(self, rows):
frame = _Frame(rows)

class _Q:
def interfaceProperties(self, **kwargs):
class _A:
def answer(self):
return self

def frame(self):
return frame
return _A()
self.q = _Q()


def _row(node, incoming=None, outgoing=None):
return {"Interface": _Interface(node), "Incoming_Filter_Name": incoming,
"Outgoing_Filter_Name": outgoing}


def test_both_directions_count_and_names_are_casefolded():
applied = policy_compliance._applied_filters(_Session([
_row("rtr-us5", incoming="ACL_IN"),
_row("rtr-us5", outgoing="acl_out"),
_row("rtr-us5"), # an interface with no filter
_row("rtr-hq", incoming="hq_in"),
]))
assert applied == {"rtr-us5": {"acl_in", "acl_out"}, "rtr-hq": {"hq_in"}}


@pytest.mark.parametrize("node, name, expected", [
("rtr-us5", "acl_in", False), # applied
("rtr-us5", "ACL_IN ", False), # Batfish resolves this to acl_in
("rtr-us5", "acl_out", True), # defined or not, applied nowhere
("rtr-hq", "acl_in", True), # applied, but on another device
("rtr-us5", "/acl/", False), # a specifier: not one name to look for
("rtr-us5", "@in(Gi0/0)", False),
])
def test_not_in_path(node, name, expected):
applied = {"rtr-us5": {"acl_in"}}
assert policy_compliance._not_in_path(node, name, applied) is expected


# --- what run() reports -----------------------------------------------------


@pytest.fixture
def one_rule(monkeypatch):
monkeypatch.setattr(policy_compliance, "POLICY_RULES", [RULE])
monkeypatch.setattr(policy_compliance.snapshot, "device_names",
lambda bf: {"rtr-us5"})
searched = []

def _search(bf, node, filt, action, headers):
searched.append(filt)
return []
monkeypatch.setattr(policy_compliance, "_search", _search)
return searched


def test_an_unapplied_filter_is_could_not_check_not_a_pass(one_rule, monkeypatch):
monkeypatch.setattr(policy_compliance, "_applied_filters",
lambda bf: {"rtr-us5": {"some_other_acl"}})

results = policy_compliance.run(object())

assert [f["status"] for f in results] == ["error"]
assert results[0]["id"] == "PC-052"
assert results[0]["summary"].startswith("Could not check policy rule")
assert "is not applied to any interface" in results[0]["evidence"]["detail"]
# The filter's answer is not asked for at all, so it cannot leak through.
assert one_rule == []


def test_the_message_blames_the_model_not_the_config(one_rule, monkeypatch):
# vendor-asa applies its ACL in a line Batfish does not parse, so "your
# config does not apply it" would be false there.
monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {})

detail = policy_compliance.run(object())[0]["evidence"]["detail"]

assert "In Batfish's model of rtr-us5" in detail
assert "check the parse warnings" in detail


def test_not_knowing_what_is_applied_is_could_not_check(one_rule, monkeypatch):
def _broken(bf):
raise RuntimeError("interfaceProperties failed")
monkeypatch.setattr(policy_compliance, "_applied_filters", _broken)

results = policy_compliance.run(object())

assert [f["status"] for f in results] == ["error"]
assert "could not find out which filters are applied" in results[0]["evidence"]["detail"]
assert one_rule == []


def test_a_specifier_filter_is_still_searched_when_the_lookup_fails(one_rule, monkeypatch):
# The guard never applies to a specifier, so its failing must not either.
monkeypatch.setattr(policy_compliance, "POLICY_RULES", [dict(RULE, filter="/acl_in/")])

def _broken(bf):
raise RuntimeError("interfaceProperties failed")
monkeypatch.setattr(policy_compliance, "_applied_filters", _broken)

results = policy_compliance.run(object())

assert [f["status"] for f in results] == ["none"]
assert one_rule == ["/acl_in/"]


def test_an_applied_filter_is_checked_as_before(one_rule, monkeypatch):
monkeypatch.setattr(policy_compliance, "_applied_filters",
lambda bf: {"rtr-us5": {"acl_in"}})

results = policy_compliance.run(object())

assert [f["status"] for f in results] == ["none"]
assert one_rule == ["acl_in"]


# --- the measurement, against real Batfish ----------------------------------


SECURE = Path(__file__).parent / "fixtures" / "rtr-us5-secure" / "configs" / "rtr-us5.cfg"


@needs_batfish
def test_real_batfish_no_green_tick_for_an_acl_applied_nowhere(tmp_path):
configs = tmp_path / "configs"
configs.mkdir()
text = SECURE.read_text()
assert "ip access-group acl_in in" in text
(configs / "rtr-us5.cfg").write_text(text.replace(" ip access-group acl_in in\n", ""))

results = [f for f in pipeline.analyse(tmp_path, check_names=["policy_compliance"])
if f["check"] == "policy_compliance"]

assert "none" not in {f["status"] for f in results}
assert {f["status"] for f in results} == {"error"}


@needs_batfish
def test_real_batfish_the_secure_fixture_still_passes():
results = [f for f in pipeline.analyse(SECURE.parent.parent,
check_names=["policy_compliance"])
if f["check"] == "policy_compliance"]

assert "error" not in {f["status"] for f in results}
3 changes: 3 additions & 0 deletions tests/test_policy_partial_checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,9 @@ def two_arm_rule(monkeypatch):
"""
monkeypatch.setattr(policy_compliance.snapshot, "device_names",
lambda bf: {"rtr-us5"})
# The filter is in the path (#430); that guard has its own tests.
monkeypatch.setattr(policy_compliance, "_not_in_path", lambda *args: False)
monkeypatch.setattr(policy_compliance, "_applied_filters", lambda bf: {})
rule = {
"number": 2,
"description": "The internal server is reachable only over HTTPS",
Expand Down
Loading
Loading