Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 69 additions & 2 deletions ai/query.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,7 @@
from pybatfish.datamodel.flow import HeaderConstraints

from analysis import findings, snapshot
from analysis.pipeline import classify_parse_status

# ------------------------------------------------------------------------------
# 1. Intent classification -- a closed set of three, nothing else
Expand Down Expand Up @@ -594,6 +595,10 @@ def _answer_reachability_question(
f"Can {source_device} reach {destination_display}?"
)

refusal, caveat = _what_was_not_read(bf, question_understood)
if refusal is not None:
return refusal

try:
frame = (
bf.q.traceroute(
Expand Down Expand Up @@ -635,12 +640,65 @@ def _answer_reachability_question(
traces = frame.iloc[0]["Traces"]
return {
"question_understood": question_understood,
"answer": _describe_traces(source_device, destination_display, traces),
"answer": _describe_traces(source_device, destination_display, traces) + caveat,
"grounded": True,
"resolved_entities": resolved_entities,
}


def _what_was_not_read(bf: Session, question_understood: str) -> "tuple[Optional[Dict[str, Any]], str]":
"""Refuse when Batfish could not read the config; caveat when it read part.

Returns (refusal, caveat). `refusal` is the whole answer to give INSTEAD
of asking Batfish anything, or None. `caveat` is a sentence to add to the
answer, or "".

WHY THIS EXISTS
An empty answer from Batfish means "none" only if Batfish read the
config. Measured on main dc93ccb, a file it could not read at all
(status UNKNOWN, zero devices) answered "are there any dead rules?"
with "No, checked the whole snapshot and found none.", grounded. That
is "we found nothing" standing in for "we could not look" -- F-4.

The rule is analyse()'s, and so is the classification
(classify_parse_status, #217): an unreadable file means no answer at
all, a partly read one means the answer AND the caveat.
"""
try:
fatal, partial = classify_parse_status(bf)
except Exception as error:
return {
"question_understood": question_understood,
"answer": (
"I could not tell what Batfish was able to read in this "
"config, so I will not answer. The underlying reason: "
+ findings.describe_error(error)
),
"grounded": False,
# Nothing was resolved, and a follow-up must not reuse anything
# from a config we could not read (see the module docstring).
"resolved_entities": None,
}, ""

if fatal:
return {
"question_understood": question_understood,
"answer": (
"Batfish could not read: " + "; ".join(fatal) + ". I cannot "
"answer about rules it did not read, so I will not say yes or no."
),
"grounded": False,
"resolved_entities": None,
}, ""

if partial:
return None, (
" Batfish could not read all of: " + "; ".join(partial)
+ ", so nothing is claimed about the lines it skipped."
)
return None, ""


# Same set analysis/checks/routing.py uses, and for the same reason,
# repeated here rather than imported so ai/ does not depend on a specific
# check module for a general Batfish-interpretation fact. pybatfish's own
Expand Down Expand Up @@ -695,6 +753,10 @@ def _describe_traces(source_device: str, destination_display: str, traces: Any)
def _answer_whole_snapshot_question(
bf: Session, *, batfish_question: str, question_understood: str
) -> Dict[str, Any]:
refusal, caveat = _what_was_not_read(bf, question_understood)
if refusal is not None:
return refusal

try:
frame = getattr(bf.q, batfish_question)().answer().frame()
except Exception as error:
Expand All @@ -709,9 +771,13 @@ def _answer_whole_snapshot_question(
}

if frame.empty:
# "The whole snapshot" only when Batfish read all of it.
return {
"question_understood": question_understood,
"answer": "No, checked the whole snapshot and found none.",
"answer": (
"No, found none in the part of the config Batfish could read." + caveat
if caveat else "No, checked the whole snapshot and found none."
),
"grounded": True,
"resolved_entities": None,
}
Expand All @@ -721,6 +787,7 @@ def _answer_whole_snapshot_question(
"answer": (
f"Yes, found {len(frame)}. See the dashboard's findings list for "
"the specific lines, this question only confirms whether any exist."
+ caveat
),
"grounded": True,
"resolved_entities": None,
Expand Down
4 changes: 4 additions & 0 deletions analysis/checks/policy_compliance.py
Original file line number Diff line number Diff line change
Expand Up @@ -694,6 +694,10 @@ def run(bf: Session) -> List[Dict[str, Any]]:
# note in the module docstring.
hits: List[Any] = []
failures: List[Exception] = []
# A rule with no queries asks Batfish nothing, and "nothing found"
# would then read as "the rule holds". It is a rule we could not check.
if not rule["queries"]:
failures.append(ValueError("this rule has no queries, so nothing was asked"))
for headers in rule["queries"]:
try:
hits.extend(_search(bf, node, filter_name, action, headers))
Expand Down
32 changes: 27 additions & 5 deletions analysis/checks/routing.py
Original file line number Diff line number Diff line change
Expand Up @@ -269,6 +269,12 @@ def _evaluate(expected: str, traces: Sequence[Any]) -> Optional[str]:
exercises both branches and specifically regression-tests the
EXITS_NETWORK case described in the module docstring.
"""
# No trace is no evidence either way. With nothing traced, both branches
# below find nothing wrong and report "holds" -- a false all-clear, so it
# is refused the same way as a bad "expected" at the bottom.
if len(traces) == 0:
raise ValueError("traceroute returned no traces, so nothing was checked")

successes = [t for t in traces if t.disposition in SUCCESS_DISPOSITIONS]
failures = [t for t in traces if t.disposition not in SUCCESS_DISPOSITIONS]
multi = len(traces) > 1
Expand All @@ -294,10 +300,9 @@ def _evaluate(expected: str, traces: Sequence[Any]) -> Optional[str]:
return detail

# A statement with anything else in "expected" is a bug in ROUTES, not a
# runtime condition -- raising here means run() surfaces it through the
# pipeline's own crash-isolation as a status="error" finding (see
# run_check() in analysis/pipeline.py) rather than silently doing nothing,
# which is exactly the failure this function exists to replace.
# runtime condition -- raising here means run() reports that one route as
# status="error" rather than silently doing nothing, which is exactly the
# failure this function exists to replace.
raise ValueError(f"route 'expected' must be REACHABLE or UNREACHABLE, got {expected!r}")


Expand Down Expand Up @@ -465,7 +470,24 @@ def run(bf: Session) -> List[Dict[str, Any]]:
continue

row = frame.iloc[0]
violation = _evaluate(route["expected"], row["Traces"])
try:
violation = _evaluate(route["expected"], row["Traces"])
except ValueError as error:
# This route cannot be judged: nothing was traced, or its "expected"
# is not a value we know. Same handling as the two cases above, so
# one bad route does not end the check for the routes after it or
# for the policy-free analyses below.
results.append(
findings.error_finding(
check=CHECK_NAME,
device=node,
summary=f"Could not check: {route['description'].lower()}",
detail=findings.describe_error(error),
source=f"{node}:{route['dst_ip']}",
number=route["number"],
)
)
continue

if violation is None:
# The statement holds. Nothing to report.
Expand Down
23 changes: 23 additions & 0 deletions analysis/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -699,6 +699,29 @@ def _restore_protected_findings(
)
continue

# Status is what a CHECK concluded, not a post-processor's to change
# (docs/severity-rules.md section 2, which says this is enforced
# here). Until it was, a stage setting status="none" turned "could
# not check" into a green tick and nothing said so.
if current.get("status") != original.get("status"):
number += 1
complaints.append(
findings.error_finding(
check=name,
device=original.get("device", "unknown"),
summary=f"The {name.replace('_', ' ')} stage changed a finding's status",
detail=(
f"{fid} was {original.get('status')} and was set to "
f"{current.get('status')}. Restored. A post-processor "
"may re-rate and re-order, never change what a check "
"concluded."
),
source="analysis/pipeline.py",
number=number,
)
)
current["status"] = original.get("status")

if original.get("status") != "error":
continue

Expand Down
Loading
Loading