diff --git a/ai/propose.py b/ai/propose.py index 0888a0a..bb094f4 100644 --- a/ai/propose.py +++ b/ai/propose.py @@ -14,6 +14,7 @@ "impact": list[dict], # F-1 findings from change_impact, [] if none "verified": bool, "warning": bool, + "opens_access": bool, # see the warning/opens_access split below "grounded": bool, "answer": str} @@ -49,11 +50,15 @@ compareFilters/differentialReachability comparison Shubham's feature (#30) already runs for a given before/after pair. That module already grades severity by direction: a change that newly OPENS traffic is - `high` (silent -- nothing breaks, so nobody looks), a change that - CLOSES traffic is `medium` (loud -- someone notices). `warning=True` - here means exactly "a high-severity finding is present in that diff" -- - running the tool the project already trusts, twice, and reading its - answer. No new judgement is introduced. + `high` (silent -- nothing breaks, so nobody looks), a filter or + reachability change that CLOSES traffic is `medium` (loud -- someone + notices). A CLOSING that breaches a written policy requirement is the + one exception (#402): it carries that rule's own `violation_severity` + instead, which can be `high` -- this PR exists because that case was + reported with the wrong banner. `warning=True` here means exactly "a + high-severity finding is present in that diff" -- running the tool + the project already trusts, twice, and reading its answer. No new + judgement is introduced. This module does NOT audit the proposed state the way change_impact's own docstring flags as unbuilt ("it will not tell you that the after @@ -311,18 +316,31 @@ def _refuse(reason: str) -> Dict[str, Any]: "impact": [], "verified": False, "warning": False, + "opens_access": False, "grounded": False, "answer": reason, } def propose_change( - request: str, before_dir: "str | Path", host: "str | None" = None + request: str, + before_dir: "str | Path", + host: "str | None" = None, + policy: Any = None, ) -> Dict[str, Any]: """Propose ONE config change from plain English, and simulate it against a scratch copy of `before_dir` -- see the module docstring for the shape, the scope limit, and why neither step here calls a model. + `policy` (#402): the user's staged policy, or None to compare against + Netwise's built-in example rules. Passed straight through to + `analyse_change()`, which already installs and clears it -- this + function does no policy handling of its own. Before this parameter + existed, propose always used the built-in rules regardless of what the + caller had staged, and `policy_compliance`'s own evidence then correctly + -- but misleadingly -- reported "no policy file was supplied", because + nothing here had ever told it otherwise. + Never raises. Every failure -- an unparseable request, an unknown device, an unreachable Batfish -- is a refusal, not an exception, the same convention `ai/query.py`'s `answer_question()` and @@ -455,7 +473,7 @@ def propose_change( after_dir = Path(tmp) / "after" shutil.copytree(before_dir, after_dir) (after_dir / "configs" / cfg_path.name).write_text(new_cfg_text, encoding="utf-8") - impact = change_impact.analyse_change(before_dir, after_dir, host=host) + impact = change_impact.analyse_change(before_dir, after_dir, host=host, policy=policy) # Shown on the dashboard as a before/after diff of the affected filter, # not just the one generated line -- a line in isolation does not say @@ -489,15 +507,48 @@ def propose_change( # know" reads as LESS alarming than the truth. Gating warning on # verified silently swallowed the proof in that direction. verified = all(f["status"] != "error" for f in impact) - warning = any( - f["status"] == "found" and f["severity"] == "high" for f in impact + high_severity_found = [ + f for f in impact if f["status"] == "found" and f["severity"] == "high" + ] + warning = bool(high_severity_found) + # Returned alongside `warning` rather than left for the caller to + # re-derive from `impact` (#405 review, Arsh + Samika). `web/static/ + # app.js` used to render every `warning=True` response with "This change + # opens access that is currently blocked", unconditionally -- correct + # when `warning` meant only an opening, wrong the moment it could also + # mean a closing breach that is still a serious effect. The same + # direction check `warning_text` below already makes, hoisted into the + # response so the one place that decides it is this function, not every + # renderer that reads the response. + opens_access = warning and all( + f.get("direction") == "opening" for f in high_severity_found ) if warning: + # The trigger stays severity-high regardless of direction (#402) -- + # under-warning on a genuinely serious change is the dangerous + # direction, same reasoning as the verified/warning split above. What + # changes is the WORDING: "opens access" is only true when every + # triggering finding actually opens something. A high-severity + # POLICY finding can instead be a requirement breached, which CLOSES + # access that should stay open -- the opposite claim, found by + # @ARSH871-bot reviewing #389. Checking the real `direction` field + # (analysis/change_impact.py) rather than assuming severity implies + # it is what avoids repeating that mistake here. + if opens_access: + warning_text = ( + "Warning: simulating this change shows it newly opens access " + "that was previously blocked -- see the impact list." + ) + else: + warning_text = ( + "Warning: simulating this change has a serious effect that " + "is not simply opening access -- see the impact list for " + "exactly what changed." + ) answer = ( f"Generated: {rule_line}, on {device}'s {acl_name!r} filter. " - "Warning: simulating this change shows it newly opens access " - "that was previously blocked -- see the impact list." + f"{warning_text}" ) if not verified: answer += ( @@ -529,6 +580,7 @@ def propose_change( "impact": impact, "verified": verified, "warning": warning, + "opens_access": opens_access, "grounded": True, "answer": answer, } @@ -602,7 +654,13 @@ def propose_and_scan( "findings": [] and "grounded": False -- there is nothing to scan when there was nothing to propose. """ - base = propose_change(request, before_dir, host=host) + # Every call to propose_change() should see the same policy regardless + # of caller (#405 review, Samika). Only base["proposed_change"] and + # base["answer"] are read below -- neither depends on policy -- so this + # has no visible effect on what this function returns today. Kept + # anyway: passing the wrong policy into a shared helper is the wrong + # default even when nothing currently reads the result. + base = propose_change(request, before_dir, host=host, policy=policy) change = base["proposed_change"] if change is None: return { diff --git a/analysis/change_impact.py b/analysis/change_impact.py index 0f38339..3320c5e 100644 --- a/analysis/change_impact.py +++ b/analysis/change_impact.py @@ -151,6 +151,36 @@ def _direction(line_action: str) -> str: return "opened" if str(line_action).upper() == "PERMIT" else "closed" +#: One vocabulary for "direction" across all three finding producers here, +#: carried on the finding as an extra key (see _attach_direction()) rather +#: than left for a caller to infer from severity or parse out of `detail`. +#: +#: WHY THIS EXISTS (#402, raised by @ARSH871-bot reviewing #389) +#: For the filter and reachability producers, severity already IS a +#: faithful proxy for direction (_OPENING_SEVERITY/_CLOSING_SEVERITY are +#: assigned directly from "opened"/"closed"). For the POLICY producer it +#: is not: severity there tracks whether a breach was newly created, and +#: a newly-created breach can be a prohibition violated (access opens) +#: OR a requirement violated (access closes -- something that must flow +#: no longer does). A caller that assumes "severity high means opened" +#: -- ai/propose.py did -- gets the direction backwards for exactly the +#: second case. A real field removes the need to infer it at all. +_DIRECTION_NORMAL_FORM = { + "opened": "opening", + "closed": "closing", + "changed": "changed", + "opening": "opening", + "tightening": "closing", +} + + +def _attach_direction(finding: Dict[str, Any], raw_direction: str) -> Dict[str, Any]: + """Set the normalised `direction` key on `finding` and return it, so + every append site reads as one line rather than three.""" + finding["direction"] = _DIRECTION_NORMAL_FORM[raw_direction] + return finding + + # Which trace dispositions count as the flow having got through. # # DELIBERATELY WIDER THAN analysis/checks/routing.py'S SET, which excludes @@ -239,23 +269,26 @@ def _filter_change_findings( severity = _CLOSING_SEVERITY results.append( - findings.make_finding( - check=CHECK_NAME, - severity=severity, - device=node, - summary=summary, - # Both lines, verbatim from Batfish. The reader needs the pair - # to see the change; one line alone is not a diff. - detail=( - f"After: {row['Line_Content']} (line {row['Line_Index']}, " - f"{row['Line_Action']}). " - f"Before: {row['Reference_Line_Content']} " - f"(line {row['Reference_Line_Index']}). " - "Both match at least one common flow and treat it differently." + _attach_direction( + findings.make_finding( + check=CHECK_NAME, + severity=severity, + device=node, + summary=summary, + # Both lines, verbatim from Batfish. The reader needs the + # pair to see the change; one line alone is not a diff. + detail=( + f"After: {row['Line_Content']} (line {row['Line_Index']}, " + f"{row['Line_Action']}). " + f"Before: {row['Reference_Line_Content']} " + f"(line {row['Reference_Line_Index']}). " + "Both match at least one common flow and treat it differently." + ), + source=f"{node}:{filter_name}", + status="found", + number=number, ), - source=f"{node}:{filter_name}", - status="found", - number=number, + direction, ) ) number += 1 @@ -297,19 +330,22 @@ def _reachability_change_findings( severity = _OPENING_SEVERITY results.append( - findings.make_finding( - check=CHECK_NAME, - severity=severity, - device=_device_of(row["Flow"]), - summary=summary, - detail=( - f"Flow {row['Flow']}. " - f"After: {_first_trace(row['Snapshot_Traces'])}. " - f"Before: {_first_trace(row['Reference_Traces'])}." + _attach_direction( + findings.make_finding( + check=CHECK_NAME, + severity=severity, + device=_device_of(row["Flow"]), + summary=summary, + detail=( + f"Flow {row['Flow']}. " + f"After: {_first_trace(row['Snapshot_Traces'])}. " + f"Before: {_first_trace(row['Reference_Traces'])}." + ), + source="differentialReachability", + status="found", + number=number, ), - source="differentialReachability", - status="found", - number=number, + direction, ) ) number += 1 @@ -436,25 +472,28 @@ def _policy_change_findings(bf: Any, number: int) -> Tuple[List[Dict[str, Any]], not_compared.append(f"{_rule_label(rule)}: more rule changes than finding numbers") continue results.append( - findings.make_finding( - check=CHECK_NAME, - severity=(rule.get("violation_severity", "high") if created else "low"), - device=rule["node"], - summary=( - f"This change BREAKS {_rule_label(rule)}" - if created else - f"This change FIXES {_rule_label(rule)}" - ), - detail=( - f"{rule['kind']} on {rule['node']} filter {rule['filter']}: " - f"{'held before, breached after' if created else 'breached before, holds after'}. " - f"Direction: {direction}. " - + (f"Violation if applied: {rule.get('violation_summary')}." if created - else "The breach this rule reports is gone in the proposed config.") + _attach_direction( + findings.make_finding( + check=CHECK_NAME, + severity=(rule.get("violation_severity", "high") if created else "low"), + device=rule["node"], + summary=( + f"This change BREAKS {_rule_label(rule)}" + if created else + f"This change FIXES {_rule_label(rule)}" + ), + detail=( + f"{rule['kind']} on {rule['node']} filter {rule['filter']}: " + f"{'held before, breached after' if created else 'breached before, holds after'}. " + f"Direction: {direction}. " + + (f"Violation if applied: {rule.get('violation_summary')}." if created + else "The breach this rule reports is gone in the proposed config.") + ), + source=f"{policy_label}; rule {rule.get('number', '?')}", + status="found", + number=number, ), - source=f"{policy_label}; rule {rule.get('number', '?')}", - status="found", - number=number, + direction, ) ) number += 1 diff --git a/docs/finding-format.md b/docs/finding-format.md index 24e90fe..68e7e9e 100644 --- a/docs/finding-format.md +++ b/docs/finding-format.md @@ -512,3 +512,49 @@ somewhere later reports it as agreed, sourced from nothing. > > **A ⬜ here is not a disagreement.** It is the absence of a recorded answer, > which is the only state this file is able to distinguish from agreement. + +### A-5 - one vocabulary for `direction`, in the field and in the words + +**PROPOSED, not ratified.** + +The structural `direction` field already uses three values in code - +`"opening"`, `"closing"`, `"changed"` - via `_DIRECTION_NORMAL_FORM` in +`analysis/change_impact.py`, and `opens_access` in `ai/propose.py` already +relies on exactly those three. That part is a de facto standard: nothing +here proposes changing it, only recording that it is agreed. + +What is not settled is the human-readable `"Direction: X"` phrase inside +`evidence.detail`, which still uses a second vocabulary for the same fact. +`analysis/change_impact.py:488` writes `"Direction: tightening"` for the +same case the structural field calls `"closing"`: `_RULE_DIRECTION` maps a +policy requirement newly breached to the raw value `"tightening"`, which +`_attach_direction()` then normalises to `"closing"` on the field, while the +sentence a person reads still says "tightening". PR #406 (not yet merged) +separately writes `"Direction: changed."` for its own case, a third, +independently chosen word for a fact the field already has a name for. + +This proposes ratifying `opening` / `closing` / `changed` as the single +vocabulary for BOTH the structural field and the `"Direction: X"` phrasing, +one set of words rather than two that happen to describe the same thing. + +**Not changing `analysis/change_impact.py:488`'s actual wording in this +PR.** That edit, replacing `tightening` with `closing` in the sentence +itself, is exactly the code change this table is meant to gate, the same +discipline A-4 uses for `cve_mapping`: the agreement is recorded before the +behaviour changes, not after. + +| Member | Why it touches them | Agreed | +|---|---|---| +| **Arsh** | Raised the inconsistency on #405's review; owns the shared pipeline this field flows through | ⬜ | +| **Ankeet** | `ai/explain.py` reads finding text when building an explanation, and a sentence using one word while the field uses another is exactly the kind of mismatch that module is built to not repeat | ⬜ | +| **Shubham** | Owns `analysis/change_impact.py`, where `_RULE_DIRECTION` and the `"Direction: tightening"` wording live | ⬜ | +| **Samika** | `opens_access` in `ai/propose.py` already reads the structural field; the dashboard banner is downstream of whichever wording this settles on | ⬜ | + +> **NOT RATIFIED - zero of four, as of 11 November.** The structural field's +> three values are already relied on and are not what this table gates. +> **`"Direction: tightening"` must not be changed to `"Direction: closing"` +> in `analysis/change_impact.py`, and #406's `"Direction: changed"` must not +> be treated as settled, until this table is full.** +> +> **A ⬜ here is not a disagreement.** It is the absence of a recorded answer, +> which is the only state this file is able to distinguish from agreement. diff --git a/docs/user-guide.md b/docs/user-guide.md index 1240645..51e2e7a 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -258,7 +258,8 @@ Netwise will: 2. Generate a candidate config line. 3. Apply it to a **disposable copy** of your network model and re-analyse. 4. Report what actually changed — and **warn** if the change opens access - that was previously blocked. + that was previously blocked, or closes access your policy says must + stay open. **Nothing is ever applied to your device.** The generated line is text for you to review. Your uploaded config is never modified; a test asserts it is diff --git a/tests/js/propose_render_harness.js b/tests/js/propose_render_harness.js index 24e12f0..822682d 100644 --- a/tests/js/propose_render_harness.js +++ b/tests/js/propose_render_harness.js @@ -338,6 +338,7 @@ const WARNING = { impact: [finding({})], verified: true, warning: true, + opens_access: true, grounded: true, answer: "Generated: permit tcp host 10.10.10.5 host 10.20.0.5 eq 443, on rtr-us5's " + @@ -345,6 +346,39 @@ const WARNING = { "access that was previously blocked -- see the impact list.", }; +// #405 review (Arsh + Samika): a PROVED high-severity CLOSING -- a policy +// requirement breached, which blocks traffic that should stay open. `warning` +// is true for the same reason as WARNING above (a serious effect was proved), +// but `opens_access` is false: the banner must not claim this opens anything. +const WARNING_CLOSING = { + request_understood: "On rtr-us5, add to 'acl_in' (at the top): deny udp ...", + proposed_change: { + device: "rtr-us5", + filter: "acl_in", + line: "deny udp any host 218.8.104.58 eq domain", + }, + impact: [ + finding({ + severity: "high", + summary: "This change BREAKS policy rule 4", + direction: "closing", + evidence: { + detail: "requirement on rtr-us5 filter acl_in: held before, breached after.", + source: "the policy file you supplied; rule 4", + }, + }), + ], + verified: true, + warning: true, + opens_access: false, + grounded: true, + answer: + "Generated: deny udp any host 218.8.104.58 eq domain, on rtr-us5's " + + "'acl_in' filter. Warning: simulating this change has a serious effect " + + "that is not simply opening access -- see the impact list for exactly " + + "what changed.", +}; + // #183's case: a PROVED opening beside an UNRELATED error. Both must show. const WARNING_AND_UNVERIFIED = { request_understood: "On rtr-us5, add to 'acl_in' (at the top): permit tcp ...", @@ -361,6 +395,7 @@ const WARNING_AND_UNVERIFIED = { ], verified: false, warning: true, + opens_access: true, grounded: true, answer: "Generated: ... Warning: ... Some of the impact analysis also could not run.", }; @@ -390,11 +425,12 @@ const MIXED_IMPACT = { impact: [ finding({ id: "CH-003", severity: "medium", status: "found" }), finding({ id: "CH-004", status: "none", severity: "low", summary: "No change" }), - finding({ id: "CH-001", severity: "high", status: "found" }), + finding({ id: "CH-001", severity: "high", status: "found", direction: "opening" }), finding({ id: "CH-000", status: "error", summary: "could not run" }), ], verified: false, warning: true, + opens_access: true, grounded: true, answer: "Generated: ...", }; @@ -446,6 +482,20 @@ const GROUNDED_JUNK_WARNING_ONLY = { answer: "Generated: ...", }; +// warning EXACTLY true, only `opens_access` malformed (#405 review). The +// generic "serious effect" wording must render, not the specific "opens +// access" claim a junk flag gives no grounds for, and not a crash. +const WARNING_JUNK_OPENS_ACCESS = { + request_understood: "On rtr-us5, add to 'acl_in' (at the top): deny tcp ...", + proposed_change: CHANGE, + impact: [finding({})], + verified: true, + warning: true, + opens_access: "yes", + grounded: true, + answer: "Generated: ...", +}; + // A device name containing markup, arriving by a path that looks like ours. const SCRIPTED = { request_understood: "On , add to 'acl_in': deny ...", @@ -511,6 +561,7 @@ const out = { clean: renderAndDescribe(CLEAN), narrows: renderAndDescribe(NARROWS), warning: renderAndDescribe(WARNING), + warningClosing: renderAndDescribe(WARNING_CLOSING), warningAndUnverified: renderAndDescribe(WARNING_AND_UNVERIFIED), unverifiedOnly: renderAndDescribe(UNVERIFIED_ONLY), mixedImpact: renderAndDescribe(MIXED_IMPACT), @@ -518,6 +569,7 @@ const out = { junkKeys: renderAndDescribe(JUNK_KEYS), groundedJunkFlags: renderAndDescribe(GROUNDED_JUNK_FLAGS), groundedJunkWarningOnly: renderAndDescribe(GROUNDED_JUNK_WARNING_ONLY), + warningJunkOpensAccess: renderAndDescribe(WARNING_JUNK_OPENS_ACCESS), scripted: renderAndDescribe(SCRIPTED), // The two strengthenings, exercised together: a real request resubmitted diff --git a/tests/test_change_against_policy.py b/tests/test_change_against_policy.py index 71a9a00..6457140 100644 --- a/tests/test_change_against_policy.py +++ b/tests/test_change_against_policy.py @@ -92,15 +92,32 @@ def test_a_rule_with_the_same_verdict_either_side_is_unchanged_not_a_finding(mon ) -@pytest.mark.parametrize("kind,before,after,direction", [ - ("prohibition", False, True, "opening"), # forbidden traffic now permitted - ("prohibition", True, False, "tightening"), - ("requirement", False, True, "tightening"), # required traffic now denied - ("requirement", True, False, "opening"), +@pytest.mark.parametrize("kind,before,after,direction,normalised", [ + ("prohibition", False, True, "opening", "opening"), # forbidden traffic now permitted + ("prohibition", True, False, "tightening", "closing"), + ("requirement", False, True, "tightening", "closing"), # required traffic now denied + ("requirement", True, False, "opening", "opening"), ]) -def test_direction_is_stated_in_this_modules_own_terms(monkeypatch, kind, before, after, direction): +def test_direction_is_stated_in_this_modules_own_terms(monkeypatch, kind, before, after, direction, normalised): found, _, _ = stage(monkeypatch, [rule(1, kind)], {1: (before, after)}) assert f"Direction: {direction}" in found[0]["evidence"]["detail"] + assert found[0]["direction"] == normalised + + +@pytest.mark.parametrize("kind,before,after,normalised", [ + ("prohibition", False, True, "opening"), + ("prohibition", True, False, "closing"), + ("requirement", False, True, "closing"), + ("requirement", True, False, "opening"), +]) +def test_direction_field_is_set_regardless_of_severity(monkeypatch, kind, before, after, normalised): + """#402: severity alone cannot be used to infer direction for a policy + finding (a high-severity REQUIREMENT breach closes access, it does not + open it) -- this is the field a caller should read instead.""" + found, _, _ = stage( + monkeypatch, [rule(1, kind, severity="high")], {1: (before, after)}, + ) + assert found[0]["direction"] == normalised # --- F-4: could-not-ask is never unchanged ---------------------------------- diff --git a/tests/test_change_impact.py b/tests/test_change_impact.py index d6bde60..e882ef3 100644 --- a/tests/test_change_impact.py +++ b/tests/test_change_impact.py @@ -34,6 +34,7 @@ import pytest from analysis import change_impact +from analysis import policy as policy_module def _trace(disposition): @@ -184,6 +185,58 @@ def test_ids_stay_inside_the_ch_prefix(monkeypatch): assert all(f["check"] == "change_impact" for f in results) +# --- the supplied policy actually reaches the real install (#405 review) --- +# +# Every test above drives analyse_change() with _policy_change_findings() +# itself stubbed to ([], [], number), so none of them touch the line that +# installs `policy`. Found in review: changing +# `policy_module.set_active_policy(policy)` to `set_active_policy(None)` +# inside analyse_change() left the WHOLE SUITE green, because nothing +# exercised the real install with a policy that would answer differently +# from the built-in rules if it were ignored. + + +def test_the_supplied_policy_is_what_gets_installed_not_dropped(monkeypatch): + """Drives the real `pc.rules_in_use()` path, not a stub of the stage that + reads it. A policy naming a device and a violation summary the built-in + rules never use, so a silently-ignored `policy` argument is detectable + by its ABSENCE from the result, not just by a call count.""" + monkeypatch.setattr(change_impact, "connect", lambda host="localhost": SimpleNamespace( + set_network=lambda *a, **k: None, set_snapshot=lambda *a, **k: None)) + monkeypatch.setattr(change_impact, "load_snapshot", lambda *a, **k: None) + monkeypatch.setattr(change_impact, "find_parse_problems", lambda bf: []) + monkeypatch.setattr(change_impact, "_filter_change_findings", + lambda bf, number: ([], number)) + monkeypatch.setattr(change_impact, "_reachability_change_findings", + lambda bf, number: ([], number)) + monkeypatch.setattr( + change_impact, "_rule_breached", + lambda bf, snapshot, rule: snapshot == change_impact.AFTER_SNAPSHOT, + ) + + policy = policy_module.load_policy({ + "device": "rtr-supplied-policy-marker", + "policy_compliance": [{ + "number": 1, "kind": "requirement", "node": "rtr-supplied-policy-marker", + "filter": "acl_in", "description": "d", "violation_severity": "high", + "violation_summary": "a rule only the supplied policy carries", + "queries": [{"dstIps": "10.20.0.9"}], + }], + }) + + results = change_impact.analyse_change("before", "after", policy=policy) + + details = " ".join(f["evidence"]["detail"] for f in results) + assert "a rule only the supplied policy carries" in details, ( + f"the supplied policy's own rule never reached the comparison -- " + f"analyse_change() used some other rule set instead: {details!r}" + ) + assert policy_module.active_policy() is None, ( + "set_active_policy() must be cleared on the way out even when the " + "stage succeeds, not just on the error paths" + ) + + def test_the_error_sentinel_cannot_collide_with_a_real_finding(): """CH-050 mirrors PC-050 -- SENTINEL_NUMBER + ERROR_NUMBER_OFFSET (#101). @@ -238,6 +291,7 @@ def test_reachability_findings_rate_an_opening_high(): assert len(results) == 1 assert results[0]["severity"] == "high" + assert results[0]["direction"] == "opening" assert "could not reach before" in results[0]["summary"] @@ -250,9 +304,59 @@ def test_reachability_findings_rate_a_tightening_medium(): assert results[0]["severity"] == "medium", ( "a change that blocks traffic is not a new exposure" ) + assert results[0]["direction"] == "closing" assert "no longer reaches" in results[0]["summary"] +def test_reachability_findings_say_changed_when_the_direction_is_unknown(): + """#405 review (Arsh): neither side counts as having got through, so this + is not an opening or a closing -- opens_access must not treat it as one.""" + bf = _FakeSessionWithFlows([_flow_row("NO_ROUTE", "DENIED_IN")]) + results, _ = change_impact._reachability_change_findings(bf, 1) + + assert len(results) == 1 + assert results[0]["severity"] == "high" + assert results[0]["direction"] == "changed" + + +class _FakeSessionWithFilterRows: + def __init__(self, rows): + self.q = SimpleNamespace(compareFilters=lambda **kw: _FakeQuestion(rows)) + + +def _filter_row(line_action): + return { + "Node": "rtr-us5", + "Filter_Name": "acl_in", + "Line_Action": line_action, + "Line_Content": f"{line_action.lower()} tcp any any eq 443", + "Line_Index": 0, + "Reference_Line_Content": "deny tcp any any eq 443", + "Reference_Line_Index": 0, + } + + +def test_filter_findings_carry_the_direction_opening(): + bf = _FakeSessionWithFilterRows([_filter_row("PERMIT")]) + results, _ = change_impact._filter_change_findings(bf, 1) + + assert len(results) == 1 + assert results[0]["severity"] == "high" + assert results[0]["direction"] == "opening" + assert "opens traffic" in results[0]["summary"] + + +def test_filter_findings_carry_the_direction_closing(): + """The mutation guard: hardcoding 'opening' here fails, not above.""" + bf = _FakeSessionWithFilterRows([_filter_row("DENY")]) + results, _ = change_impact._filter_change_findings(bf, 1) + + assert len(results) == 1 + assert results[0]["severity"] == "medium" + assert results[0]["direction"] == "closing" + assert "blocks traffic" in results[0]["summary"] + + def test_reachability_findings_number_sequentially_and_report_the_next(): bf = _FakeSessionWithFlows([ _flow_row("ACCEPTED", "DENIED_IN"), diff --git a/tests/test_propose.py b/tests/test_propose.py index 6aeb914..d615d18 100644 --- a/tests/test_propose.py +++ b/tests/test_propose.py @@ -356,10 +356,23 @@ def fake_load_snapshot(bf, config_dir, network_name, snapshot_name): def _stub_change_impact(monkeypatch, results): monkeypatch.setattr( propose.change_impact, "analyse_change", - lambda before_dir, after_dir, host="localhost": results, + lambda before_dir, after_dir, host="localhost", policy=None: results, ) +def _stub_change_impact_recording(monkeypatch, results, calls): + """Like _stub_change_impact(), but appends every call's kwargs to + `calls` -- a fake that accepts an argument is not proof it was passed + the real value, only that the signature matches (#402's own lesson: + checked by mutation-testing this file after the first version of the + fix shipped with no test that would have caught dropping `policy` + silently).""" + def fake(before_dir, after_dir, host="localhost", policy=None): + calls.append({"before_dir": before_dir, "after_dir": after_dir, "host": host, "policy": policy}) + return results + monkeypatch.setattr(propose.change_impact, "analyse_change", fake) + + GOOD_REQUEST = "block 10.10.10.5 to 10.20.0.5 on tcp/443 on rtr-us5" @@ -444,12 +457,21 @@ def test_a_device_with_multiple_distinct_acls_is_refused(monkeypatch, tmp_path): assert "acl_lan" in result["answer"] and "acl_wan" in result["answer"] -def _found(check="change_impact", severity="high", number=1): +def _found(check="change_impact", severity="high", number=1, direction="opening"): + """`direction` defaults to "opening" because every existing caller of + this helper is modelling a proven high-severity OPENING (see each + test's own name/docstring) -- #402 added the real field change_impact + now attaches to every finding it makes, and a caller that wants the + other case (a high-severity policy requirement CLOSING access) passes + direction="closing" explicitly, same as test_propose_rendering.py's + WARNING fixture models an opening by its own hand-written `answer`.""" from analysis import findings - return findings.make_finding( + f = findings.make_finding( check=check, severity=severity, device="rtr-us5", summary="s", detail="d", source="s", status="found", number=number, ) + f["direction"] = direction + return f def _none_finding(): @@ -470,6 +492,9 @@ def test_a_well_formed_request_that_opens_something_is_a_warning(monkeypatch, tm assert result["grounded"] is True assert result["verified"] is True assert result["warning"] is True + assert result["opens_access"] is True, ( + "a pure opening must set the fact app.js's banner keys on (#405 review)" + ) # after_config_text is asserted separately below -- its exact wording is # not this test's concern, and hardcoding the full banner here would # make every wording tweak fail a test unrelated to the change it is @@ -496,6 +521,92 @@ def test_a_well_formed_request_that_opens_something_is_a_warning(monkeypatch, tm assert "GENERATED BY NETWISE" in after_config_text assert "NOT APPLIED" in after_config_text assert "Warning" in result["answer"] + assert "newly opens access" in result["answer"] + + +def test_a_high_severity_closing_warns_without_claiming_it_opens_access(monkeypatch, tmp_path): + """#402: a high-severity POLICY finding can be a requirement breached, + which CLOSES access that should stay open -- the opposite of what the + old wording claimed unconditionally. The warning must still fire (an + under-warning here is the dangerous direction), but it must not say + "opens access" about a change that did the reverse.""" + before_dir = _write_snapshot(tmp_path) + _stub_batfish(monkeypatch) + _stub_change_impact(monkeypatch, [_found(severity="high", direction="closing")]) + + result = propose.propose_change(GOOD_REQUEST, before_dir) + + assert result["verified"] is True + assert result["warning"] is True, "under-warning a serious change is the dangerous direction" + assert "Warning" in result["answer"] + assert "opens access" not in result["answer"] + assert result["opens_access"] is False, ( + "a closing breach must not set the fact app.js's banner would render " + "as 'opens access' (#405 review)" + ) + + +def test_a_mix_of_opening_and_closing_does_not_claim_simple_opening(monkeypatch, tmp_path): + """A mixed impact list (one finding opens, another closes, both high + severity) must not collapse to the single-direction wording -- that + would be true of one finding and asserted about the whole change.""" + before_dir = _write_snapshot(tmp_path) + _stub_batfish(monkeypatch) + _stub_change_impact(monkeypatch, [ + _found(severity="high", direction="opening", number=1), + _found(severity="high", direction="closing", number=2), + ]) + + result = propose.propose_change(GOOD_REQUEST, before_dir) + + assert result["warning"] is True + assert "opens access" not in result["answer"] + assert result["opens_access"] is False, ( + "mixed directions must not claim a simple opening either (#405 review)" + ) + + +def test_an_unclassified_high_finding_warns_without_claiming_an_opening(monkeypatch, tmp_path): + """#405 review (Arsh): a "changed" direction is neither an opening nor a + closing -- opens_access's all() must not treat it as one by default.""" + before_dir = _write_snapshot(tmp_path) + _stub_batfish(monkeypatch) + _stub_change_impact(monkeypatch, [_found(severity="high", direction="changed")]) + + result = propose.propose_change(GOOD_REQUEST, before_dir) + + assert result["warning"] is True + assert result["opens_access"] is False + + +def test_the_caller_supplied_policy_actually_reaches_analyse_change(monkeypatch, tmp_path): + """#402's own bug, pinned directly: a stub that merely ACCEPTS a + `policy` keyword is not proof propose_change() passes its real value + through -- confirmed by mutation-testing this exact line and finding + every other test in this file still passed with the argument dropped.""" + before_dir = _write_snapshot(tmp_path) + _stub_batfish(monkeypatch) + calls = [] + sentinel_policy = object() + _stub_change_impact_recording(monkeypatch, [_found(severity="high")], calls) + + propose.propose_change(GOOD_REQUEST, before_dir, policy=sentinel_policy) + + assert len(calls) == 1 + assert calls[0]["policy"] is sentinel_policy + + +def test_omitting_policy_passes_none_through_unchanged(monkeypatch, tmp_path): + """Regression guard: a caller that does not pass policy must behave + exactly as before this feature existed.""" + before_dir = _write_snapshot(tmp_path) + _stub_batfish(monkeypatch) + calls = [] + _stub_change_impact_recording(monkeypatch, [_found(severity="high")], calls) + + propose.propose_change(GOOD_REQUEST, before_dir) + + assert calls[0]["policy"] is None def test_a_request_with_an_any_endpoint_still_generates_the_right_protocol(monkeypatch, tmp_path): @@ -806,6 +917,28 @@ def fake_analyse(config_dir, host="localhost", network_name="netwise", assert captured["policy"] is sentinel_policy +def test_full_scans_own_impact_list_also_gets_the_policy(monkeypatch, tmp_path): + """#405 review (Samika): propose_and_scan() threads `policy` into the + full-scan pipeline above, but its own call to propose_change() used to + call it with no `policy` at all. Nothing this function returns reads + propose_change()'s impact/warning/opens_access today -- it computes its + own findings via the full pipeline instead -- but every caller of + propose_change() should still see the same policy, not a different one + depending on which function reached it.""" + before_dir = _write_snapshot(tmp_path) + _stub_batfish(monkeypatch) + monkeypatch.setattr("analysis.pipeline.analyse", lambda *a, **k: []) + + calls = [] + sentinel_policy = object() + _stub_change_impact_recording(monkeypatch, [_none_finding()], calls) + + propose.propose_and_scan(GOOD_REQUEST, before_dir, policy=sentinel_policy) + + assert len(calls) == 1 + assert calls[0]["policy"] is sentinel_policy + + def test_full_scan_never_mutates_the_original_config_directory(monkeypatch, tmp_path): """Same guarantee propose_change() itself pins -- a caller of the stronger function must get the same untouched-upload promise, not a diff --git a/tests/test_propose_rendering.py b/tests/test_propose_rendering.py index d4ec4fe..a5c6b53 100644 --- a/tests/test_propose_rendering.py +++ b/tests/test_propose_rendering.py @@ -152,6 +152,29 @@ def test_a_proved_opening_renders_a_warning(rendered, case): assert "opens access" in rendered[case]["warningText"] +@needs_node +def test_a_proved_closing_warns_without_claiming_it_opens_access(rendered): + """#405 review (Arsh + Samika): `warning === true` used to always render + as "This change opens access that is currently blocked", even for a + proved high-severity CLOSING (a policy requirement breached). The banner + must still fire -- under-warning a serious change is the dangerous + direction -- but must not claim the specific, false direction.""" + assert rendered["warningClosing"]["warningPresent"] == 1 + assert "opens access" not in rendered["warningClosing"]["warningText"] + assert "serious effect" in rendered["warningClosing"]["warningText"] + + +@needs_node +def test_a_malformed_opens_access_gets_the_generic_wording_not_a_crash(rendered): + """`opens_access` junk, `warning` exactly true. Same reasoning as the + existing malformed-`warning`/`verified` cases: a junk flag gives no + grounds for the specific "opens access" claim, so it must render the + generic wording rather than that claim or a crash.""" + assert rendered["warningJunkOpensAccess"]["warningPresent"] == 1 + assert "opens access" not in rendered["warningJunkOpensAccess"]["warningText"] + assert "serious effect" in rendered["warningJunkOpensAccess"]["warningText"] + + @needs_node def test_the_warning_appears_before_the_answer(rendered): """Order is a safety property, not layout taste. "This change opens @@ -163,7 +186,9 @@ def test_the_warning_appears_before_the_answer(rendered): @needs_node -@pytest.mark.parametrize("case", ["warning", "warningAndUnverified", "mixedImpact"]) +@pytest.mark.parametrize( + "case", ["warning", "warningClosing", "warningAndUnverified", "mixedImpact"] +) def test_a_warned_response_scrolls_its_own_start_into_view(rendered, case): """Being FIRST in the DOM is not the same as being visible. diff --git a/tests/test_web_propose_endpoint.py b/tests/test_web_propose_endpoint.py index c146cb3..a3e466e 100644 --- a/tests/test_web_propose_endpoint.py +++ b/tests/test_web_propose_endpoint.py @@ -54,7 +54,7 @@ def test_a_refusal_from_propose_change_passes_through_unchanged(monkeypatch): parsing logic here.""" monkeypatch.setattr(main, "_uploaded", True) - def fake_propose_change(request, before_dir, host="localhost"): + def fake_propose_change(request, before_dir, host="localhost", policy=None): return { "request_understood": None, "proposed_change": None, @@ -84,9 +84,10 @@ def test_a_successful_proposal_delegates_to_propose_change_with_the_snapshot_dir monkeypatch.setattr(main, "_uploaded", True) captured = {} - def fake_propose_change(request, before_dir, host="localhost"): + def fake_propose_change(request, before_dir, host="localhost", policy=None): captured["request"] = request captured["before_dir"] = before_dir + captured["policy"] = policy return { "request_understood": "On rtr-us5, add to 'acl_in' (at the top): deny tcp host 10.10.10.5 host 10.20.0.5 eq 443", "proposed_change": { @@ -109,6 +110,30 @@ def fake_propose_change(request, before_dir, host="localhost"): assert body["proposed_change"]["device"] == "rtr-us5" assert captured["request"] == GOOD_REQUEST assert captured["before_dir"] == main.SNAPSHOT_DIR + assert captured["policy"] is None, "no policy staged in this test -- must pass through as None" + + +def test_a_staged_policy_reaches_propose_change(monkeypatch): + """#402: propose_change_endpoint() must pass the SAME staged policy + /api/findings already reads, not silently keep comparing against the + built-in example rules.""" + monkeypatch.setattr(main, "_uploaded", True) + sentinel_policy = object() + monkeypatch.setattr(main, "_staged_policy", lambda: sentinel_policy) + captured = {} + + def fake_propose_change(request, before_dir, host="localhost", policy=None): + captured["policy"] = policy + return { + "request_understood": None, "proposed_change": None, "impact": [], + "verified": False, "warning": False, "grounded": False, "answer": "refused", + } + + monkeypatch.setattr(main, "propose_change", fake_propose_change) + response = client.post("/api/propose", json={"request": GOOD_REQUEST}) + + assert response.status_code == 200 + assert captured["policy"] is sentinel_policy def test_impact_findings_with_status_found_get_an_explanation_attached(monkeypatch): @@ -123,7 +148,7 @@ def test_impact_findings_with_status_found_get_an_explanation_attached(monkeypat "evidence": {"detail": "d", "source": "s"}, "status": "found", } - def fake_propose_change(request, before_dir, host="localhost"): + def fake_propose_change(request, before_dir, host="localhost", policy=None): return { "request_understood": "...", "proposed_change": {"device": "rtr-us5", "filter": "acl_in", "line": "..."}, "impact": [dict(finding)], @@ -151,7 +176,7 @@ def test_an_explanation_failure_does_not_break_the_response(monkeypatch): "evidence": {"detail": "d", "source": "s"}, "status": "found", } - def fake_propose_change(request, before_dir, host="localhost"): + def fake_propose_change(request, before_dir, host="localhost", policy=None): return { "request_understood": "...", "proposed_change": {"device": "rtr-us5", "filter": "acl_in", "line": "..."}, "impact": [dict(finding)], @@ -182,7 +207,7 @@ def test_a_none_status_finding_in_impact_is_not_explained(monkeypatch): } calls = [] - def fake_propose_change(request, before_dir, host="localhost"): + def fake_propose_change(request, before_dir, host="localhost", policy=None): return { "request_understood": "...", "proposed_change": {"device": "rtr-us5", "filter": "acl_in", "line": "..."}, "impact": [dict(finding)], diff --git a/web/main.py b/web/main.py index dda5f3f..32d3108 100644 --- a/web/main.py +++ b/web/main.py @@ -2490,7 +2490,8 @@ def propose_change_endpoint(body: ProposeRequest) -> Dict[str, Any]: impact before returning it (US-13, US-14). Always returns ai.propose.propose_change()'s shape (request_understood, - proposed_change, impact, verified, warning, grounded, answer), whether + proposed_change, impact, verified, warning, opens_access, grounded, + answer), whether the request was understood or refused. Never a 500 for an operational failure -- no upload yet, an unreachable Batfish, or a request that does not resolve are all refusals in the same shape, the same @@ -2512,6 +2513,14 @@ def propose_change_endpoint(body: ProposeRequest) -> Dict[str, Any]: reusing _attach_explanations() rather than a second mechanism -- a warning naming the exact ACL lines that changed is more useful read in plain English than as a raw Batfish diff. + + PASSES THE STAGED POLICY (#402). Before this, every proposal was + checked against Netwise's built-in example rules regardless of what + the user had staged via /api/policy, and the resulting finding's own + evidence then said "no policy file was supplied" -- true of this call + specifically, misleading about the app as a whole. `_staged_policy()` + is the same accessor `/api/findings` already uses, so a staged policy + now reads the same way on every surface that checks one. """ if not _is_uploaded(): return { @@ -2520,6 +2529,7 @@ def propose_change_endpoint(body: ProposeRequest) -> Dict[str, Any]: "impact": [], "verified": False, "warning": False, + "opens_access": False, "grounded": False, "answer": ( "Upload a config first, there is nothing to propose a " @@ -2527,7 +2537,7 @@ def propose_change_endpoint(body: ProposeRequest) -> Dict[str, Any]: ), } - result = propose_change(body.request, snapshot_dir()) + result = propose_change(body.request, snapshot_dir(), policy=_staged_policy()) result["impact"] = _attach_explanations(result["impact"]) return result diff --git a/web/static/app.js b/web/static/app.js index 7f124d3..8f76ef8 100644 --- a/web/static/app.js +++ b/web/static/app.js @@ -1815,6 +1815,7 @@ function addSkippedNotes(notes) { * impact: F-1 findings, * verified: bool, * warning: bool, + * opens_access: bool, * grounded: bool, * answer: string } * @@ -1847,11 +1848,12 @@ function addProposeResponse(result, requestText) { const refused = result.grounded !== true; - // THREE INDEPENDENT FACTS, NEVER FOLDED INTO ONE. + // FOUR INDEPENDENT FACTS, NEVER FOLDED INTO ONE. // - // refused nothing ran - // warning === true a high-severity OPENING was proved in the diff - // verified !== true some of the impact analysis could not run + // refused nothing ran + // warning === true a serious (high-severity) effect was proved + // opensAccess === true that effect is specifically an OPENING + // verified !== true some of the impact analysis could not run // // #183 made `verified` and `warning` separate booleans after review found // a proved opening being suppressed by an unrelated error in the same @@ -1863,6 +1865,17 @@ function addProposeResponse(result, requestText) { // silently drop the warning, and must never silently claim verification. const warned = result.warning === true; + // `warning` alone used to be read as "this opens access" everywhere this + // pane rendered it (#405 review, Arsh + Samika). That was only ever true + // while `warning` COULD mean nothing else. ai/propose.py also sets it for + // a high-severity POLICY breach that CLOSES access -- the opposite claim + // -- so this reads the real direction fact the backend now sends rather + // than assuming. `=== true` here too: a missing or malformed flag gets + // the generic wording below, which is true whichever direction the real + // change turns out to be, rather than a specific claim ("opens access") + // that a malformed response gives no grounds for. + const opensAccess = result.opens_access === true; + // A NON-BOOLEAN `warning` IS NOT A "NO". // `=== true` alone would render a malformed flag as silence, and // silence in this pane implicitly claims "this change does not open @@ -1888,8 +1901,11 @@ function addProposeResponse(result, requestText) { el( "div", "propose-warning", - "This change opens access that is currently blocked. Read the " + - "simulated impact below before applying it." + opensAccess + ? "This change opens access that is currently blocked. Read the " + + "simulated impact below before applying it." + : "This change has a serious effect that is not simply opening " + + "access. Read the simulated impact below before applying it." ) ); }