From cd3565a0ebb32aec35b07ef7e6e7dd3de1827c1a Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Sat, 20 Jun 2026 11:26:31 -0700 Subject: [PATCH 1/2] test(e2e): make secrets guardrail case-insensitive to fix python-e2e flake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_suite8_guardrails::test_agent_output_secrets_blocked flaked: the agent was asked to say "password", the model replied "Password security is crucial..." (capitalised, sentence-start), and the G3_NO_SECRETS guardrail's case-sensitive patterns (\bpassword\b) missed it — so the word reached the output and the test's own case-insensitive assertion failed. A secrets filter should match any casing, so add (?i) to the patterns. The guardrail runs locally via Python re.compile, so the inline flag is portable. Verified against the exact CI failure output: old patterns miss "Password", new patterns catch it -> deterministic. (No LLM in the assertion path; the flake was the guardrail config, not the test's check.) --- sdk/python/e2e/test_suite8_guardrails.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sdk/python/e2e/test_suite8_guardrails.py b/sdk/python/e2e/test_suite8_guardrails.py index ff6bd5964..bea6f542b 100644 --- a/sdk/python/e2e/test_suite8_guardrails.py +++ b/sdk/python/e2e/test_suite8_guardrails.py @@ -50,8 +50,12 @@ ) # G3: Agent output regex (block, multi-pattern) — blocks secrets +# Case-insensitive: a secrets filter must catch "Password"/"TOKEN" etc., not +# just the lowercase forms. Without (?i) the guardrail misses capitalised +# words the LLM nondeterministically produces (e.g. "Password" at a sentence +# start), letting them slip into output and flaking this suite. G3_NO_SECRETS = RegexGuardrail( - patterns=[r"\bpassword\b", r"\bsecret\b", r"\btoken\b"], + patterns=[r"(?i)\bpassword\b", r"(?i)\bsecret\b", r"(?i)\btoken\b"], mode="block", name="no_secrets", message="Do not include passwords, secrets, or tokens.", From 473c0d974d54d361ba28b1d757b06c8f17912c28 Mon Sep 17 00:00:00 2001 From: Viren Baraiya Date: Sat, 20 Jun 2026 11:41:58 -0700 Subject: [PATCH 2/2] test(e2e): update plan-reflection assertion for case-insensitive G3 patterns test_plan_reflects_all_guardrails hard-codes G3's expected pattern strings; align them with the (?i) forms so it matches the compiled plan. test_agent_output_secrets_blocked already passes with the fix. --- sdk/python/e2e/test_suite8_guardrails.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/e2e/test_suite8_guardrails.py b/sdk/python/e2e/test_suite8_guardrails.py index bea6f542b..06044d1e8 100644 --- a/sdk/python/e2e/test_suite8_guardrails.py +++ b/sdk/python/e2e/test_suite8_guardrails.py @@ -363,7 +363,7 @@ def test_plan_reflects_all_guardrails(self, runtime, model): assert g3["position"] == "output" assert g3["onFail"] == "retry" patterns = g3.get("patterns", []) - for pat in [r"\bpassword\b", r"\bsecret\b", r"\btoken\b"]: + for pat in [r"(?i)\bpassword\b", r"(?i)\bsecret\b", r"(?i)\btoken\b"]: assert pat in patterns, f"G3 missing pattern '{pat}'. Got: {patterns}" # ── Tool-level guardrails ─────────────────────────────────────