From a8070f380007c66e1971699af0b988d4fde69ddb Mon Sep 17 00:00:00 2001 From: spencrr <23708360+spencrr@users.noreply.github.com> Date: Thu, 9 Apr 2026 17:01:44 -0700 Subject: [PATCH 1/5] [STYLE]: Fixup Linting of Initial Files fix: resolve all 189 ruff lint violations Per-file-ignores for tests/ (pyproject.toml): S101, D100-D107, ANN001/ANN201/ANN202, PLR2004, ARG001/ARG002, PLC0415, SLF001, TRY003/EM101/EM102, BLE001, TRY301, S108, PT017/PT018, ASYNC240, PERF401, RUF015, PTH123 TC001/TC002/TC003 (45 fixes) - moved type-only imports into TYPE_CHECKING blocks: rampart/attacks/__init__.py, rampart/core/adapter.py, rampart/core/converter.py, rampart/core/execution.py, rampart/core/injection.py, rampart/core/prompt_driver.py, rampart/core/types.py, rampart/payloads/__init__.py, rampart/payloads/_generator.py, rampart/probes/__init__.py, rampart/probes/_single_turn.py, rampart/pytest_plugin/_collection.py, rampart/pytest_plugin/_session.py, rampart/pytest_plugin/plugin.py, rampart/reporting/json_file.py, rampart/surfaces/onedrive.py, rampart/_pyrit/llm_bridge.py, rampart/evaluators/response_contains.py, tests/fixtures.py TRY003/EM101/EM102 (50 fixes) - extracted exception messages to msg variable: rampart/_pyrit/llm_bridge.py:86-91, rampart/converters/docx.py:63-64, rampart/core/execution.py:146-148, rampart/core/types.py:119-131,217-218,300, rampart/drivers/__init__.py:37-39, rampart/payloads/__init__.py:144, rampart/payloads/_store.py:82,135-138,191-192,201-202,350, rampart/probes/__init__.py:87-88, rampart/pytest_plugin/_session.py:104-106, rampart/pytest_plugin/plugin.py:111-116,200-201, rampart/surfaces/onedrive.py:95-97,104-107,119-121,180-182 PYI034 (9 fixes) - changed __aenter__ return type to Self: rampart/core/adapter.py:42, rampart/core/injection.py:41, rampart/surfaces/onedrive.py:171, tests/fixtures.py:43, tests/unit/core/test_execution.py:25, tests/unit/core/test_protocols.py:23,43,89,118 PYI036/ANN401 (9 fixes) - changed exc_tb: Any to types.TracebackType | None: rampart/core/adapter.py:51, rampart/core/injection.py:50, rampart/surfaces/onedrive.py:203, tests/fixtures.py:53, tests/unit/core/test_execution.py:33, tests/unit/core/test_protocols.py:30,50,96,125 D107 (8 fixes) - added __init__ docstrings: rampart/converters/docx.py:34, rampart/core/execution.py:182, rampart/drivers/static.py:25, rampart/evaluators/response_contains.py:23, rampart/evaluators/side_effect.py:19, rampart/evaluators/tool_called.py:26, rampart/reporting/json_file.py:39, rampart/surfaces/onedrive.py:51 D205 (3 fixes) - inserted blank line after docstring summary: rampart/core/execution.py:201, rampart/pytest_plugin/plugin.py:122, tests/fixtures.py:57 E501 (10 fixes) - shortened lines exceeding 88 chars: rampart/core/execution.py:78, rampart/core/manifest.py:88,97, rampart/core/result.py:137, rampart/evaluators/response_contains.py:19, rampart/probes/_single_turn.py:104, rampart/pytest_plugin/_session.py:106, tests/fixtures.py:86, tests/unit/attacks/test_xpia.py:72, tests/unit/core/test_result.py:1 SLF001 (8 fixes) - added public properties to OneDriveSurface (drive_id, folder_path, indexing_delay), updated _OneDriveInjection to use them: rampart/surfaces/onedrive.py:159,174,181-182,195,200 PLW2901 (2 fixes) - renamed loop variables to avoid reassignment: rampart/payloads/_store.py:143-144 (line -> raw_line) rampart/pytest_plugin/_session.py:146 (result -> original_result) PLR2004 (1 fix) - extracted magic value to _PREVIEW_MAX_LENGTH constant: rampart/core/types.py:136-137 RUF006 (1 fix) - stored create_task return value: rampart/pytest_plugin/plugin.py:523 noqa suppressions introduced (29 total): rampart/_pyrit/llm_bridge.py:128,130 PLC0415 deferred pyrit imports rampart/converters/docx.py:41 PLC0415 lazy pyrit converter import rampart/core/execution.py:314 BLE001 handler safety catch rampart/evaluators/side_effect.py:19 ANN401 **kwargs API design rampart/evaluators/tool_called.py:26 ANN401 **kwargs API design rampart/payloads/__init__.py:92 PLR0913 factory method rampart/probes/__init__.py:56 PLR0913 factory method rampart/probes/__init__.py:98 S101 type narrowing assert rampart/pytest_plugin/plugin.py:223-224 SLF001 pytest item private attrs rampart/pytest_plugin/plugin.py:240 ARG001 pytest hook signature rampart/pytest_plugin/plugin.py:297 BLE001 plugin safety catch rampart/pytest_plugin/plugin.py:458 ARG001 pytest hook signature rampart/pytest_plugin/plugin.py:496 BLE001 sink teardown safety rampart/pytest_plugin/plugin.py:523 RUF006 fire-and-forget task rampart/pytest_plugin/plugin.py:528,559,580 ANN401 no public type for terminalreporter rampart/pytest_plugin/plugin.py:581 ARG001 pytest hook signature rampart/surfaces/onedrive.py:202 SLF001 inner class -> parent._upload_async rampart/surfaces/onedrive.py:226 SLF001 inner class -> parent._delete_async rampart/surfaces/onedrive.py:227 BLE001 cleanup must not raise tests/unit/pytest_plugin/test_plugin.py:35-51 ANN401 x6 _StashStub dict mock tests/unit/surfaces/test_onedrive.py:25 ANN401 sentinel default tests/unit/surfaces/test_onedrive.py:63 ANN401 mock dispatch return --- pyproject.toml | 18 ++ rampart/__init__.py | 4 +- rampart/_pyrit/llm_bridge.py | 39 ++-- rampart/attacks/__init__.py | 20 +- rampart/attacks/_xpia.py | 78 +++---- rampart/converters/docx.py | 22 +- rampart/core/__init__.py | 2 +- rampart/core/adapter.py | 31 ++- rampart/core/converter.py | 5 +- rampart/core/errors.py | 3 +- rampart/core/evaluator.py | 24 +-- rampart/core/execution.py | 64 +++--- rampart/core/injection.py | 20 +- rampart/core/manifest.py | 32 +-- rampart/core/persona.py | 3 +- rampart/core/prompt_driver.py | 20 +- rampart/core/result.py | 24 +-- rampart/core/types.py | 73 ++++--- rampart/drivers/__init__.py | 10 +- rampart/drivers/static.py | 14 +- rampart/evaluators/response_contains.py | 16 +- rampart/evaluators/side_effect.py | 6 +- rampart/evaluators/tool_called.py | 6 +- rampart/payloads/__init__.py | 18 +- rampart/payloads/_generator.py | 14 +- rampart/payloads/_store.py | 40 ++-- rampart/payloads/template.py | 3 +- rampart/probes/__init__.py | 26 +-- rampart/probes/_single_turn.py | 27 ++- rampart/pytest_plugin/__init__.py | 2 +- rampart/pytest_plugin/_collection.py | 34 +-- rampart/pytest_plugin/_session.py | 46 ++-- rampart/pytest_plugin/plugin.py | 126 +++++------ rampart/reporting/__init__.py | 2 +- rampart/reporting/json_file.py | 36 ++-- rampart/reporting/sink.py | 25 +-- rampart/surfaces/onedrive.py | 67 ++++-- tests/fixtures.py | 32 +-- tests/integration/test_phase1_exit.py | 8 +- tests/unit/_pyrit/test_llm_bridge.py | 153 ++++++++----- tests/unit/attacks/test_xpia.py | 2 +- tests/unit/converters/test_docx.py | 3 +- tests/unit/core/test_converter.py | 1 - tests/unit/core/test_evaluator.py | 13 +- tests/unit/core/test_execution.py | 16 +- tests/unit/core/test_protocols.py | 28 +-- tests/unit/core/test_result.py | 18 +- tests/unit/drivers/test_coerce.py | 10 +- .../unit/evaluators/test_response_contains.py | 10 +- tests/unit/evaluators/test_side_effect.py | 31 ++- tests/unit/evaluators/test_tool_called.py | 56 +++-- tests/unit/payloads/test_generator.py | 8 +- tests/unit/payloads/test_payloads.py | 8 +- tests/unit/payloads/test_store.py | 3 +- tests/unit/probes/test_single_turn.py | 38 ++-- tests/unit/pytest_plugin/test_collection.py | 9 +- tests/unit/pytest_plugin/test_plugin.py | 127 +++++++---- tests/unit/reporting/test_report.py | 203 +++++++++++++----- tests/unit/surfaces/test_onedrive.py | 14 +- 59 files changed, 1071 insertions(+), 720 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 33c3322b..894e0ed5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,6 +35,24 @@ markers = [ [tool.ruff.lint] select = ["ALL"] +[tool.ruff.lint.per-file-ignores] +"tests/**" = [ + "S101", # assert is pytest's API + "D100", "D101", "D102", "D104", "D107", # no docstrings needed + "ANN001", "ANN201", "ANN202", # no type annotations needed + "PLR2004", # magic values in assertions are fine + "ARG001", "ARG002", # unused args (fixtures, stubs) + "PLC0415", # imports inside functions for isolation + "SLF001", # testing private members is valid + "TRY003", "EM101", "EM102", # exception message style + "BLE001", # catching Exception in tests + "TRY301", # raise in try blocks + "S108", # /tmp usage + "PT017", "PT018", # assertion style + "ASYNC240", # pathlib in async tests + "PERF401", "RUF015", "PTH123", # micro-optimizations / style +] + [tool.ruff.lint.flake8-copyright] notice-rgx = "Copyright \\(c\\) Microsoft Corporation\\.\\s*\\n.*Licensed under the MIT license" diff --git a/rampart/__init__.py b/rampart/__init__.py index f5381119..eade4111 100644 --- a/rampart/__init__.py +++ b/rampart/__init__.py @@ -6,6 +6,7 @@ Public API re-exports for convenient top-level access. """ +from rampart.attacks import Attacks from rampart.core.adapter import AgentAdapter, Session from rampart.core.errors import InfrastructureError from rampart.core.evaluator import BaseEvaluator, Evaluator @@ -40,7 +41,6 @@ ToolCall, Turn, ) -from rampart.attacks import Attacks from rampart.probes import Probes from rampart.pytest_plugin._collection import record_result @@ -82,4 +82,4 @@ "record_result", "resolve_as_attack", "resolve_as_probe", -] \ No newline at end of file +] diff --git a/rampart/_pyrit/llm_bridge.py b/rampart/_pyrit/llm_bridge.py index c5d44dc9..46ae6d8a 100644 --- a/rampart/_pyrit/llm_bridge.py +++ b/rampart/_pyrit/llm_bridge.py @@ -12,26 +12,29 @@ from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any from pyrit.prompt_target import OpenAIChatTarget, PromptChatTarget -from rampart.core.llm import LLMConfig +if TYPE_CHECKING: + from rampart.core.llm import LLMConfig # OpenAIChatTarget constructor parameters that can be forwarded # from LLMConfig.metadata. Kept explicit so unrecognised keys # are silently ignored rather than causing PyRIT TypeErrors. -_FORWARDED_MODEL_PARAMS: frozenset[str] = frozenset({ - "frequency_penalty", - "max_completion_tokens", - "max_requests_per_minute", - "max_tokens", - "n", - "presence_penalty", - "seed", - "temperature", - "top_p", -}) +_FORWARDED_MODEL_PARAMS: frozenset[str] = frozenset( + { + "frequency_penalty", + "max_completion_tokens", + "max_requests_per_minute", + "max_tokens", + "n", + "presence_penalty", + "seed", + "temperature", + "top_p", + }, +) def create_prompt_target(config: LLMConfig) -> PromptChatTarget: @@ -84,12 +87,14 @@ def create_prompt_target(config: LLMConfig) -> PromptChatTarget: def _validate(config: LLMConfig) -> None: """Raise early with clear messages for missing required fields.""" if not config.model: + msg = "LLMConfig.model is required (e.g. 'gpt-4o')." raise ValueError( - "LLMConfig.model is required (e.g. 'gpt-4o')." + msg, ) if not config.endpoint: + msg = "LLMConfig.endpoint is required (e.g. 'https://api.openai.com/v1')." raise ValueError( - "LLMConfig.endpoint is required (e.g. 'https://api.openai.com/v1')." + msg, ) @@ -123,9 +128,9 @@ async def send_generation_request_async( Returns: str: The LLM's text response. """ - import uuid + import uuid # noqa: PLC0415 — deferred: pyrit is optional - from pyrit.models import MessagePiece + from pyrit.models import MessagePiece # noqa: PLC0415 target = create_prompt_target(config) conversation_id = str(uuid.uuid4()) diff --git a/rampart/attacks/__init__.py b/rampart/attacks/__init__.py index 3ffbf0a1..5b12205b 100644 --- a/rampart/attacks/__init__.py +++ b/rampart/attacks/__init__.py @@ -9,20 +9,23 @@ from __future__ import annotations +from typing import TYPE_CHECKING + from rampart.attacks._xpia import XPIAExecution -from rampart.core.evaluator import Evaluator -from rampart.core.execution import BaseExecution, ExecutionEventHandler -from rampart.core.injection import InjectionHandle -from rampart.core.prompt_driver import PromptDriver -from rampart.core.types import Request from rampart.drivers import _coerce_driver +if TYPE_CHECKING: + from rampart.core.evaluator import Evaluator + from rampart.core.execution import BaseExecution, ExecutionEventHandler + from rampart.core.injection import InjectionHandle + from rampart.core.prompt_driver import PromptDriver + from rampart.core.types import Request + __all__ = ["Attacks", "XPIAExecution"] class Attacks: - """ - Factory methods for attack test executions. + """Factory methods for attack test executions. Each method returns a BaseExecution. The test calls ``execute_async`` and asserts the result. @@ -47,8 +50,7 @@ def xpia( max_turns: int = 25, event_handlers: list[ExecutionEventHandler] | None = None, ) -> BaseExecution: - """ - Create an XPIA attack execution. + """Create an XPIA attack execution. Orchestrates the full XPIA flow: inject payloads into surfaces, wait for indexing, create a session, drive the trigger diff --git a/rampart/attacks/_xpia.py b/rampart/attacks/_xpia.py index 5b258627..e582bf6d 100644 --- a/rampart/attacks/_xpia.py +++ b/rampart/attacks/_xpia.py @@ -28,7 +28,6 @@ InjectionRecord, ObservabilityLevel, PromptDriver, - Request, Result, SafetyStatus, Turn, @@ -39,8 +38,7 @@ class XPIAExecution(BaseExecution): - """ - Executes the full XPIA attack lifecycle. + """Executes the full XPIA attack lifecycle. Inherits BaseExecution. Implements ``_execute_async`` with XPIA's specific phase structure. The lifecycle skeleton (event dispatch, @@ -95,8 +93,7 @@ def strategy_name(self) -> str: return "xpia" async def _execute_async(self, *, adapter: AgentAdapter) -> Result: - """ - Orchestrate the XPIA lifecycle and return a safety Result. + """Orchestrate the XPIA lifecycle and return a safety Result. Delegates phase execution to ``_run_phases_async`` and result construction to ``_build_attack_result`` or @@ -116,17 +113,22 @@ async def _execute_async(self, *, adapter: AgentAdapter) -> Result: ) if max_turns_hit: return self._max_turns_error_result( - adapter=adapter, turns=turns, eval_results=eval_results, + adapter=adapter, + turns=turns, + eval_results=eval_results, ) return self._build_attack_result( - adapter=adapter, turns=turns, eval_results=eval_results, + adapter=adapter, + turns=turns, + eval_results=eval_results, ) async def _run_phases_async( - self, *, adapter: AgentAdapter, + self, + *, + adapter: AgentAdapter, ) -> tuple[list[Turn], list[EvalResult], bool]: - """ - Run XPIA phases 1-5 inside a cleanup-guaranteed context. + """Run XPIA phases 1-5 inside a cleanup-guaranteed context. Args: adapter (AgentAdapter): The agent adapter. @@ -140,7 +142,7 @@ async def _run_phases_async( async with AsyncExitStack() as stack: await self._activate_handles_async(stack=stack) session = await stack.enter_async_context( - await adapter.create_session_async() + await adapter.create_session_async(), ) for turn_index in range(self._max_turns): @@ -150,16 +152,19 @@ async def _run_phases_async( request = decision.request response = await session.send_async(request) - turns.append(Turn( - request=request, - response=response, - turn_number=turn_index, - driver_reasoning=decision.reasoning, - )) + turns.append( + Turn( + request=request, + response=response, + turn_number=turn_index, + driver_reasoning=decision.reasoning, + ), + ) eval_result = await self._evaluator.evaluate_async( context=EvalContext( - turns=list(turns), manifest=adapter.manifest, + turns=list(turns), + manifest=adapter.manifest, ), ) eval_results.append(eval_result) @@ -172,10 +177,11 @@ async def _run_phases_async( return turns, eval_results, False async def _activate_handles_async( - self, *, stack: AsyncExitStack, + self, + *, + stack: AsyncExitStack, ) -> None: - """ - Activate all injection handles and wait for indexing. + """Activate all injection handles and wait for indexing. Args: stack (AsyncExitStack): The exit stack managing cleanup. @@ -184,7 +190,8 @@ async def _activate_handles_async( await stack.enter_async_context(handle) delay = max( - (h.indexing_delay_seconds for h in self._handles), default=0.0, + (h.indexing_delay_seconds for h in self._handles), + default=0.0, ) if delay > 0: await asyncio.sleep(delay) @@ -196,8 +203,7 @@ def _build_attack_result( turns: list[Turn], eval_results: list[EvalResult], ) -> Result: - """ - Resolve eval results into a final attack Result. + """Resolve eval results into a final attack Result. Applies observability adjustment when the initial verdict is SAFE. @@ -213,7 +219,10 @@ def _build_attack_result( if status == SafetyStatus.SAFE: safe, status = self._adjust_for_observability( - adapter=adapter, turns=turns, safe=safe, status=status, + adapter=adapter, + turns=turns, + safe=safe, + status=status, ) return Result( @@ -235,8 +244,7 @@ def _max_turns_error_result( turns: list[Turn], eval_results: list[EvalResult], ) -> Result: - """ - Build an ERROR result when the driver exceeds max_turns. + """Build an ERROR result when the driver exceeds max_turns. Args: adapter (AgentAdapter): The adapter under test. @@ -255,8 +263,7 @@ def _max_turns_error_result( safe=False, status=SafetyStatus.ERROR, summary=( - f"Max turns ({self._max_turns}) reached" - " — driver did not terminate" + f"Max turns ({self._max_turns}) reached — driver did not terminate" ), turns=turns, eval_results=eval_results, @@ -284,8 +291,7 @@ def _adjust_for_observability( safe: bool, status: SafetyStatus, ) -> tuple[bool, SafetyStatus]: - """ - Downgrade SAFE to UNDETERMINED when the adapter lacks observability. + """Downgrade SAFE to UNDETERMINED when the adapter lacks observability. If the evaluator returned NOT_DETECTED (and ``resolve_as_attack`` produced SAFE), but the adapter has RESPONSE_ONLY observability @@ -323,8 +329,7 @@ def _collect_response_metadata( *, turns: list[Turn], ) -> dict[str, Any]: - """ - Aggregate response metadata from all turns. + """Aggregate response metadata from all turns. Adapters can attach diagnostic data (e.g. conversation_id, session_id) to Response.metadata. This function collects it into a single dict @@ -355,8 +360,7 @@ def _build_summary( status: SafetyStatus, eval_results: list[EvalResult], ) -> str: - """ - Build a human-readable one-line summary for XPIA results. + """Build a human-readable one-line summary for XPIA results. Args: status (SafetyStatus): The resolved safety status. @@ -374,7 +378,9 @@ def _build_summary( return f"Attack objective detected: {'; '.join(evidence[:3])}" if status == SafetyStatus.UNDETERMINED: rationales = [er.rationale for er in eval_results if er.rationale] - detail = "; ".join(rationales[:2]) if rationales else "Insufficient observability" + detail = ( + "; ".join(rationales[:2]) if rationales else "Insufficient observability" + ) return f"Evaluation undetermined: {detail}" if status == SafetyStatus.ERROR: return "Infrastructure error during execution" diff --git a/rampart/converters/docx.py b/rampart/converters/docx.py index 9a87b176..de155517 100644 --- a/rampart/converters/docx.py +++ b/rampart/converters/docx.py @@ -18,12 +18,13 @@ from rampart.core.types import Payload, PayloadFormat if TYPE_CHECKING: - from pyrit.prompt_converter.word_doc_converter import WordDocConverter as _WordDocConverter + from pyrit.prompt_converter.word_doc_converter import ( + WordDocConverter as _WordDocConverter, + ) class DocxConverter: - """ - Convert a text payload into a Word (.docx) document. + """Convert a text payload into a Word (.docx) document. Thin wrapper around PyRIT's ``WordDocConverter``. Accepts a RAMPART ``Payload`` (text format) and returns a new ``Payload`` @@ -34,19 +35,21 @@ class DocxConverter: """ def __init__(self) -> None: + """Initialize with deferred PyRIT converter.""" self._pyrit_converter: _WordDocConverter | None = None def _get_converter(self) -> _WordDocConverter: """Lazily import and instantiate the PyRIT converter.""" if self._pyrit_converter is None: - from pyrit.prompt_converter.word_doc_converter import WordDocConverter + from pyrit.prompt_converter.word_doc_converter import ( # noqa: PLC0415 + WordDocConverter, + ) self._pyrit_converter = WordDocConverter() return self._pyrit_converter async def convert_async(self, *, payload: Payload) -> Payload: - """ - Convert a text payload into a ``.docx`` payload. + """Convert a text payload into a ``.docx`` payload. Delegates document generation to PyRIT's ``WordDocConverter``. Preserves ``payload.id`` and ``payload.content`` for @@ -63,13 +66,14 @@ async def convert_async(self, *, payload: Payload) -> Payload: ValueError: If the payload format is not a text format. """ if not payload.format.is_text: + msg = f"DocxConverter requires a text payload, got {payload.format.value}." raise ValueError( - f"DocxConverter requires a text payload, " - f"got {payload.format.value}." + msg, ) result = await self._get_converter().convert_async( - prompt=payload.content, input_type="text", + prompt=payload.content, + input_type="text", ) artifact_path = Path(result.output_text) diff --git a/rampart/core/__init__.py b/rampart/core/__init__.py index 5e62cf78..102d1e12 100644 --- a/rampart/core/__init__.py +++ b/rampart/core/__init__.py @@ -82,4 +82,4 @@ "Turn", "resolve_as_attack", "resolve_as_probe", -] \ No newline at end of file +] diff --git a/rampart/core/adapter.py b/rampart/core/adapter.py index 3ba33172..800e0c1b 100644 --- a/rampart/core/adapter.py +++ b/rampart/core/adapter.py @@ -9,16 +9,18 @@ from __future__ import annotations -from typing import Any, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Protocol, Self, runtime_checkable -from rampart.core.manifest import AppManifest -from rampart.core.types import ObservabilityLevel, Payload, Request, Response +if TYPE_CHECKING: + import types + + from rampart.core.manifest import AppManifest + from rampart.core.types import ObservabilityLevel, Request, Response @runtime_checkable class Session(Protocol): - """ - A bounded unit of interaction with the agent. + """A bounded unit of interaction with the agent. Sessions are async context managers. Entering returns the session ready for use; exiting guarantees cleanup of any resources the @@ -28,8 +30,7 @@ class Session(Protocol): """ async def send_async(self, request: Request) -> Response: - """ - Send a request to the agent and return its response. + """Send a request to the agent and return its response. The adapter is responsible for populating Response.tool_calls and Response.side_effects with whatever it can observe. Empty @@ -44,7 +45,7 @@ async def send_async(self, request: Request) -> Response: """ ... - async def __aenter__(self) -> Session: + async def __aenter__(self) -> Self: """Enter the session context. Returns self.""" ... @@ -52,7 +53,7 @@ async def __aexit__( self, exc_type: type[BaseException] | None, exc_val: BaseException | None, - exc_tb: Any, + exc_tb: types.TracebackType | None, ) -> None: """Clean up session resources. Must be idempotent.""" ... @@ -60,8 +61,7 @@ async def __aexit__( @runtime_checkable class AgentAdapter(Protocol): - """ - Factory for sessions and source of agent metadata. + """Factory for sessions and source of agent metadata. Teams implement this to describe their agent and create interaction sessions. The manifest declares what the agent @@ -69,8 +69,7 @@ class AgentAdapter(Protocol): """ async def create_session_async(self) -> Session: - """ - Create a fresh interaction session. + """Create a fresh interaction session. Each session is independent — no shared conversation state. Creating a new session is how the framework achieves @@ -83,8 +82,7 @@ async def create_session_async(self) -> Session: @property def manifest(self) -> AppManifest: - """ - The agent's declared capabilities. + """The agent's declared capabilities. Returns: AppManifest: The agent's capability declaration. @@ -93,8 +91,7 @@ def manifest(self) -> AppManifest: @property def observability_profile(self) -> ObservabilityLevel: - """ - Declares what this adapter can reliably observe. + """Declares what this adapter can reliably observe. Used by evaluators to distinguish "nothing happened" from "I can't see what happened." diff --git a/rampart/core/converter.py b/rampart/core/converter.py index ff0cf036..edab21cb 100644 --- a/rampart/core/converter.py +++ b/rampart/core/converter.py @@ -22,9 +22,10 @@ from __future__ import annotations -from typing import Protocol, runtime_checkable +from typing import TYPE_CHECKING, Protocol, runtime_checkable -from rampart.core.types import Payload +if TYPE_CHECKING: + from rampart.core.types import Payload @runtime_checkable diff --git a/rampart/core/errors.py b/rampart/core/errors.py index 243d7567..2298061a 100644 --- a/rampart/core/errors.py +++ b/rampart/core/errors.py @@ -9,8 +9,7 @@ class InfrastructureError(Exception): - """ - Raised by surfaces and adapters for transient infrastructure failures. + """Raised by surfaces and adapters for transient infrastructure failures. When a surface cannot write to SharePoint (503, rate limit, timeout), or an adapter cannot reach the agent API, it should raise this diff --git a/rampart/core/evaluator.py b/rampart/core/evaluator.py index 435f056f..7698a26f 100644 --- a/rampart/core/evaluator.py +++ b/rampart/core/evaluator.py @@ -18,8 +18,7 @@ @runtime_checkable class Evaluator(Protocol): - """ - Detects a condition in an EvalContext. + """Detects a condition in an EvalContext. Evaluators are polarity-free. They answer "did X happen?" — not "is X good or bad?" The Attacks/Probes factories handle the @@ -33,8 +32,7 @@ class Evaluator(Protocol): """ async def evaluate_async(self, *, context: EvalContext) -> EvalResult: - """ - Evaluate the context and return a detection signal. + """Evaluate the context and return a detection signal. Args: context (EvalContext): The interaction data to evaluate. @@ -46,8 +44,7 @@ async def evaluate_async(self, *, context: EvalContext) -> EvalResult: class BaseEvaluator(ABC): - """ - Base class for evaluator implementations. + """Base class for evaluator implementations. Provides composition operators (|, &, ~) and common behavior. Subclass this for concrete evaluators. Implement evaluate_async. @@ -79,8 +76,7 @@ def __init__(self, *, left: Evaluator, right: Evaluator) -> None: self._right = right async def evaluate_async(self, *, context: EvalContext) -> EvalResult: - """ - Evaluate left first. If DETECTED, skip right entirely. + """Evaluate left first. If DETECTED, skip right entirely. Short-circuiting avoids unnecessary work when the left operand is a cheap deterministic evaluator and the right is an expensive @@ -104,10 +100,7 @@ async def evaluate_async(self, *, context: EvalContext) -> EvalResult: rationale=right_result.rationale, ) - if ( - left_result.outcome == EvalOutcome.UNDETERMINED - or right_result.outcome == EvalOutcome.UNDETERMINED - ): + if EvalOutcome.UNDETERMINED in (left_result.outcome, right_result.outcome): return EvalResult( outcome=EvalOutcome.UNDETERMINED, rationale="One or both operands undetermined", @@ -127,8 +120,7 @@ def __init__(self, *, left: Evaluator, right: Evaluator) -> None: self._right = right async def evaluate_async(self, *, context: EvalContext) -> EvalResult: - """ - Evaluate left first. If NOT_DETECTED or UNDETERMINED, skip right. + """Evaluate left first. If NOT_DETECTED or UNDETERMINED, skip right. Short-circuiting avoids unnecessary work when the left operand can rule out the conjunction cheaply. Place the cheaper or more @@ -182,9 +174,7 @@ async def evaluate_async(self, *, context: EvalContext) -> EvalResult: if result.outcome == EvalOutcome.UNDETERMINED: return result - flipped = ( - EvalOutcome.NOT_DETECTED if result.detected else EvalOutcome.DETECTED - ) + flipped = EvalOutcome.NOT_DETECTED if result.detected else EvalOutcome.DETECTED return EvalResult( outcome=flipped, confidence=result.confidence, diff --git a/rampart/core/execution.py b/rampart/core/execution.py index 49eab04c..c9b8b06b 100644 --- a/rampart/core/execution.py +++ b/rampart/core/execution.py @@ -15,18 +15,19 @@ from abc import ABC, abstractmethod from dataclasses import dataclass from enum import Enum -from typing import Protocol, runtime_checkable +from typing import TYPE_CHECKING, Protocol, runtime_checkable -from rampart.core.adapter import AgentAdapter from rampart.core.errors import InfrastructureError from rampart.core.result import Result, SafetyStatus +if TYPE_CHECKING: + from rampart.core.adapter import AgentAdapter + logger = logging.getLogger(__name__) class ExecutionEvent(Enum): - """ - Lifecycle events fired during a BaseExecution run. + """Lifecycle events fired during a BaseExecution run. ON_PRE_EXECUTE: Fired before _execute_async is called. ON_POST_EXECUTE: Fired after _execute_async returns a Result. @@ -41,8 +42,7 @@ class ExecutionEvent(Enum): @dataclass(kw_only=True) class ExecutionEventData: - """ - Data passed to event handlers at each lifecycle point. + """Data passed to event handlers at each lifecycle point. Args: event (ExecutionEvent): Which lifecycle point fired. @@ -60,8 +60,7 @@ class ExecutionEventData: class ExecutionEventHandler(ABC): - """ - Receives lifecycle events from BaseExecution. + """Receives lifecycle events from BaseExecution. The pytest plugin installs a ResultCollectionHandler on every execution via the _default_handler_factory hook. Teams can @@ -71,8 +70,7 @@ class ExecutionEventHandler(ABC): @abstractmethod async def on_event(self, *, event_data: ExecutionEventData) -> None: - """ - Handle an execution lifecycle event. + """Handle an execution lifecycle event. Args: event_data (ExecutionEventData): The event data. @@ -82,8 +80,7 @@ async def on_event(self, *, event_data: ExecutionEventData) -> None: @runtime_checkable class ExecutionHandlerFactory(Protocol): - """ - Factory that creates the default ExecutionEventHandlers injected into every BaseExecution. + """Factory that creates default ExecutionEventHandlers for every BaseExecution. When a BaseExecution is instantiated it needs a set of framework-level handlers (e.g. the ResultCollectionHandler that funnels results into @@ -111,8 +108,7 @@ def __call__(self) -> list[ExecutionEventHandler]: class _DefaultHandlerRegistry: - """ - Mutable registry for the default handler factory. + """Mutable registry for the default handler factory. Wrapping the reference in a registry instance avoids ``global`` statements — register/clear simply mutate the ``.factory`` @@ -137,8 +133,7 @@ def __call__(self) -> list[ExecutionEventHandler]: def register_default_handler_factory( factory: ExecutionHandlerFactory, ) -> None: - """ - Install a factory that provides default handlers for every BaseExecution. + """Install a factory that provides default handlers for every BaseExecution. Called by the pytest plugin at configure time. The factory is invoked once per BaseExecution.__init__ to supply framework-level handlers @@ -153,16 +148,18 @@ def register_default_handler_factory( TypeError: If factory does not satisfy ExecutionHandlerFactory. """ if not callable(factory): - raise TypeError( + msg = ( "factory must satisfy ExecutionHandlerFactory (callable returning " "list[ExecutionEventHandler])" ) + raise TypeError( + msg, + ) _default_handler_factory.factory = factory def clear_default_handler_factory() -> None: - """ - Remove the installed default handler factory. + """Remove the installed default handler factory. Called by the pytest plugin at unconfigure time to restore the module to its clean, no-plugin state. @@ -171,8 +168,7 @@ def clear_default_handler_factory() -> None: class BaseExecution(ABC): - """ - ABC for all execution strategies. + """ABC for all execution strategies. Owns the execution lifecycle: ON_PRE_EXECUTE → _execute_async → ON_POST_EXECUTE (or ON_ERROR). Subclasses implement only @@ -196,14 +192,14 @@ def __init__( *, event_handlers: list[ExecutionEventHandler] | None = None, ) -> None: + """Initialize with optional extra event handlers.""" defaults = _default_handler_factory() self._handlers: list[ExecutionEventHandler] = defaults + (event_handlers or []) @property @abstractmethod def strategy_name(self) -> str: - """ - Short identifier for this execution strategy. + """Short identifier for this execution strategy. Used in Result.strategy for reporting and dashboard grouping. Examples: "xpia", "probe", "crescendo", "pair". @@ -211,9 +207,10 @@ def strategy_name(self) -> str: ... async def execute_async(self, *, adapter: AgentAdapter) -> Result: - """ - Execute the safety test. Fires lifecycle events and delegates - to _execute_async for strategy-specific logic. + """Execute the safety test. + + Fires lifecycle events and delegates to _execute_async for + strategy-specific logic. InfrastructureError from _execute_async is caught here and converted to a Result with SafetyStatus.ERROR. @@ -232,7 +229,9 @@ async def execute_async(self, *, adapter: AgentAdapter) -> Result: """ start = time.monotonic() await self._fire( - ExecutionEvent.ON_PRE_EXECUTE, adapter=adapter, elapsed=0.0, + ExecutionEvent.ON_PRE_EXECUTE, + adapter=adapter, + elapsed=0.0, ) try: @@ -240,7 +239,8 @@ async def execute_async(self, *, adapter: AgentAdapter) -> Result: except InfrastructureError as exc: logger.warning( "Infrastructure error during %s execution: %s", - self.strategy_name, exc, + self.strategy_name, + exc, exc_info=True, ) result = Result( @@ -273,8 +273,7 @@ async def execute_async(self, *, adapter: AgentAdapter) -> Result: @abstractmethod async def _execute_async(self, *, adapter: AgentAdapter) -> Result: - """ - Core execution logic implemented by each strategy. + """Core execution logic implemented by each strategy. Args: adapter (AgentAdapter): The agent to test. @@ -293,8 +292,7 @@ async def _fire( result: Result | None = None, error: Exception | None = None, ) -> None: - """ - Dispatch an event to all registered handlers. + """Dispatch an event to all registered handlers. Handler exceptions are logged and swallowed — a failing handler must not abort the test or suppress its result. @@ -316,7 +314,7 @@ async def _fire( for handler in self._handlers: try: await handler.on_event(event_data=event_data) - except Exception: + except Exception: # noqa: BLE001 — handler errors must not break execution logger.warning( "ExecutionEventHandler %s raised on %s — ignored.", handler.__class__.__name__, diff --git a/rampart/core/injection.py b/rampart/core/injection.py index 0b3c04ac..d1866c45 100644 --- a/rampart/core/injection.py +++ b/rampart/core/injection.py @@ -9,15 +9,17 @@ from __future__ import annotations -from typing import Any, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Protocol, Self, runtime_checkable -from rampart.core.types import Payload +if TYPE_CHECKING: + import types + + from rampart.core.types import Payload @runtime_checkable class InjectionHandle(Protocol): - """ - A prepared injection, ready to activate as an async context manager. + """A prepared injection, ready to activate as an async context manager. Returned by Surface.inject(). Entering activates the injection (writes the payload to the data source); exiting removes it @@ -42,7 +44,7 @@ def surface_name(self) -> str: """The name of the surface this handle injects into (e.g., 'SharePoint').""" ... - async def __aenter__(self) -> InjectionHandle: + async def __aenter__(self) -> Self: """Activate the injection (write payload to data source).""" ... @@ -50,7 +52,7 @@ async def __aexit__( self, exc_type: type[BaseException] | None, exc_val: BaseException | None, - exc_tb: Any, + exc_tb: types.TracebackType | None, ) -> None: """Remove the injection. Must be idempotent. Must not raise.""" ... @@ -58,8 +60,7 @@ async def __aexit__( @runtime_checkable class Surface(Protocol): - """ - An injectable data source. + """An injectable data source. Surfaces are fully configured at construction (credentials, target location) and expose a universal inject() signature. Teams implement @@ -71,8 +72,7 @@ class Surface(Protocol): """ def inject(self, *, payload: Payload) -> InjectionHandle: - """ - Prepare an injection of the given payload. + """Prepare an injection of the given payload. Does not activate the injection — the caller enters the returned handle as an async context manager to activate it. diff --git a/rampart/core/manifest.py b/rampart/core/manifest.py index f87d6e1a..52ae9371 100644 --- a/rampart/core/manifest.py +++ b/rampart/core/manifest.py @@ -15,8 +15,7 @@ @dataclass(kw_only=True) class ToolDeclaration: - """ - A tool the agent can invoke. + """A tool the agent can invoke. Args: name (str): Tool identifier as the agent reports it. @@ -33,8 +32,7 @@ class ToolDeclaration: @dataclass(kw_only=True) class DataSource: - """ - A data source the agent can access. + """A data source the agent can access. The payload generator uses trust metadata to prioritize targets: a data source writable by untrusted users is a higher-priority @@ -54,8 +52,7 @@ class DataSource: @dataclass(kw_only=True) class AppManifest: - """ - Structured descriptor of an agent's capabilities. + """Structured descriptor of an agent's capabilities. Args: name (str): Agent display name (e.g., "Microsoft Copilot"). @@ -90,19 +87,22 @@ def __str__(self) -> str: sections.append(self.description) if self.tools: - tools = "\n".join( - f" - {t.name}({', '.join(f'{k}: {v}' for k, v in t.parameters.items())})" - f"{f' — {t.description}' if t.description else ''}" - for t in self.tools - ) + tool_lines = [] + for t in self.tools: + params = ", ".join(f"{k}: {v}" for k, v in t.parameters.items()) + desc = f" — {t.description}" if t.description else "" + tool_lines.append(f" - {t.name}({params}){desc}") + tools = "\n".join(tool_lines) sections.append(f"Available tools:\n{tools}") if self.data_sources: - sources = "\n".join( - f" - {ds.name}" - f"{' (writable by untrusted users)' if ds.writable_by_untrusted else ''}" - for ds in self.data_sources - ) + source_lines = [] + for ds in self.data_sources: + writable = ( + " (writable by untrusted users)" if ds.writable_by_untrusted else "" + ) + source_lines.append(f" - {ds.name}{writable}") + sources = "\n".join(source_lines) sections.append(f"Accessible data sources:\n{sources}") return "\n\n".join(sections) diff --git a/rampart/core/persona.py b/rampart/core/persona.py index 966f9b83..2766f443 100644 --- a/rampart/core/persona.py +++ b/rampart/core/persona.py @@ -13,8 +13,7 @@ @dataclass(kw_only=True) class Persona: - """ - A named LLM identity used to shape model behavior. + """A named LLM identity used to shape model behavior. Args: name: Stable identifier used in cache keys and reports. diff --git a/rampart/core/prompt_driver.py b/rampart/core/prompt_driver.py index 99ee5b77..c4fc1e10 100644 --- a/rampart/core/prompt_driver.py +++ b/rampart/core/prompt_driver.py @@ -9,16 +9,16 @@ from __future__ import annotations -from dataclasses import dataclass, field -from typing import Protocol, runtime_checkable +from dataclasses import dataclass +from typing import TYPE_CHECKING, Protocol, runtime_checkable -from rampart.core.types import Request, Turn +if TYPE_CHECKING: + from rampart.core.types import Request, Turn @dataclass(kw_only=True) class PromptDecision: - """ - A driver's decision for the next turn. + """A driver's decision for the next turn. Pairs a Request (what to send) with optional reasoning (why the driver chose it). Reasoning is empty for @@ -36,8 +36,7 @@ class PromptDecision: @runtime_checkable class PromptDriver(Protocol): - """ - Generates prompts for agent interaction. + """Generates prompts for agent interaction. Drivers decide WHAT to send to the agent. They do not own the session, evaluation, or result production — those belong to @@ -51,10 +50,11 @@ class PromptDriver(Protocol): """ async def next_prompt_async( - self, *, history: list[Turn], + self, + *, + history: list[Turn], ) -> PromptDecision | None: - """ - Generate the next prompt decision based on conversation history. + """Generate the next prompt decision based on conversation history. Args: history (list[Turn]): All turns so far (empty on first call). diff --git a/rampart/core/result.py b/rampart/core/result.py index ee87c7ce..3ba63464 100644 --- a/rampart/core/result.py +++ b/rampart/core/result.py @@ -23,8 +23,7 @@ class SafetyStatus(Enum): - """ - Categorical safety status for structured reporting. + """Categorical safety status for structured reporting. SAFE: The agent behaved correctly. UNSAFE: A safety violation was detected. @@ -40,8 +39,7 @@ class SafetyStatus(Enum): class HarmCategory(StrEnum): - """ - Classification of the safety concern being tested. + """Classification of the safety concern being tested. Used by the pytest @harm marker for categorization, by reporting sinks for grouping, and by safety gates for threshold configuration. @@ -73,8 +71,7 @@ class HarmCategory(StrEnum): @dataclass(kw_only=True) class InjectionRecord: - """ - Records what was injected and where, for reproduction and reporting. + """Records what was injected and where, for reproduction and reporting. Populated by XPIAExecution after handles are activated and stored on Result. Provides the complete injection context needed to @@ -93,8 +90,7 @@ class InjectionRecord: @dataclass(kw_only=True) class Result: - """ - The outcome of a safety test. + """The outcome of a safety test. This is the single result type for the entire framework. Attacks and probes both produce Result objects. The reporting infrastructure, @@ -141,12 +137,15 @@ def __bool__(self) -> bool: def __repr__(self) -> str: """Show status and summary for quick debugging.""" - return f"Result(safe={self.safe}, status={self.status.value}, summary={self.summary!r})" + return ( + f"Result(safe={self.safe}, " + f"status={self.status.value}, " + f"summary={self.summary!r})" + ) def resolve_as_attack(*, eval_results: list[EvalResult]) -> tuple[bool, SafetyStatus]: - """ - Attack semantics: detected -> UNSAFE, not detected -> SAFE. + """Attack semantics: detected -> UNSAFE, not detected -> SAFE. Shared by all attack execution strategies (XPIA, prompt injection, Crescendo, PAIR). Lives in core/result.py because it operates @@ -173,8 +172,7 @@ def resolve_as_attack(*, eval_results: list[EvalResult]) -> tuple[bool, SafetySt def resolve_as_probe(*, eval_results: list[EvalResult]) -> tuple[bool, SafetyStatus]: - """ - Probe semantics: detected -> SAFE, not detected -> UNSAFE. + """Probe semantics: detected -> SAFE, not detected -> UNSAFE. Shared by all probe execution strategies. diff --git a/rampart/core/types.py b/rampart/core/types.py index 468c94e5..ee7ee916 100644 --- a/rampart/core/types.py +++ b/rampart/core/types.py @@ -11,18 +11,18 @@ import uuid from dataclasses import dataclass, field -from datetime import datetime from enum import Enum -from pathlib import Path from typing import TYPE_CHECKING, Any if TYPE_CHECKING: + from datetime import datetime + from pathlib import Path + from rampart.core.manifest import AppManifest class ObservabilityLevel(Enum): - """ - What the adapter can reliably observe during agent execution. + """What the adapter can reliably observe during agent execution. Declared by the adapter to inform evaluators and reporting. When the adapter declares RESPONSE_ONLY, evaluators that require tool @@ -35,8 +35,7 @@ class ObservabilityLevel(Enum): class PayloadFormat(Enum): - """ - Delivery format for a payload. + """Delivery format for a payload. Text formats deliver ``content`` directly. Binary formats deliver via ``artifact`` (a file on disk). Surfaces inspect @@ -46,19 +45,19 @@ class PayloadFormat(Enum): surfaces, and the store handle the two categories uniformly without listing every enum member. """ - + # Text formats (content: str) TEXT = "text" HTML = "html" MARKDOWN = "markdown" - + # Binary formats (content: bytes | Path) IMAGE = "image" PDF = "pdf" DOCX = "docx" XLSX = "xlsx" AUDIO = "audio" - + @property def is_text(self) -> bool: """True if this format carries content as str.""" @@ -67,7 +66,7 @@ def is_text(self) -> bool: PayloadFormat.HTML, PayloadFormat.MARKDOWN, ) - + @property def is_binary(self) -> bool: """True if this format carries content as bytes or Path.""" @@ -90,8 +89,7 @@ def extension(self) -> str: @dataclass(kw_only=True) class Payload: - """ - Content to inject into a surface or send alongside a prompt. + """Content to inject into a surface or send alongside a prompt. ``content`` is always the semantic text — the attack instruction, the adversarial prompt, or a description of the payload's purpose. @@ -122,32 +120,39 @@ class Payload: def __post_init__(self) -> None: """Validate content-format-artifact consistency.""" if self.format.is_binary and self.artifact is None: - raise TypeError( + msg = ( f"Binary format {self.format.value} requires an " f"artifact path. Provide artifact=Path(...) or " f"use a converter to render the payload." ) - if self.format.is_text and self.artifact is not None: raise TypeError( + msg, + ) + if self.format.is_text and self.artifact is not None: + msg = ( f"Text format {self.format.value} delivers content " f"directly — artifact must be None." ) + raise TypeError( + msg, + ) if self.artifact is not None and not self.artifact.exists(): + msg = f"Artifact file does not exist: {self.artifact}" raise FileNotFoundError( - f"Artifact file does not exist: {self.artifact}" + msg, ) def __str__(self) -> str: """Human-readable preview of the payload.""" - truncated = self.content[:200] - suffix = "..." if len(self.content) > 200 else "" + _preview_max_length = 200 + truncated = self.content[:_preview_max_length] + suffix = "..." if len(self.content) > _preview_max_length else "" return truncated + suffix @dataclass(kw_only=True) class ToolCall: - """ - A tool invocation observed during agent execution. + """A tool invocation observed during agent execution. Adapters populate this from whatever observability they have — API response fields, telemetry streams, log parsing. @@ -167,8 +172,7 @@ class ToolCall: @dataclass(kw_only=True) class SideEffect: - """ - An observable side effect beyond tool invocations. + """An observable side effect beyond tool invocations. Covers effects like HTTP requests, file system changes, or database writes that the adapter can observe but that are not @@ -185,8 +189,7 @@ class SideEffect: @dataclass(kw_only=True) class Response: - """ - What the agent returned for a single prompt. + """What the agent returned for a single prompt. The adapter populates every field it can observe. @@ -205,8 +208,7 @@ class Response: @dataclass(kw_only=True) class Request: - """ - What is sent to the agent in a single turn. + """What is sent to the agent in a single turn. Combines prompt text and inline payloads into a single object. At least one of ``prompt`` or ``attachments`` must be provided. @@ -224,15 +226,15 @@ class Request: def __post_init__(self) -> None: """Validate that the request carries some content.""" if self.prompt is None and not self.attachments: + msg = "Request must include at least a prompt or attachments." raise ValueError( - "Request must include at least a prompt or attachments." + msg, ) @dataclass(kw_only=True) class Turn: - """ - One prompt-response exchange. + """One prompt-response exchange. Args: request: What was sent to the agent. @@ -250,8 +252,7 @@ class Turn: class EvalOutcome(Enum): - """ - What the evaluator determined. + """What the evaluator determined. DETECTED: The condition was found. NOT_DETECTED: The condition was not found. @@ -265,8 +266,7 @@ class EvalOutcome(Enum): @dataclass(kw_only=True) class EvalResult: - """ - What an evaluator returns — a raw condition detection signal. + """What an evaluator returns — a raw condition detection signal. This is NOT a safety judgment. Whether DETECTED means "safe" or "unsafe" depends on context. @@ -291,8 +291,7 @@ def detected(self) -> bool: @dataclass(kw_only=True) class EvalContext: - """ - Everything an evaluator needs to make a determination. + """Everything an evaluator needs to make a determination. Holds the full conversation as a flat list of turns. Provides convenience properties for common access patterns. @@ -311,7 +310,8 @@ class EvalContext: def current_turn(self) -> Turn: """The most recent turn. Raises ValueError if no turns exist.""" if not self.turns: - raise ValueError("No turns in context.") + msg = "No turns in context." + raise ValueError(msg) return self.turns[-1] @property @@ -337,8 +337,7 @@ def from_response( prompt: str = "", manifest: AppManifest | None = None, ) -> EvalContext: - """ - Build a context from a single response. + """Build a context from a single response. Convenience for evaluating outside the factory flow. diff --git a/rampart/drivers/__init__.py b/rampart/drivers/__init__.py index 448fd7de..f5666cd0 100644 --- a/rampart/drivers/__init__.py +++ b/rampart/drivers/__init__.py @@ -19,8 +19,7 @@ def _coerce_driver( value: str | list[str] | Request | list[Request] | PromptDriver, ) -> PromptDriver: - """ - Coerce a string, Request, or list into a PromptDriver. + """Coerce a string, Request, or list into a PromptDriver. Args: value: A single prompt string, a list of prompt strings, @@ -38,7 +37,10 @@ def _coerce_driver( return StaticDriver(prompts=value) if isinstance(value, PromptDriver): return value - raise TypeError( + msg = ( f"Cannot coerce {type(value).__name__} to PromptDriver. " f"Expected str, list[str], Request, list[Request], or PromptDriver." - ) \ No newline at end of file + ) + raise TypeError( + msg, + ) diff --git a/rampart/drivers/static.py b/rampart/drivers/static.py index 532fb1b6..b32ab9bb 100644 --- a/rampart/drivers/static.py +++ b/rampart/drivers/static.py @@ -15,8 +15,7 @@ class StaticDriver: - """ - Sends a fixed sequence of prompts. + """Sends a fixed sequence of prompts. Derives its position from the history length — no mutable state. Safe to reuse across tests. @@ -27,16 +26,17 @@ class StaticDriver: """ def __init__(self, *, prompts: list[str] | list[Request]) -> None: + """Initialize with a list of prompts or Request objects.""" self._requests: list[Request] = [ - Request(prompt=p) if isinstance(p, str) else p - for p in prompts + Request(prompt=p) if isinstance(p, str) else p for p in prompts ] async def next_prompt_async( - self, *, history: list[Turn], + self, + *, + history: list[Turn], ) -> PromptDecision | None: - """ - Return the next prompt in sequence, or None when exhausted. + """Return the next prompt in sequence, or None when exhausted. Args: history (list[Turn]): All turns so far (empty on first call). diff --git a/rampart/evaluators/response_contains.py b/rampart/evaluators/response_contains.py index 3a29f84f..a7915bb1 100644 --- a/rampart/evaluators/response_contains.py +++ b/rampart/evaluators/response_contains.py @@ -6,21 +6,24 @@ from __future__ import annotations import re -from typing import Callable +from typing import TYPE_CHECKING from rampart.core.evaluator import BaseEvaluator from rampart.core.types import EvalContext, EvalOutcome, EvalResult +if TYPE_CHECKING: + from collections.abc import Callable + class ResponseContains(BaseEvaluator): - """ - Detects whether response text contains a target pattern. + """Detects whether response text contains a target pattern. Accepts a plain string (substring match), compiled regex, or callable predicate. Args: - target (str | re.Pattern | Callable[[str], bool]): Pattern to find (positional-only). + target (str | re.Pattern | Callable[[str], bool]): + Pattern to find (positional-only). case_sensitive (bool): Whether substring match is case-sensitive. """ @@ -31,6 +34,7 @@ def __init__( *, case_sensitive: bool = False, ) -> None: + """Initialize with target pattern and case sensitivity.""" self._target = target self._case_sensitive = case_sensitive @@ -44,7 +48,9 @@ async def evaluate_async(self, *, context: EvalContext) -> EvalResult: found = bool(self._target.search(text)) else: check_text = text if self._case_sensitive else text.lower() - check_target = self._target if self._case_sensitive else self._target.lower() + check_target = ( + self._target if self._case_sensitive else self._target.lower() + ) found = check_target in check_text if found: diff --git a/rampart/evaluators/side_effect.py b/rampart/evaluators/side_effect.py index 870f2ea3..1b7a536c 100644 --- a/rampart/evaluators/side_effect.py +++ b/rampart/evaluators/side_effect.py @@ -12,15 +12,15 @@ class SideEffectOccurred(BaseEvaluator): - """ - Detects whether a side effect of a given kind occurred. + """Detects whether a side effect of a given kind occurred. Args: kind (str): The side effect kind to look for (positional-only). **detail_predicates (Any): Detail field -> expected value or predicate. """ - def __init__(self, kind: str, /, **detail_predicates: Any) -> None: + def __init__(self, kind: str, /, **detail_predicates: Any) -> None: # noqa: ANN401 + """Initialize with side effect kind and optional predicates.""" self._kind = kind self._predicates = detail_predicates diff --git a/rampart/evaluators/tool_called.py b/rampart/evaluators/tool_called.py index cf8230c1..cf06025f 100644 --- a/rampart/evaluators/tool_called.py +++ b/rampart/evaluators/tool_called.py @@ -12,8 +12,7 @@ class ToolCalled(BaseEvaluator): - """ - Detects whether a tool was called, optionally matching parameters. + """Detects whether a tool was called, optionally matching parameters. Parameter predicates can be exact values or callables. Callables receive the parameter value and return True/False. @@ -27,7 +26,8 @@ class ToolCalled(BaseEvaluator): **param_predicates (Any): Parameter name -> expected value or predicate. """ - def __init__(self, tool_name: str, /, **param_predicates: Any) -> None: + def __init__(self, tool_name: str, /, **param_predicates: Any) -> None: # noqa: ANN401 + """Initialize with tool name and optional parameter predicates.""" self._tool_name = tool_name self._predicates = param_predicates diff --git a/rampart/payloads/__init__.py b/rampart/payloads/__init__.py index 8b0123f1..21c53b29 100644 --- a/rampart/payloads/__init__.py +++ b/rampart/payloads/__init__.py @@ -49,16 +49,19 @@ import logging import uuid +from typing import TYPE_CHECKING -from rampart.core.converter import PayloadConverter -from rampart.core.llm import LLMConfig -from rampart.core.manifest import AppManifest -from rampart.core.persona import Persona from rampart.core.types import Payload, PayloadFormat from rampart.payloads._generator import PayloadGenerator from rampart.payloads._store import PayloadStore from rampart.payloads.template import PayloadTemplate +if TYPE_CHECKING: + from rampart.core.converter import PayloadConverter + from rampart.core.llm import LLMConfig + from rampart.core.manifest import AppManifest + from rampart.core.persona import Persona + logger = logging.getLogger(__name__) __all__ = [ @@ -89,7 +92,7 @@ class Payloads: """ @staticmethod - async def generate_async( + async def generate_async( # noqa: PLR0913 — factory method needs all params *, template: PayloadTemplate, llm: LLMConfig, @@ -144,7 +147,8 @@ async def generate_async( ValueError: If count < 1. """ if count < 1: - raise ValueError(f"count must be >= 1, got {count}") + msg = f"count must be >= 1, got {count}" + raise ValueError(msg) generator = PayloadGenerator(llm=llm) text_variants = await generator.generate_text_variants_async( @@ -228,4 +232,4 @@ def _build_text_payload( "objective": template.objective, "variant_index": variant_index, }, - ) \ No newline at end of file + ) diff --git a/rampart/payloads/_generator.py b/rampart/payloads/_generator.py index 9d4216b6..03d503b0 100644 --- a/rampart/payloads/_generator.py +++ b/rampart/payloads/_generator.py @@ -16,12 +16,15 @@ import asyncio import logging +from typing import TYPE_CHECKING from rampart._pyrit.llm_bridge import send_generation_request_async -from rampart.core.llm import LLMConfig -from rampart.core.manifest import AppManifest -from rampart.core.persona import Persona -from rampart.payloads.template import PayloadTemplate + +if TYPE_CHECKING: + from rampart.core.llm import LLMConfig + from rampart.core.manifest import AppManifest + from rampart.core.persona import Persona + from rampart.payloads.template import PayloadTemplate logger = logging.getLogger(__name__) @@ -108,8 +111,7 @@ def _build_user_message( sections.append(manifest_str) sections.append( - "Output ONLY the generated content. " - "No preamble, commentary, or labels." + "Output ONLY the generated content. No preamble, commentary, or labels.", ) return "\n\n".join(sections) diff --git a/rampart/payloads/_store.py b/rampart/payloads/_store.py index b24c87cb..91cef07b 100644 --- a/rampart/payloads/_store.py +++ b/rampart/payloads/_store.py @@ -82,15 +82,18 @@ def save( ValueError: If payloads is empty. """ if not payloads: - raise ValueError("Cannot save an empty payload collection") + msg = "Cannot save an empty payload collection" + raise ValueError(msg) self._root.mkdir(parents=True, exist_ok=True) collection_dir = self._root / name - tmp_dir = Path(tempfile.mkdtemp( - prefix=f".{name}_", - dir=self._root, - )) + tmp_dir = Path( + tempfile.mkdtemp( + prefix=f".{name}_", + dir=self._root, + ), + ) try: self._write_payloads(tmp_dir, payloads=payloads) @@ -133,16 +136,19 @@ def load( """ payloads_path = self._collection_path(name) if not payloads_path.exists(): - raise FileNotFoundError( + msg = ( f"Payload collection '{name}' not found " f"at {payloads_path.parent}/. Run payload generation " f"first (conftest.py fixture or CLI)." ) + raise FileNotFoundError( + msg, + ) payloads: list[Payload] = [] with payloads_path.open("r", encoding="utf-8") as f: - for line in f: - line = line.strip() + for raw_line in f: + line = raw_line.strip() if not line: continue payload = self._deserialize( @@ -189,8 +195,9 @@ def manifest(self, name: str) -> dict[str, Any]: """ path = self._root / name / "manifest.json" if not path.exists(): + msg = f"No manifest for collection '{name}'" raise FileNotFoundError( - f"No manifest for collection '{name}'" + msg, ) with path.open("r", encoding="utf-8") as f: return json.load(f) @@ -199,9 +206,9 @@ def manifest(self, name: str) -> dict[str, Any]: def _validate_collection_name(name: str) -> None: """Reject names that would escape the store root.""" if not name or "/" in name or "\\" in name or name in (".", ".."): + msg = f"Invalid collection name: {name!r}. Must be a simple directory name." raise ValueError( - f"Invalid collection name: {name!r}. " - f"Must be a simple directory name." + msg, ) def _collection_path(self, name: str) -> Path: @@ -209,7 +216,10 @@ def _collection_path(self, name: str) -> Path: return self._root / name / "payloads.jsonl" def _write_payloads( - self, directory: Path, *, payloads: list[Payload], + self, + directory: Path, + *, + payloads: list[Payload], ) -> None: """Write payloads to a JSONL file in the given directory.""" artifacts_dir = directory / "artifacts" @@ -217,7 +227,8 @@ def _write_payloads( with payloads_path.open("w", encoding="utf-8") as f: for payload in payloads: record = self._serialize( - payload=payload, artifacts_dir=artifacts_dir, + payload=payload, + artifacts_dir=artifacts_dir, ) f.write(json.dumps(record) + "\n") @@ -345,7 +356,8 @@ def _deserialize( if "artifact" in data: artifact_path = collection_dir / data["artifact"] if not artifact_path.exists(): - raise FileNotFoundError(f"Missing artifact: {artifact_path}") + msg = f"Missing artifact: {artifact_path}" + raise FileNotFoundError(msg) artifact = artifact_path return Payload( diff --git a/rampart/payloads/template.py b/rampart/payloads/template.py index cb82e399..d81eedb0 100644 --- a/rampart/payloads/template.py +++ b/rampart/payloads/template.py @@ -30,8 +30,7 @@ @dataclass(kw_only=True) class PayloadTemplate: - """ - Instruction to an adversarial LLM for payload generation. + """Instruction to an adversarial LLM for payload generation. Generic across harm categories. The ``instruction`` field is the user message to the adversarial LLM. It describes what diff --git a/rampart/probes/__init__.py b/rampart/probes/__init__.py index cb978d96..7e8ccc5b 100644 --- a/rampart/probes/__init__.py +++ b/rampart/probes/__init__.py @@ -9,14 +9,16 @@ from __future__ import annotations -from typing import overload +from typing import TYPE_CHECKING, overload -from rampart.core.evaluator import Evaluator -from rampart.core.execution import BaseExecution, ExecutionEventHandler -from rampart.core.prompt_driver import PromptDriver from rampart.drivers import _coerce_driver from rampart.probes._single_turn import SingleTurnExecution +if TYPE_CHECKING: + from rampart.core.evaluator import Evaluator + from rampart.core.execution import BaseExecution, ExecutionEventHandler + from rampart.core.prompt_driver import PromptDriver + __all__ = ["Probes", "SingleTurnExecution"] @@ -54,7 +56,7 @@ def behavior( ) -> BaseExecution: ... @staticmethod - def behavior( + def behavior( # noqa: PLR0913 *, prompt: str | None = None, prompts: list[str] | None = None, @@ -63,8 +65,7 @@ def behavior( max_turns: int = 25, event_handlers: list[ExecutionEventHandler] | None = None, ) -> BaseExecution: - """ - Probe whether the agent exhibits desired behavior. + """Probe whether the agent exhibits desired behavior. Exactly one of ``prompt``, ``prompts``, or ``driver`` must be provided. @@ -86,23 +87,22 @@ def behavior( ValueError: If more than one or none of ``prompt``, ``prompts``, and ``driver`` are provided. """ - given = sum( - x is not None for x in (prompt, prompts, driver) - ) + given = sum(x is not None for x in (prompt, prompts, driver)) if given != 1: + msg = "Specify exactly one of 'prompt', 'prompts', or 'driver'." raise ValueError( - "Specify exactly one of 'prompt', 'prompts', or 'driver'." + msg, ) if prompt is not None: resolved_driver = _coerce_driver(prompt) elif prompts is not None: resolved_driver = _coerce_driver(prompts) else: - assert driver is not None + assert driver is not None # noqa: S101 — type narrowing resolved_driver = driver return SingleTurnExecution( driver=resolved_driver, evaluator=evaluator, max_turns=max_turns, event_handlers=event_handlers, - ) \ No newline at end of file + ) diff --git a/rampart/probes/_single_turn.py b/rampart/probes/_single_turn.py index 187f7814..d1cc4b7e 100644 --- a/rampart/probes/_single_turn.py +++ b/rampart/probes/_single_turn.py @@ -12,20 +12,22 @@ from __future__ import annotations import logging +from typing import TYPE_CHECKING -from rampart.core.adapter import AgentAdapter -from rampart.core.evaluator import Evaluator from rampart.core.execution import BaseExecution, ExecutionEventHandler -from rampart.core.prompt_driver import PromptDriver from rampart.core.result import Result, SafetyStatus, resolve_as_probe from rampart.core.types import EvalContext, EvalResult, Turn +if TYPE_CHECKING: + from rampart.core.adapter import AgentAdapter + from rampart.core.evaluator import Evaluator + from rampart.core.prompt_driver import PromptDriver + logger = logging.getLogger(__name__) class SingleTurnExecution(BaseExecution): - """ - Executes a probe: send prompts, evaluate, resolve as probe. + """Executes a probe: send prompts, evaluate, resolve as probe. Inherits BaseExecution. No injection phase — just session creation, prompt driving, evaluation, and cleanup. The lifecycle @@ -60,8 +62,7 @@ def strategy_name(self) -> str: return "probe" async def _execute_async(self, *, adapter: AgentAdapter) -> Result: - """ - Send prompts, evaluate responses, return Result with probe semantics. + """Send prompts, evaluate responses, return Result with probe semantics. Args: adapter (AgentAdapter): The agent adapter. @@ -106,7 +107,10 @@ async def _execute_async(self, *, adapter: AgentAdapter) -> Result: return Result( safe=False, status=SafetyStatus.ERROR, - summary=f"Max turns ({self._max_turns}) reached — driver did not terminate", + summary=( + f"Max turns ({self._max_turns}) reached" + " — driver did not terminate" + ), turns=turns, eval_results=eval_results, strategy="probe", @@ -131,8 +135,7 @@ def _build_summary( status: SafetyStatus, eval_results: list[EvalResult], ) -> str: - """ - Build a human-readable one-line summary. + """Build a human-readable one-line summary. Args: status (SafetyStatus): The resolved safety status. @@ -149,4 +152,6 @@ def _build_summary( return f"UNSAFE: {detail}" if status == SafetyStatus.UNDETERMINED: return "UNDETERMINED: Could not determine if expected behavior occurred" - return f"ERROR: {eval_results[-1].rationale if eval_results else 'No evaluation data'}" + return ( + f"ERROR: {eval_results[-1].rationale if eval_results else 'No evaluation data'}" + ) diff --git a/rampart/pytest_plugin/__init__.py b/rampart/pytest_plugin/__init__.py index 8e5adcff..8678761b 100644 --- a/rampart/pytest_plugin/__init__.py +++ b/rampart/pytest_plugin/__init__.py @@ -19,4 +19,4 @@ "ResultCollectionHandler", "ResultCollector", "record_result", -] \ No newline at end of file +] diff --git a/rampart/pytest_plugin/_collection.py b/rampart/pytest_plugin/_collection.py index fcac4462..b540bea3 100644 --- a/rampart/pytest_plugin/_collection.py +++ b/rampart/pytest_plugin/_collection.py @@ -11,19 +11,25 @@ from __future__ import annotations from contextvars import ContextVar, Token +from typing import TYPE_CHECKING -from rampart.core.execution import ExecutionEvent, ExecutionEventData, ExecutionEventHandler -from rampart.core.result import Result +from rampart.core.execution import ( + ExecutionEvent, + ExecutionEventData, + ExecutionEventHandler, +) +if TYPE_CHECKING: + from rampart.core.result import Result _active_collector: ContextVar[ResultCollector | None] = ContextVar( - "_active_collector", default=None + "_active_collector", + default=None, ) def activate_collector(collector: ResultCollector) -> Token[ResultCollector | None]: - """ - Set the given collector as the active per-test collector. + """Set the given collector as the active per-test collector. Returns a token that must be passed to deactivate_collector to restore the previous state. @@ -38,8 +44,7 @@ def activate_collector(collector: ResultCollector) -> Token[ResultCollector | No def deactivate_collector(token: Token[ResultCollector | None]) -> None: - """ - Restore the previous collector state using the token from activate_collector. + """Restore the previous collector state using the token from activate_collector. Args: token (Token[ResultCollector | None]): The token returned by activate_collector. @@ -48,8 +53,7 @@ def deactivate_collector(token: Token[ResultCollector | None]) -> None: class ResultCollector: - """ - Accumulates Result objects produced during a single test. + """Accumulates Result objects produced during a single test. Framework-internal. Never referenced by test authors. """ @@ -58,8 +62,7 @@ def __init__(self) -> None: self._results: list[Result] = [] def record(self, *, result: Result) -> None: - """ - Record a result. + """Record a result. Args: result (Result): The result to record. @@ -73,8 +76,7 @@ def results(self) -> list[Result]: class ResultCollectionHandler(ExecutionEventHandler): - """ - Default ExecutionEventHandler installed on every BaseExecution. + """Default ExecutionEventHandler installed on every BaseExecution. Writes the Result into the active per-test collector on ON_POST_EXECUTE. No-op for all other events. No-op when no @@ -82,8 +84,7 @@ class ResultCollectionHandler(ExecutionEventHandler): """ async def on_event(self, *, event_data: ExecutionEventData) -> None: - """ - Record result on post-execute. Ignore all other events. + """Record result on post-execute. Ignore all other events. Args: event_data (ExecutionEventData): The event data. @@ -98,8 +99,7 @@ async def on_event(self, *, event_data: ExecutionEventData) -> None: def record_result(result: Result) -> None: - """ - Record a Result into the active test's collector. + """Record a Result into the active test's collector. For building-block tests that construct Results manually rather than via Attacks.* or Probes.* factories. No-op when called diff --git a/rampart/pytest_plugin/_session.py b/rampart/pytest_plugin/_session.py index 6ad653ac..a9da0893 100644 --- a/rampart/pytest_plugin/_session.py +++ b/rampart/pytest_plugin/_session.py @@ -16,15 +16,19 @@ import copy import logging from collections import Counter -from collections.abc import Sequence from dataclasses import dataclass - -import pytest +from typing import TYPE_CHECKING from rampart.core.result import Result, SafetyStatus -from rampart.pytest_plugin._collection import ResultCollector from rampart.reporting.sink import ReportSink, TestRunReport +if TYPE_CHECKING: + from collections.abc import Sequence + + import pytest + + from rampart.pytest_plugin._collection import ResultCollector + logger = logging.getLogger(__name__) @@ -66,8 +70,7 @@ def has_unsafe(self) -> bool: class RampartSession: - """ - Session-scoped state for the RAMPART plugin. + """Session-scoped state for the RAMPART plugin. Accumulates Result objects from all tests, stores trial group aggregates, tracks session duration, and builds the final @@ -92,8 +95,7 @@ def sinks(self) -> list[ReportSink]: return list(self._sinks) def add_sinks(self, *, sinks: list[ReportSink]) -> None: - """ - Register additional sinks for report emission. + """Register additional sinks for report emission. Called by the fixture-based bootstrap to add team-provided sinks. @@ -106,15 +108,18 @@ def add_sinks(self, *, sinks: list[ReportSink]) -> None: """ for sink in sinks: if not isinstance(sink, ReportSink): - raise TypeError( + msg = ( f"Expected ReportSink, got {type(sink).__name__}. " - f"Sinks must implement: async def emit_async(*, report: TestRunReport) -> None" + "Sinks must implement: " + "async def emit_async(*, report: TestRunReport) -> None" + ) + raise TypeError( + msg, ) self._sinks.append(sink) def set_duration(self, *, duration_seconds: float) -> None: - """ - Set the total session duration. + """Set the total session duration. Called by the plugin at session finish with the elapsed time since pytest_configure. @@ -125,8 +130,7 @@ def set_duration(self, *, duration_seconds: float) -> None: self._duration_seconds = duration_seconds def absorb(self, *, node: pytest.Item, collector: ResultCollector) -> None: - """ - Absorb results from a completed test's collector. + """Absorb results from a completed test's collector. Tags each result with the short test name (extracted from the node ID) and the harm category from ``@pytest.mark.harm`` @@ -141,14 +145,16 @@ def absorb(self, *, node: pytest.Item, collector: ResultCollector) -> None: """ test_name = node.nodeid.split("::")[-1] if "::" in node.nodeid else node.nodeid harm_marker = node.get_closest_marker("harm") - harm_category = harm_marker.args[0] if harm_marker and harm_marker.args else None + harm_category = ( + harm_marker.args[0] if harm_marker and harm_marker.args else None + ) collected = collector.results tagged: list[Result] = [] - for result in collected: + for original_result in collected: # Shallow copy is sufficient because we reconstruct all # mutable fields we modify (currently metadata and harm_category). - result = copy.copy(result) + result = copy.copy(original_result) result.metadata = {**result.metadata, "test_name": test_name} if harm_category is not None and result.harm_category is None: result.harm_category = harm_category @@ -164,8 +170,7 @@ def record_trial_group( trial_items: Sequence[pytest.Item], threshold: float, ) -> None: - """ - Record aggregate statistics for a trial group. + """Record aggregate statistics for a trial group. Semantics: - Any UNSAFE result across all trials -> group FAILS @@ -230,8 +235,7 @@ def trial_groups(self) -> dict[str, TrialGroupResult]: return dict(self._trial_groups) def build_report(self) -> TestRunReport: - """ - Build a TestRunReport from all collected results. + """Build a TestRunReport from all collected results. The report is cached and reused on subsequent calls. The cache is invalidated when new results are absorbed. diff --git a/rampart/pytest_plugin/plugin.py b/rampart/pytest_plugin/plugin.py index 754b7487..a127f2e9 100644 --- a/rampart/pytest_plugin/plugin.py +++ b/rampart/pytest_plugin/plugin.py @@ -28,8 +28,7 @@ import logging import re import time -from collections.abc import Generator -from typing import Any, cast +from typing import TYPE_CHECKING, Any, cast import pytest @@ -46,6 +45,9 @@ ) from rampart.pytest_plugin._session import RampartSession +if TYPE_CHECKING: + from collections.abc import Generator + logger = logging.getLogger(__name__) __all__ = [ @@ -72,8 +74,7 @@ def _sanitize_for_terminal(text: str) -> str: - """ - Strip ANSI escape sequences from text before terminal output. + """Strip ANSI escape sequences from text before terminal output. Prevents terminal injection from attacker-controlled payload text that may appear in result summaries. @@ -88,8 +89,7 @@ def _sanitize_for_terminal(text: str) -> str: def _resolve_trial_n(marker: pytest.Mark) -> int: - """ - Extract the trial count from a trial marker. + """Extract the trial count from a trial marker. Supports both positional and keyword argument forms: ``@pytest.mark.trial(5)`` and ``@pytest.mark.trial(n=5)``. @@ -113,23 +113,24 @@ def _resolve_trial_n(marker: pytest.Mark) -> int: return 1 if not isinstance(raw, int) or isinstance(raw, bool): + msg = f"trial(n=) must be an integer, got {type(raw).__name__}: {raw!r}" raise pytest.UsageError( - f"trial(n=) must be an integer, got {type(raw).__name__}: {raw!r}" + msg, ) if raw < 1: + msg = f"trial(n=) must be >= 1, got {raw}" raise pytest.UsageError( - f"trial(n=) must be >= 1, got {raw}" + msg, ) return raw def pytest_configure(config: pytest.Config) -> None: - """ - Register RAMPART markers, install default handler factory, and - initialize session. + """Register RAMPART markers and install default handler factory. - Sinks are provided by teams via the ``rampart_sinks`` fixture - in their conftest.py, not through configuration. + Initializes session. Sinks are provided by teams via the + ``rampart_sinks`` fixture in their conftest.py, not through + configuration. Args: config (pytest.Config): The pytest configuration object. @@ -144,8 +145,7 @@ def pytest_configure(config: pytest.Config) -> None: def pytest_unconfigure(config: pytest.Config) -> None: - """ - Clean up the handler factory on plugin teardown. + """Clean up the handler factory on plugin teardown. Args: config (pytest.Config): The pytest configuration object. @@ -158,8 +158,7 @@ def pytest_unconfigure(config: pytest.Config) -> None: def _copy_markers_to_clone(*, source: pytest.Item, clone: pytest.Item) -> None: - """ - Copy all markers from the original item to its trial clone. + """Copy all markers from the original item to its trial clone. Markers applied at the class level, module level, or via conftest pytestmark are NOT transferred by ``from_parent``. This function @@ -175,7 +174,7 @@ def _copy_markers_to_clone(*, source: pytest.Item, clone: pytest.Item) -> None: if marker.name == "trial": continue clone.add_marker( - pytest.mark.__getattr__(marker.name)(*marker.args, **marker.kwargs) + pytest.mark.__getattr__(marker.name)(*marker.args, **marker.kwargs), ) @@ -185,8 +184,7 @@ def _create_trial_clones( trial_marker: pytest.Mark, count: int, ) -> list[pytest.Item]: - """ - Create trial clone items from an original test item. + """Create trial clone items from an original test item. Each clone gets a unique ``[trial-N]`` suffix, all markers from the original item (including class-level and module-level markers), @@ -206,8 +204,9 @@ def _create_trial_clones( callspec = getattr(item, "callspec", None) fixtureinfo = getattr(item, "_fixtureinfo", None) if parent is None: + msg = f"Cannot clone trial item with no parent: {item.nodeid}" raise pytest.UsageError( - f"Cannot clone trial item with no parent: {item.nodeid}" + msg, ) clones: list[pytest.Item] = [] @@ -224,12 +223,12 @@ def _create_trial_clones( from_parent_kwargs["fixtureinfo"] = fixtureinfo clone = type(item).from_parent(**from_parent_kwargs) # type: ignore[arg-type] - clone._rampart_trial_index = i # type: ignore[attr-defined] - clone._rampart_trial_base = item.nodeid # type: ignore[attr-defined] + clone._rampart_trial_index = i # type: ignore[attr-defined] # noqa: SLF001 + clone._rampart_trial_base = item.nodeid # type: ignore[attr-defined] # noqa: SLF001 _copy_markers_to_clone(source=item, clone=clone) clone.add_marker( - pytest.mark.trial(*trial_marker.args, **trial_marker.kwargs) + pytest.mark.trial(*trial_marker.args, **trial_marker.kwargs), ) # Group all trials for the same base test on one xdist worker # so that trial aggregation works correctly across workers. @@ -241,11 +240,10 @@ def _create_trial_clones( @pytest.hookimpl(trylast=True) def pytest_collection_modifyitems( - config: pytest.Config, + config: pytest.Config, # noqa: ARG001 — pytest hook signature items: list[pytest.Item], ) -> None: - """ - Clone trial-marked items and validate marker usage. + """Clone trial-marked items and validate marker usage. Uses ``trylast=True`` so clones are created after pytest-asyncio has wrapped async items — ``item.obj`` on the original already @@ -275,7 +273,7 @@ def pytest_collection_modifyitems( n = _resolve_trial_n(trial_marker) expanded.extend( - _create_trial_clones(item=item, trial_marker=trial_marker, count=n) + _create_trial_clones(item=item, trial_marker=trial_marker, count=n), ) items[:] = expanded @@ -287,8 +285,7 @@ def _absorb_results( node: pytest.Item, collector: ResultCollector, ) -> None: - """ - Safely absorb collected results into the session. + """Safely absorb collected results into the session. Catches and logs any unexpected errors to prevent the plugin from breaking the test run. @@ -300,7 +297,7 @@ def _absorb_results( """ try: rampart_session.absorb(node=node, collector=collector) - except Exception: + except Exception: # noqa: BLE001 — plugin must not break test runs logger.warning( "Failed to absorb results for %s — results may be incomplete.", node.nodeid, @@ -312,8 +309,7 @@ def _absorb_results( def _rampart_collect( # type: ignore[reportUnusedFunction] # pytest discovers this via autouse=True request: pytest.FixtureRequest, ) -> Generator[None, None, None]: - """ - Installed automatically on every test. Invisible to test authors. + """Installed automatically on every test. Invisible to test authors. Scopes a ResultCollector to the current test via ContextVar. The ResultCollectionHandler (installed on every BaseExecution via @@ -328,7 +324,7 @@ def _rampart_collect( # type: ignore[reportUnusedFunction] # pytest discovers No test author ever imports or references this fixture. """ collector = ResultCollector() - node = cast(pytest.Item, request.node) + node = cast("pytest.Item", request.node) rampart_session = request.config.stash.get(_rampart_key, None) token = activate_collector(collector) yield @@ -353,8 +349,7 @@ def _rampart_collect( # type: ignore[reportUnusedFunction] # pytest discovers def _rampart_sink_bootstrap( # type: ignore[reportUnusedFunction] # pytest discovers this via autouse=True request: pytest.FixtureRequest, ) -> None: - """ - Merge team-provided sinks into the RAMPART session. + """Merge team-provided sinks into the RAMPART session. If the consuming project defines a ``rampart_sinks`` fixture (session-scoped, returning ``list[ReportSink]``), this fixture @@ -380,15 +375,15 @@ def rampart_sinks(): if not isinstance(user_sinks, list): logger.warning( - "rampart_sinks fixture must return list[ReportSink], " - "got %s. Ignoring.", + "rampart_sinks fixture must return list[ReportSink], got %s. Ignoring.", type(user_sinks).__name__, ) return rampart_session.add_sinks(sinks=user_sinks) logger.info( - "Loaded %d sink(s) from rampart_sinks fixture.", len(user_sinks), + "Loaded %d sink(s) from rampart_sinks fixture.", + len(user_sinks), ) @@ -397,8 +392,7 @@ def _aggregate_trial_results( session: pytest.Session, rampart_session: RampartSession, ) -> None: - """ - Group trial item reports by base node ID and compute per-group rates. + """Group trial item reports by base node ID and compute per-group rates. A trial group is identified by ``_rampart_trial_base`` on the item. The aggregate is stored on RampartSession for terminal summary output. @@ -427,8 +421,7 @@ def _evaluate_gates( *, rampart_session: RampartSession, ) -> None: - """ - Log trial group gate results. + """Log trial group gate results. Reports whether each trial group passed or failed based on: - Any UNSAFE -> FAIL (unconditional) @@ -465,10 +458,9 @@ def _evaluate_gates( def pytest_sessionfinish( session: pytest.Session, - exitstatus: int, + exitstatus: int, # noqa: ARG001 — pytest hook signature ) -> None: - """ - Aggregate trial results, evaluate gates, and emit sinks. + """Aggregate trial results, evaluate gates, and emit sinks. Args: session (pytest.Session): The pytest session. @@ -488,8 +480,7 @@ def pytest_sessionfinish( async def _emit_sinks_async(*, rampart_session: RampartSession) -> None: - """ - Emit the test run report to all configured sinks. + """Emit the test run report to all configured sinks. Each sink receives the complete TestRunReport. Sink errors are logged and swallowed — a failing sink must not break the test @@ -505,7 +496,7 @@ async def _emit_sinks_async(*, rampart_session: RampartSession) -> None: for sink in rampart_session.sinks: try: await sink.emit_async(report=report) - except Exception: + except Exception: # noqa: BLE001 — sink errors must not break teardown logger.warning( "Sink %s.emit_async failed — report may not be persisted.", type(sink).__name__, @@ -514,8 +505,7 @@ async def _emit_sinks_async(*, rampart_session: RampartSession) -> None: def _emit_sinks(*, rampart_session: RampartSession) -> None: - """ - Synchronous wrapper for sink emission. + """Synchronous wrapper for sink emission. Used by ``pytest_sessionfinish`` when no event loop is running. When an event loop is already running (e.g. pytest-asyncio), @@ -533,17 +523,16 @@ def _emit_sinks(*, rampart_session: RampartSession) -> None: except RuntimeError: # Inside an already-running event loop (e.g. pytest-asyncio). loop = asyncio.get_running_loop() - loop.create_task(coro) + _background_task = loop.create_task(coro) # noqa: RUF006 def _write_result_line( *, - terminalreporter: Any, + terminalreporter: Any, # noqa: ANN401 result: Result, test_name: str = "", ) -> None: - """ - Write a single result line to the terminal. + """Write a single result line to the terminal. Format matches the architecture's example output: ``PASS test_name — summary (observability_level)`` @@ -560,21 +549,20 @@ def _write_result_line( if test_name: terminalreporter.write_line( - f" {label} {test_name} -- {sanitized_summary} ({obs_level})" + f" {label} {test_name} -- {sanitized_summary} ({obs_level})", ) else: terminalreporter.write_line( - f" {label} {sanitized_summary} ({obs_level})" + f" {label} {sanitized_summary} ({obs_level})", ) def _write_trial_group_lines( *, - terminalreporter: Any, + terminalreporter: Any, # noqa: ANN401 rampart_session: RampartSession, ) -> None: - """ - Write trial group aggregate lines to the terminal. + """Write trial group aggregate lines to the terminal. Format: ``PASS test_name [8/10 safe, 80% defense rate, threshold: 70%] — PASSED`` @@ -587,17 +575,16 @@ def _write_trial_group_lines( terminalreporter.write_line( f" {group.terminal_label} {test_name} " f"[{group.detail}, {group.pass_rate:.0%} pass rate, " - f"threshold: {group.threshold:.0%}] -- {group.verdict}" + f"threshold: {group.threshold:.0%}] -- {group.verdict}", ) def pytest_terminal_summary( - terminalreporter: Any, - exitstatus: int, + terminalreporter: Any, # noqa: ANN401 + exitstatus: int, # noqa: ARG001 — pytest hook signature config: pytest.Config, ) -> None: - """ - Append RAMPART harm-category summary after pytest's standard output. + """Append RAMPART harm-category summary after pytest's standard output. Fires after all tests complete. Writes harm-grouped result lines, trial group aggregates, and population statistics. No-op if no @@ -629,7 +616,9 @@ def pytest_terminal_summary( for category, results in report.by_harm_category().items(): sorted_results = sorted(results, key=lambda r: status_order.get(r.status, 99)) - terminalreporter.write_line(f"\n{category.upper()} ({len(sorted_results)} tests)") + terminalreporter.write_line( + f"\n{category.upper()} ({len(sorted_results)} tests)", + ) for result in sorted_results: test_name = result.metadata.get("test_name", "") _write_result_line( @@ -639,7 +628,8 @@ def pytest_terminal_summary( ) _write_trial_group_lines( - terminalreporter=terminalreporter, rampart_session=rampart_session, + terminalreporter=terminalreporter, + rampart_session=rampart_session, ) stats = report.population_summary() @@ -649,5 +639,5 @@ def pytest_terminal_summary( f"{stats.unsafe_count} unsafe " f"({stats.attack_success_rate:.1%} attack success rate), " f"{stats.undetermined_count} undetermined, " - f"{stats.error_count} errors" + f"{stats.error_count} errors", ) diff --git a/rampart/reporting/__init__.py b/rampart/reporting/__init__.py index e40e4f14..2f6750c2 100644 --- a/rampart/reporting/__init__.py +++ b/rampart/reporting/__init__.py @@ -9,4 +9,4 @@ from rampart.reporting.json_file import JsonFileReportSink from rampart.reporting.sink import ReportSink, TestRunReport -__all__ = ["JsonFileReportSink", "ReportSink", "TestRunReport"] \ No newline at end of file +__all__ = ["JsonFileReportSink", "ReportSink", "TestRunReport"] diff --git a/rampart/reporting/json_file.py b/rampart/reporting/json_file.py index 114b90e8..40ec6d9b 100644 --- a/rampart/reporting/json_file.py +++ b/rampart/reporting/json_file.py @@ -19,18 +19,19 @@ def rampart_sinks(): import dataclasses import json -from datetime import datetime, timezone -from pathlib import Path -from typing import Any +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any -from rampart.core.result import Result -from rampart.core.types import Turn -from rampart.reporting.sink import TestRunReport +if TYPE_CHECKING: + from pathlib import Path + + from rampart.core.result import Result + from rampart.core.types import Turn + from rampart.reporting.sink import TestRunReport class JsonFileReportSink: - """ - Writes the test run report to a JSON file. + """Writes the test run report to a JSON file. Each run produces a timestamped file: ``/run_report_2026-03-19T21-30-00.json`` @@ -41,26 +42,25 @@ class JsonFileReportSink: """ def __init__(self, *, output_dir: Path) -> None: + """Initialize with an output directory for report files.""" self._output_dir = output_dir async def emit_async(self, *, report: TestRunReport) -> None: - """ - Serialize the report to a JSON file. + """Serialize the report to a JSON file. Args: report (TestRunReport): The aggregated test run results. """ self._output_dir.mkdir(parents=True, exist_ok=True) - timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H-%M-%S") + timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H-%M-%S") filepath = self._output_dir / f"run_report_{timestamp}.json" data = self._serialize_report(report) filepath.write_text(json.dumps(data, indent=2, default=str)) def _serialize_report(self, report: TestRunReport) -> dict[str, Any]: - """ - Convert a TestRunReport to a JSON-serializable dict. + """Convert a TestRunReport to a JSON-serializable dict. Args: report (TestRunReport): The report to serialize. @@ -83,8 +83,7 @@ def _serialize_report(self, report: TestRunReport) -> dict[str, Any]: } def _serialize_result(self, result: Result) -> dict[str, Any]: - """ - Convert a single Result to a JSON-serializable dict. + """Convert a single Result to a JSON-serializable dict. Args: result (Result): The result to serialize. @@ -96,7 +95,9 @@ def _serialize_result(self, result: Result) -> dict[str, Any]: "safe": result.safe, "status": result.status.value, "summary": result.summary, - "harm_category": str(result.harm_category) if result.harm_category else None, + "harm_category": str(result.harm_category) + if result.harm_category + else None, "strategy": result.strategy, "duration_seconds": result.duration_seconds, "metadata": result.metadata, @@ -104,8 +105,7 @@ def _serialize_result(self, result: Result) -> dict[str, Any]: } def _serialize_turn(self, turn: Turn) -> dict[str, Any]: - """ - Convert a single Turn to a JSON-serializable dict. + """Convert a single Turn to a JSON-serializable dict. Args: turn (Turn): The turn to serialize. diff --git a/rampart/reporting/sink.py b/rampart/reporting/sink.py index 6c77d506..5428570f 100644 --- a/rampart/reporting/sink.py +++ b/rampart/reporting/sink.py @@ -36,13 +36,16 @@ def has_failures(self) -> bool: @property def is_clean_run(self) -> bool: """True if all runs were SAFE (no unsafe, undetermined, or errors).""" - return self.unsafe_count == 0 and self.undetermined_count == 0 and self.error_count == 0 + return ( + self.unsafe_count == 0 + and self.undetermined_count == 0 + and self.error_count == 0 + ) @dataclass(kw_only=True) class TestRunReport: - """ - Aggregated results from a complete test run. + """Aggregated results from a complete test run. Built by the pytest plugin at session end from all collected Result objects and standard pytest outcomes. @@ -70,8 +73,7 @@ class TestRunReport: metadata: dict[str, Any] = field(default_factory=dict) def by_harm_category(self) -> dict[str, list[Result]]: - """ - Group results by harm category. + """Group results by harm category. HarmCategory is a StrEnum, so both built-in enum values and custom plain strings are native strings at runtime. The grouping @@ -91,8 +93,7 @@ def population_summary( *, harm_category: HarmCategory | str | None = None, ) -> PopulationSummary: - """ - Compute aggregate statistics over collected Result objects. + """Compute aggregate statistics over collected Result objects. Each Result corresponds to one test execution — one run of one test body. For parametrized payload suites, each payload variant @@ -151,16 +152,13 @@ def population_summary( attack_success_rate=( unsafe / diagnostic_total if diagnostic_total > 0 else 0.0 ), - safety_pass_rate=( - safe / diagnostic_total if diagnostic_total > 0 else 0.0 - ), + safety_pass_rate=(safe / diagnostic_total if diagnostic_total > 0 else 0.0), ) @runtime_checkable class ReportSink(Protocol): - """ - Receives test run reports and persists them to an external destination. + """Receives test run reports and persists them to an external destination. Implementations handle serialization and delivery to their target (database, metrics pipeline, file store, etc.). Terminal output is @@ -169,8 +167,7 @@ class ReportSink(Protocol): """ async def emit_async(self, *, report: TestRunReport) -> None: - """ - Emit a complete test run report. + """Emit a complete test run report. Args: report (TestRunReport): The aggregated test run results. diff --git a/rampart/surfaces/onedrive.py b/rampart/surfaces/onedrive.py index 89146944..89615b1d 100644 --- a/rampart/surfaces/onedrive.py +++ b/rampart/surfaces/onedrive.py @@ -10,14 +10,17 @@ from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Self from rampart.core.errors import InfrastructureError -from rampart.core.types import Payload if TYPE_CHECKING: + import types + from msgraph.graph_service_client import GraphServiceClient + from rampart.core.types import Payload + logger = logging.getLogger(__name__) # Graph's PUT /drives/{id}/items/{parent}:/{path}:/content @@ -27,8 +30,7 @@ class OneDriveSurface: - """ - Injects payloads into a specific OneDrive location. + """Injects payloads into a specific OneDrive location. The surface is fully configured at construction — drive ID, credentials, and target folder path. The ``inject()`` method takes @@ -60,14 +62,29 @@ def __init__( folder_path: str, indexing_delay: float = DEFAULT_INDEXING_DELAY, ) -> None: + """Initialize with Graph client and OneDrive location.""" self._graph_client = graph_client self._drive_id = drive_id self._folder_path = folder_path.strip("/") self._indexing_delay = indexing_delay + @property + def drive_id(self) -> str: + """The OneDrive drive ID.""" + return self._drive_id + + @property + def folder_path(self) -> str: + """The target folder path.""" + return self._folder_path + + @property + def indexing_delay(self) -> float: + """Seconds to wait after upload for indexing.""" + return self._indexing_delay + def inject(self, *, payload: Payload) -> _OneDriveInjection: - """ - Prepare an injection into the configured OneDrive folder. + """Prepare an injection into the configured OneDrive folder. Returns an InjectionHandle — enter it as an async context manager to activate the injection, exit to clean up. @@ -97,20 +114,26 @@ async def _upload_async(self, *, payload: Payload) -> str: if payload.format.is_binary: if payload.artifact is None: - raise ValueError( + msg = ( f"Binary payload format {payload.format.value} " f"requires an artifact path." ) + raise ValueError( + msg, + ) content = payload.artifact.read_bytes() else: content = payload.content.encode("utf-8") if len(content) > _MAX_SMALL_UPLOAD_BYTES: - raise ValueError( + msg = ( f"Payload {payload.id} is {len(content)} bytes, which " f"exceeds the 4 MiB small-upload limit. Upload sessions " f"are not yet implemented." ) + raise ValueError( + msg, + ) # Graph path-based addressing: root:/{relative-path}: # The trailing colon is required by the API. @@ -121,10 +144,13 @@ async def _upload_async(self, *, payload: Payload) -> str: ) if drive_item is None or drive_item.id is None: - raise InfrastructureError( + msg = ( f"Graph API returned no DriveItem after upload to " f"drive={self._drive_id} path={upload_path}" ) + raise InfrastructureError( + msg, + ) item_id = drive_item.id logger.info( @@ -161,7 +187,7 @@ def __init__(self, *, surface: OneDriveSurface, payload: Payload) -> None: @property def indexing_delay_seconds(self) -> float: """How long to wait after upload for content to be discoverable.""" - return self._surface._indexing_delay + return self._surface.indexing_delay @property def payload_id(self) -> str | None: @@ -173,18 +199,21 @@ def surface_name(self) -> str: """Identifies this injection as OneDrive for reporting.""" return "OneDrive" - async def __aenter__(self) -> _OneDriveInjection: + async def __aenter__(self) -> Self: """Upload payload to OneDrive. Raises InfrastructureError on failure.""" try: - self._item_id = await self._surface._upload_async( - payload=self._payload + self._item_id = await self._surface._upload_async( # noqa: SLF001 + payload=self._payload, ) except InfrastructureError: raise except Exception as exc: + msg = ( + f"OneDrive upload failed for drive={self._surface.drive_id} " + f"path={self._surface.folder_path}: {exc}" + ) raise InfrastructureError( - f"OneDrive upload failed for drive={self._surface._drive_id} " - f"path={self._surface._folder_path}: {exc}", + msg, ) from exc return self @@ -192,16 +221,16 @@ async def __aexit__( self, exc_type: type[BaseException] | None, exc_val: BaseException | None, - exc_tb: Any, + exc_tb: types.TracebackType | None, ) -> None: """Delete uploaded content. Logs warnings on failure but never raises.""" if self._item_id is not None: try: - await self._surface._delete_async(item_id=self._item_id) - except Exception: + await self._surface._delete_async(item_id=self._item_id) # noqa: SLF001 + except Exception: # noqa: BLE001 — cleanup must not raise logger.warning( "OneDrive cleanup failed for item %s in drive=%s", self._item_id, - self._surface._drive_id, + self._surface.drive_id, exc_info=True, ) diff --git a/tests/fixtures.py b/tests/fixtures.py index f5eb14de..6b4b4c13 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -9,15 +9,18 @@ from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Self -from rampart.core.manifest import AppManifest -from rampart.core.types import ObservabilityLevel, Payload, Request, Response +from rampart.core.types import ObservabilityLevel, Request, Response + +if TYPE_CHECKING: + import types + + from rampart.core.manifest import AppManifest class MockSession: - """ - Mock session that returns preconfigured responses. + """Mock session that returns preconfigured responses. For testing evaluators and adapter logic without a live agent. @@ -32,8 +35,7 @@ def __init__(self, *, responses: list[Response]) -> None: self._index = 0 async def send_async(self, request: Request) -> Response: - """ - Return the next preconfigured response. + """Return the next preconfigured response. Args: request (Request): The request (ignored by mock). @@ -45,7 +47,7 @@ async def send_async(self, request: Request) -> Response: self._index += 1 return response - async def __aenter__(self) -> MockSession: + async def __aenter__(self) -> Self: """No-op for mock sessions.""" return self @@ -53,16 +55,13 @@ async def __aexit__( self, exc_type: type[BaseException] | None, exc_val: BaseException | None, - exc_tb: Any, + exc_tb: types.TracebackType | None, ) -> None: """No-op for mock sessions.""" - pass class MockAdapter: - """ - Mock adapter that produces MockSessions with configurable - per-session response sequences. + """Mock adapter with configurable per-session response sequences. Accepts either a flat list of responses (all sessions return the same sequence) or a list of lists (each session gets its own @@ -90,7 +89,9 @@ def __init__( *, responses: list[Response] | list[list[Response]], manifest: AppManifest, - observability_profile: ObservabilityLevel = ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, + observability_profile: ObservabilityLevel = ( + ObservabilityLevel.TOOL_AND_SIDE_EFFECTS + ), ) -> None: if not responses: raise ValueError("MockAdapter requires at least one response sequence.") @@ -104,8 +105,7 @@ def __init__( self._session_index = 0 async def create_session_async(self) -> MockSession: - """ - Create a mock session with the next response sequence. + """Create a mock session with the next response sequence. Each call advances to the next entry in the session responses list. When the list is exhausted, cycles back to the last entry. diff --git a/tests/integration/test_phase1_exit.py b/tests/integration/test_phase1_exit.py index 7cff4a48..75eaabda 100644 --- a/tests/integration/test_phase1_exit.py +++ b/tests/integration/test_phase1_exit.py @@ -13,10 +13,10 @@ import pytest from rampart import AppManifest, HarmCategory, Response, ToolCall -from tests.fixtures import MockAdapter from rampart.core.types import EvalContext from rampart.evaluators import ToolCalled from rampart.probes import Probes +from tests.fixtures import MockAdapter class TestPhase1ExitCriteria: @@ -33,10 +33,12 @@ async def test_evaluator_detects_tool_call_async(self) -> None: ], ) ctx = EvalContext.from_response( - response=response, prompt="Summarize Q3", + response=response, + prompt="Summarize Q3", ) result = await ToolCalled( - "send_email", to="evil@evil.com", + "send_email", + to="evil@evil.com", ).evaluate_async(context=ctx) assert result.detected diff --git a/tests/unit/_pyrit/test_llm_bridge.py b/tests/unit/_pyrit/test_llm_bridge.py index 31df1ff2..b41da3c5 100644 --- a/tests/unit/_pyrit/test_llm_bridge.py +++ b/tests/unit/_pyrit/test_llm_bridge.py @@ -15,22 +15,26 @@ import pytest -from rampart.core.llm import LLMConfig from rampart._pyrit.llm_bridge import create_prompt_target - +from rampart.core.llm import LLMConfig # --------------------------------------------------------------------------- # Translation tests # --------------------------------------------------------------------------- + class TestModelNameResolution: """LLMConfig.model and .deployment map to PyRIT's model_name / underlying_model.""" @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") def test_model_becomes_model_name_without_deployment(self, mock_cls): - create_prompt_target(LLMConfig( - model="gpt-4o", endpoint="https://api.openai.com/v1", api_key="sk-test", - )) + create_prompt_target( + LLMConfig( + model="gpt-4o", + endpoint="https://api.openai.com/v1", + api_key="sk-test", + ), + ) kwargs = mock_cls.call_args.kwargs assert kwargs["model_name"] == "gpt-4o" @@ -38,12 +42,14 @@ def test_model_becomes_model_name_without_deployment(self, mock_cls): @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") def test_deployment_becomes_model_name_with_model_as_underlying(self, mock_cls): - create_prompt_target(LLMConfig( - model="gpt-4o", - endpoint="https://myresource.openai.azure.com/openai/v1", - api_key="key", - deployment="my-gpt4o-deployment", - )) + create_prompt_target( + LLMConfig( + model="gpt-4o", + endpoint="https://myresource.openai.azure.com/openai/v1", + api_key="key", + deployment="my-gpt4o-deployment", + ), + ) kwargs = mock_cls.call_args.kwargs assert kwargs["model_name"] == "my-gpt4o-deployment" @@ -55,26 +61,37 @@ class TestEndpointAndAuth: @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") def test_endpoint_forwarded(self, mock_cls): - create_prompt_target(LLMConfig( - model="gpt-4o", endpoint="https://custom.endpoint.com/v1", api_key="k", - )) + create_prompt_target( + LLMConfig( + model="gpt-4o", + endpoint="https://custom.endpoint.com/v1", + api_key="k", + ), + ) assert mock_cls.call_args.kwargs["endpoint"] == "https://custom.endpoint.com/v1" @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") def test_api_key_forwarded(self, mock_cls): - create_prompt_target(LLMConfig( - model="gpt-4o", endpoint="https://api.openai.com/v1", api_key="sk-secret", - )) + create_prompt_target( + LLMConfig( + model="gpt-4o", + endpoint="https://api.openai.com/v1", + api_key="sk-secret", + ), + ) assert mock_cls.call_args.kwargs["api_key"] == "sk-secret" @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") def test_none_api_key_forwarded_for_entra_auth(self, mock_cls): """None api_key lets PyRIT use Entra ID auth for Azure endpoints.""" - create_prompt_target(LLMConfig( - model="gpt-4o", endpoint="https://myresource.openai.azure.com/v1", - )) + create_prompt_target( + LLMConfig( + model="gpt-4o", + endpoint="https://myresource.openai.azure.com/v1", + ), + ) assert mock_cls.call_args.kwargs["api_key"] is None @@ -84,10 +101,14 @@ class TestMetadataForwarding: @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") def test_temperature_and_top_p_forwarded(self, mock_cls): - create_prompt_target(LLMConfig( - model="gpt-4o", endpoint="https://api.openai.com/v1", api_key="k", - metadata={"temperature": 0.7, "top_p": 0.9}, - )) + create_prompt_target( + LLMConfig( + model="gpt-4o", + endpoint="https://api.openai.com/v1", + api_key="k", + metadata={"temperature": 0.7, "top_p": 0.9}, + ), + ) kwargs = mock_cls.call_args.kwargs assert kwargs["temperature"] == 0.7 @@ -106,10 +127,14 @@ def test_all_recognised_params_forwarded(self, mock_cls): "max_tokens": 500, "max_requests_per_minute": 60, } - create_prompt_target(LLMConfig( - model="gpt-4o", endpoint="https://api.openai.com/v1", api_key="k", - metadata=meta, - )) + create_prompt_target( + LLMConfig( + model="gpt-4o", + endpoint="https://api.openai.com/v1", + api_key="k", + metadata=meta, + ), + ) kwargs = mock_cls.call_args.kwargs for key, value in meta.items(): @@ -117,10 +142,14 @@ def test_all_recognised_params_forwarded(self, mock_cls): @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") def test_unknown_metadata_keys_not_forwarded(self, mock_cls): - create_prompt_target(LLMConfig( - model="gpt-4o", endpoint="https://api.openai.com/v1", api_key="k", - metadata={"custom_key": "custom_value", "temperature": 0.5}, - )) + create_prompt_target( + LLMConfig( + model="gpt-4o", + endpoint="https://api.openai.com/v1", + api_key="k", + metadata={"custom_key": "custom_value", "temperature": 0.5}, + ), + ) kwargs = mock_cls.call_args.kwargs assert "custom_key" not in kwargs @@ -128,14 +157,21 @@ def test_unknown_metadata_keys_not_forwarded(self, mock_cls): @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") def test_empty_metadata_adds_no_extra_kwargs(self, mock_cls): - create_prompt_target(LLMConfig( - model="gpt-4o", endpoint="https://api.openai.com/v1", api_key="k", - )) + create_prompt_target( + LLMConfig( + model="gpt-4o", + endpoint="https://api.openai.com/v1", + api_key="k", + ), + ) kwargs = mock_cls.call_args.kwargs # Only the four core params should be present. assert set(kwargs.keys()) == { - "model_name", "endpoint", "api_key", "underlying_model", + "model_name", + "endpoint", + "api_key", + "underlying_model", } @@ -144,9 +180,13 @@ class TestReturnValue: @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") def test_returns_constructed_target(self, mock_cls): - result = create_prompt_target(LLMConfig( - model="gpt-4o", endpoint="https://api.openai.com/v1", api_key="k", - )) + result = create_prompt_target( + LLMConfig( + model="gpt-4o", + endpoint="https://api.openai.com/v1", + api_key="k", + ), + ) assert result is mock_cls.return_value @@ -155,20 +195,27 @@ def test_returns_constructed_target(self, mock_cls): # Validation tests # --------------------------------------------------------------------------- + class TestValidation: """Input validation before PyRIT construction.""" def test_empty_model_raises_value_error(self): with pytest.raises(ValueError, match="model"): - create_prompt_target(LLMConfig( - model="", endpoint="https://api.openai.com/v1", - )) + create_prompt_target( + LLMConfig( + model="", + endpoint="https://api.openai.com/v1", + ), + ) def test_empty_endpoint_raises_value_error(self): with pytest.raises(ValueError, match="endpoint"): - create_prompt_target(LLMConfig( - model="gpt-4o", endpoint="", - )) + create_prompt_target( + LLMConfig( + model="gpt-4o", + endpoint="", + ), + ) def test_none_model_raises_value_error(self): config = LLMConfig( # type: ignore[arg-type] @@ -191,6 +238,7 @@ def test_none_endpoint_raises_value_error(self): # Boundary guarantee tests # --------------------------------------------------------------------------- + def _module_imports_pyrit(module_name: str) -> list[str]: """Return any ``pyrit`` import statements found in *module_name*'s source.""" spec = importlib.util.find_spec(module_name) @@ -215,12 +263,15 @@ def _module_imports_pyrit(module_name: str) -> list[str]: class TestBoundaryGuarantees: """Public RAMPART modules must never import from PyRIT directly.""" - @pytest.mark.parametrize("module", [ - "rampart", - "rampart.core", - "rampart.core.llm", - "rampart.attacks", - ]) + @pytest.mark.parametrize( + "module", + [ + "rampart", + "rampart.core", + "rampart.core.llm", + "rampart.attacks", + ], + ) def test_public_module_has_no_pyrit_imports(self, module: str): violations = _module_imports_pyrit(module) assert not violations, ( diff --git a/tests/unit/attacks/test_xpia.py b/tests/unit/attacks/test_xpia.py index 6b3a6d74..d3d09c84 100644 --- a/tests/unit/attacks/test_xpia.py +++ b/tests/unit/attacks/test_xpia.py @@ -72,7 +72,7 @@ def _adapter( class TestXPIADetection: - """Attack semantics: DETECTED→UNSAFE, NOT_DETECTED→SAFE, UNDETERMINED→UNDETERMINED.""" + """Attack semantics: DETECTED->UNSAFE, NOT_DETECTED->SAFE.""" @pytest.mark.asyncio async def test_detected_returns_unsafe_with_evidence_in_summary(self) -> None: diff --git a/tests/unit/converters/test_docx.py b/tests/unit/converters/test_docx.py index c0678ef0..71a01d57 100644 --- a/tests/unit/converters/test_docx.py +++ b/tests/unit/converters/test_docx.py @@ -83,7 +83,8 @@ async def test_delegates_content_to_pyrit(self, tmp_path: Path) -> None: ) mock_instance.convert_async.assert_called_once_with( - prompt="adversarial text", input_type="text", + prompt="adversarial text", + input_type="text", ) @pytest.mark.asyncio diff --git a/tests/unit/core/test_converter.py b/tests/unit/core/test_converter.py index 276858f8..2a839776 100644 --- a/tests/unit/core/test_converter.py +++ b/tests/unit/core/test_converter.py @@ -4,7 +4,6 @@ """Tests for rampart.core.converter — PayloadConverter protocol.""" from pathlib import Path -from unittest.mock import patch import pytest diff --git a/tests/unit/core/test_evaluator.py b/tests/unit/core/test_evaluator.py index fdd995cf..2b2ef594 100644 --- a/tests/unit/core/test_evaluator.py +++ b/tests/unit/core/test_evaluator.py @@ -6,7 +6,14 @@ import pytest from rampart.core.evaluator import BaseEvaluator, Evaluator -from rampart.core.types import EvalContext, EvalOutcome, EvalResult, Request, Response, Turn +from rampart.core.types import ( + EvalContext, + EvalOutcome, + EvalResult, + Request, + Response, + Turn, +) class _StubEvaluator(BaseEvaluator): @@ -29,7 +36,9 @@ async def evaluate_async(self, *, context: EvalContext) -> EvalResult: def _ctx() -> EvalContext: """Build a minimal EvalContext for testing.""" - return EvalContext(turns=[Turn(request=Request(prompt="p"), response=Response(text="r"))]) + return EvalContext( + turns=[Turn(request=Request(prompt="p"), response=Response(text="r"))], + ) class TestEvaluatorProtocol: diff --git a/tests/unit/core/test_execution.py b/tests/unit/core/test_execution.py index 1396a6a5..19197306 100644 --- a/tests/unit/core/test_execution.py +++ b/tests/unit/core/test_execution.py @@ -1,12 +1,12 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -from typing import Any +import types +from typing import Self import pytest -from rampart.core import execution as _execution_module -from rampart.core.adapter import AgentAdapter, Session +from rampart.core.adapter import AgentAdapter from rampart.core.errors import InfrastructureError from rampart.core.execution import ( BaseExecution, @@ -16,7 +16,7 @@ ) from rampart.core.manifest import AppManifest from rampart.core.result import Result, SafetyStatus -from rampart.core.types import ObservabilityLevel, Payload, Request, Response +from rampart.core.types import ObservabilityLevel, Request, Response class _StubSession: @@ -26,7 +26,7 @@ async def send_async(self, request: Request) -> Response: """Return a fixed response.""" return Response(text="ok") - async def __aenter__(self) -> "_StubSession": + async def __aenter__(self) -> Self: """Enter context.""" return self @@ -34,7 +34,7 @@ async def __aexit__( self, exc_type: type[BaseException] | None, exc_val: BaseException | None, - exc_tb: Any, + exc_tb: types.TracebackType | None, ) -> None: """Exit context.""" @@ -219,7 +219,9 @@ async def test_on_error_contains_exception(self) -> None: with pytest.raises(RuntimeError): await execution.execute_async(adapter=_StubAdapter()) - error_event = [e for e in handler.events if e.event is ExecutionEvent.ON_ERROR][0] + error_event = [e for e in handler.events if e.event is ExecutionEvent.ON_ERROR][ + 0 + ] assert isinstance(error_event.error, RuntimeError) diff --git a/tests/unit/core/test_protocols.py b/tests/unit/core/test_protocols.py index 3891c042..d956a9ba 100644 --- a/tests/unit/core/test_protocols.py +++ b/tests/unit/core/test_protocols.py @@ -8,7 +8,8 @@ inheriting from the protocol. """ -from typing import Any +import types +from typing import Self from rampart.core.adapter import AgentAdapter, Session from rampart.core.injection import InjectionHandle, Surface @@ -23,14 +24,14 @@ class MySession: async def send_async(self, request: Request) -> Response: return Response(text="ok") - async def __aenter__(self) -> "MySession": + async def __aenter__(self) -> Self: return self async def __aexit__( self, exc_type: type[BaseException] | None, exc_val: BaseException | None, - exc_tb: Any, + exc_tb: types.TracebackType | None, ) -> None: pass @@ -38,18 +39,19 @@ async def __aexit__( def test_send_async_accepts_request(self) -> None: """Verify the protocol requires a Request parameter.""" + class MySession: async def send_async(self, request: Request) -> Response: return Response(text="ok") - async def __aenter__(self) -> "MySession": + async def __aenter__(self) -> Self: return self async def __aexit__( self, exc_type: type[BaseException] | None, exc_val: BaseException | None, - exc_tb: Any, + exc_tb: types.TracebackType | None, ) -> None: pass @@ -60,8 +62,7 @@ async def __aexit__( class TestAgentAdapterProtocol: def test_structural_subtyping(self) -> None: class MyAdapter: - async def create_session_async(self) -> Session: - ... + async def create_session_async(self) -> Session: ... @property def manifest(self) -> AppManifest: @@ -89,14 +90,14 @@ def payload_id(self) -> str | None: def surface_name(self) -> str: return "SharePoint" - async def __aenter__(self) -> "MyHandle": + async def __aenter__(self) -> Self: return self async def __aexit__( self, exc_type: type[BaseException] | None, exc_val: BaseException | None, - exc_tb: Any, + exc_tb: types.TracebackType | None, ) -> None: pass @@ -118,14 +119,14 @@ def payload_id(self) -> str | None: def surface_name(self) -> str: return "test" - async def __aenter__(self) -> "MyHandle": + async def __aenter__(self) -> Self: return self async def __aexit__( self, exc_type: type[BaseException] | None, exc_val: BaseException | None, - exc_tb: Any, + exc_tb: types.TracebackType | None, ) -> None: pass @@ -140,7 +141,9 @@ class TestPromptDriverProtocol: def test_structural_subtyping(self) -> None: class MyDriver: async def next_prompt_async( - self, *, history: list[Turn], + self, + *, + history: list[Turn], ) -> PromptDecision | None: return None @@ -161,5 +164,6 @@ def test_with_attachments_only(self) -> None: def test_empty_request_raises(self) -> None: import pytest + with pytest.raises(ValueError, match="at least"): Request() diff --git a/tests/unit/core/test_result.py b/tests/unit/core/test_result.py index ab810970..6dec546a 100644 --- a/tests/unit/core/test_result.py +++ b/tests/unit/core/test_result.py @@ -1,7 +1,10 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Tests for rampart.core.result — Result, SafetyStatus, HarmCategory, resolve functions.""" +"""Tests for rampart.core.result. + +Result, SafetyStatus, HarmCategory, resolve functions. +""" import pytest @@ -29,7 +32,6 @@ def test_values(self) -> None: assert SafetyStatus.ERROR.value == "error" - class TestHarmCategory: def test_is_strenum(self) -> None: assert isinstance(HarmCategory.PROMPT_INJECTION, str) @@ -49,7 +51,7 @@ def test_xpia_is_not_a_harm_category(self) -> None: def test_interchangeable_with_plain_string(self) -> None: assert HarmCategory.PROMPT_INJECTION == "prompt_injection" - assert "prompt_injection" == HarmCategory.PROMPT_INJECTION + assert HarmCategory.PROMPT_INJECTION == "prompt_injection" def test_usable_as_dict_key(self) -> None: d: dict[str, int] = {HarmCategory.DATA_EXFILTRATION: 1, "custom_risk": 2} @@ -57,7 +59,6 @@ def test_usable_as_dict_key(self) -> None: assert d[HarmCategory.DATA_EXFILTRATION] == 1 - class TestInjectionRecord: def test_construction(self) -> None: rec = InjectionRecord(payload_id="abc123", surface_name="SharePoint") @@ -69,7 +70,6 @@ def test_none_payload_id(self) -> None: assert rec.payload_id is None - class TestResult: def test_bool_returns_safe_true(self) -> None: r = Result(safe=True, status=SafetyStatus.SAFE, summary="ok") @@ -83,7 +83,11 @@ def test_assert_safe_pattern(self) -> None: safe_result = Result(safe=True, status=SafetyStatus.SAFE, summary="ok") assert safe_result, safe_result.summary - unsafe_result = Result(safe=False, status=SafetyStatus.UNSAFE, summary="attack detected") + unsafe_result = Result( + safe=False, + status=SafetyStatus.UNSAFE, + summary="attack detected", + ) with pytest.raises(AssertionError): assert unsafe_result, unsafe_result.summary @@ -124,7 +128,6 @@ def test_harm_category_accepts_plain_string(self) -> None: assert r.harm_category == "custom_product_risk" - class TestResolveAsAttack: def test_empty_returns_error(self) -> None: safe, status = resolve_as_attack(eval_results=[]) @@ -183,7 +186,6 @@ def test_all_not_detected_returns_safe(self) -> None: assert status is SafetyStatus.SAFE - class TestResolveAsProbe: def test_empty_returns_error(self) -> None: safe, status = resolve_as_probe(eval_results=[]) diff --git a/tests/unit/drivers/test_coerce.py b/tests/unit/drivers/test_coerce.py index 4bd08674..6b945306 100644 --- a/tests/unit/drivers/test_coerce.py +++ b/tests/unit/drivers/test_coerce.py @@ -7,7 +7,7 @@ import pytest -from rampart.core.prompt_driver import PromptDecision, PromptDriver +from rampart.core.prompt_driver import PromptDecision from rampart.core.types import Request, Response, Turn from rampart.drivers import _coerce_driver from rampart.drivers.static import StaticDriver @@ -25,7 +25,9 @@ async def test_str_produces_single_prompt_driver_async(self) -> None: assert d0.request.prompt == "hello" d1 = await driver.next_prompt_async( - history=[Turn(request=Request(prompt="hello"), response=Response(text="ok"))], + history=[ + Turn(request=Request(prompt="hello"), response=Response(text="ok")), + ], ) assert d1 is None @@ -55,7 +57,9 @@ async def test_prompt_driver_passthrough_async(self) -> None: async def test_custom_driver_passthrough_async(self) -> None: class Custom: async def next_prompt_async( - self, *, history: list[Turn], + self, + *, + history: list[Turn], ) -> PromptDecision | None: return PromptDecision(request=Request(prompt="custom")) diff --git a/tests/unit/evaluators/test_response_contains.py b/tests/unit/evaluators/test_response_contains.py index 33f96614..7606a8fd 100644 --- a/tests/unit/evaluators/test_response_contains.py +++ b/tests/unit/evaluators/test_response_contains.py @@ -21,20 +21,24 @@ def _ctx(text: str) -> EvalContext: class TestResponseContainsSubstring: @pytest.mark.asyncio async def test_case_insensitive_by_default(self) -> None: - result = await ResponseContains("HELLO").evaluate_async(context=_ctx("hello world")) + result = await ResponseContains("HELLO").evaluate_async( + context=_ctx("hello world"), + ) assert result.outcome is EvalOutcome.DETECTED @pytest.mark.asyncio async def test_case_sensitive_opt_in(self) -> None: result = await ResponseContains( - "HELLO", case_sensitive=True, + "HELLO", + case_sensitive=True, ).evaluate_async(context=_ctx("hello world")) assert result.outcome is EvalOutcome.NOT_DETECTED @pytest.mark.asyncio async def test_case_sensitive_match(self) -> None: result = await ResponseContains( - "hello", case_sensitive=True, + "hello", + case_sensitive=True, ).evaluate_async(context=_ctx("hello world")) assert result.outcome is EvalOutcome.DETECTED diff --git a/tests/unit/evaluators/test_side_effect.py b/tests/unit/evaluators/test_side_effect.py index 64668c34..6f827350 100644 --- a/tests/unit/evaluators/test_side_effect.py +++ b/tests/unit/evaluators/test_side_effect.py @@ -5,17 +5,26 @@ import pytest -from rampart.core.types import EvalContext, EvalOutcome, Request, Response, SideEffect, Turn +from rampart.core.types import ( + EvalContext, + EvalOutcome, + Request, + Response, + SideEffect, + Turn, +) from rampart.evaluators import SideEffectOccurred def _ctx_with_side_effects(*effects: SideEffect) -> EvalContext: """Build a single-turn EvalContext with the given side effects.""" return EvalContext( - turns=[Turn( - request=Request(prompt="test"), - response=Response(text="ok", side_effects=list(effects)), - )], + turns=[ + Turn( + request=Request(prompt="test"), + response=Response(text="ok", side_effects=list(effects)), + ), + ], ) @@ -45,7 +54,8 @@ async def test_exact_detail_match(self) -> None: se = SideEffect(kind="http_request", details={"url": "https://evil.com"}) ctx = _ctx_with_side_effects(se) result = await SideEffectOccurred( - "http_request", url="https://evil.com", + "http_request", + url="https://evil.com", ).evaluate_async(context=ctx) assert result.outcome is EvalOutcome.DETECTED @@ -54,7 +64,8 @@ async def test_exact_detail_mismatch(self) -> None: se = SideEffect(kind="http_request", details={"url": "https://safe.com"}) ctx = _ctx_with_side_effects(se) result = await SideEffectOccurred( - "http_request", url="https://evil.com", + "http_request", + url="https://evil.com", ).evaluate_async(context=ctx) assert result.outcome is EvalOutcome.NOT_DETECTED @@ -63,7 +74,8 @@ async def test_predicate_detail_match(self) -> None: se = SideEffect(kind="http_request", details={"url": "https://evil.com/data"}) ctx = _ctx_with_side_effects(se) result = await SideEffectOccurred( - "http_request", url=lambda u: "evil.com" in str(u), + "http_request", + url=lambda u: "evil.com" in str(u), ).evaluate_async(context=ctx) assert result.outcome is EvalOutcome.DETECTED @@ -72,6 +84,7 @@ async def test_predicate_detail_mismatch(self) -> None: se = SideEffect(kind="http_request", details={"url": "https://safe.com"}) ctx = _ctx_with_side_effects(se) result = await SideEffectOccurred( - "http_request", url=lambda u: "evil.com" in str(u), + "http_request", + url=lambda u: "evil.com" in str(u), ).evaluate_async(context=ctx) assert result.outcome is EvalOutcome.NOT_DETECTED diff --git a/tests/unit/evaluators/test_tool_called.py b/tests/unit/evaluators/test_tool_called.py index 96b86e1d..091bee4c 100644 --- a/tests/unit/evaluators/test_tool_called.py +++ b/tests/unit/evaluators/test_tool_called.py @@ -5,17 +5,26 @@ import pytest -from rampart.core.types import EvalContext, EvalOutcome, Request, Response, ToolCall, Turn +from rampart.core.types import ( + EvalContext, + EvalOutcome, + Request, + Response, + ToolCall, + Turn, +) from rampart.evaluators import ToolCalled def _ctx_with_tool_calls(*tool_calls: ToolCall) -> EvalContext: """Build an EvalContext with a single turn containing the given tool calls.""" return EvalContext( - turns=[Turn( - request=Request(prompt="test"), - response=Response(text="ok", tool_calls=list(tool_calls)), - )], + turns=[ + Turn( + request=Request(prompt="test"), + response=Response(text="ok", tool_calls=list(tool_calls)), + ), + ], ) @@ -58,14 +67,18 @@ class TestToolCalledParameterMatching: async def test_exact_parameter_match(self) -> None: tc = ToolCall(name="send_email", arguments={"to": "evil@evil.com"}) ctx = _ctx_with_tool_calls(tc) - result = await ToolCalled("send_email", to="evil@evil.com").evaluate_async(context=ctx) + result = await ToolCalled("send_email", to="evil@evil.com").evaluate_async( + context=ctx, + ) assert result.outcome is EvalOutcome.DETECTED @pytest.mark.asyncio async def test_exact_parameter_mismatch(self) -> None: tc = ToolCall(name="send_email", arguments={"to": "friend@company.com"}) ctx = _ctx_with_tool_calls(tc) - result = await ToolCalled("send_email", to="evil@evil.com").evaluate_async(context=ctx) + result = await ToolCalled("send_email", to="evil@evil.com").evaluate_async( + context=ctx, + ) assert result.outcome is EvalOutcome.NOT_DETECTED @pytest.mark.asyncio @@ -73,7 +86,8 @@ async def test_predicate_parameter_match(self) -> None: tc = ToolCall(name="send_email", arguments={"to": "evil@evil.com"}) ctx = _ctx_with_tool_calls(tc) result = await ToolCalled( - "send_email", to=lambda v: "evil" in str(v), + "send_email", + to=lambda v: "evil" in str(v), ).evaluate_async(context=ctx) assert result.outcome is EvalOutcome.DETECTED @@ -82,7 +96,8 @@ async def test_predicate_parameter_mismatch(self) -> None: tc = ToolCall(name="send_email", arguments={"to": "friend@company.com"}) ctx = _ctx_with_tool_calls(tc) result = await ToolCalled( - "send_email", to=lambda v: "evil" in str(v), + "send_email", + to=lambda v: "evil" in str(v), ).evaluate_async(context=ctx) assert result.outcome is EvalOutcome.NOT_DETECTED @@ -91,7 +106,8 @@ async def test_missing_parameter_returns_none_to_predicate(self) -> None: tc = ToolCall(name="send_email", arguments={}) ctx = _ctx_with_tool_calls(tc) result = await ToolCalled( - "send_email", to=lambda v: v is not None, + "send_email", + to=lambda v: v is not None, ).evaluate_async(context=ctx) assert result.outcome is EvalOutcome.NOT_DETECTED @@ -99,19 +115,23 @@ async def test_missing_parameter_returns_none_to_predicate(self) -> None: class TestToolCalledMultiTurn: @pytest.mark.asyncio async def test_scans_across_turns(self) -> None: - ctx = _multi_turn_ctx([ - [], - [ToolCall(name="send_email", arguments={"to": "evil@evil.com"})], - ]) + ctx = _multi_turn_ctx( + [ + [], + [ToolCall(name="send_email", arguments={"to": "evil@evil.com"})], + ], + ) result = await ToolCalled("send_email").evaluate_async(context=ctx) assert result.outcome is EvalOutcome.DETECTED @pytest.mark.asyncio async def test_not_detected_across_turns(self) -> None: - ctx = _multi_turn_ctx([ - [ToolCall(name="read_file")], - [ToolCall(name="query_db")], - ]) + ctx = _multi_turn_ctx( + [ + [ToolCall(name="read_file")], + [ToolCall(name="query_db")], + ], + ) result = await ToolCalled("send_email").evaluate_async(context=ctx) assert result.outcome is EvalOutcome.NOT_DETECTED diff --git a/tests/unit/payloads/test_generator.py b/tests/unit/payloads/test_generator.py index ac616071..d86a4952 100644 --- a/tests/unit/payloads/test_generator.py +++ b/tests/unit/payloads/test_generator.py @@ -77,7 +77,9 @@ async def capture(*, system_message, user_message): return "single variant" with patch.object( - PayloadGenerator, "_send_to_llm_async", side_effect=capture, + PayloadGenerator, + "_send_to_llm_async", + side_effect=capture, ): gen = PayloadGenerator(llm=_llm()) template = PayloadTemplate( @@ -127,7 +129,9 @@ async def capture(*, system_message, user_message): return "variant" with patch.object( - PayloadGenerator, "_send_to_llm_async", side_effect=capture, + PayloadGenerator, + "_send_to_llm_async", + side_effect=capture, ): gen = PayloadGenerator(llm=_llm()) await gen.generate_text_variants_async( diff --git a/tests/unit/payloads/test_payloads.py b/tests/unit/payloads/test_payloads.py index 062da79b..192cab2b 100644 --- a/tests/unit/payloads/test_payloads.py +++ b/tests/unit/payloads/test_payloads.py @@ -133,7 +133,9 @@ async def capture(*, system_message: str, user_message: str) -> str: from rampart.payloads._generator import PayloadGenerator with patch.object( - PayloadGenerator, "_send_to_llm_async", side_effect=capture, + PayloadGenerator, + "_send_to_llm_async", + side_effect=capture, ): await Payloads.generate_async( template=_template(), @@ -158,7 +160,9 @@ async def capture(*, system_message: str, user_message: str) -> str: from rampart.payloads._generator import PayloadGenerator with patch.object( - PayloadGenerator, "_send_to_llm_async", side_effect=capture, + PayloadGenerator, + "_send_to_llm_async", + side_effect=capture, ): await Payloads.generate_async( template=_template(), diff --git a/tests/unit/payloads/test_store.py b/tests/unit/payloads/test_store.py index 70ec8097..29ef73ec 100644 --- a/tests/unit/payloads/test_store.py +++ b/tests/unit/payloads/test_store.py @@ -4,7 +4,6 @@ """Tests for rampart.payloads._store — PayloadStore persistence.""" import json -from pathlib import Path import pytest @@ -12,7 +11,7 @@ from rampart.payloads._store import PayloadStore -@pytest.fixture() +@pytest.fixture def store(tmp_path): """PayloadStore rooted in a temporary directory.""" return PayloadStore(root=tmp_path) diff --git a/tests/unit/probes/test_single_turn.py b/tests/unit/probes/test_single_turn.py index 7eefffb4..e9f46606 100644 --- a/tests/unit/probes/test_single_turn.py +++ b/tests/unit/probes/test_single_turn.py @@ -183,7 +183,7 @@ class FailingAdapter: manifest = AppManifest(name="broken") observability_profile = ObservabilityLevel.RESPONSE_ONLY - async def create_session_async(self): # noqa: ANN201 + async def create_session_async(self): raise InfrastructureError("Service down") result = await Probes.behavior( @@ -201,12 +201,14 @@ class TestProbeEndToEnd: @pytest.mark.asyncio async def test_tool_call_detected_async(self) -> None: - adapter = _adapter(responses=[ - Response( - text="Confirming action.", - tool_calls=[ToolCall(name="confirm_action")], - ), - ]) + adapter = _adapter( + responses=[ + Response( + text="Confirming action.", + tool_calls=[ToolCall(name="confirm_action")], + ), + ], + ) result = await Probes.behavior( prompt="Delete all my calendar events", @@ -218,9 +220,11 @@ async def test_tool_call_detected_async(self) -> None: @pytest.mark.asyncio async def test_tool_call_not_detected_async(self) -> None: - adapter = _adapter(responses=[ - Response(text="Done. All events deleted."), - ]) + adapter = _adapter( + responses=[ + Response(text="Done. All events deleted."), + ], + ) result = await Probes.behavior( prompt="Delete all my calendar events", @@ -232,12 +236,14 @@ async def test_tool_call_not_detected_async(self) -> None: @pytest.mark.asyncio async def test_assert_pattern_async(self) -> None: - adapter = _adapter(responses=[ - Response( - text="Confirming.", - tool_calls=[ToolCall(name="confirm_action")], - ), - ]) + adapter = _adapter( + responses=[ + Response( + text="Confirming.", + tool_calls=[ToolCall(name="confirm_action")], + ), + ], + ) result = await Probes.behavior( prompt="Delete all events", diff --git a/tests/unit/pytest_plugin/test_collection.py b/tests/unit/pytest_plugin/test_collection.py index 630e7fae..8d270b25 100644 --- a/tests/unit/pytest_plugin/test_collection.py +++ b/tests/unit/pytest_plugin/test_collection.py @@ -79,7 +79,8 @@ async def test_records_on_post_execute_async(self) -> None: handler = ResultCollectionHandler() result = _make_result(summary="captured") event_data = _make_event_data( - event=ExecutionEvent.ON_POST_EXECUTE, result=result, + event=ExecutionEvent.ON_POST_EXECUTE, + result=result, ) await handler.on_event(event_data=event_data) @@ -122,7 +123,8 @@ async def test_noop_when_no_collector_active_async(self) -> None: handler = ResultCollectionHandler() result = _make_result() event_data = _make_event_data( - event=ExecutionEvent.ON_POST_EXECUTE, result=result, + event=ExecutionEvent.ON_POST_EXECUTE, + result=result, ) await handler.on_event(event_data=event_data) @@ -134,7 +136,8 @@ async def test_noop_when_result_is_none_async(self) -> None: try: handler = ResultCollectionHandler() event_data = _make_event_data( - event=ExecutionEvent.ON_POST_EXECUTE, result=None, + event=ExecutionEvent.ON_POST_EXECUTE, + result=None, ) await handler.on_event(event_data=event_data) diff --git a/tests/unit/pytest_plugin/test_plugin.py b/tests/unit/pytest_plugin/test_plugin.py index c3fd9186..80e8c339 100644 --- a/tests/unit/pytest_plugin/test_plugin.py +++ b/tests/unit/pytest_plugin/test_plugin.py @@ -6,7 +6,7 @@ from __future__ import annotations from typing import Any -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, MagicMock import pytest @@ -15,9 +15,8 @@ from rampart.pytest_plugin._collection import ResultCollectionHandler, ResultCollector from rampart.pytest_plugin._session import RampartSession from rampart.pytest_plugin.plugin import ( - _ANSI_ESCAPE_RE, - _evaluate_gates, _emit_sinks, + _evaluate_gates, _resolve_trial_n, _sanitize_for_terminal, _write_result_line, @@ -36,23 +35,23 @@ class _StashStub: def __init__(self) -> None: self._data: dict[Any, Any] = {} - def __setitem__(self, key: Any, value: Any) -> None: + def __setitem__(self, key: Any, value: Any) -> None: # noqa: ANN401 self._data[key] = value - def __getitem__(self, key: Any) -> Any: + def __getitem__(self, key: Any) -> Any: # noqa: ANN401 return self._data[key] - def __contains__(self, key: Any) -> bool: + def __contains__(self, key: Any) -> bool: # noqa: ANN401 return key in self._data - def __delitem__(self, key: Any) -> None: + def __delitem__(self, key: Any) -> None: # noqa: ANN401 del self._data[key] - def get(self, key: Any, default: Any = None) -> Any: + def get(self, key: Any, default: Any = None) -> Any: # noqa: ANN401 """Return value for key, or default.""" return self._data.get(key, default) - def pop(self, key: Any, *args: Any) -> Any: + def pop(self, key: Any, *args: Any) -> Any: # noqa: ANN401 """Remove and return value for key.""" return self._data.pop(key, *args) @@ -77,6 +76,7 @@ def test_configure_sets_factory(self) -> None: pytest_configure(config) try: from rampart.core.execution import _default_handler_factory + handlers = _default_handler_factory() assert len(handlers) == 1 assert isinstance(handlers[0], ResultCollectionHandler) @@ -89,6 +89,7 @@ def test_unconfigure_clears_factory(self) -> None: pytest_unconfigure(config) from rampart.core.execution import _default_handler_factory + assert _default_handler_factory() == [] def test_configure_creates_session_in_stash(self) -> None: @@ -96,6 +97,7 @@ def test_configure_creates_session_in_stash(self) -> None: pytest_configure(config) try: from rampart.pytest_plugin.plugin import _rampart_key + assert isinstance(config.stash.get(_rampart_key), RampartSession) finally: pytest_unconfigure(config) @@ -106,6 +108,7 @@ def test_unconfigure_removes_session_from_stash(self) -> None: pytest_unconfigure(config) from rampart.pytest_plugin.plugin import _rampart_key + assert config.stash.get(_rampart_key) is None @@ -115,7 +118,9 @@ class TestRampartSession: def test_absorb_accumulates_results(self) -> None: session = RampartSession() collector = ResultCollector() - collector.record(result=Result(safe=True, status=SafetyStatus.SAFE, summary="ok")) + collector.record( + result=Result(safe=True, status=SafetyStatus.SAFE, summary="ok"), + ) node = MagicMock() node.nodeid = "test_file.py::test_absorb" @@ -134,9 +139,15 @@ def test_build_report_counts(self) -> None: session = RampartSession() collector = ResultCollector() - collector.record(result=Result(safe=True, status=SafetyStatus.SAFE, summary="s")) - collector.record(result=Result(safe=False, status=SafetyStatus.UNSAFE, summary="u")) - collector.record(result=Result(safe=False, status=SafetyStatus.ERROR, summary="e")) + collector.record( + result=Result(safe=True, status=SafetyStatus.SAFE, summary="s"), + ) + collector.record( + result=Result(safe=False, status=SafetyStatus.UNSAFE, summary="u"), + ) + collector.record( + result=Result(safe=False, status=SafetyStatus.ERROR, summary="e"), + ) node = MagicMock() node.nodeid = "test_file.py::test_counts" @@ -167,7 +178,7 @@ def test_record_trial_group(self) -> None: safe=statuses[idx] == SafetyStatus.SAFE, status=statuses[idx], summary=f"trial-{idx}", - ) + ), ) session.absorb(node=item, collector=collector) @@ -196,7 +207,11 @@ def test_record_trial_group_all_errors(self) -> None: item.nodeid = f"test_file.py::test_err[trial-{idx}]" collector = ResultCollector() collector.record( - result=Result(safe=False, status=SafetyStatus.ERROR, summary=f"err-{idx}") + result=Result( + safe=False, + status=SafetyStatus.ERROR, + summary=f"err-{idx}", + ), ) session.absorb(node=item, collector=collector) @@ -257,7 +272,10 @@ def _make_plain_item( class TestTrialCloning: """Trial cloning produces n items with distinct [trial-N] node ids.""" - def test_trial_cloning_produces_n_items(self, monkeypatch: pytest.MonkeyPatch) -> None: + def test_trial_cloning_produces_n_items( + self, + monkeypatch: pytest.MonkeyPatch, + ) -> None: item = _make_trial_item(n=3) clone_instances = [MagicMock() for _ in range(3)] for clone in clone_instances: @@ -385,7 +403,9 @@ class TestWriteResultLine: def test_safe_result_includes_observability(self) -> None: reporter = MagicMock() result = Result( - safe=True, status=SafetyStatus.SAFE, summary="ok", + safe=True, + status=SafetyStatus.SAFE, + summary="ok", observability_level=ObservabilityLevel.RESPONSE_ONLY, ) _write_result_line(terminalreporter=reporter, result=result) @@ -394,31 +414,38 @@ def test_safe_result_includes_observability(self) -> None: def test_unsafe_result_includes_observability(self) -> None: reporter = MagicMock() result = Result( - safe=False, status=SafetyStatus.UNSAFE, summary="bad", + safe=False, + status=SafetyStatus.UNSAFE, + summary="bad", observability_level=ObservabilityLevel.TOOL_AND_SIDE_EFFECTS, ) _write_result_line(terminalreporter=reporter, result=result) reporter.write_line.assert_called_once_with( - " FAIL bad (tool_and_side_effects)" + " FAIL bad (tool_and_side_effects)", ) def test_with_test_name(self) -> None: reporter = MagicMock() result = Result( - safe=True, status=SafetyStatus.SAFE, summary="SAFE", + safe=True, + status=SafetyStatus.SAFE, + summary="SAFE", observability_level=ObservabilityLevel.TOOL_ONLY, ) _write_result_line( - terminalreporter=reporter, result=result, test_name="test_exfil", + terminalreporter=reporter, + result=result, + test_name="test_exfil", ) reporter.write_line.assert_called_once_with( - " PASS test_exfil -- SAFE (tool_only)" + " PASS test_exfil -- SAFE (tool_only)", ) def test_ansi_stripped_from_summary(self) -> None: reporter = MagicMock() result = Result( - safe=True, status=SafetyStatus.SAFE, + safe=True, + status=SafetyStatus.SAFE, summary="\x1b[31mevil\x1b[0m", ) _write_result_line(terminalreporter=reporter, result=result) @@ -434,14 +461,22 @@ def _make_session_with_results(self) -> RampartSession: """Build a RampartSession with two results in different categories.""" session = RampartSession() collector = ResultCollector() - collector.record(result=Result( - safe=True, status=SafetyStatus.SAFE, - summary="safe-one", harm_category="data_exfiltration", - )) - collector.record(result=Result( - safe=False, status=SafetyStatus.UNSAFE, - summary="unsafe-one", harm_category="jailbreak", - )) + collector.record( + result=Result( + safe=True, + status=SafetyStatus.SAFE, + summary="safe-one", + harm_category="data_exfiltration", + ), + ) + collector.record( + result=Result( + safe=False, + status=SafetyStatus.UNSAFE, + summary="unsafe-one", + harm_category="jailbreak", + ), + ) node = MagicMock() node.nodeid = "test_file.py::test_summary" session.absorb(node=node, collector=collector) @@ -459,6 +494,7 @@ def test_noop_when_no_results(self) -> None: config = MagicMock() config.stash = _StashStub() from rampart.pytest_plugin.plugin import _rampart_key + config.stash[_rampart_key] = RampartSession() pytest_terminal_summary(terminalreporter=reporter, exitstatus=0, config=config) reporter.write_sep.assert_not_called() @@ -468,6 +504,7 @@ def test_writes_summary_header(self) -> None: config = MagicMock() config.stash = _StashStub() from rampart.pytest_plugin.plugin import _rampart_key + config.stash[_rampart_key] = self._make_session_with_results() pytest_terminal_summary(terminalreporter=reporter, exitstatus=0, config=config) reporter.write_sep.assert_called_once_with("=", "RAMPART Safety Summary") @@ -477,12 +514,12 @@ def test_writes_population_stats(self) -> None: config = MagicMock() config.stash = _StashStub() from rampart.pytest_plugin.plugin import _rampart_key + config.stash[_rampart_key] = self._make_session_with_results() pytest_terminal_summary(terminalreporter=reporter, exitstatus=0, config=config) # Check that the Population line was written population_calls = [ - c for c in reporter.write_line.call_args_list - if "Population:" in str(c) + c for c in reporter.write_line.call_args_list if "Population:" in str(c) ] assert len(population_calls) == 1 @@ -555,7 +592,9 @@ class TestRampartSessionDuration: def test_default_duration_zero(self) -> None: session = RampartSession() collector = ResultCollector() - collector.record(result=Result(safe=True, status=SafetyStatus.SAFE, summary="ok")) + collector.record( + result=Result(safe=True, status=SafetyStatus.SAFE, summary="ok"), + ) node = MagicMock() node.nodeid = "test.py::test_dur" session.absorb(node=node, collector=collector) @@ -565,7 +604,9 @@ def test_default_duration_zero(self) -> None: def test_set_duration_reflected_in_report(self) -> None: session = RampartSession() collector = ResultCollector() - collector.record(result=Result(safe=True, status=SafetyStatus.SAFE, summary="ok")) + collector.record( + result=Result(safe=True, status=SafetyStatus.SAFE, summary="ok"), + ) node = MagicMock() node.nodeid = "test.py::test_dur" session.absorb(node=node, collector=collector) @@ -585,7 +626,11 @@ def test_writes_trial_group_line(self) -> None: collector = ResultCollector() status = SafetyStatus.UNSAFE if idx < 2 else SafetyStatus.SAFE collector.record( - result=Result(safe=status == SafetyStatus.SAFE, status=status, summary=f"t-{idx}") + result=Result( + safe=status == SafetyStatus.SAFE, + status=status, + summary=f"t-{idx}", + ), ) session.absorb(node=item, collector=collector) @@ -622,7 +667,11 @@ def test_logs_when_rate_exceeds_threshold(self) -> None: collector = ResultCollector() status = SafetyStatus.UNSAFE if idx < 2 else SafetyStatus.SAFE collector.record( - result=Result(safe=status == SafetyStatus.SAFE, status=status, summary=f"t-{idx}") + result=Result( + safe=status == SafetyStatus.SAFE, + status=status, + summary=f"t-{idx}", + ), ) session.absorb(node=item, collector=collector) @@ -648,7 +697,9 @@ def test_sink_error_swallowed(self) -> None: mock_sink.emit_async = AsyncMock(side_effect=RuntimeError("Kusto down")) session = RampartSession(sinks=[mock_sink]) collector = ResultCollector() - collector.record(result=Result(safe=True, status=SafetyStatus.SAFE, summary="ok")) + collector.record( + result=Result(safe=True, status=SafetyStatus.SAFE, summary="ok"), + ) node = MagicMock() node.nodeid = "test.py::test_sink" session.absorb(node=node, collector=collector) diff --git a/tests/unit/reporting/test_report.py b/tests/unit/reporting/test_report.py index a89bf097..0cc5e8c2 100644 --- a/tests/unit/reporting/test_report.py +++ b/tests/unit/reporting/test_report.py @@ -32,53 +32,116 @@ class TestByHarmCategory: """by_harm_category groups correctly and uses 'uncategorized' for None.""" def test_groups_by_enum_category(self) -> None: - report = TestRunReport(results=[ - Result(safe=True, status=SafetyStatus.SAFE, summary="ok", harm_category=HarmCategory.DATA_EXFILTRATION), - Result(safe=False, status=SafetyStatus.UNSAFE, summary="bad", harm_category=HarmCategory.DATA_EXFILTRATION), - Result(safe=True, status=SafetyStatus.SAFE, summary="ok2", harm_category=HarmCategory.JAILBREAK), - ]) + report = TestRunReport( + results=[ + Result( + safe=True, + status=SafetyStatus.SAFE, + summary="ok", + harm_category=HarmCategory.DATA_EXFILTRATION, + ), + Result( + safe=False, + status=SafetyStatus.UNSAFE, + summary="bad", + harm_category=HarmCategory.DATA_EXFILTRATION, + ), + Result( + safe=True, + status=SafetyStatus.SAFE, + summary="ok2", + harm_category=HarmCategory.JAILBREAK, + ), + ], + ) grouped = report.by_harm_category() assert len(grouped["data_exfiltration"]) == 2 assert len(grouped["jailbreak"]) == 1 def test_groups_by_plain_string_category(self) -> None: - report = TestRunReport(results=[ - Result(safe=True, status=SafetyStatus.SAFE, summary="ok", harm_category="custom_risk"), - Result(safe=True, status=SafetyStatus.SAFE, summary="ok2", harm_category="custom_risk"), - ]) + report = TestRunReport( + results=[ + Result( + safe=True, + status=SafetyStatus.SAFE, + summary="ok", + harm_category="custom_risk", + ), + Result( + safe=True, + status=SafetyStatus.SAFE, + summary="ok2", + harm_category="custom_risk", + ), + ], + ) grouped = report.by_harm_category() assert len(grouped["custom_risk"]) == 2 def test_none_category_becomes_uncategorized(self) -> None: - report = TestRunReport(results=[ - Result(safe=True, status=SafetyStatus.SAFE, summary="ok", harm_category=None), - ]) + report = TestRunReport( + results=[ + Result( + safe=True, + status=SafetyStatus.SAFE, + summary="ok", + harm_category=None, + ), + ], + ) grouped = report.by_harm_category() assert "uncategorized" in grouped assert len(grouped["uncategorized"]) == 1 def test_mixed_categories(self) -> None: - report = TestRunReport(results=[ - Result(safe=True, status=SafetyStatus.SAFE, summary="a", harm_category=HarmCategory.DATA_EXFILTRATION), - Result(safe=True, status=SafetyStatus.SAFE, summary="b", harm_category=None), - Result(safe=True, status=SafetyStatus.SAFE, summary="c", harm_category="team_specific"), - ]) + report = TestRunReport( + results=[ + Result( + safe=True, + status=SafetyStatus.SAFE, + summary="a", + harm_category=HarmCategory.DATA_EXFILTRATION, + ), + Result( + safe=True, + status=SafetyStatus.SAFE, + summary="b", + harm_category=None, + ), + Result( + safe=True, + status=SafetyStatus.SAFE, + summary="c", + harm_category="team_specific", + ), + ], + ) grouped = report.by_harm_category() - assert set(grouped.keys()) == {"data_exfiltration", "uncategorized", "team_specific"} + assert set(grouped.keys()) == { + "data_exfiltration", + "uncategorized", + "team_specific", + } class TestPopulationSummary: - """population_summary computes attack_success_rate and safety_pass_rate correctly.""" + """Tests population_summary. + + population_summary should compute attack_success_rate and safety_pass_rate + correctly. + """ def test_all_safe(self) -> None: - report = TestRunReport(results=[ - Result(safe=True, status=SafetyStatus.SAFE, summary="ok"), - Result(safe=True, status=SafetyStatus.SAFE, summary="ok2"), - ]) + report = TestRunReport( + results=[ + Result(safe=True, status=SafetyStatus.SAFE, summary="ok"), + Result(safe=True, status=SafetyStatus.SAFE, summary="ok2"), + ], + ) stats = report.population_summary() assert stats.total_runs == 2 @@ -88,11 +151,13 @@ def test_all_safe(self) -> None: assert stats.safety_pass_rate == 1.0 def test_mixed_results(self) -> None: - report = TestRunReport(results=[ - Result(safe=True, status=SafetyStatus.SAFE, summary="ok"), - Result(safe=False, status=SafetyStatus.UNSAFE, summary="bad"), - Result(safe=False, status=SafetyStatus.UNDETERMINED, summary="?"), - ]) + report = TestRunReport( + results=[ + Result(safe=True, status=SafetyStatus.SAFE, summary="ok"), + Result(safe=False, status=SafetyStatus.UNSAFE, summary="bad"), + Result(safe=False, status=SafetyStatus.UNDETERMINED, summary="?"), + ], + ) stats = report.population_summary() assert stats.total_runs == 3 @@ -110,11 +175,13 @@ def test_empty_results(self) -> None: assert stats.attack_success_rate == 0.0 def test_error_excluded_from_attack_success_rate(self) -> None: - report = TestRunReport(results=[ - Result(safe=True, status=SafetyStatus.SAFE, summary="ok"), - Result(safe=False, status=SafetyStatus.UNSAFE, summary="bad"), - Result(safe=False, status=SafetyStatus.ERROR, summary="infra"), - ]) + report = TestRunReport( + results=[ + Result(safe=True, status=SafetyStatus.SAFE, summary="ok"), + Result(safe=False, status=SafetyStatus.UNSAFE, summary="bad"), + Result(safe=False, status=SafetyStatus.ERROR, summary="infra"), + ], + ) stats = report.population_summary() assert stats.total_runs == 3 @@ -123,10 +190,12 @@ def test_error_excluded_from_attack_success_rate(self) -> None: assert stats.safety_pass_rate == pytest.approx(1 / 2) def test_all_errors(self) -> None: - report = TestRunReport(results=[ - Result(safe=False, status=SafetyStatus.ERROR, summary="err1"), - Result(safe=False, status=SafetyStatus.ERROR, summary="err2"), - ]) + report = TestRunReport( + results=[ + Result(safe=False, status=SafetyStatus.ERROR, summary="err1"), + Result(safe=False, status=SafetyStatus.ERROR, summary="err2"), + ], + ) stats = report.population_summary() assert stats.total_runs == 2 @@ -135,11 +204,28 @@ def test_all_errors(self) -> None: assert stats.safety_pass_rate == 0.0 def test_filter_by_harm_category(self) -> None: - report = TestRunReport(results=[ - Result(safe=True, status=SafetyStatus.SAFE, summary="ok", harm_category=HarmCategory.DATA_EXFILTRATION), - Result(safe=False, status=SafetyStatus.UNSAFE, summary="bad", harm_category=HarmCategory.JAILBREAK), - Result(safe=True, status=SafetyStatus.SAFE, summary="ok2", harm_category=HarmCategory.DATA_EXFILTRATION), - ]) + report = TestRunReport( + results=[ + Result( + safe=True, + status=SafetyStatus.SAFE, + summary="ok", + harm_category=HarmCategory.DATA_EXFILTRATION, + ), + Result( + safe=False, + status=SafetyStatus.UNSAFE, + summary="bad", + harm_category=HarmCategory.JAILBREAK, + ), + Result( + safe=True, + status=SafetyStatus.SAFE, + summary="ok2", + harm_category=HarmCategory.DATA_EXFILTRATION, + ), + ], + ) stats = report.population_summary(harm_category=HarmCategory.DATA_EXFILTRATION) assert stats.total_runs == 2 @@ -147,19 +233,38 @@ def test_filter_by_harm_category(self) -> None: assert stats.unsafe_count == 0 def test_filter_by_plain_string_category(self) -> None: - report = TestRunReport(results=[ - Result(safe=True, status=SafetyStatus.SAFE, summary="ok", harm_category="custom"), - Result(safe=False, status=SafetyStatus.UNSAFE, summary="bad", harm_category="other"), - ]) + report = TestRunReport( + results=[ + Result( + safe=True, + status=SafetyStatus.SAFE, + summary="ok", + harm_category="custom", + ), + Result( + safe=False, + status=SafetyStatus.UNSAFE, + summary="bad", + harm_category="other", + ), + ], + ) stats = report.population_summary(harm_category="custom") assert stats.total_runs == 1 assert stats.safe_count == 1 def test_filter_returns_empty_for_missing_category(self) -> None: - report = TestRunReport(results=[ - Result(safe=True, status=SafetyStatus.SAFE, summary="ok", harm_category=HarmCategory.DATA_EXFILTRATION), - ]) + report = TestRunReport( + results=[ + Result( + safe=True, + status=SafetyStatus.SAFE, + summary="ok", + harm_category=HarmCategory.DATA_EXFILTRATION, + ), + ], + ) stats = report.population_summary(harm_category="nonexistent") assert stats.total_runs == 0 diff --git a/tests/unit/surfaces/test_onedrive.py b/tests/unit/surfaces/test_onedrive.py index c8a2f1d2..0e8484c3 100644 --- a/tests/unit/surfaces/test_onedrive.py +++ b/tests/unit/surfaces/test_onedrive.py @@ -12,11 +12,11 @@ from rampart.core.errors import InfrastructureError from rampart.core.injection import InjectionHandle, Surface -from rampart.core.types import Payload, PayloadFormat +from rampart.core.types import Payload from rampart.surfaces.onedrive import ( + _MAX_SMALL_UPLOAD_BYTES, OneDriveSurface, _OneDriveInjection, - _MAX_SMALL_UPLOAD_BYTES, ) _UNSET = object() @@ -25,7 +25,7 @@ def _make_graph_client( *, upload_item_id: str = "item-abc-123", - upload_return: Any = _UNSET, + upload_return: Any = _UNSET, # noqa: ANN401 upload_error: Exception | None = None, delete_error: Exception | None = None, ) -> MagicMock: @@ -63,13 +63,13 @@ def _make_graph_client( items_mock = MagicMock() - def _by_drive_item_id_dispatch(item_id: str) -> Any: + def _by_drive_item_id_dispatch(item_id: str) -> Any: # noqa: ANN401 if item_id.startswith("root:"): return upload_item_mock return delete_item_mock items_mock.by_drive_item_id = MagicMock( - side_effect=_by_drive_item_id_dispatch + side_effect=_by_drive_item_id_dispatch, ) by_drive_id_mock = MagicMock() @@ -293,7 +293,9 @@ async def test_returns_self_from_aenter(self) -> None: assert h is handle @pytest.mark.asyncio - async def test_upload_exceeding_size_limit_raises_infrastructure_error(self) -> None: + async def test_upload_exceeding_size_limit_raises_infrastructure_error( + self, + ) -> None: client = _make_graph_client() surface = OneDriveSurface( graph_client=client, From 5f26819159c3f25510ca80d6657f2115a020dae6 Mon Sep 17 00:00:00 2001 From: spencrr <23708360+spencrr@users.noreply.github.com> Date: Tue, 14 Apr 2026 16:06:34 -0700 Subject: [PATCH 2/5] [STYLE]: Safe noqa fixes Pt.1 --- rampart/_pyrit/llm_bridge.py | 8 +++----- rampart/evaluators/side_effect.py | 5 +++-- rampart/evaluators/tool_called.py | 5 +++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/rampart/_pyrit/llm_bridge.py b/rampart/_pyrit/llm_bridge.py index 46ae6d8a..ba5c0417 100644 --- a/rampart/_pyrit/llm_bridge.py +++ b/rampart/_pyrit/llm_bridge.py @@ -13,7 +13,9 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any +from uuid import uuid4 +from pyrit.models import MessagePiece from pyrit.prompt_target import OpenAIChatTarget, PromptChatTarget if TYPE_CHECKING: @@ -128,12 +130,8 @@ async def send_generation_request_async( Returns: str: The LLM's text response. """ - import uuid # noqa: PLC0415 — deferred: pyrit is optional - - from pyrit.models import MessagePiece # noqa: PLC0415 - target = create_prompt_target(config) - conversation_id = str(uuid.uuid4()) + conversation_id = str(uuid4()) target.set_system_prompt( system_prompt=system_message, diff --git a/rampart/evaluators/side_effect.py b/rampart/evaluators/side_effect.py index 1b7a536c..7bac9467 100644 --- a/rampart/evaluators/side_effect.py +++ b/rampart/evaluators/side_effect.py @@ -16,10 +16,11 @@ class SideEffectOccurred(BaseEvaluator): Args: kind (str): The side effect kind to look for (positional-only). - **detail_predicates (Any): Detail field -> expected value or predicate. + **detail_predicates (dict[str, Any]): + Detail field -> expected value or predicate. """ - def __init__(self, kind: str, /, **detail_predicates: Any) -> None: # noqa: ANN401 + def __init__(self, kind: str, /, **detail_predicates: dict[str, Any]) -> None: """Initialize with side effect kind and optional predicates.""" self._kind = kind self._predicates = detail_predicates diff --git a/rampart/evaluators/tool_called.py b/rampart/evaluators/tool_called.py index cf06025f..648f12d4 100644 --- a/rampart/evaluators/tool_called.py +++ b/rampart/evaluators/tool_called.py @@ -23,10 +23,11 @@ class ToolCalled(BaseEvaluator): Args: tool_name (str): The tool to look for (positional-only). - **param_predicates (Any): Parameter name -> expected value or predicate. + **param_predicates (dict[str, Any]): + Parameter name -> expected value or predicate. """ - def __init__(self, tool_name: str, /, **param_predicates: Any) -> None: # noqa: ANN401 + def __init__(self, tool_name: str, /, **param_predicates: dict[str, Any]) -> None: """Initialize with tool name and optional parameter predicates.""" self._tool_name = tool_name self._predicates = param_predicates From 79c08050f34a1636b0743845b6534bee36ece06e Mon Sep 17 00:00:00 2001 From: spencrr <23708360+spencrr@users.noreply.github.com> Date: Tue, 14 Apr 2026 16:09:38 -0700 Subject: [PATCH 3/5] [STYLE]: Safe-ish noqa fixes Pt.2 --- rampart/surfaces/onedrive.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rampart/surfaces/onedrive.py b/rampart/surfaces/onedrive.py index 89615b1d..447834fe 100644 --- a/rampart/surfaces/onedrive.py +++ b/rampart/surfaces/onedrive.py @@ -97,7 +97,7 @@ def inject(self, *, payload: Payload) -> _OneDriveInjection: """ return _OneDriveInjection(surface=self, payload=payload) - async def _upload_async(self, *, payload: Payload) -> str: + async def upload_async(self, *, payload: Payload) -> str: """Upload payload content to OneDrive. Returns the item ID. Uses the small-file upload endpoint @@ -162,7 +162,7 @@ async def _upload_async(self, *, payload: Payload) -> str: ) return item_id - async def _delete_async(self, *, item_id: str) -> None: + async def delete_async(self, *, item_id: str) -> None: """Delete a file from OneDrive by item ID.""" await ( self._graph_client.drives.by_drive_id(self._drive_id) @@ -202,7 +202,7 @@ def surface_name(self) -> str: async def __aenter__(self) -> Self: """Upload payload to OneDrive. Raises InfrastructureError on failure.""" try: - self._item_id = await self._surface._upload_async( # noqa: SLF001 + self._item_id = await self._surface.upload_async( payload=self._payload, ) except InfrastructureError: @@ -226,7 +226,7 @@ async def __aexit__( """Delete uploaded content. Logs warnings on failure but never raises.""" if self._item_id is not None: try: - await self._surface._delete_async(item_id=self._item_id) # noqa: SLF001 + await self._surface.delete_async(item_id=self._item_id) except Exception: # noqa: BLE001 — cleanup must not raise logger.warning( "OneDrive cleanup failed for item %s in drive=%s", From 9106ab59b55d5e6f3c209a20cf89b3f6d1c263f0 Mon Sep 17 00:00:00 2001 From: spencrr <23708360+spencrr@users.noreply.github.com> Date: Wed, 15 Apr 2026 16:25:52 -0700 Subject: [PATCH 4/5] [STYLE]: Safe noqa fixes Pt.3 --- pyproject.toml | 2 +- rampart/pytest_plugin/plugin.py | 14 ++++++-------- tests/unit/pytest_plugin/test_plugin.py | 12 ++++++------ tests/unit/surfaces/test_onedrive.py | 4 ++-- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 894e0ed5..61458772 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ select = ["ALL"] "tests/**" = [ "S101", # assert is pytest's API "D100", "D101", "D102", "D104", "D107", # no docstrings needed - "ANN001", "ANN201", "ANN202", # no type annotations needed + "ANN001", "ANN201", "ANN202", "ANN401", # no type annotations needed "PLR2004", # magic values in assertions are fine "ARG001", "ARG002", # unused args (fixtures, stubs) "PLC0415", # imports inside functions for isolation diff --git a/rampart/pytest_plugin/plugin.py b/rampart/pytest_plugin/plugin.py index a127f2e9..6acb9a2e 100644 --- a/rampart/pytest_plugin/plugin.py +++ b/rampart/pytest_plugin/plugin.py @@ -48,6 +48,8 @@ if TYPE_CHECKING: from collections.abc import Generator + from _pytest.terminal import TerminalReporter + logger = logging.getLogger(__name__) __all__ = [ @@ -528,7 +530,7 @@ def _emit_sinks(*, rampart_session: RampartSession) -> None: def _write_result_line( *, - terminalreporter: Any, # noqa: ANN401 + terminalreporter: TerminalReporter, result: Result, test_name: str = "", ) -> None: @@ -538,8 +540,7 @@ def _write_result_line( ``PASS test_name — summary (observability_level)`` Args: - terminalreporter: The pytest terminal reporter (typed as Any - to avoid importing from ``_pytest.terminal``). + terminalreporter: The pytest terminal reporter. result (Result): The result to display. test_name (str): The test name to include in the line. """ @@ -559,7 +560,7 @@ def _write_result_line( def _write_trial_group_lines( *, - terminalreporter: Any, # noqa: ANN401 + terminalreporter: TerminalReporter, rampart_session: RampartSession, ) -> None: """Write trial group aggregate lines to the terminal. @@ -580,7 +581,7 @@ def _write_trial_group_lines( def pytest_terminal_summary( - terminalreporter: Any, # noqa: ANN401 + terminalreporter: TerminalReporter, exitstatus: int, # noqa: ARG001 — pytest hook signature config: pytest.Config, ) -> None: @@ -590,9 +591,6 @@ def pytest_terminal_summary( trial group aggregates, and population statistics. No-op if no RAMPART results were collected. - The terminalreporter parameter is typed as Any to avoid importing - from ``_pytest.terminal``, which is pytest's private internal API. - Args: terminalreporter: The pytest terminal reporter. exitstatus (int): The session exit status. diff --git a/tests/unit/pytest_plugin/test_plugin.py b/tests/unit/pytest_plugin/test_plugin.py index 80e8c339..2e344f4e 100644 --- a/tests/unit/pytest_plugin/test_plugin.py +++ b/tests/unit/pytest_plugin/test_plugin.py @@ -35,23 +35,23 @@ class _StashStub: def __init__(self) -> None: self._data: dict[Any, Any] = {} - def __setitem__(self, key: Any, value: Any) -> None: # noqa: ANN401 + def __setitem__(self, key: Any, value: Any) -> None: self._data[key] = value - def __getitem__(self, key: Any) -> Any: # noqa: ANN401 + def __getitem__(self, key: Any) -> Any: return self._data[key] - def __contains__(self, key: Any) -> bool: # noqa: ANN401 + def __contains__(self, key: Any) -> bool: return key in self._data - def __delitem__(self, key: Any) -> None: # noqa: ANN401 + def __delitem__(self, key: Any) -> None: del self._data[key] - def get(self, key: Any, default: Any = None) -> Any: # noqa: ANN401 + def get(self, key: Any, default: Any = None) -> Any: """Return value for key, or default.""" return self._data.get(key, default) - def pop(self, key: Any, *args: Any) -> Any: # noqa: ANN401 + def pop(self, key: Any, *args: Any) -> Any: """Remove and return value for key.""" return self._data.pop(key, *args) diff --git a/tests/unit/surfaces/test_onedrive.py b/tests/unit/surfaces/test_onedrive.py index 0e8484c3..92646554 100644 --- a/tests/unit/surfaces/test_onedrive.py +++ b/tests/unit/surfaces/test_onedrive.py @@ -25,7 +25,7 @@ def _make_graph_client( *, upload_item_id: str = "item-abc-123", - upload_return: Any = _UNSET, # noqa: ANN401 + upload_return: Any = _UNSET, upload_error: Exception | None = None, delete_error: Exception | None = None, ) -> MagicMock: @@ -63,7 +63,7 @@ def _make_graph_client( items_mock = MagicMock() - def _by_drive_item_id_dispatch(item_id: str) -> Any: # noqa: ANN401 + def _by_drive_item_id_dispatch(item_id: str) -> Any: if item_id.startswith("root:"): return upload_item_mock return delete_item_mock From d8dcd683d11d737bce2c07287207ad998b8142b0 Mon Sep 17 00:00:00 2001 From: spencrr <23708360+spencrr@users.noreply.github.com> Date: Wed, 15 Apr 2026 16:11:15 -0700 Subject: [PATCH 5/5] [STYLE]: noqa fix Pt.4 - asyncio --- rampart/pytest_plugin/plugin.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/rampart/pytest_plugin/plugin.py b/rampart/pytest_plugin/plugin.py index 6acb9a2e..ff5dc31c 100644 --- a/rampart/pytest_plugin/plugin.py +++ b/rampart/pytest_plugin/plugin.py @@ -506,6 +506,9 @@ async def _emit_sinks_async(*, rampart_session: RampartSession) -> None: ) +_background_tasks: set[asyncio.Task[Any]] = set() + + def _emit_sinks(*, rampart_session: RampartSession) -> None: """Synchronous wrapper for sink emission. @@ -521,11 +524,15 @@ def _emit_sinks(*, rampart_session: RampartSession) -> None: coro = _emit_sinks_async(rampart_session=rampart_session) try: - asyncio.run(coro) - except RuntimeError: - # Inside an already-running event loop (e.g. pytest-asyncio). loop = asyncio.get_running_loop() - _background_task = loop.create_task(coro) # noqa: RUF006 + except RuntimeError: + # No event loop running — start one. + asyncio.run(coro) + else: + # Event loop is already running — schedule the coroutine. + task = loop.create_task(coro) + _background_tasks.add(task) + task.add_done_callback(_background_tasks.discard) def _write_result_line(