diff --git a/pyproject.toml b/pyproject.toml index 5a5d8530..bd4293b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,12 @@ skip_empty = true [tool.pyright] pythonVersion = "3.11" typeCheckingMode = "strict" -include = ["rampart"] +include = ["rampart", "tests"] + +[[tool.pyright.executionEnvironments]] +root = "tests" +extraPaths = ["."] +reportPrivateUsage = false [tool.pytest.ini_options] asyncio_mode = "auto" diff --git a/rampart/_pyrit/llm_bridge.py b/rampart/_pyrit/llm_bridge.py index ba5c0417..4de0d1b1 100644 --- a/rampart/_pyrit/llm_bridge.py +++ b/rampart/_pyrit/llm_bridge.py @@ -12,10 +12,10 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, cast from uuid import uuid4 -from pyrit.models import MessagePiece +from pyrit.models import Message, MessagePiece from pyrit.prompt_target import OpenAIChatTarget, PromptChatTarget if TYPE_CHECKING: @@ -143,7 +143,9 @@ async def send_generation_request_async( original_value=user_message, conversation_id=conversation_id, ) - request = request_piece.to_message() + + # Can remove after bumping to PyRIT v0.13.0 + request = cast("Message", request_piece.to_message()) # pyright: ignore[reportUnknownMemberType] responses = await target.send_prompt_async(message=request) return responses[0].get_value() diff --git a/rampart/attacks/__init__.py b/rampart/attacks/__init__.py index 5b12205b..1c4f93b1 100644 --- a/rampart/attacks/__init__.py +++ b/rampart/attacks/__init__.py @@ -12,7 +12,7 @@ from typing import TYPE_CHECKING from rampart.attacks._xpia import XPIAExecution -from rampart.drivers import _coerce_driver +from rampart.drivers._utils import coerce_driver if TYPE_CHECKING: from rampart.core.evaluator import Evaluator @@ -94,7 +94,7 @@ def xpia( handles = inject else: handles = [inject] - driver = _coerce_driver(trigger) + driver = coerce_driver(trigger) return XPIAExecution( handles=handles, diff --git a/rampart/core/llm.py b/rampart/core/llm.py index 489d93c2..a0588702 100644 --- a/rampart/core/llm.py +++ b/rampart/core/llm.py @@ -32,4 +32,4 @@ class LLMConfig: endpoint: str api_key: str | None = field(default=None, repr=False) deployment: str | None = None - metadata: dict[str, Any] = field(default_factory=dict) + metadata: dict[str, Any] = field(default_factory=dict[str, Any]) diff --git a/rampart/core/manifest.py b/rampart/core/manifest.py index 52ae9371..9b2cb91c 100644 --- a/rampart/core/manifest.py +++ b/rampart/core/manifest.py @@ -26,8 +26,8 @@ class ToolDeclaration: name: str description: str = "" - parameters: dict[str, Any] = field(default_factory=dict) - permissions: list[str] = field(default_factory=list) + parameters: dict[str, Any] = field(default_factory=dict[str, Any]) + permissions: list[str] = field(default_factory=list[str]) @dataclass(kw_only=True) @@ -63,10 +63,14 @@ class AppManifest: """ name: str - tools: list[ToolDeclaration] = field(default_factory=list) - data_sources: list[DataSource] = field(default_factory=list) + tools: list[ToolDeclaration] = field( + default_factory=list[ToolDeclaration], + ) + data_sources: list[DataSource] = field( + default_factory=list[DataSource], + ) description: str = "" - metadata: dict[str, Any] = field(default_factory=dict) + metadata: dict[str, Any] = field(default_factory=dict[str, Any]) def declares_tool(self, name: str) -> bool: """Check if a tool is declared in the manifest.""" @@ -87,7 +91,7 @@ def __str__(self) -> str: sections.append(self.description) if self.tools: - tool_lines = [] + tool_lines: list[str] = [] for t in self.tools: params = ", ".join(f"{k}: {v}" for k, v in t.parameters.items()) desc = f" — {t.description}" if t.description else "" @@ -96,7 +100,7 @@ def __str__(self) -> str: sections.append(f"Available tools:\n{tools}") if self.data_sources: - source_lines = [] + source_lines: list[str] = [] for ds in self.data_sources: writable = ( " (writable by untrusted users)" if ds.writable_by_untrusted else "" diff --git a/rampart/core/result.py b/rampart/core/result.py index 3ba63464..474507dd 100644 --- a/rampart/core/result.py +++ b/rampart/core/result.py @@ -122,14 +122,18 @@ class Result: safe: bool status: SafetyStatus summary: str - turns: list[Turn] = field(default_factory=list) - eval_results: list[EvalResult] = field(default_factory=list) + turns: list[Turn] = field(default_factory=list[Turn]) + eval_results: list[EvalResult] = field( + default_factory=list[EvalResult], + ) duration_seconds: float = 0.0 harm_category: HarmCategory | str | None = None strategy: str = "" observability_level: ObservabilityLevel = ObservabilityLevel.RESPONSE_ONLY - injections: list[InjectionRecord] = field(default_factory=list) - metadata: dict[str, Any] = field(default_factory=dict) + injections: list[InjectionRecord] = field( + default_factory=list[InjectionRecord], + ) + metadata: dict[str, Any] = field(default_factory=dict[str, Any]) def __bool__(self) -> bool: """Assert-safe: bool(result) means the agent behaved safely.""" diff --git a/rampart/core/types.py b/rampart/core/types.py index ee7ee916..e9ce72fd 100644 --- a/rampart/core/types.py +++ b/rampart/core/types.py @@ -115,7 +115,7 @@ class Payload: id: str = field(default_factory=lambda: uuid.uuid4().hex[:12]) format: PayloadFormat = PayloadFormat.TEXT artifact: Path | None = None - metadata: dict[str, Any] = field(default_factory=dict) + metadata: dict[str, Any] = field(default_factory=dict[str, Any]) def __post_init__(self) -> None: """Validate content-format-artifact consistency.""" @@ -165,7 +165,7 @@ class ToolCall: """ name: str - arguments: dict[str, Any] = field(default_factory=dict) + arguments: dict[str, Any] = field(default_factory=dict[str, Any]) result: str | None = None timestamp: datetime | None = None @@ -184,7 +184,7 @@ class SideEffect: """ kind: str - details: dict[str, Any] = field(default_factory=dict) + details: dict[str, Any] = field(default_factory=dict[str, Any]) @dataclass(kw_only=True) @@ -201,9 +201,9 @@ class Response: """ text: str - tool_calls: list[ToolCall] = field(default_factory=list) - side_effects: list[SideEffect] = field(default_factory=list) - metadata: dict[str, Any] = field(default_factory=dict) + tool_calls: list[ToolCall] = field(default_factory=list[ToolCall]) + side_effects: list[SideEffect] = field(default_factory=list[SideEffect]) + metadata: dict[str, Any] = field(default_factory=dict[str, Any]) @dataclass(kw_only=True) @@ -221,7 +221,7 @@ class Request: """ prompt: str | None = None - attachments: list[Payload] = field(default_factory=list) + attachments: list[Payload] = field(default_factory=list[Payload]) def __post_init__(self) -> None: """Validate that the request carries some content.""" @@ -280,7 +280,7 @@ class EvalResult: outcome: EvalOutcome confidence: float = 1.0 - evidence: list[str] = field(default_factory=list) + evidence: list[str] = field(default_factory=list[str]) rationale: str = "" @property @@ -304,7 +304,7 @@ class EvalContext: turns: list[Turn] manifest: AppManifest | None = None - metadata: dict[str, Any] = field(default_factory=dict) + metadata: dict[str, Any] = field(default_factory=dict[str, Any]) @property def current_turn(self) -> Turn: diff --git a/rampart/drivers/__init__.py b/rampart/drivers/__init__.py index f5666cd0..adcd77cd 100644 --- a/rampart/drivers/__init__.py +++ b/rampart/drivers/__init__.py @@ -1,46 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Driver implementations. +"""Driver implementations.""" -Re-exports StaticDriver and provides the _coerce_driver helper -for ergonomic prompt/driver coercion. -""" - -from __future__ import annotations - -from rampart.core.prompt_driver import PromptDriver -from rampart.core.types import Request from rampart.drivers.static import StaticDriver -__all__ = ["StaticDriver", "_coerce_driver"] - - -def _coerce_driver( - value: str | list[str] | Request | list[Request] | PromptDriver, -) -> PromptDriver: - """Coerce a string, Request, or list into a PromptDriver. - - Args: - value: A single prompt string, a list of prompt strings, - a single Request, a list of Requests, or an existing - PromptDriver. - - Returns: - PromptDriver: A driver wrapping the input. - """ - if isinstance(value, str): - return StaticDriver(prompts=[value]) - if isinstance(value, Request): - return StaticDriver(prompts=[value]) - if isinstance(value, list): - return StaticDriver(prompts=value) - if isinstance(value, PromptDriver): - return value - msg = ( - f"Cannot coerce {type(value).__name__} to PromptDriver. " - f"Expected str, list[str], Request, list[Request], or PromptDriver." - ) - raise TypeError( - msg, - ) +__all__ = ["StaticDriver"] diff --git a/rampart/drivers/_utils.py b/rampart/drivers/_utils.py new file mode 100644 index 00000000..fc889a0e --- /dev/null +++ b/rampart/drivers/_utils.py @@ -0,0 +1,28 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from rampart.core.prompt_driver import PromptDriver +from rampart.core.types import Request +from rampart.drivers.static import StaticDriver + + +def coerce_driver( + value: str | list[str] | Request | list[Request] | PromptDriver, +) -> PromptDriver: + """Coerce a string, Request, or list into a PromptDriver. + + Args: + value: A single prompt string, a list of prompt strings, + a single Request, a list of Requests, or an existing + PromptDriver. + + Returns: + PromptDriver: A driver wrapping the input. + """ + if isinstance(value, str): + return StaticDriver(prompts=[value]) + if isinstance(value, Request): + return StaticDriver(prompts=[value]) + if isinstance(value, list): + return StaticDriver(prompts=value) + return value diff --git a/rampart/evaluators/response_contains.py b/rampart/evaluators/response_contains.py index a7915bb1..006174d6 100644 --- a/rampart/evaluators/response_contains.py +++ b/rampart/evaluators/response_contains.py @@ -29,7 +29,7 @@ class ResponseContains(BaseEvaluator): def __init__( self, - target: str | re.Pattern | Callable[[str], bool], + target: str | re.Pattern[str] | Callable[[str], bool], /, *, case_sensitive: bool = False, @@ -42,7 +42,7 @@ async def evaluate_async(self, *, context: EvalContext) -> EvalResult: """Check response text for the target pattern.""" text = context.text - if callable(self._target) and not isinstance(self._target, re.Pattern): + if callable(self._target): found = self._target(text) elif isinstance(self._target, re.Pattern): found = bool(self._target.search(text)) diff --git a/rampart/evaluators/side_effect.py b/rampart/evaluators/side_effect.py index 7bac9467..eb61f287 100644 --- a/rampart/evaluators/side_effect.py +++ b/rampart/evaluators/side_effect.py @@ -5,22 +5,30 @@ from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any from rampart.core.evaluator import BaseEvaluator from rampart.core.types import EvalContext, EvalOutcome, EvalResult, SideEffect +if TYPE_CHECKING: + from collections.abc import Callable + class SideEffectOccurred(BaseEvaluator): """Detects whether a side effect of a given kind occurred. Args: kind (str): The side effect kind to look for (positional-only). - **detail_predicates (dict[str, Any]): - Detail field -> expected value or predicate. + **detail_predicates (dict[str, Any | Callable[[Any], bool]]): + Detail field -> expected value or callable predicate. """ - def __init__(self, kind: str, /, **detail_predicates: dict[str, Any]) -> None: + def __init__( + self, + kind: str, + /, + **detail_predicates: Any | Callable[[Any], bool], # noqa: ANN401 + ) -> None: """Initialize with side effect kind and optional predicates.""" self._kind = kind self._predicates = detail_predicates diff --git a/rampart/evaluators/tool_called.py b/rampart/evaluators/tool_called.py index 648f12d4..b21f7e5f 100644 --- a/rampart/evaluators/tool_called.py +++ b/rampart/evaluators/tool_called.py @@ -5,11 +5,14 @@ from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any from rampart.core.evaluator import BaseEvaluator from rampart.core.types import EvalContext, EvalOutcome, EvalResult, ToolCall +if TYPE_CHECKING: + from collections.abc import Callable + class ToolCalled(BaseEvaluator): """Detects whether a tool was called, optionally matching parameters. @@ -23,11 +26,16 @@ class ToolCalled(BaseEvaluator): Args: tool_name (str): The tool to look for (positional-only). - **param_predicates (dict[str, Any]): + **param_predicates (dict[str, Any | Callable[[Any], bool]]): Parameter name -> expected value or predicate. """ - def __init__(self, tool_name: str, /, **param_predicates: dict[str, Any]) -> None: + def __init__( + self, + tool_name: str, + /, + **param_predicates: Any | Callable[[Any], bool], # noqa: ANN401 + ) -> None: """Initialize with tool name and optional parameter predicates.""" self._tool_name = tool_name self._predicates = param_predicates diff --git a/rampart/payloads/template.py b/rampart/payloads/template.py index d81eedb0..b09f7cc6 100644 --- a/rampart/payloads/template.py +++ b/rampart/payloads/template.py @@ -74,7 +74,7 @@ class PayloadTemplate: description: str objective: str instruction: str - variables: dict[str, str] = field(default_factory=dict) + variables: dict[str, str] = field(default_factory=dict[str, str]) def with_variables(self, **overrides: str) -> PayloadTemplate: """Return a copy with updated variable values. diff --git a/rampart/probes/__init__.py b/rampart/probes/__init__.py index 7e8ccc5b..09bda1a5 100644 --- a/rampart/probes/__init__.py +++ b/rampart/probes/__init__.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, overload -from rampart.drivers import _coerce_driver +from rampart.drivers._utils import coerce_driver from rampart.probes._single_turn import SingleTurnExecution if TYPE_CHECKING: @@ -94,9 +94,9 @@ def behavior( # noqa: PLR0913 msg, ) if prompt is not None: - resolved_driver = _coerce_driver(prompt) + resolved_driver = coerce_driver(prompt) elif prompts is not None: - resolved_driver = _coerce_driver(prompts) + resolved_driver = coerce_driver(prompts) else: assert driver is not None # noqa: S101 — type narrowing resolved_driver = driver diff --git a/rampart/pytest_plugin/_session.py b/rampart/pytest_plugin/_session.py index a9da0893..715f3143 100644 --- a/rampart/pytest_plugin/_session.py +++ b/rampart/pytest_plugin/_session.py @@ -107,7 +107,7 @@ def add_sinks(self, *, sinks: list[ReportSink]) -> None: TypeError: If any item does not satisfy ReportSink. """ for sink in sinks: - if not isinstance(sink, ReportSink): + if not isinstance(sink, ReportSink): # pyright: ignore[reportUnnecessaryIsInstance] msg = ( f"Expected ReportSink, got {type(sink).__name__}. " "Sinks must implement: " diff --git a/rampart/pytest_plugin/plugin.py b/rampart/pytest_plugin/plugin.py index ff5dc31c..c7b5a67c 100644 --- a/rampart/pytest_plugin/plugin.py +++ b/rampart/pytest_plugin/plugin.py @@ -44,6 +44,7 @@ deactivate_collector, ) from rampart.pytest_plugin._session import RampartSession +from rampart.reporting.sink import ReportSink if TYPE_CHECKING: from collections.abc import Generator @@ -215,7 +216,6 @@ def _create_trial_clones( for i in range(count): trial_name = f"{display_name}[trial-{i}]" from_parent_kwargs: dict[str, Any] = { - "parent": parent, "name": trial_name, "originalname": original_name, } @@ -224,9 +224,9 @@ def _create_trial_clones( if fixtureinfo is not None: from_parent_kwargs["fixtureinfo"] = fixtureinfo - clone = type(item).from_parent(**from_parent_kwargs) # type: ignore[arg-type] - clone._rampart_trial_index = i # type: ignore[attr-defined] # noqa: SLF001 - clone._rampart_trial_base = item.nodeid # type: ignore[attr-defined] # noqa: SLF001 + clone = type(item).from_parent(parent=parent, **from_parent_kwargs) # pyright: ignore[reportUnknownMemberType] + clone._rampart_trial_index = i # pyright: ignore[reportAttributeAccessIssue] # noqa: SLF001 + clone._rampart_trial_base = item.nodeid # pyright: ignore[reportAttributeAccessIssue] # noqa: SLF001 _copy_markers_to_clone(source=item, clone=clone) clone.add_marker( @@ -308,7 +308,7 @@ def _absorb_results( @pytest.fixture(autouse=True) -def _rampart_collect( # type: ignore[reportUnusedFunction] # pytest discovers this via autouse=True +def _rampart_collect( # pyright: ignore[reportUnusedFunction] # pytest discovers this via autouse=True request: pytest.FixtureRequest, ) -> Generator[None, None, None]: """Installed automatically on every test. Invisible to test authors. @@ -326,7 +326,7 @@ def _rampart_collect( # type: ignore[reportUnusedFunction] # pytest discovers No test author ever imports or references this fixture. """ collector = ResultCollector() - node = cast("pytest.Item", request.node) + node = cast("pytest.Item", request.node) # pyright: ignore[reportUnknownMemberType] rampart_session = request.config.stash.get(_rampart_key, None) token = activate_collector(collector) yield @@ -348,7 +348,7 @@ def _rampart_collect( # type: ignore[reportUnusedFunction] # pytest discovers @pytest.fixture(scope="session", autouse=True) -def _rampart_sink_bootstrap( # type: ignore[reportUnusedFunction] # pytest discovers this via autouse=True +def _rampart_sink_bootstrap( # pyright: ignore[reportUnusedFunction] # pytest discovers this via autouse=True request: pytest.FixtureRequest, ) -> None: """Merge team-provided sinks into the RAMPART session. @@ -382,6 +382,17 @@ def rampart_sinks(): ) return + user_sinks = cast("list[object]", user_sinks) + + if not all(isinstance(x, ReportSink) for x in user_sinks): + logger.warning( + "rampart_sinks fixture must return list[ReportSink], " + "got list with non-ReportSink items. Ignoring.", + ) + return + + user_sinks = cast("list[ReportSink]", user_sinks) + rampart_session.add_sinks(sinks=user_sinks) logger.info( "Loaded %d sink(s) from rampart_sinks fixture.", diff --git a/rampart/reporting/sink.py b/rampart/reporting/sink.py index 5428570f..ff614702 100644 --- a/rampart/reporting/sink.py +++ b/rampart/reporting/sink.py @@ -63,14 +63,14 @@ class TestRunReport: __test__ = False # Prevent pytest from collecting this dataclass as a test. - results: list[Result] = field(default_factory=list) + results: list[Result] = field(default_factory=list[Result]) total_runs: int = 0 passed: int = 0 failed: int = 0 undetermined: int = 0 errors: int = 0 duration_seconds: float = 0.0 - metadata: dict[str, Any] = field(default_factory=dict) + metadata: dict[str, Any] = field(default_factory=dict[str, Any]) def by_harm_category(self) -> dict[str, list[Result]]: """Group results by harm category. diff --git a/tests/fixtures.py b/tests/fixtures.py index 6b4b4c13..f8bb703c 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -9,7 +9,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Self +from typing import TYPE_CHECKING, Any, Self, cast from rampart.core.types import ObservabilityLevel, Request, Response @@ -96,10 +96,14 @@ def __init__( if not responses: raise ValueError("MockAdapter requires at least one response sequence.") - if isinstance(responses[0], list): - self._session_responses: list[list[Response]] = responses # type: ignore[assignment] + if all(isinstance(r, list) for r in responses) and all( + isinstance(r, Response) + for sublist in cast("list[list[Any]]", responses) + for r in sublist + ): + self._session_responses = cast("list[list[Response]]", responses) else: - self._session_responses = [responses] # type: ignore[list-item] + self._session_responses = [cast("list[Response]", responses)] self._manifest_value = manifest self._observability_profile_value = observability_profile self._session_index = 0 diff --git a/tests/unit/_pyrit/test_llm_bridge.py b/tests/unit/_pyrit/test_llm_bridge.py index b41da3c5..0333b683 100644 --- a/tests/unit/_pyrit/test_llm_bridge.py +++ b/tests/unit/_pyrit/test_llm_bridge.py @@ -11,7 +11,7 @@ import ast import importlib.util -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest @@ -27,7 +27,7 @@ class TestModelNameResolution: """LLMConfig.model and .deployment map to PyRIT's model_name / underlying_model.""" @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") - def test_model_becomes_model_name_without_deployment(self, mock_cls): + def test_model_becomes_model_name_without_deployment(self, mock_cls: Mock): create_prompt_target( LLMConfig( model="gpt-4o", @@ -41,7 +41,10 @@ def test_model_becomes_model_name_without_deployment(self, mock_cls): assert kwargs["underlying_model"] is None @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") - def test_deployment_becomes_model_name_with_model_as_underlying(self, mock_cls): + def test_deployment_becomes_model_name_with_model_as_underlying( + self, + mock_cls: Mock, + ): create_prompt_target( LLMConfig( model="gpt-4o", @@ -60,7 +63,7 @@ class TestEndpointAndAuth: """Endpoint and api_key are forwarded directly to PyRIT.""" @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") - def test_endpoint_forwarded(self, mock_cls): + def test_endpoint_forwarded(self, mock_cls: Mock): create_prompt_target( LLMConfig( model="gpt-4o", @@ -72,7 +75,7 @@ def test_endpoint_forwarded(self, mock_cls): assert mock_cls.call_args.kwargs["endpoint"] == "https://custom.endpoint.com/v1" @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") - def test_api_key_forwarded(self, mock_cls): + def test_api_key_forwarded(self, mock_cls: Mock): create_prompt_target( LLMConfig( model="gpt-4o", @@ -84,7 +87,7 @@ def test_api_key_forwarded(self, mock_cls): assert mock_cls.call_args.kwargs["api_key"] == "sk-secret" @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") - def test_none_api_key_forwarded_for_entra_auth(self, mock_cls): + def test_none_api_key_forwarded_for_entra_auth(self, mock_cls: Mock): """None api_key lets PyRIT use Entra ID auth for Azure endpoints.""" create_prompt_target( LLMConfig( @@ -100,7 +103,7 @@ class TestMetadataForwarding: """Recognised model parameters in metadata are forwarded; unknown keys are not.""" @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") - def test_temperature_and_top_p_forwarded(self, mock_cls): + def test_temperature_and_top_p_forwarded(self, mock_cls: Mock): create_prompt_target( LLMConfig( model="gpt-4o", @@ -115,7 +118,7 @@ def test_temperature_and_top_p_forwarded(self, mock_cls): assert kwargs["top_p"] == 0.9 @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") - def test_all_recognised_params_forwarded(self, mock_cls): + def test_all_recognised_params_forwarded(self, mock_cls: Mock): meta = { "temperature": 0.5, "top_p": 0.8, @@ -141,7 +144,7 @@ def test_all_recognised_params_forwarded(self, mock_cls): assert kwargs[key] == value, f"metadata[{key!r}] not forwarded" @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") - def test_unknown_metadata_keys_not_forwarded(self, mock_cls): + def test_unknown_metadata_keys_not_forwarded(self, mock_cls: Mock): create_prompt_target( LLMConfig( model="gpt-4o", @@ -156,7 +159,7 @@ def test_unknown_metadata_keys_not_forwarded(self, mock_cls): assert kwargs["temperature"] == 0.5 @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") - def test_empty_metadata_adds_no_extra_kwargs(self, mock_cls): + def test_empty_metadata_adds_no_extra_kwargs(self, mock_cls: Mock): create_prompt_target( LLMConfig( model="gpt-4o", @@ -179,7 +182,7 @@ class TestReturnValue: """create_prompt_target returns the constructed target.""" @patch("rampart._pyrit.llm_bridge.OpenAIChatTarget") - def test_returns_constructed_target(self, mock_cls): + def test_returns_constructed_target(self, mock_cls: Mock): result = create_prompt_target( LLMConfig( model="gpt-4o", @@ -218,17 +221,17 @@ def test_empty_endpoint_raises_value_error(self): ) def test_none_model_raises_value_error(self): - config = LLMConfig( # type: ignore[arg-type] - model=None, + config = LLMConfig( + model=None, # pyright: ignore[reportArgumentType] endpoint="https://api.openai.com/v1", ) with pytest.raises(ValueError, match="model"): create_prompt_target(config) def test_none_endpoint_raises_value_error(self): - config = LLMConfig( # type: ignore[arg-type] + config = LLMConfig( model="gpt-4o", - endpoint=None, + endpoint=None, # pyright: ignore[reportArgumentType] ) with pytest.raises(ValueError, match="endpoint"): create_prompt_target(config) diff --git a/tests/unit/core/test_converter.py b/tests/unit/core/test_converter.py index 2a839776..bcb3223a 100644 --- a/tests/unit/core/test_converter.py +++ b/tests/unit/core/test_converter.py @@ -35,20 +35,6 @@ async def convert_async(self, *, payload: Payload) -> Payload: ) -class _FormatChangingConverter: - """Test converter that produces a binary-format payload.""" - - async def convert_async(self, *, payload: Payload) -> Payload: - fake_path = Path("/tmp/fake.png") - return Payload( - content=payload.content, - id=payload.id, - format=PayloadFormat.IMAGE, - artifact=fake_path, - metadata={**payload.metadata, "converter": "FormatChangingConverter"}, - ) - - class TestPayloadConverterProtocol: def test_converter_satisfies_protocol(self) -> None: assert isinstance(_UpperCaseConverter(), PayloadConverter) @@ -102,7 +88,7 @@ async def test_converters_compose_sequentially(self) -> None: assert result.format is PayloadFormat.HTML @pytest.mark.asyncio - async def test_format_converter_preserves_content(self, tmp_path) -> None: + async def test_format_converter_preserves_content(self, tmp_path: Path) -> None: fake_file = tmp_path / "fake.png" fake_file.write_bytes(b"\x89PNG") diff --git a/tests/unit/core/test_execution.py b/tests/unit/core/test_execution.py index 19197306..811ad3bf 100644 --- a/tests/unit/core/test_execution.py +++ b/tests/unit/core/test_execution.py @@ -269,4 +269,4 @@ def test_register_rejects_non_callable(self) -> None: from rampart.core.execution import register_default_handler_factory with pytest.raises(TypeError, match="callable"): - register_default_handler_factory("not a function") # type: ignore[arg-type] + register_default_handler_factory("not a function") # pyright: ignore[reportArgumentType] diff --git a/tests/unit/core/test_llm.py b/tests/unit/core/test_llm.py index 6b3a7cdf..22feac61 100644 --- a/tests/unit/core/test_llm.py +++ b/tests/unit/core/test_llm.py @@ -38,15 +38,15 @@ def test_full_construction(self): def test_keyword_only(self): with pytest.raises(TypeError): - LLMConfig("gpt-4o", "https://api.example.com") # type: ignore[misc] + LLMConfig("gpt-4o", "https://api.example.com") # pyright: ignore[reportCallIssue] def test_requires_model(self): with pytest.raises(TypeError): - LLMConfig(endpoint="https://api.example.com") # type: ignore[call-arg] + LLMConfig(endpoint="https://api.example.com") # pyright: ignore[reportCallIssue] def test_requires_endpoint(self): with pytest.raises(TypeError): - LLMConfig(model="gpt-4o") # type: ignore[call-arg] + LLMConfig(model="gpt-4o") # pyright: ignore[reportCallIssue] class TestLLMConfigImmutability: @@ -55,12 +55,12 @@ class TestLLMConfigImmutability: def test_cannot_set_field(self): cfg = LLMConfig(model="gpt-4o", endpoint="https://api.example.com") with pytest.raises(AttributeError): - cfg.model = "gpt-4" # type: ignore[misc] + cfg.model = "gpt-4" # pyright: ignore[reportAttributeAccessIssue] def test_cannot_delete_field(self): cfg = LLMConfig(model="gpt-4o", endpoint="https://api.example.com") with pytest.raises(AttributeError): - del cfg.model # type: ignore[misc] + del cfg.model # pyright: ignore[reportAttributeAccessIssue] class TestLLMConfigEquality: diff --git a/tests/unit/drivers/test_coerce.py b/tests/unit/drivers/test_coerce.py index 6b945306..5803769c 100644 --- a/tests/unit/drivers/test_coerce.py +++ b/tests/unit/drivers/test_coerce.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Tests for _coerce_driver.""" +"""Tests for coerce_driver.""" from __future__ import annotations @@ -9,16 +9,16 @@ from rampart.core.prompt_driver import PromptDecision from rampart.core.types import Request, Response, Turn -from rampart.drivers import _coerce_driver +from rampart.drivers._utils import coerce_driver from rampart.drivers.static import StaticDriver class TestCoerceString: - """_coerce_driver accepts a str as a single-prompt driver.""" + """coerce_driver accepts a str as a single-prompt driver.""" @pytest.mark.asyncio async def test_str_produces_single_prompt_driver_async(self) -> None: - driver = _coerce_driver("hello") + driver = coerce_driver("hello") d0 = await driver.next_prompt_async(history=[]) assert d0 is not None @@ -33,11 +33,11 @@ async def test_str_produces_single_prompt_driver_async(self) -> None: class TestCoerceList: - """_coerce_driver accepts list[str] as a multi-prompt driver.""" + """coerce_driver accepts list[str] as a multi-prompt driver.""" @pytest.mark.asyncio async def test_list_produces_multi_prompt_driver_async(self) -> None: - driver = _coerce_driver(["a", "b"]) + driver = coerce_driver(["a", "b"]) d0 = await driver.next_prompt_async(history=[]) assert d0 is not None @@ -45,12 +45,12 @@ async def test_list_produces_multi_prompt_driver_async(self) -> None: class TestCoercePassthrough: - """_coerce_driver passes through an existing PromptDriver unchanged.""" + """coerce_driver passes through an existing PromptDriver unchanged.""" @pytest.mark.asyncio async def test_prompt_driver_passthrough_async(self) -> None: original = StaticDriver(prompts=["x"]) - result = _coerce_driver(original) + result = coerce_driver(original) assert result is original @pytest.mark.asyncio @@ -64,5 +64,5 @@ async def next_prompt_async( return PromptDecision(request=Request(prompt="custom")) custom = Custom() - result = _coerce_driver(custom) # type: ignore[arg-type] + result = coerce_driver(custom) assert result is custom diff --git a/tests/unit/payloads/test_generator.py b/tests/unit/payloads/test_generator.py index d86a4952..e64564d3 100644 --- a/tests/unit/payloads/test_generator.py +++ b/tests/unit/payloads/test_generator.py @@ -70,9 +70,9 @@ async def test_returns_one_variant_per_call(self) -> None: @pytest.mark.asyncio async def test_resolves_template_variables(self) -> None: - captured_args: dict = {} + captured_args: dict[str, str] = {} - async def capture(*, system_message, user_message): + async def capture(*, system_message: str, user_message: str): captured_args["user_message"] = user_message return "single variant" @@ -122,9 +122,9 @@ async def test_strips_whitespace(self) -> None: @pytest.mark.asyncio async def test_includes_objective_in_prompt(self) -> None: - captured_args: dict = {} + captured_args: dict[str, str] = {} - async def capture(*, system_message, user_message): + async def capture(*, system_message: str, user_message: str) -> str: captured_args["user_message"] = user_message return "variant" diff --git a/tests/unit/payloads/test_payloads.py b/tests/unit/payloads/test_payloads.py index 192cab2b..1b8b0935 100644 --- a/tests/unit/payloads/test_payloads.py +++ b/tests/unit/payloads/test_payloads.py @@ -124,7 +124,7 @@ async def test_provenance_metadata_on_payloads(self) -> None: @pytest.mark.asyncio async def test_manifest_reaches_llm_prompt(self) -> None: """Manifest tools and agent name appear in the LLM user message.""" - captured: dict = {} + captured: dict[str, str] = {} async def capture(*, system_message: str, user_message: str) -> str: captured["user_message"] = user_message @@ -151,7 +151,7 @@ async def capture(*, system_message: str, user_message: str) -> str: @pytest.mark.asyncio async def test_persona_becomes_system_message(self) -> None: """Persona system_prompt is forwarded as the LLM system message.""" - captured: dict = {} + captured: dict[str, str] = {} async def capture(*, system_message: str, user_message: str) -> str: captured["system_message"] = system_message diff --git a/tests/unit/payloads/test_store.py b/tests/unit/payloads/test_store.py index 29ef73ec..30e3cdd7 100644 --- a/tests/unit/payloads/test_store.py +++ b/tests/unit/payloads/test_store.py @@ -4,6 +4,7 @@ """Tests for rampart.payloads._store — PayloadStore persistence.""" import json +from pathlib import Path import pytest @@ -12,17 +13,17 @@ @pytest.fixture -def store(tmp_path): +def store(tmp_path: Path) -> PayloadStore: """PayloadStore rooted in a temporary directory.""" return PayloadStore(root=tmp_path) class TestPayloadStoreSave: - def test_save_empty_raises(self, store) -> None: + def test_save_empty_raises(self, store: PayloadStore) -> None: with pytest.raises(ValueError, match="empty"): store.save("col1", payloads=[]) - def test_save_stores_provenance(self, store, tmp_path) -> None: + def test_save_stores_provenance(self, store: PayloadStore, tmp_path: Path) -> None: payloads = [Payload(content="test", id="p1")] store.save( "col1", @@ -35,14 +36,18 @@ def test_save_stores_provenance(self, store, tmp_path) -> None: assert manifest["provenance"]["template"] == "email_exfil" assert manifest["count"] == 1 - def test_save_overwrites_existing(self, store) -> None: + def test_save_overwrites_existing(self, store: PayloadStore) -> None: store.save("col1", payloads=[Payload(content="old", id="p1")]) store.save("col1", payloads=[Payload(content="new", id="p2")]) loaded = store.load("col1") assert len(loaded) == 1 assert loaded[0].content == "new" - def test_save_binary_creates_artifact(self, store, tmp_path) -> None: + def test_save_binary_creates_artifact( + self, + store: PayloadStore, + tmp_path: Path, + ) -> None: source_file = tmp_path / "input.png" source_file.write_bytes(b"\x89PNG") payload = Payload( @@ -58,7 +63,7 @@ def test_save_binary_creates_artifact(self, store, tmp_path) -> None: class TestPayloadStoreLoad: - def test_load_roundtrip_text(self, store) -> None: + def test_load_roundtrip_text(self, store: PayloadStore) -> None: original = Payload( content="evil stuff", id="t1", @@ -73,7 +78,7 @@ def test_load_roundtrip_text(self, store) -> None: assert loaded[0].metadata["persona"] == "stealth" assert loaded[0].artifact is None - def test_load_roundtrip_binary(self, store, tmp_path) -> None: + def test_load_roundtrip_binary(self, store: PayloadStore, tmp_path: Path) -> None: source_file = tmp_path / "input.pdf" source_file.write_bytes(b"\x00\x01\x02") original = Payload( @@ -89,11 +94,11 @@ def test_load_roundtrip_binary(self, store, tmp_path) -> None: assert loaded[0].artifact is not None assert loaded[0].artifact.read_bytes() == b"\x00\x01\x02" - def test_load_missing_raises(self, store) -> None: + def test_load_missing_raises(self, store: PayloadStore) -> None: with pytest.raises(FileNotFoundError, match="not found"): store.load("nonexistent") - def test_load_with_format_filter(self, store) -> None: + def test_load_with_format_filter(self, store: PayloadStore) -> None: payloads = [ Payload(content="text", id="t1", format=PayloadFormat.TEXT), Payload(content="html", id="h1", format=PayloadFormat.HTML), @@ -105,18 +110,18 @@ def test_load_with_format_filter(self, store) -> None: class TestPayloadStoreCollectionManagement: - def test_list_collections(self, store) -> None: + def test_list_collections(self, store: PayloadStore) -> None: store.save("alpha", payloads=[Payload(content="x", id="p1")]) store.save("beta", payloads=[Payload(content="y", id="p2")]) collections = store.list_collections() assert collections == ["alpha", "beta"] - def test_delete_removes_collection(self, store) -> None: + def test_delete_removes_collection(self, store: PayloadStore) -> None: store.save("doomed", payloads=[Payload(content="x", id="p1")]) store.delete("doomed") assert not store.exists("doomed") - def test_manifest_roundtrip(self, store) -> None: + def test_manifest_roundtrip(self, store: PayloadStore) -> None: store.save( "col1", payloads=[Payload(content="x", id="p1")], @@ -127,13 +132,17 @@ def test_manifest_roundtrip(self, store) -> None: assert m["count"] == 1 assert m["provenance"]["template"] == "email_exfiltration" - def test_manifest_missing_raises(self, store) -> None: + def test_manifest_missing_raises(self, store: PayloadStore) -> None: with pytest.raises(FileNotFoundError, match="No manifest"): store.manifest("ghost") class TestPayloadStorePathPayload: - def test_path_based_payload_roundtrip(self, store, tmp_path) -> None: + def test_path_based_payload_roundtrip( + self, + store: PayloadStore, + tmp_path: Path, + ) -> None: source_file = tmp_path / "source.pdf" source_file.write_bytes(b"PDF_CONTENT") payload = Payload( diff --git a/tests/unit/probes/test_single_turn.py b/tests/unit/probes/test_single_turn.py index e9f46606..290553e0 100644 --- a/tests/unit/probes/test_single_turn.py +++ b/tests/unit/probes/test_single_turn.py @@ -155,7 +155,7 @@ class TestProbeParameterValidation: def test_both_prompt_and_driver_raises(self) -> None: with pytest.raises(ValueError, match="exactly one"): - Probes.behavior( + Probes.behavior( # pyright: ignore[reportCallIssue] prompt="hello", driver=StaticDriver(prompts=["driven"]), evaluator=_DetectsAlways(), @@ -163,7 +163,7 @@ def test_both_prompt_and_driver_raises(self) -> None: def test_both_prompt_and_prompts_raises(self) -> None: with pytest.raises(ValueError, match="exactly one"): - Probes.behavior( + Probes.behavior( # pyright: ignore[reportCallIssue] prompt="hello", prompts=["a", "b"], evaluator=_DetectsAlways(), @@ -171,7 +171,7 @@ def test_both_prompt_and_prompts_raises(self) -> None: def test_no_source_raises(self) -> None: with pytest.raises(ValueError, match="exactly one"): - Probes.behavior(evaluator=_DetectsAlways()) + Probes.behavior(evaluator=_DetectsAlways()) # pyright: ignore[reportCallIssue] class TestProbeInfrastructureError: diff --git a/tests/unit/pytest_plugin/test_plugin.py b/tests/unit/pytest_plugin/test_plugin.py index 2e344f4e..e929e2ca 100644 --- a/tests/unit/pytest_plugin/test_plugin.py +++ b/tests/unit/pytest_plugin/test_plugin.py @@ -196,7 +196,7 @@ def test_record_trial_group(self) -> None: assert group.unsafe == 2 assert group.errors == 1 assert group.threshold == 0.3 - assert group.pass_rate == pytest.approx(0.4) + assert group.pass_rate == pytest.approx(0.4) # pyright: ignore[reportUnknownMemberType] assert not group.passed # UNSAFE present → always fails def test_record_trial_group_all_errors(self) -> None: @@ -570,7 +570,7 @@ class NotASink: pass with pytest.raises(TypeError, match="Expected ReportSink"): - session.add_sinks(sinks=[NotASink()]) # type: ignore[list-item] + session.add_sinks(sinks=[NotASink()]) # pyright: ignore[reportArgumentType] def test_add_sinks_preserves_existing(self) -> None: """Config-loaded sinks are not lost when fixture sinks are added.""" @@ -713,7 +713,10 @@ class TestSessionFinishIntegration: def test_sets_duration(self) -> None: import time - from rampart.pytest_plugin.plugin import _rampart_key, _session_start_key + from rampart.pytest_plugin.plugin import ( + _rampart_key, + _session_start_key, + ) session_mock = MagicMock() config_stash = _StashStub() diff --git a/tests/unit/reporting/test_json_file.py b/tests/unit/reporting/test_json_file.py index 242614b2..8de57189 100644 --- a/tests/unit/reporting/test_json_file.py +++ b/tests/unit/reporting/test_json_file.py @@ -7,6 +7,7 @@ import json from pathlib import Path +from typing import Any import pytest @@ -18,8 +19,8 @@ def _result_with_turns( *, - response_metadata: dict | None = None, - result_metadata: dict | None = None, + response_metadata: dict[str, Any] | None = None, + result_metadata: dict[str, Any] | None = None, ) -> Result: """Build a Result carrying turns with optional response metadata.""" response = Response( diff --git a/tests/unit/reporting/test_report.py b/tests/unit/reporting/test_report.py index 0cc5e8c2..3e959380 100644 --- a/tests/unit/reporting/test_report.py +++ b/tests/unit/reporting/test_report.py @@ -164,8 +164,8 @@ def test_mixed_results(self) -> None: assert stats.safe_count == 1 assert stats.unsafe_count == 1 assert stats.undetermined_count == 1 - assert stats.attack_success_rate == pytest.approx(1 / 3) - assert stats.safety_pass_rate == pytest.approx(1 / 3) + assert stats.attack_success_rate == pytest.approx(1 / 3) # pyright: ignore[reportUnknownMemberType] + assert stats.safety_pass_rate == pytest.approx(1 / 3) # pyright: ignore[reportUnknownMemberType] def test_empty_results(self) -> None: report = TestRunReport() @@ -186,8 +186,8 @@ def test_error_excluded_from_attack_success_rate(self) -> None: stats = report.population_summary() assert stats.total_runs == 3 assert stats.error_count == 1 - assert stats.attack_success_rate == pytest.approx(1 / 2) - assert stats.safety_pass_rate == pytest.approx(1 / 2) + assert stats.attack_success_rate == pytest.approx(1 / 2) # pyright: ignore[reportUnknownMemberType] + assert stats.safety_pass_rate == pytest.approx(1 / 2) # pyright: ignore[reportUnknownMemberType] def test_all_errors(self) -> None: report = TestRunReport(