[FEAT]: LLM-based Prompt Driver

.github/copilot-instructions.md

@@ -16,7 +16,7 @@ RAMPART is organized as a modular framework with these main components:
 - **Payloads** (`rampart/payloads/`) — Payload generation, storage, and templating.
 - **Reporting** (`rampart/reporting/`) — Test result reporting (JSON file sink).
 - **Pytest Plugin** (`rampart/pytest_plugin/`) — Native pytest integration for test collection and session management.
-- **PyRIT Bridge** (`rampart/_pyrit/`) — Isolated boundary for all PyRIT framework interaction (see coding standards for import rules).
+- **PyRIT Bridge** (`rampart/pyrit_bridge/`) — Isolated boundary for all PyRIT framework interaction (see coding standards for import rules).
 
 ## Instruction Files
 

.github/instructions/coding-standards.instructions.md

@@ -554,10 +554,10 @@ logger.warning("Cleanup error during %s: %s", self.name, exc, exc_info=True)
 logger.info(f"Saved {len(payloads)} payloads to '{name}'")
 ```
 
-## PyRIT Boundary Isolation
+## PyRIT Bridge
 
-- **All PyRIT interaction MUST be isolated to `rampart/_pyrit/`**
-- Do NOT import PyRIT modules from anywhere else in the codebase (except `rampart/converters/` for converter wrappers)
+- Prefer grouping PyRIT-related logic under `rampart/pyrit_bridge/` to keep a clear boundary between RAMPART and PyRIT internals
+- PyRIT imports are allowed anywhere in the codebase when needed
 - PyRIT's import chain is heavy (~14s) — use lazy imports inside functions when wrapping PyRIT converters to defer the cost
 
 ```python
@@ -584,7 +584,7 @@ Before committing code, ensure:
 - [ ] Complex logic is extracted to helper methods
 - [ ] Copyright header is present
 - [ ] Log calls use `%s`-style formatting (no f-strings)
-- [ ] PyRIT imports are isolated to `rampart/_pyrit/` (or lazy in converters)
+- [ ] PyRIT logic is grouped under `rampart/pyrit_bridge/` where practical
 
 ---
 

.github/workflows/coverage.yml

@@ -40,15 +40,18 @@ jobs:
 
       - name: Run tests with coverage
         id: tests
-        # --cov-fail-under=0 overrides pyproject.toml [tool.coverage.report] fail_under
-        # so pytest doesn't exit non-zero on low coverage; threshold is checked separately below.
-        run: uv run pytest tests/unit --cov=rampart --cov-report=term-missing --cov-fail-under=0
+        # Use 'coverage run' instead of 'pytest --cov' so that coverage
+        # starts BEFORE pytest loads plugins. rampart registers a pytest
+        # plugin via the pytest11 entry point — if coverage starts after
+        # plugin loading, all module-level code imported during plugin
+        # setup is invisible to the tracer.
+        run: uv run coverage run -m pytest tests/unit -q
 
       - name: Coverage summary
         if: ${{ steps.tests.outcome == 'success' }}
         run: |
           echo '## Coverage Report' >> $GITHUB_STEP_SUMMARY
-          uv run coverage report --format=markdown >> $GITHUB_STEP_SUMMARY
+          uv run coverage report --format=markdown --fail-under=0 >> $GITHUB_STEP_SUMMARY
 
       - name: Check coverage threshold
         if: ${{ steps.tests.outcome == 'success' }}

pyproject.toml

@@ -46,6 +46,11 @@ dev = [
     "pytest-xdist[psutil]>=3.8.0",
     "ruff>=0.15.10",
 ]
+docs = [
+    "mkdocs>=1.6",
+    "mkdocs-material>=9.5",
+    "mkdocstrings[python]>=0.27",
+]
 
 [project.urls]
 Homepage = "https://github.com/microsoft/RAMPART"
@@ -55,6 +60,9 @@ Issues = "https://github.com/microsoft/RAMPART/issues"
 [project.entry-points.pytest11]
 rampart = "rampart.pytest_plugin.plugin"
 
+[tool.setuptools.package-data]
+rampart = ["drivers/prompts/*.yaml"]
+
 [tool.coverage.run]
 source = ["rampart"]
 omit = ["tests/*"]
@@ -115,5 +123,8 @@ known-first-party = ["rampart"]
 [tool.ruff.lint.pydocstyle]
 convention = "google"
 
+[tool.ruff.lint.pylint]
+max-args = 10
+
 [tool.uv.sources]
 pyrit = { git = "https://github.com/microsoft/PyRIT", rev = "6dc8b94139757390286bbce7d53c1f7e58e66e29" } # v0.13.0

rampart/__init__.py

@@ -8,7 +8,7 @@
 
 from rampart.attacks import Attacks
 from rampart.core.adapter import AgentAdapter, Session
-from rampart.core.errors import InfrastructureError
+from rampart.core.errors import DriverError, InfrastructureError
 from rampart.core.evaluator import BaseEvaluator, Evaluator
 from rampart.core.execution import (
     BaseExecution,
@@ -41,6 +41,7 @@
     ToolCall,
     Turn,
 )
+from rampart.drivers.llm import LLMDriver
 from rampart.probes import Probes
 from rampart.pytest_plugin._collection import record_result
 
@@ -51,6 +52,7 @@
     "BaseEvaluator",
     "BaseExecution",
     "DataSource",
+    "DriverError",
     "EvalContext",
     "EvalOutcome",
     "EvalResult",
@@ -62,6 +64,7 @@
     "InfrastructureError",
     "InjectionHandle",
     "InjectionRecord",
+    "LLMDriver",
     "ObservabilityLevel",
     "Payload",
     "PayloadFormat",

rampart/attacks/__init__.py

@@ -47,7 +47,7 @@ def xpia(
         inject: InjectionHandle | list[InjectionHandle] | None = None,
         trigger: str | list[str] | Request | list[Request] | PromptDriver,
         evaluator: Evaluator,
-        max_turns: int = 25,
+        max_turns: int = 5,
         event_handlers: list[ExecutionEventHandler] | None = None,
     ) -> BaseExecution:
         """Create an XPIA attack execution.
@@ -80,7 +80,7 @@ def xpia(
                 poisoned content.
             evaluator (Evaluator): What condition to check for.
             max_turns (int): Maximum prompt-response exchanges before
-                ERROR.  Defaults to 25.
+                ERROR.  Defaults to 5.
             event_handlers (list[ExecutionEventHandler] | None): Optional
                 additional handlers for custom observability.
 

rampart/attacks/_xpia.py

@@ -20,7 +20,6 @@
 from rampart.core import (
     AgentAdapter,
     BaseExecution,
-    EvalContext,
     EvalResult,
     Evaluator,
     ExecutionEventHandler,
@@ -33,6 +32,7 @@
     Turn,
     resolve_as_attack,
 )
+from rampart.core.execution import evaluate_turn_async
 
 logger = logging.getLogger(__name__)
 
@@ -96,8 +96,7 @@ async def _execute_async(self, *, adapter: AgentAdapter) -> Result:
         """Orchestrate the XPIA lifecycle and return a safety Result.
 
         Delegates phase execution to ``_run_phases_async`` and result
-        construction to ``_build_attack_result`` or
-        ``_max_turns_error_result``.
+        construction to ``_build_attack_result``.
 
         InfrastructureError is NOT caught here — it propagates to
         ``BaseExecution.execute_async``.
@@ -108,36 +107,23 @@ async def _execute_async(self, *, adapter: AgentAdapter) -> Result:
         Returns:
             Result: Safety verdict with full conversation evidence.
         """
-        turns, eval_results, max_turns_hit = await self._run_phases_async(
-            adapter=adapter,
-        )
-        if max_turns_hit:
-            return self._max_turns_error_result(
-                adapter=adapter,
-                turns=turns,
-                eval_results=eval_results,
-            )
-        return self._build_attack_result(
-            adapter=adapter,
-            turns=turns,
-            eval_results=eval_results,
-        )
+        turns = await self._run_phases_async(adapter=adapter)
+        return self._build_attack_result(adapter=adapter, turns=turns)
 
     async def _run_phases_async(
         self,
         *,
         adapter: AgentAdapter,
-    ) -> tuple[list[Turn], list[EvalResult], bool]:
+    ) -> list[Turn]:
         """Run XPIA phases 1-5 inside a cleanup-guaranteed context.
 
         Args:
             adapter (AgentAdapter): The agent adapter.
 
         Returns:
-            tuple: (turns, eval_results, max_turns_exceeded).
+            list[Turn]: Completed turns with eval_result populated.
         """
         turns: list[Turn] = []
-        eval_results: list[EvalResult] = []
 
         async with AsyncExitStack() as stack:
             await self._activate_handles_async(stack=stack)
@@ -150,31 +136,22 @@ async def _run_phases_async(
                 if decision is None:
                     break
 
-                request = decision.request
-                response = await session.send_async(request)
-                turns.append(
-                    Turn(
-                        request=request,
-                        response=response,
-                        turn_number=turn_index,
-                        driver_reasoning=decision.reasoning,
-                    ),
-                )
-
-                eval_result = await self._evaluator.evaluate_async(
-                    context=EvalContext(
-                        turns=list(turns),
-                        manifest=adapter.manifest,
-                    ),
+                response = await session.send_async(decision.request)
+                turn = await evaluate_turn_async(
+                    evaluator=self._evaluator,
+                    history=turns,
+                    request=decision.request,
+                    response=response,
+                    turn_number=turn_index,
+                    driver_reasoning=decision.reasoning,
+                    manifest=adapter.manifest,
                 )
-                eval_results.append(eval_result)
+                turns.append(turn)
 
-                if eval_result.detected:
+                if turn.eval_result and turn.eval_result.detected:
                     break
-            else:
-                return turns, eval_results, True
 
-        return turns, eval_results, False
+        return turns
 
     async def _activate_handles_async(
         self,
@@ -199,7 +176,6 @@ def _build_attack_result(
         *,
         adapter: AgentAdapter,
         turns: list[Turn],
-        eval_results: list[EvalResult],
     ) -> Result:
         """Resolve eval results into a final attack Result.
 
@@ -208,11 +184,11 @@ def _build_attack_result(
         Args:
             adapter (AgentAdapter): The adapter under test.
             turns (list[Turn]): Conversation history.
-            eval_results (list[EvalResult]): Evaluator outputs.
 
         Returns:
             Result: The final safety verdict.
         """
+        eval_results = [t.eval_result for t in turns if t.eval_result is not None]
         safe, status = resolve_as_attack(eval_results=eval_results)
 
         if status == SafetyStatus.SAFE:
@@ -228,43 +204,6 @@ def _build_attack_result(
             status=status,
             summary=_build_summary(status=status, eval_results=eval_results),
             turns=turns,
-            eval_results=eval_results,
-            strategy=self.strategy_name,
-            observability_level=adapter.observability_profile,
-            injections=self._build_injection_records(),
-            metadata=_collect_response_metadata(turns=turns),
-        )
-
-    def _max_turns_error_result(
-        self,
-        *,
-        adapter: AgentAdapter,
-        turns: list[Turn],
-        eval_results: list[EvalResult],
-    ) -> Result:
-        """Build an ERROR result when the driver exceeds max_turns.
-
-        Args:
-            adapter (AgentAdapter): The adapter under test.
-            turns (list[Turn]): Conversation history.
-            eval_results (list[EvalResult]): Evaluator outputs so far.
-
-        Returns:
-            Result: Error result with max-turns summary.
-        """
-        logger.warning(
-            "Max turns (%d) reached without driver termination. "
-            "Check PromptDriver configuration.",
-            self._max_turns,
-        )
-        return Result(
-            safe=False,
-            status=SafetyStatus.ERROR,
-            summary=(
-                f"Max turns ({self._max_turns}) reached — driver did not terminate"
-            ),
-            turns=turns,
-            eval_results=eval_results,
             strategy=self.strategy_name,
             observability_level=adapter.observability_profile,
             injections=self._build_injection_records(),

rampart/core/__init__.py

@@ -8,14 +8,15 @@
 
 from rampart.core.adapter import AgentAdapter, Session
 from rampart.core.converter import PayloadConverter
-from rampart.core.errors import InfrastructureError
+from rampart.core.errors import DriverError, InfrastructureError
 from rampart.core.evaluator import BaseEvaluator, Evaluator
 from rampart.core.execution import (
     BaseExecution,
     ExecutionEvent,
     ExecutionEventData,
     ExecutionEventHandler,
     ExecutionHandlerFactory,
+    evaluate_turn_async,
 )
 from rampart.core.injection import InjectionHandle, Surface
 from rampart.core.llm import LLMConfig
@@ -50,6 +51,7 @@
     "BaseEvaluator",
     "BaseExecution",
     "DataSource",
+    "DriverError",
     "EvalContext",
     "EvalOutcome",
     "EvalResult",
@@ -80,6 +82,7 @@
     "ToolCall",
     "ToolDeclaration",
     "Turn",
+    "evaluate_turn_async",
     "resolve_as_attack",
     "resolve_as_probe",
 ]

rampart/core/converter.py

@@ -15,7 +15,7 @@
 - **Pre-injection**: applied directly in test code before
   passing a payload to a surface.
 
-The PyRIT bridge in ``_pyrit/converter_bridge.py`` will adapt
+The PyRIT bridge in ``pyrit_bridge/converter_bridge.py`` will adapt
 ``PromptConverter`` to this protocol. Teams can also implement
 custom converters directly.
 """

rampart/core/errors.py

@@ -25,3 +25,11 @@ class InfrastructureError(Exception):
     Use ``raise InfrastructureError(...) from original_exception`` to
     preserve the causal chain via Python's native ``__cause__`` attribute.
     """
+
+
+class DriverError(Exception):
+    """Raised by a PromptDriver when it cannot produce a decision.
+
+    BaseExecution.execute_async catches this and produces a Result with
+    SafetyStatus.ERROR.
+    """