diff --git a/src/strands_evals/simulation/__init__.py b/src/strands_evals/simulation/__init__.py
index 3097b0d6..b41e3140 100644
--- a/src/strands_evals/simulation/__init__.py
+++ b/src/strands_evals/simulation/__init__.py
@@ -1,4 +1,5 @@
from .actor_simulator import ActorSimulator
+from .prompt_templates.actor_system_prompt import DEFAULT_USER_SIMULATOR_PROMPT_TEMPLATE
from .tool_simulator import ToolSimulator
# Alias for backward compatibility
@@ -8,4 +9,5 @@
"ActorSimulator",
"UserSimulator",
"ToolSimulator",
+ "DEFAULT_USER_SIMULATOR_PROMPT_TEMPLATE",
]
diff --git a/src/strands_evals/simulation/actor_simulator.py b/src/strands_evals/simulation/actor_simulator.py
index fb1d9c36..eb686b19 100644
--- a/src/strands_evals/simulation/actor_simulator.py
+++ b/src/strands_evals/simulation/actor_simulator.py
@@ -1,10 +1,11 @@
import logging
import random
+from typing import Any, cast
+from pydantic import BaseModel
from strands import Agent
from strands.agent.agent_result import AgentResult
from strands.types.content import Message
-from typing_extensions import cast
from strands_evals.case import Case
from strands_evals.simulation.profiles.actor_profile import DEFAULT_USER_PROFILE_SCHEMA
@@ -17,8 +18,7 @@
class ActorSimulator:
- """
- Simulates an actor in multi-turn conversations for agent evaluation.
+ """Simulates an actor in multi-turn conversations for agent evaluation.
ActorSimulator wraps a Strands Agent configured to behave as a specific actor
(typically a user) in conversation scenarios. It maintains conversation history,
@@ -49,8 +49,7 @@ def from_case_for_user_simulator(
model: str | None = None,
max_turns: int = 10,
) -> "ActorSimulator":
- """
- Create an ActorSimulator configured as a user simulator from a test case.
+ """Create an ActorSimulator configured as a user simulator from a test case.
Generates a realistic user profile and goal from case.input and optionally
case.metadata["task_description"], then configures the simulator with
@@ -71,22 +70,14 @@ def from_case_for_user_simulator(
from strands_evals import Case, ActorSimulator
from strands import Agent
- # Create test case
case = Case(
input="I need to book a flight to Paris",
metadata={"task_description": "Flight booking confirmed"}
)
- # Create user simulator
- user_sim = ActorSimulator.from_case_for_user_simulator(
- case=case,
- max_turns=5
- )
-
- # Create target agent to evaluate
+ user_sim = ActorSimulator.from_case_for_user_simulator(case=case, max_turns=5)
agent = Agent(system_prompt="You are a travel assistant.")
- # Run conversation
user_message = case.input
while user_sim.has_next():
agent_response = agent(user_message)
@@ -96,9 +87,6 @@ def from_case_for_user_simulator(
"""
actor_profile = cls._generate_profile_from_case(case)
- if system_prompt_template is None:
- system_prompt_template = DEFAULT_USER_SIMULATOR_PROMPT_TEMPLATE
-
return cls(
actor_profile=actor_profile,
initial_query=case.input,
@@ -110,10 +98,8 @@ def from_case_for_user_simulator(
@staticmethod
def _generate_profile_from_case(case: Case) -> ActorProfile:
- """
- Generate user profile from case.
+ """Generate user profile from case.
- Private helper for from_case_for_user_simulator factory method.
Uses case.input and optionally case.metadata["task_description"] if present.
Args:
@@ -138,13 +124,14 @@ def __init__(
self,
actor_profile: ActorProfile,
initial_query: str,
- system_prompt_template: str,
+ system_prompt_template: str | None = None,
tools: list | None = None,
model: str | None = None,
max_turns: int = 10,
+ *,
+ structured_output_model: type[BaseModel] | None = None,
):
- """
- Initialize an ActorSimulator with profile and goal.
+ """Initialize an ActorSimulator with profile and goal.
Use this constructor when you have a pre-defined ActorProfile. For automatic
profile generation from test cases, use from_case_for_user_simulator() instead.
@@ -152,53 +139,84 @@ def __init__(
Args:
actor_profile: ActorProfile object containing traits, context, and actor_goal.
initial_query: The actor's first query or message.
- system_prompt_template: Template string for system prompt. Must include {actor_profile} placeholder.
+ system_prompt_template: System prompt for the actor. Accepts two shapes:
+
+ - A template containing the `{actor_profile}` placeholder, which
+ is rendered via `str.format(actor_profile=...)` against the
+ actor's profile.
+ - An already-rendered system prompt string with no
+ `{actor_profile}` placeholder, which is used verbatim.
+
+ When `None` (the default), uses
+ `DEFAULT_USER_SIMULATOR_PROMPT_TEMPLATE` which instructs the LLM
+ to set `stop=true` on the structured response when the
+ conversation goal is met.
+
+ Pass an explicit template to override.
tools: Additional tools available to the actor. Defaults to goal completion tool only.
model: Model identifier for the underlying agent. Uses Strands default if None.
max_turns: Maximum number of conversation turns before stopping (default: 10).
+ structured_output_model: Optional Pydantic model to use for all `act()` calls.
+ Must have `message` and `stop` fields.
+ When set, `act()` uses this model by default instead of `ActorResponse`.
+ Can still be overridden per-call via `act(structured_output_model=...)`.
Example:
```python
from strands_evals.simulation import ActorSimulator
+ from pydantic import BaseModel
from strands_evals.types.simulation import ActorProfile
- # Define custom actor profile
+ class SimulatorResult(BaseModel):
+ reasoning: str = ""
+ stop: bool = False
+ message: str | None = None
+ urgency: str = "normal"
+
profile = ActorProfile(
- traits={
- "expertise_level": "expert",
- "communication_style": "technical"
- },
+ traits={"expertise_level": "expert", "communication_style": "technical"},
context="A software engineer debugging a production issue.",
actor_goal="Identify and resolve the memory leak."
)
- # Create simulator with custom profile
simulator = ActorSimulator(
actor_profile=profile,
initial_query="Our service is experiencing high memory usage.",
- system_prompt_template="You are simulating: {actor_profile}",
- max_turns=15
+ structured_output_model=SimulatorResult,
+ max_turns=15,
)
+
+ # act() uses SimulatorResult automatically
+ result = simulator.act(str(agent_response))
+ result.structured_output # SimulatorResult instance
```
"""
self.actor_profile = actor_profile
self.initial_query = initial_query
self.conversation_history: list[Message] = []
self.model_id = model
+ self.stop = False
self._turn_count = 0
- self._last_message = ""
self._max_turns = max_turns
+ self._structured_output_model = structured_output_model or ActorResponse
+
+ if structured_output_model is not None:
+ self._validate_output_model(structured_output_model)
+
+ if system_prompt_template is None:
+ system_prompt_template = DEFAULT_USER_SIMULATOR_PROMPT_TEMPLATE
- system_prompt = system_prompt_template.format(actor_profile=actor_profile.model_dump())
+ if "{actor_profile}" in system_prompt_template:
+ system_prompt = system_prompt_template.format(actor_profile=actor_profile.model_dump())
+ else:
+ system_prompt = system_prompt_template
- # Combine tools
all_tools = [get_conversation_goal_completion]
if tools:
all_tools.extend(tools)
self._initialize_conversation()
- # Create agent
self.agent = Agent(
system_prompt=system_prompt,
messages=self.conversation_history,
@@ -208,14 +226,7 @@ def __init__(
)
def _initialize_conversation(self):
- """
- Initialize the conversation history with a greeting and initial query.
-
- Sets up the conversation with a random greeting from the assistant followed
- by the actor's initial query. This establishes the conversation context.
-
- Note: This is a private method called during initialization.
- """
+ """Initialize the conversation history with a greeting and initial query."""
selected_greeting = random.choice(self.INITIAL_GREETINGS)
greeting_message = {"role": "user", "content": [{"text": selected_greeting}]}
self.conversation_history.append(greeting_message)
@@ -223,51 +234,76 @@ def _initialize_conversation(self):
initial_query_message = {"role": "assistant", "content": [{"text": self.initial_query.strip()}]}
self.conversation_history.append(initial_query_message)
- def act(self, agent_message: str) -> AgentResult:
- """
- Generate the next actor message in the conversation.
+ @staticmethod
+ def _validate_output_model(model: type) -> None:
+ """Validate that a structured output model has the required fields for the simulator."""
+ if "message" not in model.model_fields:
+ raise ValueError(f"structured_output_model {model.__name__} must have a 'message' field.")
+ if "stop" not in model.model_fields:
+ raise ValueError(f"structured_output_model {model.__name__} must have a 'stop' field.")
+
+ def act(
+ self,
+ agent_message: str,
+ *,
+ structured_output_model: type[BaseModel] | None = None,
+ ) -> AgentResult:
+ """Generate the next actor message in the conversation.
Processes the agent's message and generates a contextually appropriate
- response from the actor's perspective, maintaining consistency with the actor's
- profile and goal. The response includes reasoning about the actor's thought
- process and the actual message to send.
+ response from the actor's perspective. The response is returned as an
+ `AgentResult` whose `structured_output` is an `ActorResponse` (or the
+ caller-provided `structured_output_model`).
+
+ The provided model must have `message` and `stop` fields.
+ A `ValueError` is raised if either is missing.
Args:
- agent_message: The agent's response to react to (required).
+ agent_message: The agent's response to react to.
+ structured_output_model: Optional Pydantic model to use instead of
+ `ActorResponse`. Must have `message` and `stop` fields.
Returns:
- AgentResult containing the actor's structured response with:
- - structured_output.reasoning: Actor's internal reasoning
- - structured_output.message: Actor's response message
+ AgentResult with `structured_output` set to either `ActorResponse`
+ or the caller-provided model instance.
Example:
```python
- # Agent responds to user
- agent_response = agent("I need help booking a flight")
-
- # User simulator generates next message
- user_result = user_sim.act(str(agent_response))
-
- # Access the response
- print(user_result.structured_output.reasoning) # Why the actor responded this way
- print(user_result.structured_output.message) # The actual message
-
- # Continue conversation
- next_message = str(user_result.structured_output.message)
+ # Default usage
+ result = simulator.act(str(agent_response))
+ response = result.structured_output # ActorResponse
+ print(response.message)
+
+ # Custom structured output
+ result = simulator.act(str(agent_response), structured_output_model=MySchema)
+ my_output = result.structured_output # MySchema instance
```
"""
- response = self.agent(agent_message.strip(), structured_output_model=ActorResponse)
+ model = structured_output_model or self._structured_output_model
+ self._validate_output_model(model)
+
+ response = self.agent(agent_message.strip(), structured_output_model=model)
self._turn_count += 1
- self._last_message = str(cast(ActorResponse, response.structured_output).message)
+
+ result = cast(Any, response.structured_output)
+
+ if result.stop:
+ self.stop = True
+ if hasattr(result, "stop_reason"):
+ result.stop_reason = "goal_completed"
+ elif self._turn_count >= self._max_turns:
+ result.stop = True
+ self.stop = True
+ if hasattr(result, "stop_reason"):
+ result.stop_reason = "max_turns"
+
return response
def has_next(self) -> bool:
- """
- Check if the conversation should continue.
+ """Check if the conversation should continue.
- Returns False if the stop token () is present in the last message or if
- the maximum number of turns has been reached. Use this in a loop to control
- multi-turn conversations.
+ Returns False if the actor signalled stop or if the maximum number of
+ turns has been reached.
Returns:
True if the conversation should continue, False otherwise.
@@ -275,18 +311,10 @@ def has_next(self) -> bool:
Example:
```python
user_message = case.input
-
- # Continue conversation until completion
while user_sim.has_next():
agent_response = agent(user_message)
user_result = user_sim.act(str(agent_response))
user_message = str(user_result.structured_output.message)
-
- # Conversation ended either by:
- # - Actor including token in message
- # - Reaching max_turns limit
```
"""
- if self._turn_count >= self._max_turns:
- return False
- return "" not in self._last_message
+ return not self.stop
diff --git a/src/strands_evals/simulation/prompt_templates/actor_system_prompt.py b/src/strands_evals/simulation/prompt_templates/actor_system_prompt.py
index 2b863fd0..dfc62ad2 100644
--- a/src/strands_evals/simulation/prompt_templates/actor_system_prompt.py
+++ b/src/strands_evals/simulation/prompt_templates/actor_system_prompt.py
@@ -1,8 +1,10 @@
-"""
-Default system prompt for actor simulation.
+"""Default system prompt for actor simulation.
+
+The template instructs the actor to signal end-of-conversation by setting
+`stop=true` on the structured response.
-This module contains the default system prompt that configures the actor's behavior,
-communication style, and response protocols for realistic conversation simulation.
+The template contains a single `{actor_profile}` placeholder. The simulator
+renders it with `str.format(actor_profile=...)`.
"""
from textwrap import dedent
@@ -24,7 +26,7 @@
- Maximum 2-3 sentences
When assistant provides solutions/answers:
- - Ask follow-ups, seek clarification, or express satisfaction. Do no deviate from the User Goal.
+ - Ask follow-ups, seek clarification, or express satisfaction. Do not deviate from the User Goal.
- While following up, do not increase the conversation scope beyond your User Goal.
Communication Rules:
@@ -46,11 +48,11 @@
10. Use all relevant tools first to ground your responses, and then respond
Exit Conditions:
-1. Use get_conversation_goal_completion tool to check if your User Goal is met. When your User Goal is met:
- - Just generate "" to terminate conversation
+1. Use get_conversation_goal_completion tool to check if your User Goal is met. When your
+ User Goal is met, set stop=true in your structured response to end the conversation.
2. If conversation becomes unproductive or unsafe:
- Naturally steer back towards your User Goal
- - If this becomes impossible, just generate: "" to terminate conversation
+ - If this becomes impossible, set stop=true in your structured response to end the conversation
CRITICAL BEHAVIORAL CONSTRAINTS:
- You are ONLY a user seeking assistance, NEVER the one providing assistance.
@@ -58,7 +60,6 @@
- NEVER solve problems yourself - that's the assistant's job. Under no circumstances,
you can use your tools to solve your user goal/sub goals.
- If you find yourself writing more than 3 sentences, you're doing it wrong.
-- Generate only "" to terminate conversation
Response Format:
Generate ONLY the next SHORT message (1-3 sentences). No explanations, no solutions, no comprehensive information.""")
diff --git a/src/strands_evals/types/simulation/__init__.py b/src/strands_evals/types/simulation/__init__.py
index 13a94b00..d53fe2ba 100644
--- a/src/strands_evals/types/simulation/__init__.py
+++ b/src/strands_evals/types/simulation/__init__.py
@@ -2,4 +2,7 @@
from .actor import ActorProfile, ActorResponse
-__all__ = ["ActorProfile", "ActorResponse"]
+__all__ = [
+ "ActorProfile",
+ "ActorResponse",
+]
diff --git a/src/strands_evals/types/simulation/actor.py b/src/strands_evals/types/simulation/actor.py
index d30be945..a1a3cc19 100644
--- a/src/strands_evals/types/simulation/actor.py
+++ b/src/strands_evals/types/simulation/actor.py
@@ -1,10 +1,10 @@
-from pydantic import BaseModel, Field
-from typing_extensions import Any
+from typing import Any
+
+from pydantic import BaseModel, ConfigDict, Field
class ActorProfile(BaseModel):
- """
- Profile for actor simulation.
+ """Profile for actor simulation.
Attributes:
traits: Dictionary of actor characteristics and attributes.
@@ -22,13 +22,33 @@ class ActorProfile(BaseModel):
class ActorResponse(BaseModel):
- """
- Structured response from an actor.
+ """Default structured response from the actor simulator.
+
+ Used as the LLM structured-output schema for `ActorSimulator.act` when no
+ custom `structured_output_model` is provided. The LLM fills `reasoning`,
+ `stop`, and `message`. The simulator fills `stop_reason` after the LLM call.
Attributes:
reasoning: Internal reasoning process for the response.
- message: The actual message content from the actor.
+ stop: `True` when the actor signals the conversation should end.
+ message: The actual message content from the actor. `None` when `stop=True`.
+ stop_reason: Why the conversation ended. One of `"goal_completed"`,
+ `"max_turns"`, or `None` while ongoing. Populated by the simulator
+ after the LLM call.
"""
+ model_config = ConfigDict(arbitrary_types_allowed=True)
+
reasoning: str = Field(..., description="Reasoning for the actor's response")
- message: str = Field(..., description="Message from the actor")
+ stop: bool = Field(
+ False,
+ description="Set to true when the conversation goal is met or the conversation should end.",
+ )
+ message: str | None = Field(
+ None,
+ description="The actor's next message to the agent. Provide when stop=false; set to null when stop=true.",
+ )
+ stop_reason: str | None = Field(
+ None,
+ description='Populated by the simulator after the call. One of "goal_completed", "max_turns", or None.',
+ )
diff --git a/tests/strands_evals/simulation/test_actor_simulator.py b/tests/strands_evals/simulation/test_actor_simulator.py
index c491a6f5..59135088 100644
--- a/tests/strands_evals/simulation/test_actor_simulator.py
+++ b/tests/strands_evals/simulation/test_actor_simulator.py
@@ -3,6 +3,7 @@
from unittest.mock import MagicMock, patch
import pytest
+from pydantic import BaseModel
from strands.agent.agent_result import AgentResult
from strands_evals import Case
@@ -68,7 +69,6 @@ def test_initialize_conversation(sample_actor_profile):
@patch("strands_evals.simulation.actor_simulator.Agent")
def test_from_case_for_user_simulator(mock_agent_class, sample_case):
"""Test factory method creates simulator from case."""
- # Mock the profile generation agent
mock_profile_agent = MagicMock()
mock_profile = ActorProfile(
traits={"test": "trait"},
@@ -79,10 +79,8 @@ def test_from_case_for_user_simulator(mock_agent_class, sample_case):
mock_result.structured_output = mock_profile
mock_profile_agent.return_value = mock_result
- # Mock the main simulator agent
mock_simulator_agent = MagicMock()
- # Configure mock to return different instances
mock_agent_class.side_effect = [mock_profile_agent, mock_simulator_agent]
simulator = ActorSimulator.from_case_for_user_simulator(case=sample_case)
@@ -110,7 +108,6 @@ def test_generate_profile_from_case(mock_agent_class, sample_case):
assert profile == mock_profile
assert mock_agent.called
- # Verify structured_output_model was passed
call_args = mock_agent.call_args
assert call_args[1]["structured_output_model"] == ActorProfile
@@ -123,11 +120,11 @@ def test_act_generates_response(sample_actor_profile):
system_prompt_template="Test: {actor_profile}",
)
- # Mock the agent's response
mock_response = MagicMock(spec=AgentResult)
mock_actor_response = ActorResponse(
reasoning="Test reasoning",
message="Test response message",
+ stop=False,
)
mock_response.structured_output = mock_actor_response
simulator.agent = MagicMock(return_value=mock_response)
@@ -139,8 +136,8 @@ def test_act_generates_response(sample_actor_profile):
simulator.agent.assert_called_once()
-def test_act_uses_structured_output(sample_actor_profile):
- """Test act method requests structured output."""
+def test_act_uses_actor_response_by_default(sample_actor_profile):
+ """Test act method uses ActorResponse as default structured output model."""
simulator = ActorSimulator(
actor_profile=sample_actor_profile,
initial_query="Hello",
@@ -148,17 +145,75 @@ def test_act_uses_structured_output(sample_actor_profile):
)
mock_response = MagicMock(spec=AgentResult)
- mock_actor_response = ActorResponse(reasoning="Test", message="Test message")
+ mock_actor_response = ActorResponse(reasoning="Test", message="Test message", stop=False)
mock_response.structured_output = mock_actor_response
simulator.agent = MagicMock(return_value=mock_response)
simulator.act("Test message")
- # Verify structured_output_model parameter
call_kwargs = simulator.agent.call_args[1]
assert call_kwargs["structured_output_model"] == ActorResponse
+def test_act_with_custom_structured_output_model(sample_actor_profile):
+ """Test act passes custom structured_output_model to the agent."""
+
+ class CustomOutput(BaseModel):
+ answer: str
+ confidence: float
+ stop: bool = False
+ message: str | None = None
+
+ simulator = ActorSimulator(
+ actor_profile=sample_actor_profile,
+ initial_query="Hello",
+ system_prompt_template="Test: {actor_profile}",
+ )
+
+ mock_response = MagicMock(spec=AgentResult)
+ mock_response.structured_output = CustomOutput(answer="test", confidence=0.9, stop=False, message="hi")
+ simulator.agent = MagicMock(return_value=mock_response)
+
+ result = simulator.act("Test message", structured_output_model=CustomOutput)
+
+ call_kwargs = simulator.agent.call_args[1]
+ assert call_kwargs["structured_output_model"] == CustomOutput
+ assert result.structured_output.answer == "test"
+
+
+def test_act_custom_model_without_stop_raises(sample_actor_profile):
+ """Test act raises ValueError if custom model has no stop field."""
+
+ class NoStopModel(BaseModel):
+ message: str | None = None
+
+ simulator = ActorSimulator(
+ actor_profile=sample_actor_profile,
+ initial_query="Hello",
+ system_prompt_template="Test: {actor_profile}",
+ )
+
+ with pytest.raises(ValueError, match="must have a 'stop' field"):
+ simulator.act("Test message", structured_output_model=NoStopModel)
+
+
+def test_act_custom_model_without_message_raises(sample_actor_profile):
+ """Test act raises ValueError if custom model has no message field."""
+
+ class NoMessageModel(BaseModel):
+ answer: str = ""
+ stop: bool = False
+
+ simulator = ActorSimulator(
+ actor_profile=sample_actor_profile,
+ initial_query="Hello",
+ system_prompt_template="Test: {actor_profile}",
+ )
+
+ with pytest.raises(ValueError, match="must have a 'message' field"):
+ simulator.act("Test message", structured_output_model=NoMessageModel)
+
+
def test_has_next_returns_true_initially(sample_actor_profile):
"""Test has_next returns True before any turns."""
simulator = ActorSimulator(
@@ -179,35 +234,270 @@ def test_has_next_respects_max_turns(sample_actor_profile):
max_turns=3,
)
- # Mock responses
mock_response = MagicMock(spec=AgentResult)
- mock_actor_response = ActorResponse(reasoning="Test", message="Continue")
+ mock_actor_response = ActorResponse(reasoning="Test", message="Continue", stop=False)
mock_response.structured_output = mock_actor_response
simulator.agent = MagicMock(return_value=mock_response)
- # Simulate 3 turns with max_turns=3
for _ in range(3):
assert simulator.has_next() is True
simulator.act("Test message")
- # After 3 turns, should return False
assert simulator.has_next() is False
-def test_has_next_detects_stop_token(sample_actor_profile):
- """Test has_next returns False when stop token is present."""
+def test_has_next_detects_stop(sample_actor_profile):
+ """Test has_next returns False when actor signals stop."""
simulator = ActorSimulator(
actor_profile=sample_actor_profile,
initial_query="Hello",
system_prompt_template="Test: {actor_profile}",
)
- # Mock response with stop token
mock_response = MagicMock(spec=AgentResult)
- mock_actor_response = ActorResponse(reasoning="Done", message="Thanks! ")
+ mock_actor_response = ActorResponse(reasoning="Done", message=None, stop=True)
mock_response.structured_output = mock_actor_response
simulator.agent = MagicMock(return_value=mock_response)
- # After act with stop token, has_next should return False
simulator.act("Test message")
assert simulator.has_next() is False
+
+
+def test_act_sets_stop_reason_goal_completed(sample_actor_profile):
+ """Test act sets stop_reason to 'goal_completed' when actor signals stop."""
+ simulator = ActorSimulator(
+ actor_profile=sample_actor_profile,
+ initial_query="Hello",
+ system_prompt_template="Test: {actor_profile}",
+ )
+
+ mock_response = MagicMock(spec=AgentResult)
+ mock_actor_response = ActorResponse(reasoning="Done", message=None, stop=True)
+ mock_response.structured_output = mock_actor_response
+ simulator.agent = MagicMock(return_value=mock_response)
+
+ result = simulator.act("Test message")
+
+ assert result.structured_output.stop_reason == "goal_completed"
+
+
+def test_act_sets_stop_reason_max_turns(sample_actor_profile):
+ """Test act sets stop_reason to 'max_turns' when turn cap is reached."""
+ simulator = ActorSimulator(
+ actor_profile=sample_actor_profile,
+ initial_query="Hello",
+ system_prompt_template="Test: {actor_profile}",
+ max_turns=1,
+ )
+
+ mock_response = MagicMock(spec=AgentResult)
+ mock_actor_response = ActorResponse(reasoning="r", message="more please", stop=False)
+ mock_response.structured_output = mock_actor_response
+ simulator.agent = MagicMock(return_value=mock_response)
+
+ result = simulator.act("agent reply")
+
+ assert result.structured_output.stop is True
+ assert result.structured_output.stop_reason == "max_turns"
+
+
+def test_act_continuing_turn_no_stop_reason(sample_actor_profile):
+ """Test act leaves stop_reason as None for normal continuing turns."""
+ simulator = ActorSimulator(
+ actor_profile=sample_actor_profile,
+ initial_query="Hello",
+ system_prompt_template="Test: {actor_profile}",
+ )
+
+ mock_response = MagicMock(spec=AgentResult)
+ mock_actor_response = ActorResponse(reasoning="thinking", message="keep going", stop=False)
+ mock_response.structured_output = mock_actor_response
+ simulator.agent = MagicMock(return_value=mock_response)
+
+ result = simulator.act("agent reply")
+
+ assert result.structured_output.stop is False
+ assert result.structured_output.stop_reason is None
+ assert result.structured_output.message == "keep going"
+
+
+def test_act_custom_model_manages_stop(sample_actor_profile):
+ """When structured_output_model is provided, act() still manages stop via the stop field."""
+
+ class CustomOutput(BaseModel):
+ stop: bool = False
+ message: str | None = None
+
+ simulator = ActorSimulator(
+ actor_profile=sample_actor_profile,
+ initial_query="Hello",
+ system_prompt_template="Test: {actor_profile}",
+ )
+
+ mock_response = MagicMock(spec=AgentResult)
+ mock_response.structured_output = CustomOutput(message="done", stop=True)
+ simulator.agent = MagicMock(return_value=mock_response)
+
+ simulator.act("agent reply", structured_output_model=CustomOutput)
+
+ assert simulator.stop is True
+ assert simulator.has_next() is False
+
+
+def test_act_custom_model_max_turns(sample_actor_profile):
+ """Custom model path still enforces max_turns."""
+
+ class CustomOutput(BaseModel):
+ stop: bool = False
+ message: str | None = None
+
+ simulator = ActorSimulator(
+ actor_profile=sample_actor_profile,
+ initial_query="Hello",
+ system_prompt_template="Test: {actor_profile}",
+ max_turns=1,
+ )
+
+ mock_response = MagicMock(spec=AgentResult)
+ mock_response.structured_output = CustomOutput(message="hi", stop=False)
+ simulator.agent = MagicMock(return_value=mock_response)
+
+ simulator.act("agent reply", structured_output_model=CustomOutput)
+
+ assert simulator.stop is True
+ assert simulator.has_next() is False
+
+
+def test_system_prompt_template_none_uses_default(sample_actor_profile):
+ """When system_prompt_template is None, the default template is rendered with the profile."""
+ from strands_evals.simulation.prompt_templates.actor_system_prompt import (
+ DEFAULT_USER_SIMULATOR_PROMPT_TEMPLATE,
+ )
+
+ simulator = ActorSimulator(
+ actor_profile=sample_actor_profile,
+ initial_query="Hello",
+ )
+
+ expected = DEFAULT_USER_SIMULATOR_PROMPT_TEMPLATE.format(actor_profile=sample_actor_profile.model_dump())
+ assert simulator.agent.system_prompt == expected
+
+
+def test_system_prompt_template_prerendered_passes_through(sample_actor_profile):
+ """A template with no {actor_profile} placeholder is passed through verbatim."""
+ prerendered = "You are simulating Alice, a beginner user. Keep replies short."
+
+ simulator = ActorSimulator(
+ actor_profile=sample_actor_profile,
+ initial_query="Hello",
+ system_prompt_template=prerendered,
+ )
+
+ assert simulator.agent.system_prompt == prerendered
+
+
+def test_system_prompt_contains_stop_instruction(sample_actor_profile):
+ """Default prompt instructs the actor to set stop=true."""
+ simulator = ActorSimulator(
+ actor_profile=sample_actor_profile,
+ initial_query="Hello",
+ )
+
+ assert "stop=true" in simulator.agent.system_prompt
+
+
+def test_explicit_template_overrides_default(sample_actor_profile):
+ """Explicit system_prompt_template is used instead of the default."""
+ custom = "Custom prompt for {actor_profile}"
+ simulator = ActorSimulator(
+ actor_profile=sample_actor_profile,
+ initial_query="Hello",
+ system_prompt_template=custom,
+ )
+
+ assert simulator.agent.system_prompt == custom.format(actor_profile=sample_actor_profile.model_dump())
+
+
+def test_init_structured_output_model_used_by_act(sample_actor_profile):
+ """structured_output_model set at init is used as default for act()."""
+
+ class CustomOutput(BaseModel):
+ stop: bool = False
+ message: str | None = None
+ extra: str = "default"
+
+ simulator = ActorSimulator(
+ actor_profile=sample_actor_profile,
+ initial_query="Hello",
+ system_prompt_template="Test: {actor_profile}",
+ structured_output_model=CustomOutput,
+ )
+
+ mock_response = MagicMock(spec=AgentResult)
+ mock_response.structured_output = CustomOutput(message="hi", stop=False)
+ simulator.agent = MagicMock(return_value=mock_response)
+
+ simulator.act("agent reply")
+
+ call_kwargs = simulator.agent.call_args[1]
+ assert call_kwargs["structured_output_model"] == CustomOutput
+
+
+def test_init_structured_output_model_overridden_per_call(sample_actor_profile):
+ """Per-call structured_output_model overrides the init-level default."""
+
+ class InitModel(BaseModel):
+ stop: bool = False
+ message: str | None = None
+
+ class CallModel(BaseModel):
+ stop: bool = False
+ message: str | None = None
+ priority: int = 0
+
+ simulator = ActorSimulator(
+ actor_profile=sample_actor_profile,
+ initial_query="Hello",
+ system_prompt_template="Test: {actor_profile}",
+ structured_output_model=InitModel,
+ )
+
+ mock_response = MagicMock(spec=AgentResult)
+ mock_response.structured_output = CallModel(message="hi", stop=False)
+ simulator.agent = MagicMock(return_value=mock_response)
+
+ simulator.act("agent reply", structured_output_model=CallModel)
+
+ call_kwargs = simulator.agent.call_args[1]
+ assert call_kwargs["structured_output_model"] == CallModel
+
+
+def test_init_structured_output_model_validates_stop_field(sample_actor_profile):
+ """Init raises ValueError if structured_output_model has no stop field."""
+
+ class NoStopModel(BaseModel):
+ message: str | None = None
+
+ with pytest.raises(ValueError, match="must have a 'stop' field"):
+ ActorSimulator(
+ actor_profile=sample_actor_profile,
+ initial_query="Hello",
+ system_prompt_template="Test: {actor_profile}",
+ structured_output_model=NoStopModel,
+ )
+
+
+def test_init_structured_output_model_validates_message_field(sample_actor_profile):
+ """Init raises ValueError if structured_output_model has no message field."""
+
+ class NoMessageModel(BaseModel):
+ answer: str = ""
+ stop: bool = False
+
+ with pytest.raises(ValueError, match="must have a 'message' field"):
+ ActorSimulator(
+ actor_profile=sample_actor_profile,
+ initial_query="Hello",
+ system_prompt_template="Test: {actor_profile}",
+ structured_output_model=NoMessageModel,
+ )