Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -289,14 +289,25 @@ def format_auto_rater_prompt(

app_details = actual_invocation.app_details
if app_details:
# Determine agent name from invocation events if available,
# otherwise fall back to the first agent in app_details.
# This ensures developer_instructions are populated even when
# the agent makes zero tool calls (e.g., declining out-of-scope
# requests).
agent_name = None
if (
isinstance(actual_invocation.intermediate_data, InvocationEvents)
and actual_invocation.intermediate_data.invocation_events
):
agent_name = (
actual_invocation.intermediate_data.invocation_events[0].author
)
elif app_details.agent_details:
agent_name = next(iter(app_details.agent_details))

if agent_name:
developer_instructions = app_details.get_developer_instructions(
agent_name=actual_invocation.intermediate_data.invocation_events[
0
].author
agent_name=agent_name
)
tool_declarations = get_tool_declarations_as_json_str(app_details)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -222,3 +222,66 @@ def test_format_auto_rater_prompt_with_intermediate_data_no_tools(
prompt = evaluator.format_auto_rater_prompt(invocation, None)

assert "No intermediate steps were taken." in prompt


def test_format_auto_rater_prompt_with_app_details_empty_invocation_events(
evaluator: RubricBasedFinalResponseQualityV1Evaluator,
):
"""Tests that developer_instructions are populated even when invocation_events is empty.

This covers the case where an agent declines a request without calling any
tools (e.g., out-of-scope rejection), resulting in zero invocation events.
The judge should still receive the developer instructions to evaluate rubrics
that reference the system prompt.
"""
app_details = AppDetails(
agent_details={
"my_agent": AgentDetails(
name="my_agent",
instructions="Only answer questions about cooking. Decline all other requests.",
tool_declarations=[],
)
},
)
invocation = Invocation(
user_content=genai_types.Content(
parts=[genai_types.Part(text="What is the capital of France?")]
),
final_response=genai_types.Content(
parts=[genai_types.Part(text="I can only help with cooking.")]
),
app_details=app_details,
intermediate_data=InvocationEvents(invocation_events=[]),
)
prompt = evaluator.format_auto_rater_prompt(invocation, None)

assert "Only answer questions about cooking." in prompt
assert "I can only help with cooking." in prompt


def test_format_auto_rater_prompt_with_app_details_no_intermediate_data(
evaluator: RubricBasedFinalResponseQualityV1Evaluator,
):
"""Tests that developer_instructions are populated when intermediate_data is None."""
app_details = AppDetails(
agent_details={
"my_agent": AgentDetails(
name="my_agent",
instructions="Agent instructions here.",
tool_declarations=[],
)
},
)
invocation = Invocation(
user_content=genai_types.Content(
parts=[genai_types.Part(text="User input here.")]
),
final_response=genai_types.Content(
parts=[genai_types.Part(text="Final agent response.")]
),
app_details=app_details,
intermediate_data=None,
)
prompt = evaluator.format_auto_rater_prompt(invocation, None)

assert "Agent instructions here." in prompt