From ac71e07a60da06892fc97237a6632459419b4ef7 Mon Sep 17 00:00:00 2001 From: cppbear Date: Thu, 8 Oct 2026 17:20:46 +0800 Subject: [PATCH] fix: describe crate-local scope in unit test prompts --- scripts/check_tasks.py | 20 ++++++++++++++++++-- utgen/res/oracle_prompt.json | 1 + utgen/res/prefix_prompt.json | 1 + utgen/res/test_prompt.json | 1 + utgen/src/gene/prompt/oracle.rs | 5 +++++ utgen/src/gene/prompt/prefix.rs | 5 +++++ utgen/src/gene/prompt/prompt.rs | 15 ++++++++++++--- utgen/src/gene/prompt/test.rs | 5 +++++ 8 files changed, 48 insertions(+), 5 deletions(-) diff --git a/scripts/check_tasks.py b/scripts/check_tasks.py index f6dd9d629e..c47b927f31 100644 --- a/scripts/check_tasks.py +++ b/scripts/check_tasks.py @@ -116,9 +116,25 @@ def do_POST(self): if 'The function to be tested is presented as follows:' in user: focal = user.split('The function to be tested is presented as follows:', 1)[1] function = re.search(r'\bfn (f\d+)\(', focal).group(1) - if 'infer the test input ranges' in system: + is_input = 'infer the test input ranges' in system + is_oracle = 'generate accurate test oracles' in system + if is_input or state['integration']: + assert 'use super::*;' not in system, 'unit scope leaked into another prompt' + else: + assert 'child test module inside the crate under test' in system + assert 'use super::*;' in system and 'crate::' in system + assert 'not an external dependency' in system + if state['integration'] and not is_input: + assert 'publicly accessible' in system + if not is_oracle: + assert 'separate folder' in system + if is_oracle: + assert 'do not output complete test functions' in system.lower() + if 'Omit test oracles' in system: + assert 'Omit test oracles and assertions' in system + if is_input: answer = 'x = 1' - elif 'generate accurate test oracles' in system: + elif is_oracle: answer = 'assert_eq!(actual, 2);' else: argument = '1' if state['integration'] or state['invalid_answers'] else '"bad"' diff --git a/utgen/res/oracle_prompt.json b/utgen/res/oracle_prompt.json index 45abb633f3..80c1b702fb 100644 --- a/utgen/res/oracle_prompt.json +++ b/utgen/res/oracle_prompt.json @@ -1,5 +1,6 @@ { "system": "As a software testing expert, please generate accurate test oracles code based on the provided information. Follow these guidelines:\n1. Generate accurate test oracle/assertion codes in plain text format, one per line.\n2. Do not output complete test functions, do not repeat provided test prefixes, do not generate non-assertion statements, avoid additional explanations, and do not use Markdown.\n3. Generate necessary test oracles solely for the provided test prefixes.\n4. All assertions are independent, with no dependencies between them.\n5. Generate test oracles by interpreting the behavior of the test function through the provided test prefixes, context, and documentation.\n", + "unit": "The tests run in a child test module inside the crate under test, alongside the focal code. The test module already provides use super::*;. Use types and functions in that scope, or correct crate:: paths for items in the crate under test. The crate name shown in the context identifies the crate being tested, not an external dependency; do not use that name as an external-crate path. Actual external dependencies may still use their normal paths. Keep the output limited to the code requested for this stage; do not add a test module.\n", "integration": "6. Restrict your implementation to publicly accessible data structures, fields, or functions within the given context.\n", "context": "The context for the focal function is as follows:\n", "focal": "The function to be tested is presented as follows:\n" diff --git a/utgen/res/prefix_prompt.json b/utgen/res/prefix_prompt.json index 0a9d137e7a..75b7d7cf40 100644 --- a/utgen/res/prefix_prompt.json +++ b/utgen/res/prefix_prompt.json @@ -1,5 +1,6 @@ { "system": "As a software testing expert, please generate Rust test functions based on the following guidelines:\n1. Provide the code in plain text format, without explanations or Markdown.\n2. Omit test oracles and assertions; concentrate on generating test inputs and function calls of the focal function; do not use \"_\" for the return values of the focal function.\n3. Ensure the generated function is fully implemented and can be compiled and executed directly without any missing parts.\n4. Whenever possible, initialize the corresponding data structures using the initialization methods provided in the context.\n5. If the method under test belongs to a trait, construct appropriate structs within the test function, but avoid method overrides.\n6. If the method under test uses generics, instantiate them with suitable types based on the context.\n7. Define any necessary helper structures or implementations directly within the test function when required.\n8. Create a minimal yet comprehensive set of test functions, ensuring each test input satisfies all given constraints, with some explicitly covering edge scenarios.\n9. Do not create a test module, but include intrinsic attributes like #[test] or #[should_panic] where necessary.\n", + "unit": "The tests run in a child test module inside the crate under test, alongside the focal code. The test module already provides use super::*;. Use types and functions in that scope, or correct crate:: paths for items in the crate under test. The crate name shown in the context identifies the crate being tested, not an external dependency; do not use that name as an external-crate path. Actual external dependencies may still use their normal paths. Keep the output limited to the code requested for this stage; do not add a test module.\n", "integration": "10. Assume the tests will be placed in a separate folder within the project's root directory.\n11. Restrict your implementation to publicly accessible data structures, fields, or functions within the given context.\n12. Include only use statements that are not already present in the provided test file.\n", "context": "The context for the focal function is as follows:\n", "focal": "The function to be tested is presented as follows:\n", diff --git a/utgen/res/test_prompt.json b/utgen/res/test_prompt.json index 5874e43f5c..08ca0f49f5 100644 --- a/utgen/res/test_prompt.json +++ b/utgen/res/test_prompt.json @@ -1,5 +1,6 @@ { "system": "As a software testing expert, please generate Rust test functions based on the following guidelines:\n1. Provide the code in plain text format, without explanations or Markdown.\n2. If the method under test belongs to a trait, construct appropriate structs within the test function, but avoid method overrides. If the method under test uses generics, instantiate them with suitable types based on the context.\n3. Generate test code with minimal scope: avoid creating external structures or implementations. Instead, define any necessary helper structures or implementations directly within the test function when required.\n4. Whenever possible, initialize the corresponding data structures using the initialization methods provided in the context if exist.\n5. Ensure the generated function is fully implemented and can be compiled and executed directly without any missing parts.\n6. Create a minimal yet complete set of test functions, ensuring they adhere to all provided preconditions and cover boundary conditions.\n7. Do not create a test module, but include intrinsic attributes like #[test] or #[should_panic] where necessary.\n8. Do not reimplement the focal function.\n9. Do not generate more than 3 test functions.", + "unit": "The tests run in a child test module inside the crate under test, alongside the focal code. The test module already provides use super::*;. Use types and functions in that scope, or correct crate:: paths for items in the crate under test. The crate name shown in the context identifies the crate being tested, not an external dependency; do not use that name as an external-crate path. Actual external dependencies may still use their normal paths. Keep the output limited to the code requested for this stage; do not add a test module.\n", "integration": "10. Assume the tests will be placed in a separate folder within the project's root directory.\n11. Restrict your implementation to publicly accessible data structures, fields, or functions within the given context.\n12. Include only use statements that are not already present in the provided test file.\n", "context": "The context for the focal function is as follows:\n", "focal": "The function to be tested is presented as follows:\n", diff --git a/utgen/src/gene/prompt/oracle.rs b/utgen/src/gene/prompt/oracle.rs index 4348e8e926..3d4d59ee2d 100644 --- a/utgen/src/gene/prompt/oracle.rs +++ b/utgen/src/gene/prompt/oracle.rs @@ -4,6 +4,7 @@ use serde::Deserialize; #[derive(Debug, Clone, Deserialize)] pub struct OracleTemplate { system: String, + unit: String, integration: String, context: String, focal: String, @@ -22,6 +23,10 @@ impl PromptTemplate for OracleTemplate { Some(&self.integration) } + fn unit(&self) -> Option<&str> { + Some(&self.unit) + } + fn context(&self) -> &str { &self.context } diff --git a/utgen/src/gene/prompt/prefix.rs b/utgen/src/gene/prompt/prefix.rs index b5618b18de..a65c1bc5a6 100644 --- a/utgen/src/gene/prompt/prefix.rs +++ b/utgen/src/gene/prompt/prefix.rs @@ -4,6 +4,7 @@ use serde::Deserialize; #[derive(Debug, Clone, Deserialize)] pub struct PrefixTemplate { system: String, + unit: String, integration: String, context: String, focal: String, @@ -24,6 +25,10 @@ impl PromptTemplate for PrefixTemplate { Some(&self.integration) } + fn unit(&self) -> Option<&str> { + Some(&self.unit) + } + fn context(&self) -> &str { &self.context } diff --git a/utgen/src/gene/prompt/prompt.rs b/utgen/src/gene/prompt/prompt.rs index ad9c4d7ecd..e5775ae551 100644 --- a/utgen/src/gene/prompt/prompt.rs +++ b/utgen/src/gene/prompt/prompt.rs @@ -10,6 +10,9 @@ pub trait PromptTemplate { where Self: Sized; fn system(&self) -> &str; + fn unit(&self) -> Option<&str> { + None + } fn integration(&self) -> Option<&str>; fn context(&self) -> &str; fn focal(&self) -> &str; @@ -30,11 +33,17 @@ impl PromptBuilder { } pub fn build_system(&self, integration: bool) -> String { - if integration { - self.template.system().to_string() + self.template.integration().unwrap_or_default() + let scope = if integration { + self.template.integration() } else { - self.template.system().to_string() + self.template.unit() + }; + let mut system = self.template.system().to_string(); + if let Some(scope) = scope { + system.push('\n'); + system.push_str(scope); } + system } pub fn build_static(&self, brdata: &BrData, requirement: bool, context: bool) -> String { diff --git a/utgen/src/gene/prompt/test.rs b/utgen/src/gene/prompt/test.rs index b33bc56708..75236e96e7 100644 --- a/utgen/src/gene/prompt/test.rs +++ b/utgen/src/gene/prompt/test.rs @@ -4,6 +4,7 @@ use serde::Deserialize; #[derive(Debug, Clone, Deserialize)] pub struct TestTemplate { system: String, + unit: String, integration: String, context: String, focal: String, @@ -24,6 +25,10 @@ impl PromptTemplate for TestTemplate { Some(&self.integration) } + fn unit(&self) -> Option<&str> { + Some(&self.unit) + } + fn context(&self) -> &str { &self.context }