From 66c51dbcfe7331a963272154dbf1fb831b99116d Mon Sep 17 00:00:00 2001 From: cppbear Date: Thu, 8 Oct 2026 15:43:16 +0800 Subject: [PATCH] fix: reject model answers without test candidates --- docs/minimal-pipeline.md | 6 ++ docs/palm-rust-unit-test-generation.md | 4 +- scripts/check_tasks.py | 56 ++++++++++++++- utgen/README.md | 4 ++ utgen/src/gene/cot/mod.rs | 2 +- utgen/src/gene/cot/prefix_gen.rs | 95 ++++++++++++-------------- utgen/src/gene/cot/test_gen.rs | 95 ++++++++++++-------------- utgen/src/gene/mod.rs | 2 +- utgen/src/gene/test_ext.rs | 52 ++++++++++---- 9 files changed, 192 insertions(+), 124 deletions(-) diff --git a/docs/minimal-pipeline.md b/docs/minimal-pipeline.md index acfb59b919..37db7e65e9 100644 --- a/docs/minimal-pipeline.md +++ b/docs/minimal-pipeline.md @@ -74,3 +74,9 @@ Local HTTP tests exercise the shared generation/repair request path, including r `scripts/check_tasks.py` generates and repairs just two selected functions while other saved candidates remain in the same directories. It checks actual request counts, selected-only statistics, untouched unselected candidate files, and invocation records. Duplicate names and blank lines are accepted. Separate CLI tests reject empty/unknown lists and a zero request limit before configuration or target changes. With four function workers and a two-attempt limit, both generation and repair must send exactly two requests, return failure when more work needs a request, restore sources, and skip statistics. Local HTTP unit tests additionally cover retries consuming the limit, concurrent clones sharing it, and success on the final allowed attempt. None of these checks uses real model credentials. The bytes analysis check validates the checked-in two- and eight-function lists against the complete analysis index for a later model trial. + +## Answers Without Tests + +Before this fix, a reply such as `fn helper() {}` parsed as Rust and let generation return success with zero candidate tests. Direct-test and prefix generation now require at least one extracted `#[test]` function and reuse the existing three-attempt format retry loop. This does not require assertions in a prefix or judge the semantic quality of a test. + +The task regression selects one function and exercises recovery after a helper-only reply, exhaustion after three such replies, and interruption by the shared request budget, in both generation paths. It compares saved raw answers with the server's replies, including code fences, verifies the accepted code is normalized, checks request/token counts, and requires failures to leave no successful candidate or statistics. The recovered tests must compile, pass and cover production code, including a prefix with no assertion before oracle generation. Unit tests additionally cover invalid Rust, imports/comments without tests, empty test modules, and existing helpers/test attributes. All fixtures use fresh generation directories and local HTTP; no full-project model trial is involved. diff --git a/docs/palm-rust-unit-test-generation.md b/docs/palm-rust-unit-test-generation.md index 4aa4ee9e8e..a2955799ee 100644 --- a/docs/palm-rust-unit-test-generation.md +++ b/docs/palm-rust-unit-test-generation.md @@ -369,7 +369,9 @@ export PALM_CONFIG="$(pwd)/utgen/res/api.json" - `utgen/src/gene/test_ext.rs` - 用 `syn` 解析 LLM 返回代码。 - 提取 `use` 语句、`#[test]` 函数、测试模块、公共辅助代码。 - - 要求 LLM 返回可被 `syn::File` 解析的 Rust 代码。 + - 要求 LLM 返回可被 `syn::File` 解析且至少包含一个当前支持的 `#[test]` 函数的 Rust 代码；语法错误与未提取到测试均返回明确原因。 + +完整测试和 prefix 生成共用这次解析与提取，失败时沿用最多 3 次的格式尝试，耗尽后报告生成失败并跳过后续统计。这里只判断是否存在测试候选；prefix 阶段允许没有断言，候选能否编译和通过由后续执行判断。每次收到的原始回答先保存在 `utgen/generation/answer///test-attempt-N.txt` 或 `prefix-attempt-N.txt`，再移除代码围栏并解析；成功代码仍保存为 `code.rs` 或 `prefix.rs`。尝试文件路径会在后续命令中复用，不构成多轮实验历史；旧候选缓存也不会自动迁移，验收使用新的工作副本。 - `utgen/src/utils/insert.rs` - 根据 `InsertKind` 把测试插入文件末尾或模块末尾。 diff --git a/scripts/check_tasks.py b/scripts/check_tasks.py index 1640a7c5a5..f6dd9d629e 100644 --- a/scripts/check_tasks.py +++ b/scripts/check_tasks.py @@ -121,12 +121,18 @@ def do_POST(self): elif 'generate accurate test oracles' in system: answer = 'assert_eq!(actual, 2);' else: - argument = '1' if state['integration'] else '"bad"' + argument = '1' if state['integration'] or state['invalid_answers'] else '"bad"' body = ('let actual = ' + function + '(1);' if 'Omit test oracles' in system else f'assert_eq!({function}({argument}), 2);') imports = ('use palm_fixture::*;\nuse std::cmp::max;\nuse absent_crate::Missing;\n' if state['integration'] else '') answer = imports + '#[test]\nfn generated() {\n ' + body + '\n}\n' + if state['invalid_answers']: + with lock: + if len(state['code_answers']) < state['invalid_answers']: + answer = 'fn helper() {}' + answer = '```rust\n' + answer + '```' + state['code_answers'].append(answer) else: file = re.search(r'You can only modify lines \d+ to \d+ in file (.+?)\. For your answer', user).group(1) match = re.search(r'^\[(\d+)\](.*assert_eq!\(f\d+\("bad"\), 2\);.*)$', user, re.M) @@ -164,7 +170,8 @@ def do_POST(self): threading.Thread(target=server.serve_forever, daemon=True).start() env.update(PALM_API_BASE=f'http://127.0.0.1:{server.server_port}/v1', PALM_API_KEY='fixture', PALM_MODEL='fixture') - def run(label, command, limit=1, integration=False, missing_usage=False, fault=None, fail=False, backpressure=False): + def run(label, command, limit=1, integration=False, missing_usage=False, fault=None, fail=False, backpressure=False, + invalid_answers=0): monitor = work / label monitor.mkdir() (monitor / 'compiler.json').write_text(json.dumps(dict(active=0, peak=0, total=0, source_changes=0))) @@ -173,6 +180,7 @@ def run(label, command, limit=1, integration=False, missing_usage=False, fault=N with lock: state.clear() state.update(active=0, peak=0, total=0, errors=[], limit=limit, integration=integration, missing_usage=missing_usage, fault=fault, fault_sent=False) + state.update(invalid_answers=invalid_answers, code_answers=[]) ready.clear() watcher = None if backpressure: @@ -247,6 +255,50 @@ def results(directory, compiled, names=None): try: run('analyze', ['utgen', 'analyze', '-p', str(root)]) + # Real reply parsing, format retries and saving on a single selected function. + nmap = json.loads((root / 'brinfo/name_map.json').read_text()) + one_name = sorted(nmap)[0] + one_file = work / 'one-function.txt' + one_file.write_text(one_name + '\n') + for oracle in [False, True]: + stage = 'prefix' if oracle else 'test' + for case in ['recover', 'exhaust', 'budget']: + reset() + cap = 2 if oracle else 1 # Input inference consumes the first request in oracle mode. + extra = ['--functions-file', str(one_file)] + if oracle: + extra.append('--oracle') + if case == 'budget': + extra += ['--max-requests', str(cap)] + label = f'{stage}-without-tests-{case}' + measured = run(label, command('gen', 1, *extra), invalid_answers=1 if case == 'recover' else 3, + fail=case != 'recover') + attempts = {'recover': 2, 'exhaust': 3, 'budget': 1}[case] + expected_requests = attempts + int(oracle) + int(oracle and case == 'recover') + assert measured['total'] == expected_requests, measured + assert len(measured['code_answers']) == attempts + chain_dir = root / 'utgen/generation/answer' / nmap[one_name] / '000' + files = sorted(chain_dir.glob(f'{stage}-attempt-*.txt')) + assert len(files) == attempts, files + for index, file in enumerate(files): + assert file.read_text() == measured['code_answers'][index], file + accepted = chain_dir / ('prefix.rs' if oracle else 'code.rs') + candidate = root / 'utgen/generation/pre_fix' / (nmap[one_name] + '.json') + assert accepted.exists() == candidate.exists() == (case == 'recover') + report = json.loads((root / 'utgen/generation/gen-requests.json').read_text()) + assert report['reported_prompt_tokens'] == expected_requests, report + assert report['reported_completion_tokens'] == expected_requests, report + assert report['budget_exhausted'] == (case == 'budget'), report + log = (work / label / 'command.log').read_text() + assert 'No #[test] function found' in log, log[-2000:] + assert not (root / 'src/lib.rs.bak').exists() + if case == 'recover': + assert '```' not in accepted.read_text() and 'fn helper()' not in accepted.read_text() + if oracle: + assert 'assert' not in accepted.read_text(), 'a prefix need not contain an oracle' + results('result', 1, [one_name]) + else: + assert not (root / 'utgen/result').exists() seed = work / 'broken-candidates' for n in [1, 2, 4]: reset() diff --git a/utgen/README.md b/utgen/README.md index 6fe3ccb12c..093cba8fdc 100644 --- a/utgen/README.md +++ b/utgen/README.md @@ -151,6 +151,10 @@ utgen gen -p --requirement --context Generation validates the branch index, context index, context files, and source paths before modifying the target. Missing or inconsistent artifacts are errors. A failed generation task is reported instead of being silently lost before statistics. `--tasks 0` is rejected. +Direct-test and prefix answers must parse as Rust and contain at least one supported `#[test]` function. Syntax errors and answers containing only helpers, imports, comments, or an empty test module use the existing format retry loop, with at most three attempts. Exhaustion fails that function's generation and skips subsequent command statistics. This checks candidate presence; compilation and execution still determine whether a candidate works. Prefixes are accepted without assertions because oracle generation follows separately. + +Each answer returned by the request layer is saved before code-fence removal and parsing, under `utgen/generation/answer///test-attempt-N.txt` or `prefix-attempt-N.txt`, with N starting at 1. Logs identify the stage, chain, attempt and rejection reason. Accepted code retains the existing `code.rs` / `prefix.rs` layout. These paths are reused across invocations, not an invocation history; use fresh copies for separate trials. Existing cached candidates, including old empty candidates, are still skipped and are not migrated by this validation change. + Each generation job handles one focal function, including its condition chains and input/prefix/oracle stages sequentially. It holds a slot until its result enters the bounded queue, which also has capacity N. A single consumer validates candidates and their imports while other jobs can await model responses. Thus N limits active generation jobs, not the number of compiler processes or Cargo's internal build jobs; buffered results and the candidate currently being validated are separate. Generation may append an `ntest` dependency to the target's Cargo.toml. Existing `utgen/generation/pre_fix/.json` results are skipped, so use a fresh target copy for a different model, prompt, or generation mode. diff --git a/utgen/src/gene/cot/mod.rs b/utgen/src/gene/cot/mod.rs index d2f34b0219..b4068925ba 100644 --- a/utgen/src/gene/cot/mod.rs +++ b/utgen/src/gene/cot/mod.rs @@ -3,7 +3,7 @@ mod oracle_gen; mod prefix_gen; mod test_gen; -use super::{extract_test_functions, try_parse, Prompt, LLM}; +use super::{extract_test_functions, Prompt, LLM}; pub use input_infer::gen_input_range; pub use oracle_gen::gen_oracle; diff --git a/utgen/src/gene/cot/prefix_gen.rs b/utgen/src/gene/cot/prefix_gen.rs index 9ef482f324..0f1f2e9290 100644 --- a/utgen/src/gene/cot/prefix_gen.rs +++ b/utgen/src/gene/cot/prefix_gen.rs @@ -1,21 +1,12 @@ -use super::LLM; -use super::Prompt; -use super::{extract_test_functions, try_parse}; +use super::{LLM, Prompt, extract_test_functions}; use crate::types::{ChainTestAnswer, TestInfo}; -use log::{error, info}; +use log::error; use rand::Rng; use std::collections::HashSet; use std::fs; -use std::io::Write; use std::path::Path; use tokio::time::{Duration, sleep}; -fn postprocess(inputs: &mut Vec) { - for input in inputs.iter_mut() { - *input = input.replace("```rust", "").replace("```", ""); - } -} - pub async fn gen_prefix( llm: &LLM, answer_dir: &Path, @@ -35,16 +26,15 @@ pub async fn gen_prefix( user_pt += input_range; user_pt += &pt_info.depend_pt; - let mut parse_retry = 0; - while parse_retry < 3 { - if parse_retry != 0 { + for attempt in 1..=3 { + if attempt > 1 { let random_secs = { let mut rng = rand::rng(); rng.random_range(10..=30) }; sleep(Duration::from_secs(random_secs)).await; } - let (mut answers, usage_completion, usage_prompt) = + let (answers, usage_completion, usage_prompt) = match llm.fetch_answer(Some(system_pt), &user_pt, 1, false).await { Ok(answer) => answer, Err(error) => { @@ -55,46 +45,45 @@ pub async fn gen_prefix( completion_tokens += usage_completion; prompt_tokens += usage_prompt; - postprocess(&mut answers); - - answers.retain(|answer| try_parse(answer).is_ok()); - if answers.is_empty() { - error!("No valid answer found."); - parse_retry += 1; - continue; - } - - // info!("Answers: {:?}", answers); - let mut test_answer_list = vec![]; - let path = answer_dir.join(format!("{:03}/prefix.rs", id)); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - let mut file = fs::File::create(&path).unwrap(); - for (id, answer) in answers.into_iter().enumerate() { - let mut test_info_list = vec![]; - file.write_all(format!("// Answer {}\n\n", id).as_bytes()) - .unwrap(); - file.write_all((answer.clone() + "\n\n").as_bytes()) - .unwrap(); - - let prefixs = extract_test_functions(&answer); - for (attrs, prefix) in prefixs.test_fns { - let test_info = TestInfo::new(attrs, prefix, vec![]); - test_info_list.push(test_info); - } - let mut use_set = prefixs.uses.into_iter().collect::>(); - if integration { - use_set.remove("use super::*;"); + // LLM guarantees one nonempty answer. Keep it before stripping fences or parsing. + let raw_answer = &answers[0]; + let chain_dir = answer_dir.join(format!("{id:03}")); + fs::create_dir_all(&chain_dir).unwrap(); + fs::write( + chain_dir.join(format!("prefix-attempt-{attempt}.txt")), + raw_answer, + ) + .unwrap(); + let answer = raw_answer.replace("```rust", "").replace("```", ""); + let prefixes = match extract_test_functions(&answer) { + Ok(prefixes) => prefixes, + Err(reason) => { + error!("Prefix generation rejected chain {id}, attempt {attempt}/3: {reason}"); + continue; } - test_answer_list.push(ChainTestAnswer::new( - use_set.into_iter().collect(), - prefixs.has_test_mod, - prefixs.common, - test_info_list, - completion_tokens, - prompt_tokens, - )); + }; + fs::write( + chain_dir.join("prefix.rs"), + format!("// Answer 0\n\n{answer}\n\n"), + ) + .unwrap(); + let tests = prefixes + .test_fns + .into_iter() + .map(|(attrs, prefix)| TestInfo::new(attrs, prefix, vec![])) + .collect(); + let mut use_set = prefixes.uses.into_iter().collect::>(); + if integration { + use_set.remove("use super::*;"); } - return Some(test_answer_list); + return Some(vec![ChainTestAnswer::new( + use_set.into_iter().collect(), + prefixes.has_test_mod, + prefixes.common, + tests, + completion_tokens, + prompt_tokens, + )]); } None } diff --git a/utgen/src/gene/cot/test_gen.rs b/utgen/src/gene/cot/test_gen.rs index b460a08ac3..58f3347a7d 100644 --- a/utgen/src/gene/cot/test_gen.rs +++ b/utgen/src/gene/cot/test_gen.rs @@ -1,21 +1,12 @@ -use super::LLM; -use super::Prompt; -use super::{extract_test_functions, try_parse}; +use super::{LLM, Prompt, extract_test_functions}; use crate::types::{ChainTestAnswer, TestInfo}; -use log::{error, info}; +use log::error; use rand::Rng; use std::collections::HashSet; use std::fs; -use std::io::Write; use std::path::Path; use tokio::time::{Duration, sleep}; -fn postprocess(inputs: &mut Vec) { - for input in inputs.iter_mut() { - *input = input.replace("```rust", "").replace("```", ""); - } -} - pub async fn gen_test( llm: &LLM, answer_dir: &Path, @@ -31,16 +22,15 @@ pub async fn gen_test( let mut user_pt = static_pt.clone() + &conds.join(""); user_pt += &pt_info.depend_pt; - let mut parse_retry = 0; - while parse_retry < 3 { - if parse_retry != 0 { + for attempt in 1..=3 { + if attempt > 1 { let random_secs = { let mut rng = rand::rng(); rng.random_range(10..=30) }; sleep(Duration::from_secs(random_secs)).await; } - let (mut answers, usage_completion, usage_prompt) = + let (answers, usage_completion, usage_prompt) = match llm.fetch_answer(Some(system_pt), &user_pt, 1, false).await { Ok(answer) => answer, Err(error) => { @@ -51,46 +41,45 @@ pub async fn gen_test( completion_tokens += usage_completion; prompt_tokens += usage_prompt; - postprocess(&mut answers); - - answers.retain(|answer| try_parse(answer).is_ok()); - if answers.is_empty() { - error!("No valid answer found."); - parse_retry += 1; - continue; - } - - // info!("Answers: {:?}", answers); - let mut test_answer_list = vec![]; - let path = answer_dir.join(format!("{:03}/code.rs", id)); - fs::create_dir_all(path.parent().unwrap()).unwrap(); - let mut file = fs::File::create(&path).unwrap(); - for (id, answer) in answers.into_iter().enumerate() { - let mut test_info_list = vec![]; - file.write_all(format!("// Answer {}\n\n", id).as_bytes()) - .unwrap(); - file.write_all((answer.clone() + "\n\n").as_bytes()) - .unwrap(); - - let codes = extract_test_functions(&answer); - for (attrs, code) in codes.test_fns { - let test_info = TestInfo::new(attrs, vec![], vec![code]); - test_info_list.push(test_info); - } - let mut use_set = codes.uses.into_iter().collect::>(); - if integration { - use_set.remove("use super::*;"); + // LLM guarantees one nonempty answer. Keep it before stripping fences or parsing. + let raw_answer = &answers[0]; + let chain_dir = answer_dir.join(format!("{id:03}")); + fs::create_dir_all(&chain_dir).unwrap(); + fs::write( + chain_dir.join(format!("test-attempt-{attempt}.txt")), + raw_answer, + ) + .unwrap(); + let answer = raw_answer.replace("```rust", "").replace("```", ""); + let codes = match extract_test_functions(&answer) { + Ok(codes) => codes, + Err(reason) => { + error!("Test generation rejected chain {id}, attempt {attempt}/3: {reason}"); + continue; } - test_answer_list.push(ChainTestAnswer::new( - use_set.into_iter().collect(), - codes.has_test_mod, - codes.common, - test_info_list, - completion_tokens, - prompt_tokens, - )); + }; + fs::write( + chain_dir.join("code.rs"), + format!("// Answer 0\n\n{answer}\n\n"), + ) + .unwrap(); + let tests = codes + .test_fns + .into_iter() + .map(|(attrs, code)| TestInfo::new(attrs, vec![], vec![code])) + .collect(); + let mut use_set = codes.uses.into_iter().collect::>(); + if integration { + use_set.remove("use super::*;"); } - return Some(test_answer_list); + return Some(vec![ChainTestAnswer::new( + use_set.into_iter().collect(), + codes.has_test_mod, + codes.common, + tests, + completion_tokens, + prompt_tokens, + )]); } None } diff --git a/utgen/src/gene/mod.rs b/utgen/src/gene/mod.rs index 45eefab175..41e6898f54 100644 --- a/utgen/src/gene/mod.rs +++ b/utgen/src/gene/mod.rs @@ -15,7 +15,7 @@ use std::collections::HashMap; use std::path::Path; use std::sync::Arc; use std::{fs, io}; -use test_ext::{extract_test_functions, try_parse}; +use test_ext::extract_test_functions; use tokio::sync::{Semaphore, mpsc}; pub use llm::LLM; diff --git a/utgen/src/gene/test_ext.rs b/utgen/src/gene/test_ext.rs index 436126a7b8..bfe6a37b49 100644 --- a/utgen/src/gene/test_ext.rs +++ b/utgen/src/gene/test_ext.rs @@ -180,15 +180,9 @@ fn is_test_mod_attr(attr: &Attribute) -> bool { flag } -pub fn try_parse(source: &str) -> Result<(), ()> { - match syn::parse_str::(source) { - Ok(_) => Ok(()), - Err(_) => Err(()), - } -} - -pub fn extract_test_functions(source: &str) -> TestExtract { - let syntax_tree: syn::File = syn::parse_str(source).expect("Failed to parse source"); +pub fn extract_test_functions(source: &str) -> Result { + let syntax_tree: syn::File = + syn::parse_str(source).map_err(|error| format!("Invalid Rust answer: {error}"))?; let mut visitor = TestFnVisitor { uses: Vec::new(), has_test_mod: false, @@ -196,16 +190,19 @@ pub fn extract_test_functions(source: &str) -> TestExtract { specs: Vec::new(), }; visitor.visit_file(&syntax_tree); + if visitor.test_fns.is_empty() { + return Err("No #[test] function found in model answer".to_string()); + } let mut lines = source.lines().map(|s| s.to_string()).collect::>(); remove_lines_and_ranges(&mut lines, visitor.specs); trim_empty_lines(&mut lines); - TestExtract { + Ok(TestExtract { uses: visitor.uses, has_test_mod: visitor.has_test_mod, common: lines, test_fns: visitor.test_fns, - } + }) } #[cfg(test)] @@ -236,7 +233,7 @@ mod test { assert_eq!(2, 4); } "#; - let extract = extract_test_functions(source); + let extract = extract_test_functions(source).unwrap(); assert_eq!(extract.has_test_mod, false); println!("uses: {:?}", extract.uses); println!("common: {:?}", extract.common); @@ -276,7 +273,7 @@ mod test { } } "#; - let extract = extract_test_functions(source); + let extract = extract_test_functions(source).unwrap(); assert_eq!(extract.has_test_mod, true); println!("uses: {:?}", extract.uses); println!("common: {:?}", extract.common); @@ -286,4 +283,33 @@ mod test { println!("attrs: {:?}", extract.test_fns[1].0); // assert_eq!(uses.len(), 4); } + + #[test] + fn rejects_valid_rust_without_test_functions() { + for source in [ + "fn helper() {}", + "use std::cmp::max;", + "// No test generated", + "#[cfg(test)] mod tests {}", + ] { + let error = extract_test_functions(source).err().unwrap(); + assert!(error.contains("No #[test] function"), "{source}: {error}"); + } + } + + #[test] + fn invalid_rust_is_a_parse_error() { + let error = extract_test_functions("#[test] fn broken( {") + .err() + .unwrap(); + assert!(error.starts_with("Invalid Rust answer:"), "{error}"); + } + + #[test] + fn a_test_prefix_does_not_need_an_assertion() { + let extract = + extract_test_functions("#[test]\nfn prefix() { let actual = target(); }").unwrap(); + assert_eq!(extract.test_fns.len(), 1); + assert!(extract.test_fns[0].1.join("\n").contains("target()")); + } }