From 9fe136c76c2bb8a131fd8722a051de2d9eacfc1c Mon Sep 17 00:00:00 2001 From: cppbear Date: Wed, 7 Oct 2026 20:44:35 +0800 Subject: [PATCH 1/3] fix: preserve source positions during test preprocessing --- utgen/src/utils/preprocess.rs | 297 +++++++++++++++++++--------------- 1 file changed, 169 insertions(+), 128 deletions(-) diff --git a/utgen/src/utils/preprocess.rs b/utgen/src/utils/preprocess.rs index e7d3f49149..8f528246f2 100644 --- a/utgen/src/utils/preprocess.rs +++ b/utgen/src/utils/preprocess.rs @@ -1,167 +1,208 @@ -use std::fs::{self, File}; -use std::io::{self, Read, Write}; -use std::path::Path; -use syn::spanned::Spanned; -use syn::visit::Visit; -// use syn::Attribute; +use std::{fs, io, ops::Range, path::Path}; +use syn::{Meta, Token, punctuated::Punctuated, spanned::Spanned, visit::Visit}; use walkdir::WalkDir; -pub fn rename_tests_to_bak(dir: &Path) -> std::io::Result<()> { - for entry in WalkDir::new(dir).into_iter().filter_map(Result::ok) { - let path = entry.path(); - if path.is_dir() && path.file_name().unwrap() == "tests" { - let new_name = path.with_file_name("tests.bak"); - fs::rename(path, new_name)?; +/// Only the selected crate's integration-test directory is renamed. +pub fn rename_tests_to_bak(dir: &Path) -> io::Result<()> { + let tests = dir.join("tests"); + let backup = dir.join("tests.bak"); + if tests.exists() { + if backup.exists() { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + format!( + "{} already exists; use a fresh working copy", + backup.display() + ), + )); } + fs::rename(tests, backup)?; } Ok(()) } pub fn comment_out_tests(dir: &Path) -> io::Result<()> { - for entry in WalkDir::new(dir).into_iter().filter_map(Result::ok) { + let src = dir.join("src"); + if !src.is_dir() { + return Ok(()); + } + // Parse every file before writing any of them. + let mut changes = Vec::new(); + for entry in WalkDir::new(src) { + let entry = entry.map_err(io::Error::other)?; let path = entry.path(); - if path.is_file() - && path.extension().and_then(|s| s.to_str()) == Some("rs") - && path - .strip_prefix(dir) - .unwrap() - .components() - .next() - .map(|c| c.as_os_str()) - == Some("src".as_ref()) - { - comment_out_tests_in_file(path)?; + if entry.file_type().is_file() && path.extension().is_some_and(|e| e == "rs") { + let original = fs::read_to_string(path)?; + let prepared = prepare_source(&original).map_err(|err| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("{}: {err}", path.display()), + ) + })?; + if prepared != original { + changes.push((path.to_owned(), prepared)); + } } } + for (path, prepared) in changes { + fs::write(path, prepared)?; + } Ok(()) } +// Evaluate only what is known with cfg(test) disabled. Unknown feature/target +// predicates stay unknown, so a production module is never removed on a guess. +fn without_tests(meta: &Meta) -> Option { + match meta { + Meta::Path(path) if path.is_ident("test") => Some(false), + Meta::List(list) => { + let args = list + .parse_args_with(Punctuated::::parse_terminated) + .ok()?; + let values: Vec<_> = args.iter().map(without_tests).collect(); + if list.path.is_ident("not") && values.len() == 1 { + values[0].map(|v| !v) + } else if list.path.is_ident("all") { + if values.contains(&Some(false)) { + Some(false) + } else if values.iter().all(|v| *v == Some(true)) { + Some(true) + } else { + None + } + } else if list.path.is_ident("any") { + if values.contains(&Some(true)) { + Some(true) + } else if values.iter().all(|v| *v == Some(false)) { + Some(false) + } else { + None + } + } else { + None + } + } + _ => None, + } +} + fn is_test_module(attrs: &[syn::Attribute]) -> bool { attrs.iter().any(|attr| { attr.path().is_ident("cfg") && attr - .meta - .require_list() - .is_ok_and(|list| list.tokens.to_string().contains("test")) + .parse_args::() + .ok() + .is_some_and(|meta| without_tests(&meta) == Some(false)) }) } -struct TestCommenter { - mods_pos: Vec<(usize, usize, usize, usize)>, - functions_pos: Vec<(usize, usize, usize, usize)>, +#[derive(Default)] +struct TestRanges { + ranges: Vec>, } -impl TestCommenter { - fn new() -> Self { - Self { - mods_pos: Vec::new(), - functions_pos: Vec::new(), +impl<'ast> Visit<'ast> for TestRanges { + fn visit_item_mod(&mut self, item: &'ast syn::ItemMod) { + if is_test_module(&item.attrs) { + self.ranges.push(item.span().byte_range()); + } else { + syn::visit::visit_item_mod(self, item); } } -} -fn is_test_like_attr(attr: &syn::Attribute) -> bool { - let path = attr.path(); - - path.is_ident("test") - || path.segments.last().is_some_and(|seg| seg.ident == "test") -} -impl<'ast> Visit<'ast> for TestCommenter { - fn visit_item_mod(&mut self, i: &'ast syn::ItemMod) { - if is_test_module(&i.attrs) { - let span = i.span(); - self.mods_pos.push(( - span.start().line, - span.start().column, - span.end().line, - span.end().column, - )); + fn visit_item_fn(&mut self, item: &'ast syn::ItemFn) { + if item + .attrs + .iter() + .any(|a| a.path().segments.last().is_some_and(|s| s.ident == "test")) + { + self.ranges.push(item.span().byte_range()); + } else { + syn::visit::visit_item_fn(self, item); } - syn::visit::visit_item_mod(self, i); } +} - fn visit_item_fn(&mut self, i: &'ast syn::ItemFn) { - if i.attrs.iter().any(is_test_like_attr) { - let span = i.span(); - self.functions_pos.push(( - span.start().line, - span.start().column, - span.end().line, - span.end().column, - )); +fn prepare_source(source: &str) -> syn::Result { + let syntax = syn::parse_file(source)?; + // syn strips these prefixes before tokenizing; spans are relative to the + // remainder. Account for them when editing the original byte buffer. + let offset = if source.starts_with('\u{feff}') { + '\u{feff}'.len_utf8() + } else { + 0 + } + syntax.shebang.as_ref().map_or(0, |s| s.len()); + let mut visitor = TestRanges::default(); + visitor.visit_file(&syntax); + let mut bytes = source.as_bytes().to_vec(); + for range in visitor.ranges { + // Retain newlines and byte offsets, including CRLF and same-line items. + // Byte ranges include full UTF-8 characters; spaces keep valid UTF-8. + for byte in &mut bytes[range.start + offset..range.end + offset] { + if *byte != b'\n' && *byte != b'\r' { + *byte = b' '; + } } - syn::visit::visit_item_fn(self, i); } + Ok(String::from_utf8(bytes).expect("test ranges end on UTF-8 boundaries")) } -fn comment_out_tests_in_file(file_path: &Path) -> io::Result<()> { - let mut file = File::open(file_path)?; - let mut src = String::new(); - file.read_to_string(&mut src)?; +#[cfg(test)] +mod tests { + use super::*; - let syntax = syn::parse_file(&src).expect("Failed to parse file"); - let mut commenter = TestCommenter::new(); - commenter.visit_file(&syntax); - - let mut lines: Vec = src.lines().map(|s| s.to_string()).collect(); - for (start_line, start_col, end_line, end_col) in commenter.functions_pos.into_iter().rev().chain(commenter.mods_pos.into_iter()) { - if start_line == end_line { - let line = &mut lines[start_line - 1]; - let (before, target, after) = ( - &line[..start_col], - &line[start_col..end_col], - &line[end_col..], - ); - if after.trim_start().is_empty() { - *line = format!("{}// {}", before, target); - } else { - let indent = before - .chars() - .take_while(|&c| c.is_whitespace()) - .collect::(); - let new_line = format!("{}// {}", before, target); - let moved_line = format!("{}{}", indent, after.trim_start()); - *line = new_line; - lines.insert(start_line, moved_line); - } - } else { - let end_line_content = &mut lines[end_line - 1]; - let (target, after) = end_line_content.split_at_mut(end_col); - if !after.trim_start().is_empty() { - let indent = " ".repeat(end_col - 1); - let moved_line = format!("{}{}", indent, after.trim_start()); - *end_line_content = format!("// {}", target); - lines.insert(end_line, moved_line); - } else { - *end_line_content = format!("// {}", target); - } + #[test] + fn same_line_tests_do_not_move_production_code() { + let source = "#[test] fn old() { let s = \"中文\"; } pub fn keep() -> u8 { 1 }\r\n"; + let prepared = prepare_source(source).unwrap(); + assert_eq!(prepared.len(), source.len()); + let position = source.find("pub fn keep").unwrap(); + assert_eq!(&prepared[position..], &source[position..]); + assert!(prepared[..position].trim().is_empty()); + assert!(syn::parse_file(&prepared).is_ok()); + assert_eq!(prepare_source(&prepared).unwrap(), prepared); + } - for line_index in start_line..end_line - 1 { - let line_content = &mut lines[line_index]; - if !line_content.trim().is_empty() { - *line_content = format!("// {}", line_content); - } - } + #[test] + fn nested_ranges_leave_following_items_intact() { + let source = "#[cfg(test)] mod tests { #[test] fn a() {} mod nested { #[test] fn b() {} } } fn keep() {}\n#[tokio::test] async fn c() {}\nfn last() {}\n"; + let prepared = prepare_source(source).unwrap(); + assert_eq!(prepared.len(), source.len()); + let syntax = syn::parse_file(&prepared).unwrap(); + assert_eq!(syntax.items.len(), 2); + assert!(prepared.contains("fn keep() {}")); + assert!(prepared.contains("fn last() {}")); + } - let start_line_content = &mut lines[start_line - 1]; - let (before, target) = start_line_content.split_at_mut(start_col); - if !before.trim_end().is_empty() { - let indent = before - .chars() - .rev() - .take_while(|&c| c.is_whitespace()) - .collect::(); - let moved_line = format!("// {}{}", indent, target.trim_start()); - *start_line_content = format!("{}", before.trim_end()); - lines.insert(start_line, moved_line); - } else { - *start_line_content = format!("{}// {}", before, target); - } - } + #[test] + fn cfg_predicates_do_not_erase_production_modules() { + let source = "#[cfg(not(test))] mod normal { fn a() {} }\n#[cfg(feature = \"contest\")] mod feature { fn b() {} }\n#[cfg(any(test, feature = \"enabled\"))] mod shared { fn c() {} }\n#[cfg(all(test, feature = \"enabled\"))] mod tests { fn d() {} }\n"; + let prepared = prepare_source(source).unwrap(); + assert!(prepared.contains("mod normal")); + assert!(prepared.contains("mod feature")); + assert!(prepared.contains("mod shared")); + assert!(!prepared.contains("mod tests")); + assert_eq!(source.lines().count(), prepared.lines().count()); } - let modified_code = lines.join("\n"); - let mut file = File::create(file_path)?; - file.write_all(modified_code.as_bytes())?; + #[test] + fn files_without_tests_are_byte_identical() { + let source = "// 中文\r\npub fn keep() {}\r\n"; + assert_eq!(prepare_source(source).unwrap(), source); + } - Ok(()) + #[test] + fn bom_and_shebang_do_not_shift_ranges() { + for prefix in [ + "\u{feff}", + "#!/usr/bin/env rust-script\n", + "\u{feff}#!/usr/bin/env rust-script\n", + ] { + let source = format!("{prefix}#[test] fn old() {{}} fn keep() {{}}\n"); + let prepared = prepare_source(&source).unwrap(); + assert!(prepared.starts_with(prefix)); + let position = source.find("fn keep").unwrap(); + assert_eq!(&source[position..], &prepared[position..]); + assert_eq!(syn::parse_file(&prepared).unwrap().items.len(), 1); + } + } } From 8c8f5fac2c7dc46e44d65d836010e9fbdd2e0bd8 Mon Sep 17 00:00:00 2001 From: cppbear Date: Wed, 7 Oct 2026 20:45:45 +0800 Subject: [PATCH 2/3] fix: connect analysis and validate generation inputs --- focxt/src/collect_context/syntax_context.rs | 45 ++++---- utgen/src/analysis.rs | 112 +++++++++++++++++++ utgen/src/gene/mod.rs | 53 +++++---- utgen/src/lib.rs | 2 + utgen/src/main.rs | 56 ++++++---- utgen/src/types/sourceinfo.rs | 11 +- utgen/tests/cli_config.rs | 115 +++++++++++++++++++- 7 files changed, 323 insertions(+), 71 deletions(-) create mode 100644 utgen/src/analysis.rs diff --git a/focxt/src/collect_context/syntax_context.rs b/focxt/src/collect_context/syntax_context.rs index c8185ea6a9..78131ae1bb 100644 --- a/focxt/src/collect_context/syntax_context.rs +++ b/focxt/src/collect_context/syntax_context.rs @@ -508,37 +508,33 @@ fn get_syntax( fn get_syntax_recursively( output_path: &PathBuf, + impl_informations: &[ImplInformation], data: &CallsAndTypes, syntax_context: &mut SyntaxContext, fns: &HashMap, structs: &HashMap, ) { let mut remain_calls = data.calls.clone(); - let mut already_calls: HashSet = HashSet::new(); - let mut end_data: CallsAndTypes = CallsAndTypes::new_empty(); - while remain_calls.len() > 0 { - let call = remain_calls.pop(); - if let Some(call) = call { - already_calls.insert(call.clone()); - let call_file = output_path.join(String::from("new_callsandtypes/") + &call + ".json"); - // println!("{}", call_file.to_string_lossy()); - let mut file = File::open(call_file); - match file { - Ok(mut file) => { - let mut contents = String::new(); - file.read_to_string(&mut contents).unwrap(); - let new_data: CallsAndTypes = serde_json::from_str(&contents).unwrap(); - end_data.add_data(&new_data); - for new_call in new_data.calls.iter() { - if !already_calls.contains(new_call) { - remain_calls.push(new_call.clone()); - } - } - } - Err(_) => {} - } + let mut already_calls = HashSet::new(); + let mut end_data = data.clone(); + while let Some(call) = remain_calls.pop() { + if !already_calls.insert(call.clone()) { continue; } + // Calls contain full names; files are named by the compiler's encoding. + let Some(info) = impl_informations.iter().find(|info| info.full_name == call) else { + continue; // External functions do not have local context artifacts. + }; + let call_file = output_path.join("new_callsandtypes").join(format!("{}.json", info.encoded_name)); + if let Ok(file) = File::open(call_file) { + let new_data: CallsAndTypes = serde_json::from_reader(file).unwrap(); + remain_calls.extend(new_data.calls.iter().filter(|name| !already_calls.contains(*name)).cloned()); + end_data.add_data(&new_data); } } + end_data.calls.sort(); + end_data.calls.dedup(); + end_data.types.sort(); + end_data.types.dedup(); + let data = &end_data; for call in data.calls.iter() { let fn_data = fns.get(call); if let Some(fn_data) = fn_data { @@ -2095,6 +2091,7 @@ impl SyntaxContext { get_syntax(&data, &mut syntax_context, fns, structs); get_syntax_recursively( output_path, + impl_informations, &data, &mut syntax_context, fns, @@ -2183,6 +2180,7 @@ impl SyntaxContext { get_syntax(&data, &mut syntax_context, fns, structs); get_syntax_recursively( output_path, + impl_informations, &data, &mut syntax_context, fns, @@ -2244,6 +2242,7 @@ impl SyntaxContext { get_syntax(&data, &mut syntax_context, fns, structs); get_syntax_recursively( output_path, + impl_informations, &data, &mut syntax_context, fns, diff --git a/utgen/src/analysis.rs b/utgen/src/analysis.rs new file mode 100644 index 0000000000..be2fdd7eed --- /dev/null +++ b/utgen/src/analysis.rs @@ -0,0 +1,112 @@ +use crate::types::{BrData, RfocxtNameInformation, TestGenInfo}; +use std::{collections::HashMap, fs, io, path::Path, process::Command}; + +fn read_json(path: &Path) -> io::Result { + let data = fs::read(path).map_err(|e| io::Error::other(format!("{}: {e}", path.display())))?; + serde_json::from_slice(&data).map_err(|e| io::Error::other(format!("{}: {e}", path.display()))) +} + +/// Check the complete analysis interface before generation changes the target. +pub fn validate_analysis(project_dir: &Path, work_dir: &Path) -> io::Result<()> { + let map: HashMap = read_json(&work_dir.join("brinfo/name_map.json"))?; + let context: Vec = + read_json(&work_dir.join("focxt/impl_informations.json"))?; + if map.is_empty() { + return Err(io::Error::other("Analysis contains no focal functions")); + } + for (name, encoded) in &map { + let data: BrData = read_json( + &work_dir + .join("brinfo/brdata") + .join(format!("{encoded}.json")), + )?; + if data.name != *name { + return Err(io::Error::other(format!( + "Branch index mismatch for {name}" + ))); + } + let info = context + .iter() + .find(|info| info.full_name == *name) + .ok_or_else(|| io::Error::other(format!("No focal context index for {name}")))?; + let file = work_dir + .join("focxt") + .join(format!("{}.rs", info.encoded_name)); + if fs::read_to_string(&file)?.trim().is_empty() { + return Err(io::Error::other(format!("Empty focal context for {name}"))); + } + let source = project_dir.join(data.loc.get_file()); + if !source.is_file() { + return Err(io::Error::other(format!( + "Missing source for {name}: {}", + source.display() + ))); + } + } + // Also reject orphaned branch files, which the generator would otherwise read. + for entry in fs::read_dir(work_dir.join("brinfo/brdata"))? { + let path = entry?.path(); + if path.is_file() { + let data: BrData = read_json(&path)?; + if !map.get(&data.name).is_some_and(|encoded| { + path.file_stem() + .is_some_and(|stem| stem == encoded.as_str()) + }) { + return Err(io::Error::other(format!( + "Unindexed branch artifact: {}", + path.display() + ))); + } + } + } + Ok(()) +} + +pub fn validate_repair(project_dir: &Path, work_dir: &Path) -> io::Result<()> { + validate_analysis(project_dir, work_dir)?; + for entry in fs::read_dir(project_dir.join("utgen/generation/pre_fix"))? { + let path = entry?.path(); + if path.is_file() { + let _: TestGenInfo = read_json(&path)?; + } + } + Ok(()) +} + +fn run(command: &mut Command) -> io::Result<()> { + let description = format!("{command:?}"); + let status = command + .status() + .map_err(|e| io::Error::other(format!("Cannot run {description}: {e}")))?; + if !status.success() { + return Err(io::Error::other(format!( + "{description} failed with {status}" + ))); + } + Ok(()) +} + +/// Analyze one fresh, prepared crate. Existing output directories are rejected +/// rather than mixing old indices with newly generated artifacts. +pub fn analyze_project(work_dir: &Path) -> io::Result<()> { + if !work_dir.join("Cargo.toml").is_file() || !work_dir.join("src").is_dir() { + return Err(io::Error::other( + "Analysis requires a crate with Cargo.toml and src/", + )); + } + for directory in ["brinfo", "focxt"] { + if work_dir.join(directory).exists() { + return Err(io::Error::other(format!( + "{directory}/ already exists; analyze a fresh prepared working copy" + ))); + } + } + // RUSTC_WRAPPER does not by itself invalidate Cargo's existing check cache. + run(Command::new("cargo").arg("clean").current_dir(work_dir))?; + run(Command::new("cargo").arg("brinfo").current_dir(work_dir))?; + run(Command::new("focxt") + .arg("-c") + .arg(work_dir) + .current_dir(work_dir))?; + validate_analysis(work_dir, work_dir) +} diff --git a/utgen/src/gene/mod.rs b/utgen/src/gene/mod.rs index e9a855e5bd..735b509d75 100644 --- a/utgen/src/gene/mod.rs +++ b/utgen/src/gene/mod.rs @@ -6,13 +6,14 @@ mod test_ext; use crate::run::add_ntest_dependency; use crate::types::{BrData, RfocxtNameInformation, TestGenInfo}; +use crate::validate_analysis; use cot::{gen_input_range, gen_oracle, gen_prefix, gen_test}; use generation::{check_integration, check_unit, generation_tests}; use log::{error, info, warn}; -use prompt::{inputprompts, oracleprompts, prefixprompts, testprompts, Prompt}; +use prompt::{Prompt, inputprompts, oracleprompts, prefixprompts, testprompts}; use std::collections::HashMap; -use std::fs; use std::path::Path; +use std::{fs, io}; use test_ext::{extract_test_functions, try_parse}; use tokio::sync::mpsc; @@ -30,7 +31,8 @@ async fn generation_task( context: bool, oracle: bool, tx: mpsc::Sender, -) { +) -> io::Result<()> { + let name = brdata.name.clone(); info!("Generating tests for {}", brdata.name); let test_gen_info = generation_tests( &llm, @@ -45,9 +47,12 @@ async fn generation_task( oracle, ) .await; - if let Some(test_gen_info) = test_gen_info { - tx.send(test_gen_info).await.unwrap(); - } + let test_gen_info = + test_gen_info.ok_or_else(|| io::Error::other(format!("Generation failed for {name}")))?; + tx.send(test_gen_info) + .await + .map_err(|_| io::Error::other("Generation result channel closed"))?; + Ok(()) } pub async fn gen_tests_project( @@ -59,23 +64,15 @@ pub async fn gen_tests_project( requirement: bool, context: bool, oracle: bool, -) { +) -> io::Result<()> { + validate_analysis(project_dir, work_dir)?; + if tasks == 0 { + return Err(io::Error::other("--tasks must be greater than zero")); + } add_ntest_dependency(work_dir); let brdata_dir = work_dir.join("brinfo/brdata"); let map_path = work_dir.join("brinfo/name_map.json"); let focxt_name_informations_path = work_dir.join("focxt/impl_informations.json"); - if !map_path.exists() { - error!("{} does not exist", map_path.display()); - return; - } - if !brdata_dir.exists() { - error!("{} does not exist", brdata_dir.display()); - return; - } - if !focxt_name_informations_path.exists() { - error!("{} does not exist", focxt_name_informations_path.display()); - return; - } if brdata_dir.is_dir() { let nmap: HashMap = serde_json::from_str(&fs::read_to_string(&map_path).unwrap()).unwrap(); @@ -83,6 +80,7 @@ pub async fn gen_tests_project( serde_json::from_str(&fs::read_to_string(&focxt_name_informations_path).unwrap()) .unwrap(); let (tx, mut rx) = mpsc::channel(tasks); + let mut handles = Vec::new(); for entry in fs::read_dir(brdata_dir).unwrap() { let entry = entry.unwrap(); let brdata_path = entry.path(); @@ -120,7 +118,7 @@ pub async fn gen_tests_project( let project_dir = project_dir.to_path_buf(); let work_dir = work_dir.to_path_buf(); let llm = llm.clone(); - tokio::spawn(async move { + handles.push(tokio::spawn(async move { generation_task( llm, brdata, @@ -134,8 +132,8 @@ pub async fn gen_tests_project( oracle, tx, ) - .await; - }); + .await + })); } } drop(tx); @@ -153,7 +151,16 @@ pub async fn gen_tests_project( )); test_gen_info.dump_json(&file_path); } + for handle in handles { + handle + .await + .map_err(|e| io::Error::other(format!("Generation task failed: {e}")))??; + } } else { - error!("{} is not a directory", brdata_dir.display()); + return Err(io::Error::other(format!( + "{} is not a directory", + brdata_dir.display() + ))); } + Ok(()) } diff --git a/utgen/src/lib.rs b/utgen/src/lib.rs index b103a65a8a..d8418c2d5c 100644 --- a/utgen/src/lib.rs +++ b/utgen/src/lib.rs @@ -1,3 +1,4 @@ +mod analysis; mod config; mod gene; mod run; @@ -7,5 +8,6 @@ mod utils; pub use gene::gen_tests_project; pub use config::{ConfigError, LlmConfig}; pub use gene::LLM; +pub use analysis::{analyze_project, validate_analysis, validate_repair}; pub use run::{gen_test_rate, llm_fix, gen_test_rate_aggregated}; pub use utils::{comment_out_tests, rename_tests_to_bak}; diff --git a/utgen/src/main.rs b/utgen/src/main.rs index b88b3ea2e3..978b6f589a 100644 --- a/utgen/src/main.rs +++ b/utgen/src/main.rs @@ -1,10 +1,10 @@ use clap::{Args, Parser, Subcommand}; use log::info; -use utgen::{comment_out_tests, gen_test_rate, gen_tests_project, llm_fix, rename_tests_to_bak}; -use utgen::{LlmConfig, LLM}; use simplelog::{ColorChoice, ConfigBuilder, LevelFilter, TermLogger, TerminalMode}; use std::env; use std::path::PathBuf; +use utgen::{LLM, LlmConfig, analyze_project, validate_repair}; +use utgen::{comment_out_tests, gen_test_rate, gen_tests_project, llm_fix, rename_tests_to_bak}; /// Generate unit tests for a project #[derive(Debug, Parser)] @@ -63,7 +63,7 @@ struct Opts { /// Path to the project directory, can be relative to the current directory #[arg(short, long)] project_dir: PathBuf, - /// Path to the work directory(s), separated by commas (e.g., dir1,dir2), can be relative to the project directory, default to the project directory + /// Crate directories, separated by commas; relative to the current directory #[arg(short, long, use_value_delimiter = true)] work_dir: Vec, } @@ -81,20 +81,22 @@ fn init_log() { .unwrap(); } -fn get_dirs(options: Opts) -> (PathBuf, Vec) { - let current_dir = env::current_dir().unwrap(); - let project_dir = current_dir - .join(options.project_dir) - .canonicalize() - .unwrap(); +fn get_dirs(options: Opts) -> std::io::Result<(PathBuf, Vec)> { + let current_dir = env::current_dir()?; + let project_dir = current_dir.join(options.project_dir).canonicalize()?; let mut work_dirs = Vec::new(); for work_dir in options.work_dir { - work_dirs.push(current_dir.join(work_dir).canonicalize().unwrap()); + work_dirs.push(current_dir.join(work_dir).canonicalize()?); } if work_dirs.is_empty() { work_dirs.push(project_dir.clone()); } - (project_dir, work_dirs) + if !project_dir.is_dir() || work_dirs.iter().any(|dir| !dir.is_dir()) { + return Err(std::io::Error::other( + "Project and work paths must be directories", + )); + } + Ok((project_dir, work_dirs)) } #[tokio::main] @@ -106,7 +108,7 @@ async fn main() -> Result<(), Box> { match cli.command { // preprocess project Command::PreProcess { options } => { - let (project_dir, work_dirs) = get_dirs(options); + let (project_dir, work_dirs) = get_dirs(options)?; info!( "Preprocessing project at {}, with work directory(s) {:?}", project_dir.display(), @@ -114,21 +116,34 @@ async fn main() -> Result<(), Box> { ); // Rename integration tests - rename_tests_to_bak(&project_dir).unwrap(); - - // Comment out unit tests for work_dir in work_dirs { - comment_out_tests(&work_dir).unwrap(); + if work_dir.join("tests").exists() && work_dir.join("tests.bak").exists() { + return Err(std::io::Error::other( + "tests.bak already exists; use a fresh working copy", + ) + .into()); + } + comment_out_tests(&work_dir)?; + rename_tests_to_bak(&work_dir)?; } } // analyze project Command::Analyze { options } => { - let (project_dir, work_dirs) = get_dirs(options); + let (project_dir, work_dirs) = get_dirs(options)?; info!( "Analyzing project at {}, with work directory(s) {:?}", project_dir.display(), work_dirs ); + for work_dir in &work_dirs { + if work_dirs.len() != 1 || work_dir != &project_dir { + return Err(std::io::Error::other( + "analyze currently supports one standalone crate; pass its root with -p", + ) + .into()); + } + analyze_project(work_dir)?; + } } // generate unit tests Command::Gen { @@ -140,7 +155,7 @@ async fn main() -> Result<(), Box> { oracle, } => { let llm = LLM::new(LlmConfig::load(cli.config.as_deref())?); - let (project_dir, work_dirs) = get_dirs(options); + let (project_dir, work_dirs) = get_dirs(options)?; info!( "Generating tests for project at {}, with work directory(s) {:?}", project_dir.display(), @@ -159,7 +174,7 @@ async fn main() -> Result<(), Box> { context, oracle, ) - .await; + .await?; } // Collect coverage rate and pass rate for each work directory for work_dir in work_dirs.iter() { @@ -169,13 +184,14 @@ async fn main() -> Result<(), Box> { // fix unit tests Command::Fix { options, tasks } => { let llm = LLM::new(LlmConfig::load(cli.config.as_deref())?); - let (project_dir, work_dirs) = get_dirs(options); + let (project_dir, work_dirs) = get_dirs(options)?; info!( "Fixing tests for project at {}, with work directory(s) {:?}", project_dir.display(), work_dirs ); for work_dir in work_dirs.iter() { + validate_repair(&project_dir, work_dir)?; llm_fix(&llm, project_dir.clone(), work_dir.clone()).await; } //gen_test_rate_aggregated(&project_dir, false); diff --git a/utgen/src/types/sourceinfo.rs b/utgen/src/types/sourceinfo.rs index 2572e0670b..e433c0e822 100644 --- a/utgen/src/types/sourceinfo.rs +++ b/utgen/src/types/sourceinfo.rs @@ -214,10 +214,13 @@ impl<'de> serde::Deserialize<'de> for SourceInfo { return Err(serde::de::Error::custom("Invalid SourceInfo format")); } let file_path = parts[0].to_string(); - let start_line = parts[1].parse().unwrap(); - let start_column = parts[2].parse().unwrap(); - let end_line = parts[3].parse().unwrap(); - let end_column = parts[4].parse().unwrap(); + let start_line: usize = parts[1].parse().map_err(serde::de::Error::custom)?; + let start_column: usize = parts[2].parse().map_err(serde::de::Error::custom)?; + let end_line: usize = parts[3].parse().map_err(serde::de::Error::custom)?; + let end_column: usize = parts[4].parse().map_err(serde::de::Error::custom)?; + if start_line == 0 || start_column == 0 || end_line < start_line || end_column == 0 || (start_line == end_line && end_column < start_column) { + return Err(serde::de::Error::custom("Invalid source location bounds")); + } Ok(SourceInfo { file_path, start_line, diff --git a/utgen/tests/cli_config.rs b/utgen/tests/cli_config.rs index 0dbbad09dc..571b1892cf 100644 --- a/utgen/tests/cli_config.rs +++ b/utgen/tests/cli_config.rs @@ -48,7 +48,7 @@ fn help_and_non_model_commands_do_not_load_configuration() { vec!["--help"], vec!["gen", "--help"], vec!["fix", "--help"], - vec!["analyze", "-p", "."], + vec!["analyze", "--help"], vec!["pre-process", "-p", "."], ] { let output = fixture @@ -65,6 +65,119 @@ fn help_and_non_model_commands_do_not_load_configuration() { } } +#[test] +fn analyze_validates_the_crate_without_loading_model_configuration() { + let fixture = Fixture::new(); + let output = fixture + .command() + .args(["analyze", "-p", "."]) + .env("PALM_CONFIG", "missing.json") + .output() + .unwrap(); + assert!(!output.status.success()); + let error = String::from_utf8_lossy(&output.stderr); + assert!(error.contains("Analysis requires a crate"), "{error}"); + assert!(!error.contains("model configuration")); +} + +#[test] +fn missing_analysis_does_not_modify_the_manifest() { + let fixture = Fixture::new(); + let before = fs::read(fixture.0.join("Cargo.toml")).unwrap(); + for command in ["gen", "fix"] { + let output = fixture + .command() + .args([command, "-p", "."]) + .env("PALM_API_BASE", "http://127.0.0.1:9/v1") + .env("PALM_API_KEY", "unused") + .env("PALM_MODEL", "unused") + .output() + .unwrap(); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("name_map.json")); + assert_eq!(before, fs::read(fixture.0.join("Cargo.toml")).unwrap()); + } +} + +#[test] +fn preprocessing_only_changes_the_selected_crate() { + let fixture = Fixture::new(); + for name in ["selected", "untouched"] { + fs::create_dir_all(fixture.0.join(name).join("src")).unwrap(); + fs::create_dir_all(fixture.0.join(name).join("tests")).unwrap(); + fs::write( + fixture.0.join(name).join("src/lib.rs"), + "#[test] fn old() {} fn keep() {}\n", + ) + .unwrap(); + } + let output = fixture + .command() + .args(["pre-process", "-p", ".", "-w", "selected"]) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(fixture.0.join("selected/tests.bak").is_dir()); + assert!(!fixture.0.join("selected/tests").exists()); + assert!(fixture.0.join("untouched/tests").is_dir()); + assert_eq!( + fs::read_to_string(fixture.0.join("untouched/src/lib.rs")).unwrap(), + "#[test] fn old() {} fn keep() {}\n" + ); +} + +#[test] +fn preprocessing_refuses_to_overwrite_an_existing_backup() { + let fixture = Fixture::new(); + fs::create_dir_all(fixture.0.join("src")).unwrap(); + fs::create_dir(fixture.0.join("tests")).unwrap(); + fs::create_dir(fixture.0.join("tests.bak")).unwrap(); + fs::write(fixture.0.join("src/lib.rs"), "#[test] fn old() {}\n").unwrap(); + let output = fixture + .command() + .args(["pre-process", "-p", "."]) + .output() + .unwrap(); + assert!(!output.status.success()); + assert!(String::from_utf8_lossy(&output.stderr).contains("tests.bak already exists")); + assert_eq!( + fs::read_to_string(fixture.0.join("src/lib.rs")).unwrap(), + "#[test] fn old() {}\n" + ); +} + +#[cfg(unix)] +#[test] +fn analysis_stops_after_a_failed_tool_command() { + use std::os::unix::fs::PermissionsExt; + let fixture = Fixture::new(); + fs::create_dir(fixture.0.join("src")).unwrap(); + fs::create_dir(fixture.0.join("bin")).unwrap(); + let cargo = fixture.0.join("bin/cargo"); + fs::write( + &cargo, + "#!/bin/sh\necho fixture-cargo-failure >&2\nexit 42\n", + ) + .unwrap(); + fs::set_permissions(&cargo, fs::Permissions::from_mode(0o755)).unwrap(); + let output = fixture + .command() + .args(["analyze", "-p", "."]) + .env("PATH", fixture.0.join("bin")) + .output() + .unwrap(); + let error = String::from_utf8_lossy(&output.stderr); + assert!(!output.status.success()); + assert_eq!(error.matches("fixture-cargo-failure").count(), 1, "{error}"); + assert!(error.contains("42"), "{error}"); + assert!(!fixture.0.join("brinfo").exists()); + assert!(!fixture.0.join("focxt").exists()); +} + #[test] fn generation_and_repair_reject_missing_configuration_before_modifying_target() { let fixture = Fixture::new(); From 775501069c3d61b19fa44af4da824b348889256a Mon Sep 17 00:00:00 2001 From: cppbear Date: Wed, 7 Oct 2026 20:46:36 +0800 Subject: [PATCH 3/3] test: validate a deterministic minimal PALM pipeline --- .github/workflows/ci.yml | 4 + Cargo.toml | 2 +- README.md | 10 +- docs/README.md | 1 + docs/build-validation.md | 6 +- docs/minimal-pipeline.md | 47 ++++++ docs/palm-rust-unit-test-generation.md | 60 +++---- examples/README.md | 14 +- examples/minimal/.gitignore | 7 + examples/minimal/Cargo.lock | 185 +++++++++++++++++++++ examples/minimal/Cargo.toml | 9 + examples/minimal/README.md | 5 + examples/minimal/rust-toolchain.toml | 2 + examples/minimal/src/lib.rs | 31 ++++ examples/minimal/src/support.rs | 7 + examples/minimal/tests/original.rs | 2 + scripts/.gitignore | 1 + scripts/check_minimal.py | 218 +++++++++++++++++++++++++ utgen/README.md | 24 ++- 19 files changed, 585 insertions(+), 50 deletions(-) create mode 100644 docs/minimal-pipeline.md create mode 100644 examples/minimal/.gitignore create mode 100644 examples/minimal/Cargo.lock create mode 100644 examples/minimal/Cargo.toml create mode 100644 examples/minimal/README.md create mode 100644 examples/minimal/rust-toolchain.toml create mode 100644 examples/minimal/src/lib.rs create mode 100644 examples/minimal/src/support.rs create mode 100644 examples/minimal/tests/original.rs create mode 100644 scripts/.gitignore create mode 100644 scripts/check_minimal.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08d3737a41..04c43277f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,3 +36,7 @@ jobs: ./target/debug/utgen --help - name: Check the standalone example workspace run: cargo metadata --manifest-path examples/bytes/Cargo.toml --no-deps --format-version 1 > /dev/null + - name: Install the validated coverage tool + run: cargo +stable install cargo-llvm-cov --version 0.6.16 --locked + - name: Validate the minimal analysis and test pipeline + run: python3 scripts/check_minimal.py diff --git a/Cargo.toml b/Cargo.toml index cbc8c4b947..84fb9d29c9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,4 @@ [workspace] resolver = "2" members = ["brinfo", "focxt", "focxt/call_chain", "utgen", "build-utils"] -exclude = ["examples/bytes"] +exclude = ["examples/bytes", "examples/minimal"] diff --git a/README.md b/README.md index c5c5d0ee8c..ce019842e8 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ PALM combines program analysis with LLMs to generate Rust tests, repair compilat 2. Install [cargo-llvm-cov](https://github.com/taiki-e/cargo-llvm-cov): ```sh - cargo +stable install cargo-llvm-cov --locked + cargo +stable install cargo-llvm-cov --version 0.6.16 --locked ``` 3. Use the same nightly toolchain for the target crate. The analysis tools handle `lib` and `bin` targets. @@ -64,11 +64,11 @@ Run `docker/docker-run` from the same directory to mount the repository at `/hom ## Workflow -Run `cargo brinfo` in the target crate, then run `focxt -c ` to extract condition chains and context. The current `utgen analyze` command only logs its arguments; these analysis steps must be run explicitly. +On a fresh working copy of a standalone crate, run `utgen pre-process -p ` before `utgen analyze -p `. Analysis clears Cargo's check cache, runs brinfo and focxt, and verifies their outputs. It rejects existing `brinfo/` or `focxt/` directories so previous analysis results cannot be mixed into the run. -Use `utgen pre-process`, `utgen gen`, and `utgen fix` for preprocessing, generation, and compilation repair. Pass `--requirement --context` to `utgen gen` to include path constraints and focal context. Preprocessing and test checks modify the target tree; use a working copy of the target project. +Then use `utgen gen` and `utgen fix` for generation and compilation repair. Pass `--requirement --context` to `utgen gen` to include path constraints and focal context. Preprocessing and test checks modify the target tree; use a working copy of the target project. -See the [bytes example](examples/README.md) and [utgen usage](utgen/README.md) for the commands and result locations. +Start with the [minimal pipeline check](docs/minimal-pipeline.md), which uses fixed local model responses and verifies analysis, generation, repair, source restoration, and coverage. The [bytes example](examples/README.md) remains a larger follow-up target. See [utgen usage](utgen/README.md) for command details. ## Development checks @@ -79,7 +79,7 @@ cargo build --workspace --locked cargo test --workspace --locked ``` -Default tests use local model-response fixtures and do not contact a model service. The real-service test is opt-in; see [utgen testing](utgen/README.md#testing). Dependency downloads may still require network access. [Build validation](docs/build-validation.md) records the baseline and checks performed; the GitHub Actions workflow runs the basic checks on Linux. +Default tests use local model-response fixtures and do not contact a model service. The real-service test is opt-in; see [utgen testing](utgen/README.md#testing). Dependency downloads may still require network access. [Build validation](docs/build-validation.md) records the baseline and checks performed; the GitHub Actions workflow runs these checks and the minimal pipeline on Linux. ## Documentation diff --git a/docs/README.md b/docs/README.md index b9e326d763..093c5ec5a3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,6 +6,7 @@ PALM's implementation, technical guide, and conference materials are available i | --- | --- | | [Project README](../README.md) | Prerequisites, installation, and workspace layout. | | [Technical guide in Chinese](palm-rust-unit-test-generation.md) | Condition chains, context construction, generation, repair, and usage. | +| [Minimal pipeline](minimal-pipeline.md) | Deterministic program-analysis, generation, repair, and coverage validation. | | [bytes example](../examples/README.md) | Commands for analyzing a working copy of the bundled target. | | [utgen reference](../utgen/README.md) | Configuration, CLI behavior, and output files. | | [ASE 2025 materials](ase2025/README.md) | Original poster and presentation PDFs. | diff --git a/docs/build-validation.md b/docs/build-validation.md index f6a3bae81d..653163dc5e 100644 --- a/docs/build-validation.md +++ b/docs/build-validation.md @@ -50,14 +50,14 @@ The installer retains Cargo output, uses the repository directory to select its The proxy regression configures only child-process environments, so parallel tests do not change one another's environment. The ordinary LLM constructor retains the SDK's default proxy behavior. The fixtures inject a separate HTTP client with proxies disabled. A test-only `reqwest` dependency reuses the existing locked version; dependency versions, checksums, and lockfile format were not changed. -The [GitHub Actions workflow](../.github/workflows/ci.yml) configures the build, default tests, CLI help checks, installer syntax/help, and standalone bytes metadata checks on Ubuntu 24.04. Its YAML was parsed locally. A hosted workflow run remains to be verified after publication; this document does not claim a Linux CI pass. +The [GitHub Actions workflow](../.github/workflows/ci.yml) configures the build, default tests, CLI help checks, installer syntax/help, and standalone bytes metadata checks on Ubuntu 24.04. Its YAML was parsed locally. The [Linux branch run](https://github.com/cppbear/PALM/actions/runs/37619301888) and [PR run](https://github.com/cppbear/PALM/actions/runs/37619401144) passed before [PR #5](https://github.com/cppbear/PALM/pull/5) was merged. Subsequent pipeline validation is recorded in [minimal pipeline](minimal-pipeline.md). ## Validation Limits and Next Batch - No real model service was contacted. Local fixtures validate request construction and response handling, not model compatibility or generation quality. - The tests added here do not exercise preprocessing correctness, focal-context completeness, generated-test insertion, compilation repair, or coverage measurement end to end. -- Docker was not built: the local Docker daemon was unavailable. Linux runtime validation remains pending. +- Docker was not built: the local Docker daemon was unavailable. The basic Linux CI passed; the Docker image itself remains unverified. - The pinned toolchain and existing dependency versions were retained; `Cargo.lock` adds only the test dependency edge described above. Compatibility of the coverage tool and target-crate dependencies needs separate verification. -- `utgen analyze` remains a logging-only command. Request concurrency, source restoration, cache validity, and coverage-baseline semantics are unchanged. +- At this baseline, `utgen analyze` was a logging-only command. The following batch connects analysis and adds a minimal end-to-end check; see [minimal pipeline](minimal-pipeline.md). Concurrency, general recovery, and cache invalidation remain follow-up work. The next batch should use a small target crate to verify the relationship between prepared source files, analysis locations, function identities, focal context, and generated tests. It should establish failure propagation and coverage semantics before proceeding to a large bytes run. diff --git a/docs/minimal-pipeline.md b/docs/minimal-pipeline.md new file mode 100644 index 0000000000..0490483af0 --- /dev/null +++ b/docs/minimal-pipeline.md @@ -0,0 +1,47 @@ +# Minimal Pipeline Validation + +This batch follows the [build and configuration baseline](build-validation.md), merged through [PR #5](https://github.com/cppbear/PALM/pull/5). It validates one standalone crate with fixed local model responses, the pinned nightly, and `cargo-llvm-cov 0.6.16`. + +## Run the Check + +From the repository root: + +```sh +cargo build --workspace --locked +cargo +stable install cargo-llvm-cov --version 0.6.16 --locked +python3 scripts/check_minimal.py +``` + +No model credentials are required. The model endpoint is a temporary loopback HTTP server owned by the script; target commands receive only a dummy key. Dependency downloads may require network access. + +The script prints the temporary directory containing source copies, command logs, model-request counts, coverage, and generated results. Use `--work-dir ` to select it, `--bin-dir ` for a custom tool build directory, or `--analysis-only` to stop before coverage/generation. Work directories must be new. The [fixture](../examples/minimal/README.md) is never modified. + +## Verified Contracts + +- Original tests run on an untouched copy. A separate prepared copy is used for generated tests. +- Preprocessing removes test-only source ranges while preserving newlines and byte offsets, including a test and production function sharing one line. Repeating preprocessing is idempotent. +- `utgen analyze` succeeds even after a prior `cargo check`, since it clears the check cache before invoking the compiler wrapper. It rejects existing analysis output directories. +- brinfo and focxt agree on five complete function names. Source locations identify the exported focal code, accounting for brinfo's indentation normalization. +- The `Gauge::label` context contains its struct, the focal method, and the declarations for `classify`, `positive`, and the transitive `threshold` dependency. +- Missing context and malformed source locations fail before generation changes the manifest or source files. Source-location parse errors no longer panic. +- Six model responses produce six candidate tests. One candidate deliberately passes a string to an integer parameter and fails compilation; a numbered ChangeLog response repairs it through the actual repair path. +- Both branches of `classify` are covered. Before repair, the intentionally invalid `double` candidate has zero compilable tests. After repair, all candidates compile and pass. +- Source files and the target manifest match their prepared state after generation and repair; no source backup remains after successful completion. + +The regression script runs in Linux CI after the ordinary workspace tests. It uses the real analyzers, compiler, coverage tool, prompt builder, response parser, candidate checks, and repair machinery. Model replies are deterministic fixtures; these results do not measure LLM quality. + +## Coverage Interpretation + +`original/original-coverage.json` measures all original unit and integration tests on the untouched copy. The check requires both original `classify` branches to have executed. + +`generated/utgen/result/` and `generated/utgen/fixed_result/` contain the tool's per-function statistics for generated candidates. The script checks the known branch count and execution outcomes directly. The separate legacy `utgen/original_result.json` restores only the integration-test backup; it must not be presented as coverage of all original tests. Combined original-plus-generated coverage is not part of this check. + +## Changes and Limits + +`utgen analyze` now invokes the tools and validates their artifacts. Its supported scope is one standalone crate with `Cargo.toml` and `src/`. Generation rejects incomplete indices/context and propagates failed asynchronous generation tasks. Repair validates the analysis and pre-repair JSON before starting. + +The preprocessor parses `cfg` predicates conservatively, preserves `cfg(not(test))` and unknown production features, handles nested test ranges and UTF-8 safely, and refuses to overwrite `tests.bak`. Whitespace replacement preserves bytes/newlines, not necessarily character columns inside removed UTF-8 ranges; analysis runs after preprocessing. + +Recursive context lookup now resolves full function names through the compiler's encoded-name index and consumes the accumulated dependencies. External functions without local artifacts remain outside this source context. + +The local macOS/Apple Silicon run passed using six generation responses and one repair response. Real-model behavior, bytes-scale execution, general multi-crate support, request concurrency limits, hard process timeouts, recovery after forced termination, and parameter-aware cache invalidation remain later work. Git history is retained. diff --git a/docs/palm-rust-unit-test-generation.md b/docs/palm-rust-unit-test-generation.md index 8e2eb45669..71f4d51ca1 100644 --- a/docs/palm-rust-unit-test-generation.md +++ b/docs/palm-rust-unit-test-generation.md @@ -260,12 +260,12 @@ Rust 的一个核心挑战是 MIR 的去糖。比如高级语法 `match color` ```text utgen pre-process --project-dir [--work-dir ...] -utgen analyze --project-dir [--work-dir ...] +utgen analyze --project-dir utgen gen --project-dir [--work-dir ...] [--tasks ] [--integration] [--requirement] [--context] [--oracle] utgen fix --project-dir [--work-dir ...] [--tasks ] ``` -注意:`analyze` 子命令目前只打印日志,没有实际调用 `cargo brinfo` 或 `focxt`。实际运行时仍需要手动执行分析步骤。 +`analyze` 现在执行 `cargo clean`、`cargo brinfo` 和 `focxt`,随后核对分析索引及上下文文件。当前支持用 `-p` 指定单个独立 crate;目录中已有 `brinfo/` 或 `focxt/` 时会拒绝执行,应使用新的工作副本,避免混入旧分析结果。 参数含义: @@ -466,7 +466,7 @@ rustup component add --toolchain nightly-2025-03-19 rust-src rustc-dev llvm-tool 安装覆盖率工具: ```sh -cargo +stable install cargo-llvm-cov --locked +cargo +stable install cargo-llvm-cov --version 0.6.16 --locked ``` 目标 crate 根目录建议包含: @@ -523,41 +523,47 @@ focxt --help utgen --help ``` -### 10.3 对目标 crate 运行程序分析 - -进入目标 crate 根目录,先提取条件链: +### 10.3 预处理项目 ```sh -cargo clean -cargo brinfo +utgen pre-process -p ``` -然后构建上下文: +如果项目是 workspace 或需要只处理部分子目录: ```sh -focxt -c +utgen pre-process -p -w -w ``` -`focxt` 内部会运行 `cargo call-chain`,因此需要先安装 `focxt/call_chain`。 +预处理会: + +- 将所选 crate 根目录的 `tests` 重命名为 `tests.bak`;已有备份不会被覆盖。 +- 将 `src` 中测试专属模块,以及带 `#[test]` 或路径末段为 `test` 的属性的函数替换为空白,保持换行和字节位置。 + +模块识别解析 `cfg` 表达式,仅移除能确定在关闭 `test` 时不可用的模块;保留 `cfg(not(test))`,对未知 feature/target 条件保守处理。预处理修改目标项目源码树,没有完整的反向预处理命令,应在临时副本中运行。先预处理,再对准备好的源码分析;空白替换保持字节位置,UTF-8 字符列号仍应以处理后的源码为准。`-w` 的相对路径基于当前命令执行目录。 -### 10.4 预处理项目 +### 10.4 对目标 crate 运行程序分析 + +预处理后,推荐对新的独立 crate 副本执行: ```sh -utgen pre-process -p +utgen analyze -p ``` -如果项目是 workspace 或需要只处理部分子目录: +也可以手动进入准备好的目标 crate 根目录提取条件链。已有 `cargo check` 缓存可能使编译器包装器不执行,因此需要先清理: ```sh -utgen pre-process -p -w -w +cargo clean +cargo brinfo ``` -预处理会: +然后构建上下文: -- 将已有 `tests` 目录重命名为 `tests.bak`。 -- 注释 `src` 下被识别为测试模块的模块,以及带 `#[test]` 或路径末段为 `test` 的属性的函数。 +```sh +focxt -c +``` -模块识别使用 `cfg` 属性文本中是否包含 `test` 的判断,不是对任意条件编译表达式求值。预处理会修改目标项目源码树,且没有完整的反向预处理命令,建议在临时副本中运行。`-w` 的相对路径基于当前命令执行目录。 +`focxt` 内部会运行 `cargo call-chain`,因此需要先安装 `focxt/call_chain`。 ### 10.5 生成测试 @@ -608,20 +614,18 @@ export PALM_CONFIG="$palm_repo/utgen/res/api.json" palm_example_dir="$(mktemp -d "${TMPDIR:-/tmp}/palm-bytes.XXXXXX")" cp -R "$palm_repo/examples/bytes/." "$palm_example_dir/" cd "$palm_example_dir" -cargo clean -cargo brinfo -focxt -c "$palm_example_dir" +utgen pre-process -p "$palm_example_dir" +utgen analyze -p "$palm_example_dir" ``` 按单元测试模式生成和修复: ```sh -utgen pre-process -p "$palm_example_dir" utgen gen -p "$palm_example_dir" --requirement --context utgen fix -p "$palm_example_dir" ``` -切换到 integration、其他提示词或其他模型时,使用新的目标副本并重新分析,避免复用旧的生成缓存。更详细的示例见 [examples/README.md](../examples/README.md)。 +切换到 integration、其他提示词或其他模型时,使用新的目标副本并重新分析,避免复用旧的生成缓存。更详细的示例见 [examples/README.md](../examples/README.md)。当前完整回归使用 [minimal 示例](minimal-pipeline.md);bytes 的完整运行仍是后续验证项。 完成后重点查看临时目标目录中的以下路径: @@ -670,11 +674,11 @@ focxt/impl_informations.json focxt/.rs ``` -缺少时按顺序重新运行: +缺少或不一致时,生成器会在修改目标文件之前报错。请准备新的目标副本,再按顺序执行: ```sh -cargo brinfo -focxt -c +utgen pre-process -p +utgen analyze -p ``` ### 11.5 集成测试生成数量少 @@ -685,7 +689,7 @@ integration 模式先按 `brinfo` 的 `visible` 标志筛选函数,并且 `utg 会修改目标项目的步骤包括: -- `utgen pre-process`:重命名 `tests`,注释已有测试。 +- `utgen pre-process`:重命名所选 crate 的 `tests`,将测试源码区间替换为空白。 - `utgen gen`:可能追加 `ntest` 依赖,写入临时测试文件或插入测试做编译检查。 - `utgen fix`:临时插入测试并恢复,结束时删除备份。 diff --git a/examples/README.md b/examples/README.md index 1f93764f57..22dc71793e 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,5 +1,7 @@ # Examples +Start with the [minimal pipeline](../docs/minimal-pipeline.md) for an automated, no-credentials validation. The commands below describe the larger bytes target; the complete pipeline is currently regression-tested on the minimal crate. + ## bytes The bundled target is [tokio-rs/bytes](https://github.com/tokio-rs/bytes), version 1.10.0. It is excluded from PALM's tool workspace and retains its own source files, tests, and license. @@ -10,7 +12,7 @@ Follow the [root installation instructions](../README.md#installation), includin ### Prepare a Working Copy -Preprocessing comments out existing tests and renames test directories. Generation can add `ntest` to the target's Cargo.toml, and compilation checks temporarily insert tests. Use a separate copy to preserve the bundled example. +Preprocessing blanks existing test source ranges and renames test directories. Generation can add `ntest` to the target's Cargo.toml, and compilation checks temporarily insert tests. Use a separate copy to preserve the bundled example. Run the following from the PALM repository root, keeping the same shell for subsequent commands: @@ -22,20 +24,18 @@ cp -R "$palm_repo/examples/bytes/." "$palm_example_dir/" cd "$palm_example_dir" ``` -### Extract Information +### Preprocess and Analyze ```sh -cargo clean -cargo brinfo -focxt -c "$palm_example_dir" +utgen pre-process -p "$palm_example_dir" +utgen analyze -p "$palm_example_dir" ``` -The last command also runs `cargo call-chain`. The `utgen analyze` subcommand does not currently perform these analysis steps. +Analysis runs brinfo and focxt (including call-chain) on the prepared source. If a function lacks corresponding context, generation reports the missing artifact; support for larger targets is still being validated. ### Generate and Repair Unit Tests ```sh -utgen pre-process -p "$palm_example_dir" utgen gen -p "$palm_example_dir" --requirement --context utgen fix -p "$palm_example_dir" ``` diff --git a/examples/minimal/.gitignore b/examples/minimal/.gitignore new file mode 100644 index 0000000000..a7e65ce947 --- /dev/null +++ b/examples/minimal/.gitignore @@ -0,0 +1,7 @@ +/target/ +/brinfo/ +/focxt/ +/utgen/ +/tests.bak/ +/coverage.* +/error_output.json diff --git a/examples/minimal/Cargo.lock b/examples/minimal/Cargo.lock new file mode 100644 index 0000000000..c749970c88 --- /dev/null +++ b/examples/minimal/Cargo.lock @@ -0,0 +1,185 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "ntest" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb183f0a1da7a937f672e5ee7b7edb727bf52b8a52d531374ba8ebb9345c0330" +dependencies = [ + "ntest_test_cases", + "ntest_timeout", +] + +[[package]] +name = "ntest_test_cases" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6913433c6319ef9b2df316bb8e3db864a41724c2bb8f12555e07dc4ec69d3db1" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ntest_timeout" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9224be3459a0c1d6e9b0f42ab0e76e98b29aef5aba33c0487dfcf47ea08b5150" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "palm_fixture" +version = "0.1.0" +dependencies = [ + "ntest", +] + +[[package]] +name = "proc-macro-crate" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +dependencies = [ + "toml_edit", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_edit" +version = "0.25.15+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1340ea94a5856333492c9064b02c778b191dd2c853778d9609debdcdfea3a614" +dependencies = [ + "indexmap", + "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +dependencies = [ + "memchr", +] diff --git a/examples/minimal/Cargo.toml b/examples/minimal/Cargo.toml new file mode 100644 index 0000000000..5a1775725b --- /dev/null +++ b/examples/minimal/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "palm_fixture" +version = "0.1.0" +edition = "2021" + +[workspace] + +[dev-dependencies] +ntest = "=0.9.3" diff --git a/examples/minimal/README.md b/examples/minimal/README.md new file mode 100644 index 0000000000..caae50d058 --- /dev/null +++ b/examples/minimal/README.md @@ -0,0 +1,5 @@ +# Minimal PALM Target + +This standalone library exercises five focal functions: a branch, a struct method, an inline module, and a two-level cross-module call chain. It also includes existing unit/integration tests and a test sharing a line with production code. Its test dependencies are locked for the pinned nightly. + +Run the [minimal pipeline check](../../docs/minimal-pipeline.md) from the PALM repository root. The script copies this directory before preprocessing or analysis; generated data is not written into this fixture. diff --git a/examples/minimal/rust-toolchain.toml b/examples/minimal/rust-toolchain.toml new file mode 100644 index 0000000000..81febe9658 --- /dev/null +++ b/examples/minimal/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "nightly-2025-03-19" diff --git a/examples/minimal/src/lib.rs b/examples/minimal/src/lib.rs new file mode 100644 index 0000000000..51265c26b0 --- /dev/null +++ b/examples/minimal/src/lib.rs @@ -0,0 +1,31 @@ +mod support; + +#[cfg(test)] +mod tests { + #[test] + fn existing_unit_test() { assert_eq!(super::classify(2), 1); } +} + +pub struct Gauge { + pub value: i32, +} + +impl Gauge { + pub fn label(&self) -> i32 { + classify(self.value) + } +} + +pub fn classify(value: i32) -> i32 { + if support::positive(value) { + 1 + } else { + 0 + } +} + +pub mod nested { + pub fn double(value: i32) -> i32 { + value * 2 + } +} diff --git a/examples/minimal/src/support.rs b/examples/minimal/src/support.rs new file mode 100644 index 0000000000..184396905b --- /dev/null +++ b/examples/minimal/src/support.rs @@ -0,0 +1,7 @@ +#[test] fn before_helper() { assert!(positive(1)); } pub fn positive(value: i32) -> bool { + threshold(value) +} + +fn threshold(value: i32) -> bool { + value > 0 +} diff --git a/examples/minimal/tests/original.rs b/examples/minimal/tests/original.rs new file mode 100644 index 0000000000..190dfb4d49 --- /dev/null +++ b/examples/minimal/tests/original.rs @@ -0,0 +1,2 @@ +#[test] +fn existing_integration_test() { assert_eq!(palm_fixture::classify(-1), 0); } diff --git a/scripts/.gitignore b/scripts/.gitignore new file mode 100644 index 0000000000..a348e5040c --- /dev/null +++ b/scripts/.gitignore @@ -0,0 +1 @@ +/__pycache__/ diff --git a/scripts/check_minimal.py b/scripts/check_minimal.py new file mode 100644 index 0000000000..3b09ba311e --- /dev/null +++ b/scripts/check_minimal.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Exercise PALM on an isolated, deterministic crate; never use a real model.""" +import argparse +import hashlib +import http.server +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import threading + +REPO = Path(__file__).resolve().parents[1] + + +def snapshot(root): + return {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in sorted((root / 'src').rglob('*.rs'))} | { + 'Cargo.toml': hashlib.sha256((root / 'Cargo.toml').read_bytes()).hexdigest()} + + +def source_fragment(root, loc): + file, start, col, end, last = loc.rsplit(':', 4) + start, col, end, last = map(int, (start, col, end, last)) + lines = (root / file).read_text().splitlines() + return '\n'.join(line[(col - 1 if n == start else 0):(last - 1 if n == end else len(line))] + for n, line in enumerate(lines, 1) if start <= n <= end) + + +def check_analysis(root): + names = json.loads((root / 'brinfo/name_map.json').read_text()) + expected = {'palm_fixture::classify', 'palm_fixture::Gauge::label', + 'palm_fixture::nested::double', 'palm_fixture::support::positive', + 'palm_fixture::support::threshold'} + assert set(names) == expected, names + infos = {item['full_name']: item for item in json.loads((root / 'focxt/impl_informations.json').read_text())} + assert set(infos) == expected, infos + for name, encoded in names.items(): + data = json.loads((root / f'brinfo/brdata/{encoded}.json').read_text()) + assert data['name'] == name + # brinfo removes leading indentation when exporting focal code. + assert [line.lstrip() for line in source_fragment(root, data['loc']).splitlines()] == [line.lstrip() for line in data['code']], name + assert (root / f"focxt/{infos[name]['encoded_name']}.rs").is_file(), name + context = (root / f"focxt/{infos['palm_fixture::Gauge::label']['encoded_name']}.rs").read_text() + for declaration in ['struct Gauge', 'fn label', 'fn classify', 'fn positive', 'fn threshold']: + assert declaration in context, (declaration, context) + branch = json.loads((root / f"brinfo/brdata/{names['palm_fixture::classify']}.json").read_text()) + assert {c['conds'][0]['value'] for c in branch['cond_chains'] if c['min_set']} == {'true', 'false'} + return names + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--bin-dir', type=Path, default=REPO / 'target/debug') + parser.add_argument('--work-dir', type=Path, help='New directory for copies, logs and results') + parser.add_argument('--analysis-only', action='store_true') + args = parser.parse_args() + work = args.work_dir.resolve() if args.work_dir else Path(tempfile.mkdtemp(prefix='palm-minimal-')).resolve() + if args.work_dir: + work.mkdir(parents=True, exist_ok=False) + print(f'Validation artifacts: {work}', flush=True) + env = os.environ.copy() + env.pop('CARGO_TARGET_DIR', None) + for name in ['PALM_CONFIG', 'PALM_API_BASE', 'PALM_API_KEY', 'PALM_MODEL']: + env.pop(name, None) + env['PATH'] = str(args.bin_dir.resolve()) + os.pathsep + env['PATH'] + # Only child commands receive this override; preserve proxies for registry downloads. + for name in ['NO_PROXY', 'no_proxy']: + env[name] = ','.join(filter(None, [env.get(name, ''), '127.0.0.1', 'localhost'])) + logs = work / 'logs' + logs.mkdir() + original = work / 'original' + target = work / 'generated' + for copy in [original, target]: + shutil.copytree(REPO / 'examples/minimal', copy, + ignore=shutil.ignore_patterns('target', 'brinfo', 'focxt', 'utgen', 'tests.bak')) + + def run(label, command, cwd=target, expected=0): + print(label, flush=True) + result = subprocess.run(command, cwd=cwd, env=env, capture_output=True, text=True, timeout=240) + log = logs / (label + '.log') + log.write_text(result.stdout + result.stderr) + assert (result.returncode == 0) == (expected == 0), f'{label}: exit {result.returncode}; see {log}\n{(result.stdout + result.stderr)[-12000:]}' + return result + + run('original-tests', ['cargo', 'test', '--locked', '--tests'], original) + run('preprocess', ['utgen', 'pre-process', '-p', str(target)]) + assert (target / 'tests.bak/original.rs').exists() and not (target / 'tests').exists() + for file in ['src/lib.rs', 'src/support.rs']: + old, new = (original / file).read_bytes(), (target / file).read_bytes() + assert len(old) == len(new) + assert [i for i, b in enumerate(old) if b == 10] == [i for i, b in enumerate(new) if b == 10] + prepared = snapshot(target) + run('preprocess-again', ['utgen', 'pre-process', '-p', str(target)]) + assert snapshot(target) == prepared + run('prepared-check', ['cargo', 'check', '--locked']) + run('analyze', ['utgen', 'analyze', '-p', str(target)]) + names = check_analysis(target) + assert snapshot(target) == prepared + run('reject-stale-analysis', ['utgen', 'analyze', '-p', str(target)], expected=1) + if args.analysis_only: + print('Analysis contract passed for all five functions.', flush=True) + return + + # Invalid analysis must fail before requests or manifest/source mutations. + env.update(PALM_API_BASE='http://127.0.0.1:9/v1', PALM_API_KEY='unused', PALM_MODEL='unused') + context = target / f"focxt/{names['palm_fixture::classify']}.rs" + saved = context.read_bytes() + context.unlink() + try: + run('missing-context', ['utgen', 'gen', '-p', str(target)], expected=1) + assert snapshot(target) == prepared + assert not (target / 'utgen').exists() + finally: + context.write_bytes(saved) + artifact = target / f"brinfo/brdata/{names['palm_fixture::classify']}.json" + saved = artifact.read_bytes() + data = json.loads(saved) + data['loc'] = 'src/lib.rs:not-a-number:1:2:1' + artifact.write_text(json.dumps(data)) + try: + failure = run('malformed-location', ['utgen', 'gen', '-p', str(target)], expected=1) + assert 'panicked' not in failure.stderr + assert snapshot(target) == prepared + finally: + artifact.write_bytes(saved) + + version = run('coverage-version', ['cargo', 'llvm-cov', '--version']) + assert version.stdout.strip() == 'cargo-llvm-cov 0.6.16', version.stdout + run('original-coverage', ['cargo', 'llvm-cov', '--tests', '--branch', '--json', '--output-path', 'original-coverage.json'], original) + original_coverage = json.loads((original / 'original-coverage.json').read_text()) + original_file = next(file for data in original_coverage['data'] for file in data['files'] + if Path(file['filename']) == original / 'src/lib.rs') + condition_line = (original / 'src/lib.rs').read_text().splitlines().index(' if support::positive(value) {') + 1 + # Library and unit-test binaries may emit separate records for one location. + records = [branch for branch in original_file['branches'] if branch[0] == condition_line] + assert any(branch[4] > 0 for branch in records) and any(branch[5] > 0 for branch in records) + state = {'generation': 0, 'repair': 0, 'errors': []} + + class Model(http.server.BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_POST(self): + try: + request = json.loads(self.rfile.read(int(self.headers['Content-Length']))) + assert self.path == '/v1/chat/completions' + assert self.headers['Authorization'] == 'Bearer local-fixture-key' + user = request['messages'][-1]['content'] + if 'The function to be tested is presented as follows:' in user: + focal = user.split('The function to be tested is presented as follows:', 1)[1] + function = re.search(r'\bfn (\w+)\(', focal).group(1) + positive = '// constraint: support::positive(value) is true' in user + body = { + 'classify': f'assert_eq!(classify({2 if positive else -1}), {1 if positive else 0});', + 'label': 'assert_eq!(Gauge { value: 2 }.label(), 1);', + 'positive': 'assert!(positive(2));', + 'threshold': 'assert!(threshold(2));', + 'double': 'assert_eq!(double("bad"), 4);', + }[function] + answer = '#[test]\nfn generated() {\n ' + body + '\n}\n' + state['generation'] += 1 + else: + file = re.search(r'You can only modify lines \d+ to \d+ in file (.+?)\. For your answer', user).group(1) + match = re.search(r'^\[(\d+)\](.*assert_eq!\(double\("bad"\), 4\);.*)$', user, re.M) + assert match, 'repair prompt lacks numbered failing statement' + number, old = match.groups() + new = old.replace('double("bad")', 'double(2)') + answer = f'ChangeLog:1@{file}\nFixDescription: Use an integer argument.\nOriginalCode@{number}-{number}:\n[{number}]{old}\nFixedCode@{number}-{number}:\n[{number}]{new}\n' + state['repair'] += 1 + response = {'id': 'fixture', 'object': 'chat.completion', 'created': 0, 'model': 'fixture', + 'choices': [{'index': 0, 'message': {'role': 'assistant', 'content': answer}, 'finish_reason': 'stop'}], + 'usage': {'prompt_tokens': 1, 'completion_tokens': 1, 'total_tokens': 2}} + code = 200 + except Exception as error: + state['errors'].append(str(error)) + response = {'error': {'message': str(error), 'type': 'fixture_error'}} + code = 400 + body = json.dumps(response).encode() + self.send_response(code) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), Model) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + env.update(PALM_API_BASE=f'http://127.0.0.1:{server.server_port}/v1', PALM_API_KEY='local-fixture-key', PALM_MODEL='fixture') + try: + run('generate', ['utgen', 'gen', '-p', str(target), '--requirement', '--context']) + assert snapshot(target) == prepared + run('repair', ['utgen', 'fix', '-p', str(target)]) + assert snapshot(target) == prepared + assert not state['errors'], state + assert state['generation'] == 6 and state['repair'] >= 1, state + for directory in ['result', 'fixed_result']: + results = [json.loads(p.read_text()) for p in (target / 'utgen' / directory).glob('*.json')] + assert len(results) == 5, (directory, results) + if directory == 'fixed_result': + assert all(r['tests_compiled'] == r['tests'] and r['tests_passed'] == r['tests_run'] and r['tests_run'] > 0 for r in results), results + classify = next(r for r in results if r['function_name'] == 'palm_fixture::classify') + assert classify['branches_covered'] == 2 and classify['branches'] == 2, classify + double = json.loads((target / f"utgen/result/{names['palm_fixture::nested::double']}.json").read_text()) + assert double['tests_compiled'] == 0 and double['tests'] == 1, double + assert not list((target / 'src').rglob('*.bak')) + (work / 'summary.json').write_text(json.dumps(state, indent=2) + '\n') + print('Minimal analysis/generation/repair/coverage checks passed.', flush=True) + finally: + server.shutdown() + server.server_close() + (work / 'model-requests.json').write_text(json.dumps(state, indent=2) + '\n') + + +if __name__ == '__main__': + main() diff --git a/utgen/README.md b/utgen/README.md index c4c5f28869..77f5e89e41 100644 --- a/utgen/README.md +++ b/utgen/README.md @@ -32,7 +32,7 @@ An absolute path continues to work after changing into the target crate. Relativ Changing configuration takes effect on the next invocation without rebuilding. There are no built-in model credentials. Configuration values are not printed in configuration diagnostics. -Install the toolchain and coverage tool described in the [project README](../README.md#prerequisites). Before generation, run `cargo brinfo` in the target crate and `focxt -c ` to produce: +Install the toolchain and coverage tool described in the [project README](../README.md#prerequisites). On a fresh working copy, run `utgen pre-process -p ` and then `utgen analyze -p ` to produce: ```text brinfo/name_map.json @@ -76,12 +76,12 @@ The following is a command reference rather than captured help output. Use `utge | Command | Current behavior | | --- | --- | -| `pre-process` | Rename existing `tests` directories to `tests.bak` and comment out test modules and test functions under the selected `src` directories. | -| `analyze` | Log the selected directories. Run brinfo and focxt explicitly to perform analysis. | +| `pre-process` | Rename each selected crate's `tests/` to `tests.bak/` and replace test-only source ranges with whitespace while preserving line breaks and byte offsets. | +| `analyze` | Clear Cargo's check cache, run brinfo and focxt, and validate their function indices and artifacts. Currently supports one standalone crate passed with `-p`. | | `gen` | Generate candidates, check compilation, and collect pre-repair statistics. | | `fix` | Attempt to repair candidates with compilation errors and collect post-repair statistics using unit-test insertion. | -All four commands take `-p, --project-dir`. Use `-w, --work-dir` for individual crates in a larger project. Both relative paths are resolved against the current shell directory; work directories do not resolve against `--project-dir`. Work directories may be repeated or comma-separated and default to the project directory. +All four commands take `-p, --project-dir`. Use `-w, --work-dir` for individual crates in a larger project. Both relative paths are resolved against the current shell directory; work directories do not resolve against `--project-dir`. Work directories may be repeated or comma-separated and default to the project directory. The current `analyze` command requires a single standalone crate; use that crate as `-p`, with no separate work-directory selection. ### Preprocessing @@ -97,7 +97,17 @@ For multiple crates, use explicit work-directory paths: utgen pre-process -p -w -w ``` -The current preprocessor recognizes modules whose `cfg` attribute text contains `test`, and functions with `test` or namespaced `test` attributes. It does not provide a general reverse-preprocessing command. +The preprocessor removes modules whose `cfg` predicates are provably disabled without `test`, and functions with `test` or namespaced `test` attributes. It preserves production predicates such as `cfg(not(test))` and treats unknown feature/target predicates conservatively. Source files retain their line breaks and byte offsets. UTF-8 character columns can change inside blanked ranges; perform analysis after preprocessing. Existing `tests.bak` is never overwritten. There is no general reverse-preprocessing command. + +### Analysis + +```sh +utgen analyze -p +``` + +The target must have `Cargo.toml` and `src/`, with no existing `brinfo/` or `focxt/` directory. Use a fresh prepared copy when repeating analysis. Tool failures return a nonzero exit status. No model configuration is needed. + +For manual analysis, run `cargo clean`, `cargo brinfo`, then `focxt -c ` in the prepared crate. A prior `cargo check` can otherwise prevent the compiler wrapper from running. ### Generation @@ -113,6 +123,8 @@ utgen gen -p --requirement --context | `-i, --integration` | Generate integration tests under `tests/`, using the analysis visibility flag to select functions and compilation checks to filter candidates. Default: off. | | `-t, --tasks` | Default: 128. Currently sizes the result channel and does not enforce a strict limit on concurrent LLM requests. | +Generation validates the branch index, context index, context files, and source paths before modifying the target. Missing or inconsistent artifacts are errors. A failed generation task is reported instead of being silently lost before statistics. `--tasks 0` is rejected. + Generation may append an `ntest` dependency to the target's Cargo.toml. Existing `utgen/generation/pre_fix/.json` results are skipped, so use a fresh target copy for a different model, prompt, or generation mode. ### Repair @@ -140,4 +152,4 @@ Paths under `utgen/` below are relative to `--project-dir`: `coverage.xml`, `coverage.json`, and `error_output.json` are intermediate files in the work directory and may be deleted after parsing. The current implementation does not produce an HTML report. -See the [bytes example](../examples/README.md) for a working-copy workflow and the [Chinese technical guide](../docs/palm-rust-unit-test-generation.md) for implementation details. +See the [deterministic minimal pipeline](../docs/minimal-pipeline.md) and the [bytes example](../examples/README.md) for a working-copy workflow and the [Chinese technical guide](../docs/palm-rust-unit-test-generation.md) for implementation details.