From 40eab895293be18c69deb1bcd1b22c529bdc2363 Mon Sep 17 00:00:00 2001 From: cppbear Date: Fri, 9 Oct 2026 02:41:18 +0800 Subject: [PATCH 1/2] feat: repair and evaluate candidates in integration test mode --- README.md | 2 +- docs/minimal-pipeline.md | 3 +- docs/palm-rust-unit-test-generation.md | 10 +- examples/README.md | 3 +- scripts/check_minimal.py | 138 +++++++++- scripts/check_tasks.py | 24 +- utgen/README.md | 17 +- utgen/src/gene/generation.rs | 87 +++--- utgen/src/gene/mod.rs | 13 + utgen/src/main.rs | 9 +- utgen/src/run/coverage.rs | 35 ++- utgen/src/run/integration.rs | 339 +++++++++++++++--------- utgen/src/run/llm_fix.rs | 352 +++++++++++++++++-------- utgen/src/run/llm_fix_type.rs | 7 + utgen/src/run/mod.rs | 2 +- utgen/src/run/run.rs | 145 +++++----- utgen/src/types/testgen.rs | 23 ++ utgen/src/utils/check.rs | 13 +- 18 files changed, 830 insertions(+), 392 deletions(-) diff --git a/README.md b/README.md index bd602bd740..f492342c7d 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ To measure existing tests before preprocessing, run `utgen coverage -p ` before `utgen analyze -p `. Analysis clears Cargo's check cache, runs brinfo and focxt, and verifies their outputs. It rejects existing `brinfo/` or `focxt/` directories so previous analysis results cannot be mixed into the run. -Then use `utgen gen` and `utgen fix` for generation and compilation repair. Pass `--requirement --context` to `utgen gen` to include path constraints and focal context. Preprocessing and test checks modify the target tree; use a working copy of the target project. +Then use `utgen gen` and `utgen fix` for generation and compilation repair. Pass `--requirement --context` to `utgen gen` to include path constraints and focal context. For integration tests, pass `--integration` to both `gen` and `fix`. Preprocessing and test checks modify the target tree; use a working copy of the target project. Start with the [minimal pipeline check](docs/minimal-pipeline.md), which uses fixed local model responses and verifies analysis, generation, repair, source restoration, and coverage. The [bytes analysis check](docs/bytes-analysis.md) validates analysis on the larger bundled target without a model service. For model-based generation and repair, follow the [small model trial](examples/README.md#prepare-a-small-model-trial) using an explicit function list and separate request limits, then inspect the request reports and test statistics. See [utgen usage](utgen/README.md) for command details and supported scope. diff --git a/docs/minimal-pipeline.md b/docs/minimal-pipeline.md index 9ab845936e..494e8d6c44 100644 --- a/docs/minimal-pipeline.md +++ b/docs/minimal-pipeline.md @@ -41,6 +41,7 @@ The default binary directory is `target/debug`. The coverage tool must be availa | Coverage | Excludes test bodies and test-only helpers, preserves production coverage, and executes tests once when exporting XML and JSON. | | Cleanup | Restores sources after the covered success, assertion-failure, compilation-failure, and report-export-failure cases. | | Candidate timeout | Ordinary and `should_panic` infinite loops count as failures; an expected panic and a later ordinary test pass in unit and integration statistics. | +| Integration repair | Repairs alias/trait imports, helpers, and test bodies in integration targets; keeps sibling candidates independent; checks cached repair, mode mismatch, unselected targets, and cleanup after request failure. | | Integration directories | Original and generated tests coexist after normal runs and cached reruns; original-test compilation failure restores both directories; an existing staging directory is preserved and reported. | The integration-directory cases retain the original `tests.bak/` while generating `tests/`. They use one cached candidate and make no additional model requests. Separate timeout cases also check integration execution without an original-test backup. @@ -63,7 +64,7 @@ Explicit `coverage(on)` is respected. Expression fragments are preserved; the ch `check_tasks.py` exercises function limits N=1, N=2, and the default N=4, with more than 2N functions in each scheduling case. It checks overlapping model requests, serial compiler activity, bounded-queue backpressure, and completion after queued work resumes. Cargo's target directory is reused between scenarios. -Additional cases cover direct and input/prefix/oracle generation, integration imports, function selection, request budgets, and worker/consumer failures. Checks require source restoration, preservation of unrelated backups, retention of repair backups on failure, and skipped statistics after a candidate-stage error. +Additional cases cover direct and input/prefix/oracle generation, integration imports and repair concurrency, function selection, request budgets, and worker/consumer failures. Checks require source restoration, preservation of unrelated backups, retention of repair backups on failure, and skipped statistics after a candidate-stage error. ## Model Request Regression diff --git a/docs/palm-rust-unit-test-generation.md b/docs/palm-rust-unit-test-generation.md index 2c76ad8ab7..5866aec472 100644 --- a/docs/palm-rust-unit-test-generation.md +++ b/docs/palm-rust-unit-test-generation.md @@ -262,7 +262,7 @@ Rust 的一个核心挑战是 MIR 的去糖。比如高级语法 `match color` utgen pre-process --project-dir [--work-dir ...] utgen analyze --project-dir utgen gen --project-dir [--work-dir ...] [--tasks ] [--integration] [--requirement] [--context] [--oracle] -utgen fix --project-dir [--work-dir ...] [--tasks ] +utgen fix --project-dir [--work-dir ...] [--tasks ] [--integration] ``` `analyze` 现在执行 `cargo clean`、`cargo brinfo` 和 `focxt`，随后核对分析索引及上下文文件。当前支持用 `-p` 指定单个独立 crate；目录中已有 `brinfo/` 或 `focxt/` 时会拒绝执行，应使用新的工作副本，避免混入旧分析结果。 @@ -358,7 +358,7 @@ export PALM_CONFIG="$(pwd)/utgen/res/api.json" - `gen_full_tests` 使用 `test_prompt.json` 一次性生成测试。 - `gen_tests_cot` 使用 `input -> prefix -> oracle` 三步流程。 - `check_unit` 会临时把测试插入被测源码文件，运行 `cargo build --tests`，记录每个候选是否可编译，然后恢复文件。 - - `check_integration` 会写入 `tests/test_check.rs` 做编译检查。 + - `check_integration` 会写入临时 `tests/palm_candidate.rs`，只编译该集成测试目标。 - `utgen/src/gene/cot/*.rs` - `input_infer.rs`：生成输入范围。 @@ -620,7 +620,9 @@ utgen fix -p 修复阶段读取 `utgen/generation/pre_fix`，针对不可编译测试调用 LLM 生成 ChangeLog 并重新验证。修复结果写入 `utgen/generation/llm_fix`，覆盖率统计写入 `utgen/fixed_result`。 -`fix` 当前将测试插入源码，以单元测试方式修复和统计，没有独立的 integration 修复选项。因此 `gen --integration` 后接 `fix` 并不表示全过程保留集成测试模式。运行时断言失败也不属于当前编译修复流程的目标。 +`fix` 默认将测试插入源码，以单元测试方式修复和统计。集成测试需要在生成和修复时都传入 `--integration`：修复阶段把单个候选写入临时集成测试目标，只编译该目标；可以修改导入、模块级辅助代码和测试体，并单独保存各候选的导入与辅助代码。整体执行和逐候选覆盖率统计都保持集成测试模式，不再把候选插入源码。覆盖率阶段仍会临时添加排除测试代码的属性并恢复源文件。 + +候选缓存记录生成模式，模式不匹配时会报错；没有模式字段的旧缓存按单元测试处理，旧集成测试缓存应在新的工作副本中重新生成。两条命令还应传入相同的函数清单。运行时断言失败不属于编译修复流程的目标。 ### 10.7 示例：`examples/bytes` @@ -709,7 +711,7 @@ integration 模式先按 `brinfo` 的 `visible` 标志筛选函数，并且 `utg - `utgen pre-process`：重命名所选 crate 的 `tests`，将测试源码区间替换为空白。 - `utgen gen`：可能追加 `ntest` 依赖，写入临时测试文件或插入测试做编译检查。 -- `utgen fix`：临时插入测试并恢复，结束时删除备份。 +- `utgen fix`：默认临时插入单元测试并恢复，成功时删除本轮备份；`--integration` 使用临时集成测试文件，恢复或删除本轮临时输入。 建议用 git 查看差异： diff --git a/examples/README.md b/examples/README.md index c34bb7f45a..67a54ab88c 100644 --- a/examples/README.md +++ b/examples/README.md @@ -72,9 +72,10 @@ Use a fresh working copy and repeat preprocessing and analysis before choosing t ```sh utgen gen -p --integration --requirement --context +utgen fix -p --integration ``` -Integration mode generates files under the target's `tests/` directory after filtering functions using the analysis visibility flag and checking compilation. `utgen fix` currently uses source-inserted unit tests for repair and post-repair statistics; it does not provide a separate integration-mode repair pipeline. +Integration mode generates files under the target's `tests/` directory after filtering functions using the analysis visibility flag and checking compilation. Pass `--integration` to `fix` as well to repair and evaluate candidates in integration-test targets. Imports and helpers stay with each candidate, and both combined and per-candidate coverage retain integration scope. Keep the same function selection for both commands; mixing cached generation modes is rejected. ### Results diff --git a/scripts/check_minimal.py b/scripts/check_minimal.py index 59273e5672..f05e3e2804 100644 --- a/scripts/check_minimal.py +++ b/scripts/check_minimal.py @@ -202,7 +202,7 @@ def run(label, command, cwd=target, expected=0, timeout=240): finally: lib.write_bytes(saved_lib) assert snapshot(original) == baseline - state = {'generation': 0, 'repair': 0, 'errors': []} + state = {'generation': 0, 'repair': 0, 'errors': [], 'integration': False, 'reject_repair': False} class Model(http.server.BaseHTTPRequestHandler): def log_message(self, *_): @@ -226,19 +226,60 @@ def do_POST(self): 'double': 'assert_eq!(double("bad"), 4);', }[function] answer = 'fn generated_helper() -> i32 { 1 }\n#[test]\nfn generated() {\n assert_eq!(generated_helper(), 1);\n ' + body + '\n}\n' + if state['integration']: + answer = { + 'classify': '''use std::cmp::max as target; +use std::fmt::Write; +fn helper() -> i32 { 1 } +#[test] +fn alias_case() { + let mut buffer = Vec::new(); + buffer.write_all(b"x").unwrap(); + assert_eq!(target(2), helper()); +} +#[test] +fn sibling_case() { + assert_eq!(target(2, 1), 2); + assert_eq!(palm_fixture::classify(-1), 0); +} +''', + 'double': '''use palm_fixture::nested::double as twice; +fn helper() -> i32 { "bad-helper" } +#[test] +fn body_case() { + assert_eq!(twice("bad-body") + helper(), 5); +} +''', + }[function] state['generation'] += 1 else: file = re.search(r'You can only modify lines \d+ to \d+ in file (.+?)\. For your answer', user).group(1) - match = re.search(r'^\[(\d+)\](.*assert_eq!\(double\("bad"\), 4\);.*)$', user, re.M) - assert match, 'repair prompt lacks numbered failing statement' - number, old = match.groups() - new = old.replace('double("bad")', 'double(2)') - answer = f'ChangeLog:1@{file}\nFixDescription: Use an integer argument.\nOriginalCode@{number}-{number}:\n[{number}]{old}\nFixedCode@{number}-{number}:\n[{number}]{new}\n' + replacements = { + 'double("bad")': 'double(2)', + 'use std::cmp::max as target;': 'use palm_fixture::classify as target;', + 'use std::fmt::Write;': 'use std::io::Write;', + '"bad-helper"': '1', + '"bad-body"': '2', + } + changes = {} + for number, old in re.findall(r'^\[(\d+)\](.*)$', user, re.M): + new = old + for before, after in replacements.items(): + new = new.replace(before, after) + if new != old: + changes[number] = (old, new) + assert changes, 'repair prompt lacks numbered failing code' + answer = f'ChangeLog:1@{file}\nFixDescription: Fix candidate imports and arguments.\n' + for number, (old, new) in changes.items(): + answer += f'OriginalCode@{number}-{number}:\n[{number}]{old}\nFixedCode@{number}-{number}:\n[{number}]{new}\n' state['repair'] += 1 response = {'id': 'fixture', 'object': 'chat.completion', 'created': 0, 'model': 'fixture', 'choices': [{'index': 0, 'message': {'role': 'assistant', 'content': answer}, 'finish_reason': 'stop'}], 'usage': {'prompt_tokens': 1, 'completion_tokens': 1, 'total_tokens': 2}} code = 200 + if state['reject_repair']: + response = {'error': {'message': 'fixture request rejected'}} + code = 401 except Exception as error: state['errors'].append(str(error)) response = {'error': {'message': str(error), 'type': 'fixture_error'}} @@ -307,6 +348,8 @@ def do_POST(self): timeout_prepared = snapshot(target) requests_before = (state['generation'], state['repair']) for integration in [False, True]: + data['integration'] = integration + candidate.write_text(json.dumps(data, indent=2) + '\n') run(f'candidate-timeouts-{integration}', ['utgen', 'gen', '-p', str(target), '--functions-file', str(selection), *(['--integration'] if integration else [])], timeout=90) @@ -376,6 +419,89 @@ def integration_snapshot(): assert report['attempts'] == 0, report shutil.rmtree(target / 'tests') assert snapshot(target) == prepared + + # Exercise the complete integration path with real Rust diagnostics. + # One answer has two candidates needing different meanings of `target`. + state['integration'] = True + selected_names = [selected, 'palm_fixture::nested::double'] + integration_selection = work / 'integration-functions.txt' + integration_selection.write_text('\n'.join(selected_names) + '\n') + for directory in ['generation/pre_fix', 'generation/llm_fix']: + for name in selected_names: + (target / 'utgen' / directory / (names[name] + '.json')).unlink() + integration_args = ['-p', str(target), '--functions-file', str(integration_selection), + '--integration', '--tasks', '2', '--max-requests', '8'] + run('integration-generate-imports', ['utgen', 'gen', *integration_args]) + for name, compiled in zip(selected_names, [1, 0]): + result = json.loads((target / f'utgen/result/{names[name]}.json').read_text()) + assert result['tests_compiled'] == result['tests_run'] == compiled, result + source_before = {p: content for p, content in snapshot(target).items() if not p.startswith('tests/')} + assert source_before == prepared + # Unselected targets must neither pollute diagnostics nor be executed. + unrelated = target / 'tests/unrelated.rs' + unrelated.write_text('compile_error!("unselected test must not compile");\n') + temporary_file = target / 'tests/palm_candidate.rs' + temporary_file.write_text('compile_error!("pre-existing temporary path");\n') + before_fix = snapshot(target) + requests_before = (state['generation'], state['repair']) + wrong_mode = run('integration-reject-unit-fix', + ['utgen', 'fix', '-p', str(target), '--functions-file', str(integration_selection)], expected=1) + assert 'Test mode mismatch' in wrong_mode.stderr + assert snapshot(target) == before_fix + assert (state['generation'], state['repair']) == requests_before + wrong_mode = run('integration-reject-unit-gen', + ['utgen', 'gen', '-p', str(target), '--functions-file', str(integration_selection)], expected=1) + assert 'Test mode mismatch' in wrong_mode.stderr + assert snapshot(target) == before_fix + assert (state['generation'], state['repair']) == requests_before + run('integration-repair-imports', ['utgen', 'fix', *integration_args]) + assert state['repair'] > requests_before[1] + for name, count in zip(selected_names, [2, 1]): + result = json.loads((target / f'utgen/fixed_result/{names[name]}.json').read_text()) + assert result['tests'] == result['tests_compiled'] == result['tests_run'] == result['tests_passed'] == count, result + assert len(result['codes_lines_covered']) == count, result + assert result['lines_covered'] > 0, result + fixed_path = target / f'utgen/generation/llm_fix/{names[selected]}.json' + fixed = json.loads(fixed_path.read_text()) + tests = fixed['fn_tests'][0]['answers'][0]['chain_tests'] + assert 'use palm_fixture::classify as target;' in tests[0]['integration_contexts'][0]['uses'] + assert 'use std::io::Write;' in tests[0]['integration_contexts'][0]['uses'] + assert 'use std::cmp::max as target;' in tests[1]['integration_contexts'][0]['uses'] + assert {p: content for p, content in snapshot(target).items() if not p.startswith('tests/')} == prepared + assert unrelated.read_bytes() == before_fix['tests/unrelated.rs'] + assert temporary_file.read_bytes() == before_fix['tests/palm_candidate.rs'] + assert not list((target / 'src').rglob('*.bak')) + stable = snapshot(target) + requests_before = (state['generation'], state['repair']) + run('integration-repair-cached', ['utgen', 'fix', *integration_args]) + assert snapshot(target) == stable + assert (state['generation'], state['repair']) == requests_before + report = json.loads((target / 'utgen/generation/fix-requests.json').read_text()) + assert report['attempts'] == 0 and report['invocation']['integration'], report + source_file = target / 'src/lib.rs' + original_source = source_file.read_bytes() + source_file.write_bytes(original_source + b'\ncompile_error!("production compilation failure");\n') + broken = snapshot(target) + failure = run('integration-repair-build-error', ['utgen', 'fix', *integration_args], expected=1) + assert 'production compilation failure' in failure.stderr + assert snapshot(target) == broken + assert (state['generation'], state['repair']) == requests_before + assert not list((target / 'src').rglob('*.bak')) + source_file.write_bytes(original_source) + # Request failure must restore the temporary input without touching src/. + saved_fixed = {} + for name in selected_names: + path = target / f'utgen/generation/llm_fix/{names[name]}.json' + saved_fixed[path] = path.read_bytes() + path.unlink() + state['reject_repair'] = True + run('integration-repair-request-error', ['utgen', 'fix', *integration_args], expected=1) + state['reject_repair'] = False + assert snapshot(target) == stable + assert not list((target / 'src').rglob('*.bak')) + for path, content in saved_fixed.items(): + path.write_bytes(content) + assert not state['errors'], state (work / 'summary.json').write_text(json.dumps(state, indent=2) + '\n') print('Minimal analysis/generation/repair/coverage checks passed.', flush=True) finally: diff --git a/scripts/check_tasks.py b/scripts/check_tasks.py index 863f213b9b..b8ca2f0418 100644 --- a/scripts/check_tasks.py +++ b/scripts/check_tasks.py @@ -402,6 +402,26 @@ def results(directory, compiled, names=None): assert measured['total'] == 2 results('result', 1, selected) + # Integration repair shares the same task limit and serial compiler, + # while leaving each candidate in a separate integration target. + for name in selected: + path = pre_dir / (nmap[name] + '.json') + candidate = json.loads(path.read_text()) + for chain in candidate['fn_tests']: + for answer in chain['answers']: + for test in answer['chain_tests']: + test['codes'] = [[line.replace('(1)', '("bad")') for line in body] + for body in test['codes']] + test['can_compile'] = [{'Err': 'fixture integer argument'}] * len(test['codes']) + path.write_text(json.dumps(candidate)) + measured = run('integration-fix-selected', command('fix', 2, '--integration', *selection_args), + limit=2, integration=True) + assert measured['total'] == measured['peak'] == 2, measured + results('fixed_result', 1, selected) + measured = run('integration-fix-cached', command('fix', 2, '--integration', *selection_args), + integration=True) + assert measured['total'] == 0, measured + # More workers than requests: every clone shares the same hard cap. for mode in ['gen', 'fix']: reset(seed if mode == 'fix' else None) @@ -420,7 +440,7 @@ def results(directory, compiled, names=None): preserved = {} if oracle: (root / 'tests').mkdir() - for name in ['use_check.rs', 'test_check.rs']: + for name in ['use_check.rs', 'palm_candidate.rs']: path = root / 'tests' / name path.write_text('// existing compiler input\n') preserved[path] = path.read_bytes() @@ -431,7 +451,7 @@ def results(directory, compiled, names=None): for file in (root / 'utgen/generation/pre_fix').glob('*.json'): text = file.read_text() assert 'absent_crate' not in text and 'std::cmp::max' in text, file - for name in ['use_check.rs', 'test_check.rs']: + for name in ['use_check.rs', 'palm_candidate.rs']: path = root / 'tests' / name if path in preserved: assert path.read_bytes() == preserved[path], path diff --git a/utgen/README.md b/utgen/README.md index 6322390659..eee84c01a9 100644 --- a/utgen/README.md +++ b/utgen/README.md @@ -185,11 +185,22 @@ utgen fix -p Repair reads the generated candidates and uses compiler diagnostics to revise those that fail compilation. It does not target runtime assertion failures. `--tasks N` defaults to 4 and limits active focal-function repair jobs through result saving. Each job processes its candidates and repair rounds sequentially. -Source insertion, target cleanup, compilation/test execution, diagnostic reading, and source restoration are serialized within each command. Model requests may overlap. Cargo retains its own dependency-build parallelism. Concurrent PALM commands or experiments require separate working copies and separate target directories. +Source insertion or temporary test-file writes, compilation/test execution, diagnostic reading, and restoration are serialized within each command. Model requests may overlap. Cargo retains its own dependency-build parallelism. Concurrent PALM commands or experiments require separate working copies and separate target directories. -All workers are joined before cleanup. A task panic or infrastructure error returns a nonzero status and skips subsequent coverage statistics; candidate compilation errors remain ordinary repair outcomes. Repair restores its source backups before returning, deletes only backups created by that invocation on success, and retains them after a worker failure. Existing source backups are rejected without overwriting them. Temporary import/candidate files are restored or removed after validation, including unwinding. This is not recovery from forced process termination. Model request deadlines are described above; Cargo subprocess deadlines remain separate work. +All workers are joined before cleanup. A task panic or infrastructure error returns a nonzero status and skips subsequent coverage statistics; candidate compilation errors remain ordinary repair outcomes. Unit-mode repair restores its source backups before returning, deletes only backups created by that invocation on success, and retains them after a worker failure. Existing source backups are rejected without overwriting them. Temporary import/candidate files are restored or removed after validation, including unwinding. This is not recovery from forced process termination. Model request deadlines are described above; Cargo subprocess deadlines remain separate work. -There is no integration-mode option for repair. Candidates are inserted into source files as unit tests, and post-repair statistics use that same mode. Do not interpret `gen --integration` followed by `fix` as preserving an integration-only evaluation. +For integration tests, pass `--integration` to both commands: + +```sh +utgen gen -p --integration +utgen fix -p --integration +``` + +Integration repair compiles each candidate in a temporary `tests/palm_candidate.rs` target, without inserting tests into production source. It can repair imports, module-level helpers, and the test body. Each candidate keeps its own repaired imports/helpers, so a change does not affect siblings from the same model answer. Compilation checks do not execute tests. Repair checks saved candidates again in their integration target; successful cached repairs need no model requests. + +The resulting tests remain under `tests/`. Both combined and per-candidate statistics use integration targets, retain the five-second timeout, and exclude test code from coverage. Temporary compiler inputs are restored or removed on normal completion and ordinary failures. Coverage still temporarily annotates source to exclude test code and restores it afterward. + +Candidate files record their generation mode. Generation and repair reject a different mode instead of converting candidates. Older cache files without a mode field are treated as unit-test candidates; regenerate old integration caches in a fresh working copy. Keep the same function selection and mode for generation and repair. ## Output diff --git a/utgen/src/gene/generation.rs b/utgen/src/gene/generation.rs index 2134fcd5ea..e4ae6d1b07 100644 --- a/utgen/src/gene/generation.rs +++ b/utgen/src/gene/generation.rs @@ -1,9 +1,8 @@ -use super::{gen_input_range, gen_oracle, gen_prefix, gen_test, LLM}; +use super::{LLM, gen_input_range, gen_oracle, gen_prefix, gen_test}; use super::{inputprompts, oracleprompts, prefixprompts, testprompts}; use crate::types::{BrData, ChainTestInfo, TestGenInfo}; use crate::utils::{ - RestoreOnDrop, TemporaryFile, backup_file, cargo_check, delete_backup, insert_test, - restore_file, use_check, + RestoreOnDrop, backup_file, cargo_check, delete_backup, insert_test, restore_file, use_check, }; use log::info; use std::collections::HashSet; @@ -456,59 +455,45 @@ pub fn check_unit(test_gen: &mut TestGenInfo, project_dir: &Path, work_dir: &Pat } pub fn check_integration(test_gen: &mut TestGenInfo, work_dir: &Path) { - let file_path = test_gen.get_file(); - let stem = Path::new(&file_path).file_stem().unwrap().to_str().unwrap(); - let path = work_dir.join("tests/test_check.rs"); - let temporary = TemporaryFile::new(&path).unwrap(); - fs::create_dir_all(&path.parent().unwrap()).unwrap(); - fs::write(&path, "").unwrap(); - let name = test_gen.get_name().to_string(); - let fn_name = name.split("::").last().unwrap(); - - let mut init_use_set = test_gen - .get_use_path() - .into_iter() - .collect::>(); - use_check(&mut init_use_set, work_dir); - let mut id = 0; - for chain_test in test_gen.get_tests_mut() { - for test_answer in chain_test.get_answers_mut() { - // Model tasks only collect imports. All import compilation happens - // here, in the single validation consumer. - fs::write(&path, "").unwrap(); - let mut uses: HashSet<_> = test_answer.get_uses().iter().cloned().collect(); + use crate::run::integration::{ + CANDIDATE_TARGET, candidate_code, initial_uses, write_candidate, + }; + use crate::types::IntegrationContext; + use crate::utils::cargo_check_test; + + let initial = initial_uses(test_gen, work_dir); + for chain in test_gen.get_tests_mut() { + for answer in chain.get_answers_mut() { + let mut uses: HashSet<_> = answer.get_uses().iter().cloned().collect(); use_check(&mut uses, work_dir); let mut uses: Vec<_> = uses.into_iter().collect(); uses.sort(); - test_answer.set_uses(uses); - let mut answer_use_set = init_use_set.clone(); - answer_use_set.extend(test_answer.get_uses().clone()); - let common = test_answer.get_common().clone(); - for test_info in test_answer.get_tests_mut() { - for (num, test_code) in test_info.codes.iter().enumerate() { - let sig = format!("fn test_{}_{}_{:02}()", stem, fn_name, id); - let mut fn_code = vec!["#[test]".to_string()]; - fn_code.extend(test_info.attrs.clone()); - fn_code.push(sig); - fn_code.extend(test_code.clone()); - let mut code = answer_use_set - .clone() - .into_iter() - .collect::>() - .join("\n") - + "\n\n"; - if !common.is_empty() { - code += &(common.join("\n") + "\n\n"); - } - code += &(fn_code.join("\n") + "\n"); - fs::write(&path, code).unwrap(); - // let _ = target_clean(&work_dir); - let result = cargo_check(&work_dir); - test_info.can_compile[num] = result; - id += 1; + answer.set_uses(uses); + let context = IntegrationContext { + uses: initial + .iter() + .chain(answer.get_uses()) + .cloned() + .collect::>() + .into_iter() + .collect(), + common: answer.get_common().clone(), + }; + for test in answer.get_tests_mut() { + for (num, body) in test.codes.iter().enumerate() { + let code = candidate_code( + &context, + &test.attrs, + body, + "palm_candidate", + "test_candidate", + false, + ); + let temporary = write_candidate(work_dir, &code).unwrap(); + test.can_compile[num] = cargo_check_test(work_dir, CANDIDATE_TARGET); + temporary.finish().unwrap(); } } } } - temporary.finish().unwrap(); } diff --git a/utgen/src/gene/mod.rs b/utgen/src/gene/mod.rs index 41e6898f54..d6ba319fd0 100644 --- a/utgen/src/gene/mod.rs +++ b/utgen/src/gene/mod.rs @@ -73,6 +73,18 @@ pub async fn gen_tests_project( )); } validate_analysis(project_dir, work_dir)?; + let saved_dir = project_dir.join("utgen/generation/pre_fix"); + if saved_dir.is_dir() { + for entry in fs::read_dir(saved_dir)? { + let path = entry?.path(); + if path.is_file() { + let info = TestGenInfo::from_json(&path); + if functions.contains(info.get_name()) { + info.check_mode(integration)?; + } + } + } + } add_ntest_dependency(work_dir); let brdata_dir = work_dir.join("brinfo/brdata"); let map_path = work_dir.join("brinfo/name_map.json"); @@ -164,6 +176,7 @@ pub async fn gen_tests_project( let work_dir = work_dir.to_owned(); // Await exactly one validation at a time, without blocking model tasks. let checked = tokio::task::spawn_blocking(move || { + test_gen_info.integration = integration; if integration { check_integration(&mut test_gen_info, &work_dir); } else { diff --git a/utgen/src/main.rs b/utgen/src/main.rs index 1e200b83ae..952875e641 100644 --- a/utgen/src/main.rs +++ b/utgen/src/main.rs @@ -73,6 +73,9 @@ enum Command { Fix { #[command(flatten)] options: Opts, + /// Repair and evaluate candidates as integration tests under tests/ + #[arg(short, long)] + integration: bool, /// Maximum active function-repair tasks; compilation stays serial #[arg(short, long, default_value = "4")] tasks: NonZeroUsize, @@ -247,6 +250,7 @@ async fn main() -> Result<(), Box> { // fix unit tests Command::Fix { options, + integration, tasks, request_timeout, functions_file, @@ -271,6 +275,7 @@ async fn main() -> Result<(), Box> { work_dir.clone(), &functions, tasks.get(), + integration, ) .await?; } @@ -282,7 +287,7 @@ async fn main() -> Result<(), Box> { &project_dir.join("utgen/generation/fix-requests.json"), serde_json::json!({ "command": "fix", "functions": functions.names(), - "project_dir": project_dir, "work_dirs": work_dirs, "tasks": tasks, + "project_dir": project_dir, "work_dirs": work_dirs, "tasks": tasks, "integration": integration, "candidate_status": if repaired.is_ok() { "completed" } else { "failed" }, "error": repaired.as_ref().err().map(ToString::to_string), }), @@ -294,7 +299,7 @@ async fn main() -> Result<(), Box> { recorded?; //gen_test_rate_aggregated(&project_dir, false); for work_dir in work_dirs.iter() { - gen_test_rate(&project_dir, &work_dir, false, false, &functions); + gen_test_rate(&project_dir, &work_dir, integration, false, &functions); } } } diff --git a/utgen/src/run/coverage.rs b/utgen/src/run/coverage.rs index 87a8391e82..2d988ed4c9 100644 --- a/utgen/src/run/coverage.rs +++ b/utgen/src/run/coverage.rs @@ -181,6 +181,14 @@ fn coverage_command(work_dir: &Path, args: &[&str]) -> io::Result { /// and optionally JSON from that run. Use a standalone crate working copy. /// Test assertion failures are retained in the output; build/report errors fail. pub fn collect_coverage(work_dir: &Path, json: bool) -> io::Result { + collect_coverage_for_tests(work_dir, json, &[]) +} + +pub(crate) fn collect_coverage_for_tests( + work_dir: &Path, + json: bool, + targets: &[String], +) -> io::Result { let canonical_dir = work_dir.canonicalize()?; let work_dir = canonical_dir.as_path(); let metadata = MetadataCommand::new() @@ -231,17 +239,22 @@ pub fn collect_coverage(work_dir: &Path, json: bool) -> io::Result { } // The first command uses llvm-cov's default cleanup. --no-report would // imply --no-clean and mix profiles from different generated candidates. - let output = coverage_command( - work_dir, - &[ - "--tests", - "--ignore-run-fail", - "--branch", - "--cobertura", - "--output-path", - "coverage.xml", - ], - )?; + let mut args = Vec::new(); + if targets.is_empty() { + args.push("--tests"); + } else { + for target in targets { + args.extend(["--test", target.as_str()]); + } + } + args.extend([ + "--ignore-run-fail", + "--branch", + "--cobertura", + "--output-path", + "coverage.xml", + ]); + let output = coverage_command(work_dir, &args)?; if json { coverage_command( work_dir, diff --git a/utgen/src/run/integration.rs b/utgen/src/run/integration.rs index 7df4733d27..88b293cd01 100644 --- a/utgen/src/run/integration.rs +++ b/utgen/src/run/integration.rs @@ -1,13 +1,170 @@ -use std::{collections::HashSet, fs, path::Path}; +use std::{ + collections::{BTreeSet, HashSet}, + fs, io, + path::{Path, PathBuf}, + process::{Command, Output}, +}; use log::info; +use quote::ToTokens; +use syn::spanned::Spanned; use crate::{ run::timed_test, - types::TestGenInfo, - utils::{cargo_check, use_check}, + types::{ChainTestAnswer, IntegrationContext, TestGenInfo, TestInfo}, + utils::{TemporaryFile, cargo_check_test, use_check}, }; +pub(crate) const CANDIDATE_TARGET: &str = "palm_candidate"; + +pub(crate) fn candidate_path(work_dir: &Path) -> PathBuf { + work_dir.join(format!("tests/{CANDIDATE_TARGET}.rs")) +} + +pub(crate) fn write_candidate(work_dir: &Path, code: &[String]) -> io::Result { + let path = candidate_path(work_dir); + fs::create_dir_all(path.parent().unwrap())?; + let temporary = TemporaryFile::new(&path)?; + fs::write(&path, code.join("\n"))?; + Ok(temporary) +} + +pub(crate) fn compile_candidate(work_dir: &Path) -> io::Result { + Command::new("cargo") + .args([ + "test", + "--test", + CANDIDATE_TARGET, + "--no-run", + "--message-format", + "json", + ]) + .current_dir(work_dir) + .output() +} + +pub(crate) fn initial_uses(test_gen: &TestGenInfo, work_dir: &Path) -> Vec { + let mut uses: HashSet<_> = test_gen.get_use_path().into_iter().collect(); + uses.remove("use super::*;"); + uses.insert("use ntest::timeout;".to_string()); + use_check(&mut uses, work_dir); + uses.into_iter() + .collect::>() + .into_iter() + .collect() +} + +pub(crate) fn candidate_context( + answer: &ChainTestAnswer, + test: &TestInfo, + num: usize, + initial: &[String], +) -> IntegrationContext { + test.integration_contexts + .get(num) + .cloned() + .unwrap_or_else(|| IntegrationContext { + uses: initial + .iter() + .chain(answer.get_uses()) + .cloned() + .collect::>() + .into_iter() + .collect(), + common: answer.get_common().clone(), + }) +} + +pub(crate) fn candidate_code( + context: &IntegrationContext, + attrs: &[String], + body: &[String], + module: &str, + name: &str, + timed: bool, +) -> Vec { + let mut code = vec![format!("mod {module} {{")]; + code.extend(context.uses.clone()); + code.extend(context.common.clone()); + let signature = format!("fn {name}()"); + if timed { + code.extend(timed_test(&signature, attrs, body)); + } else { + code.push("#[test]".to_string()); + code.extend_from_slice(attrs); + code.push(signature); + code.extend_from_slice(body); + } + code.push("}".to_string()); + code +} + +// Keep imports and helpers at module scope. Only the single candidate's body +// and supporting items may change; its test declaration must remain intact. +pub(crate) fn repaired_candidate( + code: &[String], + attrs: &[String], +) -> Option<(IntegrationContext, Vec)> { + let source = code.join("\n"); + let syntax = syn::parse_file(&source).ok()?; + let [syn::Item::Mod(module)] = syntax.items.as_slice() else { + return None; + }; + if module.ident != "palm_candidate" { + return None; + } + let (_, items) = module.content.as_ref()?; + let expected = syn::parse_str::(&format!( + "#[test]\n{}\nfn test_candidate() {{}}", + attrs.join("\n") + )) + .ok()?; + let mut context = IntegrationContext { + uses: Vec::new(), + common: Vec::new(), + }; + let mut body = None; + for item in items { + let text = &source[item.span().byte_range()]; + match item { + syn::Item::Fn(function) + if function + .attrs + .iter() + .any(|attr| attr.path().is_ident("test")) => + { + if body.is_some() + || function.sig.to_token_stream().to_string() + != expected.sig.to_token_stream().to_string() + || function + .attrs + .iter() + .map(ToTokens::to_token_stream) + .map(|t| t.to_string()) + .collect::>() + != expected + .attrs + .iter() + .map(ToTokens::to_token_stream) + .map(|t| t.to_string()) + .collect::>() + { + return None; + } + body = Some( + source[function.block.span().byte_range()] + .lines() + .map(str::to_owned) + .collect(), + ); + } + syn::Item::Use(_) => context.uses.push(text.to_owned()), + _ => context.common.extend(text.lines().map(str::to_owned)), + } + } + Some((context, body?)) +} + pub struct IntegrationInfo { pub function_name: String, pub file_name: String, @@ -38,140 +195,68 @@ impl IntegrationInfo { } } -fn backup_integration(work_dir: &Path) { - let test_path = work_dir.join("/tests"); - let bak_test_path = work_dir.join("/test_bak"); - if fs::exists(&test_path).unwrap() { - fs::rename(&test_path, &bak_test_path).unwrap(); - } -} - pub fn gen_integration( test_gen_infos: &Vec, - project_dir: &Path, work_dir: &Path, ) -> Vec { - // backup_integration(work_dir); info!("Generate integration tests!"); - let mut integration_infos: Vec = Vec::new(); - for test_gen_info in test_gen_infos.iter() { - if test_gen_info.get_visibility() { - let file_path = test_gen_info.get_file(); - let stem = Path::new(&file_path).file_stem().unwrap().to_str().unwrap(); - let name = test_gen_info.get_name().to_string(); - let fn_name = name.split("::").last().unwrap(); - let rename = name - .replace("::", "_") - .replace("{", "") - .replace("}", "") - .replace("#", "_"); - let path = work_dir.join(format!("tests/test_{}_{}.rs", stem, rename)); - let mut integration_info = IntegrationInfo::new(); - integration_info.function_name = name.clone(); - integration_info.file_name = format!("{}_{}", stem, rename); - let mut file_content = String::new(); - if !fs::exists(&path.parent().unwrap()).unwrap() { - fs::create_dir_all(&path.parent().unwrap()).unwrap(); - } - let mut use_set = test_gen_info - .get_use_path() - .into_iter() - .collect::>(); - use_set.remove("use super::*;"); - use_set.insert("use ntest::timeout;".to_string()); - use_check(&mut use_set, work_dir); - let mut id = 0; - let mut tests = 0; - let mut tests_compiled = 0; - let mut tests_lines: Vec = Vec::new(); - let mut oracles = 0; - let mut oracles_compiled = 0; - let mut has_oracle_compiled = false; - for chain_test in test_gen_info.get_tests().iter() { - let mut answer_id = 0; - for answer in chain_test.get_answers().iter() { - let mut mod_content = String::new(); - let mut use_set = use_set.clone(); - use_set.extend(answer.get_uses().clone()); - let common = answer.get_common().clone(); - for test in answer.get_tests().iter() { - oracles += 1; - for (num, test_code) in test.codes.iter().enumerate() { - tests += 1; - tests_lines.push(test_code.len() as i32); - if test.can_compile[num].is_ok() { - tests_compiled += 1; - has_oracle_compiled = true; - let sig = format!("fn test_{}_{}_{:02}()", stem, rename, id); - let fn_code = timed_test(&sig, &test.attrs, test_code); - mod_content = mod_content + &fn_code.join("\n") + "\n\n"; - integration_info - .test_function_names - .push(format!("test_{}_{}_{:02}", stem, rename, id)); - id += 1; - } - if has_oracle_compiled { - oracles_compiled += 1; - has_oracle_compiled = false; - } - } - } - if mod_content != "".to_string() { - if !common.is_empty() { - mod_content = - format!( - "mod chain_{}_answer_{}{{\n", - chain_test.get_id(), - answer_id - ) + &use_set.into_iter().collect::>().join("\n") - + "\n" - + &common.join("\n") - + "\n" - + &mod_content - + "}" - + "\n\n"; - } else { - mod_content = - format!( - "mod chain_{}_answer_{}{{\n", - chain_test.get_id(), - answer_id - ) + &use_set.into_iter().collect::>().join("\n") - + "\n" - + &mod_content - + "}" - + "\n\n"; + let mut integration_infos = Vec::new(); + for test_gen in test_gen_infos.iter().filter(|info| info.get_visibility()) { + let file = test_gen.get_file(); + let stem = Path::new(&file).file_stem().unwrap().to_str().unwrap(); + let name = test_gen.get_name(); + let rename = name + .replace("::", "_") + .replace(['{', '}'], "") + .replace('#', "_"); + let mut result = IntegrationInfo::new(); + result.function_name = name.to_string(); + result.file_name = format!("test_{stem}_{rename}"); + let path = work_dir.join(format!("tests/{}.rs", result.file_name)); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + let initial = initial_uses(test_gen, work_dir); + let mut content = Vec::new(); + for chain in test_gen.get_tests() { + for answer in chain.get_answers() { + for test in answer.get_tests() { + result.oracles += 1; + let mut compiled = false; + for (num, body) in test.codes.iter().enumerate() { + result.tests += 1; + result.tests_lines.push(body.len() as i32); + if test.can_compile[num].is_ok() { + compiled = true; + let id = result.tests_compiled; + let test_name = format!("test_{stem}_{rename}_{id:02}"); + let context = candidate_context(answer, test, num, &initial); + content.extend(candidate_code( + &context, + &test.attrs, + body, + &format!("candidate_{id}"), + &test_name, + true, + )); + result.test_function_names.push(test_name); + result.tests_compiled += 1; } - file_content = file_content + &mod_content; - answer_id = answer_id + 1; } + result.oracles_compiled += i32::from(compiled); } } - let tests_compiled_rate = if tests > 0 { - tests_compiled as f64 / tests as f64 * 100.0 - } else { - 0.0 - }; - let oracles_compiled_rate = if oracles > 0 { - oracles_compiled as f64 / oracles as f64 * 100.0 - } else { - 0.0 - }; - if file_content != "".to_string() { - fs::write(&path, file_content).unwrap(); - integration_info.tests = tests; - integration_info.tests_lines = tests_lines; - integration_info.oracles = oracles; - integration_info.oracles_compiled = oracles_compiled; - integration_info.oracles_compiled_rate = oracles_compiled_rate; - integration_info.tests_compiled = tests_compiled; - integration_info.tests_compiled_rate = tests_compiled_rate; - integration_infos.push(integration_info); - if !cargo_check(work_dir).is_ok() { - panic!(); - } - } } + if result.tests > 0 { + result.tests_compiled_rate = result.tests_compiled as f64 / result.tests as f64 * 100.0; + } + if result.oracles > 0 { + result.oracles_compiled_rate = + result.oracles_compiled as f64 / result.oracles as f64 * 100.0; + } + // Also replace an older file when no candidate still compiles. + fs::write(&path, content.join("\n")).unwrap(); + cargo_check_test(work_dir, &result.file_name) + .expect("generated integration tests must compile"); + integration_infos.push(result); } integration_infos } diff --git a/utgen/src/run/llm_fix.rs b/utgen/src/run/llm_fix.rs index 3c2c5b4f12..8ee548561e 100644 --- a/utgen/src/run/llm_fix.rs +++ b/utgen/src/run/llm_fix.rs @@ -1,16 +1,15 @@ use super::{ + TIMEOUT_DERIVE, + integration, llm_fix_type::{ChangeLog, CompilerMessage, ErrorMessage, TestCode}, prepare::get_test_gen_infos, - run::{run_test, TestType}, - TIMEOUT_DERIVE, + run::{TestType, run_test}, }; use crate::{ FunctionSelection, gene::LLM, - types::{InsertKind, TestGenInfo}, - utils::{ - RestoreOnDrop, cargo_check, create_backup, insert_test, restore_file, - }, + types::{InsertKind, IntegrationContext, TestGenInfo}, + utils::{RestoreOnDrop, cargo_check, create_backup, insert_test, restore_file}, }; use log::{error, info, warn}; use rand::Rng; @@ -18,13 +17,13 @@ use serde::Deserialize; use std::{ cmp::min, collections::{BTreeSet, HashMap}, - fs::{self, create_dir_all, read_to_string, File}, + fs::{self, File, create_dir_all, read_to_string}, i32, io::{self, Write}, path::{Path, PathBuf}, sync::{ - atomic::{AtomicUsize, Ordering}, Arc, + atomic::{AtomicUsize, Ordering}, }, }; use tokio::time::Duration; @@ -136,12 +135,10 @@ async fn compilation_fix_assistant_for_an_error( file_path: &Path, test_code: &TestCode, sig: String, - test_name: String, insert_kind: InsertKind, common: &Vec, + integration_attrs: Option<&[String]>, ) -> io::Result<(Vec, TestCode, u32, u32)> { - // let mut local_error_group: Vec = Vec::new(); - // local_error_group.push(error_message.clone()); let mut iterative_time = 0; let mut min_error_set = compile_error_set.clone(); let mut min_error_content = test_code.clone(); @@ -150,12 +147,15 @@ async fn compilation_fix_assistant_for_an_error( let current_error = error_message; let rust_assistant_prompt_json_str = include_str!("../../res/rustassistant_prompt.json"); - // println!("{:#?}", rust_assistant_prompt_json_path); let rust_assistant_prompt = RustAssistantPrompt::from_json(&rust_assistant_prompt_json_str); - let preamble_prompt = rust_assistant_prompt - .rustassistant_preamble - .replace("{cmd}", "cargo test --tests") - + "\n"; + let preamble_prompt = rust_assistant_prompt.rustassistant_preamble.replace( + "{cmd}", + if integration_attrs.is_some() { + "cargo test --test palm_candidate --no-run" + } else { + "cargo test --tests" + }, + ) + "\n"; let error_snippet_prompt = current_error.code_snippets.clone(); let final_prompt = preamble_prompt.clone() + &error_snippet_prompt.join("\n") @@ -166,9 +166,6 @@ async fn compilation_fix_assistant_for_an_error( .replace("{end}", (test_code.end - 1).to_string().as_str()) + &rust_assistant_prompt .rustassistant_instructions_and_examples_for_formatting_the_changelog_output; - // let fix_prompt_path = work_path.join("fix_prompt.txt"); - // let mut file = File::create(fix_prompt_path).unwrap(); - // file.write_all(final_prompt.as_bytes()).unwrap(); let lock = FIX_LOCK.get_or_init(|| async { Mutex::new(()) }).await; let mut completion_tokens = 0; @@ -180,22 +177,8 @@ async fn compilation_fix_assistant_for_an_error( llm.get_answer(&final_prompt, 1, false).await?; completion_tokens += usage_completion; prompt_tokens += usage_prompt; - // for request_choice in request_choices.iter_mut() { - // *request_choice = "ChangeLog:".to_string() + request_choice; - // } - // let mut request_result: Vec<(Vec, TestCode)> = Vec::new(); - let mut request_choices_string = String::new(); - // for request in request_choices.iter() { - // request_choices_string = request_choices_string + request + "\n"; - // } - request_choices_string = request_choices.join("\n"); - // let fix_request_path = work_path.join("fix_request.txt"); - // let mut file = File::create(fix_request_path).unwrap(); - // file.write_all(request_choices_string.as_bytes()).unwrap(); - // println!("{:#?}", request_choices); - // let mut modified_file_right = true; + let request_choices_string = request_choices.join("\n"); - // for i in request_choices.iter() { let mut new_test_code = test_code.clone(); let mut changelog_list = llm_return_content_parser(work_path, &request_choices_string); let changelog_list_copy = changelog_list.clone(); @@ -215,82 +198,80 @@ async fn compilation_fix_assistant_for_an_error( continue; } - let template = include_str!("../../res/code_template.json"); - let code_template: Vec = serde_json::from_str(&template).unwrap(); - let mut fn_code = vec![ - "#[test]".to_string(), - TIMEOUT_DERIVE.to_string(), - sig.clone(), - ]; - fn_code.extend(new_test_code.codes.clone()); - let insert_code = if !common.is_empty() { - let mut code = common.clone(); - code.push("".to_string()); - code.extend(fn_code); - code + let compile_error_set; + if let Some(attrs) = integration_attrs { + if integration::repaired_candidate(&new_test_code.codes, attrs).is_none() { + continue; + } + let _guard = lock.lock().await; + let (updated, errors, _) = + check_integration_candidate(work_path, &new_test_code.codes)?; + new_test_code = updated; + compile_error_set = errors; } else { - fn_code - }; - let mut mod_code = code_template.clone(); - let pos = mod_code.len() - 1; - mod_code.splice(pos..pos, insert_code); + let template = include_str!("../../res/code_template.json"); + let code_template: Vec = serde_json::from_str(&template).unwrap(); + let mut fn_code = vec![ + "#[test]".to_string(), + TIMEOUT_DERIVE.to_string(), + sig.clone(), + ]; + fn_code.extend(new_test_code.codes.clone()); + let insert_code = if !common.is_empty() { + let mut code = common.clone(); + code.push("".to_string()); + code.extend(fn_code); + code + } else { + fn_code + }; + let mut mod_code = code_template.clone(); + let pos = mod_code.len() - 1; + mod_code.splice(pos..pos, insert_code); - let guard = lock.lock().await; - restore_file(file_path); - let restore = RestoreOnDrop(file_path); - insert_test(insert_kind, Path::new(&file_path), &mod_code); - // let _ = target_clean(&work_path); + let guard = lock.lock().await; + restore_file(file_path); + let restore = RestoreOnDrop(file_path); + insert_test(insert_kind, Path::new(&file_path), &mod_code); + // let _ = target_clean(&work_path); - let test_type = TestType::Error; - run_test(project_path, work_path, test_type, false, false); + let test_type = TestType::Error; + run_test(project_path, work_path, test_type, false, false); - let compiler_message_set = compiler_error_parser_from_json(work_path); + let compiler_message_set = compiler_error_parser_from_json(work_path); - let mut compile_error_set: Vec = Vec::new(); - for compiler_message in compiler_message_set.iter() { - let compile_error = ErrorMessage::new(work_path, compiler_message); - compile_error_set.push(compile_error); - } + let mut errors: Vec = Vec::new(); + for compiler_message in compiler_message_set.iter() { + let compile_error = ErrorMessage::new(work_path, compiler_message); + errors.push(compile_error); + } - // request_result.push((compile_error_set, new_test_code)); - // } - // if request_result.len() != 0 { - new_test_code = TestCode::new(&file_path, &new_test_code.codes); - restore_file(Path::new(&file_path)); - drop(restore); - drop(guard); - // let mut min_set_index = MAX; - // let mut i = 0; - // for request_r in request_result.iter() { - // if request_r.0.len() < min_set_num { - // min_set_num = request_r.0.len(); - // min_set_index = i; - // } - // i += 1; - // } + new_test_code = TestCode::new(&file_path, &new_test_code.codes); + restore_file(Path::new(&file_path)); + drop(restore); + drop(guard); + compile_error_set = errors; + } if compile_error_set.len() < min_set_num { min_error_set = compile_error_set; min_error_content = new_test_code; min_set_num = min_error_set.len(); } - // min_error_set = request_result[min_set_index].0.clone(); - // min_error_content = request_result[min_set_index].1.clone(); if min_set_num == 0 { - return Ok((min_error_set, min_error_content, completion_tokens, prompt_tokens)); + return Ok(( + min_error_set, + min_error_content, + completion_tokens, + prompt_tokens, + )); } - - // let template = include_str!("../../res/code_template.json"); - // let code_template: Vec = serde_json::from_str(&template).unwrap(); - // let mut fn_code = vec!["#[test]".to_string(), sig.clone()]; - // fn_code.extend(min_error_content.codes.clone()); - // let mut mod_code = code_template.clone(); - // let pos = mod_code.len() - 1; - // mod_code.splice(pos..pos, fn_code); - // restore_file(Path::new(&file_path)); - // insert_test(insert_kind, Path::new(&file_path), &mod_code); - // } } - return Ok((min_error_set, min_error_content, completion_tokens, prompt_tokens)); + return Ok(( + min_error_set, + min_error_content, + completion_tokens, + prompt_tokens, + )); } async fn compilation_fix_assistant_for_one_fn( @@ -347,7 +328,6 @@ async fn compilation_fix_assistant_for_one_fn( // let _ = target_clean(&work_path); let test_type = TestType::Error; - let test_name = format!("test_{}", fn_name); run_test(&project_dir, &work_path, test_type, false, false); // restore_file(&file_path); @@ -379,8 +359,8 @@ async fn compilation_fix_assistant_for_one_fn( let (new_error_set, new_error_content, usage_completion, usage_prompt) = compilation_fix_assistant_for_an_error( llm, random_error, &compile_error_set, &project_dir, &work_path, - &file_path, &test_file_content, sig.clone(), test_name.clone(), - insert_kind, &common, + &file_path, &test_file_content, sig.clone(), + insert_kind, &common, None, ).await.map_err(|error| io::Error::other(format!( "Repair model request failed for {name}: {error}" )))?; @@ -442,6 +422,147 @@ async fn compilation_fix_assistant_for_one_fn( Ok(test_gen_info) } +fn check_integration_candidate( + work_dir: &Path, + code: &[String], +) -> io::Result<(TestCode, Vec, Result<(), String>)> { + let temporary = integration::write_candidate(work_dir, code)?; + let output = integration::compile_candidate(work_dir)?; + let mut errors = Vec::new(); + for line in String::from_utf8_lossy(&output.stdout).lines() { + let Ok(value) = serde_json::from_str::(line) else { + continue; + }; + if value["reason"] == "compiler-message" && value["message"]["level"] == "error" { + let message: CompilerMessage = + serde_json::from_value(value).map_err(io::Error::other)?; + if message.references_file(work_dir, &integration::candidate_path(work_dir)) { + errors.push(ErrorMessage::new(work_dir, &message)); + } + } + } + let status = if output.status.success() { + Ok(()) + } else { + let details = format!( + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + if errors.is_empty() { + return Err(io::Error::other(format!( + "Integration compilation failed: {details}" + ))); + } + Err(details) + }; + let test_code = TestCode::new(&integration::candidate_path(work_dir), &code.to_vec()); + temporary.finish()?; + Ok((test_code, errors, status)) +} + +async fn fix_integration_function( + llm: &LLM, + project_dir: &Path, + work_dir: &Path, + mut test_gen: TestGenInfo, +) -> io::Result { + let lock = FIX_LOCK.get_or_init(|| async { Mutex::new(()) }).await; + let initial = { + let _guard = lock.lock().await; + integration::initial_uses(&test_gen, work_dir) + }; + let start = Instant::now(); + for chain in test_gen.get_tests_mut() { + for answer in chain.get_answers_mut() { + let context = IntegrationContext { + uses: initial + .iter() + .chain(answer.get_uses()) + .cloned() + .collect::>() + .into_iter() + .collect(), + common: answer.get_common().clone(), + }; + let mut completion = answer.get_completion_tokens(); + let mut prompt = answer.get_prompt_tokens(); + for test in answer.get_tests_mut() { + if test.integration_contexts.is_empty() { + test.integration_contexts = vec![context.clone(); test.codes.len()]; + } + for num in 0..test.codes.len() { + let code = integration::candidate_code( + &test.integration_contexts[num], + &test.attrs, + &test.codes[num], + "palm_candidate", + "test_candidate", + false, + ); + // Revalidate in the actual target, including cached successful candidates. + let (mut current, mut errors, status) = { + let _guard = lock.lock().await; + check_integration_candidate(work_dir, &code)? + }; + test.can_compile[num] = status; + if test.can_compile[num].is_ok() || test.repaired[num] { + continue; + } + let rounds = min(errors.len() + 2, 10); + for _ in 0..rounds { + if errors.is_empty() { + break; + } + let error = &errors[0]; + let (next_errors, next_code, used_completion, used_prompt) = + compilation_fix_assistant_for_an_error( + llm, + error, + &errors, + project_dir, + work_dir, + &integration::candidate_path(work_dir), + ¤t, + "fn test_candidate()".to_string(), + InsertKind::EOF, + &Vec::new(), + Some(&test.attrs), + ) + .await?; + errors = next_errors; + current = next_code; + completion += used_completion; + prompt += used_prompt; + } + let (context, body) = + integration::repaired_candidate(¤t.codes, &test.attrs).ok_or_else( + || io::Error::other("Integration repair changed the test declaration"), + )?; + test.integration_contexts[num] = context; + test.codes[num] = body; + test.repaired[num] = true; + test.can_compile[num] = if errors.is_empty() { + Ok(()) + } else { + Err(errors + .iter() + .flat_map(|error| error.code_snippets.clone()) + .collect::>() + .join("\n")) + }; + if start.elapsed() >= Duration::from_secs(FIX_TIMEOUT) { + return Ok(test_gen); + } + } + } + answer.set_completion_tokens(completion); + answer.set_prompt_tokens(prompt); + } + } + Ok(test_gen) +} + fn merge_common_in_code(test_gen_infos: &mut Vec) { for test_gen_info in test_gen_infos.iter_mut() { for fn_test in test_gen_info.get_tests_mut() { @@ -466,6 +587,7 @@ pub async fn llm_fix( work_path: PathBuf, functions: &FunctionSelection, tasks: usize, + integration: bool, ) -> io::Result<()> { if tasks == 0 || tasks > Semaphore::MAX_PERMITS { return Err(io::Error::other( @@ -478,11 +600,18 @@ pub async fn llm_fix( serde_json::from_str(&fs::read_to_string(&map_path).unwrap()).unwrap(); // Prefer saved repair progress, then add candidates not yet copied from generation. + let generated = get_test_gen_infos(&project_path, true); + for info in generated + .iter() + .filter(|info| functions.contains(info.get_name())) + { + info.check_mode(integration)?; + } let mut test_gen_infos = get_test_gen_infos(&project_path, false); let saved: BTreeSet<_> = test_gen_infos.iter() .map(|info| info.get_name().to_owned()).collect(); test_gen_infos.extend( - get_test_gen_infos(&project_path, true).into_iter() + generated.into_iter() .filter(|info| !saved.contains(info.get_name())), ); test_gen_infos.retain(|info| { @@ -503,7 +632,12 @@ pub async fn llm_fix( } } } - merge_common_in_code(&mut test_gen_infos); + for info in &test_gen_infos { + info.check_mode(integration)?; + } + if !integration { + merge_common_in_code(&mut test_gen_infos); + } create_dir_all(&parent_dir)?; for info in &test_gen_infos { let json_path = parent_dir.join(nmap.get(info.get_name()).unwrap().to_owned() + ".json"); @@ -516,7 +650,7 @@ pub async fn llm_fix( .map(|info| project_path.join(info.get_file())) .collect(); let mut backups = Vec::new(); - for source in sources { + for source in sources.into_iter().filter(|_| !integration) { match create_backup(&source) { Ok(backup) => backups.push((source, backup)), Err(error) => { @@ -545,13 +679,13 @@ pub async fn llm_fix( let llm = llm.clone(); handles.push(tokio::spawn(async move { let _permit = slots.acquire_owned().await.map_err(io::Error::other)?; - let test_gen_info = compilation_fix_assistant_for_one_fn( - &llm, - project_path_clone, - work_path, - test_gen_info, - ) - .await?; + let test_gen_info = if integration { + fix_integration_function(&llm, &project_path_clone, &work_path, test_gen_info).await? + } else { + compilation_fix_assistant_for_one_fn( + &llm, project_path_clone, work_path, test_gen_info, + ).await? + }; let json_path = parent_dir_clone.join(encoded_name + ".json"); test_gen_info.dump_json(&json_path); let completed = counter_clone.fetch_add(1, Ordering::Relaxed) + 1; diff --git a/utgen/src/run/llm_fix_type.rs b/utgen/src/run/llm_fix_type.rs index 7946295464..31a64797ac 100644 --- a/utgen/src/run/llm_fix_type.rs +++ b/utgen/src/run/llm_fix_type.rs @@ -44,6 +44,13 @@ impl CompilerMessage { pub fn has_spans(&self) -> bool { return self.message.has_spans(); } + + pub fn references_file(&self, work_dir: &Path, file: &Path) -> bool { + self.message + .spans + .iter() + .any(|span| work_dir.join(&span.file_name) == file) + } } #[derive(Debug, Clone, Deserialize)] diff --git a/utgen/src/run/mod.rs b/utgen/src/run/mod.rs index ec6ae5b3c3..c306d44e8f 100644 --- a/utgen/src/run/mod.rs +++ b/utgen/src/run/mod.rs @@ -1,7 +1,7 @@ static TIMEOUT_DERIVE: &str = "#[timeout(5000)]"; mod coverage; mod coverage_json; -mod integration; +pub(crate) mod integration; mod llm_fix; mod llm_fix_type; mod prepare; diff --git a/utgen/src/run/run.rs b/utgen/src/run/run.rs index c76eb9571b..a586590731 100644 --- a/utgen/src/run/run.rs +++ b/utgen/src/run/run.rs @@ -359,12 +359,23 @@ pub fn gen_test_rate( gen_coverage_rate_for_original_tests(project_dir, work_dir, &test_gen_infos); } } else { - let integration_infos = gen_integration(&test_gen_infos, project_dir, work_dir); + let integration_infos = gen_integration(&test_gen_infos, work_dir); let mut test_rate_infos: Vec = Vec::new(); - let test_type = TestType::CoverageRate; let start_time = SystemTime::now(); // let _ = target_clean(work_dir); - let test_output = run_test(project_dir, work_dir, test_type, true, true); + if integration_infos.is_empty() { + return; + } + let targets = integration_infos + .iter() + .map(|info| info.file_name.clone()) + .collect::>(); + let output = super::coverage::collect_coverage_for_tests(work_dir, true, &targets) + .expect("Failed to collect integration coverage"); + let test_output = String::from_utf8_lossy(&output.stdout) + .lines() + .map(str::to_owned) + .collect::>(); let end_time = SystemTime::now(); let test_time = end_time.duration_since(start_time).unwrap().as_secs_f64(); for integration_info in integration_infos.iter() { @@ -455,17 +466,16 @@ pub fn gen_test_rate( if already_test_rate_info.function_name == integration_info.function_name { already_test_rate_info.tests_run = tests; already_test_rate_info.tests_passed = passed_tests; - already_test_rate_info.tests_passed_rate = - passed_tests as f64 / tests as f64 * 100.0; + already_test_rate_info.tests_passed_rate = if tests > 0 { + passed_tests as f64 / tests as f64 * 100.0 + } else { + 0.0 + }; break; } } } // let _ = target_clean(work_dir); - let test_path = work_dir.join("tests"); - let bak_test_path2 = work_dir.join("tests.bak2"); - let generated_tests = MovedTestDirectory::new(&test_path, &bak_test_path2) - .expect("cannot temporarily move generated tests"); gen_codes_lines_and_branches_covered( project_dir, work_dir, @@ -480,6 +490,10 @@ pub fn gen_test_rate( false, test_time, ); + let test_path = work_dir.join("tests"); + let bak_test_path2 = work_dir.join("tests.bak2"); + let generated_tests = MovedTestDirectory::new(&test_path, &bak_test_path2) + .expect("cannot temporarily move generated tests"); if is_pre { gen_coverage_rate_for_original_tests(project_dir, work_dir, &test_gen_infos); } @@ -1188,72 +1202,61 @@ fn gen_codes_lines_and_branches_covered( test_gen_infos: &Vec, test_rate_infos: &mut Vec, ) { - for test_rate_info in test_rate_infos.iter_mut() { - let function_name = test_rate_info.function_name.clone(); - for test_gen_info in test_gen_infos.iter() { - if function_name != test_gen_info.get_name().to_string() { - continue; - } - let file_rela = test_gen_info.get_file(); - let file_path = project_dir.join(&file_rela); - let name = test_gen_info.get_name().to_string(); - let fn_name = name.split("::").last().unwrap(); - info!("Generate codes lines and branches for {}", fn_name); - let insert_kind = test_gen_info.get_insert_kind(); - let template = include_str!("../../res/code_template.json"); - let code_template: Vec = serde_json::from_str(template).unwrap(); - backup_file(Path::new(&file_path)); - for chain_test in test_gen_info.get_tests().iter() { - for answer in chain_test.get_answers().iter() { - for test in answer.get_tests().iter() { - for (num, test_code) in test.codes.iter().enumerate() { - if test.can_compile[num].is_ok() { - let mut mod_code = code_template.clone(); - let mut common = answer.get_common().clone(); - common.push("".to_string()); - let pos = mod_code.len() - 1; - mod_code.splice(pos..pos, common); - let sig = format!("fn test_{}_{:02}()", fn_name, num); - info!("Running test fn test_{}_{:02}()", fn_name, num); - let fn_code = timed_test(&sig, &test.attrs, test_code); - let pos = mod_code.len() - 1; - mod_code.splice(pos..pos, fn_code); - insert_test(insert_kind, Path::new(&file_path), &mod_code); - // let _ = target_clean(work_dir); - let test_type = TestType::CoverageRate; - let test_name = format!("test_{}", fn_name); - run_test(project_dir, work_dir, test_type, true, false); - let loc = test_gen_info.get_loc(); - let begin = loc.get_startline() as i32; - let end = loc.get_endline() as i32; - gen_one_codes_lines( - project_dir, - work_dir, - &name, - &file_path, - begin, - end, - &test_code, - test_rate_info, - ); - gen_one_codes_branchs( - project_dir, - work_dir, - &function_name, - &file_path, - begin, - end, - &test_code, - test_rate_info, - ); - restore_file(Path::new(&file_path)); - } + for rate in test_rate_infos { + let info = test_gen_infos + .iter() + .find(|info| info.get_name() == rate.function_name) + .unwrap(); + let initial = integration::initial_uses(info, work_dir); + let file_path = project_dir.join(info.get_file()); + let loc = info.get_loc(); + for chain in info.get_tests() { + for answer in chain.get_answers() { + for test in answer.get_tests() { + for (num, body) in test.codes.iter().enumerate() { + if test.can_compile[num].is_err() { + continue; } + let context = integration::candidate_context(answer, test, num, &initial); + let code = integration::candidate_code( + &context, + &test.attrs, + body, + "palm_candidate", + "test_candidate", + true, + ); + let temporary = integration::write_candidate(work_dir, &code).unwrap(); + super::coverage::collect_coverage_for_tests( + work_dir, + true, + &[integration::CANDIDATE_TARGET.to_string()], + ) + .expect("Failed to collect candidate coverage"); + gen_one_codes_lines( + project_dir, + work_dir, + &info.get_name().to_string(), + &file_path, + loc.get_startline() as i32, + loc.get_endline() as i32, + body, + rate, + ); + gen_one_codes_branchs( + project_dir, + work_dir, + &info.get_name().to_string(), + &file_path, + loc.get_startline() as i32, + loc.get_endline() as i32, + body, + rate, + ); + temporary.finish().unwrap(); } } } - delete_backup(&file_path); - break; } } } diff --git a/utgen/src/types/testgen.rs b/utgen/src/types/testgen.rs index afc7a464e7..c968867eee 100644 --- a/utgen/src/types/testgen.rs +++ b/utgen/src/types/testgen.rs @@ -26,6 +26,8 @@ pub enum InsertKind { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TestGenInfo { + #[serde(default)] + pub integration: bool, name: String, name_with_impl: String, mod_info: ModInfo, @@ -44,6 +46,7 @@ impl TestGenInfo { fn_tests: Vec, ) -> TestGenInfo { TestGenInfo { + integration: false, name, name_with_impl, mod_info, @@ -63,6 +66,16 @@ impl TestGenInfo { } } + pub fn check_mode(&self, integration: bool) -> std::io::Result<()> { + if self.integration != integration { + return Err(std::io::Error::other(format!( + "Test mode mismatch for {}; use the same --integration setting for gen and fix, or generate in a fresh working copy", + self.name + ))); + } + Ok(()) + } + pub fn get_file(&self) -> String { self.loc.get_file() } @@ -255,6 +268,12 @@ impl ChainTestAnswer { } } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IntegrationContext { + pub uses: Vec, + pub common: Vec, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TestInfo { pub attrs: Vec, @@ -263,6 +282,9 @@ pub struct TestInfo { pub codes: Vec>, pub can_compile: Vec>, pub repaired: Vec, + // Repair may change imports/helpers for one candidate without changing its siblings. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub integration_contexts: Vec, } impl TestInfo { @@ -275,6 +297,7 @@ impl TestInfo { codes, can_compile: vec![Ok(()); len], repaired: vec![false; len], + integration_contexts: Vec::new(), } } diff --git a/utgen/src/utils/check.rs b/utgen/src/utils/check.rs index bc88285d82..bb587dc55a 100644 --- a/utgen/src/utils/check.rs +++ b/utgen/src/utils/check.rs @@ -5,8 +5,17 @@ use std::path::Path; use std::process::Command; pub fn cargo_check(work_dir: &Path) -> Result<(), String> { + cargo_build(work_dir, &["--tests"]) +} + +pub(crate) fn cargo_check_test(work_dir: &Path, target: &str) -> Result<(), String> { + cargo_build(work_dir, &["--test", target]) +} + +fn cargo_build(work_dir: &Path, targets: &[&str]) -> Result<(), String> { let output = Command::new("cargo") - .args(["build", "--tests"]) + .arg("build") + .args(targets) .current_dir(work_dir) .output() .expect("failed to execute process"); @@ -30,7 +39,7 @@ pub fn use_check(use_set: &mut HashSet, work_dir: &Path) { use_set.retain(|x| { let code = format!("{}\n", x); fs::write(&path, code).unwrap(); - cargo_check(&work_dir).is_ok() + cargo_check_test(work_dir, "use_check").is_ok() }); temporary.finish().unwrap(); } From 76c6052253ac1577f6d4f530d47c31caedc317ab Mon Sep 17 00:00:00 2001 From: cppbear Date: Fri, 9 Oct 2026 02:43:10 +0800 Subject: [PATCH 2/2] fix: keep the integration repair test declaration active --- utgen/src/run/integration.rs | 39 +++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/utgen/src/run/integration.rs b/utgen/src/run/integration.rs index 88b293cd01..5167050bac 100644 --- a/utgen/src/run/integration.rs +++ b/utgen/src/run/integration.rs @@ -110,7 +110,7 @@ pub(crate) fn repaired_candidate( let [syn::Item::Mod(module)] = syntax.items.as_slice() else { return None; }; - if module.ident != "palm_candidate" { + if module.ident != "palm_candidate" || !module.attrs.is_empty() || !syntax.attrs.is_empty() { return None; } let (_, items) = module.content.as_ref()?; @@ -195,10 +195,7 @@ impl IntegrationInfo { } } -pub fn gen_integration( - test_gen_infos: &Vec, - work_dir: &Path, -) -> Vec { +pub fn gen_integration(test_gen_infos: &Vec, work_dir: &Path) -> Vec { info!("Generate integration tests!"); let mut integration_infos = Vec::new(); for test_gen in test_gen_infos.iter().filter(|info| info.get_visibility()) { @@ -260,3 +257,35 @@ pub fn gen_integration( } integration_infos } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn repair_cannot_disable_the_candidate_to_hide_a_compile_error() { + let context = IntegrationContext { + uses: vec![], + common: vec![], + }; + let code = candidate_code( + &context, + &[], + &["{ missing(); }".to_string()], + "palm_candidate", + "test_candidate", + false, + ); + assert!(repaired_candidate(&code, &[]).is_some()); + // Rust would accept this by compiling no tests. It must not be saved + // as a successful repair and then reconstructed without the cfg. + let mut disabled = code.clone(); + disabled.insert(0, "#[cfg(any())]".to_string()); + assert!(repaired_candidate(&disabled, &[]).is_none()); + let removed = code + .into_iter() + .filter(|line| line != "#[test]") + .collect::>(); + assert!(repaired_candidate(&removed, &[]).is_none()); + } +}