From a90d485ef4fc5b63592f9aa52e066f7c3293f96e Mon Sep 17 00:00:00 2001 From: cppbear Date: Thu, 8 Oct 2026 21:51:23 +0800 Subject: [PATCH] fix: restore timeouts for generated test statistics --- docs/minimal-pipeline.md | 2 + docs/palm-rust-unit-test-generation.md | 2 + scripts/check_minimal.py | 59 +++++++++++++++++++++++++- utgen/README.md | 2 + utgen/src/run/integration.rs | 14 +----- utgen/src/run/mod.rs | 32 +++++++++++++- utgen/src/run/run.rs | 26 ++---------- 7 files changed, 99 insertions(+), 38 deletions(-) diff --git a/docs/minimal-pipeline.md b/docs/minimal-pipeline.md index e183d6c33a..083bad78a6 100644 --- a/docs/minimal-pipeline.md +++ b/docs/minimal-pipeline.md @@ -36,6 +36,8 @@ The regression script runs in Linux CI after the ordinary workspace tests. It us Generation, repair, and statistics no longer call the partial `target_clean` helper between candidates. Cargo reuses build artifacts as sources are replaced and restored; analysis still clears the check cache, and coverage retains its own profile cleanup. The task regression preserves the target directory across scenarios and checks compilation failures followed by successful repairs. +Cached-candidate checks exercise the shared five-second test timeout in unit and integration statistics without additional model requests. Ordinary and `should_panic` infinite loops must count as failures, while an expected panic and a later normal test pass. Both production branches must remain covered and temporary source edits must be restored. The regression harness terminates its subprocess group if a check hangs; this external deadline is separate from the candidate timeout. + ## Coverage Interpretation `original/original-coverage.json` and `.xml` measure production code exercised by all original unit and integration tests through `utgen coverage`. Test bodies and helpers contribute to neither the coverage numerator nor denominator. The check requires both original `classify` branches to have executed. LLVM can emit separate records for the same branch location in different test binaries; the check unions those outcomes instead of requiring both in one record. diff --git a/docs/palm-rust-unit-test-generation.md b/docs/palm-rust-unit-test-generation.md index 9b705edb32..f73cdd9a8b 100644 --- a/docs/palm-rust-unit-test-generation.md +++ b/docs/palm-rust-unit-test-generation.md @@ -287,6 +287,8 @@ utgen fix --project-dir [--work-dir ...] [--tasks ] 任务异常会在等待所有任务结束后报告，并跳过后续覆盖率统计。修复成功时只删除本次创建的备份；任务失败时恢复源码并保留这些备份，已有备份不会被覆盖。候选本身编译不通过仍按测试结果处理。模型请求现在统一执行：`--request-timeout` 默认每次尝试 180 秒，包含响应体读取；网络错误、429（余额不足除外）和 5xx 最多尝试 3 次，其他 HTTP 错误或空回答直接报错。格式纠正和编译修复轮次单独计算，不再叠加网络重试。缺少 `usage` 的有效回答可以继续，但请求统计标记为不完整；生成和修复分别写入 `utgen/generation/gen-requests.json`、`fix-requests.json`。该期限不涵盖 Cargo 子进程，也不提供跨进程锁或强制终止恢复。 +生成候选的测试体通过 `ntest` 统一限时 5 秒，单元测试统计、集成测试和修复执行均使用此期限。超时候选计为失败，后续候选可以继续运行。统计路径在超时范围内检查预期 panic，避免超时本身使 `should_panic` 测试误判通过；预期 panic 沿用现有的任意 panic 判定。该限制不包含 Cargo 编译、模型请求，也不是任意子进程的强制终止机制。 + 小规模试跑可在 `gen`、`fix` 中分别传入同一份 `--functions-file`，不会自动沿用上次清单。空清单和未知名称在修改目标前报错；修复时缺少所选函数的候选结果也会报错。`--max-requests` 包含所有并发任务、生成阶段、格式纠正、修复轮次及网络重试的请求尝试，计数与额度检查使用同一个共享计数器。额度用完且仍需请求时，命令失败，等待已有请求收尾并恢复源码；恰好用完额度且工作全部完成则成功。请求报告记录模型、选项、去重后的函数清单、候选处理状态和用量；候选状态不等于测试通过或覆盖率统计完成。未选函数的已有文件保留，跨实验应使用新副本。[2 函数和 8 函数清单及操作说明](../examples/README.md#prepare-a-small-model-trial)提供限定函数范围和请求额度的用法；先检查请求报告，再查看对应函数的编译、执行和覆盖率统计。 ### 8.2 LLM 配置 diff --git a/scripts/check_minimal.py b/scripts/check_minimal.py index bc3bf65ceb..7062f339ee 100644 --- a/scripts/check_minimal.py +++ b/scripts/check_minimal.py @@ -7,6 +7,7 @@ from pathlib import Path import re import shutil +import signal import subprocess import tempfile import threading @@ -78,10 +79,19 @@ def main(): shutil.copytree(REPO / 'examples/minimal', copy, ignore=shutil.ignore_patterns('target', 'brinfo', 'focxt', 'utgen', 'tests.bak')) - def run(label, command, cwd=target, expected=0): + def run(label, command, cwd=target, expected=0, timeout=240): print(label, flush=True) - result = subprocess.run(command, cwd=cwd, env=env, capture_output=True, text=True, timeout=240) log = logs / (label + '.log') + process = subprocess.Popen(command, cwd=cwd, env=env, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, start_new_session=True) + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + stdout, stderr = process.communicate() + log.write_text(stdout + stderr) + raise AssertionError(f'{label}: timeout; see {log}') + result = subprocess.CompletedProcess(command, process.returncode, stdout, stderr) log.write_text(result.stdout + result.stderr) assert (result.returncode == 0) == (expected == 0), f'{label}: exit {result.returncode}; see {log}\n{(result.stdout + result.stderr)[-12000:]}' return result @@ -269,6 +279,51 @@ def do_POST(self): double = json.loads((target / f"utgen/result/{names['palm_fixture::nested::double']}.json").read_text()) assert double['tests_compiled'] == 0 and double['tests'] == 1, double assert not list((target / 'src').rglob('*.bak')) + + # Replay cached candidates on one function: ordinary and should_panic + # loops must fail, and later passing tests must still execute. + selected = 'palm_fixture::classify' + selection = work / 'timeout-function.txt' + selection.write_text(selected + '\n') + candidate = target / f'utgen/generation/pre_fix/{names[selected]}.json' + data = json.loads(candidate.read_text()) + data['fn_tests'] = data['fn_tests'][:1] + answer = data['fn_tests'][0]['answers'][0] + answer['chain_tests'] = [ + dict(attrs=attrs, prefix=[], oracles=[], codes=[['{', body, '}']], + can_compile=[{'Ok': None}], repaired=[False]) + for attrs, body in [ + ([], 'assert_eq!(classify(2), 1); loop { std::hint::spin_loop(); }'), + (['#[should_panic]'], 'loop { std::hint::spin_loop(); }'), + (['#[should_panic(expected = "expected")]'], 'panic!("expected");'), + ([], 'assert_eq!(classify(-1), 0);'), + ] + ] + candidate.write_text(json.dumps(data, indent=2) + '\n') + # Keep original-test coverage separate when switching this fixture to + # integration mode, which creates its own tests directory. + original_tests = work / 'timeout-original-tests' + shutil.move(target / 'tests.bak', original_tests) + timeout_prepared = snapshot(target) + requests_before = (state['generation'], state['repair']) + for integration in [False, True]: + run(f'candidate-timeouts-{integration}', + ['utgen', 'gen', '-p', str(target), '--functions-file', str(selection), + *(['--integration'] if integration else [])], timeout=90) + result = json.loads((target / f'utgen/result/{names[selected]}.json').read_text()) + assert result['tests'] == result['tests_compiled'] == result['tests_run'] == 4, result + assert result['tests_passed'] == 2, result + assert result['branches_covered'] == result['branches'] == 2, result + if integration: + assert not (target / 'tests.bak2').exists() + shutil.rmtree(target / 'tests') + assert snapshot(target) == timeout_prepared + assert not list((target / 'src').rglob('*.bak')) + assert (state['generation'], state['repair']) == requests_before, state + report = json.loads((target / 'utgen/generation/gen-requests.json').read_text()) + assert report['attempts'] == 0, report + shutil.move(original_tests, target / 'tests.bak') + assert snapshot(target) == prepared (work / 'summary.json').write_text(json.dumps(state, indent=2) + '\n') print('Minimal analysis/generation/repair/coverage checks passed.', flush=True) finally: diff --git a/utgen/README.md b/utgen/README.md index b31c793af4..7783ded216 100644 --- a/utgen/README.md +++ b/utgen/README.md @@ -155,6 +155,8 @@ Generation validates the branch index, context index, context files, and source Direct-test and prefix answers must parse as Rust and contain at least one supported `#[test]` function. Syntax errors and answers containing only helpers, imports, comments, or an empty test module use the existing format retry loop, with at most three attempts and a fixed one-second wait before each retry. The first attempt starts without a wait. Exhaustion fails that function's generation and skips subsequent command statistics. This checks candidate presence; compilation and execution still determine whether a candidate works. Prefixes are accepted without assertions because oracle generation follows separately. +Generated candidate execution uses a five-second `ntest` timeout, including unit-test statistics and integration tests. A timed-out candidate counts as failed and later candidates can still run. Statistics check expected panics inside the timeout boundary, so a timeout cannot satisfy `should_panic`; the existing any-panic expectation is retained. This limit covers the test body, not Cargo compilation, model requests, or arbitrary child processes. + Each answer returned by the request layer is saved before code-fence removal and parsing, under `utgen/generation/answer///test-attempt-N.txt` or `prefix-attempt-N.txt`, with N starting at 1. Logs identify the stage, chain, attempt and rejection reason. Accepted code retains the existing `code.rs` / `prefix.rs` layout. These paths are reused across invocations, not an invocation history; use fresh copies for separate trials. Existing cached candidates, including old empty candidates, are still skipped and are not migrated by this validation change. Each generation job handles one focal function, including its condition chains and input/prefix/oracle stages sequentially. It holds a slot until its result enters the bounded queue, which also has capacity N. A single consumer validates candidates and their imports while other jobs can await model responses. Thus N limits active generation jobs, not the number of compiler processes or Cargo's internal build jobs; buffered results and the candidate currently being validated are separate. diff --git a/utgen/src/run/integration.rs b/utgen/src/run/integration.rs index 060d5c742b..7df4733d27 100644 --- a/utgen/src/run/integration.rs +++ b/utgen/src/run/integration.rs @@ -3,7 +3,7 @@ use std::{collections::HashSet, fs, path::Path}; use log::info; use crate::{ - run::TIMEOUT_DERIVE, + run::timed_test, types::TestGenInfo, utils::{cargo_check, use_check}, }; @@ -103,17 +103,7 @@ pub fn gen_integration( tests_compiled += 1; has_oracle_compiled = true; let sig = format!("fn test_{}_{}_{:02}()", stem, rename, id); - let mut fn_code = - vec!["#[test]".to_string(), TIMEOUT_DERIVE.to_string()]; - fn_code.extend(test.attrs.clone().iter().map(|attr| { - if attr.contains("#[should_panic(") { - return "#[should_panic]".to_string(); - } else { - attr.clone() - } - })); - fn_code.push(sig); - fn_code.extend(test_code.clone()); + let fn_code = timed_test(&sig, &test.attrs, test_code); mod_content = mod_content + &fn_code.join("\n") + "\n\n"; integration_info .test_function_names diff --git a/utgen/src/run/mod.rs b/utgen/src/run/mod.rs index bcea5a703d..ec6ae5b3c3 100644 --- a/utgen/src/run/mod.rs +++ b/utgen/src/run/mod.rs @@ -1,4 +1,4 @@ -static TIMEOUT_DERIVE: &str = "#[timeout(1000)]"; +static TIMEOUT_DERIVE: &str = "#[timeout(5000)]"; mod coverage; mod coverage_json; mod integration; @@ -15,3 +15,33 @@ pub use run::gen_test_rate; pub use run::gen_test_rate_aggregated; pub use coverage::collect_coverage; + +/// Assemble a statistics test with the shared execution limit. +fn timed_test(signature: &str, attrs: &[String], body: &[String]) -> Vec { + use quote::ToTokens; + use syn::parse::Parser; + + let attrs = syn::Attribute::parse_outer + .parse_str(&attrs.join("\n")) + .expect("generated test attributes must be valid Rust"); + let should_panic = attrs.iter().any(|attr| attr.path().is_ident("should_panic")); + let mut code = vec!["#[test]".to_string(), TIMEOUT_DERIVE.to_string()]; + code.extend( + attrs.iter() + .filter(|attr| !attr.path().is_ident("should_panic")) + .map(|attr| attr.to_token_stream().to_string()), + ); + code.push(signature.to_string()); + if should_panic { + // Keep the existing any-panic expectation inside the timeout boundary, + // so the timeout's own panic cannot make a hanging test pass. + code.push("{".to_string()); + code.push("assert!(std::panic::catch_unwind(||".to_string()); + code.extend_from_slice(body); + code.push(").is_err(), \"test did not panic as expected\");".to_string()); + code.push("}".to_string()); + } else { + code.extend_from_slice(body); + } + code +} diff --git a/utgen/src/run/run.rs b/utgen/src/run/run.rs index 4a2d7f6b3c..b942b8db70 100644 --- a/utgen/src/run/run.rs +++ b/utgen/src/run/run.rs @@ -5,7 +5,7 @@ use super::{ }; use crate::{ FunctionSelection, - run::TIMEOUT_DERIVE, + run::timed_test, types::TestGenInfo, utils::{backup_file, delete_backup, insert_test, restore_file}, run::run_all::{prepare_all_unit_tests_for_one_shot_run}, @@ -801,17 +801,7 @@ fn gen_coverage_and_pass_rate( mod_code.splice(pos..pos, common); let sig = format!("fn test_{}_{:02}()", fn_name, num); info!("Running test fn test_{}_{:02}()", fn_name, tests); - let mut fn_code = - vec!["#[test]".to_string()]; - fn_code.extend(test.attrs.clone().iter().map(|attr| { - if attr.contains("#[should_panic(") { - return "#[should_panic]".to_string(); - } else { - attr.clone() - } - })); - fn_code.push(sig); - fn_code.extend(test_code.clone()); + let fn_code = timed_test(&sig, &test.attrs, test_code); let pos = mod_code.len() - 1; mod_code.splice(pos..pos, fn_code); insert_test(insert_kind, Path::new(&file_path), &mod_code); @@ -1178,17 +1168,7 @@ fn gen_codes_lines_and_branches_covered( mod_code.splice(pos..pos, common); let sig = format!("fn test_{}_{:02}()", fn_name, num); info!("Running test fn test_{}_{:02}()", fn_name, num); - let mut fn_code = - vec!["#[test]".to_string()]; - fn_code.extend(test.attrs.clone().iter().map(|attr| { - if attr.contains("#[should_panic(") { - return "#[should_panic]".to_string(); - } else { - attr.clone() - } - })); - fn_code.push(sig); - fn_code.extend(test_code.clone()); + let fn_code = timed_test(&sig, &test.attrs, test_code); let pos = mod_code.len() - 1; mod_code.splice(pos..pos, fn_code); insert_test(insert_kind, Path::new(&file_path), &mod_code);