From c07d67df572081eaf3270993e0aae82c81760b39 Mon Sep 17 00:00:00 2001 From: cppbear Date: Wed, 7 Oct 2026 21:38:12 +0800 Subject: [PATCH] fix: exclude test code from coverage and reuse one run --- README.md | 2 + docs/minimal-pipeline.md | 12 +- docs/palm-rust-unit-test-generation.md | 8 +- examples/minimal/README.md | 2 +- examples/minimal/src/lib.rs | 9 +- examples/minimal/src/support.rs | 5 +- scripts/check_minimal.py | 77 ++++++++- utgen/README.md | 23 ++- utgen/src/lib.rs | 2 +- utgen/src/main.rs | 14 +- utgen/src/run/coverage.rs | 207 +++++++++++++++++++++++++ utgen/src/run/mod.rs | 3 + utgen/src/run/run.rs | 30 +--- utgen/src/utils/preprocess.rs | 23 ++- utgen/tests/cli_config.rs | 1 + 15 files changed, 363 insertions(+), 55 deletions(-) create mode 100644 utgen/src/run/coverage.rs diff --git a/README.md b/README.md index ce019842e8..3bcf611808 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,8 @@ Run `docker/docker-run` from the same directory to mount the repository at `/hom ## Workflow +To measure existing tests before preprocessing, run `utgen coverage -p `. It runs tests once, excludes test code from coverage, and writes `coverage.xml` and `coverage.json` without model configuration. Keep this baseline copy separate from the generation copy. + On a fresh working copy of a standalone crate, run `utgen pre-process -p ` before `utgen analyze -p `. Analysis clears Cargo's check cache, runs brinfo and focxt, and verifies their outputs. It rejects existing `brinfo/` or `focxt/` directories so previous analysis results cannot be mixed into the run. Then use `utgen gen` and `utgen fix` for generation and compilation repair. Pass `--requirement --context` to `utgen gen` to include path constraints and focal context. Preprocessing and test checks modify the target tree; use a working copy of the target project. diff --git a/docs/minimal-pipeline.md b/docs/minimal-pipeline.md index 0490483af0..d683a0d587 100644 --- a/docs/minimal-pipeline.md +++ b/docs/minimal-pipeline.md @@ -18,21 +18,23 @@ The script prints the temporary directory containing source copies, command logs ## Verified Contracts -- Original tests run on an untouched copy. A separate prepared copy is used for generated tests. +- Original tests run on a separate original copy; coverage annotations are temporary and restored. A separate prepared copy is used for generated tests. - Preprocessing removes test-only source ranges while preserving newlines and byte offsets, including a test and production function sharing one line. Repeating preprocessing is idempotent. - `utgen analyze` succeeds even after a prior `cargo check`, since it clears the check cache before invoking the compiler wrapper. It rejects existing analysis output directories. - brinfo and focxt agree on five complete function names. Source locations identify the exported focal code, accounting for brinfo's indentation normalization. - The `Gauge::label` context contains its struct, the focal method, and the declarations for `classify`, `positive`, and the transitive `threshold` dependency. - Missing context and malformed source locations fail before generation changes the manifest or source files. Source-location parse errors no longer panic. - Six model responses produce six candidate tests. One candidate deliberately passes a string to an integer parameter and fails compilation; a numbered ChangeLog response repairs it through the actual repair path. +- Coverage executes each test body once for XML and JSON. Test functions, module helpers, standalone test-only helpers, and generated `llmtests` records are excluded; production coverage remains. +- Sources are restored after successful coverage, assertion failures, compilation failures, and JSON export failures. - Both branches of `classify` are covered. Before repair, the intentionally invalid `double` candidate has zero compilable tests. After repair, all candidates compile and pass. -- Source files and the target manifest match their prepared state after generation and repair; no source backup remains after successful completion. +- Direct byte comparisons verify that source files and the target manifest match their prepared state after generation and repair; no source backup remains after successful completion. The regression script runs in Linux CI after the ordinary workspace tests. It uses the real analyzers, compiler, coverage tool, prompt builder, response parser, candidate checks, and repair machinery. Model replies are deterministic fixtures; these results do not measure LLM quality. ## Coverage Interpretation -`original/original-coverage.json` measures all original unit and integration tests on the untouched copy. The check requires both original `classify` branches to have executed. +`original/original-coverage.json` and `.xml` measure production code exercised by all original unit and integration tests through `utgen coverage`. Test bodies and helpers contribute to neither the coverage numerator nor denominator. The check requires both original `classify` branches to have executed. LLVM can emit separate records for the same branch location in different test binaries; the check unions those outcomes instead of requiring both in one record. `generated/utgen/result/` and `generated/utgen/fixed_result/` contain the tool's per-function statistics for generated candidates. The script checks the known branch count and execution outcomes directly. The separate legacy `utgen/original_result.json` restores only the integration-test backup; it must not be presented as coverage of all original tests. Combined original-plus-generated coverage is not part of this check. @@ -44,4 +46,6 @@ The preprocessor parses `cfg` predicates conservatively, preserves `cfg(not(test Recursive context lookup now resolves full function names through the compiler's encoded-name index and consumes the accumulated dependencies. External functions without local artifacts remain outside this source context. -The local macOS/Apple Silicon run passed using six generation responses and one repair response. Real-model behavior, bytes-scale execution, general multi-crate support, request concurrency limits, hard process timeouts, recovery after forced termination, and parameter-aware cache invalidation remain later work. Git history is retained. +The local macOS/Apple Silicon run passed using six generation responses and one repair response. Real-model behavior, bytes-scale execution, general multi-crate support, request concurrency limits, hard process timeouts, recovery after forced termination, and explicit run/resume behavior remain later work. Git history is retained. + +The coverage entry temporarily annotates sources with `#[coverage(off)]` and enables the nightly feature at crate roots without adding newlines. It preserves compiler flags, restores original bytes on returned errors, and performs no hashing or source fingerprint checks. `cargo llvm-cov report --json` reuses the first run's data; the first XML-producing command retains the tool's normal profile cleanup between candidates. This does not provide crash recovery or general macro expansion. See the [coverage command reference](../utgen/README.md#coverage-of-existing-tests) for scope. diff --git a/docs/palm-rust-unit-test-generation.md b/docs/palm-rust-unit-test-generation.md index 71f4d51ca1..93a06cbdfe 100644 --- a/docs/palm-rust-unit-test-generation.md +++ b/docs/palm-rust-unit-test-generation.md @@ -381,7 +381,7 @@ export PALM_CONFIG="$(pwd)/utgen/res/api.json" ```sh cargo llvm-cov --tests --ignore-run-fail --branch --cobertura --output-path coverage.xml - cargo llvm-cov --tests --ignore-run-fail --branch --json --output-path coverage.json + cargo llvm-cov report --json --output-path coverage.json ``` - 错误模式下执行： @@ -392,6 +392,12 @@ export PALM_CONFIG="$(pwd)/utgen/res/api.json" - `gen_test_rate` 读取生成结果，运行覆盖率，计算每个被测函数的测试数、编译通过数、运行通过数、行覆盖和分支覆盖。 +- `utgen/src/run/coverage.rs` + - 覆盖率入口复用预处理的测试识别规则，在测试模块、独立测试函数及 `cfg(test)` 辅助函数前临时添加 `#[coverage(off)]`。整个测试模块（含生成的 `llmtests`）排除后，其中的辅助代码也不计入覆盖率分子和分母；被调用的生产函数仍参与统计。 + - 在 crate 根文件同一行临时开启 `coverage_attribute`，保持原行号，不覆盖编译 flags。成功或编译/导出报错返回前恢复源文件原始内容；强制终止后的恢复留待后续处理。 + - 测试只执行一次，先输出 XML，再由 `report --json` 导出同一份运行数据。第一条命令保留默认清理，避免候选之间累积旧覆盖率。 + - `utgen coverage -p <原始 crate 工作副本>` 可在预处理前测量原有测试，无需模型配置和分析产物；结果保留在该副本的 `coverage.xml`、`coverage.json`。断言失败仍导出覆盖率，编译或报告错误则返回失败。当前支持源码位于 `src/` 的独立 crate，不展开宏，也不自动推断未标记的公共辅助函数是否仅用于测试。 + - `utgen/src/run/coverage_json.rs` - 解析 `coverage.json` 中的 branch 信息。 diff --git a/examples/minimal/README.md b/examples/minimal/README.md index caae50d058..0850d9f5a0 100644 --- a/examples/minimal/README.md +++ b/examples/minimal/README.md @@ -1,5 +1,5 @@ # Minimal PALM Target -This standalone library exercises five focal functions: a branch, a struct method, an inline module, and a two-level cross-module call chain. It also includes existing unit/integration tests and a test sharing a line with production code. Its test dependencies are locked for the pinned nightly. +This standalone library exercises five focal functions: a branch, a struct method, an inline module, and a two-level cross-module call chain. It also includes existing unit/integration tests and a test sharing a line with production code, and test-only helpers. A helper can record a test execution for the coverage regression, which checks that XML and JSON do not cause duplicate runs. Its test dependencies are locked for the pinned nightly. Run the [minimal pipeline check](../../docs/minimal-pipeline.md) from the PALM repository root. The script copies this directory before preprocessing or analysis; generated data is not written into this fixture. diff --git a/examples/minimal/src/lib.rs b/examples/minimal/src/lib.rs index 51265c26b0..e9976e3322 100644 --- a/examples/minimal/src/lib.rs +++ b/examples/minimal/src/lib.rs @@ -2,8 +2,15 @@ mod support; #[cfg(test)] mod tests { + fn record_run() { + if let Ok(path) = std::env::var("PALM_FIXTURE_RUN_LOG") { + use std::io::Write; + std::fs::OpenOptions::new().create(true).append(true).open(path).unwrap() + .write_all(b"x").unwrap(); + } + } #[test] - fn existing_unit_test() { assert_eq!(super::classify(2), 1); } + fn existing_unit_test() { record_run(); assert_eq!(super::classify(2), 1); } } pub struct Gauge { diff --git a/examples/minimal/src/support.rs b/examples/minimal/src/support.rs index 184396905b..2d9f3312bf 100644 --- a/examples/minimal/src/support.rs +++ b/examples/minimal/src/support.rs @@ -1,7 +1,10 @@ -#[test] fn before_helper() { assert!(positive(1)); } pub fn positive(value: i32) -> bool { +#[test] fn before_helper() { assert!(test_only_helper()); } pub fn positive(value: i32) -> bool { threshold(value) } fn threshold(value: i32) -> bool { value > 0 } + +#[cfg(test)] +fn test_only_helper() -> bool { positive(1) } diff --git a/scripts/check_minimal.py b/scripts/check_minimal.py index 3b09ba311e..bc3bf65ceb 100644 --- a/scripts/check_minimal.py +++ b/scripts/check_minimal.py @@ -1,7 +1,6 @@ #!/usr/bin/env python3 """Exercise PALM on an isolated, deterministic crate; never use a real model.""" import argparse -import hashlib import http.server import json import os @@ -11,14 +10,16 @@ import subprocess import tempfile import threading +import xml.etree.ElementTree as ET REPO = Path(__file__).resolve().parents[1] def snapshot(root): - return {str(p.relative_to(root)): hashlib.sha256(p.read_bytes()).hexdigest() - for p in sorted((root / 'src').rglob('*.rs'))} | { - 'Cargo.toml': hashlib.sha256((root / 'Cargo.toml').read_bytes()).hexdigest()} + return {str(p.relative_to(root)): p.read_bytes() + for directory in ['src', 'tests', 'tests.bak'] + for p in sorted((root / directory).rglob('*.rs'))} | { + 'Cargo.toml': (root / 'Cargo.toml').read_bytes()} def source_fragment(root, loc): @@ -129,14 +130,68 @@ def run(label, command, cwd=target, expected=0): version = run('coverage-version', ['cargo', 'llvm-cov', '--version']) assert version.stdout.strip() == 'cargo-llvm-cov 0.6.16', version.stdout - run('original-coverage', ['cargo', 'llvm-cov', '--tests', '--branch', '--json', '--output-path', 'original-coverage.json'], original) - original_coverage = json.loads((original / 'original-coverage.json').read_text()) + baseline = snapshot(original) + # A test side effect must occur once even though both formats are exported. + counter = work / 'coverage-runs.txt' + env['PALM_FIXTURE_RUN_LOG'] = str(counter) + coverage_run = run('original-coverage', ['utgen', 'coverage', '-p', str(original)], original) + assert counter.read_text() == 'x', 'coverage ran the test body more than once' + env.pop('PALM_FIXTURE_RUN_LOG') + assert snapshot(original) == baseline + assert '2 passed; 0 failed' in coverage_run.stdout and '1 passed; 0 failed' in coverage_run.stdout + original_coverage = json.loads((original / 'coverage.json').read_text()) + (original / 'coverage.json').rename(original / 'original-coverage.json') + (original / 'coverage.xml').rename(original / 'original-coverage.xml') + covered_functions = [function['name'] for data in original_coverage['data'] for function in data['functions']] + assert not any(marker in name for name in covered_functions + for marker in ['existing_unit_test', 'before_helper', 'test_only_helper', 'record_run', 'existing_integration_test']), covered_functions + assert any('classify' in name for name in covered_functions), covered_functions + xml = ET.parse(original / 'original-coverage.xml') + assert not any(marker in method.get('name', '') for method in xml.iter('method') + for marker in ['existing_unit_test', 'before_helper', 'test_only_helper', 'record_run']) + # Test-only lines must disappear from both the numerator and denominator. + lib_lines = (original / 'src/lib.rs').read_text().splitlines() + test_line = next(i for i, line in enumerate(lib_lines, 1) if 'fn existing_unit_test' in line) + for file in (file for data in original_coverage['data'] for file in data['files']): + if Path(file['filename']) == original / 'src/lib.rs': + assert not any(segment[0] == test_line and segment[3] for segment in file['segments']) + for cls in xml.iter('class'): + if cls.get('filename', '').endswith('src/lib.rs'): + assert not any(int(line.get('number')) == test_line for line in cls.findall('./lines/line')) original_file = next(file for data in original_coverage['data'] for file in data['files'] if Path(file['filename']) == original / 'src/lib.rs') condition_line = (original / 'src/lib.rs').read_text().splitlines().index(' if support::positive(value) {') + 1 # Library and unit-test binaries may emit separate records for one location. records = [branch for branch in original_file['branches'] if branch[0] == condition_line] assert any(branch[4] > 0 for branch in records) and any(branch[5] > 0 for branch in records) + # Preserve assertion failures as test outcomes, but stop on build errors; + # temporary coverage attributes must be removed in either case. + lib = original / 'src/lib.rs' + saved_lib = lib.read_bytes() + try: + lib.write_bytes(saved_lib.replace(b'assert_eq!(super::classify(2), 1)', b'assert_eq!(super::classify(2), 99)')) + failed_test_source = snapshot(original) + failed_test = run('coverage-failed-test', ['utgen', 'coverage', '-p', str(original)], original) + assert '1 passed; 1 failed' in failed_test.stdout + assert snapshot(original) == failed_test_source + lib.write_bytes(saved_lib + b'\ncompile_error!("coverage build failure fixture");\n') + failed_build_source = snapshot(original) + failure = run('coverage-failed-build', ['utgen', 'coverage', '-p', str(original)], original, expected=1) + assert 'coverage build failure fixture' in failure.stderr and 'panicked' not in failure.stderr + assert snapshot(original) == failed_build_source + lib.write_bytes(saved_lib) + # Block only the JSON export, after the tests and XML export succeed. + (original / 'coverage.json').unlink() + (original / 'coverage.json').mkdir() + try: + report_failure = run('coverage-failed-report', ['utgen', 'coverage', '-p', str(original)], original, expected=1) + assert 'report --json' in report_failure.stderr and 'panicked' not in report_failure.stderr + assert snapshot(original) == baseline + finally: + (original / 'coverage.json').rmdir() + finally: + lib.write_bytes(saved_lib) + assert snapshot(original) == baseline state = {'generation': 0, 'repair': 0, 'errors': []} class Model(http.server.BaseHTTPRequestHandler): @@ -160,7 +215,7 @@ def do_POST(self): 'threshold': 'assert!(threshold(2));', 'double': 'assert_eq!(double("bad"), 4);', }[function] - answer = '#[test]\nfn generated() {\n ' + body + '\n}\n' + answer = 'fn generated_helper() -> i32 { 1 }\n#[test]\nfn generated() {\n assert_eq!(generated_helper(), 1);\n ' + body + '\n}\n' state['generation'] += 1 else: file = re.search(r'You can only modify lines \d+ to \d+ in file (.+?)\. For your answer', user).group(1) @@ -194,6 +249,12 @@ def do_POST(self): assert snapshot(target) == prepared run('repair', ['utgen', 'fix', '-p', str(target)]) assert snapshot(target) == prepared + # The last repaired candidate's raw data remains after per-function parsing. + # Exporting it again must not run tests or contain llmtests/helper records. + run('generated-coverage-report', ['cargo', 'llvm-cov', 'report', '--json', '--output-path', 'generated-coverage.json']) + generated_coverage = json.loads((target / 'generated-coverage.json').read_text()) + generated_functions = [function['name'] for data in generated_coverage['data'] for function in data['functions']] + assert generated_functions and not any('llmtests' in name or 'generated_helper' in name for name in generated_functions), generated_functions assert not state['errors'], state assert state['generation'] == 6 and state['repair'] >= 1, state for directory in ['result', 'fixed_result']: @@ -203,6 +264,8 @@ def do_POST(self): assert all(r['tests_compiled'] == r['tests'] and r['tests_passed'] == r['tests_run'] and r['tests_run'] > 0 for r in results), results classify = next(r for r in results if r['function_name'] == 'palm_fixture::classify') assert classify['branches_covered'] == 2 and classify['branches'] == 2, classify + assert all(not (branch['positive'] and branch['negative']) + for _, branches in classify['codes_branches_covered'] for branch in branches), classify double = json.loads((target / f"utgen/result/{names['palm_fixture::nested::double']}.json").read_text()) assert double['tests_compiled'] == 0 and double['tests'] == 1, double assert not list((target / 'src').rglob('*.bak')) diff --git a/utgen/README.md b/utgen/README.md index 77f5e89e41..7e92ae5260 100644 --- a/utgen/README.md +++ b/utgen/README.md @@ -4,7 +4,7 @@ utgen generates Rust tests from condition chains and focal context, checks compi ## Prerequisites -Model configuration is loaded at runtime, once per `gen` or `fix` command. Building, installing, running help, `pre-process`, and `analyze` do not require it. +Model configuration is loaded at runtime, once per `gen` or `fix` command. Building, installing, running help, `pre-process`, `analyze`, and `coverage` do not require it. Copy [api.example.json](res/api.example.json) to `utgen/res/api.json` from the repository root, then replace its placeholders: @@ -78,10 +78,11 @@ The following is a command reference rather than captured help output. Use `utge | --- | --- | | `pre-process` | Rename each selected crate's `tests/` to `tests.bak/` and replace test-only source ranges with whitespace while preserving line breaks and byte offsets. | | `analyze` | Clear Cargo's check cache, run brinfo and focxt, and validate their function indices and artifacts. Currently supports one standalone crate passed with `-p`. | +| `coverage` | Run existing tests once and export XML/JSON with test code excluded. Supports one standalone crate passed with `-p`. | | `gen` | Generate candidates, check compilation, and collect pre-repair statistics. | | `fix` | Attempt to repair candidates with compilation errors and collect post-repair statistics using unit-test insertion. | -All four commands take `-p, --project-dir`. Use `-w, --work-dir` for individual crates in a larger project. Both relative paths are resolved against the current shell directory; work directories do not resolve against `--project-dir`. Work directories may be repeated or comma-separated and default to the project directory. The current `analyze` command requires a single standalone crate; use that crate as `-p`, with no separate work-directory selection. +All commands take `-p, --project-dir`. Except for `coverage`, commands also accept `-w, --work-dir` for individual crates in a larger project. Both relative paths are resolved against the current shell directory; work directories do not resolve against `--project-dir`. Work directories may be repeated or comma-separated and default to the project directory. The current `analyze` command requires a single standalone crate; use that crate as `-p`, with no separate work-directory selection. ### Preprocessing @@ -109,6 +110,22 @@ The target must have `Cargo.toml` and `src/`, with no existing `brinfo/` or `foc For manual analysis, run `cargo clean`, `cargo brinfo`, then `focxt -c ` in the prepared crate. A prior `cargo check` can otherwise prevent the compiler wrapper from running. +### Coverage of existing tests + +On an original working copy, before preprocessing: + +```sh +utgen coverage -p +``` + +This command needs neither model configuration nor analysis artifacts. It writes `coverage.xml` and `coverage.json` in the crate directory from a single test run. Assertion failures remain visible in test output and coverage is still exported; compilation or report-export failures return a nonzero exit status. A failed command does not guarantee valid output files. + +The same coverage entry point is used by generation and repair. It temporarily adds `#[coverage(off)]` to test-only modules (including generated `llmtests`), standalone test functions, and `#[cfg(test)]` helper functions. Module exclusion also covers nested helpers. Production functions called by tests remain instrumented; mixed production/test files are never excluded wholesale. Integration-test files use cargo-llvm-cov's default directory exclusions. Doc tests are not run by `--tests`. + +The nightly feature gate is inserted temporarily on the same line at crate roots, leaving compiler flags and Cargo configuration untouched. Ordinary compilation and repair need no coverage attributes. Coverage annotations are removed by restoring the bytes saved at coverage entry, after success or a returned build/report error. Temporary insertions preserve line numbers, but columns and byte offsets can change while measuring; use working copies and do not edit them concurrently. Recovery for the broader generation/repair backup flow and forced termination is still pending. + +Scope is a standalone crate with sources under `src/` and Cargo target entry files. Test identification reuses preprocessing's conservative `cfg` rules; macros are not expanded, and unmarked helpers outside test-only modules cannot be inferred as test-only. The pinned nightly and cargo-llvm-cov version are required. + ### Generation ```sh @@ -150,6 +167,6 @@ Paths under `utgen/` below are relative to `--project-dir`: | `utgen/fixed_result/` | Post-repair coverage and execution statistics per focal function. | | `utgen/original_result.json` | Comparison statistics when the original integration-test backup is available. | -`coverage.xml`, `coverage.json`, and `error_output.json` are intermediate files in the work directory and may be deleted after parsing. The current implementation does not produce an HTML report. +`coverage.xml`, `coverage.json`, and `error_output.json` are intermediate files in the work directory and may be deleted after generation/repair parses them. The standalone `coverage` command retains both reports. The current implementation does not produce an HTML report. See the [deterministic minimal pipeline](../docs/minimal-pipeline.md) and the [bytes example](../examples/README.md) for a working-copy workflow and the [Chinese technical guide](../docs/palm-rust-unit-test-generation.md) for implementation details. diff --git a/utgen/src/lib.rs b/utgen/src/lib.rs index d8418c2d5c..0cd0e43779 100644 --- a/utgen/src/lib.rs +++ b/utgen/src/lib.rs @@ -9,5 +9,5 @@ pub use gene::gen_tests_project; pub use config::{ConfigError, LlmConfig}; pub use gene::LLM; pub use analysis::{analyze_project, validate_analysis, validate_repair}; -pub use run::{gen_test_rate, llm_fix, gen_test_rate_aggregated}; +pub use run::{collect_coverage, gen_test_rate, llm_fix, gen_test_rate_aggregated}; pub use utils::{comment_out_tests, rename_tests_to_bak}; diff --git a/utgen/src/main.rs b/utgen/src/main.rs index 978b6f589a..97ba96cb83 100644 --- a/utgen/src/main.rs +++ b/utgen/src/main.rs @@ -3,7 +3,7 @@ use log::info; use simplelog::{ColorChoice, ConfigBuilder, LevelFilter, TermLogger, TerminalMode}; use std::env; use std::path::PathBuf; -use utgen::{LLM, LlmConfig, analyze_project, validate_repair}; +use utgen::{LLM, LlmConfig, analyze_project, collect_coverage, validate_repair}; use utgen::{comment_out_tests, gen_test_rate, gen_tests_project, llm_fix, rename_tests_to_bak}; /// Generate unit tests for a project @@ -28,6 +28,12 @@ enum Command { #[command(flatten)] options: Opts, }, + /// Run existing tests once and export coverage excluding test code (no model needed) + Coverage { + /// Standalone crate working copy + #[arg(short, long)] + project_dir: PathBuf, + }, /// Generate and run tests for the project to collect coverage data Gen { #[command(flatten)] @@ -145,6 +151,12 @@ async fn main() -> Result<(), Box> { analyze_project(work_dir)?; } } + Command::Coverage { project_dir } => { + let output = collect_coverage(&project_dir.canonicalize()?, true)?; + use std::io::Write; + std::io::stdout().write_all(&output.stdout)?; + std::io::stderr().write_all(&output.stderr)?; + } // generate unit tests Command::Gen { options, diff --git a/utgen/src/run/coverage.rs b/utgen/src/run/coverage.rs new file mode 100644 index 0000000000..2de585c385 --- /dev/null +++ b/utgen/src/run/coverage.rs @@ -0,0 +1,207 @@ +use crate::utils::test_ranges; +use cargo_metadata::{MetadataCommand, TargetKind}; +use std::{ + collections::BTreeSet, + fs, io, + path::{Path, PathBuf}, + process::{Command, Output}, +}; +use syn::{Meta, Token, punctuated::Punctuated}; +use walkdir::WalkDir; + +fn has_coverage_feature(meta: &Meta) -> bool { + let Meta::List(list) = meta else { return false }; + let Ok(args) = list.parse_args_with(Punctuated::::parse_terminated) else { + return false; + }; + if list.path.is_ident("feature") { + args.iter() + .any(|arg| arg.path().is_ident("coverage_attribute")) + } else if list.path.is_ident("cfg_attr") + && args.first().is_some_and(|arg| { + arg.path().is_ident("coverage") || arg.path().is_ident("coverage_nightly") + }) + { + args.iter().skip(1).any(has_coverage_feature) + } else { + false + } +} + +fn annotate_source(source: &str, crate_root: bool) -> syn::Result { + let mut annotated = source.to_owned(); + // Reverse edits keep the original byte offsets valid. Inserting no newline + // preserves focal line ranges, including production following a same-line test. + for range in test_ranges(source)?.into_iter().rev() { + annotated.insert_str(range.start, "#[coverage(off)] "); + } + if crate_root { + let syntax = syn::parse_file(source)?; + if !syntax + .attrs + .iter() + .any(|attr| has_coverage_feature(&attr.meta)) + { + let bom = if source.starts_with('\u{feff}') { 3 } else { 0 }; + let offset = if syntax.shebang.is_some() { + source.find('\n').map_or(source.len(), |i| i + 1) + } else { + bom + }; + annotated.insert_str(offset, "#![feature(coverage_attribute)] "); + } + } + Ok(annotated) +} + +fn coverage_command(work_dir: &Path, args: &[&str]) -> io::Result { + let output = Command::new("cargo") + .arg("llvm-cov") + .args(args) + .current_dir(work_dir) + .output()?; + if !output.status.success() { + return Err(io::Error::other(format!( + "cargo llvm-cov {} failed ({}):\n{}\n{}", + args.join(" "), + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ))); + } + Ok(output) +} + +/// Run tests once, excluding test code from instrumentation, then export XML +/// and optionally JSON from that run. Use a standalone crate working copy. +/// Test assertion failures are retained in the output; build/report errors fail. +pub fn collect_coverage(work_dir: &Path, json: bool) -> io::Result { + let canonical_dir = work_dir.canonicalize()?; + let work_dir = canonical_dir.as_path(); + let metadata = MetadataCommand::new() + .manifest_path(work_dir.join("Cargo.toml")) + .current_dir(work_dir) + .no_deps() + .exec() + .map_err(io::Error::other)?; + let package = metadata.root_package().ok_or_else(|| { + io::Error::other("coverage requires a standalone crate with Cargo.toml and src/") + })?; + if metadata.workspace_members.len() != 1 || !work_dir.join("src").is_dir() { + return Err(io::Error::other( + "coverage currently supports one standalone crate", + )); + } + let roots: BTreeSet = package + .targets + .iter() + .filter(|target| !target.is_kind(TargetKind::CustomBuild)) + .map(|target| target.src_path.clone().into_std_path_buf()) + .collect(); + let mut files = roots.clone(); + for entry in WalkDir::new(work_dir.join("src")) { + let entry = entry.map_err(io::Error::other)?; + if entry.file_type().is_file() && entry.path().extension().is_some_and(|e| e == "rs") { + files.insert(entry.into_path()); + } + } + // Prepare all edits before writing. Keep original bytes in memory, without + // sharing the generation pipeline's .bak files or introducing fingerprints. + let mut changes = Vec::new(); + for path in files { + let original = fs::read_to_string(&path)?; + let annotated = annotate_source(&original, roots.contains(&path)).map_err(|err| { + io::Error::new( + io::ErrorKind::InvalidData, + format!("{}: {err}", path.display()), + ) + })?; + if annotated != original { + changes.push((path, original, annotated)); + } + } + let result = (|| { + for (path, _, annotated) in &changes { + fs::write(path, annotated)?; + } + // The first command uses llvm-cov's default cleanup. --no-report would + // imply --no-clean and mix profiles from different generated candidates. + let output = coverage_command( + work_dir, + &[ + "--tests", + "--ignore-run-fail", + "--branch", + "--cobertura", + "--output-path", + "coverage.xml", + ], + )?; + if json { + coverage_command( + work_dir, + &["report", "--json", "--output-path", "coverage.json"], + )?; + } + Ok(output) + })(); + // Restore even when compilation, test launch, or report export failed. + // Try every changed file before reporting a restoration error. + let mut restore_error = None; + for (path, original, _) in changes { + if let Err(err) = fs::write(&path, original) { + restore_error.get_or_insert_with(|| { + io::Error::other(format!("failed to restore {}: {err}", path.display())) + }); + } + } + if let Some(err) = restore_error { + return Err(err); + } + result +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_modules_helpers_and_same_line_tests_keep_production_lines() { + let source = "#[cfg(test)] mod tests { fn helper() {} #[test] fn a() {} }\r\n#[cfg(test)] fn helper() {}\r\n#[test] fn b() {} pub fn production() {}\r\n#[cfg(any(test, feature = \"shared\"))] mod shared {}\r\n"; + let result = annotate_source(source, true).unwrap(); + assert_eq!(result.matches("#[coverage(off)]").count(), 3); + assert_eq!(result.matches('\r').count(), source.matches('\r').count()); + assert_eq!(result.lines().count(), source.lines().count()); + assert!( + result + .lines() + .nth(2) + .unwrap() + .ends_with("pub fn production() {}") + ); + assert!(result.lines().nth(3).unwrap().starts_with("#[cfg(any(")); + syn::parse_file(&result).unwrap(); + } + + #[test] + fn existing_coverage_feature_is_not_duplicated() { + for gate in [ + "#![feature(coverage_attribute)]", + "#![cfg_attr(coverage_nightly, feature(coverage_attribute))]", + ] { + let source = format!("{gate}\n#[test] fn test() {{}}\n"); + let result = annotate_source(&source, true).unwrap(); + assert_eq!(result.matches("coverage_attribute").count(), 1); + } + } + + #[test] + fn bom_and_shebang_precede_the_feature_gate() { + let prefix = "\u{feff}#!/usr/bin/env rust-script\n"; + let source = format!("{prefix}#[test] fn test() {{}} fn keep() {{}}\n"); + let result = annotate_source(&source, true).unwrap(); + assert!(result.starts_with(&format!("{prefix}#![feature(coverage_attribute)]"))); + assert_eq!(result.lines().count(), source.lines().count()); + syn::parse_file(&result).unwrap(); + } +} diff --git a/utgen/src/run/mod.rs b/utgen/src/run/mod.rs index 1dda1fd538..bcea5a703d 100644 --- a/utgen/src/run/mod.rs +++ b/utgen/src/run/mod.rs @@ -1,4 +1,5 @@ static TIMEOUT_DERIVE: &str = "#[timeout(1000)]"; +mod coverage; mod coverage_json; mod integration; mod llm_fix; @@ -12,3 +13,5 @@ pub use prepare::add_ntest_dependency; // pub use run::cargo_clean; pub use run::gen_test_rate; pub use run::gen_test_rate_aggregated; + +pub use coverage::collect_coverage; diff --git a/utgen/src/run/run.rs b/utgen/src/run/run.rs index c607e83079..099513215a 100644 --- a/utgen/src/run/run.rs +++ b/utgen/src/run/run.rs @@ -43,19 +43,8 @@ pub fn run_test( .output() .expect("Failed to run tests"); } else { - let coverage_output = Command::new("cargo") - .args([ - "llvm-cov", - "--tests", - "--ignore-run-fail", - "--branch", - "--cobertura", - "--output-path", - "coverage.xml", - ]) // RUST_BACKTRACE=full cargo llvm-cov --tests --ignore-run-fail --branch --cobertura - .current_dir(work_path) - .output() - .expect("Failed to generate coverage report"); + let coverage_output = super::coverage::collect_coverage(work_path, is_json) + .expect("Failed to collect coverage"); if !is_integration { return_value = String::from_utf8_lossy(&coverage_output.stdout) .to_string() @@ -71,21 +60,6 @@ pub fn run_test( .map(|line| line.to_string()) .collect::>(); } - if is_json { - let json_output = Command::new("cargo") - .args([ - "llvm-cov", - "--tests", - "--ignore-run-fail", - "--branch", - "--json", - "--output-path", - "coverage.json", - ]) - .current_dir(work_path) - .output() - .expect("Failed to generate coverage report"); - } } return return_value; } diff --git a/utgen/src/utils/preprocess.rs b/utgen/src/utils/preprocess.rs index 8f528246f2..8e5bcd7bef 100644 --- a/utgen/src/utils/preprocess.rs +++ b/utgen/src/utils/preprocess.rs @@ -110,10 +110,11 @@ impl<'ast> Visit<'ast> for TestRanges { } } fn visit_item_fn(&mut self, item: &'ast syn::ItemFn) { - if item - .attrs - .iter() - .any(|a| a.path().segments.last().is_some_and(|s| s.ident == "test")) + if is_test_module(&item.attrs) + || item + .attrs + .iter() + .any(|a| a.path().segments.last().is_some_and(|s| s.ident == "test")) { self.ranges.push(item.span().byte_range()); } else { @@ -122,7 +123,7 @@ impl<'ast> Visit<'ast> for TestRanges { } } -fn prepare_source(source: &str) -> syn::Result { +pub(crate) fn test_ranges(source: &str) -> syn::Result>> { let syntax = syn::parse_file(source)?; // syn strips these prefixes before tokenizing; spans are relative to the // remainder. Account for them when editing the original byte buffer. @@ -133,11 +134,19 @@ fn prepare_source(source: &str) -> syn::Result { } + syntax.shebang.as_ref().map_or(0, |s| s.len()); let mut visitor = TestRanges::default(); visitor.visit_file(&syntax); + Ok(visitor + .ranges + .into_iter() + .map(|range| range.start + offset..range.end + offset) + .collect()) +} + +fn prepare_source(source: &str) -> syn::Result { let mut bytes = source.as_bytes().to_vec(); - for range in visitor.ranges { + for range in test_ranges(source)? { // Retain newlines and byte offsets, including CRLF and same-line items. // Byte ranges include full UTF-8 characters; spaces keep valid UTF-8. - for byte in &mut bytes[range.start + offset..range.end + offset] { + for byte in &mut bytes[range] { if *byte != b'\n' && *byte != b'\r' { *byte = b' '; } diff --git a/utgen/tests/cli_config.rs b/utgen/tests/cli_config.rs index 571b1892cf..9d73bf89ac 100644 --- a/utgen/tests/cli_config.rs +++ b/utgen/tests/cli_config.rs @@ -49,6 +49,7 @@ fn help_and_non_model_commands_do_not_load_configuration() { vec!["gen", "--help"], vec!["fix", "--help"], vec!["analyze", "--help"], + vec!["coverage", "--help"], vec!["pre-process", "-p", "."], ] { let output = fixture