diff --git a/src/timmytest/analysis.py b/src/timmytest/analysis.py index ef5620f..65d0dea 100644 --- a/src/timmytest/analysis.py +++ b/src/timmytest/analysis.py @@ -126,7 +126,7 @@ def analyze_project( ) # 4. Execute tests if requested - if execute_tests and test_modules: + if execute_tests and test_cmd: # `timeout_seconds` is already resolved by the caller when it can be; the # config value is the fallback for callers that pass 0/None. effective_timeout = timeout_seconds or cfg.timeout_seconds diff --git a/src/timmytest/detector/gap_analyzer.py b/src/timmytest/detector/gap_analyzer.py index c02c55b..0134107 100644 --- a/src/timmytest/detector/gap_analyzer.py +++ b/src/timmytest/detector/gap_analyzer.py @@ -81,7 +81,8 @@ def _prepare_tests(test_modules: list[TestModule]) -> list[_PreparedTest]: stem=stem, clean_stem=clean_stem, parent_names=frozenset(p.lower() for p in test_path.parts[:-1]), - usable=bool(test.test_functions) or named_like_test, + usable=bool(test.test_functions or test.imported_modules) + or (named_like_test and test.line_count > 0), ) ) return prepared @@ -91,6 +92,7 @@ def _find_matching_test( source: SourceModule, test_modules: list[TestModule], prepared: list[_PreparedTest] | None = None, + require_import: bool = False, ) -> TestModule | None: """ Find if a source module has a corresponding test file using exact naming, @@ -107,6 +109,12 @@ def _find_matching_test( for entry in prepared if prepared is not None else _prepare_tests(test_modules): if not entry.usable: continue + if require_import: + if entry.module.rel_path == source.rel_path and entry.module.test_functions: + return entry.module + if any(_import_targets_module(imp, source) for imp in entry.module.imported_modules): + return entry.module + continue test_stem = entry.stem test_clean_stem = entry.clean_stem @@ -191,12 +199,19 @@ def analyze_test_gaps( total_sources = len(source_modules) if total_sources == 0: - return [], 100.0 + return [], 0.0 prepared_tests = _prepare_tests(test_modules) + stem_counts: dict[str, int] = {} + for src in source_modules: + stem = Path(src.rel_path).stem.lower() + stem_counts[stem] = stem_counts.get(stem, 0) + 1 for src in source_modules: - matching_test = _find_matching_test(src, test_modules, prepared_tests) + matching_test = _find_matching_test( + src, test_modules, prepared_tests, + require_import=stem_counts[Path(src.rel_path).stem.lower()] > 1, + ) if matching_test: covered_count += 1 else: diff --git a/src/timmytest/detector/scanner.py b/src/timmytest/detector/scanner.py index fa23dfe..dc757b2 100644 --- a/src/timmytest/detector/scanner.py +++ b/src/timmytest/detector/scanner.py @@ -1,6 +1,7 @@ """AST and regex-based source and test file scanner with multi-language AST extraction.""" import ast +import fnmatch import re from collections.abc import Iterator from pathlib import Path @@ -13,6 +14,7 @@ TestFramework, TestModule, ) +from timmytest.registry.loader import load_registry #: Re-exported for callers that have always imported it from here. IGNORED_DIRS = walk.IGNORED_DIRS @@ -98,7 +100,9 @@ def iter_project_files( yield current / filename -def _is_test_file(path: Path, root: Path | None = None) -> bool: +def _is_test_file( + path: Path, root: Path | None = None, test_patterns: tuple[str, ...] = () +) -> bool: """Determine if a file is a test file. ``root`` bounds the directory-name check to the project. Without it, every @@ -121,6 +125,16 @@ def _is_test_file(path: Path, root: Path | None = None) -> bool: if any(part in TEST_DIR_NAMES for part in parent_names): return True + # The registry's file patterns are the per-framework source of truth. + # Entries describing inline syntax (for example Rust's "#[test]") are not + # filenames and must not be fed to a glob matcher. + if any( + not any(char in pattern for char in " #{") + and fnmatch.fnmatchcase(name, pattern.lower()) + for pattern in test_patterns + ): + return True + return bool( name.startswith("test_") or name.startswith("test-") @@ -140,23 +154,23 @@ def _is_test_file(path: Path, root: Path | None = None) -> bool: or name.endswith("test.cs") or name.endswith("test.java") # Kotlin / Scala / Swift / Dart / Elixir / Haskell - or name.endswith("Test.kt") - or name.endswith("Tests.kt") - or name.endswith("Spec.scala") - or name.endswith("Suite.scala") - or name.endswith("Test.scala") - or name.endswith("Tests.swift") - or name.endswith("Test.swift") + or name.endswith("test.kt") + or name.endswith("tests.kt") + or name.endswith("spec.scala") + or name.endswith("suite.scala") + or name.endswith("test.scala") + or name.endswith("tests.swift") + or name.endswith("test.swift") or name.endswith("_test.dart") or name.endswith("_test.exs") - or name.endswith("Spec.hs") - or name.endswith("Test.hs") + or name.endswith("spec.hs") + or name.endswith("test.hs") # C / C++ or name.endswith("_test.c") or name.endswith("_test.cpp") or name.endswith("_test.cc") - or name.endswith("Test.cpp") - or name.endswith("Test.cc") + or name.endswith("test.cpp") + or name.endswith("test.cc") # Lua / Crystal / Clojure or name.endswith("_spec.lua") or name.endswith("_spec.cr") @@ -636,9 +650,17 @@ def scan_project_structure( ".v", } + registry = load_registry() + selected: dict = next((entry for entry in registry["ecosystems"] if entry.get("id") == ecosystem.value), {}) + valid_extensions.update(str(ext).lower() for ext in selected.get("extensions", [])) + selected_framework: dict = next( + (entry for entry in selected.get("frameworks", []) if entry.get("id") == framework.value), {} + ) + test_patterns = tuple(str(pattern) for pattern in selected_framework.get("test_patterns", [])) + for item in iter_project_files(root, effective_ignored_dirs, effective_ignored_files, valid_extensions): rel_path = item.relative_to(root).as_posix() - is_test = _is_test_file(item, root) + is_test = _is_test_file(item, root, test_patterns) if is_test: if item.suffix == ".py": @@ -662,6 +684,22 @@ def scan_project_structure( else: funcs, details, classes, imports, lines = _parse_generic_source(item) + # Rust unit tests commonly live beside the implementation in the + # same .rs file. Keep the file as source, and record its test blocks + # separately so a real inline test can be associated with it. + if item.suffix.lower() == ".rs": + inline_tests, _, _ = _parse_generic_test(item) + if inline_tests: + test_modules.append( + TestModule( + rel_path=rel_path, + abs_path=str(item), + framework=framework, + test_functions=inline_tests, + line_count=lines, + ) + ) + lower_name = item.stem.lower() is_entry = lower_name in {"main", "app", "cli", "index", "server", "runner"} is_util = "util" in lower_name or "helper" in lower_name or "tool" in lower_name diff --git a/src/timmytest/registry/ecosystems.yaml b/src/timmytest/registry/ecosystems.yaml index fcd38bf..1aa6da3 100644 --- a/src/timmytest/registry/ecosystems.yaml +++ b/src/timmytest/registry/ecosystems.yaml @@ -66,7 +66,7 @@ ecosystems: - id: node name: Node.js / JavaScript / TypeScript - extensions: [".js", ".ts", ".jsx", ".tsx", ".mjs", ".cjs"] + extensions: [".js", ".ts", ".jsx", ".tsx", ".mjs", ".cjs", ".mts", ".cts", ".vue", ".svelte", ".astro"] config_files: [package.json, deno.json, deno.jsonc, tsconfig.json, vitest.config.ts, vitest.config.js, jest.config.js, jest.config.ts, pnpm-lock.yaml, yarn.lock, bun.lockb, bun.lock] test_dirs: [tests, test, __tests__, specs, spec] @@ -91,13 +91,13 @@ ecosystems: dependency_in: [vitest] script_contains: [vitest] command: "{runner} vitest run" - test_patterns: ["*.test.ts", "*.test.js", "*.spec.ts", "*.spec.js", "*.test.tsx", "*.test.jsx"] + test_patterns: ["*.test.ts", "*.test.js", "*.spec.ts", "*.spec.js", "*.test.tsx", "*.test.jsx", "*.test.mjs", "*.spec.mjs", "*.test.cjs", "*.spec.cjs", "*.test.mts", "*.spec.mts", "*.test.cts", "*.spec.cts"] - id: jest name: Jest dependency_in: [jest] script_contains: [jest] command: "{runner} jest" - test_patterns: ["*.test.ts", "*.test.js", "*.spec.ts", "*.spec.js", "*.test.tsx", "*.test.jsx"] + test_patterns: ["*.test.ts", "*.test.js", "*.spec.ts", "*.spec.js", "*.test.tsx", "*.test.jsx", "*.test.mjs", "*.spec.mjs", "*.test.cjs", "*.spec.cjs", "*.test.mts", "*.spec.mts", "*.test.cts", "*.spec.cts"] - id: mocha name: Mocha dependency_in: [mocha] @@ -112,7 +112,7 @@ ecosystems: name: package.json test script default: true command: "{pm} test" - test_patterns: ["*.test.js", "*.spec.js", "*.test.ts", "*.spec.ts"] + test_patterns: ["*.test.js", "*.spec.js", "*.test.ts", "*.spec.ts", "*.test.mjs", "*.spec.mjs", "*.test.cjs", "*.spec.cjs", "*.test.mts", "*.spec.mts", "*.test.cts", "*.spec.cts"] - id: rust name: Rust diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 9293e17..1c8256e 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -104,6 +104,7 @@ def test_a_fixture_file_is_not_a_test_suite(): def test_a_test_named_file_without_parsed_functions_still_counts(): """Languages whose test bodies the scanner cannot parse must not regress.""" unparsed = _test_mod("tests/auth_test.sh", functions=[]) + unparsed.line_count = 5 assert _find_matching_test(_src("src/auth.py"), [unparsed]) is not None diff --git a/tests/test_discovery_accuracy.py b/tests/test_discovery_accuracy.py new file mode 100644 index 0000000..7256274 --- /dev/null +++ b/tests/test_discovery_accuracy.py @@ -0,0 +1,71 @@ +"""Failures that must not be reported as complete test coverage.""" + +from timmytest import analysis +from timmytest.detector.gap_analyzer import analyze_test_gaps +from timmytest.detector.models import Ecosystem, SourceModule, TestFramework, TestModule, TestRunResult +from timmytest.detector.scanner import scan_project_structure + + +def test_empty_project_has_no_readiness_evidence(tmp_path): + assert analyze_test_gaps([], [], Ecosystem.PYTHON, tmp_path) == ([], 0.0) + + +def test_empty_named_test_does_not_cover_source(tmp_path): + source = SourceModule(rel_path="src/payments.py", abs_path="", language="py") + test = TestModule(rel_path="tests/test_payments.py", abs_path="", framework=TestFramework.PYTEST) + gaps, score = analyze_test_gaps([source], [test], Ecosystem.PYTHON, tmp_path) + assert [gap.source_module for gap in gaps] == ["src/payments.py"] + assert score == 0.0 + + +def test_duplicate_stems_need_import_evidence(tmp_path): + sources = [ + SourceModule(rel_path="auth/utils.py", abs_path="", language="py"), + SourceModule(rel_path="billing/utils.py", abs_path="", language="py"), + ] + test = TestModule( + rel_path="tests/test_utils.py", abs_path="", framework=TestFramework.PYTEST, + test_functions=["test_format"], imported_modules=["auth.utils"], + ) + gaps, score = analyze_test_gaps(sources, [test], Ecosystem.PYTHON, tmp_path) + assert [gap.source_module for gap in gaps] == ["billing/utils.py"] + assert score < 100 + + +def test_registry_patterns_find_java_integration_test(tmp_path): + (tmp_path / "src").mkdir() + (tmp_path / "src" / "Main.java").write_text("class Main {}", encoding="utf-8") + (tmp_path / "src" / "MainIT.java").write_text("class MainIT {}", encoding="utf-8") + _, tests = scan_project_structure(tmp_path, Ecosystem.JAVA, TestFramework.MAVEN) + assert [test.rel_path for test in tests] == ["src/MainIT.java"] + + +def test_node_modern_extensions_are_scanned(tmp_path): + (tmp_path / "widget.vue").write_text("", encoding="utf-8") + (tmp_path / "widget.test.mjs").write_text('test("widget", () => {})', encoding="utf-8") + sources, tests = scan_project_structure(tmp_path, Ecosystem.NODE, TestFramework.VITEST) + assert [source.rel_path for source in sources] == ["widget.vue"] + assert [test.rel_path for test in tests] == ["widget.test.mjs"] + + +def test_command_runs_even_without_discovered_test_file(monkeypatch, tmp_path): + (tmp_path / "Cargo.toml").write_text('[package]\nname="sample"\nversion="0.1.0"\n', encoding="utf-8") + (tmp_path / "src").mkdir() + (tmp_path / "src" / "lib.rs").write_text("#[cfg(test)] mod tests { #[test] fn works() {} }", encoding="utf-8") + called = [] + monkeypatch.setattr(analysis, "run_project_tests", lambda **kwargs: called.append(kwargs) or TestRunResult(ecosystem=Ecosystem.RUST, framework=TestFramework.CARGO, command="cargo test", total=0, has_executed=True)) + monkeypatch.setattr(analysis, "enrich_test_failures", lambda result: result) + analysis.analyze_project(tmp_path, execute_tests=True) + assert len(called) == 1 + + +def test_rust_inline_test_is_both_source_and_test(tmp_path): + (tmp_path / "lib.rs").write_text( + "pub fn add(a: i32, b: i32) -> i32 { a + b }\n" + "#[cfg(test)] mod tests { #[test]\nfn adds() { assert_eq!(super::add(1, 2), 3); } }", + encoding="utf-8", + ) + sources, tests = scan_project_structure(tmp_path, Ecosystem.RUST, TestFramework.CARGO) + assert [source.rel_path for source in sources] == ["lib.rs"] + assert [test.rel_path for test in tests] == ["lib.rs"] + assert tests[0].test_functions == ["adds"]