Improve test discovery accuracy and run behavior

src/timmytest/analysis.py

@@ -126,7 +126,7 @@ def analyze_project(
     )
 
     # 4. Execute tests if requested
-    if execute_tests and test_modules:
+    if execute_tests and test_cmd:
         # `timeout_seconds` is already resolved by the caller when it can be; the
         # config value is the fallback for callers that pass 0/None.
         effective_timeout = timeout_seconds or cfg.timeout_seconds

src/timmytest/detector/gap_analyzer.py

@@ -81,7 +81,8 @@ def _prepare_tests(test_modules: list[TestModule]) -> list[_PreparedTest]:
                 stem=stem,
                 clean_stem=clean_stem,
                 parent_names=frozenset(p.lower() for p in test_path.parts[:-1]),
-                usable=bool(test.test_functions) or named_like_test,
+                usable=bool(test.test_functions or test.imported_modules)
+                or (named_like_test and test.line_count > 0),
             )
         )
     return prepared
@@ -91,6 +92,7 @@ def _find_matching_test(
     source: SourceModule,
     test_modules: list[TestModule],
     prepared: list[_PreparedTest] | None = None,
+    require_import: bool = False,
 ) -> TestModule | None:
     """
     Find if a source module has a corresponding test file using exact naming,
@@ -107,6 +109,12 @@ def _find_matching_test(
     for entry in prepared if prepared is not None else _prepare_tests(test_modules):
         if not entry.usable:
             continue
+        if require_import:
+            if entry.module.rel_path == source.rel_path and entry.module.test_functions:
+                return entry.module
+            if any(_import_targets_module(imp, source) for imp in entry.module.imported_modules):
+                return entry.module
+            continue
 
         test_stem = entry.stem
         test_clean_stem = entry.clean_stem
@@ -191,12 +199,19 @@ def analyze_test_gaps(
     total_sources = len(source_modules)
 
     if total_sources == 0:
-        return [], 100.0
+        return [], 0.0
 
     prepared_tests = _prepare_tests(test_modules)
+    stem_counts: dict[str, int] = {}
+    for src in source_modules:
+        stem = Path(src.rel_path).stem.lower()
+        stem_counts[stem] = stem_counts.get(stem, 0) + 1
 
     for src in source_modules:
-        matching_test = _find_matching_test(src, test_modules, prepared_tests)
+        matching_test = _find_matching_test(
+            src, test_modules, prepared_tests,
+            require_import=stem_counts[Path(src.rel_path).stem.lower()] > 1,
+        )
         if matching_test:
             covered_count += 1
         else:

src/timmytest/detector/scanner.py

@@ -1,6 +1,7 @@
 """AST and regex-based source and test file scanner with multi-language AST extraction."""
 
 import ast
+import fnmatch
 import re
 from collections.abc import Iterator
 from pathlib import Path
@@ -13,6 +14,7 @@
     TestFramework,
     TestModule,
 )
+from timmytest.registry.loader import load_registry
 
 #: Re-exported for callers that have always imported it from here.
 IGNORED_DIRS = walk.IGNORED_DIRS
@@ -98,7 +100,9 @@ def iter_project_files(
             yield current / filename
 
 
-def _is_test_file(path: Path, root: Path | None = None) -> bool:
+def _is_test_file(
+    path: Path, root: Path | None = None, test_patterns: tuple[str, ...] = ()
+) -> bool:
     """Determine if a file is a test file.
 
     ``root`` bounds the directory-name check to the project. Without it, every
@@ -121,6 +125,16 @@ def _is_test_file(path: Path, root: Path | None = None) -> bool:
     if any(part in TEST_DIR_NAMES for part in parent_names):
         return True
 
+    # The registry's file patterns are the per-framework source of truth.
+    # Entries describing inline syntax (for example Rust's "#[test]") are not
+    # filenames and must not be fed to a glob matcher.
+    if any(
+        not any(char in pattern for char in " #{")
+        and fnmatch.fnmatchcase(name, pattern.lower())
+        for pattern in test_patterns
+    ):
+        return True
+
     return bool(
         name.startswith("test_")
         or name.startswith("test-")
@@ -140,23 +154,23 @@ def _is_test_file(path: Path, root: Path | None = None) -> bool:
         or name.endswith("test.cs")
         or name.endswith("test.java")
         # Kotlin / Scala / Swift / Dart / Elixir / Haskell
-        or name.endswith("Test.kt")
-        or name.endswith("Tests.kt")
-        or name.endswith("Spec.scala")
-        or name.endswith("Suite.scala")
-        or name.endswith("Test.scala")
-        or name.endswith("Tests.swift")
-        or name.endswith("Test.swift")
+        or name.endswith("test.kt")
+        or name.endswith("tests.kt")
+        or name.endswith("spec.scala")
+        or name.endswith("suite.scala")
+        or name.endswith("test.scala")
+        or name.endswith("tests.swift")
+        or name.endswith("test.swift")
         or name.endswith("_test.dart")
         or name.endswith("_test.exs")
-        or name.endswith("Spec.hs")
-        or name.endswith("Test.hs")
+        or name.endswith("spec.hs")
+        or name.endswith("test.hs")
         # C / C++
         or name.endswith("_test.c")
         or name.endswith("_test.cpp")
         or name.endswith("_test.cc")
-        or name.endswith("Test.cpp")
-        or name.endswith("Test.cc")
+        or name.endswith("test.cpp")
+        or name.endswith("test.cc")
         # Lua / Crystal / Clojure
         or name.endswith("_spec.lua")
         or name.endswith("_spec.cr")
@@ -636,9 +650,17 @@ def scan_project_structure(
         ".v",
     }
 
+    registry = load_registry()
+    selected: dict = next((entry for entry in registry["ecosystems"] if entry.get("id") == ecosystem.value), {})
+    valid_extensions.update(str(ext).lower() for ext in selected.get("extensions", []))
+    selected_framework: dict = next(
+        (entry for entry in selected.get("frameworks", []) if entry.get("id") == framework.value), {}
+    )
+    test_patterns = tuple(str(pattern) for pattern in selected_framework.get("test_patterns", []))
+
     for item in iter_project_files(root, effective_ignored_dirs, effective_ignored_files, valid_extensions):
         rel_path = item.relative_to(root).as_posix()
-        is_test = _is_test_file(item, root)
+        is_test = _is_test_file(item, root, test_patterns)
 
         if is_test:
             if item.suffix == ".py":
@@ -662,6 +684,22 @@ def scan_project_structure(
             else:
                 funcs, details, classes, imports, lines = _parse_generic_source(item)
 
+            # Rust unit tests commonly live beside the implementation in the
+            # same .rs file. Keep the file as source, and record its test blocks
+            # separately so a real inline test can be associated with it.
+            if item.suffix.lower() == ".rs":
+                inline_tests, _, _ = _parse_generic_test(item)
+                if inline_tests:
+                    test_modules.append(
+                        TestModule(
+                            rel_path=rel_path,
+                            abs_path=str(item),
+                            framework=framework,
+                            test_functions=inline_tests,
+                            line_count=lines,
+                        )
+                    )
+
             lower_name = item.stem.lower()
             is_entry = lower_name in {"main", "app", "cli", "index", "server", "runner"}
             is_util = "util" in lower_name or "helper" in lower_name or "tool" in lower_name

src/timmytest/registry/ecosystems.yaml

@@ -66,7 +66,7 @@ ecosystems:
 
   - id: node
     name: Node.js / JavaScript / TypeScript
-    extensions: [".js", ".ts", ".jsx", ".tsx", ".mjs", ".cjs"]
+    extensions: [".js", ".ts", ".jsx", ".tsx", ".mjs", ".cjs", ".mts", ".cts", ".vue", ".svelte", ".astro"]
     config_files:
       [package.json, deno.json, deno.jsonc, tsconfig.json, vitest.config.ts, vitest.config.js, jest.config.js, jest.config.ts, pnpm-lock.yaml, yarn.lock, bun.lockb, bun.lock]
     test_dirs: [tests, test, __tests__, specs, spec]
@@ -91,13 +91,13 @@ ecosystems:
         dependency_in: [vitest]
         script_contains: [vitest]
         command: "{runner} vitest run"
-        test_patterns: ["*.test.ts", "*.test.js", "*.spec.ts", "*.spec.js", "*.test.tsx", "*.test.jsx"]
+        test_patterns: ["*.test.ts", "*.test.js", "*.spec.ts", "*.spec.js", "*.test.tsx", "*.test.jsx", "*.test.mjs", "*.spec.mjs", "*.test.cjs", "*.spec.cjs", "*.test.mts", "*.spec.mts", "*.test.cts", "*.spec.cts"]
       - id: jest
         name: Jest
         dependency_in: [jest]
         script_contains: [jest]
         command: "{runner} jest"
-        test_patterns: ["*.test.ts", "*.test.js", "*.spec.ts", "*.spec.js", "*.test.tsx", "*.test.jsx"]
+        test_patterns: ["*.test.ts", "*.test.js", "*.spec.ts", "*.spec.js", "*.test.tsx", "*.test.jsx", "*.test.mjs", "*.spec.mjs", "*.test.cjs", "*.spec.cjs", "*.test.mts", "*.spec.mts", "*.test.cts", "*.spec.cts"]
       - id: mocha
         name: Mocha
         dependency_in: [mocha]
@@ -112,7 +112,7 @@ ecosystems:
         name: package.json test script
         default: true
         command: "{pm} test"
-        test_patterns: ["*.test.js", "*.spec.js", "*.test.ts", "*.spec.ts"]
+        test_patterns: ["*.test.js", "*.spec.js", "*.test.ts", "*.spec.ts", "*.test.mjs", "*.spec.mjs", "*.test.cjs", "*.spec.cjs", "*.test.mts", "*.spec.mts", "*.test.cts", "*.spec.cts"]
 
   - id: rust
     name: Rust

tests/test_audit_regressions.py

@@ -104,6 +104,7 @@ def test_a_fixture_file_is_not_a_test_suite():
 def test_a_test_named_file_without_parsed_functions_still_counts():
     """Languages whose test bodies the scanner cannot parse must not regress."""
     unparsed = _test_mod("tests/auth_test.sh", functions=[])
+    unparsed.line_count = 5
     assert _find_matching_test(_src("src/auth.py"), [unparsed]) is not None
 
 

tests/test_discovery_accuracy.py

@@ -0,0 +1,71 @@
+"""Failures that must not be reported as complete test coverage."""
+
+from timmytest import analysis
+from timmytest.detector.gap_analyzer import analyze_test_gaps
+from timmytest.detector.models import Ecosystem, SourceModule, TestFramework, TestModule, TestRunResult
+from timmytest.detector.scanner import scan_project_structure
+
+
+def test_empty_project_has_no_readiness_evidence(tmp_path):
+    assert analyze_test_gaps([], [], Ecosystem.PYTHON, tmp_path) == ([], 0.0)
+
+
+def test_empty_named_test_does_not_cover_source(tmp_path):
+    source = SourceModule(rel_path="src/payments.py", abs_path="", language="py")
+    test = TestModule(rel_path="tests/test_payments.py", abs_path="", framework=TestFramework.PYTEST)
+    gaps, score = analyze_test_gaps([source], [test], Ecosystem.PYTHON, tmp_path)
+    assert [gap.source_module for gap in gaps] == ["src/payments.py"]
+    assert score == 0.0
+
+
+def test_duplicate_stems_need_import_evidence(tmp_path):
+    sources = [
+        SourceModule(rel_path="auth/utils.py", abs_path="", language="py"),
+        SourceModule(rel_path="billing/utils.py", abs_path="", language="py"),
+    ]
+    test = TestModule(
+        rel_path="tests/test_utils.py", abs_path="", framework=TestFramework.PYTEST,
+        test_functions=["test_format"], imported_modules=["auth.utils"],
+    )
+    gaps, score = analyze_test_gaps(sources, [test], Ecosystem.PYTHON, tmp_path)
+    assert [gap.source_module for gap in gaps] == ["billing/utils.py"]
+    assert score < 100
+
+
+def test_registry_patterns_find_java_integration_test(tmp_path):
+    (tmp_path / "src").mkdir()
+    (tmp_path / "src" / "Main.java").write_text("class Main {}", encoding="utf-8")
+    (tmp_path / "src" / "MainIT.java").write_text("class MainIT {}", encoding="utf-8")
+    _, tests = scan_project_structure(tmp_path, Ecosystem.JAVA, TestFramework.MAVEN)
+    assert [test.rel_path for test in tests] == ["src/MainIT.java"]
+
+
+def test_node_modern_extensions_are_scanned(tmp_path):
+    (tmp_path / "widget.vue").write_text("<script>export default {}</script>", encoding="utf-8")
+    (tmp_path / "widget.test.mjs").write_text('test("widget", () => {})', encoding="utf-8")
+    sources, tests = scan_project_structure(tmp_path, Ecosystem.NODE, TestFramework.VITEST)
+    assert [source.rel_path for source in sources] == ["widget.vue"]
+    assert [test.rel_path for test in tests] == ["widget.test.mjs"]
+
+
+def test_command_runs_even_without_discovered_test_file(monkeypatch, tmp_path):
+    (tmp_path / "Cargo.toml").write_text('[package]\nname="sample"\nversion="0.1.0"\n', encoding="utf-8")
+    (tmp_path / "src").mkdir()
+    (tmp_path / "src" / "lib.rs").write_text("#[cfg(test)] mod tests { #[test] fn works() {} }", encoding="utf-8")
+    called = []
+    monkeypatch.setattr(analysis, "run_project_tests", lambda **kwargs: called.append(kwargs) or TestRunResult(ecosystem=Ecosystem.RUST, framework=TestFramework.CARGO, command="cargo test", total=0, has_executed=True))
+    monkeypatch.setattr(analysis, "enrich_test_failures", lambda result: result)
+    analysis.analyze_project(tmp_path, execute_tests=True)
+    assert len(called) == 1
+
+
+def test_rust_inline_test_is_both_source_and_test(tmp_path):
+    (tmp_path / "lib.rs").write_text(
+        "pub fn add(a: i32, b: i32) -> i32 { a + b }\n"
+        "#[cfg(test)] mod tests { #[test]\nfn adds() { assert_eq!(super::add(1, 2), 3); } }",
+        encoding="utf-8",
+    )
+    sources, tests = scan_project_structure(tmp_path, Ecosystem.RUST, TestFramework.CARGO)
+    assert [source.rel_path for source in sources] == ["lib.rs"]
+    assert [test.rel_path for test in tests] == ["lib.rs"]
+    assert tests[0].test_functions == ["adds"]