From 35730d0cce7df719db0afbccd193dff546328d45 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Nov 2025 07:16:00 +0000 Subject: [PATCH 1/2] Fix smart filtering to include valuable source files (Issue #12) Problem: When cloning repositories like vue3-openlayers, only 9 files were captured using smart filtering. The filtering was too aggressive, excluding source files that contain valuable documentation like TypeScript definitions, component files, and entry points. Solution (TDD approach): 1. Added failing test reproducing the issue for component library structure 2. Enhanced smart filtering logic to include: - TypeScript definition files (.d.ts) with type information - Entry point files (index.ts, index.js, index.mjs) defining API surface - Component files (.vue, .tsx, .jsx) with component documentation - Changed from substring to exact directory name matching - Removed blanket exclusion of src/, lib/, components/ directories 3. Updated existing tests to reflect new behavior Impact: Component libraries and TypeScript projects now get comprehensive coverage including source files with documentation value, while still excluding build artifacts, dependencies, and test files. Fixes #12 --- .../src/__tests__/git-repo-loader.test.ts | 24 +++-- .../src/__tests__/smart-filtering.test.ts | 96 ++++++++++++++++++- .../src/content/git-repo-loader.ts | 39 ++++++-- 3 files changed, 140 insertions(+), 19 deletions(-) diff --git a/packages/content-loader/src/__tests__/git-repo-loader.test.ts b/packages/content-loader/src/__tests__/git-repo-loader.test.ts index 3a9ea88..712f2df 100644 --- a/packages/content-loader/src/__tests__/git-repo-loader.test.ts +++ b/packages/content-loader/src/__tests__/git-repo-loader.test.ts @@ -181,7 +181,7 @@ describe("Git Repository Loading - User Workflows", () => { "docs/getting-started.md", "docs/api/authentication.md", "guides/tutorial.rst", - "src/index.js", // Should be excluded + "src/index.js", // Should be included (Issue #12 - entry point files) "package.json", // Should be excluded "CHANGELOG.md", // Should be excluded (project metadata) "LICENSE", // Should be excluded (project metadata) @@ -198,6 +198,7 @@ describe("Git Repository Loading - User Workflows", () => { "docs/getting-started.md", "docs/api/authentication.md", "guides/tutorial.rst", + "src/index.js", // Now included (Issue #12 fix) ]); }); @@ -246,15 +247,18 @@ describe("Git Repository Loading - User Workflows", () => { }, { file: "AUTHORS.txt", expected: false, reason: "authors metadata" }, - // Source code - should be excluded - { file: "src/index.js", expected: false, reason: "source code" }, - { file: "lib/utils.ts", expected: false, reason: "library code" }, + // Entry point and component files - now included (Issue #12) + { file: "src/index.js", expected: true, reason: "entry point file" }, { file: "components/Button.tsx", - expected: false, - reason: "component code", + expected: true, + reason: "component file", }, + // Regular source code - should still be excluded + { file: "lib/utils.ts", expected: false, reason: "library code" }, + { file: "src/helpers.ts", expected: false, reason: "utility code" }, + // Build artifacts - should be excluded { file: "node_modules/lodash.js", @@ -295,7 +299,7 @@ describe("Git Repository Loading - User Workflows", () => { const testFiles = [ "README.md", // Should be included "docs/guide.md", // Should be included - "src/index.js", // Should be excluded + "src/index.js", // Should be included (Issue #12 - entry point) "package.json", // Should be excluded "examples/demo.js", // Should be included ]; @@ -306,7 +310,7 @@ describe("Git Repository Loading - User Workflows", () => { expect(filtered).toContain("README.md"); expect(filtered).toContain("docs/guide.md"); expect(filtered).toContain("examples/demo.js"); - expect(filtered).not.toContain("src/index.js"); + expect(filtered).toContain("src/index.js"); // Now included (Issue #12) expect(filtered).not.toContain("package.json"); // Verify the architecture supports smart filtering as default @@ -337,7 +341,7 @@ describe("Git Repository Loading - User Workflows", () => { const mockFiles = [ "README.md", // Should be included "docs/getting-started.md", // Should be included - "src/index.js", // Should be excluded + "src/index.js", // Should be included (Issue #12 - entry point) "package.json", // Should be excluded "examples/demo.js", // Should be included "node_modules/lib.js", // Should be excluded @@ -350,9 +354,9 @@ describe("Git Repository Loading - User Workflows", () => { expect(filtered).toEqual([ "README.md", "docs/getting-started.md", + "src/index.js", // Now included (Issue #12 fix) "examples/demo.js", ]); - expect(filtered).not.toContain("src/index.js"); expect(filtered).not.toContain("package.json"); expect(filtered).not.toContain("node_modules/lib.js"); expect(filtered).not.toContain(".github/template.md"); diff --git a/packages/content-loader/src/__tests__/smart-filtering.test.ts b/packages/content-loader/src/__tests__/smart-filtering.test.ts index 6047932..4807c88 100644 --- a/packages/content-loader/src/__tests__/smart-filtering.test.ts +++ b/packages/content-loader/src/__tests__/smart-filtering.test.ts @@ -43,12 +43,21 @@ describe("Smart Content Filtering - REQ-18", () => { ); }); - test("should exclude config and source files", () => { + test("should exclude config files but include entry points", () => { + // Config files should be excluded expect((loader as any).isDocumentationFile("package.json")).toBe(false); expect((loader as any).isDocumentationFile(".postcssrc.json")).toBe(false); expect((loader as any).isDocumentationFile("config.ts")).toBe(false); expect((loader as any).isDocumentationFile("styles.css")).toBe(false); - expect((loader as any).isDocumentationFile("index.ts")).toBe(false); + + // Entry point files should be included (Issue #12) + expect((loader as any).isDocumentationFile("index.ts")).toBe(true); + expect((loader as any).isDocumentationFile("src/index.ts")).toBe(true); + expect((loader as any).isDocumentationFile("index.js")).toBe(true); + + // Regular source files should still be excluded + expect((loader as any).isDocumentationFile("src/utils.ts")).toBe(false); + expect((loader as any).isDocumentationFile("src/helpers.ts")).toBe(false); }); test("should include files in examples directory", () => { @@ -80,4 +89,87 @@ describe("Smart Content Filtering - REQ-18", () => { expect(filtered).toEqual(["README.md", "docs/api.md", "examples/demo.js"]); }); + + test("should include comprehensive documentation for component libraries - Issue #12", () => { + // Simulate a component library structure like vue3-openlayers + // These libraries have documentation spread across source files, not just in docs/ + const componentLibraryFiles = [ + // Basic docs (currently included) + "README.md", + "docs/getting-started.md", + "docs/api.md", + "examples/basic-map.html", + "examples/markers.js", + + // TypeScript definitions with valuable type information (currently excluded) + "src/index.d.ts", + "src/types.d.ts", + "src/components/Map.d.ts", + + // Source files with JSDoc documentation (currently excluded) + "src/index.ts", + "src/components/OlMap.vue", + "src/components/OlMarker.vue", + "src/utils/helpers.ts", + + // Component files that demonstrate API usage (currently excluded) + "src/components/layers/OlVectorLayer.vue", + "src/components/layers/OlTileLayer.vue", + + // Example files in src (currently excluded due to src/ exclusion) + "src/examples/basic.ts", + "src/examples/advanced.ts", + + // These should still be excluded + "node_modules/some-lib/index.js", + "dist/bundle.js", + "CHANGELOG.md", + "LICENSE", + ".github/workflows/ci.yml", + "package.json", + "tsconfig.json", + ]; + + const filtered = (loader as any).filterDocumentationFiles( + componentLibraryFiles, + ); + + // Expected: More than just the 5 basic doc files + // Should include TypeScript definitions, source files with documentation value, + // and example files to provide comprehensive knowledge about the library + const expectedIncludes = [ + "README.md", + "docs/getting-started.md", + "docs/api.md", + "examples/basic-map.html", + "examples/markers.js", + // These TypeScript definition files should be included for type information + "src/index.d.ts", + "src/types.d.ts", + "src/components/Map.d.ts", + // Main entry points and component files should be included + "src/index.ts", + "src/components/OlMap.vue", + "src/components/OlMarker.vue", + // Example files provide valuable usage documentation + "src/examples/basic.ts", + "src/examples/advanced.ts", + ]; + + // The filtered result should include all expected files + for (const expectedFile of expectedIncludes) { + expect(filtered).toContain(expectedFile); + } + + // Should still exclude build artifacts and config files + expect(filtered).not.toContain("node_modules/some-lib/index.js"); + expect(filtered).not.toContain("dist/bundle.js"); + expect(filtered).not.toContain("CHANGELOG.md"); + expect(filtered).not.toContain("package.json"); + expect(filtered).not.toContain("tsconfig.json"); + expect(filtered).not.toContain(".github/workflows/ci.yml"); + + // Verify we get comprehensive coverage (more than just 9 files) + expect(filtered.length).toBeGreaterThanOrEqual(expectedIncludes.length); + }); }); diff --git a/packages/content-loader/src/content/git-repo-loader.ts b/packages/content-loader/src/content/git-repo-loader.ts index d05f414..3a90d12 100644 --- a/packages/content-loader/src/content/git-repo-loader.ts +++ b/packages/content-loader/src/content/git-repo-loader.ts @@ -351,8 +351,13 @@ export class GitRepoLoader extends ContentLoader { return false; } - // Exclude source code, build, and development directories (REQ-18) - const excludedDirPatterns = [ + // Normalize directory path for consistent matching (use forward slashes) + const normalizedDir = directory.split(path.sep).join("/"); + const pathParts = normalizedDir.split("/"); + + // Exclude build, dependency, and development directories (REQ-18) + // Use exact directory name matching, not substring matching + const excludedDirs = [ "node_modules", "vendor", ".git", @@ -360,17 +365,17 @@ export class GitRepoLoader extends ContentLoader { "dist", "target", ".cache", - "src", - "lib", - "components", "__tests__", + "test", + "tests", ".github", ".vscode", ".idea", ]; - for (const pattern of excludedDirPatterns) { - if (directory.includes(pattern)) { + // Check if any path segment matches excluded directories + for (const excludedDir of excludedDirs) { + if (pathParts.includes(excludedDir)) { return false; } } @@ -403,6 +408,26 @@ export class GitRepoLoader extends ContentLoader { return !excludedInExamples.includes(extension); } + // Include TypeScript definition files - they contain valuable type information (Issue #12) + if (extension === ".d.ts" || filename.endsWith(".d.ts")) { + return true; + } + + // Include entry point files that define the API surface (Issue #12) + if ( + filename === "index.ts" || + filename === "index.js" || + filename === "index.mjs" + ) { + return true; + } + + // Include component files that often contain valuable documentation (Issue #12) + const componentExtensions = [".vue", ".tsx", ".jsx"]; + if (componentExtensions.includes(extension)) { + return true; + } + return false; } From ecd31d2541a60139247bcd88b6151eab472188be Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Nov 2025 08:07:23 +0000 Subject: [PATCH 2/2] Simplify smart filtering to use generic approach (Issue #12) Changed from tech-specific filtering to a simpler, more generic approach: What's included: - All markdown, text, asciidoc files (.md, .mdx, .rst, .txt, .adoc, .asciidoc) - README files anywhere - ALL files from examples/ and samples/ directories (except binaries) What's excluded: - Source code files (src/, lib/, components/) - Build artifacts (dist/, build/, node_modules/) - Test files (__tests__/, test/, tests/) - Dev config (.github/, .vscode/, .idea/) - Project metadata (LICENSE, CHANGELOG, CONTRIBUTING, etc.) This approach works for any repository type without being specific to TypeScript, Vue, React, or other technologies. Fixes #12 --- .../src/__tests__/git-repo-loader.test.ts | 25 +++-- .../src/__tests__/smart-filtering.test.ts | 92 +++++++++---------- .../src/content/git-repo-loader.ts | 27 +----- 3 files changed, 58 insertions(+), 86 deletions(-) diff --git a/packages/content-loader/src/__tests__/git-repo-loader.test.ts b/packages/content-loader/src/__tests__/git-repo-loader.test.ts index 712f2df..b8ac762 100644 --- a/packages/content-loader/src/__tests__/git-repo-loader.test.ts +++ b/packages/content-loader/src/__tests__/git-repo-loader.test.ts @@ -181,7 +181,7 @@ describe("Git Repository Loading - User Workflows", () => { "docs/getting-started.md", "docs/api/authentication.md", "guides/tutorial.rst", - "src/index.js", // Should be included (Issue #12 - entry point files) + "src/index.js", // Should be excluded (source file) "package.json", // Should be excluded "CHANGELOG.md", // Should be excluded (project metadata) "LICENSE", // Should be excluded (project metadata) @@ -198,7 +198,6 @@ describe("Git Repository Loading - User Workflows", () => { "docs/getting-started.md", "docs/api/authentication.md", "guides/tutorial.rst", - "src/index.js", // Now included (Issue #12 fix) ]); }); @@ -247,18 +246,16 @@ describe("Git Repository Loading - User Workflows", () => { }, { file: "AUTHORS.txt", expected: false, reason: "authors metadata" }, - // Entry point and component files - now included (Issue #12) - { file: "src/index.js", expected: true, reason: "entry point file" }, + // Source code - should be excluded + { file: "src/index.js", expected: false, reason: "source code" }, + { file: "lib/utils.ts", expected: false, reason: "library code" }, + { file: "src/helpers.ts", expected: false, reason: "utility code" }, { file: "components/Button.tsx", - expected: true, - reason: "component file", + expected: false, + reason: "component code", }, - // Regular source code - should still be excluded - { file: "lib/utils.ts", expected: false, reason: "library code" }, - { file: "src/helpers.ts", expected: false, reason: "utility code" }, - // Build artifacts - should be excluded { file: "node_modules/lodash.js", @@ -299,7 +296,7 @@ describe("Git Repository Loading - User Workflows", () => { const testFiles = [ "README.md", // Should be included "docs/guide.md", // Should be included - "src/index.js", // Should be included (Issue #12 - entry point) + "src/index.js", // Should be excluded (source file) "package.json", // Should be excluded "examples/demo.js", // Should be included ]; @@ -310,7 +307,7 @@ describe("Git Repository Loading - User Workflows", () => { expect(filtered).toContain("README.md"); expect(filtered).toContain("docs/guide.md"); expect(filtered).toContain("examples/demo.js"); - expect(filtered).toContain("src/index.js"); // Now included (Issue #12) + expect(filtered).not.toContain("src/index.js"); expect(filtered).not.toContain("package.json"); // Verify the architecture supports smart filtering as default @@ -341,7 +338,7 @@ describe("Git Repository Loading - User Workflows", () => { const mockFiles = [ "README.md", // Should be included "docs/getting-started.md", // Should be included - "src/index.js", // Should be included (Issue #12 - entry point) + "src/index.js", // Should be excluded (source file) "package.json", // Should be excluded "examples/demo.js", // Should be included "node_modules/lib.js", // Should be excluded @@ -354,9 +351,9 @@ describe("Git Repository Loading - User Workflows", () => { expect(filtered).toEqual([ "README.md", "docs/getting-started.md", - "src/index.js", // Now included (Issue #12 fix) "examples/demo.js", ]); + expect(filtered).not.toContain("src/index.js"); expect(filtered).not.toContain("package.json"); expect(filtered).not.toContain("node_modules/lib.js"); expect(filtered).not.toContain(".github/template.md"); diff --git a/packages/content-loader/src/__tests__/smart-filtering.test.ts b/packages/content-loader/src/__tests__/smart-filtering.test.ts index 4807c88..5f1ba0c 100644 --- a/packages/content-loader/src/__tests__/smart-filtering.test.ts +++ b/packages/content-loader/src/__tests__/smart-filtering.test.ts @@ -43,19 +43,16 @@ describe("Smart Content Filtering - REQ-18", () => { ); }); - test("should exclude config files but include entry points", () => { + test("should exclude config and source files", () => { // Config files should be excluded expect((loader as any).isDocumentationFile("package.json")).toBe(false); expect((loader as any).isDocumentationFile(".postcssrc.json")).toBe(false); expect((loader as any).isDocumentationFile("config.ts")).toBe(false); expect((loader as any).isDocumentationFile("styles.css")).toBe(false); - // Entry point files should be included (Issue #12) - expect((loader as any).isDocumentationFile("index.ts")).toBe(true); - expect((loader as any).isDocumentationFile("src/index.ts")).toBe(true); - expect((loader as any).isDocumentationFile("index.js")).toBe(true); - - // Regular source files should still be excluded + // Source files should be excluded + expect((loader as any).isDocumentationFile("index.ts")).toBe(false); + expect((loader as any).isDocumentationFile("src/index.ts")).toBe(false); expect((loader as any).isDocumentationFile("src/utils.ts")).toBe(false); expect((loader as any).isDocumentationFile("src/helpers.ts")).toBe(false); }); @@ -90,70 +87,62 @@ describe("Smart Content Filtering - REQ-18", () => { expect(filtered).toEqual(["README.md", "docs/api.md", "examples/demo.js"]); }); - test("should include comprehensive documentation for component libraries - Issue #12", () => { - // Simulate a component library structure like vue3-openlayers - // These libraries have documentation spread across source files, not just in docs/ - const componentLibraryFiles = [ - // Basic docs (currently included) + test("should include comprehensive documentation using generic approach - Issue #12", () => { + // Generic approach: Include markdown/text/asciidoc files + ALL files from examples/samples + const repositoryFiles = [ + // Documentation files (markdown, text, asciidoc) - always included "README.md", "docs/getting-started.md", "docs/api.md", + "guides/tutorial.rst", + "notes.txt", + "architecture.adoc", + + // Files in examples/samples directories - all included (Issue #12) "examples/basic-map.html", "examples/markers.js", + "examples/demo.py", + "examples/config.json", + "samples/advanced-usage.ts", + "samples/quickstart.java", - // TypeScript definitions with valuable type information (currently excluded) - "src/index.d.ts", - "src/types.d.ts", - "src/components/Map.d.ts", - - // Source files with JSDoc documentation (currently excluded) + // Source files - should be excluded "src/index.ts", - "src/components/OlMap.vue", - "src/components/OlMarker.vue", "src/utils/helpers.ts", + "lib/core.js", - // Component files that demonstrate API usage (currently excluded) - "src/components/layers/OlVectorLayer.vue", - "src/components/layers/OlTileLayer.vue", - - // Example files in src (currently excluded due to src/ exclusion) - "src/examples/basic.ts", - "src/examples/advanced.ts", - - // These should still be excluded + // Build artifacts and dependencies - should be excluded "node_modules/some-lib/index.js", "dist/bundle.js", + "build/output.js", + + // Project metadata - should be excluded "CHANGELOG.md", "LICENSE", - ".github/workflows/ci.yml", + "CONTRIBUTING.md", + + // Config files - should be excluded "package.json", "tsconfig.json", + ".github/workflows/ci.yml", ]; - const filtered = (loader as any).filterDocumentationFiles( - componentLibraryFiles, - ); + const filtered = (loader as any).filterDocumentationFiles(repositoryFiles); - // Expected: More than just the 5 basic doc files - // Should include TypeScript definitions, source files with documentation value, - // and example files to provide comprehensive knowledge about the library + // Expected: Documentation files + all files from examples/samples const expectedIncludes = [ "README.md", "docs/getting-started.md", "docs/api.md", + "guides/tutorial.rst", + "notes.txt", + "architecture.adoc", "examples/basic-map.html", "examples/markers.js", - // These TypeScript definition files should be included for type information - "src/index.d.ts", - "src/types.d.ts", - "src/components/Map.d.ts", - // Main entry points and component files should be included - "src/index.ts", - "src/components/OlMap.vue", - "src/components/OlMarker.vue", - // Example files provide valuable usage documentation - "src/examples/basic.ts", - "src/examples/advanced.ts", + "examples/demo.py", + "examples/config.json", + "samples/advanced-usage.ts", + "samples/quickstart.java", ]; // The filtered result should include all expected files @@ -161,7 +150,12 @@ describe("Smart Content Filtering - REQ-18", () => { expect(filtered).toContain(expectedFile); } - // Should still exclude build artifacts and config files + // Should exclude source files + expect(filtered).not.toContain("src/index.ts"); + expect(filtered).not.toContain("src/utils/helpers.ts"); + expect(filtered).not.toContain("lib/core.js"); + + // Should exclude build artifacts and config files expect(filtered).not.toContain("node_modules/some-lib/index.js"); expect(filtered).not.toContain("dist/bundle.js"); expect(filtered).not.toContain("CHANGELOG.md"); @@ -169,7 +163,7 @@ describe("Smart Content Filtering - REQ-18", () => { expect(filtered).not.toContain("tsconfig.json"); expect(filtered).not.toContain(".github/workflows/ci.yml"); - // Verify we get comprehensive coverage (more than just 9 files) + // Verify we get comprehensive coverage expect(filtered.length).toBeGreaterThanOrEqual(expectedIncludes.length); }); }); diff --git a/packages/content-loader/src/content/git-repo-loader.ts b/packages/content-loader/src/content/git-repo-loader.ts index 3a90d12..8b27136 100644 --- a/packages/content-loader/src/content/git-repo-loader.ts +++ b/packages/content-loader/src/content/git-repo-loader.ts @@ -391,10 +391,11 @@ export class GitRepoLoader extends ContentLoader { return true; } - // Special case: examples directory - include other file types as they're often documentation (REQ-18) - const isInExamples = /\b(examples?)\b/i.test(directory); + // Special case: examples/samples directory - include ALL file types (Issue #12) + // These directories contain code that demonstrates usage patterns + const isInExamples = /\b(examples?|samples?)\b/i.test(directory); if (isInExamples) { - // In examples, exclude only binary files + // In examples/samples, exclude only binary files const excludedInExamples = [ ".exe", ".bin", @@ -408,26 +409,6 @@ export class GitRepoLoader extends ContentLoader { return !excludedInExamples.includes(extension); } - // Include TypeScript definition files - they contain valuable type information (Issue #12) - if (extension === ".d.ts" || filename.endsWith(".d.ts")) { - return true; - } - - // Include entry point files that define the API surface (Issue #12) - if ( - filename === "index.ts" || - filename === "index.js" || - filename === "index.mjs" - ) { - return true; - } - - // Include component files that often contain valuable documentation (Issue #12) - const componentExtensions = [".vue", ".tsx", ".jsx"]; - if (componentExtensions.includes(extension)) { - return true; - } - return false; }