diff --git a/packages/content-loader/src/__tests__/git-repo-loader.test.ts b/packages/content-loader/src/__tests__/git-repo-loader.test.ts index 3a9ea88..b8ac762 100644 --- a/packages/content-loader/src/__tests__/git-repo-loader.test.ts +++ b/packages/content-loader/src/__tests__/git-repo-loader.test.ts @@ -181,7 +181,7 @@ describe("Git Repository Loading - User Workflows", () => { "docs/getting-started.md", "docs/api/authentication.md", "guides/tutorial.rst", - "src/index.js", // Should be excluded + "src/index.js", // Should be excluded (source file) "package.json", // Should be excluded "CHANGELOG.md", // Should be excluded (project metadata) "LICENSE", // Should be excluded (project metadata) @@ -249,6 +249,7 @@ describe("Git Repository Loading - User Workflows", () => { // Source code - should be excluded { file: "src/index.js", expected: false, reason: "source code" }, { file: "lib/utils.ts", expected: false, reason: "library code" }, + { file: "src/helpers.ts", expected: false, reason: "utility code" }, { file: "components/Button.tsx", expected: false, @@ -295,7 +296,7 @@ describe("Git Repository Loading - User Workflows", () => { const testFiles = [ "README.md", // Should be included "docs/guide.md", // Should be included - "src/index.js", // Should be excluded + "src/index.js", // Should be excluded (source file) "package.json", // Should be excluded "examples/demo.js", // Should be included ]; @@ -337,7 +338,7 @@ describe("Git Repository Loading - User Workflows", () => { const mockFiles = [ "README.md", // Should be included "docs/getting-started.md", // Should be included - "src/index.js", // Should be excluded + "src/index.js", // Should be excluded (source file) "package.json", // Should be excluded "examples/demo.js", // Should be included "node_modules/lib.js", // Should be excluded diff --git a/packages/content-loader/src/__tests__/smart-filtering.test.ts b/packages/content-loader/src/__tests__/smart-filtering.test.ts index 6047932..5f1ba0c 100644 --- a/packages/content-loader/src/__tests__/smart-filtering.test.ts +++ b/packages/content-loader/src/__tests__/smart-filtering.test.ts @@ -44,11 +44,17 @@ describe("Smart Content Filtering - REQ-18", () => { }); test("should exclude config and source files", () => { + // Config files should be excluded expect((loader as any).isDocumentationFile("package.json")).toBe(false); expect((loader as any).isDocumentationFile(".postcssrc.json")).toBe(false); expect((loader as any).isDocumentationFile("config.ts")).toBe(false); expect((loader as any).isDocumentationFile("styles.css")).toBe(false); + + // Source files should be excluded expect((loader as any).isDocumentationFile("index.ts")).toBe(false); + expect((loader as any).isDocumentationFile("src/index.ts")).toBe(false); + expect((loader as any).isDocumentationFile("src/utils.ts")).toBe(false); + expect((loader as any).isDocumentationFile("src/helpers.ts")).toBe(false); }); test("should include files in examples directory", () => { @@ -80,4 +86,84 @@ describe("Smart Content Filtering - REQ-18", () => { expect(filtered).toEqual(["README.md", "docs/api.md", "examples/demo.js"]); }); + + test("should include comprehensive documentation using generic approach - Issue #12", () => { + // Generic approach: Include markdown/text/asciidoc files + ALL files from examples/samples + const repositoryFiles = [ + // Documentation files (markdown, text, asciidoc) - always included + "README.md", + "docs/getting-started.md", + "docs/api.md", + "guides/tutorial.rst", + "notes.txt", + "architecture.adoc", + + // Files in examples/samples directories - all included (Issue #12) + "examples/basic-map.html", + "examples/markers.js", + "examples/demo.py", + "examples/config.json", + "samples/advanced-usage.ts", + "samples/quickstart.java", + + // Source files - should be excluded + "src/index.ts", + "src/utils/helpers.ts", + "lib/core.js", + + // Build artifacts and dependencies - should be excluded + "node_modules/some-lib/index.js", + "dist/bundle.js", + "build/output.js", + + // Project metadata - should be excluded + "CHANGELOG.md", + "LICENSE", + "CONTRIBUTING.md", + + // Config files - should be excluded + "package.json", + "tsconfig.json", + ".github/workflows/ci.yml", + ]; + + const filtered = (loader as any).filterDocumentationFiles(repositoryFiles); + + // Expected: Documentation files + all files from examples/samples + const expectedIncludes = [ + "README.md", + "docs/getting-started.md", + "docs/api.md", + "guides/tutorial.rst", + "notes.txt", + "architecture.adoc", + "examples/basic-map.html", + "examples/markers.js", + "examples/demo.py", + "examples/config.json", + "samples/advanced-usage.ts", + "samples/quickstart.java", + ]; + + // The filtered result should include all expected files + for (const expectedFile of expectedIncludes) { + expect(filtered).toContain(expectedFile); + } + + // Should exclude source files + expect(filtered).not.toContain("src/index.ts"); + expect(filtered).not.toContain("src/utils/helpers.ts"); + expect(filtered).not.toContain("lib/core.js"); + + // Should exclude build artifacts and config files + expect(filtered).not.toContain("node_modules/some-lib/index.js"); + expect(filtered).not.toContain("dist/bundle.js"); + expect(filtered).not.toContain("CHANGELOG.md"); + expect(filtered).not.toContain("package.json"); + expect(filtered).not.toContain("tsconfig.json"); + expect(filtered).not.toContain(".github/workflows/ci.yml"); + + // Verify we get comprehensive coverage + expect(filtered.length).toBeGreaterThanOrEqual(expectedIncludes.length); + }); }); diff --git a/packages/content-loader/src/content/git-repo-loader.ts b/packages/content-loader/src/content/git-repo-loader.ts index d05f414..8b27136 100644 --- a/packages/content-loader/src/content/git-repo-loader.ts +++ b/packages/content-loader/src/content/git-repo-loader.ts @@ -351,8 +351,13 @@ export class GitRepoLoader extends ContentLoader { return false; } - // Exclude source code, build, and development directories (REQ-18) - const excludedDirPatterns = [ + // Normalize directory path for consistent matching (use forward slashes) + const normalizedDir = directory.split(path.sep).join("/"); + const pathParts = normalizedDir.split("/"); + + // Exclude build, dependency, and development directories (REQ-18) + // Use exact directory name matching, not substring matching + const excludedDirs = [ "node_modules", "vendor", ".git", @@ -360,17 +365,17 @@ export class GitRepoLoader extends ContentLoader { "dist", "target", ".cache", - "src", - "lib", - "components", "__tests__", + "test", + "tests", ".github", ".vscode", ".idea", ]; - for (const pattern of excludedDirPatterns) { - if (directory.includes(pattern)) { + // Check if any path segment matches excluded directories + for (const excludedDir of excludedDirs) { + if (pathParts.includes(excludedDir)) { return false; } } @@ -386,10 +391,11 @@ export class GitRepoLoader extends ContentLoader { return true; } - // Special case: examples directory - include other file types as they're often documentation (REQ-18) - const isInExamples = /\b(examples?)\b/i.test(directory); + // Special case: examples/samples directory - include ALL file types (Issue #12) + // These directories contain code that demonstrates usage patterns + const isInExamples = /\b(examples?|samples?)\b/i.test(directory); if (isInExamples) { - // In examples, exclude only binary files + // In examples/samples, exclude only binary files const excludedInExamples = [ ".exe", ".bin",