Fix path handling for git repos and improve init command

packages/cli/src/__tests__/init-command.test.ts

@@ -0,0 +1,234 @@
+/**
+ * Init Command - Behavior tests for force re-init and path discovery
+ */
+
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+import { promises as fs } from "node:fs";
+import * as path from "node:path";
+import { tmpdir } from "node:os";
+import { initCommand } from "../commands/init.js";
+
+describe("Init Command - Force Re-initialization", () => {
+  let testDir: string;
+  let configPath: string;
+
+  beforeEach(async () => {
+    // Create temporary test directory
+    testDir = path.join(tmpdir(), `agentic-init-test-${Date.now()}`);
+    await fs.mkdir(testDir, { recursive: true });
+
+    // Create .knowledge directory and config
+    const knowledgeDir = path.join(testDir, ".knowledge");
+    await fs.mkdir(knowledgeDir, { recursive: true });
+    configPath = path.join(knowledgeDir, "config.yaml");
+
+    // Create basic config
+    const config = `version: "1.0"
+docsets:
+  - id: test-docset
+    name: Test Docset
+    description: Test
+    sources:
+      - url: https://github.com/test/repo.git
+        type: git_repo
+        branch: main
+        paths:
+          - docs/
+`;
+    await fs.writeFile(configPath, config);
+  });
+
+  afterEach(async () => {
+    // Cleanup
+    await fs.rm(testDir, { recursive: true, force: true });
+  });
+
+  describe("--force flag behavior", () => {
+    it("should clear existing directory before re-initialization", async () => {
+      // Create docset directory with old files
+      const docsetPath = path.join(
+        testDir,
+        ".knowledge",
+        "docsets",
+        "test-docset",
+      );
+      await fs.mkdir(docsetPath, { recursive: true });
+
+      // Add old files that should be removed
+      await fs.writeFile(path.join(docsetPath, "old-file.md"), "old content");
+      await fs.writeFile(
+        path.join(docsetPath, "old-file2.md"),
+        "old content 2",
+      );
+      await fs.mkdir(path.join(docsetPath, "old-directory"), {
+        recursive: true,
+      });
+      await fs.writeFile(
+        path.join(docsetPath, "old-directory", "nested.md"),
+        "nested old content",
+      );
+
+      // Verify old files exist
+      const filesBefore = await fs.readdir(docsetPath);
+      expect(filesBefore).toContain("old-file.md");
+      expect(filesBefore).toContain("old-file2.md");
+      expect(filesBefore).toContain("old-directory");
+
+      // This test will fail initially - we're doing TDD
+      // The implementation should clear the directory when --force is used
+      // For now, we're just documenting the expected behavior
+    });
+
+    it("should NOT update config paths when using only --force", async () => {
+      // The --force flag should only clear and re-initialize
+      // It should NOT modify the config file
+
+      const configBefore = await fs.readFile(configPath, "utf-8");
+
+      // After running with --force, config should be unchanged
+      // (This test documents that --force and path discovery are separate)
+    });
+  });
+});
+
+describe("Init Command - Path Discovery", () => {
+  let testDir: string;
+  let configPath: string;
+
+  beforeEach(async () => {
+    testDir = path.join(tmpdir(), `agentic-path-test-${Date.now()}`);
+    await fs.mkdir(testDir, { recursive: true });
+
+    const knowledgeDir = path.join(testDir, ".knowledge");
+    await fs.mkdir(knowledgeDir, { recursive: true });
+    configPath = path.join(knowledgeDir, "config.yaml");
+
+    // Create config without paths specified
+    const config = `version: "1.0"
+docsets:
+  - id: test-docset
+    name: Test Docset
+    description: Test
+    sources:
+      - url: https://github.com/test/repo.git
+        type: git_repo
+        branch: main
+`;
+    await fs.writeFile(configPath, config);
+  });
+
+  afterEach(async () => {
+    await fs.rm(testDir, { recursive: true, force: true });
+  });
+
+  describe("--discover-paths flag", () => {
+    it("should update config with discovered directory patterns", async () => {
+      // When using --discover-paths, the system should:
+      // 1. Extract files using smart filtering
+      // 2. Analyze which directories contain the files
+      // 3. Update config with directory patterns (not individual files)
+      // This test will initially fail - we're doing TDD
+    });
+
+    it("should store directory patterns not individual file paths", async () => {
+      // Given extracted files:
+      // - docs/guide/intro.md
+      // - docs/guide/advanced.md
+      // - docs/api/reference.md
+      // - README.md
+      // - examples/basic.js
+      // - examples/advanced.js
+      //
+      // Should update config with:
+      // paths:
+      //   - README.md
+      //   - docs/
+      //   - examples/
+      //
+      // NOT with all individual file paths
+    });
+
+    it("should work independently of --force flag", async () => {
+      // You should be able to use --discover-paths without --force
+      // to update the config without re-initializing
+    });
+
+    it("should work together with --force flag", async () => {
+      // You should be able to use both flags together:
+      // --force: clear and re-initialize
+      // --discover-paths: update config with discovered patterns
+    });
+  });
+});
+
+describe("Path Pattern Discovery Function", () => {
+  // Unit tests for the function that converts file lists to directory patterns
+
+  it("should convert file list to directory patterns", () => {
+    const files = [
+      "README.md",
+      "docs/guide/intro.md",
+      "docs/guide/advanced.md",
+      "docs/api/reference.md",
+      "docs/api/endpoints.md",
+      "examples/basic.js",
+      "examples/advanced.js",
+      "src/index.ts",
+    ];
+
+    // Expected output: directory patterns
+    const expected = [
+      "README.md", // Single file at root
+      "docs/", // Multiple files in docs tree
+      "examples/", // Multiple files in examples
+      "src/", // Files in src
+    ];
+
+    // This will fail initially - function doesn't exist yet
+    // const result = discoverDirectoryPatterns(files);
+    // expect(result).toEqual(expected);
+  });
+
+  it("should handle nested directories efficiently", () => {
+    const files = [
+      "docs/en/guide/intro.md",
+      "docs/en/guide/advanced.md",
+      "docs/en/api/reference.md",
+      "docs/fr/guide/intro.md",
+    ];
+
+    // Should identify "docs/" as the common pattern
+    const expected = ["docs/"];
+
+    // const result = discoverDirectoryPatterns(files);
+    // expect(result).toEqual(expected);
+  });
+
+  it("should keep single files as-is", () => {
+    const files = [
+      "README.md",
+      "LICENSE",
+      "docs/guide/intro.md",
+      "docs/guide/advanced.md",
+    ];
+
+    const expected = [
+      "README.md",
+      "LICENSE",
+      "docs/", // Multiple files
+    ];
+
+    // const result = discoverDirectoryPatterns(files);
+    // expect(result).toEqual(expected);
+  });
+
+  it("should handle files in root directory", () => {
+    const files = ["README.md", "CONTRIBUTING.md", "LICENSE"];
+
+    // All single files in root - keep as individual files
+    const expected = ["README.md", "CONTRIBUTING.md", "LICENSE"];
+
+    // const result = discoverDirectoryPatterns(files);
+    // expect(result).toEqual(expected);
+  });
+});

packages/cli/src/commands/init.ts

@@ -10,6 +10,9 @@ import {
   ConfigManager,
   calculateLocalPath,
   ensureKnowledgeGitignoreSync,
+  discoverDirectoryPatterns,
+  safelyClearDirectory,
+  getDirectoryInfo,
 } from "@codemcp/knowledge-core";
 import {
   GitRepoLoader,
@@ -21,8 +24,16 @@ export const initCommand = new Command("init")
   .argument("<docset-id>", "ID of the docset to initialize")
   .option("-c, --config <path>", "Path to configuration file")
   .option("--force", "Force re-initialization even if already exists", false)
+  .option(
+    "--discover-paths",
+    "Discover and update config with directory patterns from extracted files",
+    false,
+  )
   .action(
-    async (docsetId: string, options: { config?: string; force: boolean }) => {
+    async (
+      docsetId: string,
+      options: { config?: string; force: boolean; discoverPaths: boolean },
+    ) => {
       console.log(chalk.blue("🚀 Agentic Knowledge Integration Test"));
 
       try {
@@ -82,6 +93,30 @@ export const initCommand = new Command("init")
           return;
         }
 
+        // Clear directory for force re-initialization
+        if (existsAlready && options.force) {
+          // Get info about what we're clearing (for logging)
+          const dirInfo = await getDirectoryInfo(localPath);
+
+          console.log(chalk.yellow("🗑️  Clearing existing directory..."));
+          console.log(
+            chalk.gray(
+              `    Removing: ${dirInfo.files} files, ${dirInfo.directories} dirs, ${dirInfo.symlinks} symlinks`,
+            ),
+          );
+
+          if (dirInfo.symlinks > 0) {
+            console.log(
+              chalk.gray(
+                "    ⚠️  Note: Symlinks will be removed, but source files are preserved",
+              ),
+            );
+          }
+
+          // Safely clear directory (preserves source files for symlinked folders)
+          await safelyClearDirectory(localPath);
+        }
+
         // Create target directory
         await fs.mkdir(localPath, { recursive: true });
 
@@ -164,6 +199,9 @@ export const initCommand = new Command("init")
             // Import symlink utilities
             const { createSymlinks } = await import("@codemcp/knowledge-core");
 
+            // Note: directory is already cleared above if --force is used,
+            // so no need to call removeSymlinks here
+
             const configDir = path.dirname(configPath);
             const projectRoot = path.dirname(configDir);
 
@@ -251,18 +289,29 @@ export const initCommand = new Command("init")
           JSON.stringify(overallMetadata, null, 2),
         );
 
-        // Update configuration with discovered paths (only if paths were discovered and force flag used)
-        if (allDiscoveredPaths.length > 0 && options.force) {
+        // Update configuration with discovered paths (only if --discover-paths flag used)
+        if (allDiscoveredPaths.length > 0 && options.discoverPaths) {
           console.log(
             chalk.yellow(
-              `\n📝 Updating configuration with discovered paths...`,
+              `\n📝 Discovering directory patterns from extracted files...`,
+            ),
+          );
+
+          // Convert file list to directory patterns
+          const directoryPatterns =
+            discoverDirectoryPatterns(allDiscoveredPaths);
+
+          console.log(
+            chalk.gray(
+              `    Found ${allDiscoveredPaths.length} files → ${directoryPatterns.length} patterns`,
             ),
           );
+
           try {
-            await configManager.updateDocsetPaths(docsetId, allDiscoveredPaths);
+            await configManager.updateDocsetPaths(docsetId, directoryPatterns);
             console.log(
               chalk.green(
-                `    ✅ Updated config with ${allDiscoveredPaths.length} discovered paths`,
+                `    ✅ Updated config with discovered patterns: ${directoryPatterns.slice(0, 5).join(", ")}${directoryPatterns.length > 5 ? "..." : ""}`,
               ),
             );
           } catch (configError) {