feat: Full support for archive formats

.knowledge/config.yaml

@@ -22,3 +22,12 @@ docsets:
           - README.md
           - docs/en
           - template/
+  - id: pipeship-full
+    name: pipeship-full-docs
+    description: >-
+      Zip archive:
+      https://bahnhub.tech.rz.db.de/artifactory/pipeship-generic-stage-dev-local/docs-as-code/latest/pipeship-docs.tar.gz
+    sources:
+      - type: zip
+        url: >-
+          https://bahnhub.tech.rz.db.de/artifactory/pipeship-generic-stage-dev-local/docs-as-code/latest/pipeship-docs.tar.gz

packages/cli/src/commands/create.ts

@@ -6,19 +6,26 @@ import { Command } from "commander";
 import chalk from "chalk";
 import { promises as fs } from "node:fs";
 import * as path from "node:path";
+import { URL } from "node:url";
 import { ConfigManager } from "@codemcp/knowledge-core";
 import type { DocsetConfig } from "@codemcp/knowledge-core";
 
 export const createCommand = new Command("create")
   .description("Create a new docset using presets")
-  .requiredOption("--preset <type>", "Preset type: git-repo or local-folder")
+  .requiredOption(
+    "--preset <type>",
+    "Preset type: git-repo, local-folder, or archive",
+  )
   .requiredOption("--id <id>", "Unique docset ID")
   .requiredOption("--name <name>", "Human-readable docset name")
   .option("--description <desc>", "Docset description")
-  .option("--url <url>", "Git repository URL (required for git-repo preset)")
+  .option(
+    "--url <url>",
+    "Git repository URL (git-repo) or archive file URL (archive preset)",
+  )
   .option(
     "--path <path>",
-    "Local folder path (required for local-folder preset)",
+    "Local folder path (local-folder) or local archive file path (archive preset)",
   )
   .option("--branch <branch>", "Git branch (default: main)", "main")
   .action(async (options) => {
@@ -59,9 +66,11 @@ export const createCommand = new Command("create")
         newDocset = await createGitRepoDocset(options);
       } else if (options.preset === "local-folder") {
         newDocset = await createLocalFolderDocset(options);
+      } else if (options.preset === "archive") {
+        newDocset = await createArchiveDocset(options);
       } else {
         throw new Error(
-          `Unknown preset: ${options.preset}. Use 'git-repo' or 'local-folder'`,
+          `Unknown preset: ${options.preset}. Use 'git-repo', 'local-folder', or 'archive'`,
         );
       }
 
@@ -140,3 +149,63 @@ async function createLocalFolderDocset(options: any): Promise<DocsetConfig> {
     ],
   };
 }
+
+async function createArchiveDocset(options: any): Promise<DocsetConfig> {
+  if (!options.path && !options.url) {
+    throw new Error("Either --path or --url is required for archive preset");
+  }
+
+  // If path is provided, validate it exists
+  if (options.path) {
+    const fullPath = path.resolve(options.path);
+    try {
+      const stat = await fs.stat(fullPath);
+      if (!stat.isFile()) {
+        throw new Error(`Path is not a file: ${options.path}`);
+      }
+      const lowerPath = options.path.toLowerCase();
+      if (
+        !lowerPath.endsWith(".zip") &&
+        !lowerPath.endsWith(".tar.gz") &&
+        !lowerPath.endsWith(".tgz")
+      ) {
+        throw new Error(
+          `File is not a supported archive format (zip, tar.gz): ${options.path}`,
+        );
+      }
+    } catch {
+      throw new Error(`Path does not exist or is invalid: ${options.path}`);
+    }
+  }
+
+  // If URL is provided, validate it's a valid URL
+  if (options.url) {
+    try {
+      new URL(options.url);
+    } catch {
+      throw new Error(`Invalid URL format: ${options.url}`);
+    }
+  }
+
+  const source: any = {
+    type: "archive",
+  };
+
+  if (options.path) {
+    source.path = options.path;
+  }
+  if (options.url) {
+    source.url = options.url;
+  }
+  if (options.paths) {
+    source.paths = options.paths.split(",");
+  }
+
+  return {
+    id: options.id,
+    name: options.name,
+    description:
+      options.description || `Archive: ${options.path || options.url}`,
+    sources: [source],
+  };
+}

packages/cli/src/commands/init.ts

@@ -16,7 +16,7 @@ import {
 } from "@codemcp/knowledge-core";
 import {
   GitRepoLoader,
-  ZipLoader,
+  ArchiveLoader,
   WebSourceType,
 } from "@codemcp/knowledge-content-loader";
 
@@ -267,16 +267,18 @@ export const initCommand = new Command("init")
               path.join(localPath, `.agentic-source-${index}.json`),
               JSON.stringify(metadata, null, 2),
             );
-          } else if (source.type === "zip") {
-            // Handle zip file initialization
-            const loader = new ZipLoader();
+          } else if (source.type === "archive") {
+            // Handle archive file initialization (zip, tar.gz, etc.)
+            const loader = new ArchiveLoader();
             const sourceUrl = source.url || source.path || "";
 
-            console.log(chalk.gray(`  Using ZipLoader for zip extraction`));
+            console.log(
+              chalk.gray(`  Using ArchiveLoader for archive extraction`),
+            );
 
             const webSourceConfig = {
               url: sourceUrl,
-              type: WebSourceType.ZIP,
+              type: WebSourceType.ARCHIVE,
               options: {
                 paths: source.paths || [],
               },
@@ -286,15 +288,15 @@ export const initCommand = new Command("init")
             const validation = loader.validateConfig(webSourceConfig);
             if (validation !== true) {
               throw new Error(
-                `Invalid zip source configuration: ${validation}`,
+                `Invalid archive source configuration: ${validation}`,
               );
             }
 
-            // Load content using ZipLoader
+            // Load content using ArchiveLoader
             const result = await loader.load(webSourceConfig, localPath);
 
             if (!result.success) {
-              throw new Error(`Zip loading failed: ${result.error}`);
+              throw new Error(`Archive loading failed: ${result.error}`);
             }
 
             // Collect discovered paths for config update
@@ -303,7 +305,7 @@ export const initCommand = new Command("init")
             totalFiles += result.files.length;
             console.log(
               chalk.green(
-                `    ✅ Extracted ${result.files.length} files from zip`,
+                `    ✅ Extracted ${result.files.length} files from archive`,
               ),
             );
 

packages/cli/src/commands/refresh.ts

@@ -14,7 +14,10 @@ import {
   calculateLocalPath,
   ensureKnowledgeGitignoreSync,
 } from "@codemcp/knowledge-core";
-import { ZipLoader, WebSourceType } from "@codemcp/knowledge-content-loader";
+import {
+  ArchiveLoader,
+  WebSourceType,
+} from "@codemcp/knowledge-content-loader";
 
 interface DocsetMetadata {
   docset_id: string;
@@ -169,8 +172,8 @@ async function refreshDocset(
         );
         totalFiles += sourceFiles.files_count;
         refreshedSources.push(sourceFiles);
-      } else if (source.type === "zip") {
-        const sourceFiles = await refreshZipSource(
+      } else if (source.type === "archive") {
+        const sourceFiles = await refreshArchiveSource(
           source,
           localPath,
           index,
@@ -372,7 +375,7 @@ async function refreshGitSource(
   }
 }
 
-async function refreshZipSource(
+async function refreshArchiveSource(
   source: any,
   localPath: string,
   index: number,
@@ -393,10 +396,10 @@ async function refreshZipSource(
   }
 
   const sourceUrl = source.url || source.path || "";
-  const loader = new ZipLoader();
+  const loader = new ArchiveLoader();
   const webSourceConfig = {
     url: sourceUrl,
-    type: WebSourceType.ZIP,
+    type: WebSourceType.ARCHIVE,
     options: {
       paths: source.paths || [],
     },
@@ -439,12 +442,12 @@ async function refreshZipSource(
   const result = await loader.load(webSourceConfig, localPath);
 
   if (!result.success) {
-    throw new Error(`Zip refresh failed: ${result.error}`);
+    throw new Error(`Archive refresh failed: ${result.error}`);
   }
 
   const metadata: SourceMetadata = {
     source_url: sourceUrl,
-    source_type: "zip",
+    source_type: "archive",
     downloaded_at: new Date().toISOString(),
     files_count: result.files.length,
     files: result.files,

packages/content-loader/package.json

@@ -30,12 +30,14 @@
   },
   "dependencies": {
     "adm-zip": "0.5.16",
-    "simple-git": "^3.22.0"
+    "simple-git": "^3.22.0",
+    "tar": "7.5.9"
   },
   "devDependencies": {
     "@eslint/js": "^9.34.0",
     "@types/adm-zip": "0.5.7",
     "@types/node": "^24.3.0",
+    "@types/tar": "7.0.87",
     "eslint": "^9.34.0",
     "rimraf": "^6.0.1",
     "typescript": "^5.9.2",