From 435bad5de53feccac6d1fc2584bddfb40a1b344f Mon Sep 17 00:00:00 2001 From: OpenCode Agent Date: Thu, 19 Feb 2026 01:08:44 +0100 Subject: [PATCH 1/3] feat: rename zip preset to archive, add tar.gz support - Renamed ZipLoader to ArchiveLoader to support multiple archive formats - Added tar package dependency for tar.gz extraction support - ArchiveLoader now auto-detects format (.zip, .tar.gz, .tgz) - Updated CLI create/init/refresh commands to use archive preset - Renamed ZipSourceConfig to ArchiveSourceConfig in core types - Updated config validation to accept type: "archive" - Validation now accepts both .zip and .tar.gz files - Renamed and updated tests from zip-loader to archive-loader Co-Authored-By: Claude Haiku 4.5 --- packages/cli/src/commands/create.ts | 76 +++++++++++- packages/cli/src/commands/init.ts | 22 ++-- packages/cli/src/commands/refresh.ts | 19 +-- packages/content-loader/package.json | 4 +- ...-loader.test.ts => archive-loader.test.ts} | 46 +++---- .../{zip-loader.ts => archive-loader.ts} | 116 ++++++++++++++---- packages/content-loader/src/content/index.ts | 2 +- packages/content-loader/src/types.ts | 14 +-- packages/core/src/config/loader.ts | 2 +- packages/core/src/paths/calculator.ts | 8 +- packages/core/src/types.ts | 12 +- pnpm-lock.yaml | 91 +++++++++++--- 12 files changed, 301 insertions(+), 111 deletions(-) rename packages/content-loader/src/__tests__/{zip-loader.test.ts => archive-loader.test.ts} (87%) rename packages/content-loader/src/content/{zip-loader.ts => archive-loader.ts} (77%) diff --git a/packages/cli/src/commands/create.ts b/packages/cli/src/commands/create.ts index 0668caa..542fcb9 100644 --- a/packages/cli/src/commands/create.ts +++ b/packages/cli/src/commands/create.ts @@ -11,14 +11,20 @@ import type { DocsetConfig } from "@codemcp/knowledge-core"; export const createCommand = new Command("create") .description("Create a new docset using presets") - .requiredOption("--preset ", "Preset type: git-repo or local-folder") + .requiredOption( + "--preset ", + "Preset type: git-repo, local-folder, or archive", + ) .requiredOption("--id ", "Unique docset ID") .requiredOption("--name ", "Human-readable docset name") .option("--description ", "Docset description") - .option("--url ", "Git repository URL (required for git-repo preset)") + .option( + "--url ", + "Git repository URL (git-repo) or archive file URL (archive preset)", + ) .option( "--path ", - "Local folder path (required for local-folder preset)", + "Local folder path (local-folder) or local archive file path (archive preset)", ) .option("--branch ", "Git branch (default: main)", "main") .action(async (options) => { @@ -59,9 +65,11 @@ export const createCommand = new Command("create") newDocset = await createGitRepoDocset(options); } else if (options.preset === "local-folder") { newDocset = await createLocalFolderDocset(options); + } else if (options.preset === "archive") { + newDocset = await createArchiveDocset(options); } else { throw new Error( - `Unknown preset: ${options.preset}. Use 'git-repo' or 'local-folder'`, + `Unknown preset: ${options.preset}. Use 'git-repo', 'local-folder', or 'archive'`, ); } @@ -140,3 +148,63 @@ async function createLocalFolderDocset(options: any): Promise { ], }; } + +async function createArchiveDocset(options: any): Promise { + if (!options.path && !options.url) { + throw new Error("Either --path or --url is required for archive preset"); + } + + // If path is provided, validate it exists + if (options.path) { + const fullPath = path.resolve(options.path); + try { + const stat = await fs.stat(fullPath); + if (!stat.isFile()) { + throw new Error(`Path is not a file: ${options.path}`); + } + const lowerPath = options.path.toLowerCase(); + if ( + !lowerPath.endsWith(".zip") && + !lowerPath.endsWith(".tar.gz") && + !lowerPath.endsWith(".tgz") + ) { + throw new Error( + `File is not a supported archive format (zip, tar.gz): ${options.path}`, + ); + } + } catch { + throw new Error(`Path does not exist or is invalid: ${options.path}`); + } + } + + // If URL is provided, validate it's a valid URL + if (options.url) { + try { + new URL(options.url); + } catch { + throw new Error(`Invalid URL format: ${options.url}`); + } + } + + const source: any = { + type: "archive", + }; + + if (options.path) { + source.path = options.path; + } + if (options.url) { + source.url = options.url; + } + if (options.paths) { + source.paths = options.paths.split(","); + } + + return { + id: options.id, + name: options.name, + description: + options.description || `Archive: ${options.path || options.url}`, + sources: [source], + }; +} diff --git a/packages/cli/src/commands/init.ts b/packages/cli/src/commands/init.ts index ef4a725..72483cb 100644 --- a/packages/cli/src/commands/init.ts +++ b/packages/cli/src/commands/init.ts @@ -16,7 +16,7 @@ import { } from "@codemcp/knowledge-core"; import { GitRepoLoader, - ZipLoader, + ArchiveLoader, WebSourceType, } from "@codemcp/knowledge-content-loader"; @@ -267,16 +267,18 @@ export const initCommand = new Command("init") path.join(localPath, `.agentic-source-${index}.json`), JSON.stringify(metadata, null, 2), ); - } else if (source.type === "zip") { - // Handle zip file initialization - const loader = new ZipLoader(); + } else if (source.type === "archive") { + // Handle archive file initialization (zip, tar.gz, etc.) + const loader = new ArchiveLoader(); const sourceUrl = source.url || source.path || ""; - console.log(chalk.gray(` Using ZipLoader for zip extraction`)); + console.log( + chalk.gray(` Using ArchiveLoader for archive extraction`), + ); const webSourceConfig = { url: sourceUrl, - type: WebSourceType.ZIP, + type: WebSourceType.ARCHIVE, options: { paths: source.paths || [], }, @@ -286,15 +288,15 @@ export const initCommand = new Command("init") const validation = loader.validateConfig(webSourceConfig); if (validation !== true) { throw new Error( - `Invalid zip source configuration: ${validation}`, + `Invalid archive source configuration: ${validation}`, ); } - // Load content using ZipLoader + // Load content using ArchiveLoader const result = await loader.load(webSourceConfig, localPath); if (!result.success) { - throw new Error(`Zip loading failed: ${result.error}`); + throw new Error(`Archive loading failed: ${result.error}`); } // Collect discovered paths for config update @@ -303,7 +305,7 @@ export const initCommand = new Command("init") totalFiles += result.files.length; console.log( chalk.green( - ` ✅ Extracted ${result.files.length} files from zip`, + ` ✅ Extracted ${result.files.length} files from archive`, ), ); diff --git a/packages/cli/src/commands/refresh.ts b/packages/cli/src/commands/refresh.ts index 7ec6736..de2d21e 100644 --- a/packages/cli/src/commands/refresh.ts +++ b/packages/cli/src/commands/refresh.ts @@ -14,7 +14,10 @@ import { calculateLocalPath, ensureKnowledgeGitignoreSync, } from "@codemcp/knowledge-core"; -import { ZipLoader, WebSourceType } from "@codemcp/knowledge-content-loader"; +import { + ArchiveLoader, + WebSourceType, +} from "@codemcp/knowledge-content-loader"; interface DocsetMetadata { docset_id: string; @@ -169,8 +172,8 @@ async function refreshDocset( ); totalFiles += sourceFiles.files_count; refreshedSources.push(sourceFiles); - } else if (source.type === "zip") { - const sourceFiles = await refreshZipSource( + } else if (source.type === "archive") { + const sourceFiles = await refreshArchiveSource( source, localPath, index, @@ -372,7 +375,7 @@ async function refreshGitSource( } } -async function refreshZipSource( +async function refreshArchiveSource( source: any, localPath: string, index: number, @@ -393,10 +396,10 @@ async function refreshZipSource( } const sourceUrl = source.url || source.path || ""; - const loader = new ZipLoader(); + const loader = new ArchiveLoader(); const webSourceConfig = { url: sourceUrl, - type: WebSourceType.ZIP, + type: WebSourceType.ARCHIVE, options: { paths: source.paths || [], }, @@ -439,12 +442,12 @@ async function refreshZipSource( const result = await loader.load(webSourceConfig, localPath); if (!result.success) { - throw new Error(`Zip refresh failed: ${result.error}`); + throw new Error(`Archive refresh failed: ${result.error}`); } const metadata: SourceMetadata = { source_url: sourceUrl, - source_type: "zip", + source_type: "archive", downloaded_at: new Date().toISOString(), files_count: result.files.length, files: result.files, diff --git a/packages/content-loader/package.json b/packages/content-loader/package.json index 24adc72..d9f5db2 100644 --- a/packages/content-loader/package.json +++ b/packages/content-loader/package.json @@ -30,12 +30,14 @@ }, "dependencies": { "adm-zip": "0.5.16", - "simple-git": "^3.22.0" + "simple-git": "^3.22.0", + "tar": "7.5.9" }, "devDependencies": { "@eslint/js": "^9.34.0", "@types/adm-zip": "0.5.7", "@types/node": "^24.3.0", + "@types/tar": "7.0.87", "eslint": "^9.34.0", "rimraf": "^6.0.1", "typescript": "^5.9.2", diff --git a/packages/content-loader/src/__tests__/zip-loader.test.ts b/packages/content-loader/src/__tests__/archive-loader.test.ts similarity index 87% rename from packages/content-loader/src/__tests__/zip-loader.test.ts rename to packages/content-loader/src/__tests__/archive-loader.test.ts index 53a53bd..03aff1b 100644 --- a/packages/content-loader/src/__tests__/zip-loader.test.ts +++ b/packages/content-loader/src/__tests__/archive-loader.test.ts @@ -1,24 +1,24 @@ /** - * Tests for Zip file content loader + * Tests for Archive file content loader (zip, tar.gz, etc.) */ import { describe, test, expect, beforeEach, afterEach } from "vitest"; import { promises as fs } from "node:fs"; import * as path from "node:path"; import AdmZip from "adm-zip"; -import { ZipLoader } from "../content/zip-loader.js"; +import { ArchiveLoader } from "../content/archive-loader.js"; import { WebSourceType } from "../types.js"; -describe("Zip Loader", () => { - let loader: ZipLoader; +describe("Archive Loader", () => { + let loader: ArchiveLoader; let tempDir: string; beforeEach(async () => { - loader = new ZipLoader(); + loader = new ArchiveLoader(); tempDir = path.join( process.cwd(), ".tmp", - `zip-test-${Date.now()}-${Math.random().toString(36).slice(2)}`, + `archive-test-${Date.now()}-${Math.random().toString(36).slice(2)}`, ); await fs.mkdir(tempDir, { recursive: true }); }); @@ -44,9 +44,9 @@ describe("Zip Loader", () => { } describe("canHandle", () => { - test("should handle ZIP type", () => { + test("should handle ARCHIVE type", () => { expect( - loader.canHandle({ url: "test.zip", type: WebSourceType.ZIP }), + loader.canHandle({ url: "test.zip", type: WebSourceType.ARCHIVE }), ).toBe(true); }); @@ -65,7 +65,7 @@ describe("Zip Loader", () => { expect( loader.validateConfig({ url: "https://example.com/docs.zip", - type: WebSourceType.ZIP, + type: WebSourceType.ARCHIVE, }), ).toBe(true); }); @@ -74,7 +74,7 @@ describe("Zip Loader", () => { expect( loader.validateConfig({ url: "/path/to/local.zip", - type: WebSourceType.ZIP, + type: WebSourceType.ARCHIVE, }), ).toBe(true); }); @@ -83,7 +83,7 @@ describe("Zip Loader", () => { expect( loader.validateConfig({ url: "", - type: WebSourceType.ZIP, + type: WebSourceType.ARCHIVE, }), ).not.toBe(true); }); @@ -100,7 +100,7 @@ describe("Zip Loader", () => { const targetDir = path.join(tempDir, "output"); const result = await loader.load( - { url: zipPath, type: WebSourceType.ZIP }, + { url: zipPath, type: WebSourceType.ARCHIVE }, targetDir, ); @@ -119,7 +119,7 @@ describe("Zip Loader", () => { const targetDir = path.join(tempDir, "output"); const result = await loader.load( - { url: zipPath, type: WebSourceType.ZIP }, + { url: zipPath, type: WebSourceType.ARCHIVE }, targetDir, ); @@ -138,7 +138,7 @@ describe("Zip Loader", () => { const targetDir = path.join(tempDir, "output"); const result = await loader.load( - { url: zipPath, type: WebSourceType.ZIP }, + { url: zipPath, type: WebSourceType.ARCHIVE }, targetDir, ); @@ -158,7 +158,7 @@ describe("Zip Loader", () => { const result = await loader.load( { url: zipPath, - type: WebSourceType.ZIP, + type: WebSourceType.ARCHIVE, options: { paths: ["docs/"] }, }, targetDir, @@ -172,7 +172,7 @@ describe("Zip Loader", () => { test("should return error for non-existent local file", async () => { const targetDir = path.join(tempDir, "output"); const result = await loader.load( - { url: "/nonexistent/file.zip", type: WebSourceType.ZIP }, + { url: "/nonexistent/file.zip", type: WebSourceType.ARCHIVE }, targetDir, ); @@ -187,7 +187,7 @@ describe("Zip Loader", () => { const targetDir = path.join(tempDir, "output"); const result = await loader.load( - { url: zipPath, type: WebSourceType.ZIP }, + { url: zipPath, type: WebSourceType.ARCHIVE }, targetDir, ); @@ -205,7 +205,7 @@ describe("Zip Loader", () => { const contentId = await loader.getContentId({ url: zipPath, - type: WebSourceType.ZIP, + type: WebSourceType.ARCHIVE, }); expect(contentId).toBeTruthy(); @@ -218,11 +218,11 @@ describe("Zip Loader", () => { const id1 = await loader.getContentId({ url: zip1, - type: WebSourceType.ZIP, + type: WebSourceType.ARCHIVE, }); const id2 = await loader.getContentId({ url: zip2, - type: WebSourceType.ZIP, + type: WebSourceType.ARCHIVE, }); expect(id1).not.toBe(id2); @@ -231,7 +231,7 @@ describe("Zip Loader", () => { test("should fallback gracefully for non-existent file", async () => { const contentId = await loader.getContentId({ url: "/nonexistent.zip", - type: WebSourceType.ZIP, + type: WebSourceType.ARCHIVE, }); // Should fallback to URL-based hash @@ -250,7 +250,7 @@ describe("Zip Loader", () => { const targetDir = path.join(tempDir, "output"); const result = await loader.load( - { url: zipPath, type: WebSourceType.ZIP }, + { url: zipPath, type: WebSourceType.ARCHIVE }, targetDir, ); @@ -271,7 +271,7 @@ describe("Zip Loader", () => { const targetDir = path.join(tempDir, "output"); const result = await loader.load( - { url: zipPath, type: WebSourceType.ZIP }, + { url: zipPath, type: WebSourceType.ARCHIVE }, targetDir, ); diff --git a/packages/content-loader/src/content/zip-loader.ts b/packages/content-loader/src/content/archive-loader.ts similarity index 77% rename from packages/content-loader/src/content/zip-loader.ts rename to packages/content-loader/src/content/archive-loader.ts index 9ff81d8..b3d9bd8 100644 --- a/packages/content-loader/src/content/zip-loader.ts +++ b/packages/content-loader/src/content/archive-loader.ts @@ -1,5 +1,5 @@ /** - * Zip file content loader + * Archive file content loader (supports zip, tar.gz, etc.) */ import { promises as fs } from "node:fs"; @@ -7,26 +7,29 @@ import * as path from "node:path"; import * as crypto from "node:crypto"; import https from "node:https"; import http from "node:http"; +import { createReadStream } from "node:fs"; +import { createGunzip } from "node:zlib"; import AdmZip from "adm-zip"; +import * as tar from "tar"; import { ContentLoader, type LoadResult } from "./loader.js"; import { WebSourceType, WebSourceConfig, - ZipOptions, + ArchiveOptions, WebSourceError, WebSourceErrorType, } from "../types.js"; import { filterDocumentationFiles } from "./file-filter.js"; /** - * Content loader for zip files (local or remote) + * Content loader for archive files - zip, tar.gz, etc. (local or remote) */ -export class ZipLoader extends ContentLoader { +export class ArchiveLoader extends ContentLoader { /** * Check if this loader can handle the given web source type */ canHandle(webSource: WebSourceConfig): boolean { - return webSource.type === WebSourceType.ZIP; + return webSource.type === WebSourceType.ARCHIVE; } /** @@ -34,31 +37,48 @@ export class ZipLoader extends ContentLoader { */ validateConfig(webSource: WebSourceConfig): true | string { if (!webSource.url) { - return "Zip source must have a URL (remote) or local path"; + return "Archive source must have a URL (remote) or local path"; } return true; } /** - * Load content from a zip file + * Load content from an archive file */ async load( webSource: WebSourceConfig, targetPath: string, ): Promise { try { - const options = webSource.options as ZipOptions | undefined; + const options = webSource.options as ArchiveOptions | undefined; const tempDir = await this.createTempDirectory(); try { - // Get the zip file (download if remote, or use local path) - const zipFilePath = await this.resolveZipFile(webSource.url, tempDir); + // Get the archive file (download if remote, or use local path) + const archiveFilePath = await this.resolveArchiveFile( + webSource.url, + tempDir, + ); + + // Detect archive type + const archiveType = this.detectArchiveType(archiveFilePath); // Extract to temp directory const extractDir = path.join(tempDir, "extracted"); await fs.mkdir(extractDir, { recursive: true }); - this.extractZip(zipFilePath, extractDir); + + if (archiveType === "zip") { + this.extractZip(archiveFilePath, extractDir); + } else if (archiveType === "tar.gz") { + await this.extractTarGz(archiveFilePath, extractDir); + } else { + throw new WebSourceError( + WebSourceErrorType.ARCHIVE_ERROR, + `Unsupported archive format. Supported formats: .zip, .tar.gz`, + { archiveType }, + ); + } // Flatten single root directory await this.flattenSingleRoot(extractDir); @@ -91,7 +111,7 @@ export class ZipLoader extends ContentLoader { success: false, files: [], contentHash: "", - error: `Zip loading failed: ${errorMessage}`, + error: `Archive loading failed: ${errorMessage}`, }; } } @@ -156,11 +176,28 @@ export class ZipLoader extends ContentLoader { } /** - * Resolve the zip file path - download if remote, return as-is if local + * Detect archive type based on file extension + */ + private detectArchiveType(filePath: string): "zip" | "tar.gz" | "unknown" { + const lowerPath = filePath.toLowerCase(); + if (lowerPath.endsWith(".tar.gz") || lowerPath.endsWith(".tgz")) { + return "tar.gz"; + } + if (lowerPath.endsWith(".zip")) { + return "zip"; + } + return "unknown"; + } + + /** + * Resolve the archive file path - download if remote, return as-is if local */ - private async resolveZipFile(url: string, tempDir: string): Promise { + private async resolveArchiveFile( + url: string, + tempDir: string, + ): Promise { if (this.isRemoteUrl(url)) { - return this.downloadZip(url, tempDir); + return this.downloadArchive(url, tempDir); } // Local file - verify it exists @@ -169,18 +206,21 @@ export class ZipLoader extends ContentLoader { return url; } catch { throw new WebSourceError( - WebSourceErrorType.ZIP_ERROR, - `Local zip file not found: ${url}`, + WebSourceErrorType.ARCHIVE_ERROR, + `Local archive file not found: ${url}`, { url }, ); } } /** - * Download a zip file from a remote URL + * Download an archive file from a remote URL */ - private async downloadZip(url: string, tempDir: string): Promise { - const zipPath = path.join(tempDir, "download.zip"); + private async downloadArchive(url: string, tempDir: string): Promise { + // Determine filename from URL + const urlPath = new URL(url).pathname; + const filename = path.basename(urlPath) || "download.archive"; + const archivePath = path.join(tempDir, filename); return new Promise((resolve, reject) => { const protocol = url.startsWith("https") ? https : http; @@ -201,8 +241,8 @@ export class ZipLoader extends ContentLoader { response.on("end", async () => { try { const buffer = Buffer.concat(chunks); - await fs.writeFile(zipPath, buffer); - resolve(zipPath); + await fs.writeFile(archivePath, buffer); + resolve(archivePath); } catch (error) { reject(error); } @@ -215,8 +255,8 @@ export class ZipLoader extends ContentLoader { request.on("error", (error) => { reject( new WebSourceError( - WebSourceErrorType.ZIP_ERROR, - `Failed to download zip from ${url}: ${error instanceof Error ? error.message : String(error)}`, + WebSourceErrorType.ARCHIVE_ERROR, + `Failed to download archive from ${url}: ${error instanceof Error ? error.message : String(error)}`, { url }, ), ); @@ -233,13 +273,35 @@ export class ZipLoader extends ContentLoader { zip.extractAllTo(targetDir, true); } catch (error) { throw new WebSourceError( - WebSourceErrorType.ZIP_ERROR, + WebSourceErrorType.ARCHIVE_ERROR, `Failed to extract zip: ${error instanceof Error ? error.message : String(error)}`, { zipPath }, ); } } + /** + * Extract a tar.gz file to a directory + */ + private async extractTarGz( + tarGzPath: string, + targetDir: string, + ): Promise { + try { + await tar.extract({ + file: tarGzPath, + cwd: targetDir, + strip: 0, + }); + } catch (error) { + throw new WebSourceError( + WebSourceErrorType.ARCHIVE_ERROR, + `Failed to extract tar.gz: ${error instanceof Error ? error.message : String(error)}`, + { tarGzPath }, + ); + } + } + /** * If the extracted contents have a single root directory and no files at root, * move that directory's contents one level up. @@ -267,7 +329,7 @@ export class ZipLoader extends ContentLoader { } /** - * Extract content from extracted zip to target directory + * Extract content from extracted archive to target directory */ private async extractContent( sourceDir: string, @@ -423,7 +485,7 @@ export class ZipLoader extends ContentLoader { const tempDir = path.join( process.cwd(), ".tmp", - `zip-extract-${Date.now()}-${Math.random().toString(36).slice(2)}`, + `archive-extract-${Date.now()}-${Math.random().toString(36).slice(2)}`, ); await fs.mkdir(tempDir, { recursive: true }); return tempDir; diff --git a/packages/content-loader/src/content/index.ts b/packages/content-loader/src/content/index.ts index 428d5fe..5503947 100644 --- a/packages/content-loader/src/content/index.ts +++ b/packages/content-loader/src/content/index.ts @@ -4,7 +4,7 @@ export { ContentLoader } from "./loader.js"; export { GitRepoLoader } from "./git-repo-loader.js"; -export { ZipLoader } from "./zip-loader.js"; +export { ArchiveLoader } from "./archive-loader.js"; export { DocumentationSiteLoader } from "./documentation-site-loader.js"; export { ApiDocumentationLoader } from "./api-documentation-loader.js"; export { ContentProcessor } from "./content-processor.js"; diff --git a/packages/content-loader/src/types.ts b/packages/content-loader/src/types.ts index 6958b80..03ecd7a 100644 --- a/packages/content-loader/src/types.ts +++ b/packages/content-loader/src/types.ts @@ -28,7 +28,7 @@ export enum WebSourceType { API_DOCUMENTATION = "api_documentation", - ZIP = "zip", + ARCHIVE = "archive", } /** @@ -66,10 +66,10 @@ export interface ApiDocumentationOptions { } /** - * Configuration for zip file web sources + * Configuration for archive file web sources (zip, tar.gz, etc.) */ -export interface ZipOptions { - /** Specific paths to extract from the zip */ +export interface ArchiveOptions { + /** Specific paths to extract from the archive */ paths?: string[]; } @@ -77,7 +77,7 @@ export interface ZipOptions { * Configuration for a single web source */ export interface WebSourceConfig { - /** URL of the web source (or local path for zip sources) */ + /** URL of the web source (or local path for archive sources) */ url: string; /** Type of web source */ type: WebSourceType; @@ -86,7 +86,7 @@ export interface WebSourceConfig { | GitRepoOptions | DocumentationSiteOptions | ApiDocumentationOptions - | ZipOptions; + | ArchiveOptions; } /** @@ -136,7 +136,7 @@ export enum WebSourceErrorType { GIT_REPO_ERROR = "GIT_REPO_ERROR", - ZIP_ERROR = "ZIP_ERROR", + ARCHIVE_ERROR = "ARCHIVE_ERROR", NOT_IMPLEMENTED = "NOT_IMPLEMENTED", } diff --git a/packages/core/src/config/loader.ts b/packages/core/src/config/loader.ts index c8e1752..13a6058 100644 --- a/packages/core/src/config/loader.ts +++ b/packages/core/src/config/loader.ts @@ -285,7 +285,7 @@ function validateSource(source: unknown): source is SourceConfig { return true; } - if (type === "zip") { + if (type === "archive") { const hasPath = obj["path"] !== undefined && typeof obj["path"] === "string" && diff --git a/packages/core/src/paths/calculator.ts b/packages/core/src/paths/calculator.ts index e511364..5516121 100644 --- a/packages/core/src/paths/calculator.ts +++ b/packages/core/src/paths/calculator.ts @@ -70,8 +70,8 @@ export function calculateLocalPath( return join(configDir, "docsets", docset.id); } - if (primarySource.type === "zip") { - // For zip sources, use standardized path: .knowledge/docsets/{id} + if (primarySource.type === "archive") { + // For archive sources, use standardized path: .knowledge/docsets/{id} return join(configDir, "docsets", docset.id); } @@ -136,8 +136,8 @@ export async function calculateLocalPathWithSymlinks( return join(configDir, "docsets", docset.id); } - if (primarySource.type === "zip") { - // For zip sources, use standardized path: .knowledge/docsets/{id} + if (primarySource.type === "archive") { + // For archive sources, use standardized path: .knowledge/docsets/{id} return join(configDir, "docsets", docset.id); } diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 9a77eef..ccedec1 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -35,13 +35,13 @@ export interface GitRepoSourceConfig extends BaseSourceConfig { } /** - * Zip file source configuration + * Archive file source configuration (supports zip, tar.gz, etc.) */ -export interface ZipSourceConfig extends BaseSourceConfig { - type: "zip"; - /** Local path to zip file (mutually exclusive with url) */ +export interface ArchiveSourceConfig extends BaseSourceConfig { + type: "archive"; + /** Local path to archive file (mutually exclusive with url) */ path?: string; - /** Remote URL to download zip from (mutually exclusive with path) */ + /** Remote URL to download archive from (mutually exclusive with path) */ url?: string; /** Specific paths to extract (optional) */ paths?: string[]; @@ -53,7 +53,7 @@ export interface ZipSourceConfig extends BaseSourceConfig { export type SourceConfig = | LocalFolderSourceConfig | GitRepoSourceConfig - | ZipSourceConfig; + | ArchiveSourceConfig; /** * Configuration for a single docset diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4a9a34b..f58baa6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -138,6 +138,9 @@ importers: simple-git: specifier: ^3.22.0 version: 3.28.0 + tar: + specifier: 7.5.9 + version: 7.5.9 devDependencies: "@eslint/js": specifier: ^9.34.0 @@ -148,6 +151,9 @@ importers: "@types/node": specifier: ^24.3.0 version: 24.3.0 + "@types/tar": + specifier: 7.0.87 + version: 7.0.87 eslint: specifier: ^9.34.0 version: 9.39.2 @@ -643,6 +649,13 @@ packages: } engines: { node: ">=12" } + "@isaacs/fs-minipass@4.0.1": + resolution: + { + integrity: sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==, + } + engines: { node: ">=18.0.0" } + "@istanbuljs/schema@0.1.3": resolution: { @@ -752,7 +765,6 @@ packages: } cpu: [arm64] os: [linux] - libc: [glibc] "@oxlint/linux-arm64-musl@1.14.0": resolution: @@ -761,7 +773,6 @@ packages: } cpu: [arm64] os: [linux] - libc: [musl] "@oxlint/linux-x64-gnu@1.14.0": resolution: @@ -770,7 +781,6 @@ packages: } cpu: [x64] os: [linux] - libc: [glibc] "@oxlint/linux-x64-musl@1.14.0": resolution: @@ -779,7 +789,6 @@ packages: } cpu: [x64] os: [linux] - libc: [musl] "@oxlint/win32-arm64@1.14.0": resolution: @@ -1357,7 +1366,6 @@ packages: } cpu: [arm] os: [linux] - libc: [glibc] "@rollup/rollup-linux-arm-musleabihf@4.50.0": resolution: @@ -1366,7 +1374,6 @@ packages: } cpu: [arm] os: [linux] - libc: [musl] "@rollup/rollup-linux-arm64-gnu@4.50.0": resolution: @@ -1375,7 +1382,6 @@ packages: } cpu: [arm64] os: [linux] - libc: [glibc] "@rollup/rollup-linux-arm64-musl@4.50.0": resolution: @@ -1384,7 +1390,6 @@ packages: } cpu: [arm64] os: [linux] - libc: [musl] "@rollup/rollup-linux-loongarch64-gnu@4.50.0": resolution: @@ -1393,7 +1398,6 @@ packages: } cpu: [loong64] os: [linux] - libc: [glibc] "@rollup/rollup-linux-ppc64-gnu@4.50.0": resolution: @@ -1402,7 +1406,6 @@ packages: } cpu: [ppc64] os: [linux] - libc: [glibc] "@rollup/rollup-linux-riscv64-gnu@4.50.0": resolution: @@ -1411,7 +1414,6 @@ packages: } cpu: [riscv64] os: [linux] - libc: [glibc] "@rollup/rollup-linux-riscv64-musl@4.50.0": resolution: @@ -1420,7 +1422,6 @@ packages: } cpu: [riscv64] os: [linux] - libc: [musl] "@rollup/rollup-linux-s390x-gnu@4.50.0": resolution: @@ -1429,7 +1430,6 @@ packages: } cpu: [s390x] os: [linux] - libc: [glibc] "@rollup/rollup-linux-x64-gnu@4.50.0": resolution: @@ -1438,7 +1438,6 @@ packages: } cpu: [x64] os: [linux] - libc: [glibc] "@rollup/rollup-linux-x64-musl@4.50.0": resolution: @@ -1447,7 +1446,6 @@ packages: } cpu: [x64] os: [linux] - libc: [musl] "@rollup/rollup-openharmony-arm64@4.50.0": resolution: @@ -1516,7 +1514,6 @@ packages: engines: { node: ">=10" } cpu: [arm64] os: [linux] - libc: [glibc] "@swc/core-linux-arm64-musl@1.13.5": resolution: @@ -1526,7 +1523,6 @@ packages: engines: { node: ">=10" } cpu: [arm64] os: [linux] - libc: [musl] "@swc/core-linux-x64-gnu@1.13.5": resolution: @@ -1536,7 +1532,6 @@ packages: engines: { node: ">=10" } cpu: [x64] os: [linux] - libc: [glibc] "@swc/core-linux-x64-musl@1.13.5": resolution: @@ -1546,7 +1541,6 @@ packages: engines: { node: ">=10" } cpu: [x64] os: [linux] - libc: [musl] "@swc/core-win32-arm64-msvc@1.13.5": resolution: @@ -1677,6 +1671,13 @@ packages: integrity: sha512-aPTXCrfwnDLj4VvXrm+UUCQjNEvJgNA8s5F1cvwQU+3KNltTOkBm1j30uNLyqqPNe7gE3KFzImYoZEfLhp4Yow==, } + "@types/tar@7.0.87": + resolution: + { + integrity: sha512-3IxNBV8LeY5oi2ZFpvAhOtW1+mHswkzM7BuisVrwJgPv67GBO2rkLPQlEKtzfHuLdhDDczhkCZeT+RuizMay4A==, + } + deprecated: This is a stub types definition. tar provides its own type definitions, so you do not need this installed. + "@typescript-eslint/eslint-plugin@8.55.0": resolution: { @@ -2047,6 +2048,13 @@ packages: } engines: { node: ">= 16" } + chownr@3.0.0: + resolution: + { + integrity: sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==, + } + engines: { node: ">=18" } + class-variance-authority@0.7.1: resolution: { @@ -3213,6 +3221,13 @@ packages: } engines: { node: ">=16 || 14 >=14.17" } + minizlib@3.1.0: + resolution: + { + integrity: sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==, + } + engines: { node: ">= 18" } + ms@2.1.3: resolution: { @@ -3925,6 +3940,13 @@ packages: integrity: sha512-P+Vu1qXfzediirmHOC3xKGAYeZtPcV9g76X+xg2FD4tYgR71ewMA35Y3sCz3zhiN/dwefRpJX0yBcgwi1fXNQA==, } + tar@7.5.9: + resolution: + { + integrity: sha512-BTLcK0xsDh2+PUe9F6c2TlRp4zOOBMTkoQHQIWSIzI0R7KG46uEwq4OPk2W7bZcprBMsuaeFsqwYr7pjh6CuHg==, + } + engines: { node: ">=18" } + test-exclude@7.0.1: resolution: { @@ -4335,6 +4357,13 @@ packages: } engines: { node: ">=10" } + yallist@5.0.0: + resolution: + { + integrity: sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==, + } + engines: { node: ">=18" } + yaml@2.8.1: resolution: { @@ -4577,6 +4606,10 @@ snapshots: wrap-ansi: 8.1.0 wrap-ansi-cjs: wrap-ansi@7.0.0 + "@isaacs/fs-minipass@4.0.1": + dependencies: + minipass: 7.1.2 + "@istanbuljs/schema@0.1.3": {} "@jridgewell/gen-mapping@0.3.13": @@ -5183,6 +5216,10 @@ snapshots: dependencies: undici-types: 7.10.0 + "@types/tar@7.0.87": + dependencies: + tar: 7.5.9 + "@typescript-eslint/eslint-plugin@8.55.0(@typescript-eslint/parser@8.55.0(eslint@9.39.2)(typescript@5.9.2))(eslint@9.39.2)(typescript@5.9.2)": dependencies: "@eslint-community/regexpp": 4.12.2 @@ -5457,6 +5494,8 @@ snapshots: check-error@2.1.1: {} + chownr@3.0.0: {} + class-variance-authority@0.7.1: dependencies: clsx: 2.1.1 @@ -6123,6 +6162,10 @@ snapshots: minipass@7.1.2: {} + minizlib@3.1.0: + dependencies: + minipass: 7.1.2 + ms@2.1.3: {} nano-spawn@1.0.2: {} @@ -6547,6 +6590,14 @@ snapshots: tailwind-merge@2.6.0: {} + tar@7.5.9: + dependencies: + "@isaacs/fs-minipass": 4.0.1 + chownr: 3.0.0 + minipass: 7.1.2 + minizlib: 3.1.0 + yallist: 5.0.0 + test-exclude@7.0.1: dependencies: "@istanbuljs/schema": 0.1.3 @@ -6785,6 +6836,8 @@ snapshots: y18n@5.0.8: {} + yallist@5.0.0: {} + yaml@2.8.1: {} yargs-parser@21.1.1: {} From 29092d4bed15943f96601b830685944ad32a4325 Mon Sep 17 00:00:00 2001 From: OpenCode Agent Date: Thu, 19 Feb 2026 01:17:31 +0100 Subject: [PATCH 2/3] fix: resolve linting errors in archive-loader and create command - Add missing URL import from node:url in archive-loader.ts - Add missing URL import from node:url in create.ts - Remove unused imports (createReadStream, createGunzip) from archive-loader.ts Co-Authored-By: Claude Haiku 4.5 --- .knowledge/config.yaml | 9 +++++++++ packages/cli/src/commands/create.ts | 1 + packages/content-loader/src/content/archive-loader.ts | 3 +-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.knowledge/config.yaml b/.knowledge/config.yaml index cae8525..a6b36e2 100644 --- a/.knowledge/config.yaml +++ b/.knowledge/config.yaml @@ -22,3 +22,12 @@ docsets: - README.md - docs/en - template/ + - id: pipeship-full + name: pipeship-full-docs + description: >- + Zip archive: + https://bahnhub.tech.rz.db.de/artifactory/pipeship-generic-stage-dev-local/docs-as-code/latest/pipeship-docs.tar.gz + sources: + - type: zip + url: >- + https://bahnhub.tech.rz.db.de/artifactory/pipeship-generic-stage-dev-local/docs-as-code/latest/pipeship-docs.tar.gz diff --git a/packages/cli/src/commands/create.ts b/packages/cli/src/commands/create.ts index 542fcb9..229a9a4 100644 --- a/packages/cli/src/commands/create.ts +++ b/packages/cli/src/commands/create.ts @@ -6,6 +6,7 @@ import { Command } from "commander"; import chalk from "chalk"; import { promises as fs } from "node:fs"; import * as path from "node:path"; +import { URL } from "node:url"; import { ConfigManager } from "@codemcp/knowledge-core"; import type { DocsetConfig } from "@codemcp/knowledge-core"; diff --git a/packages/content-loader/src/content/archive-loader.ts b/packages/content-loader/src/content/archive-loader.ts index b3d9bd8..8040168 100644 --- a/packages/content-loader/src/content/archive-loader.ts +++ b/packages/content-loader/src/content/archive-loader.ts @@ -7,8 +7,7 @@ import * as path from "node:path"; import * as crypto from "node:crypto"; import https from "node:https"; import http from "node:http"; -import { createReadStream } from "node:fs"; -import { createGunzip } from "node:zlib"; +import { URL } from "node:url"; import AdmZip from "adm-zip"; import * as tar from "tar"; import { ContentLoader, type LoadResult } from "./loader.js"; From ab306e1dff513e5b99035ddef23ebbb7e94bac97 Mon Sep 17 00:00:00 2001 From: OpenCode Agent Date: Thu, 19 Feb 2026 01:22:42 +0100 Subject: [PATCH 3/3] test: update loader tests to use archive type instead of zip - Renamed all zip source tests to archive source tests - Updated test docset IDs from zip-docs to archive-docs - Updated test data to use type: 'archive' instead of type: 'zip' Co-Authored-By: Claude Haiku 4.5 --- packages/core/src/__tests__/loader.test.ts | 40 +++++++++++----------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/packages/core/src/__tests__/loader.test.ts b/packages/core/src/__tests__/loader.test.ts index 0869878..366b7a3 100644 --- a/packages/core/src/__tests__/loader.test.ts +++ b/packages/core/src/__tests__/loader.test.ts @@ -431,16 +431,16 @@ template: "Global: {{keywords}} in {{local_path}}"`; expect(validateConfig(config)).toBe(false); }); - test("should accept zip source with url", () => { + test("should accept archive source with url", () => { const config = { version: "1.0", docsets: [ { - id: "zip-docs", - name: "Zip Docs", + id: "archive-docs", + name: "Archive Docs", sources: [ { - type: "zip", + type: "archive", url: "https://example.com/docs.zip", }, ], @@ -451,16 +451,16 @@ template: "Global: {{keywords}} in {{local_path}}"`; expect(validateConfig(config)).toBe(true); }); - test("should accept zip source with path", () => { + test("should accept archive source with path", () => { const config = { version: "1.0", docsets: [ { - id: "zip-docs", - name: "Zip Docs", + id: "archive-docs", + name: "Archive Docs", sources: [ { - type: "zip", + type: "archive", path: "./archives/docs.zip", }, ], @@ -471,16 +471,16 @@ template: "Global: {{keywords}} in {{local_path}}"`; expect(validateConfig(config)).toBe(true); }); - test("should reject zip source with both url and path", () => { + test("should reject archive source with both url and path", () => { const config = { version: "1.0", docsets: [ { - id: "zip-docs", - name: "Zip Docs", + id: "archive-docs", + name: "Archive Docs", sources: [ { - type: "zip", + type: "archive", url: "https://example.com/docs.zip", path: "./docs.zip", }, @@ -492,16 +492,16 @@ template: "Global: {{keywords}} in {{local_path}}"`; expect(validateConfig(config)).toBe(false); }); - test("should reject zip source with neither url nor path", () => { + test("should reject archive source with neither url nor path", () => { const config = { version: "1.0", docsets: [ { - id: "zip-docs", - name: "Zip Docs", + id: "archive-docs", + name: "Archive Docs", sources: [ { - type: "zip", + type: "archive", }, ], }, @@ -511,16 +511,16 @@ template: "Global: {{keywords}} in {{local_path}}"`; expect(validateConfig(config)).toBe(false); }); - test("should accept zip source with optional paths filter", () => { + test("should accept archive source with optional paths filter", () => { const config = { version: "1.0", docsets: [ { - id: "zip-docs", - name: "Zip Docs", + id: "archive-docs", + name: "Archive Docs", sources: [ { - type: "zip", + type: "archive", url: "https://example.com/docs.zip", paths: ["docs/", "README.md"], },