diff --git a/docs/guide/knowledge.md b/docs/guide/knowledge.md index e85dea593..e1d85dc34 100644 --- a/docs/guide/knowledge.md +++ b/docs/guide/knowledge.md @@ -114,6 +114,7 @@ teamai recall promote --dry-run ```bash teamai import --dir /path/to/project teamai import --dir /path/to/project --incremental +teamai import --dir /path/to/project --max-files 1000 teamai import --from-repo https://github.com/org/repo teamai import --from-org myorg teamai import --from-repo-list repos.yaml @@ -127,6 +128,8 @@ teamai import --from-repo https://github.com/org/repo --skip-enrich `--incremental` re-extracts only the files changed since the last import; each codebase keeps that baseline in its own `evidence/code//`. A remote repo's slug is `____`, shared by `--from-repo`, `--from-mr` and `teamai ci extract-mr`; `--dir` uses the repo name. +`--max-files` raises the cap on source files collected per repository (default 200). Files are collected key-file first; a source with more files is truncated, and the run warns with both counts instead of dropping the rest silently. + After an import, check and enrich that same `teamwiki/` with `teamai codebase`: ```bash diff --git a/docs/guide/zh-CN/knowledge.md b/docs/guide/zh-CN/knowledge.md index ac64efaea..a069d2ebb 100644 --- a/docs/guide/zh-CN/knowledge.md +++ b/docs/guide/zh-CN/knowledge.md @@ -114,6 +114,7 @@ teamai recall promote --dry-run ```bash teamai import --dir /path/to/project teamai import --dir /path/to/project --incremental +teamai import --dir /path/to/project --max-files 1000 teamai import --from-repo https://github.com/org/repo teamai import --from-org myorg teamai import --from-repo-list repos.yaml @@ -127,6 +128,8 @@ teamai import --from-repo https://github.com/org/repo --skip-enrich `--incremental` 只重新提取上次导入后改过的文件，基线随每个代码库存在它自己的 `evidence/code//` 里。远端仓库的 slug 是 `____`，`--from-repo`、`--from-mr` 和 `teamai ci extract-mr` 用的都是它；`--dir` 用仓库名。 +`--max-files` 调整单个仓库采集源码文件的上限（默认 200）。采集按关键文件优先；文件数超出上限时会截断，并输出同时带两个计数的警告，不再静默丢弃。 + 导入之后，用 `teamai codebase` 检查和加工同一份 `teamwiki/`： ```bash diff --git a/skill-data/core/references/commands.md b/skill-data/core/references/commands.md index 42358abde..4bc051658 100644 --- a/skill-data/core/references/commands.md +++ b/skill-data/core/references/commands.md @@ -403,6 +403,7 @@ Generated: do not edit by hand. Regenerate with - `--concurrency ` (hidden) — Concurrent repos for --from-repo-list (default 3) - `--incremental` — Re-extract only files changed since the last import (with --dir, --from-repo or --from-repo-list; remote repos also reuse the cached clone) - `--skip-enrich` — Skip AI enrichment (only clone + extract + graph, no LLM calls) + - `--max-files ` — Max source files collected per repository (default 200) - `--from-org ` — List repos under an org and generate a repo whitelist - `--max-repos ` (hidden) — Cap on repos pulled from --from-org (default 200) - `--exclude-archived` (hidden) — Exclude archived repos from --from-org (default true) diff --git a/src/__tests__/codebase-extract-fallback.test.ts b/src/__tests__/codebase-extract-fallback.test.ts index 94f350906..984dc6fe5 100644 --- a/src/__tests__/codebase-extract-fallback.test.ts +++ b/src/__tests__/codebase-extract-fallback.test.ts @@ -513,3 +513,51 @@ describe('a direct extractCodebase call stamps origin on the graph it writes (#9 expect(scoped?.edges ?? []).toHaveLength(0); }); }); + +function createManyFileFixture(count: number): string { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-1194-')); + temporaryDirectories.push(root); + fs.mkdirSync(path.join(root, 'src')); + for (let index = 0; index < count; index += 1) { + fs.writeFileSync( + path.join(root, 'src', `module-${String(index).padStart(2, '0')}.ts`), + `export const value${index} = ${index};\n`, + ); + } + return root; +} + +function captureConsoleLog(): string[] { + const lines: string[] = []; + vi.spyOn(console, 'log').mockImplementation((...args: unknown[]) => { + lines.push(args.map((arg) => String(arg)).join(' ')); + }); + return lines; +} + +describe('max-files cap on collected source files (#1194)', () => { + it('collects at most maxFiles files and warns with the counts when the source is larger', async () => { + const root = createManyFileFixture(5); + const output = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-1194-out-')); + temporaryDirectories.push(output); + const lines = captureConsoleLog(); + + await extractCodebase({ path: root, outputRoot: output, project: 'many', maxFiles: 2, skipEnrich: true }); + + expect(lines).toContain(' Files: 2'); + expect(lines.some((line) => line.includes('5 source files matched, but only the first 2 were collected'))).toBe(true); + expect(lines.some((line) => line.includes('--max-files'))).toBe(true); + }); + + it('does not warn when every candidate fits under maxFiles', async () => { + const root = createManyFileFixture(3); + const output = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-1194-out-')); + temporaryDirectories.push(output); + const lines = captureConsoleLog(); + + await extractCodebase({ path: root, outputRoot: output, project: 'few', maxFiles: 3, skipEnrich: true }); + + expect(lines).toContain(' Files: 3'); + expect(lines.some((line) => line.includes('were collected'))).toBe(false); + }); +}); diff --git a/src/codebase-extract.ts b/src/codebase-extract.ts index 63bf6b658..52e66ee0f 100644 --- a/src/codebase-extract.ts +++ b/src/codebase-extract.ts @@ -763,7 +763,12 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise files.length) { + console.log(chalk.yellow( + `[extract] ${project}: ${totalCandidates} source files matched, but only the first ${files.length} were collected. Raise the limit with --max-files .`, + )); + } if (files.length === 0 && !changedFiles) { // 全量模式下无文件 if (opts.json) { diff --git a/src/import-codebase.ts b/src/import-codebase.ts index 802552126..366467aca 100644 --- a/src/import-codebase.ts +++ b/src/import-codebase.ts @@ -20,6 +20,8 @@ export interface ImportCodebaseOptions { /** Re-extract only what changed since this codebase's published baseline. */ incremental?: boolean; skipEnrich?: boolean; + /** Cap on source files collected for this codebase (default 200). */ + maxFiles?: number; repoUrl?: string; branch?: string; /** MR/PR recorded as ingested in source-manifest.json. */ @@ -97,6 +99,7 @@ export async function importCodebase(opts: ImportCodebaseOptions): Promise const { url, depth = 1, forceSsh = false, forceAnonymous = false, explicitDomain, dryRun = false, output, - incremental = false, skipAutoPush = false, skipEnrich = false, sourceMrUrl, + incremental = false, skipAutoPush = false, skipEnrich = false, maxFiles, sourceMrUrl, } = opts; // 1. Parse provider and repo info @@ -385,7 +387,7 @@ export async function importFromRepo(opts: ImportFromRepoOptions): Promise let updated: boolean; try { updated = await importCodebase({ - sourceDir: cacheDir, slug, wikiRoot: teamwikiRoot, incremental, skipEnrich, + sourceDir: cacheDir, slug, wikiRoot: teamwikiRoot, incremental, skipEnrich, maxFiles, repoUrl: url, branch: cloneBranch === 'HEAD' ? undefined : cloneBranch, sourceMrUrl, diff --git a/src/import.ts b/src/import.ts index e29b91990..b07ed4c62 100644 --- a/src/import.ts +++ b/src/import.ts @@ -66,6 +66,8 @@ interface ImportOptions extends GlobalOptions { requireReview?: boolean; /** --skip-enrich: skip AI enrichment, only do clone + extract + graph */ skipEnrich?: boolean; + /** --max-files: cap on source files collected per repository (string, requires parseInt; default 200) */ + maxFiles?: string; } /** What one `--from-mr` URL produced: its draft, where it was written, and the repo it names. */ @@ -76,6 +78,16 @@ interface MrExtraction { learningFile?: string; } +/** Parse --max-files: a positive integer, or undefined to keep the extractor's default. */ +function parseMaxFiles(value: string | undefined): number | undefined { + if (value === undefined) return undefined; + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 1) { + throw new Error('--max-files must be a positive integer'); + } + return parsed; +} + /** * Main entry point for the import command, orchestrating dir, MR, org, and other import flows. * @@ -85,6 +97,7 @@ export async function importCmd(opts: ImportOptions): Promise { // Loaded here, not at startup: only this command shows a task list. const { Listr, PRESET_TIMER } = await import('listr2'); try { + const maxFiles = parseMaxFiles(opts.maxFiles); if (opts.fromOrg) { // 分支：--from-org ，组织级一键初始化 const tasks = new Listr([ @@ -129,6 +142,7 @@ export async function importCmd(opts: ImportOptions): Promise { output: opts.output, incremental: opts.incremental ?? false, skipEnrich: opts.skipEnrich ?? false, + maxFiles, }); }, rendererOptions: { persistentOutput: true }, @@ -154,6 +168,7 @@ export async function importCmd(opts: ImportOptions): Promise { output: opts.output, incremental: opts.incremental ?? false, skipEnrich: opts.skipEnrich ?? false, + maxFiles, }); task.title = `Batch import complete: ${result.succeeded} succeeded, ${result.failed.length} failed, ${result.skipped.length} skipped`; if (result.failed.length > 0) process.exitCode = 1; @@ -405,6 +420,7 @@ export async function importCmd(opts: ImportOptions): Promise { wikiRoot, incremental: opts.incremental ?? false, skipEnrich: opts.skipEnrich ?? false, + maxFiles, }); if (!team) { log.info(`Output written: ${wikiRoot}`); diff --git a/src/index.ts b/src/index.ts index 61e2a46f9..679802134 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1246,6 +1246,7 @@ program .addOption(new Option('--concurrency ', 'Concurrent repos for --from-repo-list (default 3)').default('3').hideHelp()) .option('--incremental', 'Re-extract only files changed since the last import (with --dir, --from-repo or --from-repo-list; remote repos also reuse the cached clone)') .option('--skip-enrich', 'Skip AI enrichment (only clone + extract + graph, no LLM calls)') + .option('--max-files ', 'Max source files collected per repository (default 200)') .option('--from-org ', 'List repos under an org and generate a repo whitelist') .addOption(new Option('--max-repos ', 'Cap on repos pulled from --from-org (default 200)').default('200').hideHelp()) .addOption(new Option('--exclude-archived', 'Exclude archived repos from --from-org (default true)').hideHelp()) diff --git a/src/wiki-engine/code-knowledge/code-collector.ts b/src/wiki-engine/code-knowledge/code-collector.ts index 2b703c223..cd07b1b68 100644 --- a/src/wiki-engine/code-knowledge/code-collector.ts +++ b/src/wiki-engine/code-knowledge/code-collector.ts @@ -65,7 +65,7 @@ export interface CollectCodeOptions { changedFiles?: string[]; } -export async function collectCode(options: CollectCodeOptions): Promise<{ manifest: CodeCollectionManifest; files: CodeCollectedFile[] }> { +export async function collectCode(options: CollectCodeOptions): Promise<{ manifest: CodeCollectionManifest; files: CodeCollectedFile[]; totalCandidates: number }> { const root = path.resolve(options.root); const filePaths: string[] = []; await walk(root, filePaths, options.includeTests ?? false); @@ -119,7 +119,8 @@ export async function collectCode(options: CollectCodeOptions): Promise<{ manife collectedAt: new Date().toISOString(), files: files.map(({ content: _content, ...file }) => file) }, - files + files, + totalCandidates: filtered.length }; }