Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions docs/guide/knowledge.md
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ teamai recall promote <learningId> --dry-run
```bash
teamai import --dir /path/to/project
teamai import --dir /path/to/project --incremental
teamai import --dir /path/to/project --max-files 1000
teamai import --from-repo https://github.com/org/repo
teamai import --from-org myorg
teamai import --from-repo-list repos.yaml
Expand All @@ -127,6 +128,8 @@ teamai import --from-repo https://github.com/org/repo --skip-enrich

`--incremental` re-extracts only the files changed since the last import; each codebase keeps that baseline in its own `evidence/code/<slug>/`. A remote repo's slug is `<provider>__<owner>__<repo>`, shared by `--from-repo`, `--from-mr` and `teamai ci extract-mr`; `--dir` uses the repo name.

`--max-files` raises the cap on source files collected per repository (default 200). Files are collected key-file first; a source with more files is truncated, and the run warns with both counts instead of dropping the rest silently.

After an import, check and enrich that same `teamwiki/` with `teamai codebase`:

```bash
Expand Down
3 changes: 3 additions & 0 deletions docs/guide/zh-CN/knowledge.md
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ teamai recall promote <learningId> --dry-run
```bash
teamai import --dir /path/to/project
teamai import --dir /path/to/project --incremental
teamai import --dir /path/to/project --max-files 1000
teamai import --from-repo https://github.com/org/repo
teamai import --from-org myorg
teamai import --from-repo-list repos.yaml
Expand All @@ -127,6 +128,8 @@ teamai import --from-repo https://github.com/org/repo --skip-enrich

`--incremental` 只重新提取上次导入后改过的文件,基线随每个代码库存在它自己的 `evidence/code/<slug>/` 里。远端仓库的 slug 是 `<provider>__<owner>__<repo>`,`--from-repo`、`--from-mr` 和 `teamai ci extract-mr` 用的都是它;`--dir` 用仓库名。

`--max-files` 调整单个仓库采集源码文件的上限(默认 200)。采集按关键文件优先;文件数超出上限时会截断,并输出同时带两个计数的警告,不再静默丢弃。

导入之后,用 `teamai codebase` 检查和加工同一份 `teamwiki/`:

```bash
Expand Down
1 change: 1 addition & 0 deletions skill-data/core/references/commands.md
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,7 @@ Generated: do not edit by hand. Regenerate with
- `--concurrency <n>` (hidden) — Concurrent repos for --from-repo-list (default 3)
- `--incremental` — Re-extract only files changed since the last import (with --dir, --from-repo or --from-repo-list; remote repos also reuse the cached clone)
- `--skip-enrich` — Skip AI enrichment (only clone + extract + graph, no LLM calls)
- `--max-files <n>` — Max source files collected per repository (default 200)
- `--from-org <org>` — List repos under an org and generate a repo whitelist
- `--max-repos <n>` (hidden) — Cap on repos pulled from --from-org (default 200)
- `--exclude-archived` (hidden) — Exclude archived repos from --from-org (default true)
Expand Down
48 changes: 48 additions & 0 deletions src/__tests__/codebase-extract-fallback.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -513,3 +513,51 @@ describe('a direct extractCodebase call stamps origin on the graph it writes (#9
expect(scoped?.edges ?? []).toHaveLength(0);
});
});

function createManyFileFixture(count: number): string {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-1194-'));
temporaryDirectories.push(root);
fs.mkdirSync(path.join(root, 'src'));
for (let index = 0; index < count; index += 1) {
fs.writeFileSync(
path.join(root, 'src', `module-${String(index).padStart(2, '0')}.ts`),
`export const value${index} = ${index};\n`,
);
}
return root;
}

function captureConsoleLog(): string[] {
const lines: string[] = [];
vi.spyOn(console, 'log').mockImplementation((...args: unknown[]) => {
lines.push(args.map((arg) => String(arg)).join(' '));
});
return lines;
}

describe('max-files cap on collected source files (#1194)', () => {
it('collects at most maxFiles files and warns with the counts when the source is larger', async () => {
const root = createManyFileFixture(5);
const output = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-1194-out-'));
temporaryDirectories.push(output);
const lines = captureConsoleLog();

await extractCodebase({ path: root, outputRoot: output, project: 'many', maxFiles: 2, skipEnrich: true });

expect(lines).toContain(' Files: 2');
expect(lines.some((line) => line.includes('5 source files matched, but only the first 2 were collected'))).toBe(true);
expect(lines.some((line) => line.includes('--max-files'))).toBe(true);
});

it('does not warn when every candidate fits under maxFiles', async () => {
const root = createManyFileFixture(3);
const output = fs.mkdtempSync(path.join(os.tmpdir(), 'teamai-1194-out-'));
temporaryDirectories.push(output);
const lines = captureConsoleLog();

await extractCodebase({ path: root, outputRoot: output, project: 'few', maxFiles: 3, skipEnrich: true });

expect(lines).toContain(' Files: 3');
expect(lines.some((line) => line.includes('were collected'))).toBe(false);
});
});
7 changes: 6 additions & 1 deletion src/codebase-extract.ts
Original file line number Diff line number Diff line change
Expand Up @@ -763,7 +763,12 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise<boo
}
}

const { files, manifest: collectionManifest } = await collectCode({ root, maxFiles, changedFiles });
const { files, manifest: collectionManifest, totalCandidates } = await collectCode({ root, maxFiles, changedFiles });
if (!opts.json && totalCandidates > files.length) {
console.log(chalk.yellow(
`[extract] ${project}: ${totalCandidates} source files matched, but only the first ${files.length} were collected. Raise the limit with --max-files <n>.`,
));
}
if (files.length === 0 && !changedFiles) {
// 全量模式下无文件
if (opts.json) {
Expand Down
3 changes: 3 additions & 0 deletions src/import-codebase.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ export interface ImportCodebaseOptions {
/** Re-extract only what changed since this codebase's published baseline. */
incremental?: boolean;
skipEnrich?: boolean;
/** Cap on source files collected for this codebase (default 200). */
maxFiles?: number;
repoUrl?: string;
branch?: string;
/** MR/PR recorded as ingested in source-manifest.json. */
Expand Down Expand Up @@ -97,6 +99,7 @@ export async function importCodebase(opts: ImportCodebaseOptions): Promise<boole
json: false,
incremental: seeded,
skipEnrich: opts.skipEnrich,
maxFiles: opts.maxFiles,
repoUrl: opts.repoUrl,
branch: opts.branch,
sourceMrUrl: opts.sourceMrUrl,
Expand Down
4 changes: 4 additions & 0 deletions src/import-repo-list.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ export interface ImportFromRepoListOptions {
incremental?: boolean;
/** 跳过 AI enrichment(只做 clone + extract + graph,不调用 LLM) */
skipEnrich?: boolean;
/** 每个仓库采集源码文件的上限(默认 200) */
maxFiles?: number;
}

/** importFromRepoList 汇总结果。 */
Expand Down Expand Up @@ -70,6 +72,7 @@ export async function importFromRepoList(
output,
incremental = false,
skipEnrich = false,
maxFiles,
} = opts;

// 1. 加载白名单
Expand Down Expand Up @@ -121,6 +124,7 @@ export async function importFromRepoList(
incremental,
skipAutoPush: true,
skipEnrich,
maxFiles,
});
succeeded.push(1);
} catch (err) {
Expand Down
6 changes: 4 additions & 2 deletions src/import-repo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ export interface ImportFromRepoOptions {
skipAutoPush?: boolean;
/** Skip AI enrichment (only clone + extract + graph, no LLM calls) */
skipEnrich?: boolean;
/** Cap on source files collected for this repo (default 200) */
maxFiles?: number;
/** Source MR/PR URL to record as ingested in source-manifest.json (P5). */
sourceMrUrl?: string | readonly string[];
}
Expand Down Expand Up @@ -243,7 +245,7 @@ export async function importFromRepo(opts: ImportFromRepoOptions): Promise<void>
const {
url, depth = 1, forceSsh = false, forceAnonymous = false,
explicitDomain, dryRun = false, output,
incremental = false, skipAutoPush = false, skipEnrich = false, sourceMrUrl,
incremental = false, skipAutoPush = false, skipEnrich = false, maxFiles, sourceMrUrl,
} = opts;

// 1. Parse provider and repo info
Expand Down Expand Up @@ -385,7 +387,7 @@ export async function importFromRepo(opts: ImportFromRepoOptions): Promise<void>
let updated: boolean;
try {
updated = await importCodebase({
sourceDir: cacheDir, slug, wikiRoot: teamwikiRoot, incremental, skipEnrich,
sourceDir: cacheDir, slug, wikiRoot: teamwikiRoot, incremental, skipEnrich, maxFiles,
repoUrl: url,
branch: cloneBranch === 'HEAD' ? undefined : cloneBranch,
sourceMrUrl,
Expand Down
16 changes: 16 additions & 0 deletions src/import.ts
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ interface ImportOptions extends GlobalOptions {
requireReview?: boolean;
/** --skip-enrich: skip AI enrichment, only do clone + extract + graph */
skipEnrich?: boolean;
/** --max-files: cap on source files collected per repository (string, requires parseInt; default 200) */
maxFiles?: string;
}

/** What one `--from-mr` URL produced: its draft, where it was written, and the repo it names. */
Expand All @@ -76,6 +78,16 @@ interface MrExtraction {
learningFile?: string;
}

/** Parse --max-files: a positive integer, or undefined to keep the extractor's default. */
function parseMaxFiles(value: string | undefined): number | undefined {
if (value === undefined) return undefined;
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed < 1) {
throw new Error('--max-files must be a positive integer');
}
return parsed;
}

/**
* Main entry point for the import command, orchestrating dir, MR, org, and other import flows.
*
Expand All @@ -85,6 +97,7 @@ export async function importCmd(opts: ImportOptions): Promise<void> {
// Loaded here, not at startup: only this command shows a task list.
const { Listr, PRESET_TIMER } = await import('listr2');
try {
const maxFiles = parseMaxFiles(opts.maxFiles);
if (opts.fromOrg) {
// 分支:--from-org <org>,组织级一键初始化
const tasks = new Listr([
Expand Down Expand Up @@ -129,6 +142,7 @@ export async function importCmd(opts: ImportOptions): Promise<void> {
output: opts.output,
incremental: opts.incremental ?? false,
skipEnrich: opts.skipEnrich ?? false,
maxFiles,
});
},
rendererOptions: { persistentOutput: true },
Expand All @@ -154,6 +168,7 @@ export async function importCmd(opts: ImportOptions): Promise<void> {
output: opts.output,
incremental: opts.incremental ?? false,
skipEnrich: opts.skipEnrich ?? false,
maxFiles,
});
task.title = `Batch import complete: ${result.succeeded} succeeded, ${result.failed.length} failed, ${result.skipped.length} skipped`;
if (result.failed.length > 0) process.exitCode = 1;
Expand Down Expand Up @@ -405,6 +420,7 @@ export async function importCmd(opts: ImportOptions): Promise<void> {
wikiRoot,
incremental: opts.incremental ?? false,
skipEnrich: opts.skipEnrich ?? false,
maxFiles,
});
if (!team) {
log.info(`Output written: ${wikiRoot}`);
Expand Down
1 change: 1 addition & 0 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1246,6 +1246,7 @@ program
.addOption(new Option('--concurrency <n>', 'Concurrent repos for --from-repo-list (default 3)').default('3').hideHelp())
.option('--incremental', 'Re-extract only files changed since the last import (with --dir, --from-repo or --from-repo-list; remote repos also reuse the cached clone)')
.option('--skip-enrich', 'Skip AI enrichment (only clone + extract + graph, no LLM calls)')
.option('--max-files <n>', 'Max source files collected per repository (default 200)')
.option('--from-org <org>', 'List repos under an org and generate a repo whitelist')
.addOption(new Option('--max-repos <n>', 'Cap on repos pulled from --from-org (default 200)').default('200').hideHelp())
.addOption(new Option('--exclude-archived', 'Exclude archived repos from --from-org (default true)').hideHelp())
Expand Down
5 changes: 3 additions & 2 deletions src/wiki-engine/code-knowledge/code-collector.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ export interface CollectCodeOptions {
changedFiles?: string[];
}

export async function collectCode(options: CollectCodeOptions): Promise<{ manifest: CodeCollectionManifest; files: CodeCollectedFile[] }> {
export async function collectCode(options: CollectCodeOptions): Promise<{ manifest: CodeCollectionManifest; files: CodeCollectedFile[]; totalCandidates: number }> {
const root = path.resolve(options.root);
const filePaths: string[] = [];
await walk(root, filePaths, options.includeTests ?? false);
Expand Down Expand Up @@ -119,7 +119,8 @@ export async function collectCode(options: CollectCodeOptions): Promise<{ manife
collectedAt: new Date().toISOString(),
files: files.map(({ content: _content, ...file }) => file)
},
files
files,
totalCandidates: filtered.length
};
}

Expand Down
Loading