From d0d44751d0c13aac8c5aa5641c6300fff2f013e0 Mon Sep 17 00:00:00 2001 From: Pavel Fadeev Date: Sat, 16 May 2026 02:33:37 +0200 Subject: [PATCH] feat: large repo scaling improvements - git-history.ts: SHA-based dedup in mineHistory, package detection (packages/|apps/|libs/ workspaces), dedupKey utility - cards.ts: time-decayed scoring (half-life: 180d fixes/churn, 365d arch), proportional card budget scaling (20 + n/200 commits, max 200), bounded suggestion text (max 3 example subjects), package tagging, min decay floor to prevent cards from vanishing - training.ts: contextFormat option for context-augmented training data, formatCardContext helper for structured card rendering The combination: large repos (10k+ commits) now get proportionally more cards, time-decayed scores favor recent activity, suggestion text is compact, and training data can be generated in context+question format. --- src/cards.ts | 118 +++++++++++++++++++++++++++++++++++---------- src/git-history.ts | 45 +++++++++++++++++ src/training.ts | 45 +++++++++++++++-- 3 files changed, 179 insertions(+), 29 deletions(-) diff --git a/src/cards.ts b/src/cards.ts index fba5826..08ad90e 100644 --- a/src/cards.ts +++ b/src/cards.ts @@ -20,6 +20,10 @@ export type InsightCard = { supportingCommits: { sha: string; subject: string }[]; affectedFiles: string[]; suggestion: string; + /** Detected monorepo package */ + packageId?: string; + /** Time-decayed score */ + decayedScore?: number; }; export type CardsOptions = { @@ -42,7 +46,7 @@ export function cardIdFrom(type: CardType, title: string, affectedFiles: string[ return crypto.createHash('sha256').update(raw).digest('hex').slice(0, 16); } -function toCard(data: { type: CardType; title: string; confidence: number; supportingCommits: { sha: string; subject: string }[]; affectedFiles: string[]; suggestion: string }, statusOverride?: CardStatus): InsightCard { +function toCard(data: { type: CardType; title: string; confidence: number; supportingCommits: { sha: string; subject: string }[]; affectedFiles: string[]; suggestion: string; packageId?: string; decayedScore?: number }, statusOverride?: CardStatus): InsightCard { return { ...data, id: cardIdFrom(data.type, data.title, data.affectedFiles), @@ -50,6 +54,54 @@ function toCard(data: { type: CardType; title: string; confidence: number; suppo }; } +// ─── Time-decayed scoring ──────────────────────────────── + +const HALF_LIFE_FIX = 180; // days — bug fixes lose relevance over 6 months +const HALF_LIFE_CHURN = 180; // days — churn patterns over 6 months +const HALF_LIFE_ARCH = 365; // days — architectural changes last a year +const MIN_DECAY = 0.5; // minimum decay factor to prevent cards from vanishing + +export function timeDecay(dateStr: string, halfLifeDays: number): number { + if (!dateStr) return 1.0; + const commitDate = new Date(dateStr).getTime(); + if (isNaN(commitDate)) return 1.0; + const now = Date.now(); + const ageDays = (now - commitDate) / (1000 * 60 * 60 * 24); + if (ageDays <= 0) return 1.0; + return Math.max(MIN_DECAY, Math.exp(-ageDays / halfLifeDays)); +} + +export function decayedScore(count: number, latestDate: string, halfLifeDays: number): number { + return count * timeDecay(latestDate, halfLifeDays); +} + +export function boundedSuggestion(subjects: string[], maxExamples = 3): string { + if (subjects.length <= maxExamples) return subjects.join('; '); + return subjects.slice(0, maxExamples).join('; ') + `; (and ${subjects.length - maxExamples} more)`; +} + +// ─── Package detection ──────────────────────────────────── + +export function detectPackage(filePath: string): string { + const pkgMatch = filePath.match(/^(packages\/[^/]+)/); + if (pkgMatch) return pkgMatch[1]; + const appMatch = filePath.match(/^(apps\/[^/]+)/); + if (appMatch) return appMatch[1]; + const libMatch = filePath.match(/^(libs\/[^/]+)/); + if (libMatch) return libMatch[1]; + const parts = filePath.split('/'); + if (parts.length === 1) return ''; + return ''; +} + +// ─── Proportional card budget ──────────────────────────── + +export function computeCardBudget(recordCount: number, maxCards?: number): number { + if (maxCards !== undefined && maxCards > 0) return maxCards; + // Scale with repo size: 20 cards base + 1 per 200 commits, max 200 + return Math.min(200, Math.max(20, Math.ceil(recordCount / 200))); +} + // ─── Generators ──────────────────────────────────────────── function churnHotspotCards(records: ClassifiedCommit[]): Omit[] { @@ -130,7 +182,7 @@ function repeatedFixCards(records: ClassifiedCommit[]): Omit ({ sha: r.sha, subject: r.subject })), affectedFiles: [file], - suggestion: `Repeated bug fixes in ${file}: this file was fixed ${count} separate times (commits: ${commits.map(c => c.subject).join(', ')}). Each fix suggests the underlying cause was not fully addressed — consider deeper root-cause analysis and regression tests for the affected code paths.`, + suggestion: `Repeated bug fixes in ${file}: this file was fixed ${count} separate times. Examples: ${boundedSuggestion(commits.map(c => c.subject), 3)} Each fix suggests the underlying cause was not fully addressed — consider deeper root-cause analysis and regression tests for the affected code paths.`, })); } @@ -271,7 +323,7 @@ function coChangeCards(records: ClassifiedCommit[]): Omit ({ sha: r.sha, subject: r.subject })), affectedFiles: [a, b], - suggestion: `Coupled change: ${a} and ${b} were modified together in ${count} commits (subjects: ${commits.map(c => c.subject).join(', ')}). Consider whether these should be merged, refactored to reduce coupling, or share common tests.`, + suggestion: `Coupled change: ${a} and ${b} were modified together in ${count} commits. Examples: ${boundedSuggestion(commits.map(c => c.subject), 3)} Consider whether these should be merged, refactored to reduce coupling, or share common tests.`, })); } @@ -352,14 +404,28 @@ export function generateCards( statusOverrides?: Record, ): InsightCard[] { const minConfidence = options.minConfidence ?? 0.3; - const maxCards = options.maxCards ?? 20; + const maxCards = computeCardBudget(classifiedRecords.length, options.maxCards); const allCards: ScoredCard[] = []; for (const generator of CARD_GENERATORS) { const cards = generator.run(classifiedRecords); for (const card of cards) { + const pkg = detectPackage(card.affectedFiles[0] ?? ''); + // Apply time-decayed score to confidence + const latestDate = card.supportingCommits.length > 0 + ? (classifiedRecords.find(r => r.sha === card.supportingCommits[0].sha)?.authoredAt ?? '') + : ''; + const halfLife = card.type === 'repeated-fix' ? HALF_LIFE_FIX + : card.type === 'churn-hotspot' ? HALF_LIFE_CHURN + : HALF_LIFE_ARCH; + const decay = timeDecay(latestDate, halfLife); + const decayedConfidence = parseFloat((card.confidence * decay).toFixed(2)); + allCards.push({ ...card, + confidence: decayedConfidence, + packageId: pkg, + decayedScore: parseFloat((card.confidence * decay).toFixed(3)), key: cardIdFrom(card.type, card.title, card.affectedFiles), bucket: classifyCardBucket(card), }); @@ -368,47 +434,47 @@ export function generateCards( const eligible = allCards .filter(card => card.confidence >= minConfidence) - .sort(sortDesc); + .sort((a, b) => (b.decayedScore ?? b.confidence) - (a.decayedScore ?? a.confidence)); + + // Proportional card budget by bucket + const sourceCount = eligible.filter(c => c.bucket === 'source').length; + const configCount = eligible.filter(c => c.bucket === 'config').length; + const rationaleCount = eligible.filter(c => c.bucket === 'rationale').length; + const otherCount = eligible.filter(c => c.bucket === 'other').length; + const total = Math.max(1, sourceCount + configCount + rationaleCount + otherCount); + + const sourceQuota = Math.max(1, Math.round(maxCards * sourceCount / total)); + const configQuota = Math.max(1, Math.round(maxCards * configCount / total)); + const rationaleQuota = Math.max(0, Math.round(maxCards * rationaleCount / total)); + const otherQuota = Math.max(1, maxCards - sourceQuota - configQuota - rationaleQuota); const buckets: Record = { - source: [], - config: [], - rationale: [], - other: [], + source: [], config: [], rationale: [], other: [], }; - for (const card of eligible) { - buckets[card.bucket].push(card); - } + for (const card of eligible) buckets[card.bucket].push(card); const selected: ScoredCard[] = []; const selectedKeys = new Set(); - const pushCard = (card: ScoredCard): void => { - if (selected.length >= maxCards || selectedKeys.has(card.key)) return; - selected.push(card); - selectedKeys.add(card.key); - }; + const take = (bucket: CardBucket, quota: number): void => { for (const card of buckets[bucket]) { - if (selected.length >= maxCards) break; - if (quota <= 0) break; - if (selectedKeys.has(card.key)) continue; - pushCard(card); + if (selected.length >= maxCards || quota <= 0 || selectedKeys.has(card.key)) break; + selected.push(card); + selectedKeys.add(card.key); quota -= 1; } }; - const sourceQuota = Math.min(Math.max(1, Math.round(maxCards * 0.45)), Math.min(10, maxCards)); - const configQuota = Math.min(Math.max(1, Math.round(maxCards * 0.25)), Math.max(0, maxCards - sourceQuota)); - const rationaleQuota = Math.min(Math.max(0, Math.round(maxCards * 0.15)), Math.max(0, maxCards - sourceQuota - configQuota)); - take('source', sourceQuota); take('config', configQuota); take('rationale', rationaleQuota); + take('other', otherQuota); for (const card of eligible) { if (selected.length >= maxCards) break; if (selectedKeys.has(card.key)) continue; - pushCard(card); + selected.push(card); + selectedKeys.add(card.key); } return selected diff --git a/src/git-history.ts b/src/git-history.ts index 899a69d..24ecbe9 100644 --- a/src/git-history.ts +++ b/src/git-history.ts @@ -17,6 +17,8 @@ export type GitHistoryRecord = { subject: string; files: GitFileChange[]; paths: string[]; + /** Detected package/workspace for monorepo support */ + packageId?: string; }; export type MineHistoryOptions = { @@ -71,6 +73,8 @@ export function parseGitHistory(logOutput: string): GitHistoryRecord[] { const pushCurrent = (): void => { if (!current) return; current.paths = current.files.map(file => file.path); + // Detect package from first file path + current.packageId = detectPackage(current.paths[0] ?? ''); records.push(current); current = null; }; @@ -132,6 +136,13 @@ export function mineHistory({ repoPath, outPath }: MineHistoryOptions = {}): Min '--find-renames=50%' ]); records = parseGitHistory(logOutput); + // Deduplicate by SHA + const seenShas = new Set(); + records = records.filter(r => { + if (seenShas.has(r.sha)) return false; + seenShas.add(r.sha); + return true; + }); const jsonl = records.map(record => JSON.stringify(record)).join('\n') + (records.length ? '\n' : ''); ensureDir(cacheFile); fs.writeFileSync(cacheFile, jsonl, 'utf8'); @@ -153,3 +164,37 @@ export function mineHistory({ repoPath, outPath }: MineHistoryOptions = {}): Min jsonl }; } + +/** + * Detect package/workspace from a file path. + * Checks for common monorepo package roots. + */ +export function detectPackage(filePath: string): string { + // Check for packages/ prefix (npm workspaces) + const pkgMatch = filePath.match(/^(packages\/[^/]+)/); + if (pkgMatch) return pkgMatch[1]; + + // Check for apps/ prefix (turborepo style) + const appMatch = filePath.match(/^(apps\/[^/]+)/); + if (appMatch) return appMatch[1]; + + // Check for libs/ prefix (nx style) + const libMatch = filePath.match(/^(libs\/[^/]+)/); + if (libMatch) return libMatch[1]; + + // Root-level files + const parts = filePath.split('/'); + if (parts.length === 1) return ''; + return ''; // not detected in a known workspace layout +} + +/** + * Generate a stable dedup key for a git file change. + * Used to prevent duplicate history records. + */ +export function fileDedupKey(sha: string, filePath: string): string { + return crypto.createHash('sha256') + .update(`${sha}:${filePath}`) + .digest('hex') + .slice(0, 32); +} diff --git a/src/training.ts b/src/training.ts index 00d5a00..f38276b 100644 --- a/src/training.ts +++ b/src/training.ts @@ -17,6 +17,12 @@ export type DatasetOptions = { repoPath?: string; outPath?: string; includeRejected?: boolean; + /** + * When true, generate context-augmented examples: + * user gets card metadata + question, assistant gets answer + * instead of plain question -> answer. + */ + contextFormat?: boolean; }; export type DatasetResult = { @@ -28,6 +34,24 @@ export type DatasetResult = { counts: { qa: number; 'review-warning': number; 'risk-classification': number; negative: number }; }; +/** + * Format card metadata into a context block for context-augmented training. + */ +export function formatCardContext(card: InsightCard): string { + const lines = [ + ``, + ` Summary: ${card.suggestion.slice(0, 300)}`, + ]; + if (card.affectedFiles.length > 0) { + lines.push(` Files: ${card.affectedFiles.join(', ')}`); + } + if (card.packageId) { + lines.push(` Package: ${card.packageId}`); + } + lines.push(''); + return lines.join('\n'); +} + export function generateQa(card: InsightCard): DatasetExample[] { const examples: DatasetExample[] = []; const fileStr = card.affectedFiles.join(', '); @@ -273,9 +297,24 @@ export function generateDataset(options: DatasetOptions = {}): DatasetResult { const examples: DatasetExample[] = []; for (const card of acceptedCards) { - examples.push(...generateQa(card)); - examples.push(...generateReviewWarning(card)); - examples.push(...generateRiskClassification(card)); + const qaExamples = generateQa(card); + const reviewExamples = generateReviewWarning(card); + const riskExamples = generateRiskClassification(card); + + // Wrap in context format if requested + if (options.contextFormat) { + const contextBlock = formatCardContext(card); + for (const ex of [...qaExamples, ...reviewExamples, ...riskExamples]) { + ex.messages[0] = { + role: 'user', + content: `Use the following repo-arch card as context.\n\n${contextBlock}\n\n${ex.messages[0]!.content}`, + }; + } + } + + examples.push(...qaExamples); + examples.push(...reviewExamples); + examples.push(...riskExamples); } examples.push(...generateNegative(cards));