scanner.js

process.env.UV_THREADPOOL_SIZE = 128; 

const { fdir } = require('fdir');
const fs = require('fs').promises;
const path = require('path');

const MINIMUM_FILE_SIZE = 1024 * 500; 
const BATCH_SIZE = 2500; // Processes files in chunks to prevent memory freezing

async function scanDirectory(dir, onProgress) {
	const normalizedTarget = path.resolve(dir);
	
	const api = new fdir().withFullPaths().crawl(normalizedTarget);
	const files = await api.withPromise(); 
	
	const totalFiles = files.length;
	let scannedCount = 0;

	const root = {
		name: path.basename(normalizedTarget) || normalizedTarget,
		path: normalizedTarget,
		value: 0,
		children: [],
		rolledUpSize: 0
	};

	const dirMap = new Map();
	dirMap.set(normalizedTarget, root);

	function getOrCreateDirNode(dirPath) {
		if (dirMap.has(dirPath)) return dirMap.get(dirPath);
		
		const parentPath = path.dirname(dirPath);
		if (parentPath === dirPath) return root; 
		
		const parentNode = getOrCreateDirNode(parentPath);
		
		const dirNode = {
			name: path.basename(dirPath),
			path: dirPath,
			value: 0,
			children: [],
			rolledUpSize: 0
		};
		
		parentNode.children.push(dirNode);
		dirMap.set(dirPath, dirNode);
		return dirNode;
	}

	const seenInodes = new Set();

	// The Batching Loop: Eliminates GC thrashing and IPC flooding
	for (let i = 0; i < totalFiles; i += BATCH_SIZE) {
		const batch = files.slice(i, i + BATCH_SIZE);

		await Promise.all(batch.map(async (filePath) => {
			try {
				// Use regular lstat if bigint isn't strictly required for your UI
				const stats = await fs.lstat(filePath); 
				
				const uniqueId = `${stats.dev}-${stats.ino}`;
				if (seenInodes.has(uniqueId)) return;
				seenInodes.add(uniqueId);

				const fileSize = Number(stats.size);
				const dirPath = path.dirname(filePath);
				const dirNode = getOrCreateDirNode(dirPath);
				
				if (fileSize > MINIMUM_FILE_SIZE) {
					dirNode.children.push({
						name: path.basename(filePath),
						path: filePath,
						value: fileSize
					});
				} else {
					// Fix: ensure rolledUpSize is initialized as a number
					dirNode.rolledUpSize = (dirNode.rolledUpSize || 0) + fileSize;
				}
			} catch (e) {
				// Locked files or permission errors
			} finally {
				scannedCount++;
			}
		}));

		// Fire UI update ONCE per batch. Passes data for your progress bar!
		if (onProgress) {
			onProgress({
				currentPath: batch[batch.length - 1], // Show the last file in the batch
				scanned: scannedCount,
				total: totalFiles
			});
		}
	}

	// Roll-up injection
	function injectRollups(node) {
		if (node.rolledUpSize > 0) {
			node.children.push({
				name: "📦 Small Files (< 500KB)",
				path: node.path, 
				value: node.rolledUpSize
			});
		}
		for (const child of node.children) {
			if (child.children) {
				injectRollups(child);
			}
		}
		delete node.rolledUpSize; 
	}
	
	injectRollups(root);

	// Pruning Logic
	const totalDiskSize = root.value + root.children.reduce((acc, cur) => acc + (cur.value || 0), 0);
	
	function bubbleUpSizes(node) {
		let size = node.value || 0;
		if (node.children) {
			for (const child of node.children) {
				size += bubbleUpSizes(child);
			}
		}
		node.totalSize = size;
		return size;
	}
	
	function pruneTree(node) {
		if (!node.children) return;
		
		if (node.totalSize / totalDiskSize < 0.001) {
			node.children = []; 
		} else {
			for (const child of node.children) {
				pruneTree(child);
			}
		}
	}

	bubbleUpSizes(root);
	pruneTree(root);

	return root;
}

module.exports = { scanDirectory };