From 11c19f58fdd686238d9ddc508b76dd200ee5152c Mon Sep 17 00:00:00 2001 From: "Prof. Mohit Tiwari" Date: Mon, 14 Sep 2026 22:58:13 +0530 Subject: [PATCH 01/87] fix(repo-map): detect nested Next.js router roots --- .../src/symbols/detectFrameworkRoutes.ts | 64 +++++++++---------- 1 file changed, 30 insertions(+), 34 deletions(-) diff --git a/packages/repo-map/src/symbols/detectFrameworkRoutes.ts b/packages/repo-map/src/symbols/detectFrameworkRoutes.ts index 5d9a0a3..179ad8c 100644 --- a/packages/repo-map/src/symbols/detectFrameworkRoutes.ts +++ b/packages/repo-map/src/symbols/detectFrameworkRoutes.ts @@ -1,6 +1,26 @@ import type { WorkspaceScanResult, RepoMapRoute, RepoMapFile } from "@wma/core"; import { readFileSync } from "node:fs"; +function nextRouterRelativePath(relativePath: string): { router: "app" | "pages"; path: string } | null { + const segments = relativePath.split("/"); + for (let index = 0; index < segments.length; index += 1) { + if (segments[index] === "src" && (segments[index + 1] === "app" || segments[index + 1] === "pages")) { + return { router: segments[index + 1] as "app" | "pages", path: segments.slice(index + 2).join("/") }; + } + if (segments[index] === "app" || segments[index] === "pages") { + return { router: segments[index] as "app" | "pages", path: segments.slice(index + 1).join("/") }; + } + } + return null; +} + +function normalizeNextRoute(route: string): string { + return route + .replace(/\/index$/, "") + .replace(/\[\.\.\.(\w+)\]/g, ":$1*") + .replace(/\[(\w+)\]/g, ":$1"); +} + export function detectFrameworkRoutes( scanResult: WorkspaceScanResult, files: RepoMapFile[], @@ -11,55 +31,31 @@ export function detectFrameworkRoutes( for (const file of scanResult.files) { if (!pathSet.has(file.relativePath)) continue; const rp = file.relativePath.replace(/\\/g, "/"); + const routerPath = nextRouterRelativePath(rp); + if (!routerPath) continue; - // Next.js App Router (matches both root app/page.tsx and nested app/some/page.tsx) - const appPageMatch = rp.match(/^app\/(?:(.*)\/)?page\.(?:js|jsx|ts|tsx)$/); - if (appPageMatch) { - const route = appPageMatch[1] ?? ""; + if (routerPath.router === "app") { + const match = routerPath.path.match(/^(?:(.*)\/)?(page|layout|route)\.(?:js|jsx|ts|tsx)$/); + if (!match) continue; + const route = normalizeNextRoute(match[1] ?? ""); routes.push({ relativePath: rp, - routePattern: `/${route.replace(/\/$/, "")}`, - framework: "nextjs", - reason: `Next.js App Router page: ${rp}`, - }); - continue; - } - - const appLayoutMatch = rp.match(/^app\/(?:(.*)\/)?layout\.(?:js|jsx|ts|tsx)$/); - if (appLayoutMatch) { - const route = appLayoutMatch[1] ?? ""; - routes.push({ - relativePath: rp, - routePattern: `/${route.replace(/\/$/, "")}`, - framework: "nextjs", - reason: `Next.js App Router layout: ${rp}`, - }); - continue; - } - - const appRouteMatch = rp.match(/^app\/(?:(.*)\/)?route\.(?:js|ts)$/); - if (appRouteMatch) { - const route = appRouteMatch[1] ?? ""; - routes.push({ - relativePath: rp, - routePattern: `/${route.replace(/\/$/, "")}`, + routePattern: `/${route}`, framework: "nextjs", - reason: `Next.js App Router API route: ${rp}`, + reason: `Next.js App Router ${match[2]}: ${rp}`, }); continue; } - // Next.js Pages Router - const pagesMatch = rp.match(/^pages\/(.*)(?:\.js|\.jsx|\.ts|\.tsx)$/); + const pagesMatch = routerPath.path.match(/^(.*)(?:\.js|\.jsx|\.ts|\.tsx)$/); if (pagesMatch) { - const route = pagesMatch[1].replace(/\/index$/, "").replace(/\[\.\.\.(\w+)\]/g, ":$1*").replace(/\[(\w+)\]/g, ":$1"); + const route = normalizeNextRoute(pagesMatch[1]); routes.push({ relativePath: rp, routePattern: `/${route}`, framework: "nextjs", reason: `Next.js Pages Router: ${rp}`, }); - continue; } } From 0e48c7bc92e1169644367c83498c3ee641b7b2e1 Mon Sep 17 00:00:00 2001 From: "Prof. Mohit Tiwari" Date: Mon, 14 Sep 2026 22:58:30 +0530 Subject: [PATCH 02/87] test(repo-map): cover src and monorepo Next.js routes --- .../symbols/detectFrameworkRoutes.test.ts | 129 ++++-------------- 1 file changed, 30 insertions(+), 99 deletions(-) diff --git a/packages/repo-map/tests/symbols/detectFrameworkRoutes.test.ts b/packages/repo-map/tests/symbols/detectFrameworkRoutes.test.ts index e5e9efe..47320dd 100644 --- a/packages/repo-map/tests/symbols/detectFrameworkRoutes.test.ts +++ b/packages/repo-map/tests/symbols/detectFrameworkRoutes.test.ts @@ -7,116 +7,47 @@ import { tmpdir } from "node:os"; let tempDir: string; -beforeAll(() => { - tempDir = mkdtempSync(join(tmpdir(), "wma-routes-test-")); -}); - -afterAll(() => { - rmSync(tempDir, { recursive: true, force: true }); -}); +beforeAll(() => { tempDir = mkdtempSync(join(tmpdir(), "wma-routes-test-")); }); +afterAll(() => { rmSync(tempDir, { recursive: true, force: true }); }); function makeFileInfo(relativePath: string, extension: string, content?: string): WorkspaceFileInfo { - return { - path: join(tempDir, relativePath.replace(/\//g, "\\")), - relativePath: relativePath.replace(/\\/g, "/"), - extension, - language: extension === ".ts" ? "TypeScript" : extension === ".tsx" ? "TypeScript React" : "Unknown", - bytes: content ? Buffer.byteLength(content, "utf-8") : 100, - estimatedTokens: 25, - included: true, - riskFlags: [], - }; -} - -function makeRepoMapFile(relativePath: string): RepoMapFile { - return { - relativePath: relativePath.replace(/\\/g, "/"), - estimatedTokens: 100, - reason: "test", - priority: 5, - }; + return { path: join(tempDir, relativePath.replace(/\//g, "\\")), relativePath: relativePath.replace(/\\/g, "/"), extension, language: extension === ".ts" ? "TypeScript" : extension === ".tsx" ? "TypeScript React" : "Unknown", bytes: content ? Buffer.byteLength(content, "utf-8") : 100, estimatedTokens: 25, included: true, riskFlags: [] }; } +function makeRepoMapFile(relativePath: string): RepoMapFile { return { relativePath: relativePath.replace(/\\/g, "/"), estimatedTokens: 100, reason: "test", priority: 5 }; } +function scan(files: WorkspaceFileInfo[]): WorkspaceScanResult { return { rootPath: tempDir, scannedAt: new Date().toISOString(), totalFiles: files.length, includedFiles: files.length, excludedFiles: 0, totalBytes: files.length * 100, includedBytes: files.length * 100, totalEstimatedTokens: files.length * 25, includedTokens: files.length * 25, files, folders: [], languages: [], warnings: [], riskFiles: [] }; } describe("detectFrameworkRoutes", () => { - it("detects Next.js app routes", () => { - const files = [makeFileInfo("app/page.jsx", ".jsx"), makeFileInfo("app/api/hello/route.js", ".js")]; - const important = files.map((f) => makeRepoMapFile(f.relativePath)); - const scanResult: WorkspaceScanResult = { - rootPath: tempDir, - scannedAt: new Date().toISOString(), - totalFiles: 2, - includedFiles: 2, - excludedFiles: 0, - totalBytes: 200, - includedBytes: 200, - totalEstimatedTokens: 50, - includedTokens: 50, - files, - folders: [], - languages: [], - warnings: [], - riskFiles: [], - }; - const routes = detectFrameworkRoutes(scanResult, important); - expect(routes.length).toBeGreaterThanOrEqual(2); - expect(routes.some((r) => r.relativePath === "app/page.jsx")).toBe(true); - expect(routes.some((r) => r.relativePath === "app/api/hello/route.js")).toBe(true); + it("detects root, src and package-nested Next.js app routes relative to the router root", () => { + const files = [ + makeFileInfo("app/page.jsx", ".jsx"), + makeFileInfo("src/app/users/[id]/page.tsx", ".tsx"), + makeFileInfo("apps/dashboard/app/docs/page.tsx", ".tsx"), + makeFileInfo("packages/admin/src/app/api/users/route.ts", ".ts"), + ]; + const routes = detectFrameworkRoutes(scan(files), files.map((f) => makeRepoMapFile(f.relativePath))); + expect(routes.find((r) => r.relativePath === "app/page.jsx")?.routePattern).toBe("/"); + expect(routes.find((r) => r.relativePath === "src/app/users/[id]/page.tsx")?.routePattern).toBe("/users/:id"); + expect(routes.find((r) => r.relativePath === "apps/dashboard/app/docs/page.tsx")?.routePattern).toBe("/docs"); + expect(routes.find((r) => r.relativePath === "packages/admin/src/app/api/users/route.ts")?.routePattern).toBe("/api/users"); + }); + + it("detects src and package-nested Pages Router paths", () => { + const files = [makeFileInfo("src/pages/index.tsx", ".tsx"), makeFileInfo("apps/site/pages/blog/[slug].tsx", ".tsx")]; + const routes = detectFrameworkRoutes(scan(files), files.map((f) => makeRepoMapFile(f.relativePath))); + expect(routes.find((r) => r.relativePath === "src/pages/index.tsx")?.routePattern).toBe("/"); + expect(routes.find((r) => r.relativePath === "apps/site/pages/blog/[slug].tsx")?.routePattern).toBe("/blog/:slug"); }); - it("detects Next.js pages routes", () => { - const files = [makeFileInfo("pages/index.tsx", ".tsx")]; - const important = files.map((f) => makeRepoMapFile(f.relativePath)); - const scanResult: WorkspaceScanResult = { - rootPath: tempDir, - scannedAt: new Date().toISOString(), - totalFiles: 1, - includedFiles: 1, - excludedFiles: 0, - totalBytes: 100, - includedBytes: 100, - totalEstimatedTokens: 25, - includedTokens: 25, - files, - folders: [], - languages: [], - warnings: [], - riskFiles: [], - }; - const routes = detectFrameworkRoutes(scanResult, important); - expect(routes.some((r) => r.framework === "nextjs")).toBe(true); + it("does not treat an unrelated app directory without Next.js route files as a route", () => { + const files = [makeFileInfo("lib/app/helpers.ts", ".ts")]; + expect(detectFrameworkRoutes(scan(files), files.map((f) => makeRepoMapFile(f.relativePath)))).toEqual([]); }); it("detects Express-style routes from file content", () => { - const content = ` -const app = express(); -app.get("/api/health", (req, res) => {}); -app.post("/api/data", (req, res) => {}); -const router = Router(); -router.get("/users", (req, res) => {}); -`; + const content = `const app = express();\napp.get("/api/health", (req, res) => {});\napp.post("/api/data", (req, res) => {});\nconst router = Router();\nrouter.get("/users", (req, res) => {});`; writeFileSync(join(tempDir, "server.ts"), content, "utf-8"); const files = [makeFileInfo("server.ts", ".ts", content)]; - const important = files.map((f) => makeRepoMapFile(f.relativePath)); - const scanResult: WorkspaceScanResult = { - rootPath: tempDir, - scannedAt: new Date().toISOString(), - totalFiles: 1, - includedFiles: 1, - excludedFiles: 0, - totalBytes: Buffer.byteLength(content, "utf-8"), - includedBytes: Buffer.byteLength(content, "utf-8"), - totalEstimatedTokens: 50, - includedTokens: 50, - files, - folders: [], - languages: [], - warnings: [], - riskFiles: [], - }; - const routes = detectFrameworkRoutes(scanResult, important); - const patterns = routes.map((r) => r.routePattern); - expect(patterns).toContain("/api/health"); - expect(patterns).toContain("/api/data"); - expect(patterns).toContain("/users"); + const patterns = detectFrameworkRoutes(scan(files), files.map((f) => makeRepoMapFile(f.relativePath))).map((r) => r.routePattern); + expect(patterns).toContain("/api/health"); expect(patterns).toContain("/api/data"); expect(patterns).toContain("/users"); }); }); From f0d3f552abc2c856f75c8702abb8f607d29f7ef3 Mon Sep 17 00:00:00 2001 From: "Prof. Mohit Tiwari" Date: Mon, 14 Sep 2026 23:04:51 +0530 Subject: [PATCH 03/87] fix: harden nested Next.js route normalization --- .../src/symbols/detectFrameworkRoutes.ts | 55 +++++++++++++------ 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/packages/repo-map/src/symbols/detectFrameworkRoutes.ts b/packages/repo-map/src/symbols/detectFrameworkRoutes.ts index 179ad8c..979b646 100644 --- a/packages/repo-map/src/symbols/detectFrameworkRoutes.ts +++ b/packages/repo-map/src/symbols/detectFrameworkRoutes.ts @@ -1,26 +1,48 @@ import type { WorkspaceScanResult, RepoMapRoute, RepoMapFile } from "@wma/core"; import { readFileSync } from "node:fs"; -function nextRouterRelativePath(relativePath: string): { router: "app" | "pages"; path: string } | null { +function nextRouterRelativePath(relativePath: string, pathSet: Set): { router: "app" | "pages"; path: string } | null { const segments = relativePath.split("/"); for (let index = 0; index < segments.length; index += 1) { - if (segments[index] === "src" && (segments[index + 1] === "app" || segments[index + 1] === "pages")) { - return { router: segments[index + 1] as "app" | "pages", path: segments.slice(index + 2).join("/") }; - } - if (segments[index] === "app" || segments[index] === "pages") { - return { router: segments[index] as "app" | "pages", path: segments.slice(index + 1).join("/") }; - } + const isSrcRouter = segments[index] === "src" && (segments[index + 1] === "app" || segments[index + 1] === "pages"); + const isDirectRouter = segments[index] === "app" || segments[index] === "pages"; + if (!isSrcRouter && !isDirectRouter) continue; + + const routerIndex = isSrcRouter ? index + 1 : index; + const rootSegments = segments.slice(0, isSrcRouter ? index : routerIndex); + const root = rootSegments.join("/"); + const verifiedRoot = root === "" || pathSet.has(`${root}/package.json`) || ["next.config.js", "next.config.mjs", "next.config.ts"].some((name) => pathSet.has(root ? `${root}/${name}` : name)); + if (!verifiedRoot) continue; + + return { + router: segments[routerIndex] as "app" | "pages", + path: segments.slice(routerIndex + 1).join("/"), + }; } return null; } -function normalizeNextRoute(route: string): string { +function normalizeDynamicSegments(route: string): string { return route - .replace(/\/index$/, "") + .replace(/\[\[\.\.\.(\w+)\]\]/g, ":$1*?") .replace(/\[\.\.\.(\w+)\]/g, ":$1*") .replace(/\[(\w+)\]/g, ":$1"); } +function normalizeAppRoute(route: string): string | null { + const segments = route.split("/").filter(Boolean); + if (segments.some((segment) => segment.startsWith("_"))) return null; + return normalizeDynamicSegments(segments.filter((segment) => !/^\(.+\)$/.test(segment)).join("/")); +} + +function normalizePagesRoute(route: string): string | null { + const segments = route.split("/").filter(Boolean); + const leaf = segments.at(-1); + if (leaf && ["_app", "_document", "_error"].includes(leaf)) return null; + if (leaf === "index") segments.pop(); + return normalizeDynamicSegments(segments.join("/")); +} + export function detectFrameworkRoutes( scanResult: WorkspaceScanResult, files: RepoMapFile[], @@ -31,13 +53,14 @@ export function detectFrameworkRoutes( for (const file of scanResult.files) { if (!pathSet.has(file.relativePath)) continue; const rp = file.relativePath.replace(/\\/g, "/"); - const routerPath = nextRouterRelativePath(rp); + const routerPath = nextRouterRelativePath(rp, pathSet); if (!routerPath) continue; if (routerPath.router === "app") { const match = routerPath.path.match(/^(?:(.*)\/)?(page|layout|route)\.(?:js|jsx|ts|tsx)$/); if (!match) continue; - const route = normalizeNextRoute(match[1] ?? ""); + const route = normalizeAppRoute(match[1] ?? ""); + if (route === null) continue; routes.push({ relativePath: rp, routePattern: `/${route}`, @@ -49,7 +72,8 @@ export function detectFrameworkRoutes( const pagesMatch = routerPath.path.match(/^(.*)(?:\.js|\.jsx|\.ts|\.tsx)$/); if (pagesMatch) { - const route = normalizeNextRoute(pagesMatch[1]); + const route = normalizePagesRoute(pagesMatch[1]); + if (route === null) continue; routes.push({ relativePath: rp, routePattern: `/${route}`, @@ -68,12 +92,7 @@ export function detectFrameworkRoutes( const routeRegex = /(?:app|router)\.(get|post|put|delete|patch|use)\s*\(\s*['"`]([^'"`]+)['"`]/g; let m: RegExpExecArray | null; while ((m = routeRegex.exec(content)) !== null) { - routes.push({ - relativePath: file.relativePath, - routePattern: m[2], - framework: "express", - reason: `${m[1].toUpperCase()} ${m[2]}`, - }); + routes.push({ relativePath: file.relativePath, routePattern: m[2], framework: "express", reason: `${m[1].toUpperCase()} ${m[2]}` }); } } catch { // Skip files that can't be read From feaf079d16687792b6de26284c0eb06c4638804d Mon Sep 17 00:00:00 2001 From: "P.R.S" Date: Tue, 15 Sep 2026 19:32:06 +0530 Subject: [PATCH 04/87] fix(recommender): correct cached-token double-counting * fix(recommender): correct cached-token double-counting in estimateCost() * fix(recommender): reject negative cachedInputTokens in estimateCost() --- packages/recommender/src/estimateCost.ts | 9 ++- .../recommender/tests/estimateCost.test.ts | 81 ++++++++++++++++++- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/packages/recommender/src/estimateCost.ts b/packages/recommender/src/estimateCost.ts index b5b285f..0a4100d 100644 --- a/packages/recommender/src/estimateCost.ts +++ b/packages/recommender/src/estimateCost.ts @@ -10,7 +10,14 @@ export interface EstimateCostOptions { export function estimateCost(options: EstimateCostOptions): CostEstimate { const { model, inputTokens, outputTokens, cachedInputTokens = 0 } = options; - const inputCost = (inputTokens / 1_000_000) * model.inputPricePerMillion; + if (cachedInputTokens < 0 || cachedInputTokens > inputTokens) { + throw new RangeError( + `cachedInputTokens (${cachedInputTokens}) must be between 0 and inputTokens (${inputTokens})`, + ); + } + + const nonCachedInputTokens = inputTokens - cachedInputTokens; + const inputCost = (nonCachedInputTokens / 1_000_000) * model.inputPricePerMillion; const cachedInputCost = (cachedInputTokens / 1_000_000) * (model.cachedInputPricePerMillion ?? model.inputPricePerMillion); const outputCost = (outputTokens / 1_000_000) * model.outputPricePerMillion; const totalCost = inputCost + cachedInputCost + outputCost; diff --git a/packages/recommender/tests/estimateCost.test.ts b/packages/recommender/tests/estimateCost.test.ts index f78fc57..91c461b 100644 --- a/packages/recommender/tests/estimateCost.test.ts +++ b/packages/recommender/tests/estimateCost.test.ts @@ -39,8 +39,9 @@ describe("estimateCost", () => { outputTokens: 100_000, cachedInputTokens: 500_000, }); + expect(result.inputCost).toBe(1.25); expect(result.cachedInputCost).toBe(0.625); - expect(result.totalCost).toBeCloseTo(2.5 + 0.625 + 1.0, 4); + expect(result.totalCost).toBe(1.25 + 0.625 + 1.0); }); it("should handle free model with zero prices", () => { @@ -69,7 +70,9 @@ describe("estimateCost", () => { outputTokens: 100_000, cachedInputTokens: 200_000, }); + expect(result.inputCost).toBe(2.0); expect(result.cachedInputCost).toBe(0.5); + expect(result.totalCost).toBe(2.0 + 0.5 + 1.0); }); it("should default cachedInputTokens to 0", () => { @@ -100,6 +103,82 @@ describe("estimateCost", () => { expect(result.inputCost).toBeGreaterThan(0); expect(result.outputCost).toBeGreaterThan(0); }); + + it("should charge zero cached cost when cachedInputTokens is 0", () => { + const result = estimateCost({ + model: testModel, + inputTokens: 200_000, + outputTokens: 40_000, + cachedInputTokens: 0, + }); + expect(result.inputCost).toBe(0.5); + expect(result.cachedInputCost).toBe(0); + expect(result.totalCost).toBe(0.5 + 0.4); + }); + + it("should correctly split cached and non-cached input tokens", () => { + const result = estimateCost({ + model: testModel, + inputTokens: 100_000, + outputTokens: 20_000, + cachedInputTokens: 60_000, + }); + expect(result.inputCost).toBe(0.1); + expect(result.cachedInputCost).toBe(0.075); + expect(result.outputCost).toBe(0.2); + expect(result.totalCost).toBe(0.1 + 0.075 + 0.2); + }); + + it("should treat fully cached input as zero input cost", () => { + const result = estimateCost({ + model: testModel, + inputTokens: 80_000, + outputTokens: 10_000, + cachedInputTokens: 80_000, + }); + expect(result.inputCost).toBe(0); + expect(result.cachedInputCost).toBe(0.1); + expect(result.outputCost).toBe(0.1); + expect(result.totalCost).toBe(0.2); + }); + + it("should throw when cachedInputTokens exceeds inputTokens", () => { + expect(() => + estimateCost({ + model: testModel, + inputTokens: 50_000, + outputTokens: 10_000, + cachedInputTokens: 60_000, + }), + ).toThrow(RangeError); + expect(() => + estimateCost({ + model: testModel, + inputTokens: 50_000, + outputTokens: 10_000, + cachedInputTokens: 60_000, + }), + ).toThrow("cachedInputTokens (60000) must be between 0 and inputTokens (50000)"); + }); + + it("should throw when cachedInputTokens is negative", () => { + expect(() => + estimateCost({ + model: testModel, + inputTokens: 100_000, + outputTokens: 10_000, + cachedInputTokens: -5000, + }), + ).toThrow(RangeError); + expect(() => + estimateCost({ + model: testModel, + inputTokens: 100_000, + outputTokens: 10_000, + cachedInputTokens: -5000, + }), + ).toThrow("cachedInputTokens (-5000) must be between 0 and inputTokens (100000)"); + }); }); describe("estimateSessionCost", () => { From 19614ff29e0458affadeb141af5a252e925f8d78 Mon Sep 17 00:00:00 2001 From: Divyanshu Kumar Date: Tue, 15 Sep 2026 19:32:09 +0530 Subject: [PATCH 05/87] fix(scanner): classify generated assets correctly * fix(scanner): detect minified and bundled files as generated isGeneratedFile compared path.extname() against compound suffixes like .min.js, but extname only returns the final extension (.js), so the check never matched. Generated assets outside build directories were counted as source, inflating workspace token counts and cost estimates. Match against the basename suffix instead, and correct the test that was asserting the broken behavior. Fixes #7 * test(scanner): add scan-level regression for generated file exclusion Adds a scanWorkspace()-level test covering the outcome from #7: a generated asset outside a build directory (src/vendor.min.js) is flagged as generated, excluded from the workspace, and its tokens are omitted from includedTokens. This fails against the pre-fix classifier and passes against it, so it guards the regression rather than testing classifyFile() in isolation. --- packages/scanner/src/classifyFile.ts | 9 +++++---- packages/scanner/tests/classifyFile.test.ts | 16 +++++++++++++++- packages/scanner/tests/scanWorkspace.test.ts | 17 +++++++++++++++++ 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/packages/scanner/src/classifyFile.ts b/packages/scanner/src/classifyFile.ts index f445eb2..5b27fcd 100644 --- a/packages/scanner/src/classifyFile.ts +++ b/packages/scanner/src/classifyFile.ts @@ -57,7 +57,7 @@ const EXTENSION_LANGUAGE_MAP: Record = { ".lock": "Lockfile", }; -const GENERATED_EXTENSIONS = new Set([".min.js", ".min.css", ".bundle.js", ".chunk.js"]); +const GENERATED_SUFFIXES = [".min.js", ".min.css", ".bundle.js", ".chunk.js"]; const BINARY_EXTENSIONS = new Set([ ".png", ".jpg", ".jpeg", ".gif", ".ico", ".webp", ".mp4", ".mp3", ".woff", ".woff2", ".ttf", ".eot", @@ -105,9 +105,10 @@ function isSecretFile(relativePath: string): boolean { } function isGeneratedFile(relativePath: string): boolean { - const ext = path.extname(relativePath); - if (GENERATED_EXTENSIONS.has(ext)) return true; - return false; + // path.extname() only returns the final extension (".js" for "app.min.js"), + // so these compound suffixes have to be matched against the whole basename. + const name = path.basename(relativePath).toLowerCase(); + return GENERATED_SUFFIXES.some(suffix => name.endsWith(suffix)); } function isLockfile(relativePath: string): boolean { diff --git a/packages/scanner/tests/classifyFile.test.ts b/packages/scanner/tests/classifyFile.test.ts index 145c934..fe1cc1a 100644 --- a/packages/scanner/tests/classifyFile.test.ts +++ b/packages/scanner/tests/classifyFile.test.ts @@ -65,7 +65,21 @@ describe("classifyFile", () => { it("should classify generated extensions", () => { const result = classifyFile("script.min.js", 1000); - expect(result.isGenerated).toBe(false); + expect(result.isGenerated).toBe(true); + expect(result.riskFlags).toContain("generated"); + }); + + it("should flag minified and bundled assets outside build directories", () => { + for (const file of ["public/vendor.min.js", "assets/app.bundle.js", "static/js/main.chunk.js", "web/styles.min.css"]) { + const result = classifyFile(file, 1000); + expect(result.isGenerated, file).toBe(true); + expect(result.riskFlags, file).toContain("generated"); + } + }); + + it("should not treat ordinary sources as generated", () => { + expect(classifyFile("src/bundler.js", 1000).isGenerated).toBe(false); + expect(classifyFile("src/minimap.css", 1000).isGenerated).toBe(false); }); it("should detect database-dump risk", () => { diff --git a/packages/scanner/tests/scanWorkspace.test.ts b/packages/scanner/tests/scanWorkspace.test.ts index 2f7bde1..423108e 100644 --- a/packages/scanner/tests/scanWorkspace.test.ts +++ b/packages/scanner/tests/scanWorkspace.test.ts @@ -80,6 +80,23 @@ describe("scanWorkspace traversal", () => { }); expect(result.files[0].estimatedTokens).toBe(7); }); + + it("excludes generated assets outside build directories from the workspace total", async () => { + const root = await tempDirectory("tcalc-generated-"); + const source = path.join(root, "src"); + await mkdir(source); + await writeFile(path.join(source, "index.ts"), "export const value = 1;"); + await writeFile(path.join(source, "vendor.min.js"), `${"a".repeat(2000)};`); + + const result = await scanWorkspace({ rootPath: root }); + + const vendor = result.files.find((file) => file.relativePath === "src/vendor.min.js"); + const index = result.files.find((file) => file.relativePath === "src/index.ts"); + expect(vendor?.riskFlags).toContain("generated"); + expect(vendor?.included).toBe(false); + expect(vendor?.estimatedTokens).toBeGreaterThan(0); + expect(result.includedTokens).toBe(index?.estimatedTokens); + }); }); async function tempDirectory(prefix: string): Promise { From c6a7971023ae112e8d366266fd68e0ddae5d03b6 Mon Sep 17 00:00:00 2001 From: Divyanshu Kumar Date: Tue, 15 Sep 2026 19:32:13 +0530 Subject: [PATCH 06/87] fix(dashboard): protect saved report reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(dashboard): require authentication for saved report reads GET /api/reports and GET /api/reports/:id returned data without checking the bearer token that POST already required. Anyone on a non-loopback host could enumerate and read saved reports, workspace paths and file-level metadata included. Reads now need the same bearer token as uploads. The dashboard UI is a static export, so it can't embed the token — instead, browser reads exchange the token once for an opaque session id at POST /api/session, returned as an HttpOnly, SameSite=Strict cookie. That cookie only grants reads. Uploads stay bearer-only, so a cross-site page can't use the cookie to forge a write. Added TCALC_DASHBOARD_PUBLIC_READ for deployments that want reports readable without authentication, opt-in only. Also moved token comparison into one shared helper using crypto.timingSafeEqual, replacing the non-constant-time check the POST path had before. Fixes #8 * fix(dashboard): fail closed when the token is unset, even with public read on denyRead() checked publicRead before checking whether a token was configured, so removing TCALC_DASHBOARD_TOKEN while TCALC_DASHBOARD_PUBLIC_READ was still set left reports readable anyway — the opposite of what the docs promise (unset token means 503 for everything). Reordered so the token check runs first: an unset token now always returns 503 for reads, regardless of publicRead. The flag only ever relaxes access while a token exists. Verified the new test actually guards this: reverting the ordering makes it the only failing test in the suite. * docs(dashboard): require HTTPS on non-loopback deployments HttpOnly and SameSite=Strict protect the session cookie from page scripts and cross-site misuse, but neither protects the token or cookie in transit. Over plain HTTP, both travel in cleartext and are readable by anyone on the network path. Added a note requiring TLS (typically via a reverse proxy) for any deployment where TCALC_DASHBOARD_HOST is not 127.0.0.1, and clarified that the server marks the cookie Secure once it detects HTTPS, directly or via x-forwarded-proto. --- apps/dashboard/components/ReportViewer.tsx | 57 ++++++++-- apps/dashboard/lib/docs.ts | 5 +- apps/dashboard/src/server.mjs | 124 ++++++++++++++++++++- apps/dashboard/tests/server.test.mjs | 118 +++++++++++++++++++- docs/team-dashboard.md | 48 ++++++++ 5 files changed, 334 insertions(+), 18 deletions(-) diff --git a/apps/dashboard/components/ReportViewer.tsx b/apps/dashboard/components/ReportViewer.tsx index cf5c229..e8dc20c 100644 --- a/apps/dashboard/components/ReportViewer.tsx +++ b/apps/dashboard/components/ReportViewer.tsx @@ -1,6 +1,6 @@ "use client"; -import { useEffect, useRef, useState } from "react"; +import { useEffect, useRef, useState, type FormEvent } from "react"; type ModelPick = { displayName?: string; modelId?: string; provider?: string; tier?: string; score?: number | { totalScore?: number } }; type TCalcReport = { @@ -46,13 +46,43 @@ export default function ReportViewer() { const [status, setStatus] = useState("Showing a safe example report."); const [error, setError] = useState(""); const [savedReports, setSavedReports] = useState([]); + const [needsUnlock, setNeedsUnlock] = useState(false); + const [unlockError, setUnlockError] = useState(""); - useEffect(() => { - fetch("/api/reports") - .then((response) => response.ok ? response.json() : Promise.reject()) - .then((items) => { if (Array.isArray(items)) setSavedReports(items); }) - .catch(() => undefined); - }, []); + // Saved reports are private by default. A 401 means the operator can unlock reads by exchanging + // the dashboard token for an HttpOnly session cookie, so the token itself never lives in this bundle. + const loadSavedReports = async () => { + try { + const response = await fetch("/api/reports", { credentials: "same-origin" }); + if (response.status === 401) { setNeedsUnlock(true); return; } + if (!response.ok) { setNeedsUnlock(false); return; } + const items = await response.json(); + if (Array.isArray(items)) setSavedReports(items); + setNeedsUnlock(false); + } catch { /* The optional report service is not running. */ } + }; + + useEffect(() => { void loadSavedReports(); }, []); + + const onUnlock = async (event: FormEvent) => { + event.preventDefault(); + const form = event.currentTarget; + const field = new FormData(form).get("token"); + setUnlockError(""); + try { + const response = await fetch("/api/session", { + method: "POST", + credentials: "same-origin", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ token: typeof field === "string" ? field : "" }), + }); + if (!response.ok) throw new Error(response.status === 401 ? "That dashboard token was not accepted." : "Could not start a session."); + form.reset(); + await loadSavedReports(); + } catch (caught) { + setUnlockError(caught instanceof Error ? caught.message : String(caught)); + } + }; const openReport = (value: unknown, message: string) => { if (!isReport(value)) throw new Error("This is not a valid TCalc report (schema 1.0.0). "); @@ -108,9 +138,20 @@ export default function ReportViewer() { } + {needsUnlock &&
+

Saved team reports

+

These reports are private. Enter the dashboard token to read them in this browser.

+
void onUnlock(event)}> + + + +
+ {unlockError ?

{unlockError}

: null} +
} {savedReports.length > 0 &&

Saved team reports

{savedReports.map((saved) =>
` : ""} + ${ + recommendation + ? `
Top recommendationsFit
${[ + recommendation.cheapestSufficient, + recommendation.balanced, + recommendation.highConfidence, + ] + .map( + (model) => + `
${escapeHtml(model.displayName)}${escapeHtml(model.tier.replaceAll("-", " "))}${Math.round(model.score.totalScore * 100)}
`, + ) + .join( + "", + )}
` + : "" + }
Workspace tools
- ` : `

${workspace ? "Ready to measure this workspace" : "Open a folder to begin"}

${workspace ? "Scan locally to see token usage, ranked models, and agent-ready actions." : "TCalc needs one open workspace folder. No source leaves your machine."}

${workspace ? `` : ""}
`} + ` + : `

${workspace ? "Ready to measure this workspace" : "Open a folder to begin"}

${workspace ? "Scan locally to see token usage, ranked models, and agent-ready actions." : "TCalc needs one open workspace folder. No source leaves your machine."}

${workspace ? `` : ""}
` + }
${scan ? `` : ""}
- + `; } } function compact(value: number): string { - return Intl.NumberFormat(undefined, { notation: "compact", maximumFractionDigits: 1 }).format(value); + return Intl.NumberFormat(undefined, { + notation: "compact", + maximumFractionDigits: 1, + }).format(value); } function relativeTime(value: string): string { @@ -90,5 +134,11 @@ function relativeTime(value: string): string { } function escapeHtml(value: string): string { - return value.replace(/[&<>"']/g, (character) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[character]!); + return value.replace( + /[&<>"']/g, + (character) => + ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[ + character + ]!, + ); } diff --git a/apps/vscode-extension/src/workspaceContext.ts b/apps/vscode-extension/src/workspaceContext.ts new file mode 100644 index 0000000..3a8735f --- /dev/null +++ b/apps/vscode-extension/src/workspaceContext.ts @@ -0,0 +1,100 @@ +import * as vscode from "vscode"; +import path from "node:path"; +import type { + ModelInfo, + RecommendationResult, + WorkspaceScanResult, +} from "@wma/core"; + +const WORKSPACE_STATE_KEY = "wma.workspaceState"; +const ACTIVE_ROOT_KEY = "wma.activeRootPath"; + +export interface RootWorkspaceState { + scan?: WorkspaceScanResult; + recommendation?: RecommendationResult | null; + models?: ModelInfo[]; + repoMapPath?: string; +} + +export function normalizeRootPath(rootPath: string): string { + const normalized = path.normalize(path.resolve(rootPath)); + return process.platform === "win32" ? normalized.toLowerCase() : normalized; +} + +export async function selectWorkspaceRoot( + rootPath?: string, +): Promise { + if (rootPath) return rootPath; + + const folders = vscode.workspace.workspaceFolders; + if (!folders || folders.length === 0) { + vscode.window.showErrorMessage("Open a workspace folder first."); + return undefined; + } + + if (folders.length === 1) return folders[0].uri.fsPath; + + const selected = await vscode.window.showQuickPick( + folders.map((folder) => ({ + label: folder.name, + description: folder.uri.fsPath, + rootPath: folder.uri.fsPath, + })), + { placeHolder: "Select a workspace folder" }, + ); + + return selected?.rootPath; +} + +export async function setActiveWorkspaceRoot( + context: vscode.ExtensionContext, + rootPath: string, +): Promise { + await context.workspaceState.update(ACTIVE_ROOT_KEY, rootPath); +} + +export function getActiveWorkspaceRoot( + context: vscode.ExtensionContext, +): string | undefined { + const activeRoot = context.workspaceState.get(ACTIVE_ROOT_KEY); + if (!activeRoot) return undefined; + + const folders = vscode.workspace.workspaceFolders; + if (!folders || folders.length === 0) return undefined; + + const activeKey = normalizeRootPath(activeRoot); + const currentFolder = folders.find( + (folder) => normalizeRootPath(folder.uri.fsPath) === activeKey, + ); + return currentFolder?.uri.fsPath ?? folders[0].uri.fsPath; +} + +export function getWorkspaceState( + context: vscode.ExtensionContext, + rootPath: string, +): RootWorkspaceState { + const allState = + context.workspaceState.get>( + WORKSPACE_STATE_KEY, + ) ?? {}; + return allState[normalizeRootPath(rootPath)] ?? {}; +} + +export async function updateWorkspaceState( + context: vscode.ExtensionContext, + rootPath: string, + update: Partial, +): Promise { + const allState = + context.workspaceState.get>( + WORKSPACE_STATE_KEY, + ) ?? {}; + const key = normalizeRootPath(rootPath); + await context.workspaceState.update(WORKSPACE_STATE_KEY, { + ...allState, + [key]: { + ...allState[key], + ...update, + }, + }); +} diff --git a/apps/vscode-extension/tests/multiRootCommands.test.ts b/apps/vscode-extension/tests/multiRootCommands.test.ts new file mode 100644 index 0000000..256c140 --- /dev/null +++ b/apps/vscode-extension/tests/multiRootCommands.test.ts @@ -0,0 +1,191 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const roots = { + alpha: "C:/work/alpha", + beta: "C:/work/beta", +}; + +const goals = new Map([ + [roots.alpha, "Goal A"], + [roots.beta, "Goal B"], +]); +const registeredCommands = new Map unknown>(); +const scanGoals: string[] = []; +const workspaceValues = new Map(); + +const vscodeMock = { + ConfigurationTarget: { WorkspaceFolder: "workspace-folder" }, + ProgressLocation: { Notification: "notification" }, + Uri: { + file: (fsPath: string) => ({ fsPath }), + }, + workspace: { + workspaceFolders: [ + { name: "alpha", uri: { fsPath: roots.alpha } }, + { name: "beta", uri: { fsPath: roots.beta } }, + ], + getConfiguration: vi.fn((_: string, resource?: { fsPath: string }) => { + const rootPath = resource?.fsPath; + return { + get: (key: string) => + (key === "defaultGoal" && rootPath + ? goals.get(rootPath) + : undefined) as T | undefined, + update: async (key: string, value: string, target: unknown) => { + expect(key).toBe("defaultGoal"); + expect(target).toBe(vscodeMock.ConfigurationTarget.WorkspaceFolder); + goals.set(rootPath!, value); + }, + }; + }), + }, + commands: { + registerCommand: vi.fn( + (id: string, handler: (...args: any[]) => unknown) => { + registeredCommands.set(id, handler); + return { dispose: vi.fn() }; + }, + ), + executeCommand: vi.fn(), + }, + window: { + showQuickPick: vi.fn(), + showInformationMessage: vi.fn(), + showWarningMessage: vi.fn(), + withProgress: vi.fn( + async ( + _options: unknown, + callback: (progress: unknown, token: unknown) => Promise, + ) => + callback( + { report: vi.fn() }, + { + isCancellationRequested: false, + onCancellationRequested: () => ({ dispose: vi.fn() }), + }, + ), + ), + }, +}; + +vi.mock("vscode", () => vscodeMock, { virtual: true }); +vi.mock("@wma/scanner", () => ({ + scanWorkspace: vi.fn(async ({ rootPath }: { rootPath: string }) => ({ + rootPath, + scannedAt: new Date().toISOString(), + includedFiles: 1, + includedTokens: 100, + totalEstimatedTokens: 100, + warnings: [], + })), +})); +vi.mock("@wma/model-catalog", () => ({ + applyModelProfile: (models: unknown[]) => models, + loadModelCatalog: () => ({ models: [{ id: "model-1" }] }), + validateModelCatalog: () => [], +})); +vi.mock("@wma/recommender", () => ({ + recommendModels: vi.fn(({ goal }: { goal: string }) => { + scanGoals.push(goal); + return null; + }), +})); +vi.mock("@wma/core", () => ({ + getActiveModelProfile: () => undefined, + loadTeamPolicy: async () => undefined, + WORKSPACE_GOALS: ["Goal A", "Goal B", "Goal B Updated"], +})); + +const { registerScanWorkspaceCommand } = + await import("../src/commands/scanWorkspaceCommand.js"); +const { registerSetWorkspaceGoalCommand } = + await import("../src/commands/setWorkspaceGoalCommand.js"); +const { TCalcSidebarProvider } = + await import("../src/views/sidebarProvider.js"); +const { getWorkspaceState } = await import("../src/workspaceContext.js"); + +function createContext() { + return { + extensionUri: { fsPath: "C:/extension" }, + workspaceState: { + get: (key: string) => workspaceValues.get(key) as T | undefined, + update: async (key: string, value: unknown) => { + workspaceValues.set(key, value); + }, + }, + } as never; +} + +async function runCommand(id: string, ...args: any[]): Promise { + await registeredCommands.get(id)?.(...args); +} + +describe("VS Code multi-root commands", () => { + beforeEach(() => { + goals.set(roots.alpha, "Goal A"); + goals.set(roots.beta, "Goal B"); + workspaceValues.clear(); + registeredCommands.clear(); + scanGoals.length = 0; + vi.clearAllMocks(); + }); + + it("isolates goals, scans, state, and sidebar display per workspace root", async () => { + const context = createContext(); + registerSetWorkspaceGoalCommand(context); + registerScanWorkspaceCommand(context); + + vscodeMock.window.showQuickPick.mockResolvedValueOnce({ + label: "Goal A", + }); + await runCommand("workspaceModelAdvisor.setWorkspaceGoal", roots.alpha); + expect(vscodeMock.window.showQuickPick).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ label: "Goal A", description: "current" }), + ]), + { placeHolder: "Select workspace goal" }, + ); + + vscodeMock.window.showQuickPick.mockResolvedValueOnce({ + label: "Goal B Updated", + }); + await runCommand("workspaceModelAdvisor.setWorkspaceGoal", roots.beta); + expect(vscodeMock.window.showQuickPick.mock.calls[1][0]).toEqual( + expect.arrayContaining([ + expect.objectContaining({ label: "Goal B", description: "current" }), + ]), + ); + expect(goals.get(roots.alpha)).toBe("Goal A"); + expect(goals.get(roots.beta)).toBe("Goal B Updated"); + expect(vscodeMock.workspace.getConfiguration).toHaveBeenCalledWith("wma", { + fsPath: roots.beta, + }); + + await runCommand("workspaceModelAdvisor.scanWorkspace", roots.alpha); + await runCommand("workspaceModelAdvisor.scanWorkspace", roots.beta); + + expect(scanGoals).toEqual(["Goal A", "Goal B Updated"]); + expect(getWorkspaceState(context, roots.alpha).scan).toMatchObject({ + rootPath: roots.alpha, + }); + expect(getWorkspaceState(context, roots.beta).scan).toMatchObject({ + rootPath: roots.beta, + }); + + let sidebarHtml = ""; + const view = { + webview: { + options: undefined, + onDidReceiveMessage: vi.fn(), + set html(value: string) { + sidebarHtml = value; + }, + }, + } as never; + const sidebar = new TCalcSidebarProvider(context); + await runCommand("workspaceModelAdvisor.scanWorkspace", roots.beta); + sidebar.resolveWebviewView(view); + expect(sidebarHtml).toContain("Goal B Updated"); + expect(sidebarHtml).not.toContain("Goal A"); + }); +}); diff --git a/apps/vscode-extension/tests/workspaceContext.test.ts b/apps/vscode-extension/tests/workspaceContext.test.ts new file mode 100644 index 0000000..e744e9a --- /dev/null +++ b/apps/vscode-extension/tests/workspaceContext.test.ts @@ -0,0 +1,134 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const vscodeMock = { + workspace: { + workspaceFolders: undefined as + Array<{ name: string; uri: { fsPath: string } }> | undefined, + }, + window: { + showErrorMessage: vi.fn(), + showQuickPick: vi.fn(), + }, +}; + +vi.mock("vscode", () => vscodeMock, { virtual: true }); + +const { + getActiveWorkspaceRoot, + getWorkspaceState, + normalizeRootPath, + selectWorkspaceRoot, + updateWorkspaceState, +} = await import("../src/workspaceContext.js"); + +function createContext() { + const values = new Map(); + return { + workspaceState: { + get: (key: string) => values.get(key) as T | undefined, + update: async (key: string, value: unknown) => { + values.set(key, value); + }, + }, + } as never; +} + +describe("workspace context", () => { + beforeEach(() => { + vscodeMock.workspace.workspaceFolders = undefined; + vscodeMock.window.showErrorMessage.mockReset(); + vscodeMock.window.showQuickPick.mockReset(); + }); + + it("uses the only workspace root without showing a picker", async () => { + const root = { name: "alpha", uri: { fsPath: "C:/work/alpha" } }; + vscodeMock.workspace.workspaceFolders = [root]; + + await expect(selectWorkspaceRoot()).resolves.toBe(root.uri.fsPath); + expect(vscodeMock.window.showQuickPick).not.toHaveBeenCalled(); + }); + + it("returns the selected root from a multi-root workspace", async () => { + const roots = [ + { name: "alpha", uri: { fsPath: "C:/work/alpha" } }, + { name: "beta", uri: { fsPath: "C:/work/beta" } }, + ]; + vscodeMock.workspace.workspaceFolders = roots; + vscodeMock.window.showQuickPick.mockResolvedValue({ + rootPath: roots[1].uri.fsPath, + }); + + await expect(selectWorkspaceRoot()).resolves.toBe(roots[1].uri.fsPath); + expect(vscodeMock.window.showQuickPick).toHaveBeenCalledWith( + [ + { + label: "alpha", + description: "C:/work/alpha", + rootPath: "C:/work/alpha", + }, + { + label: "beta", + description: "C:/work/beta", + rootPath: "C:/work/beta", + }, + ], + { placeHolder: "Select a workspace folder" }, + ); + }); + + it("stops cleanly when root selection is cancelled or no workspace is open", async () => { + vscodeMock.workspace.workspaceFolders = [ + { name: "alpha", uri: { fsPath: "C:/work/alpha" } }, + { name: "beta", uri: { fsPath: "C:/work/beta" } }, + ]; + vscodeMock.window.showQuickPick.mockResolvedValue(undefined); + await expect(selectWorkspaceRoot()).resolves.toBeUndefined(); + + vscodeMock.workspace.workspaceFolders = undefined; + await expect(selectWorkspaceRoot()).resolves.toBeUndefined(); + expect(vscodeMock.window.showErrorMessage).toHaveBeenCalledWith( + "Open a workspace folder first.", + ); + }); + + it("keeps state independent for two normalized roots", async () => { + const context = createContext(); + const rootA = "C:/work/alpha"; + const rootB = "C:/work/beta"; + + await updateWorkspaceState(context, rootA, { + scan: { rootPath: rootA } as never, + repoMapPath: "alpha-map.md", + }); + await updateWorkspaceState(context, rootB, { + scan: { rootPath: rootB } as never, + repoMapPath: "beta-map.md", + }); + + expect(getWorkspaceState(context, rootA)).toMatchObject({ + scan: { rootPath: rootA }, + repoMapPath: "alpha-map.md", + }); + expect(getWorkspaceState(context, rootB)).toMatchObject({ + scan: { rootPath: rootB }, + repoMapPath: "beta-map.md", + }); + expect(getWorkspaceState(context, "C:/work/missing")).toEqual({}); + expect(normalizeRootPath(`${rootA}/`)).toBe(normalizeRootPath(rootA)); + }); + + it("falls back when the active root is no longer available", async () => { + const context = createContext(); + const fallbackRoot = { name: "beta", uri: { fsPath: "C:/work/beta" } }; + await context.workspaceState.update( + "wma.activeRootPath", + "C:/work/removed", + ); + vscodeMock.workspace.workspaceFolders = [fallbackRoot]; + + expect(getActiveWorkspaceRoot(context)).toBe(fallbackRoot.uri.fsPath); + + vscodeMock.workspace.workspaceFolders = undefined; + expect(getActiveWorkspaceRoot(context)).toBeUndefined(); + }); +}); From ec499f80d07c4a18edcd7fc11a299c95daedece6 Mon Sep 17 00:00:00 2001 From: kspraveen-07 Date: Sun, 11 Oct 2026 13:18:42 +0530 Subject: [PATCH 70/87] fix(repo-map): parse comma-separated Python imports (#183) * fix(repo-map): parse comma-separated Python imports * fix(repo-map): ignore comments in Python imports --------- Co-authored-by: Sandesh Prakash Dawkhar <107756643+Sandesh13fr@users.noreply.github.com> --- .../repo-map/src/symbols/parseFileSymbols.ts | 14 ++++- .../tests/symbols/extractSymbols.test.ts | 62 +++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/packages/repo-map/src/symbols/parseFileSymbols.ts b/packages/repo-map/src/symbols/parseFileSymbols.ts index f35b17e..8c4db3d 100644 --- a/packages/repo-map/src/symbols/parseFileSymbols.ts +++ b/packages/repo-map/src/symbols/parseFileSymbols.ts @@ -57,10 +57,20 @@ function extractJavaScriptImports(content: string, relativePath: string, ext: st function extractPythonImports(content: string, relativePath: string): RepoMapImport[] { const imports: RepoMapImport[] = []; - const importRegex = /^import\s+(\S+)/gm; + const importRegex = /^import[ \t]+([^#\r\n]*?)[ \t]*(?:#.*)?$/gm; let m: RegExpExecArray | null; + while ((m = importRegex.exec(content)) !== null) { - imports.push({ source: m[1], relativePath, kind: "import" }); + const modules = m[1].split(",").map((item) => { + const moduleName = item.trim().split(/\s+as\s+/)[0]; + return moduleName; + }); + + for (const source of modules) { + if (source) { + imports.push({ source, relativePath, kind: "import" }); + } + } } const fromImportRegex = /^from\s+(\S+)\s+import/gm; diff --git a/packages/repo-map/tests/symbols/extractSymbols.test.ts b/packages/repo-map/tests/symbols/extractSymbols.test.ts index 447baa4..2d67807 100644 --- a/packages/repo-map/tests/symbols/extractSymbols.test.ts +++ b/packages/repo-map/tests/symbols/extractSymbols.test.ts @@ -177,6 +177,68 @@ async def fetch(): expect(result.imports.some((i) => i.source === "os")).toBe(true); expect(result.imports.some((i) => i.source === "typing")).toBe(true); }); + it("parses comma-separated Python imports and aliases", () => { + const content = ` +import os, sys +import alpha as a, beta as b +`; + const f = join(tempDir, "imports.py"); + writeFileSync(f, content, "utf-8"); + const file = makeFileInfo("src/imports.py", ".py", "imports.py"); + + const result = parseFileSymbols(file, true); + + expect(result.imports.map((i) => i.source)).toEqual([ + "os", + "sys", + "alpha", + "beta", + ]); + }); + + it("parses Python imports across multiple lines", () => { + const content = ` +import os +import sys, json +import alpha as a +import beta as b +`; + const f = join(tempDir, "multiline-imports.py"); + writeFileSync(f, content, "utf-8"); + const file = makeFileInfo("src/multiline-imports.py", ".py", "multiline-imports.py"); + + const result = parseFileSymbols(file, true); + + expect(result.imports.map((i) => i.source)).toEqual([ + "os", + "sys", + "json", + "alpha", + "beta", + ]); + }); + it("ignores comments when parsing Python imports", () => { + const content = ` +import os # comment +import os, sys # note, fake +import alpha as a, beta as b # aliases +import numpy.linalg as la # dotted module +`; + const f = join(tempDir, "commented-imports.py"); + writeFileSync(f, content, "utf-8"); + const file = makeFileInfo("src/commented-imports.py", ".py", "commented-imports.py"); + + const result = parseFileSymbols(file, true); + + expect(result.imports.map((i) => i.source)).toEqual([ + "os", + "os", + "sys", + "alpha", + "beta", + "numpy.linalg", + ]); + }); }); describe("parseFileSymbols - errors", () => { From 6cc7cdfab978d32beff07b39e6cc0ba418ba9b79 Mon Sep 17 00:00:00 2001 From: teajas Date: Sun, 11 Oct 2026 16:06:06 +0530 Subject: [PATCH 71/87] fix(repo-map): support JS/TS module extensions (#187) * fix(repo-map): parse JS and TS module extensions Closes #171. Add .mjs, .cjs, .mts, and .cts to repo-map symbol eligibility and dispatch. * test(repo-map): cover module-extension symbol extraction Add parser and workspace-eligibility regressions for .mjs, .cjs, .mts, and .cts. --- .../repo-map/src/symbols/extractSymbols.ts | 152 +++-- .../repo-map/src/symbols/parseFileSymbols.ts | 164 ++--- .../tests/symbols/extractSymbols.test.ts | 599 ++++++++++-------- 3 files changed, 486 insertions(+), 429 deletions(-) diff --git a/packages/repo-map/src/symbols/extractSymbols.ts b/packages/repo-map/src/symbols/extractSymbols.ts index 295fcba..f586fe2 100644 --- a/packages/repo-map/src/symbols/extractSymbols.ts +++ b/packages/repo-map/src/symbols/extractSymbols.ts @@ -1,69 +1,83 @@ -import type { WorkspaceScanResult, RepoMapSymbol, RepoMapImport, RepoMapRoute, RepoMapFile, RepoMapOptions } from "@wma/core"; -import { parseFileSymbols } from "./parseFileSymbols.js"; -import { detectFrameworkRoutes } from "./detectFrameworkRoutes.js"; -import { rankSymbols } from "./rankSymbols.js"; -import { resolveImports } from "./resolveImports.js"; - -export interface SymbolExtractionResult { - symbols: RepoMapSymbol[]; - imports: RepoMapImport[]; - routes: RepoMapRoute[]; - warnings: string[]; -} - -const SYMBOL_EXTS = new Set([".ts", ".tsx", ".js", ".jsx", ".py", ".java", ".kt", ".go", ".rs"]); - -export function extractSymbols( - scanResult: WorkspaceScanResult, - files: RepoMapFile[], - options: RepoMapOptions, -): SymbolExtractionResult { - const result: SymbolExtractionResult = { - symbols: [], - imports: [], - routes: [], - warnings: [], - }; - - if (options.enableSymbolExtraction === false) { - return result; - } - - const maxBytes = options.maxParseFileBytes ?? 256_000; - const maxSymbols = options.maxSymbols ?? 100; - const includeImports = options.includeImports !== false; - const includeRoutes = options.includeRoutes !== false; - - const fileSet = new Set(files.map((f) => f.relativePath)); - const eligible = scanResult.files.filter((f) => { - if (!fileSet.has(f.relativePath)) return false; - if (f.excludedReason) return false; - if (f.riskFlags.includes("generated")) return false; - if (f.riskFlags.includes("binary")) return false; - if (f.riskFlags.includes("secret")) return false; - if (f.riskFlags.includes("lockfile")) return false; - if (!SYMBOL_EXTS.has(f.extension)) return false; - if (f.bytes > maxBytes) return false; - return true; - }); - - for (const file of eligible) { - const fileSymbols = parseFileSymbols(file, includeImports); - result.symbols.push(...fileSymbols.symbols); - result.imports.push(...fileSymbols.imports); - - if (fileSymbols.error) { - result.warnings.push(`Failed to parse ${file.relativePath}: ${fileSymbols.error}`); - } - } - - if (includeRoutes) { - const routes = detectFrameworkRoutes(scanResult, files); - result.routes.push(...routes); - } - - result.symbols = rankSymbols(result.symbols, options.goal).slice(0, maxSymbols); - result.imports = resolveImports(result.imports, scanResult); - - return result; -} +import type { WorkspaceScanResult, RepoMapSymbol, RepoMapImport, RepoMapRoute, RepoMapFile, RepoMapOptions } from "@wma/core"; +import { parseFileSymbols } from "./parseFileSymbols.js"; +import { detectFrameworkRoutes } from "./detectFrameworkRoutes.js"; +import { rankSymbols } from "./rankSymbols.js"; +import { resolveImports } from "./resolveImports.js"; + +export interface SymbolExtractionResult { + symbols: RepoMapSymbol[]; + imports: RepoMapImport[]; + routes: RepoMapRoute[]; + warnings: string[]; +} + +const SYMBOL_EXTS = new Set([ + ".ts", + ".tsx", + ".mts", + ".cts", + ".js", + ".jsx", + ".mjs", + ".cjs", + ".py", + ".java", + ".kt", + ".go", + ".rs", +]); + +export function extractSymbols( + scanResult: WorkspaceScanResult, + files: RepoMapFile[], + options: RepoMapOptions, +): SymbolExtractionResult { + const result: SymbolExtractionResult = { + symbols: [], + imports: [], + routes: [], + warnings: [], + }; + + if (options.enableSymbolExtraction === false) { + return result; + } + + const maxBytes = options.maxParseFileBytes ?? 256_000; + const maxSymbols = options.maxSymbols ?? 100; + const includeImports = options.includeImports !== false; + const includeRoutes = options.includeRoutes !== false; + + const fileSet = new Set(files.map((f) => f.relativePath)); + const eligible = scanResult.files.filter((f) => { + if (!fileSet.has(f.relativePath)) return false; + if (f.excludedReason) return false; + if (f.riskFlags.includes("generated")) return false; + if (f.riskFlags.includes("binary")) return false; + if (f.riskFlags.includes("secret")) return false; + if (f.riskFlags.includes("lockfile")) return false; + if (!SYMBOL_EXTS.has(f.extension)) return false; + if (f.bytes > maxBytes) return false; + return true; + }); + + for (const file of eligible) { + const fileSymbols = parseFileSymbols(file, includeImports); + result.symbols.push(...fileSymbols.symbols); + result.imports.push(...fileSymbols.imports); + + if (fileSymbols.error) { + result.warnings.push(`Failed to parse ${file.relativePath}: ${fileSymbols.error}`); + } + } + + if (includeRoutes) { + const routes = detectFrameworkRoutes(scanResult, files); + result.routes.push(...routes); + } + + result.symbols = rankSymbols(result.symbols, options.goal).slice(0, maxSymbols); + result.imports = resolveImports(result.imports, scanResult); + + return result; +} diff --git a/packages/repo-map/src/symbols/parseFileSymbols.ts b/packages/repo-map/src/symbols/parseFileSymbols.ts index 8c4db3d..862debb 100644 --- a/packages/repo-map/src/symbols/parseFileSymbols.ts +++ b/packages/repo-map/src/symbols/parseFileSymbols.ts @@ -1,82 +1,82 @@ -import type { WorkspaceFileInfo, RepoMapSymbol, RepoMapImport } from "@wma/core"; -import { readFileSync } from "node:fs"; -import { extractTypeScriptImports, extractTypeScriptSymbols } from "./parsers/typescriptSymbols.js"; -import { extractJavaScriptSymbols } from "./parsers/javascriptSymbols.js"; -import { extractPythonSymbols } from "./parsers/pythonSymbols.js"; -import { extractCLikeSymbols } from "./parsers/cLikeSymbols.js"; - -export interface ParsedFileSymbols { - symbols: RepoMapSymbol[]; - imports: RepoMapImport[]; - error: string | null; -} - -export function parseFileSymbols(file: WorkspaceFileInfo, includeImports: boolean): ParsedFileSymbols { - try { - const content = readFileSync(file.path, "utf-8"); - const ext = file.extension; - - let symbols: RepoMapSymbol[] = []; - let imports: RepoMapImport[] = []; - let error: string | null = null; - - try { - if (ext === ".ts" || ext === ".tsx") { - symbols = extractTypeScriptSymbols(content, file.relativePath, ext); - if (includeImports) { - imports = extractTypeScriptImports(content, file.relativePath, ext); - } - } else if (ext === ".js" || ext === ".jsx") { - symbols = extractJavaScriptSymbols(content, file.relativePath, ext); - if (includeImports) { - imports = extractJavaScriptImports(content, file.relativePath, ext); - } - } else if (ext === ".py") { - symbols = extractPythonSymbols(content, file.relativePath); - if (includeImports) { - imports = extractPythonImports(content, file.relativePath); - } - } else if ([".java", ".kt", ".go", ".rs"].includes(ext)) { - symbols = extractCLikeSymbols(content, file.relativePath, ext); - } - } catch (parseErr) { - error = parseErr instanceof Error ? parseErr.message : String(parseErr); - return { symbols: [], imports: [], error }; - } - - return { symbols, imports, error: null }; - } catch (readErr: any) { - return { symbols: [], imports: [], error: readErr?.message ?? String(readErr) }; - } -} - -function extractJavaScriptImports(content: string, relativePath: string, ext: string): RepoMapImport[] { - return extractTypeScriptImports(content, relativePath, ext); -} - -function extractPythonImports(content: string, relativePath: string): RepoMapImport[] { - const imports: RepoMapImport[] = []; - - const importRegex = /^import[ \t]+([^#\r\n]*?)[ \t]*(?:#.*)?$/gm; - let m: RegExpExecArray | null; - - while ((m = importRegex.exec(content)) !== null) { - const modules = m[1].split(",").map((item) => { - const moduleName = item.trim().split(/\s+as\s+/)[0]; - return moduleName; - }); - - for (const source of modules) { - if (source) { - imports.push({ source, relativePath, kind: "import" }); - } - } - } - - const fromImportRegex = /^from\s+(\S+)\s+import/gm; - while ((m = fromImportRegex.exec(content)) !== null) { - imports.push({ source: m[1], relativePath, kind: "import" }); - } - - return imports; -} +import type { WorkspaceFileInfo, RepoMapSymbol, RepoMapImport } from "@wma/core"; +import { readFileSync } from "node:fs"; +import { extractTypeScriptImports, extractTypeScriptSymbols } from "./parsers/typescriptSymbols.js"; +import { extractJavaScriptSymbols } from "./parsers/javascriptSymbols.js"; +import { extractPythonSymbols } from "./parsers/pythonSymbols.js"; +import { extractCLikeSymbols } from "./parsers/cLikeSymbols.js"; + +export interface ParsedFileSymbols { + symbols: RepoMapSymbol[]; + imports: RepoMapImport[]; + error: string | null; +} + +export function parseFileSymbols(file: WorkspaceFileInfo, includeImports: boolean): ParsedFileSymbols { + try { + const content = readFileSync(file.path, "utf-8"); + const ext = file.extension; + + let symbols: RepoMapSymbol[] = []; + let imports: RepoMapImport[] = []; + let error: string | null = null; + + try { + if ([".ts", ".tsx", ".mts", ".cts"].includes(ext)) { + symbols = extractTypeScriptSymbols(content, file.relativePath, ext); + if (includeImports) { + imports = extractTypeScriptImports(content, file.relativePath, ext); + } + } else if ([".js", ".jsx", ".mjs", ".cjs"].includes(ext)) { + symbols = extractJavaScriptSymbols(content, file.relativePath, ext); + if (includeImports) { + imports = extractJavaScriptImports(content, file.relativePath, ext); + } + } else if (ext === ".py") { + symbols = extractPythonSymbols(content, file.relativePath); + if (includeImports) { + imports = extractPythonImports(content, file.relativePath); + } + } else if ([".java", ".kt", ".go", ".rs"].includes(ext)) { + symbols = extractCLikeSymbols(content, file.relativePath, ext); + } + } catch (parseErr) { + error = parseErr instanceof Error ? parseErr.message : String(parseErr); + return { symbols: [], imports: [], error }; + } + + return { symbols, imports, error: null }; + } catch (readErr: any) { + return { symbols: [], imports: [], error: readErr?.message ?? String(readErr) }; + } +} + +function extractJavaScriptImports(content: string, relativePath: string, ext: string): RepoMapImport[] { + return extractTypeScriptImports(content, relativePath, ext); +} + +function extractPythonImports(content: string, relativePath: string): RepoMapImport[] { + const imports: RepoMapImport[] = []; + + const importRegex = /^import[ \t]+([^#\r\n]*?)[ \t]*(?:#.*)?$/gm; + let m: RegExpExecArray | null; + + while ((m = importRegex.exec(content)) !== null) { + const modules = m[1].split(",").map((item) => { + const moduleName = item.trim().split(/\s+as\s+/)[0]; + return moduleName; + }); + + for (const source of modules) { + if (source) { + imports.push({ source, relativePath, kind: "import" }); + } + } + } + + const fromImportRegex = /^from\s+(\S+)\s+import/gm; + while ((m = fromImportRegex.exec(content)) !== null) { + imports.push({ source: m[1], relativePath, kind: "import" }); + } + + return imports; +} diff --git a/packages/repo-map/tests/symbols/extractSymbols.test.ts b/packages/repo-map/tests/symbols/extractSymbols.test.ts index 2d67807..dbd0945 100644 --- a/packages/repo-map/tests/symbols/extractSymbols.test.ts +++ b/packages/repo-map/tests/symbols/extractSymbols.test.ts @@ -1,278 +1,321 @@ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; -import { parseFileSymbols } from "../../src/symbols/parseFileSymbols.js"; -import type { WorkspaceFileInfo } from "@wma/core"; -import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; -import { join } from "node:path"; -import { tmpdir } from "node:os"; - -let tempDir: string; - -beforeAll(() => { - tempDir = mkdtempSync(join(tmpdir(), "wma-symbols-test-")); -}); - -afterAll(() => { - rmSync(tempDir, { recursive: true, force: true }); -}); - -function makeFileInfo(relativePath: string, ext: string, name: string, bytes = 1000): WorkspaceFileInfo { - return { - path: join(tempDir, name), - relativePath, - extension: ext, - language: ext === ".py" ? "Python" : ext === ".tsx" || ext === ".jsx" ? "TypeScript React" : "TypeScript", - bytes, - estimatedTokens: Math.floor(bytes / 4), - included: true, - riskFlags: [], - }; -} - -describe("parseFileSymbols - TypeScript", () => { - it("extracts TS functions, classes, interfaces, types", () => { - const content = ` -export function hello(name: string): string { - return "hello " + name; -} - -class MyClass { - doStuff() {} -} - -interface MyInterface { - name: string; -} - -type MyType = string | number; - -enum MyEnum { - A, - B, -} - -const myConst = 42; -`; - const f = join(tempDir, "test.ts"); - writeFileSync(f, content, "utf-8"); - const file = makeFileInfo("src/test.ts", ".ts", "test.ts"); - const result = parseFileSymbols(file, true); - expect(result.error).toBeNull(); - const names = result.symbols.map((s) => s.name); - expect(names).toContain("hello"); - expect(names).toContain("MyClass"); - expect(names).toContain("MyInterface"); - expect(names).toContain("MyType"); - expect(names).toContain("MyEnum"); - expect(names).toContain("myConst"); - expect(names).toContain("MyClass.doStuff"); - - const hello = result.symbols.find((s) => s.name === "hello")!; - expect(hello.exported).toBe(true); - expect(hello.kind).toBe("function"); - }); - - it("detects async functions", () => { - const content = "export async function fetchData() { return null; }"; - const f = join(tempDir, "test2.ts"); - writeFileSync(f, content, "utf-8"); - const file = makeFileInfo("src/test.ts", ".ts", "test2.ts"); - const result = parseFileSymbols(file, true); - const fn = result.symbols.find((s) => s.name === "fetchData"); - expect(fn).toBeDefined(); - expect(fn!.async).toBe(true); - }); - - it("extracts TSX React component-like symbols", () => { - const content = ` -const MyButton: React.FC = () => { return