diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 8098965..9b3b88d 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,7 +5,7 @@ Explain the problem, the approach, and any user-visible behavior. Link the issue ## Contribution prerequisite - [ ] I starred the [TCalc repository](https://github.com/Sandesh13fr/TCalc) before starting this contribution. -- [ ] I checked the issue and existing PRs for overlapping work. I understand that assignment is not required and any contributor may propose a focused solution. +- [ ] The linked issue is assigned to me, and this is my only open PR (drafts included). - [ ] This PR targets the `Development` branch (unless a maintainer explicitly approved another base). ## Affected areas @@ -31,6 +31,7 @@ List the exact commands run and their results. If a relevant check was not run, - [ ] `pnpm test` - [ ] `pnpm lint` - [ ] `pnpm typecheck` +- [ ] The `CI Success` status check passes - [ ] CLI/MCP smoke, VSIX packaging, or JetBrains verification as applicable ## Review notes diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8466be3..be84fdd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -308,3 +308,15 @@ jobs: - name: Check extension metadata run: pnpm check:extension-metadata + + ci-success: + name: CI Success + runs-on: ubuntu-latest + needs: [build, test, cli-smoke, vsix-package, no-telemetry, extension-metadata] + if: always() + steps: + - name: Verify all required jobs succeeded + if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }} + run: exit 1 + - name: All checks passed + run: echo "All required CI jobs passed." diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..d37c5b6 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,12 @@ +# Code of Conduct + +Everyone is welcome to participate, including people who are new to programming, Git, or open source. + +- Be patient, respectful, and specific. Critique code and ideas, not people. +- Ask questions freely and help others without ridicule or pressure. +- Do not harass, threaten, discriminate, post sexualized content, or share someone else's private information. +- Respect issue assignments and first-time contributor reservations. Do not pressure maintainers to merge unreviewed work or bypass checks. + +Maintainers may edit or remove harmful public content, ask participants to change their behavior, or restrict participation for serious or repeated violations. These expectations apply to repository issues, pull requests, reviews, and discussions. + +Do not publish sensitive details in an issue. Use GitHub's **Report content** / **Report abuse** controls for harassment or other platform violations. Report security vulnerabilities through the process in [SECURITY.md](./SECURITY.md), not in public threads. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e02711d..a8e195f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,22 @@ Thank you for contributing. TCalc is local-first: workspace source must not be uploaded or sent to an external service by default. +## Your first open-source contribution + +You are welcome here, even if you have never opened a pull request. Questions are part of the process, not a reason to hesitate. + +- Browse [available beginner tasks](https://github.com/Sandesh13fr/TCalc/issues?q=is%3Aissue%20is%3Aopen%20label%3A%22good%20first%20issue%22%20label%3Aup-for-grabs%20no%3Aassignee). +- `good first issue` means approachable work with a bounded scope. `up-for-grabs` means it is available; check the assignee and recent comments before starting. +- `first-timers-only` tasks are reserved for people making their first open-source pull request, not just their first TCalc contribution. Experienced contributors should choose another issue. +- Comment with a short approach and the test/check you plan to use. For documentation, one sentence is enough. Mention if this is your first open-source PR and ask for help with any unfamiliar step. +- Wait for assignment before working. If you are assigned, the maintainer will remove `up-for-grabs`. Please do not work on someone else's assigned issue. +- Fork the repository, create a branch from `Development`, make only the assigned change, and open a PR targeting `Development` with `Fixes #` in its description. +- Only one open PR per external contributor is allowed. Keep drafts on your own branch until your current PR is resolved. +- For a documentation-only task, you may use GitHub's web editor without installing the project. Preview Markdown and check the links and commands you changed. Say which checks you ran; the maintainer will still require the relevant CI checks before merging. +- If setup fails or feedback is confusing, comment on the issue or PR with the error and ask for the next step. Never post tokens, passwords, private workspace paths, or source from a private project. + +For a guided practice run, see [First Contributions](https://github.com/firstcontributions/first-contributions). Our approach follows [First Timers Only](https://www.firsttimersonly.com/): small tasks and patient help with the contribution process. Please follow our [Code of Conduct](./CODE_OF_CONDUCT.md). + ## Set up - Node.js 22 (the version used in CI) @@ -9,8 +25,11 @@ Thank you for contributing. TCalc is local-first: workspace source must not be u - JDK 21 if you change the JetBrains plugin ```bash -git clone https://github.com/Sandesh13fr/TCalc.git +git clone https://github.com//TCalc.git cd TCalc +git remote add upstream https://github.com/Sandesh13fr/TCalc.git +git fetch upstream +git switch -c my-first-fix upstream/Development corepack enable pnpm install --frozen-lockfile pnpm build @@ -42,6 +61,10 @@ Select **every** affected area in the PR template. A change in `packages/recomme 6. Fill out the PR template: affected areas, issue link, behavior change, tests run with results, documentation, and privacy or compatibility impact. If a check was not run, say why. Add screenshots or a short recording for visible web/editor changes. 7. Do not commit generated `dist/`, VSIX, `.next/`, or JetBrains `build/` files. +## Required status checks + +Pull requests targeting **`Development`** must pass the **CI Success** status check before they can be merged. This aggregate check ensures that the required CI jobs complete successfully. + ## Project layout ```text diff --git a/README.md b/README.md index f0af840..f477e02 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,8 @@ See the [VSIX installation guide](./docs/install-vsix.md) for the local package Build once, then scan, compare, or generate only the context you need: +Building from source needs **Node.js 22** and **pnpm 11.5.2**. For the fork, clone, and setup walkthrough, see [CONTRIBUTING.md](./CONTRIBUTING.md). + ```bash pnpm install pnpm build @@ -76,6 +78,40 @@ pnpm cli rules ./my-project --target codex --mode repo-map-first Other commands generate Markdown/JSON reports, validate model catalogs, create MCP configs, and start the local MCP server. Run `pnpm cli --help` for the complete command list. +### Ignore sources and precedence + +TCalc applies ignore patterns in this order, with later matching rules taking precedence under normal gitignore semantics: + +1. root `.gitignore`; +2. root `.cursorignore`; +3. root `.aiderignore`; +4. root `.continueignore`; +5. root `.tcalcignore`; +6. any explicit additional ignore files supplied by an integration; +7. nested `.gitignore` files, scoped to their containing directory; +8. workspace `exclude` patterns from TCalc configuration. + +Nested ignore discovery skips version-control metadata and directories already excluded by higher-level rules. As with Git, a file cannot be re-included when its parent directory remains ignored. File-size, binary, generated-file, secret, and other safety exclusions are applied separately after path ignores and cannot be weakened by a negated ignore rule. + +## Choose a privacy mode + +TCalc applies the same privacy setting in its CLI, VS Code UI, and shared configuration: + +| Mode | Recommendation guarantee | +| --- | --- | +| `local-first` (default) | Only catalog entries with `supportsLocal: true` are eligible. Cloud models and hybrid entries without a verified local execution path are rejected. If none qualify, TCalc reports that no eligible model is available instead of silently falling back to cloud execution. | +| `cloud-ok` | Cloud, hybrid, and local catalog entries may be recommended according to fit, quality, and cost. This expands recommendation eligibility; it does not upload source code or invoke a provider by itself. | + +Set `privacyMode` in the workspace's `.tcalc.json` or use the corresponding privacy selector exposed by the CLI/VS Code integration. For example: + +```json +{ + "privacyMode": "local-first" +} +``` + +Workspace scanning, token estimation, repo-map generation, and reports remain local in both modes. Selecting `cloud-ok` only permits cloud-capable models to appear in recommendations; any later provider execution and credentials are controlled by the user's chosen external tool. + ## Connect your coding agent TCalc can expose the same scan, recommendation, repo-map, rules, and report capabilities over stdio MCP. It also generates agent rule files and client configuration without uploading source code. @@ -129,6 +165,12 @@ pnpm cli mcp ## Project guides +### First contribution? Start here + +[![First-timers-only friendly](https://img.shields.io/badge/first--timers--only-friendly-blue.svg?style=flat-square)](https://www.firsttimersonly.com/) + +Browse [available beginner issues](https://github.com/Sandesh13fr/TCalc/issues?q=is%3Aissue%20is%3Aopen%20label%3A%22good%20first%20issue%22%20label%3Aup-for-grabs%20no%3Aassignee) and read the [first-contribution walkthrough](./CONTRIBUTING.md#your-first-open-source-contribution). Tasks labelled `first-timers-only` are reserved for your first open-source PR. Comment to request assignment before starting; maintainers can help with setup and review feedback. + ### Troubleshooting VS Code test output - Missing `catalogs/models.json` or `.tcalc/models.json` files are optional workspace overrides; TCalc uses the catalogue bundled with the extension when neither exists. diff --git a/apps/cli/src/commands/catalog.ts b/apps/cli/src/commands/catalog.ts index 2998c99..8af66f4 100644 --- a/apps/cli/src/commands/catalog.ts +++ b/apps/cli/src/commands/catalog.ts @@ -5,6 +5,8 @@ export interface CatalogValidateOptions { path?: string; format?: string; debug?: boolean; + maxAgeDays?: number; + now?: Date; } export async function executeCatalogFetch(url: string): Promise { @@ -26,7 +28,7 @@ export async function executeCatalogValidate(options: CatalogValidateOptions): P }; } - const errors = [...validateModelCatalog(catalog.models), ...validateCatalogFreshness(catalog)]; + const errors = [...validateModelCatalog(catalog.models), ...validateCatalogFreshness(catalog, options.maxAgeDays, options.now)]; if (errors.length > 0) { const msg = options.format === "json" ? JSON.stringify({ valid: false, catalogPath, modelCount: catalog.models.length, errors }, null, 2) diff --git a/apps/cli/src/commands/init.ts b/apps/cli/src/commands/init.ts new file mode 100644 index 0000000..7771210 --- /dev/null +++ b/apps/cli/src/commands/init.ts @@ -0,0 +1,149 @@ +import { access, writeFile, readFile } from "node:fs/promises"; +import path from "node:path"; +import { DEFAULT_CONFIG, type PrivacySetting, type WorkspaceGoal, type WmaConfig } from "@wma/core"; +import { resolveTargetPath } from "../utils/paths.js"; +import { CliError } from "../utils/errors.js"; + +export interface InitOptions { + target?: string; + goal?: string; + privacy?: string; + force?: boolean; + dryRun?: boolean; +} + +export interface DetectedStack { + primaryLanguage: "typescript" | "javascript" | "python" | "rust" | "go" | "generic"; + isMonorepo: boolean; + detectedFrameworks: string[]; + recommendedExcludes: string[]; + recommendedBudget: number; +} + +async function fileExists(filePath: string): Promise { + try { + await access(filePath); + return true; + } catch { + return false; + } +} + +export async function detectProjectStack(rootPath: string): Promise { + const hasPkgJson = await fileExists(path.join(rootPath, "package.json")); + const hasTsConfig = await fileExists(path.join(rootPath, "tsconfig.json")); + const hasPyProject = await fileExists(path.join(rootPath, "pyproject.toml")); + const hasRequirements = await fileExists(path.join(rootPath, "requirements.txt")); + const hasCargo = await fileExists(path.join(rootPath, "Cargo.toml")); + const hasGoMod = await fileExists(path.join(rootPath, "go.mod")); + const hasPnpmWorkspace = await fileExists(path.join(rootPath, "pnpm-workspace.yaml")); + const hasNx = await fileExists(path.join(rootPath, "nx.json")); + const hasTurbo = await fileExists(path.join(rootPath, "turbo.json")); + + const isMonorepo = hasPnpmWorkspace || hasNx || hasTurbo; + const detectedFrameworks: string[] = []; + const recommendedExcludes: string[] = [...DEFAULT_CONFIG.exclude]; + + let primaryLanguage: DetectedStack["primaryLanguage"] = "generic"; + + if (hasCargo) { + primaryLanguage = "rust"; + recommendedExcludes.push("target"); + } else if (hasGoMod) { + primaryLanguage = "go"; + recommendedExcludes.push("vendor", "bin"); + } else if (hasPyProject || hasRequirements) { + primaryLanguage = "python"; + recommendedExcludes.push(".venv", "venv", "__pycache__", ".pytest_cache", ".ruff_cache", "*.pyc"); + } else if (hasPkgJson || hasTsConfig) { + primaryLanguage = hasTsConfig ? "typescript" : "javascript"; + recommendedExcludes.push("dist", "build", ".next", ".turbo", "coverage", "storybook-static"); + + // Inspect package.json for frameworks + if (hasPkgJson) { + try { + const pkgRaw = await readFile(path.join(rootPath, "package.json"), "utf8"); + const pkg = JSON.parse(pkgRaw) as { dependencies?: Record; devDependencies?: Record }; + const allDeps = { ...pkg.dependencies, ...pkg.devDependencies }; + if (allDeps["next"]) detectedFrameworks.push("Next.js"); + if (allDeps["vite"]) detectedFrameworks.push("Vite"); + if (allDeps["react"]) detectedFrameworks.push("React"); + if (allDeps["vue"]) detectedFrameworks.push("Vue"); + if (allDeps["vitest"] || allDeps["jest"]) detectedFrameworks.push("Testing (Vitest/Jest)"); + } catch { + // Ignore JSON read errors + } + } + } + + // Deduplicate excludes + const uniqueExcludes = [...new Set(recommendedExcludes)]; + const recommendedBudget = isMonorepo ? 200_000 : 128_000; + + return { + primaryLanguage, + isMonorepo, + detectedFrameworks, + recommendedExcludes: uniqueExcludes, + recommendedBudget, + }; +} + +export async function executeInit(options: InitOptions): Promise { + const rootPath = resolveTargetPath(options.target); + const configPath = path.join(rootPath, ".workspace-model-advisor.json"); + + const stack = await detectProjectStack(rootPath); + + const goal = (options.goal as WorkspaceGoal) ?? DEFAULT_CONFIG.defaultGoal; + const privacy = (options.privacy as PrivacySetting) ?? (stack.primaryLanguage === "rust" ? "local-first" : DEFAULT_CONFIG.privacyMode); + + const newConfig: Partial = { + defaultGoal: goal, + privacyMode: privacy, + exclude: stack.recommendedExcludes, + tokenBudget: { + defaultContextBudget: stack.recommendedBudget, + maxFullWorkspaceScan: stack.recommendedBudget * 2, + warnAt: Math.round(stack.recommendedBudget * 0.8), + }, + }; + + const formattedJson = JSON.stringify(newConfig, null, 2); + + if (options.dryRun) { + return [ + `[DRY RUN] Inferred configuration for ${stack.primaryLanguage} project (${stack.isMonorepo ? "Monorepo" : "Single Package"}):`, + formattedJson, + ].join("\n"); + } + + const alreadyExists = await fileExists(configPath); + if (alreadyExists && !options.force) { + throw new CliError( + `Configuration file already exists at ${configPath}. Use --force to overwrite it.`, + ); + } + + await writeFile(configPath, formattedJson, "utf8"); + + const lines: string[] = [ + `Initialized TCalc workspace configuration!`, + `File written: ${configPath}`, + ``, + `Project Diagnostics:`, + `- Primary Language: ${stack.primaryLanguage}`, + `- Monorepo: ${stack.isMonorepo ? "Yes" : "No"}`, + stack.detectedFrameworks.length > 0 ? `- Frameworks: ${stack.detectedFrameworks.join(", ")}` : "", + `- Calibrated Context Budget: ${stack.recommendedBudget.toLocaleString()} tokens`, + `- Default Goal: ${goal}`, + `- Privacy Mode: ${privacy}`, + `- Tailored Excludes: ${stack.recommendedExcludes.length} rules active`, + ``, + `You can now run:`, + ` wma scan`, + ` wma recommend`, + ].filter(Boolean); + + return lines.join("\n"); +} diff --git a/apps/cli/src/commands/recommend.ts b/apps/cli/src/commands/recommend.ts index ffefff3..e48c8e2 100644 --- a/apps/cli/src/commands/recommend.ts +++ b/apps/cli/src/commands/recommend.ts @@ -36,11 +36,11 @@ export async function executeRecommend(options: RecommendOptions): Promise { try { recommendation = recommendModels({ models, + preferredModelIds: profile?.preferredModelIds, workspaceTokens: scanResult.includedTokens, goal, privacyMode: privacy, - preferredModelIds: profile?.preferredModelIds, }); - } catch { - // proceed without recommendations + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + scanResult.warnings.push(`Recommendations unavailable: ${reason}`); + if (options.debug) console.warn(`Recommendation failed: ${reason}`); } } diff --git a/apps/cli/src/commands/scan.ts b/apps/cli/src/commands/scan.ts index 2c51d17..2697d61 100644 --- a/apps/cli/src/commands/scan.ts +++ b/apps/cli/src/commands/scan.ts @@ -1,4 +1,5 @@ import { scanWorkspace } from "@wma/scanner"; +import { escapeMarkdownText, markdownCode } from "@wma/core"; import { formatScanTable, formatScanJson } from "../utils/output.js"; import { resolveTargetPath } from "../utils/paths.js"; import { loadWorkspaceConfig } from "../utils/loadWorkspaceConfig.js"; @@ -46,7 +47,7 @@ function formatScanMarkdown(scan: import("@wma/core").WorkspaceScanResult): stri const lines: string[] = []; lines.push("# Workspace Scan Report"); lines.push(""); - lines.push(`**Workspace Root:** \`${scan.rootPath}\``); + lines.push(`**Workspace Root:** ${markdownCode(scan.rootPath)}`); lines.push(`**Scanned At:** ${scan.scannedAt}`); lines.push(""); lines.push("## Summary"); @@ -72,7 +73,7 @@ function formatScanMarkdown(scan: import("@wma/core").WorkspaceScanResult): stri lines.push("| --- | ---:| ---:|"); for (const f of sortedFiles) { const pc = (scan.includedTokens > 0 ? (f.estimatedTokens / scan.includedTokens) * 100 : 0).toFixed(1); - lines.push(`| \`${f.relativePath}\` | ${f.estimatedTokens.toLocaleString()} | ${pc}% |`); + lines.push(`| ${markdownCode(f.relativePath)} | ${f.estimatedTokens.toLocaleString()} | ${pc}% |`); } lines.push(""); } @@ -85,7 +86,7 @@ function formatScanMarkdown(scan: import("@wma/core").WorkspaceScanResult): stri const sortedLangs = [...scan.languages].sort((a, b) => b.totalTokens - a.totalTokens); for (const l of sortedLangs) { const pc = (l.percentage * 100).toFixed(1); - lines.push(`| ${l.language} | ${l.fileCount} | ${l.totalTokens.toLocaleString()} | ${pc}% |`); + lines.push(`| ${escapeMarkdownText(l.language)} | ${l.fileCount} | ${l.totalTokens.toLocaleString()} | ${pc}% |`); } lines.push(""); } @@ -94,7 +95,7 @@ function formatScanMarkdown(scan: import("@wma/core").WorkspaceScanResult): stri lines.push("## Warnings"); lines.push(""); for (const w of scan.warnings) { - lines.push(`- ${w}`); + lines.push(`- ${escapeMarkdownText(w)}`); } lines.push(""); } diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts index efe8b17..7d96529 100644 --- a/apps/cli/src/index.ts +++ b/apps/cli/src/index.ts @@ -15,6 +15,7 @@ import { handleError, CliError } from "./utils/errors.js"; import { parseAgentTarget, parseGoal, parseMcpTarget, parseOptimizationMode, parsePositiveInteger, parsePrivacyMode, parseReportFormat, parseScanFormat, parseTableFormat } from "./utils/options.js"; import { startServer } from "@wma/mcp-server"; import { executeCompact } from "./commands/compact.js"; +import { executeInit } from "./commands/init.js"; const program = new Command(); @@ -22,7 +23,38 @@ program .name("wma") .description("TCalc — local-first workspace analysis and AI agent optimization") .version("0.1.4") - .option("--debug", "Show stack traces on error"); + .option("--debug", "Show stack traces on error") + .addHelpText( + "after", + ` +Examples: + $ wma scan ./my-project + $ wma recommend ./my-project + $ wma repo-map ./my-project + $ wma rules ./my-project --target cursor +`, + ); +program + .command("init [path]") + .description("Initialize a tailored .workspace-model-advisor.json config file") + .option("--goal ", "Workspace goal", parseGoal) + .option("--privacy ", "Privacy mode (local-first|cloud-ok)", parsePrivacyMode) + .option("--force", "Overwrite an existing configuration file") + .option("--dry-run", "Preview generated configuration without writing to disk") + .action(async (target, opts) => { + try { + const output = await executeInit({ + target, + goal: opts.goal, + privacy: opts.privacy, + force: opts.force, + dryRun: opts.dryRun, + }); + console.log(output); + } catch (err) { + handleError(err, program.opts().debug); + } + }); program .command("scan [path]") @@ -225,11 +257,13 @@ catalogCmd .command("validate [path]") .description("Validate model catalog JSON") .option("--format ", "Output format (table|json)", parseTableFormat, "table") + .option("--max-age-days ", "Maximum catalog age in days", parsePositiveInteger, 90) .action(async (target, opts) => { try { const result = await executeCatalogValidate({ path: target, format: opts.format, + maxAgeDays: opts.maxAgeDays, debug: program.opts().debug, }); console.log(result.message); diff --git a/apps/cli/src/utils/loadWorkspaceConfig.ts b/apps/cli/src/utils/loadWorkspaceConfig.ts index 6752b0e..cc1ab18 100644 --- a/apps/cli/src/utils/loadWorkspaceConfig.ts +++ b/apps/cli/src/utils/loadWorkspaceConfig.ts @@ -1,23 +1,80 @@ import { readFile } from "node:fs/promises"; import { DEFAULT_CONFIG, isPrivacySetting, isWorkspaceGoal, loadTeamPolicy, type WmaConfig } from "@wma/core"; import { findWorkspaceConfigPath } from "./paths.js"; +import { CliError } from "./errors.js"; export async function loadWorkspaceConfig(rootPath: string): Promise { const configPath = findWorkspaceConfigPath(rootPath); const teamPolicy = await loadTeamPolicy(rootPath); + let parsed: unknown; + try { const content = await readFile(configPath, "utf-8"); - const parsed = JSON.parse(content) as Partial; - const config: WmaConfig = { - ...DEFAULT_CONFIG, - ...parsed, - defaultGoal: isWorkspaceGoal(parsed.defaultGoal) ? parsed.defaultGoal : DEFAULT_CONFIG.defaultGoal, - privacyMode: isPrivacySetting(parsed.privacyMode) ? parsed.privacyMode : DEFAULT_CONFIG.privacyMode, - }; - return applyTeamPolicy(config, teamPolicy); - } catch { - return applyTeamPolicy(DEFAULT_CONFIG, teamPolicy); + parsed = JSON.parse(content); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return applyTeamPolicy(DEFAULT_CONFIG, teamPolicy); + } + throw new CliError(`Failed to load workspace config at ${configPath}: ${errorMessage(error)}`); } + + return applyTeamPolicy(parseWorkspaceConfig(parsed, configPath), teamPolicy); +} + +function parseWorkspaceConfig(value: unknown, configPath: string): WmaConfig { + if (!isRecord(value)) { + throw new CliError(`Workspace config at ${configPath} must be a JSON object.`); + } + + const config: WmaConfig = { + ...DEFAULT_CONFIG, + ...value, + defaultGoal: parseDefaultGoal(value.defaultGoal, configPath), + privacyMode: parsePrivacyMode(value.privacyMode, configPath), + exclude: parseStringArray(value.exclude, DEFAULT_CONFIG.exclude, "exclude", configPath), + tokenBudget: parseTokenBudget(value.tokenBudget, configPath), + }; + + return config; +} + +function parseDefaultGoal(value: unknown, configPath: string): WmaConfig["defaultGoal"] { + if (value === undefined) return DEFAULT_CONFIG.defaultGoal; + if (!isWorkspaceGoal(value)) throw new CliError(`Workspace config at ${configPath} has invalid defaultGoal.`); + return value; +} + +function parsePrivacyMode(value: unknown, configPath: string): WmaConfig["privacyMode"] { + if (value === undefined) return DEFAULT_CONFIG.privacyMode; + if (!isPrivacySetting(value)) throw new CliError(`Workspace config at ${configPath} has invalid privacyMode.`); + return value; +} + +function parseTokenBudget(value: unknown, configPath: string): WmaConfig["tokenBudget"] { + if (value === undefined) return DEFAULT_CONFIG.tokenBudget; + if (!isRecord(value)) throw new CliError(`Workspace config at ${configPath} tokenBudget must be an object.`); + + return { + defaultContextBudget: parsePositiveInteger(value.defaultContextBudget, DEFAULT_CONFIG.tokenBudget.defaultContextBudget, "tokenBudget.defaultContextBudget", configPath), + maxFullWorkspaceScan: parsePositiveInteger(value.maxFullWorkspaceScan, DEFAULT_CONFIG.tokenBudget.maxFullWorkspaceScan, "tokenBudget.maxFullWorkspaceScan", configPath), + warnAt: parsePositiveInteger(value.warnAt, DEFAULT_CONFIG.tokenBudget.warnAt, "tokenBudget.warnAt", configPath), + }; +} + +function parsePositiveInteger(value: unknown, fallback: number, field: string, configPath: string): number { + if (value === undefined) return fallback; + if (typeof value !== "number" || !Number.isSafeInteger(value) || value <= 0) { + throw new CliError(`Workspace config at ${configPath} has invalid ${field}; expected a positive integer.`); + } + return value; +} + +function parseStringArray(value: unknown, fallback: string[], field: string, configPath: string): string[] { + if (value === undefined) return fallback; + if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || item.length === 0)) { + throw new CliError(`Workspace config at ${configPath} has invalid ${field}; expected an array of non-empty strings.`); + } + return value; } function applyTeamPolicy(config: WmaConfig, teamPolicy: Awaited>): WmaConfig { @@ -36,3 +93,11 @@ function applyTeamPolicy(config: WmaConfig, teamPolicy: Awaited { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/apps/cli/tests/catalogCommand.test.ts b/apps/cli/tests/catalogCommand.test.ts index 8c427a1..aafc8fb 100644 --- a/apps/cli/tests/catalogCommand.test.ts +++ b/apps/cli/tests/catalogCommand.test.ts @@ -1,11 +1,17 @@ import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; import { executeCatalogValidate } from "../src/commands/catalog.js"; +// Pin "now" to the bundled catalog's own date so these output tests don't +// start failing as the calendar moves (#218). Freshness has its own tests. +const now = new Date(`${JSON.parse(readFileSync("catalogs/models.json", "utf-8")).updatedAt}T00:00:00Z`); + describe("catalog validate command", () => { it("validates valid catalog", async () => { // The bundled catalog from the workspace catalogs directory const result = await executeCatalogValidate({ path: "catalogs/models.json", + now, }); expect(result.valid).toBe(true); expect(result.exitCode).toBe(0); @@ -16,12 +22,23 @@ describe("catalog validate command", () => { const result = await executeCatalogValidate({ path: "catalogs/models.json", format: "json", + now, }); const payload = JSON.parse(result.message); expect(payload).toMatchObject({ valid: true, errors: [] }); expect(payload.modelCount).toBeGreaterThanOrEqual(20); }); + it("reports a catalog older than --max-age-days", async () => { + const result = await executeCatalogValidate({ + path: "catalogs/models.json", + maxAgeDays: 30, + now: new Date(now.getTime() + 31 * 86_400_000), + }); + expect(result.valid).toBe(false); + expect(result.message).toContain("31 days old"); + }); + it("fails on missing catalog", async () => { const result = await executeCatalogValidate({ path: "catalogs/nonexistent.json", diff --git a/apps/cli/tests/help.test.ts b/apps/cli/tests/help.test.ts new file mode 100644 index 0000000..7293c47 --- /dev/null +++ b/apps/cli/tests/help.test.ts @@ -0,0 +1,19 @@ +import { execFileSync } from "node:child_process"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; + +const cliPath = path.resolve("apps/cli/dist/index.js"); + +describe("CLI help", () => { + it("shows common usage examples", () => { + const output = execFileSync(process.execPath, [cliPath, "--help"], { + encoding: "utf-8", + }); + + expect(output).toContain("Examples:"); + expect(output).toContain("$ wma scan ./my-project"); + expect(output).toContain("$ wma recommend ./my-project"); + expect(output).toContain("$ wma repo-map ./my-project"); + expect(output).toContain("$ wma rules ./my-project --target cursor"); + }); +}); \ No newline at end of file diff --git a/apps/cli/tests/initCommand.test.ts b/apps/cli/tests/initCommand.test.ts new file mode 100644 index 0000000..5d0214b --- /dev/null +++ b/apps/cli/tests/initCommand.test.ts @@ -0,0 +1,159 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtemp, rm, writeFile, readFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { detectProjectStack, executeInit } from "../src/commands/init.js"; +import { CliError } from "../src/utils/errors.js"; +import { DEFAULT_CONFIG } from "@wma/core"; + +describe("initCommand wizard", () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(path.join(tmpdir(), "wma-init-test-")); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + describe("detectProjectStack", () => { + it("detects TypeScript project with framework dependencies", async () => { + await writeFile( + path.join(tempDir, "package.json"), + JSON.stringify({ + dependencies: { next: "14.0.0", react: "18.0.0" }, + devDependencies: { vitest: "^1.0.0" }, + }), + ); + await writeFile(path.join(tempDir, "tsconfig.json"), "{}"); + + const stack = await detectProjectStack(tempDir); + expect(stack.primaryLanguage).toBe("typescript"); + expect(stack.isMonorepo).toBe(false); + expect(stack.detectedFrameworks).toContain("Next.js"); + expect(stack.detectedFrameworks).toContain("React"); + expect(stack.detectedFrameworks).toContain("Testing (Vitest/Jest)"); + expect(stack.recommendedExcludes).toContain("dist"); + expect(stack.recommendedExcludes).toContain(".next"); + expect(stack.recommendedBudget).toBe(128_000); + }); + + it("detects Python project and adds virtualenv excludes", async () => { + await writeFile(path.join(tempDir, "pyproject.toml"), "[project]\nname='my-app'"); + + const stack = await detectProjectStack(tempDir); + expect(stack.primaryLanguage).toBe("python"); + expect(stack.recommendedExcludes).toContain(".venv"); + expect(stack.recommendedExcludes).toContain("__pycache__"); + }); + + it("detects Rust project and configures target exclusion", async () => { + await writeFile(path.join(tempDir, "Cargo.toml"), '[package]\nname = "my-crate"'); + + const stack = await detectProjectStack(tempDir); + expect(stack.primaryLanguage).toBe("rust"); + expect(stack.recommendedExcludes).toContain("target"); + }); + + it("detects Go project and configures vendor exclusion", async () => { + await writeFile(path.join(tempDir, "go.mod"), "module example.com/app\ngo 1.22"); + + const stack = await detectProjectStack(tempDir); + expect(stack.primaryLanguage).toBe("go"); + expect(stack.recommendedExcludes).toContain("vendor"); + expect(stack.recommendedExcludes).toContain("bin"); + }); + + it("detects monorepo workspaces and scales token budget", async () => { + await writeFile(path.join(tempDir, "pnpm-workspace.yaml"), "packages:\n - 'packages/*'"); + + const stack = await detectProjectStack(tempDir); + expect(stack.isMonorepo).toBe(true); + expect(stack.recommendedBudget).toBe(200_000); + }); + }); + + describe("executeInit", () => { + it("returns JSON preview in dry-run mode without creating disk file", async () => { + const output = await executeInit({ + target: tempDir, + dryRun: true, + }); + + expect(output).toContain("[DRY RUN]"); + expect(output).toContain('"defaultGoal"'); + + const configFile = path.join(tempDir, ".workspace-model-advisor.json"); + let exists = false; + try { + await readFile(configFile); + exists = true; + } catch { + exists = false; + } + expect(exists).toBe(false); + }); + + it("writes formatted configuration file on disk with custom parameters", async () => { + await writeFile(path.join(tempDir, "Cargo.toml"), ""); + + const output = await executeInit({ + target: tempDir, + goal: "refactor", + privacy: "local-first", + }); + + expect(output).toContain("Initialized TCalc workspace configuration!"); + expect(output).toContain("File written:"); + + const configFile = path.join(tempDir, ".workspace-model-advisor.json"); + const content = await readFile(configFile, "utf8"); + const parsed = JSON.parse(content); + + expect(parsed.defaultGoal).toBe("refactor"); + expect(parsed.privacyMode).toBe("local-first"); + expect(parsed.exclude).toContain("target"); + }); + + it("writes only known configuration keys", async () => { + await executeInit({ target: tempDir }); + + const configFile = path.join(tempDir, ".workspace-model-advisor.json"); + const parsed = JSON.parse(await readFile(configFile, "utf8")); + + // The repository does not publish a schema.json, so a $schema URL would 404 in editors. + expect(parsed).not.toHaveProperty("$schema"); + for (const key of Object.keys(parsed)) { + expect(Object.keys(DEFAULT_CONFIG)).toContain(key); + } + }); + + it("fails when configuration file already exists and force flag is absent", async () => { + const configFile = path.join(tempDir, ".workspace-model-advisor.json"); + await writeFile(configFile, '{"existing": true}', "utf8"); + + await expect( + executeInit({ + target: tempDir, + }), + ).rejects.toThrow(CliError); + }); + + it("overwrites configuration when force flag is set", async () => { + const configFile = path.join(tempDir, ".workspace-model-advisor.json"); + await writeFile(configFile, '{"existing": true}', "utf8"); + + const output = await executeInit({ + target: tempDir, + force: true, + }); + + expect(output).toContain("Initialized TCalc workspace configuration!"); + const content = await readFile(configFile, "utf8"); + const parsed = JSON.parse(content); + expect(parsed.existing).toBeUndefined(); + expect(parsed.defaultGoal).toBeDefined(); + }); + }); +}); diff --git a/apps/cli/tests/recommendCommand.test.ts b/apps/cli/tests/recommendCommand.test.ts index 0b8008a..42ee536 100644 --- a/apps/cli/tests/recommendCommand.test.ts +++ b/apps/cli/tests/recommendCommand.test.ts @@ -1,6 +1,63 @@ -import { describe, it, expect } from "vitest"; +import { afterEach, describe, it, expect } from "vitest"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { executeRecommend } from "../src/commands/recommend.js"; +const cleanup: string[] = []; +afterEach(() => cleanup.splice(0).forEach((directory) => rmSync(directory, { recursive: true, force: true }))); + +// Two models identical in every scored dimension, so only the tiebreak can separate them. The +// preferred id sorts after the other, so localeCompare actively picks the wrong one when the +// preference fails to reach the recommender. +function tiedModel(id: string) { + return { + id, + displayName: `Tied ${id}`, + provider: "test", + contextWindow: 128000, + maxOutputTokens: 4096, + inputPricePerMillion: 0.02, + cachedInputPricePerMillion: 0.01, + outputPricePerMillion: 0.04, + supportsTools: false, + supportsImages: false, + supportsLocal: true, + // local-first is the default privacy mode, so cloud models would be filtered out entirely. + privacyMode: "local", + codingScore: 90, + reasoningScore: 90, + latencyScore: 10, + updatedAt: "2026-07-01", + }; +} + +function makeWorkspace(): string { + const root = mkdtempSync(path.join(tmpdir(), "tcalc-cli-team-")); + cleanup.push(root); + mkdirSync(path.join(root, ".tcalc")); + mkdirSync(path.join(root, "src")); + writeFileSync(path.join(root, "src", "index.ts"), "export const value = 1;"); + writeFileSync(path.join(root, "src", "helper.ts"), "export const helper = () => value;"); + // resolveCatalog probes /catalogs/models.json then /.tcalc/models.json. + writeFileSync( + path.join(root, ".tcalc", "models.json"), + JSON.stringify({ version: "1.0", updatedAt: "2026-07-01", models: [tiedModel("alpha/tied"), tiedModel("zeta/tied")] }), + ); + return root; +} + +function writeTeamPolicy(root: string, preferredModelIds: string[]): void { + writeFileSync( + path.join(root, ".tcalc", "team.json"), + JSON.stringify({ + schemaVersion: "1.0", + modelProfiles: [{ id: "team", preferredModelIds }], + activeProfile: "team", + }), + ); +} + describe("recommend command", () => { it("recommends models from catalog", async () => { const output = await executeRecommend({ @@ -24,6 +81,31 @@ describe("recommend command", () => { expect(parsed.goal).toBeDefined(); }); + it("threads preferredModelIds from .tcalc/team.json through to the recommendation", async () => { + const root = makeWorkspace(); + + // Control: with no team policy on disk, the alphabetical tiebreak decides. + const baseline = JSON.parse(await executeRecommend({ target: root, format: "json" })); + expect(baseline.cheapestSufficient.modelId).toBe("alpha/tied"); + + // The only thing that changes is the file. This covers the whole chain: team.json is read by + // loadTeamPolicy, parsed into a profile, looked up by getActiveModelProfile, applied by + // applyModelProfile, and passed into recommendModels by the command itself. A break in any one + // of those layers, including the call-site wiring, leaves this assertion at "alpha/tied". + writeTeamPolicy(root, ["zeta/tied"]); + + const preferred = JSON.parse(await executeRecommend({ target: root, format: "json" })); + expect(preferred.cheapestSufficient.modelId).toBe("zeta/tied"); + }); + + it("leaves the recommendation alone when the policy prefers an unknown model", async () => { + const root = makeWorkspace(); + writeTeamPolicy(root, ["not-in-catalog/model"]); + + const result = JSON.parse(await executeRecommend({ target: root, format: "json" })); + expect(result.cheapestSufficient.modelId).toBe("alpha/tied"); + }); + it("returns table format output", async () => { const output = await executeRecommend({ target: "fixtures/small-node-app", diff --git a/apps/cli/tests/repoMapCommand.test.ts b/apps/cli/tests/repoMapCommand.test.ts index 01a7576..e54e518 100644 --- a/apps/cli/tests/repoMapCommand.test.ts +++ b/apps/cli/tests/repoMapCommand.test.ts @@ -1,5 +1,9 @@ import { describe, it, expect } from "vitest"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { executeRepoMap } from "../src/commands/repoMap.js"; +import { executeScan } from "../src/commands/scan.js"; describe("repo-map command", () => { it("generates markdown repo map", async () => { @@ -73,4 +77,80 @@ describe("repo-map command", () => { const parsed = JSON.parse(output); expect(parsed.symbols.length).toBeLessThanOrEqual(2); }); + + it("keeps scanner-classified generated files in the repo map end to end", async () => { + const root = await mkdtemp(path.join(tmpdir(), "tcalc-cli-generated-")); + try { + await mkdir(path.join(root, "src")); + await writeFile(path.join(root, "src", "index.ts"), "export const value = 1;"); + await writeFile(path.join(root, "src", "vendor.min.js"), `${"a".repeat(4000)};`); + + const parsed = JSON.parse(await executeRepoMap({ target: root, format: "json" })); + + // A real scan has to have excluded it from the token budget. reason/priority identify which + // branch produced the entry: the excluded pass carries the scanner's reason and priority 0, + // while an included low-priority file would read "Low-priority" at priority 10. + const excluded = parsed.excludedFiles.find((f) => f.relativePath === "src/vendor.min.js"); + expect(excluded).toBeDefined(); + expect(excluded.reason).toBe("generated"); + expect(excluded.priority).toBe(0); + + // Despite that exclusion, the generated-file label and its agent guidance survive. + expect(parsed.generatedFiles.map((f) => f.relativePath)).toContain("src/vendor.min.js"); + expect(parsed.agentInstructions).toContain("There are 1 generated file(s). Avoid editing them."); + + // Exclusion is still a token-budget decision, so it stays out of include recommendations. + expect(parsed.importantFiles.map((f) => f.relativePath)).not.toContain("src/vendor.min.js"); + expect(parsed.recommendedInclude.map((f) => f.relativePath)).not.toContain("src/vendor.min.js"); + + // Appearing in several overlapping lists must not produce repeated exclude advice. + const recommended = parsed.recommendedExclude.map((f) => f.relativePath); + expect(new Set(recommended).size).toBe(recommended.length); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it("keeps nested-ignored directories out of scan tokens and the emitted repo map", async () => { + const root = await mkdtemp(path.join(tmpdir(), "tcalc-cli-nested-ignore-")); + try { + await mkdir(path.join(root, "packages", "app", "src"), { recursive: true }); + await mkdir(path.join(root, "packages", "app", "generated"), { recursive: true }); + await writeFile(path.join(root, "packages", "app", ".gitignore"), "generated/\n"); + await writeFile(path.join(root, "packages", "app", "src", "index.ts"), "export const included = true;\n"); + await writeFile( + path.join(root, "packages", "app", "generated", "ignored.ts"), + `export const ignored = "${"token-heavy ".repeat(1000)}";\n`, + ); + + const scan = JSON.parse(await executeScan({ target: root, format: "json" })); + const included = scan.files.find( + (file: { relativePath: string }) => file.relativePath === "packages/app/src/index.ts", + ); + + expect(included).toBeDefined(); + const nonGeneratedFiles = scan.files.filter( + (file: { relativePath: string }) => !file.relativePath.startsWith("packages/app/generated/"), + ); + const nonGeneratedTokens = nonGeneratedFiles.reduce( + (sum: number, file: { estimatedTokens: number }) => sum + file.estimatedTokens, + 0, + ); + expect(scan.includedTokens).toBe(nonGeneratedTokens); + expect(scan.files.map((file: { relativePath: string }) => file.relativePath)) + .not.toContain("packages/app/generated/ignored.ts"); + + const repoMap = JSON.parse(await executeRepoMap({ + target: root, + format: "json", + noSymbols: true, + })); + expect(repoMap.workspaceTotalTokens).toBe(scan.includedTokens); + expect(JSON.stringify(repoMap)).not.toContain("packages/app/generated/ignored.ts"); + expect(JSON.stringify(repoMap)).not.toContain("token-heavy"); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + }); diff --git a/apps/cli/tests/scanCommand.test.ts b/apps/cli/tests/scanCommand.test.ts index dd0f120..f5a7f8b 100644 --- a/apps/cli/tests/scanCommand.test.ts +++ b/apps/cli/tests/scanCommand.test.ts @@ -1,6 +1,8 @@ import { describe, it, expect, vi } from "vitest"; import { executeScan } from "../src/commands/scan.js"; import path from "node:path"; +import { mkdtemp, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; const fixturePath = path.resolve("fixtures/small-node-app"); @@ -38,6 +40,30 @@ describe("scan command", () => { expect(output).toContain("Excluded Files:"); }); + it("surfaces malformed workspace config instead of silently using defaults", async () => { + const root = await mkdtemp(path.join(tmpdir(), "tcalc-bad-config-")); + try { + await writeFile(path.join(root, "index.ts"), "export const value = 1;\n"); + await writeFile(path.join(root, ".tcalc.json"), "{ \"exclude\": [\"secret/**\"], }"); + + await expect(executeScan({ target: root, format: "json" })).rejects.toThrow(/Failed to load workspace config/); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + + it("rejects invalid workspace config field types", async () => { + const root = await mkdtemp(path.join(tmpdir(), "tcalc-invalid-config-")); + try { + await writeFile(path.join(root, "index.ts"), "export const value = 1;\n"); + await writeFile(path.join(root, ".tcalc.json"), JSON.stringify({ exclude: "dist/**" })); + + await expect(executeScan({ target: root, format: "json" })).rejects.toThrow(/invalid exclude/); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + it("scan handles default path (cwd)", async () => { const cwd = vi.spyOn(process, "cwd").mockReturnValue(fixturePath); try { @@ -49,4 +75,30 @@ describe("scan command", () => { cwd.mockRestore(); } }); + it("escapes Markdown-sensitive paths and warnings", async () => { + const root = await mkdtemp(path.join(tmpdir(), "tcalc-markdown-")); + const target = await mkdtemp(path.join(tmpdir(), "tcalc-missing-")); + + try { + await writeFile(path.join(root, "report`name.ts"), "export const value = 1;\n"); + + await symlink( + target, + path.join(root, "warning`link"), + process.platform === "win32" ? "junction" : "dir", + ); + await rm(target, { recursive: true, force: true }); + + const output = await executeScan({ + target: root, + format: "markdown", + }); + + expect(output).toContain("`` report`name.ts ``"); + expect(output).toContain("warning\\`link"); + } finally { + await rm(root, { recursive: true, force: true }); + await rm(target, { recursive: true, force: true }); + } + }); }); diff --git a/apps/dashboard/components/ReportViewer.tsx b/apps/dashboard/components/ReportViewer.tsx index cf5c229..e8dc20c 100644 --- a/apps/dashboard/components/ReportViewer.tsx +++ b/apps/dashboard/components/ReportViewer.tsx @@ -1,6 +1,6 @@ "use client"; -import { useEffect, useRef, useState } from "react"; +import { useEffect, useRef, useState, type FormEvent } from "react"; type ModelPick = { displayName?: string; modelId?: string; provider?: string; tier?: string; score?: number | { totalScore?: number } }; type TCalcReport = { @@ -46,13 +46,43 @@ export default function ReportViewer() { const [status, setStatus] = useState("Showing a safe example report."); const [error, setError] = useState(""); const [savedReports, setSavedReports] = useState([]); + const [needsUnlock, setNeedsUnlock] = useState(false); + const [unlockError, setUnlockError] = useState(""); - useEffect(() => { - fetch("/api/reports") - .then((response) => response.ok ? response.json() : Promise.reject()) - .then((items) => { if (Array.isArray(items)) setSavedReports(items); }) - .catch(() => undefined); - }, []); + // Saved reports are private by default. A 401 means the operator can unlock reads by exchanging + // the dashboard token for an HttpOnly session cookie, so the token itself never lives in this bundle. + const loadSavedReports = async () => { + try { + const response = await fetch("/api/reports", { credentials: "same-origin" }); + if (response.status === 401) { setNeedsUnlock(true); return; } + if (!response.ok) { setNeedsUnlock(false); return; } + const items = await response.json(); + if (Array.isArray(items)) setSavedReports(items); + setNeedsUnlock(false); + } catch { /* The optional report service is not running. */ } + }; + + useEffect(() => { void loadSavedReports(); }, []); + + const onUnlock = async (event: FormEvent) => { + event.preventDefault(); + const form = event.currentTarget; + const field = new FormData(form).get("token"); + setUnlockError(""); + try { + const response = await fetch("/api/session", { + method: "POST", + credentials: "same-origin", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ token: typeof field === "string" ? field : "" }), + }); + if (!response.ok) throw new Error(response.status === 401 ? "That dashboard token was not accepted." : "Could not start a session."); + form.reset(); + await loadSavedReports(); + } catch (caught) { + setUnlockError(caught instanceof Error ? caught.message : String(caught)); + } + }; const openReport = (value: unknown, message: string) => { if (!isReport(value)) throw new Error("This is not a valid TCalc report (schema 1.0.0). "); @@ -108,9 +138,20 @@ export default function ReportViewer() { } + {needsUnlock &&
+

Saved team reports

+

These reports are private. Enter the dashboard token to read them in this browser.

+
void onUnlock(event)}> + + + +
+ {unlockError ?

{unlockError}

: null} +
} {savedReports.length > 0 &&

Saved team reports

{savedReports.map((saved) =>
` : ""} + ${ + recommendation + ? `
Top recommendationsFit
${[ + recommendation.cheapestSufficient, + recommendation.balanced, + recommendation.highConfidence, + ] + .map( + (model) => + `
${escapeHtml(model.displayName)}${escapeHtml(model.tier.replaceAll("-", " "))}${Math.round(model.score.totalScore * 100)}
`, + ) + .join( + "", + )}
` + : "" + }
Workspace tools
- ` : `

${workspace ? "Ready to measure this workspace" : "Open a folder to begin"}

${workspace ? "Scan locally to see token usage, ranked models, and agent-ready actions." : "TCalc needs one open workspace folder. No source leaves your machine."}

${workspace ? `` : ""}
`} + ` + : `

${workspace ? "Ready to measure this workspace" : "Open a folder to begin"}

${workspace ? "Scan locally to see token usage, ranked models, and agent-ready actions." : "TCalc needs one open workspace folder. No source leaves your machine."}

${workspace ? `` : ""}
` + }
${scan ? `` : ""}
- + `; } } function compact(value: number): string { - return Intl.NumberFormat(undefined, { notation: "compact", maximumFractionDigits: 1 }).format(value); + return Intl.NumberFormat(undefined, { + notation: "compact", + maximumFractionDigits: 1, + }).format(value); } function relativeTime(value: string): string { @@ -90,5 +134,11 @@ function relativeTime(value: string): string { } function escapeHtml(value: string): string { - return value.replace(/[&<>"']/g, (character) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[character]!); + return value.replace( + /[&<>"']/g, + (character) => + ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[ + character + ]!, + ); } diff --git a/apps/vscode-extension/src/workspaceContext.ts b/apps/vscode-extension/src/workspaceContext.ts new file mode 100644 index 0000000..3a8735f --- /dev/null +++ b/apps/vscode-extension/src/workspaceContext.ts @@ -0,0 +1,100 @@ +import * as vscode from "vscode"; +import path from "node:path"; +import type { + ModelInfo, + RecommendationResult, + WorkspaceScanResult, +} from "@wma/core"; + +const WORKSPACE_STATE_KEY = "wma.workspaceState"; +const ACTIVE_ROOT_KEY = "wma.activeRootPath"; + +export interface RootWorkspaceState { + scan?: WorkspaceScanResult; + recommendation?: RecommendationResult | null; + models?: ModelInfo[]; + repoMapPath?: string; +} + +export function normalizeRootPath(rootPath: string): string { + const normalized = path.normalize(path.resolve(rootPath)); + return process.platform === "win32" ? normalized.toLowerCase() : normalized; +} + +export async function selectWorkspaceRoot( + rootPath?: string, +): Promise { + if (rootPath) return rootPath; + + const folders = vscode.workspace.workspaceFolders; + if (!folders || folders.length === 0) { + vscode.window.showErrorMessage("Open a workspace folder first."); + return undefined; + } + + if (folders.length === 1) return folders[0].uri.fsPath; + + const selected = await vscode.window.showQuickPick( + folders.map((folder) => ({ + label: folder.name, + description: folder.uri.fsPath, + rootPath: folder.uri.fsPath, + })), + { placeHolder: "Select a workspace folder" }, + ); + + return selected?.rootPath; +} + +export async function setActiveWorkspaceRoot( + context: vscode.ExtensionContext, + rootPath: string, +): Promise { + await context.workspaceState.update(ACTIVE_ROOT_KEY, rootPath); +} + +export function getActiveWorkspaceRoot( + context: vscode.ExtensionContext, +): string | undefined { + const activeRoot = context.workspaceState.get(ACTIVE_ROOT_KEY); + if (!activeRoot) return undefined; + + const folders = vscode.workspace.workspaceFolders; + if (!folders || folders.length === 0) return undefined; + + const activeKey = normalizeRootPath(activeRoot); + const currentFolder = folders.find( + (folder) => normalizeRootPath(folder.uri.fsPath) === activeKey, + ); + return currentFolder?.uri.fsPath ?? folders[0].uri.fsPath; +} + +export function getWorkspaceState( + context: vscode.ExtensionContext, + rootPath: string, +): RootWorkspaceState { + const allState = + context.workspaceState.get>( + WORKSPACE_STATE_KEY, + ) ?? {}; + return allState[normalizeRootPath(rootPath)] ?? {}; +} + +export async function updateWorkspaceState( + context: vscode.ExtensionContext, + rootPath: string, + update: Partial, +): Promise { + const allState = + context.workspaceState.get>( + WORKSPACE_STATE_KEY, + ) ?? {}; + const key = normalizeRootPath(rootPath); + await context.workspaceState.update(WORKSPACE_STATE_KEY, { + ...allState, + [key]: { + ...allState[key], + ...update, + }, + }); +} diff --git a/apps/vscode-extension/tests/multiRootCommands.test.ts b/apps/vscode-extension/tests/multiRootCommands.test.ts new file mode 100644 index 0000000..3227d0f --- /dev/null +++ b/apps/vscode-extension/tests/multiRootCommands.test.ts @@ -0,0 +1,270 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const roots = { + alpha: "C:/work/alpha", + beta: "C:/work/beta", +}; + +const goals = new Map([ + [roots.alpha, "Goal A"], + [roots.beta, "Goal B"], +]); +const registeredCommands = new Map unknown>(); +const scanGoals: string[] = []; +const workspaceValues = new Map(); + +const vscodeMock = { + ConfigurationTarget: { WorkspaceFolder: "workspace-folder" }, + ProgressLocation: { Notification: "notification" }, + Uri: { + file: (fsPath: string) => ({ fsPath }), + }, + workspace: { + openTextDocument: vi.fn(async () => ({})), + fs: { writeFile: vi.fn(async () => undefined) }, + workspaceFolders: [ + { name: "alpha", uri: { fsPath: roots.alpha } }, + { name: "beta", uri: { fsPath: roots.beta } }, + ], + getConfiguration: vi.fn((_: string, resource?: { fsPath: string }) => { + const rootPath = resource?.fsPath; + return { + get: (key: string) => + (key === "defaultGoal" && rootPath + ? goals.get(rootPath) + : undefined) as T | undefined, + update: async (key: string, value: string, target: unknown) => { + expect(key).toBe("defaultGoal"); + expect(target).toBe(vscodeMock.ConfigurationTarget.WorkspaceFolder); + goals.set(rootPath!, value); + }, + }; + }), + }, + commands: { + registerCommand: vi.fn( + (id: string, handler: (...args: any[]) => unknown) => { + registeredCommands.set(id, handler); + return { dispose: vi.fn() }; + }, + ), + executeCommand: vi.fn(), + }, + window: { + showTextDocument: vi.fn(), + showSaveDialog: vi.fn(), + showErrorMessage: vi.fn(), + showQuickPick: vi.fn(), + showInformationMessage: vi.fn(), + showWarningMessage: vi.fn(), + withProgress: vi.fn( + async ( + _options: unknown, + callback: (progress: unknown, token: unknown) => Promise, + ) => + callback( + { report: vi.fn() }, + { + isCancellationRequested: false, + onCancellationRequested: () => ({ dispose: vi.fn() }), + }, + ), + ), + }, +}; + +vi.mock("vscode", () => vscodeMock, { virtual: true }); +vi.mock("@wma/scanner", () => ({ + scanWorkspace: vi.fn(async ({ rootPath }: { rootPath: string }) => ({ + rootPath, + scannedAt: new Date().toISOString(), + includedFiles: 1, + includedTokens: 100, + totalEstimatedTokens: 100, + warnings: [], + })), +})); +vi.mock("@wma/model-catalog", () => ({ + applyModelProfile: (models: unknown[]) => models, + loadModelCatalog: () => ({ models: [{ id: "model-1" }] }), + validateModelCatalog: () => [], +})); +vi.mock("@wma/recommender", () => ({ + recommendModels: vi.fn(({ goal }: { goal: string }) => { + scanGoals.push(goal); + return null; + }), +})); +vi.mock("@wma/core", () => ({ + getActiveModelProfile: () => undefined, + loadTeamPolicy: async () => undefined, + WORKSPACE_GOALS: ["Goal A", "Goal B", "Goal B Updated"], +})); +vi.mock("@wma/repo-map", () => ({ + createRepoMap: vi.fn(), + formatRepoMapMarkdown: vi.fn(() => "# Basic map"), +})); + +const { registerScanWorkspaceCommand } = + await import("../src/commands/scanWorkspaceCommand.js"); +const { registerSetWorkspaceGoalCommand } = + await import("../src/commands/setWorkspaceGoalCommand.js"); +const { TCalcSidebarProvider } = + await import("../src/views/sidebarProvider.js"); +const { getWorkspaceState } = await import("../src/workspaceContext.js"); +const { updateWorkspaceState } = await import("../src/workspaceContext.js"); +const { createRepoMap } = await import("@wma/repo-map"); +const { registerGenerateRepoMapCommand } = + await import("../src/commands/generateRepoMapCommand.js"); + +function createContext() { + return { + extensionUri: { fsPath: "C:/extension" }, + workspaceState: { + get: (key: string) => workspaceValues.get(key) as T | undefined, + update: async (key: string, value: unknown) => { + workspaceValues.set(key, value); + }, + }, + } as never; +} + +async function runCommand(id: string, ...args: any[]): Promise { + await registeredCommands.get(id)?.(...args); +} + +describe("VS Code multi-root commands", () => { + beforeEach(() => { + goals.set(roots.alpha, "Goal A"); + goals.set(roots.beta, "Goal B"); + workspaceValues.clear(); + registeredCommands.clear(); + scanGoals.length = 0; + vi.clearAllMocks(); + }); + + it("isolates goals, scans, state, and sidebar display per workspace root", async () => { + const context = createContext(); + registerSetWorkspaceGoalCommand(context); + registerScanWorkspaceCommand(context); + + vscodeMock.window.showQuickPick.mockResolvedValueOnce({ + label: "Goal A", + }); + await runCommand("workspaceModelAdvisor.setWorkspaceGoal", roots.alpha); + expect(vscodeMock.window.showQuickPick).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ label: "Goal A", description: "current" }), + ]), + { placeHolder: "Select workspace goal" }, + ); + + vscodeMock.window.showQuickPick.mockResolvedValueOnce({ + label: "Goal B Updated", + }); + await runCommand("workspaceModelAdvisor.setWorkspaceGoal", roots.beta); + expect(vscodeMock.window.showQuickPick.mock.calls[1][0]).toEqual( + expect.arrayContaining([ + expect.objectContaining({ label: "Goal B", description: "current" }), + ]), + ); + expect(goals.get(roots.alpha)).toBe("Goal A"); + expect(goals.get(roots.beta)).toBe("Goal B Updated"); + expect(vscodeMock.workspace.getConfiguration).toHaveBeenCalledWith("wma", { + fsPath: roots.beta, + }); + + await runCommand("workspaceModelAdvisor.scanWorkspace", roots.alpha); + await runCommand("workspaceModelAdvisor.scanWorkspace", roots.beta); + + expect(scanGoals).toEqual(["Goal A", "Goal B Updated"]); + expect(getWorkspaceState(context, roots.alpha).scan).toMatchObject({ + rootPath: roots.alpha, + }); + expect(getWorkspaceState(context, roots.beta).scan).toMatchObject({ + rootPath: roots.beta, + }); + + let sidebarHtml = ""; + const view = { + webview: { + options: undefined, + onDidReceiveMessage: vi.fn(), + set html(value: string) { + sidebarHtml = value; + }, + }, + } as never; + const sidebar = new TCalcSidebarProvider(context); + await runCommand("workspaceModelAdvisor.scanWorkspace", roots.beta); + sidebar.resolveWebviewView(view); + expect(sidebarHtml).toContain("Goal B Updated"); + expect(sidebarHtml).not.toContain("Goal A"); + }); + + it.each(["save", "cancel", "write failure"])( + "preserves fallback document and root state on %s", + async (outcome) => { + const context = createContext(); + const scan = { rootPath: roots.beta } as never; + await updateWorkspaceState(context, roots.beta, { scan }); + registerGenerateRepoMapCommand(context); + vscodeMock.window.showQuickPick + .mockResolvedValueOnce({ value: true }) + .mockResolvedValueOnce({ value: 8000 }); + vi.mocked(createRepoMap) + .mockImplementationOnce(() => { + throw new Error("analysis failed"); + }) + .mockReturnValueOnce({} as never); + const saveUri = { fsPath: "C:/saved/map.md" }; + vscodeMock.window.showSaveDialog.mockResolvedValueOnce( + outcome === "cancel" ? undefined : saveUri, + ); + if (outcome === "write failure") { + vscodeMock.workspace.fs.writeFile.mockRejectedValueOnce( + new Error("disk full"), + ); + } + + await runCommand("workspaceModelAdvisor.generateRepoMap", roots.beta); + + expect(createRepoMap).toHaveBeenNthCalledWith(2, scan, { + tokenBudget: 8000, + enableSymbolExtraction: false, + }); + expect(vscodeMock.workspace.openTextDocument).toHaveBeenCalledWith({ + content: "# Basic map", + language: "markdown", + }); + expect(vscodeMock.window.showSaveDialog).toHaveBeenCalledWith( + expect.objectContaining({ + defaultUri: { + fsPath: expect.stringMatching(/beta[\\/]repo-map\.md$/), + }, + }), + ); + expect( + getWorkspaceState(context, roots.alpha).repoMapPath, + ).toBeUndefined(); + expect(getWorkspaceState(context, roots.beta).repoMapPath).toBe( + outcome === "save" ? saveUri.fsPath : undefined, + ); + expect(vscodeMock.workspace.fs.writeFile).toHaveBeenCalledTimes( + outcome === "cancel" ? 0 : 1, + ); + if (outcome !== "cancel") { + expect(vscodeMock.workspace.fs.writeFile).toHaveBeenCalledWith( + saveUri, + new TextEncoder().encode("# Basic map"), + ); + } + if (outcome === "write failure") { + expect(vscodeMock.window.showErrorMessage).toHaveBeenCalledWith( + "Failed to save repo map: disk full", + ); + expect(vscodeMock.window.showInformationMessage).not.toHaveBeenCalled(); + } + }, + ); +}); diff --git a/apps/vscode-extension/tests/workspaceContext.test.ts b/apps/vscode-extension/tests/workspaceContext.test.ts new file mode 100644 index 0000000..e744e9a --- /dev/null +++ b/apps/vscode-extension/tests/workspaceContext.test.ts @@ -0,0 +1,134 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const vscodeMock = { + workspace: { + workspaceFolders: undefined as + Array<{ name: string; uri: { fsPath: string } }> | undefined, + }, + window: { + showErrorMessage: vi.fn(), + showQuickPick: vi.fn(), + }, +}; + +vi.mock("vscode", () => vscodeMock, { virtual: true }); + +const { + getActiveWorkspaceRoot, + getWorkspaceState, + normalizeRootPath, + selectWorkspaceRoot, + updateWorkspaceState, +} = await import("../src/workspaceContext.js"); + +function createContext() { + const values = new Map(); + return { + workspaceState: { + get: (key: string) => values.get(key) as T | undefined, + update: async (key: string, value: unknown) => { + values.set(key, value); + }, + }, + } as never; +} + +describe("workspace context", () => { + beforeEach(() => { + vscodeMock.workspace.workspaceFolders = undefined; + vscodeMock.window.showErrorMessage.mockReset(); + vscodeMock.window.showQuickPick.mockReset(); + }); + + it("uses the only workspace root without showing a picker", async () => { + const root = { name: "alpha", uri: { fsPath: "C:/work/alpha" } }; + vscodeMock.workspace.workspaceFolders = [root]; + + await expect(selectWorkspaceRoot()).resolves.toBe(root.uri.fsPath); + expect(vscodeMock.window.showQuickPick).not.toHaveBeenCalled(); + }); + + it("returns the selected root from a multi-root workspace", async () => { + const roots = [ + { name: "alpha", uri: { fsPath: "C:/work/alpha" } }, + { name: "beta", uri: { fsPath: "C:/work/beta" } }, + ]; + vscodeMock.workspace.workspaceFolders = roots; + vscodeMock.window.showQuickPick.mockResolvedValue({ + rootPath: roots[1].uri.fsPath, + }); + + await expect(selectWorkspaceRoot()).resolves.toBe(roots[1].uri.fsPath); + expect(vscodeMock.window.showQuickPick).toHaveBeenCalledWith( + [ + { + label: "alpha", + description: "C:/work/alpha", + rootPath: "C:/work/alpha", + }, + { + label: "beta", + description: "C:/work/beta", + rootPath: "C:/work/beta", + }, + ], + { placeHolder: "Select a workspace folder" }, + ); + }); + + it("stops cleanly when root selection is cancelled or no workspace is open", async () => { + vscodeMock.workspace.workspaceFolders = [ + { name: "alpha", uri: { fsPath: "C:/work/alpha" } }, + { name: "beta", uri: { fsPath: "C:/work/beta" } }, + ]; + vscodeMock.window.showQuickPick.mockResolvedValue(undefined); + await expect(selectWorkspaceRoot()).resolves.toBeUndefined(); + + vscodeMock.workspace.workspaceFolders = undefined; + await expect(selectWorkspaceRoot()).resolves.toBeUndefined(); + expect(vscodeMock.window.showErrorMessage).toHaveBeenCalledWith( + "Open a workspace folder first.", + ); + }); + + it("keeps state independent for two normalized roots", async () => { + const context = createContext(); + const rootA = "C:/work/alpha"; + const rootB = "C:/work/beta"; + + await updateWorkspaceState(context, rootA, { + scan: { rootPath: rootA } as never, + repoMapPath: "alpha-map.md", + }); + await updateWorkspaceState(context, rootB, { + scan: { rootPath: rootB } as never, + repoMapPath: "beta-map.md", + }); + + expect(getWorkspaceState(context, rootA)).toMatchObject({ + scan: { rootPath: rootA }, + repoMapPath: "alpha-map.md", + }); + expect(getWorkspaceState(context, rootB)).toMatchObject({ + scan: { rootPath: rootB }, + repoMapPath: "beta-map.md", + }); + expect(getWorkspaceState(context, "C:/work/missing")).toEqual({}); + expect(normalizeRootPath(`${rootA}/`)).toBe(normalizeRootPath(rootA)); + }); + + it("falls back when the active root is no longer available", async () => { + const context = createContext(); + const fallbackRoot = { name: "beta", uri: { fsPath: "C:/work/beta" } }; + await context.workspaceState.update( + "wma.activeRootPath", + "C:/work/removed", + ); + vscodeMock.workspace.workspaceFolders = [fallbackRoot]; + + expect(getActiveWorkspaceRoot(context)).toBe(fallbackRoot.uri.fsPath); + + vscodeMock.workspace.workspaceFolders = undefined; + expect(getActiveWorkspaceRoot(context)).toBeUndefined(); + }); +}); diff --git a/docs/team-dashboard.md b/docs/team-dashboard.md index 3931c7a..f3ea8ea 100644 --- a/docs/team-dashboard.md +++ b/docs/team-dashboard.md @@ -8,3 +8,51 @@ pnpm --filter @wma/dashboard start ``` Upload a report with `Authorization: Bearer ` to `POST /api/reports`. Set `TCALC_DATA_DIR`, `TCALC_DASHBOARD_HOST`, and `PORT` when self-hosting on a low-cost Node service. + +## Who can read saved reports + +Saved reports are **private by default**. A report listing exposes workspace paths, per-language +totals, and file-level metadata, so both `GET /api/reports` and `GET /api/reports/:id` require +authentication. + +| Caller | Read reports | Upload reports | +| --- | --- | --- | +| `Authorization: Bearer ` | Yes | Yes | +| Browser session cookie | Yes | No | +| No credentials | No | No | +| No credentials, with `TCALC_DASHBOARD_PUBLIC_READ` enabled | Yes | No | + +If `TCALC_DASHBOARD_TOKEN` is unset, the service has no way to authenticate anyone: uploads and +reads both return `503`, and nothing stored in `TCALC_DATA_DIR` is served. + +### Reading reports in a browser + +The dashboard UI is a static bundle, so it never contains the token. When the saved-reports list +returns `401`, the page offers an unlock form. It sends the token once to `POST /api/session`, which +returns an opaque session id in an `HttpOnly`, `SameSite=Strict` cookie that expires after 12 hours. +The token itself is never stored in the browser and page scripts cannot read the cookie. + +That cookie grants **read-only** access. Uploads always require the bearer token, so a cross-site +page cannot use an operator's cookie to forge a report. `DELETE /api/session` ends the session. + +Any non-loopback deployment — anywhere `TCALC_DASHBOARD_HOST` is not `127.0.0.1` — must be served +over HTTPS, typically behind a trusted reverse proxy that terminates TLS. `HttpOnly` and +`SameSite=Strict` protect the session cookie from page scripts and cross-site misuse, but they do +not protect the token or the cookie in transit: over plain HTTP the operator's token and every +request carrying the cookie travel in cleartext, readable by anyone on the network path. Only TLS +protects data in transit. The server marks the cookie `Secure` once it sees an HTTPS request, +directly or through `x-forwarded-proto`. + +### Opting into public reads + +Set `TCALC_DASHBOARD_PUBLIC_READ` to `1` (or `true`, `yes`, `on`) only when saved reports are +deliberately meant to be world-readable — for example an internal demo with synthetic data: + +```powershell +$env:TCALC_DASHBOARD_PUBLIC_READ = "1" +``` + +This allows unauthenticated `GET` on every stored report to anyone who can reach the port. It never +enables unauthenticated uploads. The service logs a warning at startup while it is on. Leave it +unset for any deployment holding real workspace data, especially with a non-loopback +`TCALC_DASHBOARD_HOST`. diff --git a/packages/agent-rules/src/generateCopilotInstructions.ts b/packages/agent-rules/src/generateCopilotInstructions.ts new file mode 100644 index 0000000..62a2085 --- /dev/null +++ b/packages/agent-rules/src/generateCopilotInstructions.ts @@ -0,0 +1,76 @@ +import type { OptimizationMode, AgentRulesOutput } from "@wma/core"; +import { getTokenBudget } from "./generateRules.js"; + +export interface CopilotInstructionsOptions { + mode?: OptimizationMode; + workspaceTokens?: number; + projectDescription?: string; + codingConventions?: string[]; +} + +export function generateCopilotInstructions( + options: CopilotInstructionsOptions = {}, +): AgentRulesOutput { + const mode = options.mode ?? "normal"; + const tokens = options.workspaceTokens ?? 0; + const description = options.projectDescription ?? "Local-first token and context optimization workspace"; + + const avoidPatterns = [ + "node_modules/", + ".git/", + "dist/", + "build/", + "*.min.js", + "*.bundle.js", + "coverage/", + ]; + + const lines: string[] = [ + `# GitHub Copilot Instructions (.github/copilot-instructions.md)`, + `# Generated by TCalc (Local Workspace Token Calculator & Optimizer)`, + ``, + `## Project Overview`, + `- Description: ${description}`, + `- Workspace Token Budget: ~${tokens.toLocaleString("en-US")} tokens`, + `- Optimization Preference: ${mode}`, + ``, + `## Coding Conventions & Style`, + `- Language: TypeScript strict mode enabled throughout.`, + `- Modularity: Functional programming preferred over deep class inheritance.`, + `- Documentation: Preserve existing TSDoc/JSDoc comments on unchanged code.`, + `- Safety: Zero telemetry, zero external network calls in core packages.`, + ]; + + if (options.codingConventions && options.codingConventions.length > 0) { + for (const conv of options.codingConventions) { + lines.push(`- ${conv}`); + } + } + + lines.push( + ``, + `## Pull Request Readiness`, + `- Always verify code with unit tests before declaring completion.`, + `- Ensure changes have zero merge conflicts against upstream branches.`, + `- Avoid introducing new external dependencies without explicit approval.`, + ``, + `## Context Guardrails & Excluded Artifacts`, + ); + + for (const p of avoidPatterns) { + lines.push(`- Exclude from suggestions: \`${p}\``); + } + + lines.push(""); + + const content = lines.join("\n"); + + return { + target: "generic" as any, + fileName: ".github/copilot-instructions.md", + content, + mode, + tokenBudget: getTokenBudget(mode), + avoidFiles: avoidPatterns, + }; +} diff --git a/packages/agent-rules/src/generateGeminiRules.ts b/packages/agent-rules/src/generateGeminiRules.ts new file mode 100644 index 0000000..db7e074 --- /dev/null +++ b/packages/agent-rules/src/generateGeminiRules.ts @@ -0,0 +1,76 @@ +import type { OptimizationMode, AgentRulesOutput } from "@wma/core"; +import { getTokenBudget } from "./generateRules.js"; + +export interface GeminiRulesOptions { + mode?: OptimizationMode; + workspaceTokens?: number; + systemRole?: string; + toolingNotes?: string[]; +} + +export function generateGeminiRules(options: GeminiRulesOptions = {}): AgentRulesOutput { + const mode = options.mode ?? "normal"; + const tokens = options.workspaceTokens ?? 0; + const systemRole = options.systemRole ?? "Senior Full-Stack Architect & Optimization Specialist"; + + const avoidPatterns = [ + ".git/", + "node_modules/", + "dist/", + "build/", + "*.min.js", + "*.lock", + "*.log", + "*.tmp", + ]; + + const lines: string[] = [ + `# Gemini System Prompt & Guidelines (GEMINI.md)`, + `# Generated by TCalc (Local Workspace Token Calculator & Optimizer)`, + ``, + `## Persona & Role`, + `- Role: ${systemRole}`, + `- Workspace Token Footprint: ~${tokens.toLocaleString("en-US")} tokens`, + `- Mode: ${mode}`, + ``, + `## Reasoning & Planning Policy`, + `- For complex multi-file refactoring, enter Planning Mode and create an implementation plan before edits.`, + `- Verify code behavior using unit tests before concluding tasks.`, + `- Strictly preserve documentation integrity and comment headers.`, + ``, + `## Tooling & Execution Directives`, + ]; + + if (options.toolingNotes && options.toolingNotes.length > 0) { + for (const note of options.toolingNotes) { + lines.push(`- ${note}`); + } + } else { + lines.push( + `- Prefer precise file replacement chunks over full-file overwrites.`, + `- Do not run destructive git commands or commit directly without verification.`, + ); + } + + lines.push( + ``, + `## Token Boundaries & Excluded Paths`, + ); + + for (const p of avoidPatterns) { + lines.push(`- Ignore from workspace search: \`${p}\``); + } + + lines.push(""); + + const content = lines.join("\n"); + + return { + target: "generic" as any, + fileName: "GEMINI.md", + content, + mode, + tokenBudget: getTokenBudget(mode), + avoidFiles: avoidPatterns, + }; +} diff --git a/packages/agent-rules/src/generateRules.ts b/packages/agent-rules/src/generateRules.ts index c4142aa..1a639e1 100644 --- a/packages/agent-rules/src/generateRules.ts +++ b/packages/agent-rules/src/generateRules.ts @@ -32,7 +32,7 @@ function getFileName(target: AgentTarget): string { } } -function getTokenBudget(mode: OptimizationMode): number { +export function getTokenBudget(mode: OptimizationMode): number { switch (mode) { case "concise": return 16000; diff --git a/packages/agent-rules/src/generateWindsurfRules.ts b/packages/agent-rules/src/generateWindsurfRules.ts new file mode 100644 index 0000000..b1e4b82 --- /dev/null +++ b/packages/agent-rules/src/generateWindsurfRules.ts @@ -0,0 +1,103 @@ +import type { OptimizationMode, AgentRulesOutput } from "@wma/core"; +import { getTokenBudget } from "./generateRules.js"; + +export interface WindsurfRulesOptions { + mode?: OptimizationMode; + workspaceTokens?: number; + modelRecommendations?: string[]; + testCommand?: string; + buildCommand?: string; +} + +export function generateWindsurfRules(options: WindsurfRulesOptions = {}): AgentRulesOutput { + const mode = options.mode ?? "normal"; + const tokens = options.workspaceTokens ?? 0; + const testCmd = options.testCommand ?? "pnpm test"; + const buildCmd = options.buildCommand ?? "pnpm build"; + + const avoidPatterns = [ + "dist/", + "node_modules/", + ".git/", + "*.min.js", + "*.bundle.js", + "*.lock", + "*.log", + "coverage/", + ]; + + const lines: string[] = [ + `# Windsurf Workspace Rules (.windsurfrules)`, + `# Generated by TCalc (Local Workspace Token Calculator & Optimizer)`, + ``, + `## Workspace Context & Token Budget`, + `- Estimated Workspace Size: ~${tokens.toLocaleString("en-US")} tokens`, + `- Optimization Mode: ${mode}`, + ]; + + if (options.modelRecommendations && options.modelRecommendations.length > 0) { + lines.push(`- Recommended Models: ${options.modelRecommendations.join(", ")}`); + } + + lines.push( + ``, + `## Cascade Agent Execution Guidelines`, + `- Read workspace architecture before making invasive modifications.`, + `- Use focused edits; prefer targeted replacements over full-file rewrites.`, + `- Never modify or regenerate files matching ignore patterns below.`, + `- Maintain existing code style, documentation comments, and typing annotations.`, + ``, + `## Verification & Quality Gates`, + `- Build Command: \`${buildCmd}\``, + `- Test Command: \`${testCmd}\``, + `- Run tests after every discrete component change to verify no regression.`, + ``, + `## File Exclusion Boundaries`, + ); + + for (const pattern of avoidPatterns) { + lines.push(`- Avoid reading/indexing: \`${pattern}\``); + } + + lines.push( + ``, + `## Behavioral Mode Instructions`, + ); + + switch (mode) { + case "concise": + lines.push( + `- Be extremely direct. State the change and rationale in 2-3 sentences.`, + `- Omit conversational filler, disclaimers, and pleasantries.`, + ); + break; + case "patch-only": + lines.push( + `- Output only unified diffs or minimal code replacement chunks.`, + `- Zero conversational explanations unless specifically requested.`, + ); + break; + case "caveman": + lines.push( + `- Terse bullet points only. Maximum brevity.`, + ); + break; + default: + lines.push( + `- Provide balanced explanations with clear code snippets and verification notes.`, + ); + } + + lines.push(""); + + const content = lines.join("\n"); + + return { + target: "generic" as any, + fileName: ".windsurfrules", + content, + mode, + tokenBudget: getTokenBudget(mode), + avoidFiles: avoidPatterns, + }; +} diff --git a/packages/agent-rules/src/index.ts b/packages/agent-rules/src/index.ts index 08f9f27..ef40c04 100644 --- a/packages/agent-rules/src/index.ts +++ b/packages/agent-rules/src/index.ts @@ -1,3 +1,8 @@ export { generateAgentRules } from "./generateRules.js"; export { compactCompletedGoal } from "./compactGoal.js"; export type { GoalCompactionInput, GoalCompactionOutput } from "./compactGoal.js"; +export { generateWindsurfRules } from "./generateWindsurfRules.js"; +export { generateCopilotInstructions } from "./generateCopilotInstructions.js"; +export { generateGeminiRules } from "./generateGeminiRules.js"; +export { generateMultiFormatRules } from "./multiFormatRules.js"; +export type { ExtendedAgentFormat, MultiFormatRulesOptions } from "./multiFormatRules.js"; diff --git a/packages/agent-rules/src/multiFormatRules.ts b/packages/agent-rules/src/multiFormatRules.ts new file mode 100644 index 0000000..b284874 --- /dev/null +++ b/packages/agent-rules/src/multiFormatRules.ts @@ -0,0 +1,101 @@ +import type { OptimizationMode, AgentRulesOutput } from "@wma/core"; +import { generateAgentRules } from "./generateRules.js"; +import { generateWindsurfRules } from "./generateWindsurfRules.js"; +import { generateCopilotInstructions } from "./generateCopilotInstructions.js"; +import { generateGeminiRules } from "./generateGeminiRules.js"; + +export type ExtendedAgentFormat = + | "cursor" + | "claude-code" + | "generic" + | "windsurf" + | "copilot" + | "gemini" + | "all"; + +export interface MultiFormatRulesOptions { + mode?: OptimizationMode; + workspaceTokens?: number; + modelRecommendations?: string[]; + testCommand?: string; + buildCommand?: string; + projectDescription?: string; + codingConventions?: string[]; + systemRole?: string; +} + +export function generateMultiFormatRules( + format: ExtendedAgentFormat, + options: MultiFormatRulesOptions = {}, +): AgentRulesOutput | Record { + const mode = options.mode ?? "normal"; + + switch (format) { + case "cursor": + return generateAgentRules({ + target: "cursor", + mode, + workspaceTokens: options.workspaceTokens, + modelRecommendations: options.modelRecommendations, + }); + case "claude-code": + return generateAgentRules({ + target: "claude-code", + mode, + workspaceTokens: options.workspaceTokens, + modelRecommendations: options.modelRecommendations, + }); + case "generic": + return generateAgentRules({ + target: "generic", + mode, + workspaceTokens: options.workspaceTokens, + modelRecommendations: options.modelRecommendations, + }); + case "windsurf": + return generateWindsurfRules({ + mode, + workspaceTokens: options.workspaceTokens, + modelRecommendations: options.modelRecommendations, + testCommand: options.testCommand, + buildCommand: options.buildCommand, + }); + case "copilot": + return generateCopilotInstructions({ + mode, + workspaceTokens: options.workspaceTokens, + projectDescription: options.projectDescription, + codingConventions: options.codingConventions, + }); + case "gemini": + return generateGeminiRules({ + mode, + workspaceTokens: options.workspaceTokens, + systemRole: options.systemRole, + }); + case "all": + return { + cursor: generateAgentRules({ + target: "cursor", + mode, + workspaceTokens: options.workspaceTokens, + modelRecommendations: options.modelRecommendations, + }), + claude: generateAgentRules({ + target: "claude-code", + mode, + workspaceTokens: options.workspaceTokens, + modelRecommendations: options.modelRecommendations, + }), + generic: generateAgentRules({ + target: "generic", + mode, + workspaceTokens: options.workspaceTokens, + modelRecommendations: options.modelRecommendations, + }), + windsurf: generateWindsurfRules(options), + copilot: generateCopilotInstructions(options), + gemini: generateGeminiRules(options), + }; + } +} diff --git a/packages/agent-rules/tests/multiFormatRules.test.ts b/packages/agent-rules/tests/multiFormatRules.test.ts new file mode 100644 index 0000000..da3c99c --- /dev/null +++ b/packages/agent-rules/tests/multiFormatRules.test.ts @@ -0,0 +1,128 @@ +import { describe, it, expect } from "vitest"; +import { + generateWindsurfRules, + generateCopilotInstructions, + generateGeminiRules, + generateMultiFormatRules, +} from "../src/index.js"; +import { isOptimizationMode, type AgentRulesOutput } from "@wma/core"; + +describe("MultiFormat Agent Rules", () => { + it("generates valid Windsurf rules with custom build and test commands", () => { + const rules = generateWindsurfRules({ + mode: "concise", + workspaceTokens: 45000, + testCommand: "pnpm test:unit", + buildCommand: "pnpm build:fast", + modelRecommendations: ["Claude 3.7 Sonnet", "GPT-4o"], + }); + + expect(rules.fileName).toBe(".windsurfrules"); + expect(rules.content).toContain("# Windsurf Workspace Rules"); + expect(rules.content).toContain("~45,000 tokens"); + expect(rules.content).toContain("Claude 3.7 Sonnet"); + expect(rules.content).toContain("pnpm test:unit"); + expect(rules.content).toContain("pnpm build:fast"); + expect(rules.content).toContain("Be extremely direct"); + expect(rules.avoidFiles).toContain("dist/"); + expect(rules.tokenBudget).toBe(16000); + }); + + it("generates GitHub Copilot instructions with PR readiness guidelines", () => { + const rules = generateCopilotInstructions({ + mode: "patch-only", + workspaceTokens: 32000, + projectDescription: "TCalc token estimation workspace", + codingConventions: ["Use Vitest for all unit tests", "No external telemetry"], + }); + + expect(rules.fileName).toBe(".github/copilot-instructions.md"); + expect(rules.content).toContain("# GitHub Copilot Instructions"); + expect(rules.content).toContain("TCalc token estimation workspace"); + expect(rules.content).toContain("Use Vitest for all unit tests"); + expect(rules.content).toContain("No external telemetry"); + expect(rules.content).toContain("## Pull Request Readiness"); + expect(rules.avoidFiles).toContain("node_modules/"); + }); + + it("generates Gemini system instructions with Planning Mode guidelines", () => { + const rules = generateGeminiRules({ + mode: "normal", + workspaceTokens: 120000, + systemRole: "Staff AI Solutions Architect", + toolingNotes: ["Preserve all docstrings", "Execute Vitest checks"], + }); + + expect(rules.fileName).toBe("GEMINI.md"); + expect(rules.content).toContain("# Gemini System Prompt & Guidelines"); + expect(rules.content).toContain("Staff AI Solutions Architect"); + expect(rules.content).toContain("~120,000 tokens"); + expect(rules.content).toContain("Planning Mode"); + expect(rules.content).toContain("Preserve all docstrings"); + }); + + it("supports caveman mode in Windsurf rules", () => { + const rules = generateWindsurfRules({ mode: "caveman" }); + expect(rules.content).toContain("Terse bullet points only. Maximum brevity."); + }); + + it("dispatches single formats via generateMultiFormatRules", () => { + const windsurf = generateMultiFormatRules("windsurf", { workspaceTokens: 1000 }) as AgentRulesOutput; + expect(windsurf.fileName).toBe(".windsurfrules"); + + const copilot = generateMultiFormatRules("copilot", { workspaceTokens: 2000 }) as AgentRulesOutput; + expect(copilot.fileName).toBe(".github/copilot-instructions.md"); + + const gemini = generateMultiFormatRules("gemini", { workspaceTokens: 3000 }) as AgentRulesOutput; + expect(gemini.fileName).toBe("GEMINI.md"); + + const cursor = generateMultiFormatRules("cursor", { workspaceTokens: 4000 }) as AgentRulesOutput; + expect(cursor.fileName).toBe(".cursorrules"); + }); + + it("bundles all rule formats when requested format is 'all'", () => { + const bundle = generateMultiFormatRules("all", { + workspaceTokens: 50000, + mode: "concise", + }) as Record; + + expect(bundle).toHaveProperty("cursor"); + expect(bundle).toHaveProperty("claude"); + expect(bundle).toHaveProperty("generic"); + expect(bundle).toHaveProperty("windsurf"); + expect(bundle).toHaveProperty("copilot"); + expect(bundle).toHaveProperty("gemini"); + + expect(bundle.windsurf.fileName).toBe(".windsurfrules"); + expect(bundle.copilot.fileName).toBe(".github/copilot-instructions.md"); + expect(bundle.gemini.fileName).toBe("GEMINI.md"); + }); + it("preserves model recommendations when all rule formats are generated", () => { + const recommendations = ["Claude 3.7 Sonnet", "GPT-4o"]; + + const bundle = generateMultiFormatRules("all", { + workspaceTokens: 50000, + mode: "concise", + modelRecommendations: recommendations, + }) as Record; + + expect(bundle.cursor.content).toContain("Claude 3.7 Sonnet"); + expect(bundle.cursor.content).toContain("GPT-4o"); + + expect(bundle.claude.content).toContain("Claude 3.7 Sonnet"); + expect(bundle.claude.content).toContain("GPT-4o"); + + expect(bundle.generic.content).toContain("Claude 3.7 Sonnet"); + expect(bundle.generic.content).toContain("GPT-4o"); + }); + it("defaults every format to a valid optimization mode and token budget", () => { + const bundle = generateMultiFormatRules("all", { workspaceTokens: 1000 }) as Record; + + for (const output of Object.values(bundle)) { + expect(output.mode).toBe("normal"); + expect(isOptimizationMode(output.mode)).toBe(true); + expect(output.tokenBudget).toBe(64000); + expect(output.content).not.toContain("Mode: balanced"); + } + }); +}); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 34d0c2c..e8cf754 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -9,3 +9,4 @@ export type * from "./types/repoMap.js"; export * from "./config/defaultConfig.js"; export * from "./markdown.js"; export * from "./teamPolicy.js"; +export type * from "./types/drift.js"; diff --git a/packages/core/src/teamPolicy.ts b/packages/core/src/teamPolicy.ts index c97754b..d143359 100644 --- a/packages/core/src/teamPolicy.ts +++ b/packages/core/src/teamPolicy.ts @@ -1,6 +1,11 @@ import { readFile } from "node:fs/promises"; import path from "node:path"; -import { isPrivacySetting, isWorkspaceGoal, type PrivacySetting, type WorkspaceGoal } from "./types/options.js"; +import { + isPrivacySetting, + isWorkspaceGoal, + type PrivacySetting, + type WorkspaceGoal, +} from "./types/options.js"; export const TEAM_POLICY_SCHEMA_VERSION = "1.0" as const; @@ -21,9 +26,15 @@ export interface TeamPolicy { activeProfile?: string; } -export async function loadTeamPolicy(rootPath: string): Promise { +export async function loadTeamPolicy( + rootPath: string, +): Promise { try { - return parseTeamPolicy(JSON.parse(await readFile(path.join(rootPath, ".tcalc", "team.json"), "utf8"))); + return parseTeamPolicy( + JSON.parse( + await readFile(path.join(rootPath, ".tcalc", "team.json"), "utf8"), + ), + ); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; throw error; @@ -31,14 +42,37 @@ export async function loadTeamPolicy(rootPath: string): Promise profile.id === value.activeProfile)) throw new TypeError(`Team policy activeProfile "${value.activeProfile}" was not found`); + const profiles = + value.modelProfiles === undefined + ? undefined + : parseProfiles(value.modelProfiles); + if (value.activeProfile !== undefined) { + if (typeof value.activeProfile !== "string") + throw new TypeError("Team policy activeProfile must be a string"); + if (!value.activeProfile.trim()) + throw new TypeError("Team policy activeProfile must not be blank"); + if (!profiles?.some((profile) => profile.id === value.activeProfile)) + throw new TypeError( + `Team policy activeProfile "${value.activeProfile}" was not found`, + ); + } return { schemaVersion: TEAM_POLICY_SCHEMA_VERSION, defaultGoal: value.defaultGoal as WorkspaceGoal | undefined, @@ -50,29 +84,49 @@ export function parseTeamPolicy(value: unknown): TeamPolicy { }; } -export function getActiveModelProfile(policy: TeamPolicy | undefined): ModelProfile | undefined { - return policy?.modelProfiles?.find((profile) => profile.id === policy.activeProfile); +export function getActiveModelProfile( + policy: TeamPolicy | undefined, +): ModelProfile | undefined { + return policy?.modelProfiles?.find( + (profile) => profile.id === policy.activeProfile, + ); } function parseProfiles(value: unknown): ModelProfile[] { - if (!Array.isArray(value)) throw new TypeError("Team policy modelProfiles must be an array"); + if (!Array.isArray(value)) + throw new TypeError("Team policy modelProfiles must be an array"); const ids = new Set(); return value.map((entry) => { - if (!isRecord(entry) || typeof entry.id !== "string" || !entry.id) throw new TypeError("Every model profile needs an id"); - if (ids.has(entry.id)) throw new TypeError(`Duplicate model profile id "${entry.id}"`); + if (!isRecord(entry) || typeof entry.id !== "string" || !entry.id.trim()) + throw new TypeError("Every model profile needs an id"); + if (ids.has(entry.id)) + throw new TypeError(`Duplicate model profile id "${entry.id}"`); ids.add(entry.id); return { id: entry.id, - allowedProviders: optionalStrings(entry.allowedProviders, "allowedProviders"), - allowedModelIds: optionalStrings(entry.allowedModelIds, "allowedModelIds"), - preferredModelIds: optionalStrings(entry.preferredModelIds, "preferredModelIds"), + allowedProviders: optionalStrings( + entry.allowedProviders, + "allowedProviders", + ), + allowedModelIds: optionalStrings( + entry.allowedModelIds, + "allowedModelIds", + ), + preferredModelIds: optionalStrings( + entry.preferredModelIds, + "preferredModelIds", + ), }; }); } function optionalStrings(value: unknown, field: string): string[] | undefined { if (value === undefined) return undefined; - if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || !item)) throw new TypeError(`Team policy ${field} must contain strings`); + if ( + !Array.isArray(value) || + value.some((item) => typeof item !== "string" || !item) + ) + throw new TypeError(`Team policy ${field} must contain strings`); return value; } diff --git a/packages/core/src/types/drift.ts b/packages/core/src/types/drift.ts new file mode 100644 index 0000000..9f901ed --- /dev/null +++ b/packages/core/src/types/drift.ts @@ -0,0 +1,45 @@ +export type DriftStatus = "added" | "removed" | "modified" | "unchanged"; + +export interface FileDriftDelta { + relativePath: string; + status: DriftStatus; + baselineTokens: number; + currentTokens: number; + tokenDelta: number; + percentChange: number; + riskFlagsAdded: string[]; + riskFlagsRemoved: string[]; +} + +export interface DirectoryDriftDelta { + directory: string; + baselineTokens: number; + currentTokens: number; + tokenDelta: number; + percentChange: number; + fileCountDelta: number; +} + +export interface DriftPolicyConfig { + maxTokenIncrease?: number; + maxDriftPercentage?: number; + alertOnNewRiskFlags?: boolean; + maxFileCountIncrease?: number; +} + +export interface TokenDriftReport { + baselineTotalTokens: number; + currentTotalTokens: number; + netTokenDelta: number; + netPercentChange: number; + filesAdded: number; + filesRemoved: number; + filesModified: number; + filesUnchanged: number; + topGrowingFiles: FileDriftDelta[]; + topShrinkingFiles: FileDriftDelta[]; + directoryDrifts: DirectoryDriftDelta[]; + violations: string[]; + passed: boolean; + generatedAt: string; +} diff --git a/packages/core/src/types/options.ts b/packages/core/src/types/options.ts index fdacb7d..63882f6 100644 --- a/packages/core/src/types/options.ts +++ b/packages/core/src/types/options.ts @@ -42,6 +42,7 @@ export const OPTIMIZATION_MODES = [ export type OptimizationMode = (typeof OPTIMIZATION_MODES)[number]; +// local-first means strict local execution: only models with verified supportsLocal=true are eligible. export const PRIVACY_SETTINGS = ["local-first", "cloud-ok"] as const; export type PrivacySetting = (typeof PRIVACY_SETTINGS)[number]; diff --git a/packages/core/tests/teamPolicy.test.ts b/packages/core/tests/teamPolicy.test.ts index 2f90325..159f99b 100644 --- a/packages/core/tests/teamPolicy.test.ts +++ b/packages/core/tests/teamPolicy.test.ts @@ -14,6 +14,96 @@ describe("team policies", () => { }); it("rejects an unknown active profile", () => { - expect(() => parseTeamPolicy({ schemaVersion: "1.0", activeProfile: "missing", modelProfiles: [] })).toThrow(/not found/); + expect(() => + parseTeamPolicy({ + schemaVersion: "1.0", + activeProfile: "missing", + modelProfiles: [], + }), + ).toThrow(/not found/); + }); + + it("allows an omitted active profile", () => { + const policy = parseTeamPolicy({ + schemaVersion: "1.0", + modelProfiles: [{ id: "local", allowedProviders: ["local"] }], + }); + expect(policy.activeProfile).toBeUndefined(); + expect(getActiveModelProfile(policy)).toBeUndefined(); + }); + + it("rejects an empty active profile string", () => { + expect(() => + parseTeamPolicy({ + schemaVersion: "1.0", + modelProfiles: [{ id: "local", allowedProviders: ["local"] }], + activeProfile: "", + }), + ).toThrow(/Team policy activeProfile must not be blank/); + }); + + it("rejects a whitespace-only active profile string", () => { + expect(() => + parseTeamPolicy({ + schemaVersion: "1.0", + modelProfiles: [{ id: "local", allowedProviders: ["local"] }], + activeProfile: " ", + }), + ).toThrow(/Team policy activeProfile must not be blank/); + }); + + it("rejects an active profile when modelProfiles is omitted", () => { + expect(() => + parseTeamPolicy({ + schemaVersion: "1.0", + activeProfile: "local", + }), + ).toThrow(/not found/); + }); + + it("rejects a non-string active profile", () => { + expect(() => + parseTeamPolicy({ + schemaVersion: "1.0", + modelProfiles: [{ id: "local" }], + activeProfile: 123, + }), + ).toThrow(/must be a string/); + }); + + it("rejects an empty model profile id", () => { + expect(() => + parseTeamPolicy({ + schemaVersion: "1.0", + modelProfiles: [{ id: "" }], + }), + ).toThrow(/needs an id/i); + }); + + it("rejects a whitespace-only model profile id", () => { + expect(() => + parseTeamPolicy({ + schemaVersion: "1.0", + modelProfiles: [{ id: " " }], + }), + ).toThrow(/needs an id/i); + }); + + it("rejects a model profile missing an id", () => { + expect(() => + parseTeamPolicy({ + schemaVersion: "1.0", + modelProfiles: [{}], + }), + ).toThrow(/needs an id/i); + }); + + it("rejects duplicate model profile ids", () => { + expect(() => + parseTeamPolicy({ + schemaVersion: "1.0", + modelProfiles: [{ id: "local" }, { id: "local" }], + }), + ).toThrow(/duplicate model profile id/i); }); }); diff --git a/packages/mcp-server/package.json b/packages/mcp-server/package.json index 21bd5bc..1127eda 100644 --- a/packages/mcp-server/package.json +++ b/packages/mcp-server/package.json @@ -25,6 +25,7 @@ "@wma/repo-map": "workspace:*", "@wma/reports": "workspace:*", "@wma/scanner": "workspace:*", + "@wma/tokenizers": "workspace:*", "zod": "^3.23.0" }, "devDependencies": { diff --git a/packages/mcp-server/src/server.ts b/packages/mcp-server/src/server.ts index 9c7c621..58f5ecc 100644 --- a/packages/mcp-server/src/server.ts +++ b/packages/mcp-server/src/server.ts @@ -35,6 +35,7 @@ import { type ValidateModelCatalogInput, } from "./tools/validateModelCatalogTool.js"; import { handleCompactCompletedGoal } from "./tools/compactCompletedGoalTool.js"; +import { handleEstimateFilesTokens } from "./tools/estimateFilesTokensTool.js"; import { readWorkspaceSummary } from "./resources/workspaceSummaryResource.js"; import { readModelCatalog } from "./resources/modelCatalogResource.js"; import { @@ -69,6 +70,7 @@ export function createServer(): Server { rootPath: { type: "string", description: "Root path to scan (defaults to current working directory)" }, goal: { type: "string", enum: [...WORKSPACE_GOALS], description: "Workspace goal included in the summary" }, privacyMode: { type: "string", enum: [...PRIVACY_SETTINGS] }, + timeoutMs: { type: "integer", minimum: 1, maximum: 300000, description: "Scan timeout in milliseconds (default 60000, max 300000)" }, }, }, }, @@ -156,6 +158,34 @@ export function createServer(): Server { }, }, }, + { + name: "estimate_files_tokens", + description: "Estimate token count and context window fit for an explicit batch of workspace files.", + inputSchema: { + type: "object", + required: ["filePaths"], + properties: { + filePaths: { + type: "array", + items: { type: "string" }, + description: "List of file paths relative to workspace root", + }, + workspaceRoot: { + type: "string", + description: "Target workspace root path (defaults to current allowed root)", + }, + targetModel: { + type: "string", + description: "Optional model name to evaluate context window compatibility", + }, + contextBudget: { + type: "integer", + minimum: 1, + description: "Optional token budget to evaluate utilization percentage", + }, + }, + }, + }, ], }; }); @@ -178,6 +208,8 @@ export function createServer(): Server { return handleValidateModelCatalog(args ?? {}); case "compact_completed_goal": return handleCompactCompletedGoal(args ?? {}); + case "estimate_files_tokens": + return handleEstimateFilesTokens(args ?? {}); default: return { content: [ diff --git a/packages/mcp-server/src/tools/createRepoMapTool.ts b/packages/mcp-server/src/tools/createRepoMapTool.ts index 16e0017..59b9d66 100644 --- a/packages/mcp-server/src/tools/createRepoMapTool.ts +++ b/packages/mcp-server/src/tools/createRepoMapTool.ts @@ -29,7 +29,7 @@ export async function handleCreateRepoMap(input: Record) { const repoMap = createRepoMap(scanResult, { tokenBudget, - goal: (policy?.defaultGoal ?? parsed.goal) as WorkspaceGoal | undefined, + goal: (parsed.goal ?? policy?.defaultGoal) as WorkspaceGoal | undefined, enableSymbolExtraction: parsed.enableSymbolExtraction ?? true, maxSymbols: parsed.maxSymbols ?? 500, maxParseFileBytes: 200000, diff --git a/packages/mcp-server/src/tools/estimateFilesTokensTool.ts b/packages/mcp-server/src/tools/estimateFilesTokensTool.ts new file mode 100644 index 0000000..ffe80d7 --- /dev/null +++ b/packages/mcp-server/src/tools/estimateFilesTokensTool.ts @@ -0,0 +1,179 @@ +import fs from "node:fs"; +import path from "node:path"; +import { estimateFileTokens } from "@wma/tokenizers"; +import { z } from "zod"; +import { validateRootPath } from "../utils/safeRootPath.js"; + +export const EstimateFilesTokensInputSchema = z.object({ + filePaths: z.array(z.string()).min(1, "At least one file path must be specified"), + workspaceRoot: z.string().optional(), + targetModel: z.string().optional(), + contextBudget: z.number().int().positive().optional(), +}); + +export type EstimateFilesTokensInput = z.infer; + +export interface FileTokenDetail { + filePath: string; + relativePath: string; + bytes: number; + estimatedTokens: number; + exists: boolean; + error?: string; +} + +export interface EstimateFilesTokensOutput { + workspaceRoot: string; + totalFiles: number; + existingFiles: number; + totalBytes: number; + totalTokens: number; + targetModel?: string; + contextBudget?: number; + fitsBudget?: boolean; + budgetUtilizationPercent?: number; + files: FileTokenDetail[]; +} + +function isContained(root: string, target: string): boolean { + const rel = path.relative(root, target); + return rel === "" || (!rel.startsWith(`..${path.sep}`) && rel !== ".." && !path.isAbsolute(rel)); +} + +export async function handleEstimateFilesTokens(args: unknown): Promise<{ + content: Array<{ type: "text"; text: string }>; + isError?: boolean; +}> { + const parseResult = EstimateFilesTokensInputSchema.safeParse(args); + if (!parseResult.success) { + return { + content: [ + { + type: "text" as const, + text: `Invalid arguments: ${parseResult.error.message}`, + }, + ], + isError: true, + }; + } + + const { filePaths, workspaceRoot: explicitRoot, targetModel, contextBudget } = parseResult.data; + let root: string; + try { + root = validateRootPath(explicitRoot); + } catch (err) { + return { + content: [ + { + type: "text" as const, + text: err instanceof Error ? err.message : String(err), + }, + ], + isError: true, + }; + } + + const fileDetails: FileTokenDetail[] = []; + let totalBytes = 0; + let totalTokens = 0; + let existingFiles = 0; + + for (const fp of filePaths) { + const resolvedPath = path.isAbsolute(fp) ? path.resolve(fp) : path.resolve(root, fp); + + if (!isContained(root, resolvedPath)) { + return { + content: [ + { + type: "text" as const, + text: `Path traversal denied: '${fp}' resolves outside workspace boundary '${root}'`, + }, + ], + isError: true, + }; + } + + const rel = path.relative(root, resolvedPath).replace(/\\/g, "/"); + + try { + if (!fs.existsSync(resolvedPath)) { + fileDetails.push({ + filePath: resolvedPath, + relativePath: rel, + bytes: 0, + estimatedTokens: 0, + exists: false, + error: "File not found", + }); + continue; + } + + const stat = fs.statSync(resolvedPath); + if (stat.isDirectory()) { + fileDetails.push({ + filePath: resolvedPath, + relativePath: rel, + bytes: stat.size, + estimatedTokens: 0, + exists: true, + error: "Path is a directory, not a file", + }); + continue; + } + + const content = fs.readFileSync(resolvedPath, "utf-8"); + const tokens = estimateFileTokens(content, rel); + + existingFiles++; + totalBytes += stat.size; + totalTokens += tokens; + + fileDetails.push({ + filePath: resolvedPath, + relativePath: rel, + bytes: stat.size, + estimatedTokens: tokens, + exists: true, + }); + } catch (err) { + fileDetails.push({ + filePath: resolvedPath, + relativePath: rel, + bytes: 0, + estimatedTokens: 0, + exists: false, + error: err instanceof Error ? err.message : String(err), + }); + } + } + + let fitsBudget: boolean | undefined; + let budgetUtilizationPercent: number | undefined; + + if (contextBudget !== undefined && contextBudget > 0) { + fitsBudget = totalTokens <= contextBudget; + budgetUtilizationPercent = Number(((totalTokens / contextBudget) * 100).toFixed(2)); + } + + const output: EstimateFilesTokensOutput = { + workspaceRoot: root, + totalFiles: filePaths.length, + existingFiles, + totalBytes, + totalTokens, + targetModel, + contextBudget, + fitsBudget, + budgetUtilizationPercent, + files: fileDetails, + }; + + return { + content: [ + { + type: "text" as const, + text: JSON.stringify(output, null, 2), + }, + ], + }; +} diff --git a/packages/mcp-server/src/tools/generateAgentRulesTool.ts b/packages/mcp-server/src/tools/generateAgentRulesTool.ts index d94785c..2fc406d 100644 --- a/packages/mcp-server/src/tools/generateAgentRulesTool.ts +++ b/packages/mcp-server/src/tools/generateAgentRulesTool.ts @@ -31,7 +31,7 @@ export async function handleGenerateAgentRules(input: Record) { const target = parsed.target ?? "generic"; const mode = parsed.mode ?? "repo-map-first"; const policy = await loadTeamPolicy(rootPath); - const goal = policy?.defaultGoal ?? parsed.goal ?? "build-mvp"; + const goal = parsed.goal ?? policy?.defaultGoal ?? "build-mvp"; const privacyMode = policy?.privacyMode ?? parsed.privacyMode ?? "local-first"; const scanResult = await scanWorkspace({ rootPath, userExcludePatterns: policy?.exclude }); @@ -44,10 +44,10 @@ export async function handleGenerateAgentRules(input: Record) { const recommendation = models.length > 0 ? recommendModels({ models, + preferredModelIds: profile?.preferredModelIds, workspaceTokens: scanResult.includedTokens, goal, privacyMode, - preferredModelIds: profile?.preferredModelIds, }) : null; diff --git a/packages/mcp-server/src/tools/generateReportTool.ts b/packages/mcp-server/src/tools/generateReportTool.ts index 09cc601..35a746c 100644 --- a/packages/mcp-server/src/tools/generateReportTool.ts +++ b/packages/mcp-server/src/tools/generateReportTool.ts @@ -24,7 +24,7 @@ export async function handleGenerateReport(input: Record) { const rootPath = validateRootPath(parsed.rootPath); const policy = await loadTeamPolicy(rootPath); - const goal = policy?.defaultGoal ?? parsed.goal ?? "build-mvp"; + const goal = parsed.goal ?? policy?.defaultGoal ?? "build-mvp"; const privacyMode = policy?.privacyMode ?? parsed.privacyMode ?? "local-first"; const format = parsed.format ?? "markdown"; const includeRepoMap = parsed.includeRepoMap ?? false; @@ -44,11 +44,11 @@ export async function handleGenerateReport(input: Record) { if (models.length > 0) { recommendation = recommendModels({ models, + preferredModelIds: profile?.preferredModelIds, workspaceTokens: scanResult.includedTokens, goal, privacyMode, budget: policy?.maxTokenBudget, - preferredModelIds: profile?.preferredModelIds, }); setLatestRecommendation(recommendation); } diff --git a/packages/mcp-server/src/tools/recommendModelsTool.ts b/packages/mcp-server/src/tools/recommendModelsTool.ts index 3e3c4bf..8ddfd70 100644 --- a/packages/mcp-server/src/tools/recommendModelsTool.ts +++ b/packages/mcp-server/src/tools/recommendModelsTool.ts @@ -22,7 +22,7 @@ export async function handleRecommendModels(input: Record) { const rootPath = validateRootPath(parsed.rootPath); const policy = await loadTeamPolicy(rootPath); - const goal = policy?.defaultGoal ?? parsed.goal ?? "build-mvp"; + const goal = parsed.goal ?? policy?.defaultGoal ?? "build-mvp"; const privacyMode = policy?.privacyMode ?? parsed.privacyMode ?? "local-first"; let catalogPath: string; try { @@ -56,11 +56,11 @@ export async function handleRecommendModels(input: Record) { const recommendation = recommendModels({ models, + preferredModelIds: profile?.preferredModelIds, workspaceTokens: scanResult.includedTokens, goal, privacyMode, budget: policy?.maxTokenBudget ? Math.min(parsed.tokenBudget ?? policy.maxTokenBudget, policy.maxTokenBudget) : parsed.tokenBudget, - preferredModelIds: profile?.preferredModelIds, }); setLatestRecommendation(recommendation); diff --git a/packages/mcp-server/src/tools/scanWorkspaceTool.ts b/packages/mcp-server/src/tools/scanWorkspaceTool.ts index d291715..8e50628 100644 --- a/packages/mcp-server/src/tools/scanWorkspaceTool.ts +++ b/packages/mcp-server/src/tools/scanWorkspaceTool.ts @@ -9,6 +9,7 @@ const ScanWorkspaceInputSchema = z.object({ rootPath: z.string().optional(), goal: z.enum(WORKSPACE_GOALS).optional(), privacyMode: z.enum(PRIVACY_SETTINGS).optional(), + timeoutMs: z.number().int().positive().max(300_000).optional(), }).strict(); export type ScanWorkspaceInput = z.infer; @@ -22,6 +23,7 @@ export async function handleScanWorkspace(input: Record) { const scanResult = await scanWorkspace({ rootPath, userExcludePatterns: policy?.exclude, + signal: AbortSignal.timeout(parsed.timeoutMs ?? 60_000), }); setLatestScan(scanResult); diff --git a/packages/mcp-server/src/utils/compactResults.ts b/packages/mcp-server/src/utils/compactResults.ts index c819e77..fab41a5 100644 --- a/packages/mcp-server/src/utils/compactResults.ts +++ b/packages/mcp-server/src/utils/compactResults.ts @@ -12,27 +12,22 @@ export interface CompactWorkspaceSummary { warnings: string[]; } +export interface CompactModelRecommendation { + modelId: string; + displayName: string; + cost: string; + score: number; + overflowRisk: number; + warnings: string[]; + optimizationSuggestions: string[]; +} + export interface CompactRecommendationSummary { goal: string; workspaceTokens: number; - cheapestSufficient: { - modelId: string; - displayName: string; - cost: string; - score: number; - }; - balanced: { - modelId: string; - displayName: string; - cost: string; - score: number; - }; - highConfidence: { - modelId: string; - displayName: string; - cost: string; - score: number; - }; + cheapestSufficient: CompactModelRecommendation; + balanced: CompactModelRecommendation; + highConfidence: CompactModelRecommendation; rejected: string[]; assumptions: string[]; } @@ -87,18 +82,27 @@ export function createCompactRecommendationSummary(rec: RecommendationResult): C displayName: rec.cheapestSufficient.displayName, cost: formatCost(rec.cheapestSufficient.costEstimate.totalCost), score: rec.cheapestSufficient.score.totalScore, + overflowRisk: rec.cheapestSufficient.overflowRisk, + warnings: rec.cheapestSufficient.warnings, + optimizationSuggestions: rec.cheapestSufficient.optimizationSuggestions, }, balanced: { modelId: rec.balanced.modelId, displayName: rec.balanced.displayName, cost: formatCost(rec.balanced.costEstimate.totalCost), score: rec.balanced.score.totalScore, + overflowRisk: rec.balanced.overflowRisk, + warnings: rec.balanced.warnings, + optimizationSuggestions: rec.balanced.optimizationSuggestions, }, highConfidence: { modelId: rec.highConfidence.modelId, displayName: rec.highConfidence.displayName, cost: formatCost(rec.highConfidence.costEstimate.totalCost), score: rec.highConfidence.score.totalScore, + overflowRisk: rec.highConfidence.overflowRisk, + warnings: rec.highConfidence.warnings, + optimizationSuggestions: rec.highConfidence.optimizationSuggestions, }, rejected: rec.rejected, assumptions: rec.assumptions, diff --git a/packages/mcp-server/tests/estimateFilesTokensTool.test.ts b/packages/mcp-server/tests/estimateFilesTokensTool.test.ts new file mode 100644 index 0000000..5e2b0a9 --- /dev/null +++ b/packages/mcp-server/tests/estimateFilesTokensTool.test.ts @@ -0,0 +1,147 @@ +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { mkdtemp, rm, writeFile, mkdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { handleEstimateFilesTokens } from "../src/tools/estimateFilesTokensTool.js"; +import { createServer } from "../src/server.js"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +import { scanWorkspace } from "@wma/scanner"; + +describe("estimateFilesTokensTool", () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(path.join(tmpdir(), "tcalc-batch-tokens-")); + await mkdir(path.join(tempDir, "src"), { recursive: true }); + await writeFile(path.join(tempDir, "src", "index.ts"), "export const a = 1;\nexport const b = 2;\n"); + await writeFile(path.join(tempDir, "src", "utils.ts"), "export function add(x: number, y: number) { return x + y; }\n"); + process.env.WMA_ALLOWED_ROOT = tempDir; + }); + + afterEach(async () => { + delete process.env.WMA_ALLOWED_ROOT; + await rm(tempDir, { recursive: true, force: true }); + }); + + it("fails on invalid input missing filePaths", async () => { + const result = await handleEstimateFilesTokens({}); + expect(result.isError).toBe(true); + expect(result.content[0].text).toContain("Invalid arguments"); + }); + + it("fails on empty filePaths array", async () => { + const result = await handleEstimateFilesTokens({ filePaths: [] }); + expect(result.isError).toBe(true); + expect(result.content[0].text).toContain("At least one file path must be specified"); + }); + + it("estimates tokens for existing files accurately", async () => { + const result = await handleEstimateFilesTokens({ + workspaceRoot: tempDir, + filePaths: ["src/index.ts", "src/utils.ts"], + contextBudget: 1000, + }); + + expect(result.isError).toBeUndefined(); + const parsed = JSON.parse(result.content[0].text); + + expect(parsed.totalFiles).toBe(2); + expect(parsed.existingFiles).toBe(2); + expect(parsed.totalTokens).toBeGreaterThan(5); + expect(parsed.fitsBudget).toBe(true); + expect(parsed.budgetUtilizationPercent).toBeGreaterThan(0); + expect(parsed.files[0].relativePath).toBe("src/index.ts"); + expect(parsed.files[0].exists).toBe(true); + }); + + it("matches scanner token estimates for the same file", async () => { + const relativePath = "src/token-estimate.md"; + const content = "Shared tokenizer estimates should agree across packages.\n"; + await writeFile(path.join(tempDir, relativePath), content); + + const result = await handleEstimateFilesTokens({ + workspaceRoot: tempDir, + filePaths: [relativePath], + }); + const parsed = JSON.parse(result.content[0].text); + const scan = await scanWorkspace({ rootPath: tempDir }); + const scannedFile = scan.files.find((file) => file.relativePath === relativePath); + + expect(scannedFile).toBeDefined(); + expect(parsed.files[0].estimatedTokens).toBe(scannedFile!.estimatedTokens); + }); + + it("gracefully marks missing files as exists: false without aborting batch", async () => { + const result = await handleEstimateFilesTokens({ + workspaceRoot: tempDir, + filePaths: ["src/index.ts", "src/nonexistent.ts"], + }); + + expect(result.isError).toBeUndefined(); + const parsed = JSON.parse(result.content[0].text); + + expect(parsed.totalFiles).toBe(2); + expect(parsed.existingFiles).toBe(1); + const missing = parsed.files.find((f: any) => f.relativePath === "src/nonexistent.ts"); + expect(missing.exists).toBe(false); + expect(missing.error).toBe("File not found"); + }); + + it("rejects path traversal attempts outside workspace", async () => { + const result = await handleEstimateFilesTokens({ + workspaceRoot: tempDir, + filePaths: ["../../etc/passwd"], + }); + + expect(result.isError).toBe(true); + expect(result.content[0].text).toContain("Path traversal denied"); + }); + + it("advertises tool in server discovery schema", async () => { + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + const server = createServer(); + const client = new Client({ name: "tcalc-test", version: "1.0.0" }); + + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const { tools } = await client.listTools(); + + const tool = tools.find((t) => t.name === "estimate_files_tokens"); + expect(tool).toBeDefined(); + expect(tool!.description).toContain("Estimate token count"); + expect((tool!.inputSchema as any).required).toContain("filePaths"); + } finally { + await client.close(); + await server.close(); + } + }); + + it("invokes tool via client callTool", async () => { + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + const server = createServer(); + const client = new Client({ name: "tcalc-test", version: "1.0.0" }); + + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + + const res = (await client.callTool({ + name: "estimate_files_tokens", + arguments: { + workspaceRoot: tempDir, + filePaths: ["src/index.ts"], + }, + })) as any; + + expect(res.isError).toBeFalsy(); + const parsed = JSON.parse(res.content[0].text); + expect(parsed.totalFiles).toBe(1); + expect(parsed.existingFiles).toBe(1); + } finally { + await client.close(); + await server.close(); + } + }); +}); diff --git a/packages/mcp-server/tests/recommendModelsTool.test.ts b/packages/mcp-server/tests/recommendModelsTool.test.ts index 142f5a2..84d9cfd 100644 --- a/packages/mcp-server/tests/recommendModelsTool.test.ts +++ b/packages/mcp-server/tests/recommendModelsTool.test.ts @@ -2,7 +2,7 @@ import { describe, it, expect, beforeAll } from "vitest"; import { handleRecommendModels } from "../src/tools/recommendModelsTool.js"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync, readFileSync } from "node:fs"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const fixturePath = path.resolve(__dirname, "../../../fixtures/small-node-app"); @@ -25,6 +25,66 @@ describe("recommendModelsTool", () => { expect(parsed.highConfidence).toBeDefined(); expect(parsed.rejected).toBeInstanceOf(Array); expect(parsed.assumptions).toBeInstanceOf(Array); + for (const tier of [ + parsed.cheapestSufficient, + parsed.balanced, + parsed.highConfidence, + ]) { + expect(tier.overflowRisk).toBe(0); + expect(tier.warnings).toBeInstanceOf(Array); + expect(tier.optimizationSuggestions).toBeInstanceOf(Array); + } + + expect(parsed).not.toHaveProperty("allScored"); + expect(JSON.stringify(parsed).length).toBeLessThan(10_000); + }); + + it("preserves overflow warnings when no model fits", async () => { + const tempDir = mkdtempSync( + path.join(process.cwd(), ".tcalc-overflow-catalog-"), + ); + + try { + const catalog = JSON.parse(readFileSync(catalogPath, "utf-8")); + + catalog.models = catalog.models.map((model: Record) => ({ + ...model, + contextWindow: 8, + maxOutputTokens: 4, + })); + + const customCatalogPath = path.join(tempDir, "models.json"); + writeFileSync(customCatalogPath, JSON.stringify(catalog)); + + const result = await handleRecommendModels({ + rootPath: fixturePath, + goal: "build-mvp", + privacyMode: "cloud-ok", + catalogPath: customCatalogPath, + }); + + const parsed = JSON.parse(result.content[0].text); + const tiers = [ + parsed.cheapestSufficient, + parsed.balanced, + parsed.highConfidence, + ]; + + for (const tier of tiers) { + expect(tier.overflowRisk).toBeGreaterThan(0); + expect(tier.warnings).toContain( + "Context window insufficient — overflow likely; consider repo-map-first strategy", + ); + expect(tier.optimizationSuggestions).toContain( + "Use a repo map instead of full workspace to reduce context", + ); + } + + expect(parsed).not.toHaveProperty("allScored"); + expect(JSON.stringify(parsed).length).toBeLessThan(10_000); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } }); it("should handle missing catalog gracefully", async () => { diff --git a/packages/mcp-server/tests/scanWorkspaceTool.test.ts b/packages/mcp-server/tests/scanWorkspaceTool.test.ts index a3794fb..cb2bc34 100644 --- a/packages/mcp-server/tests/scanWorkspaceTool.test.ts +++ b/packages/mcp-server/tests/scanWorkspaceTool.test.ts @@ -1,5 +1,8 @@ import { describe, it, expect, beforeAll, vi } from "vitest"; import { handleScanWorkspace } from "../src/tools/scanWorkspaceTool.js"; +import { createServer } from "../src/server.js"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -55,4 +58,35 @@ describe("scanWorkspaceTool", () => { handleScanWorkspace({ rootPath: fixturePath, privacyMode: "invalid" as any }) ).rejects.toThrow(); }); + + it("should accept a configurable timeoutMs", async () => { + const result = await handleScanWorkspace({ rootPath: fixturePath, timeoutMs: 5000 }); + expect(result).toBeDefined(); + }); + + it("should reject out-of-range timeoutMs", async () => { + await expect(handleScanWorkspace({ rootPath: fixturePath, timeoutMs: 0 })).rejects.toThrow(); + await expect(handleScanWorkspace({ rootPath: fixturePath, timeoutMs: 300001 })).rejects.toThrow(); + }); + + it("should advertise timeoutMs in tool discovery with matching constraints", async () => { + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + const server = createServer(); + const client = new Client({ name: "tcalc-test", version: "1.0.0" }); + try { + await server.connect(serverTransport); + await client.connect(clientTransport); + const { tools } = await client.listTools(); + const scanTool = tools.find((t) => t.name === "scan_workspace"); + expect(scanTool).toBeDefined(); + const props = (scanTool!.inputSchema as any).properties as Record; + expect(props.timeoutMs).toBeDefined(); + expect(props.timeoutMs.minimum).toBe(1); + expect(props.timeoutMs.maximum).toBe(300000); + expect(props.timeoutMs.description).toMatch(/60000/); + } finally { + await client.close(); + await server.close(); + } + }); }); diff --git a/packages/mcp-server/tests/teamPolicy.test.ts b/packages/mcp-server/tests/teamPolicy.test.ts index 715e5fd..d9885d5 100644 --- a/packages/mcp-server/tests/teamPolicy.test.ts +++ b/packages/mcp-server/tests/teamPolicy.test.ts @@ -2,6 +2,11 @@ import { afterEach, describe, expect, it } from "vitest"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import path from "node:path"; import { handleScanWorkspace } from "../src/tools/scanWorkspaceTool.js"; +import { handleRecommendModels } from "../src/tools/recommendModelsTool.js"; +import { handleGenerateReport } from "../src/tools/generateReportTool.js"; +import { handleGenerateAgentRules } from "../src/tools/generateAgentRulesTool.js"; +import { handleCreateRepoMap } from "../src/tools/createRepoMapTool.js"; +import { getLatestRecommendation } from "../src/state.js"; const cleanup: string[] = []; afterEach(() => cleanup.splice(0).forEach((directory) => rmSync(directory, { recursive: true, force: true }))); @@ -20,3 +25,46 @@ describe("MCP team policy", () => { expect(summary.excludedFiles).toBe(1); }); }); + +describe("MCP team policy defaultGoal", () => { + function workspaceWithDefaultGoal(): string { + const root = mkdtempSync(path.join(process.cwd(), ".tcalc-team-goal-")); + cleanup.push(root); + mkdirSync(path.join(root, ".tcalc")); + writeFileSync(path.join(root, ".tcalc", "team.json"), JSON.stringify({ schemaVersion: "1.0", defaultGoal: "add-feature" })); + writeFileSync(path.join(root, "index.ts"), "export const value = 1;"); + return root; + } + + it("recommend_models uses an explicit goal and falls back to defaultGoal", async () => { + const root = workspaceWithDefaultGoal(); + const explicit = await handleRecommendModels({ rootPath: root, goal: "debug" }); + expect(JSON.parse(explicit.content[0].text).goal).toBe("debug"); + const fallback = await handleRecommendModels({ rootPath: root }); + expect(JSON.parse(fallback.content[0].text).goal).toBe("add-feature"); + }); + + it("generate_report uses an explicit goal and falls back to defaultGoal", async () => { + const root = workspaceWithDefaultGoal(); + await handleGenerateReport({ rootPath: root, format: "json", goal: "debug" }); + expect(getLatestRecommendation()?.goal).toBe("debug"); + await handleGenerateReport({ rootPath: root, format: "json" }); + expect(getLatestRecommendation()?.goal).toBe("add-feature"); + }); + + it("generate_agent_rules uses an explicit goal and falls back to defaultGoal", async () => { + const root = workspaceWithDefaultGoal(); + await handleGenerateAgentRules({ rootPath: root, goal: "debug" }); + expect(getLatestRecommendation()?.goal).toBe("debug"); + await handleGenerateAgentRules({ rootPath: root }); + expect(getLatestRecommendation()?.goal).toBe("add-feature"); + }); + + it("create_repo_map uses an explicit goal and falls back to defaultGoal", async () => { + const root = workspaceWithDefaultGoal(); + const explicit = await handleCreateRepoMap({ rootPath: root, format: "json", goal: "debug" }); + expect(JSON.parse(explicit.content[0].text).agentInstructions).toContain("The current workspace goal is: debug"); + const fallback = await handleCreateRepoMap({ rootPath: root, format: "json" }); + expect(JSON.parse(fallback.content[0].text).agentInstructions).toContain("The current workspace goal is: add-feature"); + }); +}); diff --git a/packages/model-catalog/src/index.ts b/packages/model-catalog/src/index.ts index c9db9b2..3dcf16a 100644 --- a/packages/model-catalog/src/index.ts +++ b/packages/model-catalog/src/index.ts @@ -11,3 +11,13 @@ export { export { applyModelProfile } from "./applyModelProfile.js"; export { fetchCatalogFeed } from "./fetchCatalogFeed.js"; export type { CatalogFeedOptions } from "./fetchCatalogFeed.js"; + +export { queryModelCatalog } from "./queryModelCatalog.js"; +export type { + ModelQueryFilter, + ModelSortField, + ModelSortDirection, + ModelQueryOptions, + CatalogQueryAggregate, + CatalogQueryResult, +} from "./types/catalogQuery.js"; diff --git a/packages/model-catalog/src/loadModelCatalog.ts b/packages/model-catalog/src/loadModelCatalog.ts index 2741ec1..cb2eb90 100644 --- a/packages/model-catalog/src/loadModelCatalog.ts +++ b/packages/model-catalog/src/loadModelCatalog.ts @@ -1,30 +1,29 @@ -import { readFileSync, existsSync } from "node:fs"; +import { readFileSync } from "node:fs"; import path from "node:path"; import type { ModelInfo, ModelCatalog } from "@wma/core"; export function loadModelCatalog( catalogDirOrFile: string, - options: { warnIfMissing?: boolean } = {}, + options: { warnIfMissing?: boolean; strict?: boolean } = {}, ): ModelCatalog { + const resolved = path.resolve(catalogDirOrFile); + // Decide models.json location lexically (no existsSync) so non-ENOENT + // filesystem errors (ENOTDIR, EACCES, ...) are not suppressed. + const modelsPath = resolved.endsWith(".json") + ? resolved + : path.join(resolved, "models.json"); try { - const resolved = path.resolve(catalogDirOrFile); - - const modelsPath = existsSync(resolved) && !resolved.endsWith(".json") - ? path.join(resolved, "models.json") - : resolved; - - if (!existsSync(modelsPath)) { - if (options.warnIfMissing !== false) console.warn(`Catalog file not found: ${modelsPath}`); - return { version: "1.0", updatedAt: new Date().toISOString().split("T")[0], models: [] }; - } - const raw = readFileSync(modelsPath, "utf-8"); const parsed = JSON.parse(raw); return parseModelCatalog(parsed); } catch (error) { - console.error(`Failed to load model catalog:`, error); - return { version: "1.0", updatedAt: new Date().toISOString().split("T")[0], models: [] }; + if ((error as NodeJS.ErrnoException)?.code === "ENOENT") { + if (options.strict) throw error; + if (options.warnIfMissing !== false) console.warn(`Catalog file not found: ${modelsPath}`); + return { version: "1.0", updatedAt: new Date().toISOString().split("T")[0], models: [] }; + } + throw error; } } diff --git a/packages/model-catalog/src/queryModelCatalog.ts b/packages/model-catalog/src/queryModelCatalog.ts new file mode 100644 index 0000000..c430f54 --- /dev/null +++ b/packages/model-catalog/src/queryModelCatalog.ts @@ -0,0 +1,234 @@ +import type { ModelInfo } from "@wma/core"; +import type { + CatalogQueryAggregate, + CatalogQueryResult, + ModelQueryOptions, + ModelSortDirection, + ModelSortField, +} from "./types/catalogQuery.js"; + +function calculateCompositeScore(model: ModelInfo): number { + const coding = model.codingScore ?? 50; + const blendedPrice = (model.inputPricePerMillion * 3 + model.outputPricePerMillion) / 4; + const costScore = blendedPrice === 0 ? 100 : Math.max(0, 100 - blendedPrice * 6); + return Number(((coding * 0.7) + (costScore * 0.3)).toFixed(2)); +} + +function matchesSearchTerm(model: ModelInfo, term: string): boolean { + const cleanTerm = term.trim().toLowerCase(); + if (!cleanTerm) return true; + + const idMatch = model.id.toLowerCase().includes(cleanTerm); + const nameMatch = model.displayName.toLowerCase().includes(cleanTerm); + const providerMatch = model.provider.toLowerCase().includes(cleanTerm); + + return idMatch || nameMatch || providerMatch; +} + +function sortModels( + models: ModelInfo[], + field: ModelSortField = "compositeValue", + direction: ModelSortDirection = "desc", +): ModelInfo[] { + const sorted = [...models]; + + sorted.sort((a, b) => { + let cmp = 0; + + switch (field) { + case "cost": { + const costA = (a.inputPricePerMillion * 3 + a.outputPricePerMillion) / 4; + const costB = (b.inputPricePerMillion * 3 + b.outputPricePerMillion) / 4; + cmp = costA - costB; + break; + } + case "context": { + cmp = a.contextWindow - b.contextWindow; + break; + } + case "codingScore": { + cmp = (a.codingScore ?? 0) - (b.codingScore ?? 0); + break; + } + case "latencyScore": { + cmp = (a.latencyScore ?? 0) - (b.latencyScore ?? 0); + break; + } + case "compositeValue": { + cmp = calculateCompositeScore(a) - calculateCompositeScore(b); + break; + } + case "name": { + cmp = a.displayName.localeCompare(b.displayName); + break; + } + } + + return direction === "asc" ? cmp : -cmp; + }); + + return sorted; +} + +function computeAggregates(models: ModelInfo[]): CatalogQueryAggregate { + if (models.length === 0) { + return { + totalMatched: 0, + providerCounts: {}, + averageInputPrice: 0, + averageOutputPrice: 0, + averageCodingScore: 0, + localModelsCount: 0, + cloudModelsCount: 0, + }; + } + + const providerCounts: Record = {}; + let totalInputPrice = 0; + let totalOutputPrice = 0; + let totalCodingScore = 0; + let scoredCount = 0; + let localCount = 0; + let cloudCount = 0; + + for (const m of models) { + providerCounts[m.provider] = (providerCounts[m.provider] ?? 0) + 1; + totalInputPrice += m.inputPricePerMillion; + totalOutputPrice += m.outputPricePerMillion; + + if (m.codingScore !== null && m.codingScore !== undefined) { + totalCodingScore += m.codingScore; + scoredCount++; + } + + if (m.supportsLocal || m.privacyMode === "local") { + localCount++; + } else { + cloudCount++; + } + } + + return { + totalMatched: models.length, + providerCounts, + averageInputPrice: Number((totalInputPrice / models.length).toFixed(3)), + averageOutputPrice: Number((totalOutputPrice / models.length).toFixed(3)), + averageCodingScore: scoredCount > 0 ? Number((totalCodingScore / scoredCount).toFixed(1)) : 0, + localModelsCount: localCount, + cloudModelsCount: cloudCount, + }; +} + +/** + * Defaults to offset 0 and limit 50. Invalid pagination values throw a + * RangeError rather than being normalized. + */ +export function queryModelCatalog( + models: ModelInfo[], + options: ModelQueryOptions = {}, +): CatalogQueryResult { + const { + filter = {}, + sortBy = "compositeValue", + sortDirection = "desc", + offset = 0, + limit = 50, + } = options; + + if (!Number.isFinite(offset) || !Number.isInteger(offset) || offset < 0) { + throw new RangeError("offset must be a non-negative integer"); + } + + if (!Number.isFinite(limit) || !Number.isInteger(limit) || limit <= 0) { + throw new RangeError("limit must be a positive integer"); + } + + const filtered = models.filter((model) => { + if (filter.searchTerm && !matchesSearchTerm(model, filter.searchTerm)) { + return false; + } + + if (filter.providers && filter.providers.length > 0) { + const matchProvider = filter.providers.some( + (p) => p.toLowerCase() === model.provider.toLowerCase(), + ); + if (!matchProvider) return false; + } + + if (filter.excludeProviders && filter.excludeProviders.length > 0) { + const isExcluded = filter.excludeProviders.some( + (p) => p.toLowerCase() === model.provider.toLowerCase(), + ); + if (isExcluded) return false; + } + + if (filter.privacyModes && filter.privacyModes.length > 0) { + if (!filter.privacyModes.includes(model.privacyMode)) { + return false; + } + } + + if (filter.supportsTools !== undefined && model.supportsTools !== filter.supportsTools) { + return false; + } + + if (filter.supportsImages !== undefined && model.supportsImages !== filter.supportsImages) { + return false; + } + + if (filter.supportsLocal !== undefined && model.supportsLocal !== filter.supportsLocal) { + return false; + } + + if (filter.minContextWindow !== undefined && model.contextWindow < filter.minContextWindow) { + return false; + } + + if (filter.maxContextWindow !== undefined && model.contextWindow > filter.maxContextWindow) { + return false; + } + + if ( + filter.maxInputPricePerMillion !== undefined && + model.inputPricePerMillion > filter.maxInputPricePerMillion + ) { + return false; + } + + if ( + filter.maxOutputPricePerMillion !== undefined && + model.outputPricePerMillion > filter.maxOutputPricePerMillion + ) { + return false; + } + + if (filter.minCodingScore !== undefined) { + if (model.codingScore === null || model.codingScore < filter.minCodingScore) { + return false; + } + } + + if (filter.minLatencyScore !== undefined) { + if (model.latencyScore === null || model.latencyScore < filter.minLatencyScore) { + return false; + } + } + + return true; + }); + + const aggregates = computeAggregates(filtered); + const sorted = sortModels(filtered, sortBy, sortDirection); + + const paginated = sorted.slice(offset, offset + limit); + const hasMore = offset + limit < sorted.length; + + return { + models: paginated, + totalMatched: sorted.length, + offset, + limit, + hasMore, + aggregates, + }; +} diff --git a/packages/model-catalog/src/types/catalogQuery.ts b/packages/model-catalog/src/types/catalogQuery.ts new file mode 100644 index 0000000..001f014 --- /dev/null +++ b/packages/model-catalog/src/types/catalogQuery.ts @@ -0,0 +1,54 @@ +import type { ModelInfo, PrivacyMode } from "@wma/core"; + +export interface ModelQueryFilter { + searchTerm?: string; + providers?: string[]; + excludeProviders?: string[]; + privacyModes?: PrivacyMode[]; + supportsTools?: boolean; + supportsImages?: boolean; + supportsLocal?: boolean; + minContextWindow?: number; + maxContextWindow?: number; + maxInputPricePerMillion?: number; + maxOutputPricePerMillion?: number; + minCodingScore?: number; + minLatencyScore?: number; +} + +export type ModelSortField = + | "cost" + | "context" + | "codingScore" + | "latencyScore" + | "compositeValue" + | "name"; + +export type ModelSortDirection = "asc" | "desc"; + +export interface ModelQueryOptions { + filter?: ModelQueryFilter; + sortBy?: ModelSortField; + sortDirection?: ModelSortDirection; + limit?: number; + offset?: number; +} + +export interface CatalogQueryAggregate { + totalMatched: number; + providerCounts: Record; + averageInputPrice: number; + averageOutputPrice: number; + averageCodingScore: number; + localModelsCount: number; + cloudModelsCount: number; +} + +export interface CatalogQueryResult { + models: ModelInfo[]; + totalMatched: number; + offset: number; + limit: number; + hasMore: boolean; + aggregates: CatalogQueryAggregate; +} diff --git a/packages/model-catalog/tests/loadModelCatalog.test.ts b/packages/model-catalog/tests/loadModelCatalog.test.ts index b13c72f..e6262ef 100644 --- a/packages/model-catalog/tests/loadModelCatalog.test.ts +++ b/packages/model-catalog/tests/loadModelCatalog.test.ts @@ -53,10 +53,21 @@ describe("loadModelCatalog", () => { expect(result.models[0].id).toBe("a"); }); - it("should return empty catalog for invalid JSON", () => { + it("should throw for invalid JSON instead of returning an empty catalog", () => { const filePath = join(tmpDir, "bad.json"); writeFileSync(filePath, "not valid json"); - const result = loadModelCatalog(filePath); - expect(result.models).toEqual([]); + expect(() => loadModelCatalog(filePath)).toThrow(SyntaxError); + }); + + it("should rethrow non-ENOENT errors (e.g. EISDIR) instead of returning empty", () => { + const dirAsFile = join(tmpDir, "dir-as-file"); + mkdirSync(dirAsFile); + // models.json is a directory, so readFileSync throws EISDIR (non-ENOENT). + mkdirSync(join(dirAsFile, "models.json")); + expect(() => loadModelCatalog(dirAsFile)).toThrow(); + }); + + it("should throw for missing files in strict mode", () => { + expect(() => loadModelCatalog(join(tmpDir, "missing-strict"), { strict: true })).toThrow(); }); }); diff --git a/packages/model-catalog/tests/queryModelCatalog.test.ts b/packages/model-catalog/tests/queryModelCatalog.test.ts new file mode 100644 index 0000000..cb45ac1 --- /dev/null +++ b/packages/model-catalog/tests/queryModelCatalog.test.ts @@ -0,0 +1,263 @@ +import { describe, it, expect } from "vitest"; +import type { ModelInfo } from "@wma/core"; +import { queryModelCatalog } from "../src/queryModelCatalog.js"; + +const mockModels: ModelInfo[] = [ + { + id: "claude-3-5-sonnet", + displayName: "Claude 3.5 Sonnet", + provider: "Anthropic", + contextWindow: 200_000, + maxOutputTokens: 8192, + inputPricePerMillion: 3.0, + cachedInputPricePerMillion: 0.3, + outputPricePerMillion: 15.0, + supportsTools: true, + supportsImages: true, + supportsLocal: false, + privacyMode: "cloud", + codingScore: 92, + latencyScore: 75, + updatedAt: "2025-01-01", + }, + { + id: "claude-3-5-haiku", + displayName: "Claude 3.5 Haiku", + provider: "Anthropic", + contextWindow: 200_000, + maxOutputTokens: 8192, + inputPricePerMillion: 0.8, + cachedInputPricePerMillion: 0.08, + outputPricePerMillion: 4.0, + supportsTools: true, + supportsImages: false, + supportsLocal: false, + privacyMode: "cloud", + codingScore: 78, + latencyScore: 95, + updatedAt: "2025-01-01", + }, + { + id: "gpt-4o", + displayName: "GPT-4o", + provider: "OpenAI", + contextWindow: 128_000, + maxOutputTokens: 16384, + inputPricePerMillion: 2.5, + cachedInputPricePerMillion: 1.25, + outputPricePerMillion: 10.0, + supportsTools: true, + supportsImages: true, + supportsLocal: false, + privacyMode: "cloud", + codingScore: 88, + latencyScore: 80, + updatedAt: "2025-01-01", + }, + { + id: "gpt-4o-mini", + displayName: "GPT-4o mini", + provider: "OpenAI", + contextWindow: 128_000, + maxOutputTokens: 16384, + inputPricePerMillion: 0.15, + cachedInputPricePerMillion: 0.075, + outputPricePerMillion: 0.6, + supportsTools: true, + supportsImages: true, + supportsLocal: false, + privacyMode: "cloud", + codingScore: 75, + latencyScore: 92, + updatedAt: "2025-01-01", + }, + { + id: "llama-3-3-70b-instruct", + displayName: "Llama 3.3 70B (Ollama)", + provider: "Ollama", + contextWindow: 128_000, + maxOutputTokens: 8192, + inputPricePerMillion: 0, + cachedInputPricePerMillion: 0, + outputPricePerMillion: 0, + supportsTools: true, + supportsImages: false, + supportsLocal: true, + privacyMode: "local", + codingScore: 79, + latencyScore: 60, + updatedAt: "2025-01-01", + }, + { + id: "gemini-1-5-pro", + displayName: "Gemini 1.5 Pro", + provider: "Google", + contextWindow: 2_000_000, + maxOutputTokens: 8192, + inputPricePerMillion: 3.5, + cachedInputPricePerMillion: 0.875, + outputPricePerMillion: 10.5, + supportsTools: true, + supportsImages: true, + supportsLocal: false, + privacyMode: "cloud", + codingScore: 86, + latencyScore: 70, + updatedAt: "2025-01-01", + }, +]; + +describe("queryModelCatalog", () => { + it("returns all models with aggregates when no filters provided", () => { + const result = queryModelCatalog(mockModels); + + expect(result.totalMatched).toBe(6); + expect(result.models).toHaveLength(6); + expect(result.aggregates.totalMatched).toBe(6); + expect(result.aggregates.localModelsCount).toBe(1); + expect(result.aggregates.cloudModelsCount).toBe(5); + expect(result.aggregates.providerCounts["Anthropic"]).toBe(2); + expect(result.aggregates.providerCounts["OpenAI"]).toBe(2); + }); + + it("filters by fuzzy search term across id, displayName, and provider", () => { + const searchResult = queryModelCatalog(mockModels, { + filter: { searchTerm: "sonnet" }, + }); + + expect(searchResult.totalMatched).toBe(1); + expect(searchResult.models[0].id).toBe("claude-3-5-sonnet"); + + const providerResult = queryModelCatalog(mockModels, { + filter: { searchTerm: "google" }, + }); + + expect(providerResult.totalMatched).toBe(1); + expect(providerResult.models[0].provider).toBe("Google"); + }); + + it("filters by provider inclusion and exclusion", () => { + const included = queryModelCatalog(mockModels, { + filter: { providers: ["Anthropic", "Ollama"] }, + }); + expect(included.totalMatched).toBe(3); + + const excluded = queryModelCatalog(mockModels, { + filter: { excludeProviders: ["OpenAI", "Google"] }, + }); + expect(excluded.totalMatched).toBe(3); + expect(excluded.models.every((m) => m.provider !== "OpenAI" && m.provider !== "Google")).toBe(true); + }); + + it("filters by local support and privacy mode", () => { + const localOnly = queryModelCatalog(mockModels, { + filter: { supportsLocal: true }, + }); + + expect(localOnly.totalMatched).toBe(1); + expect(localOnly.models[0].id).toBe("llama-3-3-70b-instruct"); + }); + + it("filters by minimum context window and maximum price thresholds", () => { + const largeContextBudget = queryModelCatalog(mockModels, { + filter: { + minContextWindow: 200_000, + maxInputPricePerMillion: 1.0, + }, + }); + + expect(largeContextBudget.totalMatched).toBe(1); + expect(largeContextBudget.models[0].id).toBe("claude-3-5-haiku"); + }); + + it("filters by minimum coding intelligence score", () => { + const highReasoning = queryModelCatalog(mockModels, { + filter: { minCodingScore: 87 }, + }); + + expect(highReasoning.totalMatched).toBe(2); + const ids = highReasoning.models.map((m) => m.id); + expect(ids).toContain("claude-3-5-sonnet"); + expect(ids).toContain("gpt-4o"); + }); + + it("sorts by cost ascending (cheapest first)", () => { + const sortedCost = queryModelCatalog(mockModels, { + sortBy: "cost", + sortDirection: "asc", + }); + + expect(sortedCost.models[0].id).toBe("llama-3-3-70b-instruct"); // 0 cost + expect(sortedCost.models[1].id).toBe("gpt-4o-mini"); // cheapest cloud + }); + + it("sorts by context capacity descending", () => { + const sortedContext = queryModelCatalog(mockModels, { + sortBy: "context", + sortDirection: "desc", + }); + + expect(sortedContext.models[0].id).toBe("gemini-1-5-pro"); + expect(sortedContext.models[0].contextWindow).toBe(2_000_000); + }); + + it("sorts by coding score descending", () => { + const sortedScore = queryModelCatalog(mockModels, { + sortBy: "codingScore", + sortDirection: "desc", + }); + + expect(sortedScore.models[0].id).toBe("claude-3-5-sonnet"); + }); + + it("supports pagination with limit and offset", () => { + const page1 = queryModelCatalog(mockModels, { + sortBy: "name", + sortDirection: "asc", + limit: 2, + offset: 0, + }); + + expect(page1.models).toHaveLength(2); + expect(page1.totalMatched).toBe(6); + expect(page1.hasMore).toBe(true); + + const page2 = queryModelCatalog(mockModels, { + sortBy: "name", + sortDirection: "asc", + limit: 2, + offset: 2, + }); + + expect(page2.models).toHaveLength(2); + expect(page2.models[0].id).not.toBe(page1.models[0].id); + }); + + it.each([-1, 1.5, NaN, Infinity, -Infinity])( + "rejects invalid offset %s", + (offset) => { + expect(() => queryModelCatalog(mockModels, { offset })).toThrow( + "offset must be a non-negative integer", + ); + }, + ); + + it.each([0, -1, 1.5, NaN, Infinity, -Infinity])( + "rejects invalid limit %s", + (limit) => { + expect(() => queryModelCatalog(mockModels, { limit })).toThrow( + "limit must be a positive integer", + ); + }, + ); + + it("returns consistent metadata when offset is beyond the results", () => { + const result = queryModelCatalog(mockModels, { offset: 100, limit: 2 }); + + expect(result.models).toEqual([]); + expect(result.totalMatched).toBe(6); + expect(result.offset).toBe(100); + expect(result.limit).toBe(2); + expect(result.hasMore).toBe(false); + }); +}); diff --git a/packages/recommender/src/costOptimizationAdvisor.ts b/packages/recommender/src/costOptimizationAdvisor.ts new file mode 100644 index 0000000..8f8bdc7 --- /dev/null +++ b/packages/recommender/src/costOptimizationAdvisor.ts @@ -0,0 +1,375 @@ +import type { ModelInfo } from "@wma/core"; +import { estimateCost } from "./estimateCost.js"; +import type { + CostOptimizationAdvice, + CostOptimizationOptions, + HybridDispatchStrategy, + HybridTieredDispatchRule, + ModelSwitchPlan, + WorkloadProfile, +} from "./types/costAdvisor.js"; + +const DEFAULT_WORKLOAD: WorkloadProfile = { + name: "Standard Full-Stack Dev Workflow", + description: "Average code completion, refactoring, and agentic multi-turn debugging sessions.", + dailyRunsPerDev: 15, + avgInputTokens: 48_000, + avgOutputTokens: 2_000, + cacheHitRatio: 0.5, +}; + +function requireFiniteResult(name: string, value: number): number { + if (!Number.isFinite(value)) { + throw new RangeError(`${name} must be finite after calculation (got ${value})`); + } + return value; +} + +/** @internal Allocate validated run totals without overallocating fractional workloads. */ +function allocateHybridRuns(totalTeamRuns: number) { + const triageRuns = Math.min(Math.round(totalTeamRuns * 0.6), totalTeamRuns); + const editingRuns = Math.min(Math.round(totalTeamRuns * 0.3), totalTeamRuns - triageRuns); + const reasoningRuns = totalTeamRuns - triageRuns - editingRuns; + return { triageRuns, editingRuns, reasoningRuns }; +} + +function calculateSingleRunCost(model: ModelInfo, workload: WorkloadProfile): number { + // Rounding must not exceed the input total when token averages are fractional. + const cachedTokens = Math.min( + workload.avgInputTokens, + Math.round(workload.avgInputTokens * (workload.cacheHitRatio ?? 0)), + ); + const estimate = estimateCost({ + model, + inputTokens: workload.avgInputTokens, + outputTokens: workload.avgOutputTokens, + cachedInputTokens: cachedTokens, + }); + return estimate.totalCost; +} + +function evaluateContextFit(model: ModelInfo, requiredContext: number): number { + if (requiredContext === 0) return 1.0; + if (model.contextWindow < requiredContext) { + return Math.max(0.1, Number((model.contextWindow / requiredContext).toFixed(2))); + } + return 1.0; +} + +function determineRecommendedRoles(model: ModelInfo): string[] { + const roles: string[] = []; + + if (model.inputPricePerMillion === 0 && model.outputPricePerMillion === 0) { + roles.push("Local / Offline Development", "Free CI Automated Linting & Triage"); + } else if (model.inputPricePerMillion <= 0.5) { + roles.push("High-Frequency Quick Edits", "Pre-Commit Code Reviews", "Token-Dense Workspace Summaries"); + } else if (model.inputPricePerMillion <= 3.0) { + roles.push("Daily General Agent Programming", "Test Case Generation", "Feature Implementation"); + } else { + roles.push("Complex Architectural Refactoring", "High-Stakes Security Audits", "Deep Multi-File Reasoning"); + } + + if (model.contextWindow >= 1_000_000) { + roles.push("Whole-Repository Context Ingestion"); + } else if (model.contextWindow >= 128_000) { + roles.push("Multi-Module Context Analysis"); + } + + return roles; +} + +/** + * A model is genuinely free when every price that can contribute to a run is zero. + * This is decided from pricing, never from a rounded monthly spend, so a paid model whose + * monthly total rounds to $0.00 is not mistaken for a free one. + */ +function isFreeModel(model: ModelInfo): boolean { + return ( + model.inputPricePerMillion === 0 && + (model.cachedInputPricePerMillion ?? model.inputPricePerMillion) === 0 && + model.outputPricePerMillion === 0 + ); +} + +function formatMonthlyUsd(amount: number): string { + return Number(amount.toFixed(2)) === 0 ? "less than $0.01" : `$${amount.toFixed(2)}`; +} + +interface TradeoffContext { + /** Unrounded baseline monthly cost. */ + baselineCost: number; + /** Unrounded candidate monthly cost. */ + targetCost: number; + baselineIsFree: boolean; + hasRuns: boolean; +} + +function buildTradeoffSummary(model: ModelInfo, context: TradeoffContext): string { + const { baselineCost, targetCost, baselineIsFree, hasRuns } = context; + const diff = targetCost - baselineCost; + + if (diff === 0) { + if (!hasRuns) { + return `No runs are projected for this workload, so switching to ${model.displayName} has no cost impact.`; + } + return `Cost-neutral alternative to ${model.displayName} with identical economic footprint.`; + } + // A percentage change is undefined when the baseline spends nothing, so report finite absolute added cost instead. + if (baselineCost <= 0) { + const baselineLabel = baselineIsFree ? "zero-cost baseline" : "baseline's zero projected spend"; + return `Adds ${formatMonthlyUsd(diff)}/mo over the ${baselineLabel} to adopt ${model.displayName} (percentage uplift undefined).`; + } + + const percent = Math.round((Math.abs(diff) / baselineCost) * 100); + + if (percent === 0) { + return diff < 0 + ? `Costs less than 1% less than the baseline for ${model.displayName}.` + : `Costs less than 1% more than the baseline for ${model.displayName}.`; + } + + if (diff < 0) { + return `Delivers ${percent}% cost reduction. Ideal for offloading standard repetitive development prompts.`; + } + return `Invests ${percent}% higher spend for superior frontier reasoning capability on complex edge-cases.`; +} + +/** + * Internal view of a candidate that keeps the unrounded numbers. The public `ModelSwitchPlan` + * carries cent-rounded values, which must not drive savings decisions. + */ +interface PlanEvaluation { + plan: ModelSwitchPlan; + runCost: number; + monthlySpend: number; + /** Positive when the candidate is cheaper than the baseline. */ + monthlySavings: number; +} + +export function analyzeCostOptimization(options: CostOptimizationOptions): CostOptimizationAdvice { + const { + baselineModel, + candidateModels, + developerCount = 5, + workingDaysPerMonth = 22, + workload = DEFAULT_WORKLOAD, + minContextWindowRequired = workload.avgInputTokens + workload.avgOutputTokens, + } = options; + + for (const [name, value] of [ + ["developerCount", developerCount], + ["workingDaysPerMonth", workingDaysPerMonth], + ["workload.dailyRunsPerDev", workload.dailyRunsPerDev], + ["workload.avgInputTokens", workload.avgInputTokens], + ["workload.avgOutputTokens", workload.avgOutputTokens], + ["minContextWindowRequired", minContextWindowRequired], + ] as const) { + if (!Number.isFinite(value) || value < 0) { + throw new RangeError(`${name} must be a finite, non-negative number (got ${value})`); + } + } + const { cacheHitRatio } = workload; + if ( + cacheHitRatio !== undefined && + (!Number.isFinite(cacheHitRatio) || cacheHitRatio < 0 || cacheHitRatio > 1) + ) { + throw new RangeError( + `workload.cacheHitRatio must be a finite number between 0 and 1 inclusive (got ${cacheHitRatio})`, + ); + } + + const totalRunsPerDev = requireFiniteResult("totalRunsPerDev", workload.dailyRunsPerDev * workingDaysPerMonth); + const totalTeamRuns = requireFiniteResult("totalTeamRuns", totalRunsPerDev * developerCount); + const hasRuns = totalTeamRuns > 0; + + const baselineRunCost = requireFiniteResult("baselineRunCost", calculateSingleRunCost(baselineModel, workload)); + const baselineMonthlyCost = requireFiniteResult("baselineMonthlyCost", baselineRunCost * totalTeamRuns); + const baselineMonthlySpend = Number(baselineMonthlyCost.toFixed(2)); + const baselineIsFree = isFreeModel(baselineModel); + + const validCandidates = candidateModels.filter( + (c) => c.id !== baselineModel.id && c.contextWindow >= Math.round(minContextWindowRequired * 0.75), + ); + + const evaluations: PlanEvaluation[] = validCandidates.map((candidate) => { + const requireFiniteCandidateResult = (field: string, value: number) => + requireFiniteResult(`candidate ${candidate.id} ${field}`, value); + const runCost = requireFiniteCandidateResult("runCost", calculateSingleRunCost(candidate, workload)); + const monthlySpend = requireFiniteCandidateResult("monthlySpend", runCost * totalTeamRuns); + const savings = requireFiniteCandidateResult("monthlySavings", baselineMonthlyCost - monthlySpend); + const candidateMonthlySpend = Number(monthlySpend.toFixed(2)); + const monthlyCostPerDev = Number( + requireFiniteCandidateResult("monthlyCostPerDev", runCost * totalRunsPerDev).toFixed(2), + ); + const monthlySavings = Number(savings.toFixed(2)); + const savingsPercent = + baselineMonthlyCost > 0 + ? Number(requireFiniteCandidateResult("savingsPercent", (savings / baselineMonthlyCost) * 100).toFixed(1)) + : 0; + const costMultiplier = + baselineMonthlyCost > 0 + ? Number(requireFiniteCandidateResult("costMultiplier", monthlySpend / baselineMonthlyCost).toFixed(2)) + : 1; + const contextFitScore = evaluateContextFit(candidate, minContextWindowRequired); + + return { + runCost, + monthlySpend, + monthlySavings: savings, + plan: { + targetModelId: candidate.id, + targetModelName: candidate.displayName, + targetProvider: candidate.provider, + monthlyCostPerDev, + totalMonthlySpend: candidateMonthlySpend, + monthlySavings, + savingsPercent, + costMultiplier, + contextFitScore, + recommendedRoles: determineRecommendedRoles(candidate), + tradeoffSummary: buildTradeoffSummary(candidate, { + baselineCost: baselineMonthlyCost, + targetCost: monthlySpend, + baselineIsFree, + hasRuns, + }), + }, + }; + }); + + // Sort by highest monthly savings (unrounded, so sub-cent differences still order correctly). + evaluations.sort((a, b) => b.monthlySavings - a.monthlySavings); + + // Every generated candidate stays visible, including ones that cost more than the baseline. + const alternativePlans = evaluations.map((e) => e.plan); + + // Overall cheapest candidate. It may not save money; it is not the recommendation. + const cheapestViablePlan = + evaluations.length > 0 + ? [...evaluations].sort((a, b) => a.monthlySpend - b.monthlySpend)[0].plan + : undefined; + + // Only candidates that strictly cost less than the baseline may be recommended or offloaded to. + // Shared by the single-plan recommendation and the hybrid tiers (already sorted by savings). + const savingEvaluations = evaluations.filter((e) => e.monthlySavings > 0); + const savingPlans = savingEvaluations.map((e) => e.plan); + const cheapestSavingEvaluation = + savingEvaluations.length > 0 + ? [...savingEvaluations].sort((a, b) => a.monthlySpend - b.monthlySpend)[0] + : undefined; + const cheapestSavingPlan = cheapestSavingEvaluation?.plan; + + // Recommended plan balances savings (savingsPercent > 20%) while maintaining high contextFitScore >= 0.9. + // With no saving candidate there is no recommendation and the baseline stays in place. + const recommendedPlan = + savingPlans.find((p) => p.savingsPercent >= 20 && p.contextFitScore >= 0.9) ?? cheapestSavingPlan; + + // Compute Hybrid Tiered Strategy (60% fast/triage, 30% balanced editing, 10% complex frontier) + const fastEvaluation = + savingEvaluations.find((e) => e.plan.savingsPercent > 40) ?? cheapestSavingEvaluation; + const balancedEvaluation = + savingEvaluations.find((e) => e.plan.savingsPercent >= 10 && e.plan.savingsPercent <= 40) ?? + savingEvaluations[0]; + const frontierModel = baselineModel; + + const hybridRules: HybridTieredDispatchRule[] = []; + let blendedMonthlyCost = 0; + let hybridSavings = 0; + let unroundedHybridSavings = 0; + let hybridSummary: string; + + if (fastEvaluation && balancedEvaluation) { + const fastCandidate = fastEvaluation.plan; + const balancedCandidate = balancedEvaluation.plan; + const { triageRuns, editingRuns, reasoningRuns } = allocateHybridRuns(totalTeamRuns); + + const triageCost = requireFiniteResult("hybrid triageCost", fastEvaluation.runCost * triageRuns); + const editingCost = requireFiniteResult("hybrid editingCost", balancedEvaluation.runCost * editingRuns); + const reasoningCost = requireFiniteResult("hybrid reasoningCost", baselineRunCost * reasoningRuns); + const blendedCost = requireFiniteResult("hybrid blendedCost", triageCost + editingCost + reasoningCost); + + blendedMonthlyCost = Number(blendedCost.toFixed(2)); + unroundedHybridSavings = requireFiniteResult("hybrid monthlySavings", baselineMonthlyCost - blendedCost); + hybridSavings = Number(unroundedHybridSavings.toFixed(2)); + + hybridRules.push({ + tier: "fast_triage", + trafficPercentage: 60, + modelId: fastCandidate.targetModelId, + modelName: fastCandidate.targetModelName, + provider: fastCandidate.targetProvider, + allocatedMonthlyCost: Number(triageCost.toFixed(2)), + rationale: "Route file exploration, lint fixes, commit messages, and token scans to high-throughput tier.", + }); + + hybridRules.push({ + tier: "balanced_editing", + trafficPercentage: 30, + modelId: balancedCandidate.targetModelId, + modelName: balancedCandidate.targetModelName, + provider: balancedCandidate.targetProvider, + allocatedMonthlyCost: Number(editingCost.toFixed(2)), + rationale: "Route standard feature implementation, unit test generation, and pull request changesets.", + }); + + hybridRules.push({ + tier: "complex_reasoning", + trafficPercentage: 10, + modelId: frontierModel.id, + modelName: frontierModel.displayName, + provider: frontierModel.provider, + allocatedMonthlyCost: Number(reasoningCost.toFixed(2)), + rationale: "Reserve top-tier model for large architectural redesigns, debugging elusive race conditions.", + }); + + const hybridSavingsPercent = + baselineMonthlyCost > 0 + ? Number(requireFiniteResult("hybrid savingsPercent", (unroundedHybridSavings / baselineMonthlyCost) * 100).toFixed(1)) + : 0; + hybridSummary = `Tiered dispatching delivers ${formatMonthlyUsd(unroundedHybridSavings)}/mo (${hybridSavingsPercent}%) savings while preserving top-tier model precision for high-complexity prompts.`; + } else { + blendedMonthlyCost = baselineMonthlySpend; + hybridRules.push({ + tier: "balanced_editing", + trafficPercentage: 100, + modelId: baselineModel.id, + modelName: baselineModel.displayName, + provider: baselineModel.provider, + allocatedMonthlyCost: baselineMonthlySpend, + rationale: + evaluations.length === 0 + ? "All traffic routed to baseline model (no suitable multi-tier alternatives provided)." + : "All traffic routed to baseline model (no candidate model costs less than the baseline).", + }); + hybridSummary = + evaluations.length === 0 + ? `No alternative models were available, so all traffic stays on ${baselineModel.displayName} and tiered dispatching adds no savings.` + : `No candidate model costs less than ${baselineModel.displayName}, so all traffic stays on the baseline and tiered dispatching adds no savings.`; + } + + const hybridSavingsPercent = + baselineMonthlyCost > 0 + ? Number(requireFiniteResult("hybrid savingsPercent", (unroundedHybridSavings / baselineMonthlyCost) * 100).toFixed(1)) + : 0; + + const hybridStrategy: HybridDispatchStrategy = { + rules: hybridRules, + blendedMonthlyCost, + monthlySavings: hybridSavings, + savingsPercent: hybridSavingsPercent, + summary: hybridSummary, + }; + + return { + baselineModel, + developerCount, + workingDaysPerMonth, + workload, + baselineMonthlySpend, + alternativePlans, + cheapestViablePlan, + recommendedPlan, + hybridStrategy, + timestamp: new Date().toISOString(), + }; +} diff --git a/packages/recommender/src/estimateCost.ts b/packages/recommender/src/estimateCost.ts index 0a4100d..5812885 100644 --- a/packages/recommender/src/estimateCost.ts +++ b/packages/recommender/src/estimateCost.ts @@ -10,6 +10,22 @@ export interface EstimateCostOptions { export function estimateCost(options: EstimateCostOptions): CostEstimate { const { model, inputTokens, outputTokens, cachedInputTokens = 0 } = options; + if (!Number.isFinite(inputTokens) || inputTokens < 0) { + throw new RangeError( + `inputTokens must be a finite, non-negative number (got ${inputTokens})`, + ); + } + if (!Number.isFinite(outputTokens) || outputTokens < 0) { + throw new RangeError( + `outputTokens must be a finite, non-negative number (got ${outputTokens})`, + ); + } + if (!Number.isFinite(cachedInputTokens)) { + throw new RangeError( + `cachedInputTokens must be a finite number (got ${cachedInputTokens})`, + ); + } + if (cachedInputTokens < 0 || cachedInputTokens > inputTokens) { throw new RangeError( `cachedInputTokens (${cachedInputTokens}) must be between 0 and inputTokens (${inputTokens})`, diff --git a/packages/recommender/src/index.ts b/packages/recommender/src/index.ts index d119df1..2202886 100644 --- a/packages/recommender/src/index.ts +++ b/packages/recommender/src/index.ts @@ -2,3 +2,12 @@ export { estimateCost, estimateSessionCost } from "./estimateCost.js"; export type { EstimateCostOptions } from "./estimateCost.js"; export { recommendModels } from "./recommendModels.js"; export type { RecommendModelsOptions } from "./recommendModels.js"; +export { analyzeCostOptimization } from "./costOptimizationAdvisor.js"; +export type { + CostOptimizationAdvice, + CostOptimizationOptions, + HybridDispatchStrategy, + HybridTieredDispatchRule, + ModelSwitchPlan, + WorkloadProfile, +} from "./types/costAdvisor.js"; diff --git a/packages/recommender/src/recommendModels.ts b/packages/recommender/src/recommendModels.ts index 2015327..49d3169 100644 --- a/packages/recommender/src/recommendModels.ts +++ b/packages/recommender/src/recommendModels.ts @@ -8,10 +8,11 @@ export interface RecommendModelsOptions { outputTokens?: number; budget?: number; privacyMode?: PrivacySetting; - /** IDs of models the team has designated as preferred. Used only as a final - * tiebreaker after cost, totalScore, and confidenceScore are all equal. - * Cannot make an ineligible or objectively lower-ranked model win. */ - preferredModelIds?: string[]; + /** + * Model ids a team policy prefers. Used only to break ties between candidates the scoring logic + * already ranks equally; it never changes how a model is scored or whether it is eligible. + */ + preferredModelIds?: readonly string[]; } const DEFAULT_OUTPUT_TOKENS: Record = { @@ -41,12 +42,24 @@ const GOAL_DIFFICULTY: Record = { }; export function recommendModels(options: RecommendModelsOptions): RecommendationResult { - const { models, workspaceTokens, goal, outputTokens: optOutputTokens, privacyMode = "local-first", budget, preferredModelIds } = options; - const preferred = new Set(preferredModelIds ?? []); + const { models, workspaceTokens, goal, outputTokens: optOutputTokens, privacyMode = "local-first", budget } = options; + const preferredIds = new Set(options.preferredModelIds ?? []); + const preferenceRank = (a: { model: ModelInfo }, b: { model: ModelInfo }): number => + Number(preferredIds.has(b.model.id)) - Number(preferredIds.has(a.model.id)); + + if (!Number.isFinite(workspaceTokens) || workspaceTokens < 0) { + throw new RangeError(`workspaceTokens must be a finite number >= 0 (got ${workspaceTokens})`); + } + if (budget !== undefined && (!Number.isFinite(budget) || budget < 0)) { + throw new RangeError(`budget must be a finite number >= 0 (got ${budget})`); + } + if (optOutputTokens !== undefined && (!Number.isFinite(optOutputTokens) || optOutputTokens <= 0)) { + throw new RangeError(`outputTokens must be a finite number > 0 (got ${optOutputTokens})`); + } const contextTokens = budget === undefined ? workspaceTokens : Math.min(workspaceTokens, budget); const contextNeeded = Math.round(contextTokens * 1.2); - const outputTokens = optOutputTokens ?? DEFAULT_OUTPUT_TOKENS[goal]; + const outputTokens = optOutputTokens ?? DEFAULT_OUTPUT_TOKENS[goal] ?? 4000; const assumptions: string[] = [ `Output tokens estimated for "${goal}" goal: ${outputTokens}`, @@ -54,6 +67,9 @@ export function recommendModels(options: RecommendModelsOptions): Recommendation budget !== undefined ? `Context limited to user-specified budget of ${budget} tokens` : `Using full workspace tokens (${workspaceTokens}) as context`, + privacyMode === "local-first" + ? "Local-first privacy mode only considers models with verified local execution support" + : "Cloud-capable models may be considered", ]; const rejected: string[] = []; @@ -61,7 +77,7 @@ export function recommendModels(options: RecommendModelsOptions): Recommendation const overflowing: Array<{ model: ModelInfo; score: ModelScore; cost: CostEstimate }> = []; for (const model of models) { - if (privacyMode === "local-first" && model.privacyMode === "cloud") { + if (privacyMode === "local-first" && !model.supportsLocal) { continue; } @@ -116,16 +132,14 @@ export function recommendModels(options: RecommendModelsOptions): Recommendation const usedIds = new Set(); const duplicationReason = "Same model selected for multiple tiers due to limited fitting candidates"; - // Preference tiebreaker: only consulted when all preceding substantive criteria are equal. - const byPreference = (a: { model: ModelInfo }, b: { model: ModelInfo }): number => - Number(preferred.has(b.model.id)) - Number(preferred.has(a.model.id)) - || a.model.id.localeCompare(b.model.id); + // Team preference is the last resort before the alphabetical tiebreak: it only separates + // candidates the scoring keys above have already ranked equally. const cheapestRanked = [...candidates].sort((a, b) => - a.cost.totalCost - b.cost.totalCost || b.score.totalScore - a.score.totalScore || byPreference(a, b)); + a.cost.totalCost - b.cost.totalCost || b.score.totalScore - a.score.totalScore || preferenceRank(a, b) || a.model.id.localeCompare(b.model.id)); const balancedRanked = [...candidates].sort((a, b) => - b.score.totalScore - a.score.totalScore || a.cost.totalCost - b.cost.totalCost || byPreference(a, b)); + b.score.totalScore - a.score.totalScore || a.cost.totalCost - b.cost.totalCost || preferenceRank(a, b) || a.model.id.localeCompare(b.model.id)); const confidenceRanked = [...candidates].sort((a, b) => - confidenceScore(b) - confidenceScore(a) || b.score.totalScore - a.score.totalScore || byPreference(a, b)); + confidenceScore(b) - confidenceScore(a) || b.score.totalScore - a.score.totalScore || preferenceRank(a, b) || a.model.id.localeCompare(b.model.id)); const pickUnused = (ranked: typeof candidates): typeof candidates[0] => { const selected = ranked.find((candidate) => !usedIds.has(candidate.model.id)) ?? ranked[0]; @@ -184,6 +198,7 @@ function confidenceScore(candidate: { model: ModelInfo; score: ModelScore }): nu } function calculateContextFit(model: ModelInfo, contextNeeded: number): number { + if (contextNeeded <= 0) return 1.0; if (model.contextWindow >= contextNeeded * 2) return 1.0; if (model.contextWindow >= contextNeeded) return 0.8; if (model.contextWindow >= contextNeeded * 0.75) return 0.5; @@ -203,6 +218,7 @@ function calculateCostEfficiency(cost: CostEstimate): number { } function calculateOverflowRisk(model: ModelInfo, contextNeeded: number): number { + if (contextNeeded <= 0) return 0; if (model.contextWindow >= contextNeeded) return 0; return Math.min(1, 1 - model.contextWindow / contextNeeded); } diff --git a/packages/recommender/src/types/costAdvisor.ts b/packages/recommender/src/types/costAdvisor.ts new file mode 100644 index 0000000..69ad543 --- /dev/null +++ b/packages/recommender/src/types/costAdvisor.ts @@ -0,0 +1,64 @@ +import type { ModelInfo } from "@wma/core"; + +export interface WorkloadProfile { + name: string; + description?: string; + dailyRunsPerDev: number; + avgInputTokens: number; + avgOutputTokens: number; + cacheHitRatio?: number; +} + +export interface ModelSwitchPlan { + targetModelId: string; + targetModelName: string; + targetProvider: string; + monthlyCostPerDev: number; + totalMonthlySpend: number; + monthlySavings: number; + savingsPercent: number; + costMultiplier: number; + contextFitScore: number; + recommendedRoles: string[]; + tradeoffSummary: string; +} + +export interface HybridTieredDispatchRule { + tier: "fast_triage" | "balanced_editing" | "complex_reasoning"; + trafficPercentage: number; + modelId: string; + modelName: string; + provider: string; + allocatedMonthlyCost: number; + rationale: string; +} + +export interface HybridDispatchStrategy { + rules: HybridTieredDispatchRule[]; + blendedMonthlyCost: number; + monthlySavings: number; + savingsPercent: number; + summary: string; +} + +export interface CostOptimizationOptions { + baselineModel: ModelInfo; + candidateModels: ModelInfo[]; + developerCount?: number; + workingDaysPerMonth?: number; + workload?: WorkloadProfile; + minContextWindowRequired?: number; +} + +export interface CostOptimizationAdvice { + baselineModel: ModelInfo; + developerCount: number; + workingDaysPerMonth: number; + workload: WorkloadProfile; + baselineMonthlySpend: number; + alternativePlans: ModelSwitchPlan[]; + cheapestViablePlan?: ModelSwitchPlan; + recommendedPlan?: ModelSwitchPlan; + hybridStrategy: HybridDispatchStrategy; + timestamp: string; +} diff --git a/packages/recommender/tests/costOptimizationAdvisor.test.ts b/packages/recommender/tests/costOptimizationAdvisor.test.ts new file mode 100644 index 0000000..6ff8f21 --- /dev/null +++ b/packages/recommender/tests/costOptimizationAdvisor.test.ts @@ -0,0 +1,801 @@ +import { describe, it, expect } from "vitest"; +import type { ModelInfo } from "@wma/core"; +import { analyzeCostOptimization } from "../src/costOptimizationAdvisor.js"; +import type { CostOptimizationOptions } from "../src/types/costAdvisor.js"; + +const sonnetModel: ModelInfo = { + id: "claude-3-5-sonnet", + displayName: "Claude 3.5 Sonnet", + provider: "Anthropic", + contextWindow: 200_000, + maxOutputTokens: 8192, + inputPricePerMillion: 3.0, + cachedInputPricePerMillion: 0.3, + outputPricePerMillion: 15.0, + supportsTools: true, + supportsImages: true, + supportsLocal: false, + privacyMode: "cloud", + codingScore: 92, + latencyScore: 75, + updatedAt: "2025-01-01", +}; + +const haikuModel: ModelInfo = { + id: "claude-3-5-haiku", + displayName: "Claude 3.5 Haiku", + provider: "Anthropic", + contextWindow: 200_000, + maxOutputTokens: 8192, + inputPricePerMillion: 0.8, + cachedInputPricePerMillion: 0.08, + outputPricePerMillion: 4.0, + supportsTools: true, + supportsImages: false, + supportsLocal: false, + privacyMode: "cloud", + codingScore: 78, + latencyScore: 95, + updatedAt: "2025-01-01", +}; + +const gpt4oMini: ModelInfo = { + id: "gpt-4o-mini", + displayName: "GPT-4o mini", + provider: "OpenAI", + contextWindow: 128_000, + maxOutputTokens: 16384, + inputPricePerMillion: 0.15, + cachedInputPricePerMillion: 0.075, + outputPricePerMillion: 0.6, + supportsTools: true, + supportsImages: true, + supportsLocal: false, + privacyMode: "cloud", + codingScore: 75, + latencyScore: 92, + updatedAt: "2025-01-01", +}; + +const localLlama: ModelInfo = { + id: "llama-3-3-70b-local", + displayName: "Llama 3.3 70B (Ollama)", + provider: "Ollama", + contextWindow: 128_000, + maxOutputTokens: 8192, + inputPricePerMillion: 0, + cachedInputPricePerMillion: 0, + outputPricePerMillion: 0, + supportsTools: true, + supportsImages: false, + supportsLocal: true, + privacyMode: "local", + codingScore: 79, + latencyScore: 60, + updatedAt: "2025-01-01", +}; + +const tinyModel: ModelInfo = { + id: "tiny-legacy-model", + displayName: "Tiny Legacy Model", + provider: "Legacy", + contextWindow: 8000, // Too small for standard workload + maxOutputTokens: 2048, + inputPricePerMillion: 0.1, + cachedInputPricePerMillion: null, + outputPricePerMillion: 0.2, + supportsTools: false, + supportsImages: false, + supportsLocal: false, + privacyMode: "cloud", + codingScore: 40, + latencyScore: 90, + updatedAt: "2025-01-01", +}; + +describe("analyzeCostOptimization", () => { + describe("input validation (#175)", () => { + const validOptions: CostOptimizationOptions = { + baselineModel: sonnetModel, + candidateModels: [haikuModel], + developerCount: 4, + workingDaysPerMonth: 20, + workload: { + name: "Validation control", + dailyRunsPerDev: 10, + avgInputTokens: 50_000, + avgOutputTokens: 2_000, + cacheHitRatio: 0.5, + }, + }; + + for (const field of [ + "developerCount", + "workingDaysPerMonth", + "dailyRunsPerDev", + "avgInputTokens", + "avgOutputTokens", + "minContextWindowRequired", + ] as const) { + it.each([-1, NaN, Infinity, -Infinity])(`rejects invalid ${field}: %s`, (value) => { + const isWorkloadField = + field === "dailyRunsPerDev" || field === "avgInputTokens" || field === "avgOutputTokens"; + const options = isWorkloadField + ? { ...validOptions, workload: { ...validOptions.workload!, [field]: value } } + : { ...validOptions, [field]: value }; + const name = isWorkloadField ? `workload.${field}` : field; + expect(() => analyzeCostOptimization(options)).toThrow( + new RangeError(`${name} must be a finite, non-negative number (got ${value})`), + ); + }); + } + + it.each([-0.1, 1.1, NaN, Infinity, -Infinity])("rejects invalid cacheHitRatio: %s", (cacheHitRatio) => { + expect(() => analyzeCostOptimization({ + ...validOptions, + workload: { ...validOptions.workload!, cacheHitRatio }, + })).toThrow(new RangeError( + `workload.cacheHitRatio must be a finite number between 0 and 1 inclusive (got ${cacheHitRatio})`, + )); + }); + + it.each([ + [undefined, 144], + [0, 144], + [0.5, 90], + [1, 36], + ])("preserves normal costs with cacheHitRatio %s", (cacheHitRatio, expectedSpend) => { + const advice = analyzeCostOptimization({ + ...validOptions, + workload: { ...validOptions.workload!, cacheHitRatio }, + }); + expect(advice.baselineMonthlySpend).toBe(expectedSpend); + expect(advice.recommendedPlan?.targetModelId).toBe(haikuModel.id); + expect(advice.hybridStrategy.monthlySavings).toBeGreaterThan(0); + }); + + it("accepts fractional team and workload inputs without rounding cached tokens above input", () => { + const advice = analyzeCostOptimization({ + ...validOptions, + developerCount: 1.5, + workingDaysPerMonth: 2.5, + minContextWindowRequired: 0.5, + workload: { name: "Fractional", dailyRunsPerDev: 2.5, avgInputTokens: 0.75, avgOutputTokens: 0.5, cacheHitRatio: 1 }, + }); + expect(advice.developerCount).toBe(1.5); + expect(advice.workingDaysPerMonth).toBe(2.5); + expect(Number.isFinite(advice.baselineMonthlySpend)).toBe(true); + expect(advice.alternativePlans[0].contextFitScore).toBe(1); + }); + + it.each([undefined, 0])("handles zero token workload with context requirement %s", (minContextWindowRequired) => { + const advice = analyzeCostOptimization({ + ...validOptions, + candidateModels: [{ ...haikuModel, contextWindow: 0 }], + minContextWindowRequired, + workload: { name: "No tokens", dailyRunsPerDev: 10, avgInputTokens: 0, avgOutputTokens: 0 }, + }); + expect(advice.baselineMonthlySpend).toBe(0); + expect(advice.alternativePlans).toHaveLength(1); + expect(advice.alternativePlans[0].contextFitScore).toBe(1); + expect(advice.recommendedPlan).toBeUndefined(); + expect(advice.hybridStrategy.monthlySavings).toBe(0); + }); + }); + + describe("calculation overflow (#175)", () => { + const options: CostOptimizationOptions = { + baselineModel: sonnetModel, + candidateModels: [], + developerCount: 1, + workingDaysPerMonth: 1, + minContextWindowRequired: 0, + workload: { + name: "Overflow regression", + dailyRunsPerDev: 1, + avgInputTokens: 1_000_000, + avgOutputTokens: 0, + cacheHitRatio: 0, + }, + }; + const expensiveModel: ModelInfo = { ...haikuModel, inputPricePerMillion: Number.MAX_VALUE }; + + const cases: [string, CostOptimizationOptions][] = [ + ["totalRunsPerDev", { + ...options, + developerCount: 0, + workingDaysPerMonth: 2, + workload: { ...options.workload!, dailyRunsPerDev: Number.MAX_VALUE }, + }], + ["totalTeamRuns", { + ...options, + developerCount: 2, + workload: { ...options.workload!, dailyRunsPerDev: Number.MAX_VALUE }, + }], + ["baselineRunCost", { + ...options, + baselineModel: expensiveModel, + workload: { ...options.workload!, avgInputTokens: 2_000_000 }, + }], + ["baselineMonthlyCost", { ...options, developerCount: Number.MAX_VALUE }], + [`candidate ${haikuModel.id} runCost`, { + ...options, + baselineModel: localLlama, + candidateModels: [expensiveModel], + workload: { ...options.workload!, avgInputTokens: 2_000_000 }, + }], + [`candidate ${sonnetModel.id} monthlySpend`, { + ...options, + baselineModel: localLlama, + candidateModels: [sonnetModel], + developerCount: Number.MAX_VALUE, + }], + [`candidate ${sonnetModel.id} monthlyCostPerDev`, { + ...options, + baselineModel: localLlama, + candidateModels: [sonnetModel], + developerCount: 0, + workingDaysPerMonth: Number.MAX_VALUE, + }], + ]; + + it.each(cases)("rejects overflowing %s from finite inputs", (field, input) => { + const run = () => analyzeCostOptimization(input); + expect(run).toThrow(RangeError); + expect(run).toThrow(`${field} must be finite after calculation (got Infinity)`); + }); + + it("rejects an overflowing percentage even when both costs are finite", () => { + const run = () => analyzeCostOptimization({ + ...options, + baselineModel: { ...sonnetModel, inputPricePerMillion: Number.MIN_VALUE }, + candidateModels: [{ ...haikuModel, inputPricePerMillion: 1e308 }], + }); + expect(run).toThrow(RangeError); + expect(run).toThrow(`candidate ${haikuModel.id} savingsPercent must be finite after calculation (got -Infinity)`); + }); + + it("accepts very large finite run totals when calculated costs remain finite", () => { + const advice = analyzeCostOptimization({ + ...options, + baselineModel: localLlama, + candidateModels: [{ ...localLlama, id: "another-free-model" }], + developerCount: Number.MAX_VALUE, + }); + expect(advice.baselineMonthlySpend).toBe(0); + expect(advice.alternativePlans[0]).toMatchObject({ + totalMonthlySpend: 0, + monthlySavings: 0, + savingsPercent: 0, + costMultiplier: 1, + contextFitScore: 1, + }); + expect(advice.recommendedPlan).toBeUndefined(); + expect(advice.hybridStrategy.monthlySavings).toBe(0); + }); + + it("rejects hybrid blendedCost overflow after finite integer-run tier costs", () => { + const totalRuns = 556; + // Adjacent finite run costs: monthly totals remain finite, but summing the + // rounded tier allocations overflows at the upper floating-point boundary. + const baselineRunCost = 3.2332610339250284e305; + const candidateRunCost = 3.233261033925028e305; + const [triageRuns, editingRuns, reasoningRuns] = [334, 167, 55]; + const tierCosts = [ + candidateRunCost * triageRuns, + candidateRunCost * editingRuns, + baselineRunCost * reasoningRuns, + ]; + expect(Number.isFinite(baselineRunCost * totalRuns)).toBe(true); + expect(Number.isFinite(candidateRunCost * totalRuns)).toBe(true); + expect(candidateRunCost * totalRuns).toBeLessThan(baselineRunCost * totalRuns); + expect(tierCosts.every(Number.isFinite)).toBe(true); + expect(tierCosts.reduce((sum, cost) => sum + cost, 0)).toBe(Infinity); + + const run = () => analyzeCostOptimization({ + ...options, + baselineModel: { ...sonnetModel, inputPricePerMillion: baselineRunCost }, + candidateModels: [{ ...haikuModel, inputPricePerMillion: candidateRunCost }], + workload: { ...options.workload!, dailyRunsPerDev: totalRuns }, + }); + expect(run).toThrow(RangeError); + expect(run).toThrow("hybrid blendedCost must be finite after calculation (got Infinity)"); + }); + }); + + it("does not overallocate a fractional 0.9-run workload or claim more than 100% savings", () => { + const totalRuns = 0.9; + const advice = analyzeCostOptimization({ + baselineModel: sonnetModel, + candidateModels: [localLlama], + developerCount: 1, + workingDaysPerMonth: 1, + minContextWindowRequired: 0, + workload: { + name: "Fractional monthly runs", + dailyRunsPerDev: totalRuns, + avgInputTokens: 1_000_000, + avgOutputTokens: 0, + cacheHitRatio: 0, + }, + }); + expect(advice.baselineMonthlySpend).toBe(2.7); + expect(advice.hybridStrategy.rules).toHaveLength(3); + for (const rule of advice.hybridStrategy.rules) expect(rule.allocatedMonthlyCost).toBeGreaterThanOrEqual(0); + expect(advice.hybridStrategy.blendedMonthlyCost).toBe(0); + expect(advice.hybridStrategy.monthlySavings).toBe(2.7); + expect(advice.hybridStrategy.savingsPercent).toBe(100); + expect(advice.hybridStrategy.savingsPercent).toBeLessThanOrEqual(100); + }); + + it("infers non-negative fractional allocations from $1/run candidate and $3/run baseline tier costs", () => { + const totalRuns = 0.9; + const advice = analyzeCostOptimization({ + baselineModel: sonnetModel, + candidateModels: [{ ...haikuModel, inputPricePerMillion: 1 }], + developerCount: 1, + workingDaysPerMonth: 1, + minContextWindowRequired: 0, + workload: { + name: "Observable fractional allocations", + dailyRunsPerDev: totalRuns, + avgInputTokens: 1_000_000, + avgOutputTokens: 0, + cacheHitRatio: 0, + }, + }); + expect(advice.hybridStrategy.rules).toHaveLength(3); + const allocations = advice.hybridStrategy.rules.map((rule) => + rule.allocatedMonthlyCost / (rule.modelId === sonnetModel.id ? 3 : 1), + ); + expect(allocations).toEqual([0.9, 0, 0]); + for (const runs of allocations) expect(runs).toBeGreaterThanOrEqual(0); + expect(allocations.reduce((sum, runs) => sum + runs, 0)).toBe(totalRuns); + expect(advice.baselineMonthlySpend).toBe(2.7); + for (const rule of advice.hybridStrategy.rules) expect(rule.allocatedMonthlyCost).toBeGreaterThanOrEqual(0); + expect(advice.hybridStrategy.blendedMonthlyCost).toBe(0.9); + expect(advice.hybridStrategy.monthlySavings).toBe(1.8); + expect(advice.hybridStrategy.savingsPercent).toBeLessThanOrEqual(100); + }); + + it("computes baseline monthly spend for developer team accurately", () => { + const advice = analyzeCostOptimization({ + baselineModel: sonnetModel, + candidateModels: [haikuModel, gpt4oMini], + developerCount: 4, + workingDaysPerMonth: 20, + workload: { + name: "Custom Agent Load", + dailyRunsPerDev: 10, + avgInputTokens: 50_000, + avgOutputTokens: 2_000, + cacheHitRatio: 0.5, + }, + }); + + expect(advice.developerCount).toBe(4); + expect(advice.workingDaysPerMonth).toBe(20); + // 4 devs * 20 days * 10 runs = 800 runs + // input non-cached = 25,000 => (25000/1M)*3 = $0.075 + // input cached = 25,000 => (25000/1M)*0.3 = $0.0075 + // output = 2,000 => (2000/1M)*15 = $0.03 + // per run = 0.075 + 0.0075 + 0.03 = 0.1125 + // 800 runs * 0.1125 = 90.00 + expect(advice.baselineMonthlySpend).toBe(90); + }); + + it("evaluates candidate plans and detects savings against baseline", () => { + const advice = analyzeCostOptimization({ + baselineModel: sonnetModel, + candidateModels: [haikuModel, gpt4oMini, localLlama], + developerCount: 5, + workingDaysPerMonth: 22, + }); + + expect(advice.alternativePlans.length).toBe(3); + + const miniPlan = advice.alternativePlans.find((p) => p.targetModelId === "gpt-4o-mini"); + expect(miniPlan).toBeDefined(); + expect(miniPlan!.monthlySavings).toBeGreaterThan(0); + expect(miniPlan!.savingsPercent).toBeGreaterThan(80); + expect(miniPlan!.costMultiplier).toBeLessThan(0.2); + + const localPlan = advice.alternativePlans.find((p) => p.targetModelId === "llama-3-3-70b-local"); + expect(localPlan).toBeDefined(); + expect(localPlan!.totalMonthlySpend).toBe(0); + expect(localPlan!.savingsPercent).toBe(100); + expect(localPlan!.recommendedRoles).toContain("Local / Offline Development"); + }); + + it("filters out candidate models whose context window is insufficient", () => { + const advice = analyzeCostOptimization({ + baselineModel: sonnetModel, + candidateModels: [haikuModel, tinyModel], + workload: { + name: "Large Prompt Workflow", + dailyRunsPerDev: 10, + avgInputTokens: 50_000, + avgOutputTokens: 2_000, + }, + minContextWindowRequired: 52_000, + }); + + // tinyModel has 8000 context window, 8000 < 52000 * 0.75 (39000), so excluded + expect(advice.alternativePlans.some((p) => p.targetModelId === "tiny-legacy-model")).toBe(false); + expect(advice.alternativePlans.some((p) => p.targetModelId === "claude-3-5-haiku")).toBe(true); + }); + + it("identifies cheapestViablePlan and recommendedPlan", () => { + const advice = analyzeCostOptimization({ + baselineModel: sonnetModel, + candidateModels: [haikuModel, gpt4oMini], + }); + + expect(advice.cheapestViablePlan).toBeDefined(); + expect(advice.cheapestViablePlan?.targetModelId).toBe("gpt-4o-mini"); + expect(advice.recommendedPlan).toBeDefined(); + }); + + it("constructs a 3-tier hybrid dispatch strategy with blended cost calculation", () => { + const advice = analyzeCostOptimization({ + baselineModel: sonnetModel, + candidateModels: [haikuModel, gpt4oMini], + }); + + expect(advice.hybridStrategy.rules.length).toBe(3); + const triageTier = advice.hybridStrategy.rules.find((r) => r.tier === "fast_triage"); + const editTier = advice.hybridStrategy.rules.find((r) => r.tier === "balanced_editing"); + const reasoningTier = advice.hybridStrategy.rules.find((r) => r.tier === "complex_reasoning"); + + expect(triageTier).toBeDefined(); + expect(triageTier?.trafficPercentage).toBe(60); + expect(editTier?.trafficPercentage).toBe(30); + expect(reasoningTier?.trafficPercentage).toBe(10); + expect(reasoningTier?.modelId).toBe(sonnetModel.id); + + expect(advice.hybridStrategy.blendedMonthlyCost).toBeLessThan(advice.baselineMonthlySpend); + expect(advice.hybridStrategy.monthlySavings).toBeGreaterThan(0); + expect(advice.hybridStrategy.summary).toContain("Tiered dispatching delivers"); + }); + + it("handles fallback gracefully when no candidate models are provided", () => { + const advice = analyzeCostOptimization({ + baselineModel: sonnetModel, + candidateModels: [], + }); + + expect(advice.alternativePlans).toHaveLength(0); + expect(advice.cheapestViablePlan).toBeUndefined(); + expect(advice.hybridStrategy.blendedMonthlyCost).toBe(advice.baselineMonthlySpend); + expect(advice.hybridStrategy.rules[0].tier).toBe("balanced_editing"); + expect(advice.hybridStrategy.rules[0].trafficPercentage).toBe(100); + }); + + it("reports finite absolute added cost when the baseline model is free", () => { + const advice = analyzeCostOptimization({ + baselineModel: localLlama, + candidateModels: [haikuModel], + developerCount: 5, + workingDaysPerMonth: 22, + }); + + const plan = advice.alternativePlans.find((p) => p.targetModelId === "claude-3-5-haiku"); + expect(plan).toBeDefined(); + expect(plan!.totalMonthlySpend).toBeGreaterThan(0); + expect(plan!.tradeoffSummary).not.toContain("Infinity"); + expect(plan!.tradeoffSummary).toContain("Adds $"); + expect(plan!.tradeoffSummary).toContain("/mo over the zero-cost baseline"); + }); + + it("treats a zero-to-zero comparison as cost-neutral instead of a percentage", () => { + const secondFreeModel: ModelInfo = { + ...localLlama, + id: "llama-3-3-8b-local", + displayName: "Llama 3.3 8B (Ollama)", + }; + const advice = analyzeCostOptimization({ + baselineModel: localLlama, + candidateModels: [secondFreeModel], + developerCount: 5, + workingDaysPerMonth: 22, + }); + + const plan = advice.alternativePlans[0]; + expect(plan).toBeDefined(); + expect(plan!.totalMonthlySpend).toBe(0); + expect(plan!.tradeoffSummary).toContain("Cost-neutral"); + expect(plan!.tradeoffSummary).not.toContain("Infinity"); + }); + describe("positive-savings-only recommendations (#105)", () => { + function expectAllNumbersFinite(value: unknown, path = "advice"): void { + if (typeof value === "number") { + expect(Number.isFinite(value), `${path} must be finite`).toBe(true); + } else if (Array.isArray(value)) { + value.forEach((item, i) => expectAllNumbersFinite(item, `${path}[${i}]`)); + } else if (value !== null && typeof value === "object") { + for (const [key, child] of Object.entries(value)) { + expectAllNumbersFinite(child, `${path}.${key}`); + } + } + } + + function expectNoInvalidNumberText(advice: ReturnType): void { + const texts = [ + advice.hybridStrategy.summary, + ...advice.alternativePlans.map((p) => p.tradeoffSummary), + ]; + for (const text of texts) { + expect(text).not.toMatch(/Infinity|NaN/); + } + } + + it("keeps every candidate in alternativePlans but recommends none when all cost more than the baseline", () => { + const advice = analyzeCostOptimization({ + baselineModel: gpt4oMini, + candidateModels: [sonnetModel, haikuModel], + developerCount: 5, + workingDaysPerMonth: 22, + }); + + expect(advice.alternativePlans.map((p) => p.targetModelId).sort()).toEqual([ + "claude-3-5-haiku", + "claude-3-5-sonnet", + ]); + for (const plan of advice.alternativePlans) { + expect(plan.monthlySavings).toBeLessThan(0); + expect(plan.costMultiplier).toBeGreaterThan(1); + expect(plan.tradeoffSummary).toContain("higher spend"); + } + + expect(advice.recommendedPlan).toBeUndefined(); + + // Baseline fallback for the hybrid strategy: all traffic stays on the baseline. + expect(advice.hybridStrategy.rules).toHaveLength(1); + expect(advice.hybridStrategy.rules[0]).toMatchObject({ + tier: "balanced_editing", + trafficPercentage: 100, + modelId: gpt4oMini.id, + allocatedMonthlyCost: advice.baselineMonthlySpend, + }); + expect(advice.hybridStrategy.blendedMonthlyCost).toBe(advice.baselineMonthlySpend); + expect(advice.hybridStrategy.monthlySavings).toBe(0); + expect(advice.hybridStrategy.savingsPercent).toBe(0); + expect(advice.hybridStrategy.summary).not.toContain("Tiered dispatching delivers"); + expect(advice.hybridStrategy.summary).toContain(gpt4oMini.displayName); + expectAllNumbersFinite(advice); + }); + + it("does not recommend a candidate that costs exactly the same as the baseline", () => { + const sameCostClone: ModelInfo = { ...sonnetModel, id: "sonnet-clone", displayName: "Sonnet Clone" }; + const advice = analyzeCostOptimization({ + baselineModel: sonnetModel, + candidateModels: [sameCostClone], + }); + + expect(advice.alternativePlans).toHaveLength(1); + expect(advice.alternativePlans[0].monthlySavings).toBe(0); + expect(advice.recommendedPlan).toBeUndefined(); + expect(advice.hybridStrategy.rules).toHaveLength(1); + expect(advice.hybridStrategy.rules[0].modelId).toBe(sonnetModel.id); + }); + + it("handles a truly free baseline without dividing by zero or claiming savings", () => { + const advice = analyzeCostOptimization({ + baselineModel: localLlama, + candidateModels: [haikuModel, gpt4oMini], + developerCount: 5, + workingDaysPerMonth: 22, + }); + + expect(advice.baselineMonthlySpend).toBe(0); + expect(advice.alternativePlans).toHaveLength(2); + for (const plan of advice.alternativePlans) { + expect(plan.tradeoffSummary).toMatch(/^Adds \$\d+\.\d{2}\/mo over the zero-cost baseline/); + expect(plan.tradeoffSummary).toContain("percentage uplift undefined"); + expect(plan.tradeoffSummary).not.toMatch(/Infinity|NaN|%\s+(cost reduction|higher spend)/); + expect(plan.savingsPercent).toBe(0); + expect(plan.costMultiplier).toBe(1); + expect(plan.monthlySavings).toBeLessThan(0); + } + + expect(advice.recommendedPlan).toBeUndefined(); + expect(advice.hybridStrategy.rules).toHaveLength(1); + expect(advice.hybridStrategy.rules[0].modelId).toBe(localLlama.id); + expect(advice.hybridStrategy.blendedMonthlyCost).toBe(0); + expect(advice.hybridStrategy.savingsPercent).toBe(0); + expectNoInvalidNumberText(advice); + expectAllNumbersFinite(advice); + }); + + it("describes a sub-cent addition over a free baseline in absolute terms", () => { + const microPriced: ModelInfo = { + ...haikuModel, + id: "micro-priced", + displayName: "Micro Priced", + inputPricePerMillion: 0.0001, + cachedInputPricePerMillion: 0.0001, + outputPricePerMillion: 0.0001, + }; + const advice = analyzeCostOptimization({ + baselineModel: localLlama, + candidateModels: [microPriced], + developerCount: 1, + workingDaysPerMonth: 1, + workload: { name: "Tiny", dailyRunsPerDev: 1, avgInputTokens: 1000, avgOutputTokens: 100, cacheHitRatio: 0 }, + }); + + const plan = advice.alternativePlans[0]; + expect(plan.totalMonthlySpend).toBe(0); + expect(plan.tradeoffSummary).toContain("Adds less than $0.01/mo over the zero-cost baseline"); + expect(advice.recommendedPlan).toBeUndefined(); + }); + + it("stays valid with zero runs: no NaN or infinities, candidates kept, baseline fallback", () => { + for (const zeroRuns of [ + { developerCount: 0, workingDaysPerMonth: 22 }, + { developerCount: 5, workingDaysPerMonth: 0 }, + { workload: { name: "No daily runs", dailyRunsPerDev: 0, avgInputTokens: 48_000, avgOutputTokens: 2_000, cacheHitRatio: 0.5 } }, + ]) { + const advice = analyzeCostOptimization({ + baselineModel: sonnetModel, + candidateModels: [haikuModel, gpt4oMini], + ...zeroRuns, + }); + + expect(advice.baselineMonthlySpend).toBe(0); + expect(advice.alternativePlans).toHaveLength(2); + for (const plan of advice.alternativePlans) { + expect(plan.totalMonthlySpend).toBe(0); + expect(plan.monthlySavings).toBe(0); + expect(plan.savingsPercent).toBe(0); + expect(plan.costMultiplier).toBe(1); + expect(plan.tradeoffSummary).toContain("No runs are projected"); + // A paid model with no usage is not described as a free baseline. + expect(plan.tradeoffSummary).not.toContain("zero-cost baseline"); + } + expect(advice.recommendedPlan).toBeUndefined(); + expect(advice.hybridStrategy.rules).toHaveLength(1); + expect(advice.hybridStrategy.rules[0].modelId).toBe(sonnetModel.id); + expect(advice.hybridStrategy.blendedMonthlyCost).toBe(0); + expect(advice.hybridStrategy.monthlySavings).toBe(0); + expect(advice.hybridStrategy.savingsPercent).toBe(0); + expect(advice.hybridStrategy.summary).not.toContain("Tiered dispatching delivers"); + expectNoInvalidNumberText(advice); + expectAllNumbersFinite(advice); + } + }); + + describe("sub-cent paid baseline", () => { + // 10 runs/month at 1,100 tokens: baseline costs ~$0.00011/mo, which rounds to $0.00. + const subCentWorkload = { + name: "Sub-cent", + dailyRunsPerDev: 10, + avgInputTokens: 1000, + avgOutputTokens: 100, + cacheHitRatio: 0, + }; + const subCentBaseline: ModelInfo = { + ...haikuModel, + id: "sub-cent-baseline", + displayName: "Sub-Cent Baseline", + inputPricePerMillion: 0.01, + cachedInputPricePerMillion: 0.01, + outputPricePerMillion: 0.01, + }; + const run = (candidateModels: ModelInfo[]) => + analyzeCostOptimization({ + baselineModel: subCentBaseline, + candidateModels, + developerCount: 1, + workingDaysPerMonth: 1, + workload: subCentWorkload, + }); + + it("is not treated as free when a pricier candidate rounds to a visible cent amount", () => { + const pricier: ModelInfo = { + ...haikuModel, + id: "pricier", + displayName: "Pricier", + inputPricePerMillion: 10, + cachedInputPricePerMillion: 10, + outputPricePerMillion: 10, + }; + const advice = run([pricier]); + + expect(advice.baselineMonthlySpend).toBe(0); // displayed value rounds to $0.00 + const plan = advice.alternativePlans[0]; + expect(plan.tradeoffSummary).not.toContain("zero-cost"); + expect(plan.tradeoffSummary).not.toContain("undefined"); + expect(plan.tradeoffSummary).toMatch(/^Invests \d+% higher spend/); + expect(plan.costMultiplier).toBeGreaterThan(1); + expect(advice.recommendedPlan).toBeUndefined(); + expect(advice.hybridStrategy.rules[0].modelId).toBe(subCentBaseline.id); + expectAllNumbersFinite(advice); + }); + + it("still recognises real unrounded savings against the paid baseline", () => { + const cheaper: ModelInfo = { + ...haikuModel, + id: "cheaper", + displayName: "Cheaper", + inputPricePerMillion: 0.0025, + cachedInputPricePerMillion: 0.0025, + outputPricePerMillion: 0.0025, + }; + const advice = run([cheaper]); + + expect(advice.baselineMonthlySpend).toBe(0); + const plan = advice.alternativePlans[0]; + expect(plan.totalMonthlySpend).toBe(0); // rounded display only + expect(plan.savingsPercent).toBe(75); + expect(plan.costMultiplier).toBe(0.25); + expect(plan.tradeoffSummary).toContain("Delivers 75% cost reduction"); + expect(plan.tradeoffSummary).not.toContain("zero-cost"); + expect(advice.recommendedPlan?.targetModelId).toBe("cheaper"); + expect(advice.hybridStrategy.savingsPercent).toBe(67.5); + expect(advice.hybridStrategy.summary).toContain("less than $0.01/mo (67.5%) savings"); + expectAllNumbersFinite(advice); + }); + }); + + it("assigns no hybrid offload traffic and uses the baseline when only pricier candidates exist", () => { + const advice = analyzeCostOptimization({ + baselineModel: haikuModel, + candidateModels: [sonnetModel], + }); + + expect(advice.alternativePlans).toHaveLength(1); + expect(advice.alternativePlans[0].monthlySavings).toBeLessThan(0); + + const { rules } = advice.hybridStrategy; + expect(rules).toHaveLength(1); + expect(rules[0].modelId).toBe(haikuModel.id); + expect(rules[0].trafficPercentage).toBe(100); + expect(rules.some((r) => r.modelId === sonnetModel.id)).toBe(false); + expect(rules.some((r) => r.tier === "fast_triage")).toBe(false); + expect(advice.hybridStrategy.blendedMonthlyCost).toBe(advice.baselineMonthlySpend); + expect(advice.hybridStrategy.monthlySavings).toBe(0); + expect(advice.recommendedPlan).toBeUndefined(); + }); + + it("leaves pricier candidates out of the hybrid tiers when a cheaper one exists", () => { + // Baseline is Haiku: Sonnet is pricier, GPT-4o mini is cheaper. + const advice = analyzeCostOptimization({ + baselineModel: haikuModel, + candidateModels: [sonnetModel, gpt4oMini], + }); + + expect(advice.alternativePlans.map((p) => p.targetModelId).sort()).toEqual([ + "claude-3-5-sonnet", + "gpt-4o-mini", + ]); + expect(advice.hybridStrategy.rules.map((r) => r.modelId)).not.toContain(sonnetModel.id); + + const triage = advice.hybridStrategy.rules.find((r) => r.tier === "fast_triage"); + const editing = advice.hybridStrategy.rules.find((r) => r.tier === "balanced_editing"); + const reasoning = advice.hybridStrategy.rules.find((r) => r.tier === "complex_reasoning"); + expect(triage?.modelId).toBe("gpt-4o-mini"); + expect(editing?.modelId).toBe("gpt-4o-mini"); + expect(reasoning?.modelId).toBe(haikuModel.id); + expect(advice.hybridStrategy.blendedMonthlyCost).toBeLessThan(advice.baselineMonthlySpend); + expect(advice.hybridStrategy.monthlySavings).toBeGreaterThan(0); + expect(advice.recommendedPlan?.targetModelId).toBe("gpt-4o-mini"); + }); + + it("keeps recommending a cheaper alternative (positive-savings control)", () => { + const advice = analyzeCostOptimization({ + baselineModel: sonnetModel, + candidateModels: [haikuModel, gpt4oMini, localLlama], + }); + + expect(advice.recommendedPlan).toBeDefined(); + expect(advice.recommendedPlan!.monthlySavings).toBeGreaterThan(0); + expect(advice.alternativePlans).toContain(advice.recommendedPlan); + expect(advice.recommendedPlan!.savingsPercent).toBeGreaterThanOrEqual(20); + expect(advice.hybridStrategy.rules).toHaveLength(3); + expect(advice.hybridStrategy.monthlySavings).toBeGreaterThan(0); + expect(advice.hybridStrategy.summary).toContain("Tiered dispatching delivers"); + expectAllNumbersFinite(advice); + }); + }); +}); diff --git a/packages/recommender/tests/estimateCost.test.ts b/packages/recommender/tests/estimateCost.test.ts index 91c461b..a56f620 100644 --- a/packages/recommender/tests/estimateCost.test.ts +++ b/packages/recommender/tests/estimateCost.test.ts @@ -59,6 +59,46 @@ describe("estimateCost", () => { expect(result.totalCost).toBe(0); }); + it.each([ + ["negative inputTokens", { inputTokens: -1 }], + ["negative outputTokens", { outputTokens: -100 }], + ["NaN inputTokens", { inputTokens: Number.NaN }], + ["NaN outputTokens", { outputTokens: Number.NaN }], + ["NaN cachedInputTokens", { cachedInputTokens: Number.NaN }], + ["infinite inputTokens", { inputTokens: Number.POSITIVE_INFINITY }], + ["infinite outputTokens", { outputTokens: Number.POSITIVE_INFINITY }], + ["infinite cachedInputTokens", { cachedInputTokens: Number.POSITIVE_INFINITY }], + ])("should reject %s", (_label, overrides) => { + expect(() => + estimateCost({ + model: testModel, + inputTokens: 1_000, + outputTokens: 100, + ...overrides, + }), + ).toThrow(RangeError); + }); + + it("should accept zero token counts at the boundary", () => { + const result = estimateCost({ + model: testModel, + inputTokens: 0, + outputTokens: 0, + cachedInputTokens: 0, + }); + expect(result.totalCost).toBe(0); + }); + + it("should accept valid fractional token counts", () => { + const result = estimateCost({ + model: testModel, + inputTokens: 1500.5, + outputTokens: 250.25, + }); + expect(Number.isFinite(result.totalCost)).toBe(true); + expect(result.totalCost).toBeGreaterThan(0); + }); + it("should fall back to input price when cached price is null", () => { const modelNoCache: ModelInfo = { ...testModel, diff --git a/packages/recommender/tests/recommendModels.test.ts b/packages/recommender/tests/recommendModels.test.ts index fb2f1ec..9c3ea08 100644 --- a/packages/recommender/tests/recommendModels.test.ts +++ b/packages/recommender/tests/recommendModels.test.ts @@ -84,6 +84,56 @@ const weakModel: ModelInfo = { outputPricePerMillion: 8, }; +// Two candidates identical in every scored dimension, so only the tiebreak separates them. +// The preferred id sorts after the other, making localeCompare pick the wrong one without a +// preference tiebreak. +const tiedBase: ModelInfo = { + id: "alpha/tied", + displayName: "Tied Model", + provider: "test", + contextWindow: 128000, + maxOutputTokens: 4096, + inputPricePerMillion: 0.02, + cachedInputPricePerMillion: 0.01, + outputPricePerMillion: 0.04, + supportsTools: false, + supportsImages: false, + supportsLocal: true, + privacyMode: "local", + codingScore: 90, + reasoningScore: 90, + latencyScore: 10, + updatedAt: "2025-01-01", +}; + +const tiedAlpha: ModelInfo = { ...tiedBase, id: "alpha/tied" }; +const tiedZeta: ModelInfo = { ...tiedBase, id: "zeta/tied" }; + +// Strictly cheapest, so it consumes the cheapest-sufficient tier and leaves the tied pair to +// contest the later tiers. +const cheapestDecoy: ModelInfo = { + ...tiedBase, + id: "decoy/cheapest", + displayName: "Cheapest Decoy", + inputPricePerMillion: 0.001, + cachedInputPricePerMillion: 0.0005, + outputPricePerMillion: 0.002, + codingScore: 20, + reasoningScore: 20, +}; + +// Higher totalScore than the tied pair (tool support and latency) but lower confidenceScore +// (weaker coding/reasoning), so it consumes the balanced tier without touching high-confidence. +const balancedDecoy: ModelInfo = { + ...tiedBase, + id: "decoy/balanced", + displayName: "Balanced Decoy", + supportsTools: true, + latencyScore: 100, + codingScore: 80, + reasoningScore: 80, +}; + describe("recommendModels", () => { it("should select cheapest sufficient model based on total cost", () => { const result = recommendModels({ @@ -136,6 +186,45 @@ describe("recommendModels", () => { expect(bigInAll).toBeUndefined(); }); + it("skips hybrid models without local execution support in local-first privacy mode", () => { + const remoteHybrid: ModelInfo = { + ...midModel, + id: "remote-hybrid", + displayName: "Remote Hybrid", + privacyMode: "hybrid", + supportsLocal: false, + }; + + const result = recommendModels({ + models: [cheapModel, remoteHybrid], + workspaceTokens: 5000, + goal: "debug", + privacyMode: "local-first", + }); + + expect(result.allScored.map((m) => m.modelId)).toContain("cheap"); + expect(result.allScored.map((m) => m.modelId)).not.toContain("remote-hybrid"); + }); + + it("allows hybrid models with verified local execution support in local-first privacy mode", () => { + const localHybrid: ModelInfo = { + ...midModel, + id: "local-hybrid", + displayName: "Local Hybrid", + privacyMode: "hybrid", + supportsLocal: true, + }; + + const result = recommendModels({ + models: [localHybrid], + workspaceTokens: 5000, + goal: "debug", + privacyMode: "local-first", + }); + + expect(result.allScored.map((m) => m.modelId)).toContain("local-hybrid"); + }); + it("reports when privacy filtering leaves no eligible models", () => { expect(() => recommendModels({ models: [bigModel], @@ -251,6 +340,7 @@ describe("recommendModels", () => { models: [cheapModel, midModel], workspaceTokens: 5000, goal: "debug", + privacyMode: "cloud-ok", }); const tierIds = [ result.cheapestSufficient.modelId, @@ -284,6 +374,7 @@ describe("recommendModels", () => { models: [cheapModel, midModel], workspaceTokens: 5000, goal: "debug", + privacyMode: "cloud-ok", }); const tierIds = [ result.cheapestSufficient.modelId, @@ -294,6 +385,60 @@ describe("recommendModels", () => { expect(uniqueIds.size).toBeGreaterThanOrEqual(2); }); + it("breaks cheapest-tier ties toward preferred models", () => { + const options = { models: [tiedAlpha, tiedZeta], workspaceTokens: 5000, goal: "debug" as const }; + + expect(recommendModels(options).cheapestSufficient.modelId).toBe("alpha/tied"); + expect(recommendModels({ ...options, preferredModelIds: ["zeta/tied"] }).cheapestSufficient.modelId) + .toBe("zeta/tied"); + }); + + it("breaks balanced-tier ties toward preferred models", () => { + const options = { models: [tiedAlpha, tiedZeta, cheapestDecoy], workspaceTokens: 5000, goal: "debug" as const }; + + const baseline = recommendModels(options); + expect(baseline.cheapestSufficient.modelId).toBe("decoy/cheapest"); + expect(baseline.balanced.modelId).toBe("alpha/tied"); + + const preferred = recommendModels({ ...options, preferredModelIds: ["zeta/tied"] }); + expect(preferred.balanced.modelId).toBe("zeta/tied"); + }); + + it("breaks high-confidence ties toward preferred models", () => { + const options = { + models: [tiedAlpha, tiedZeta, cheapestDecoy, balancedDecoy], + workspaceTokens: 5000, + goal: "debug" as const, + }; + + const baseline = recommendModels(options); + expect(baseline.cheapestSufficient.modelId).toBe("decoy/cheapest"); + expect(baseline.balanced.modelId).toBe("decoy/balanced"); + expect(baseline.highConfidence.modelId).toBe("alpha/tied"); + + const preferred = recommendModels({ ...options, preferredModelIds: ["zeta/tied"] }); + expect(preferred.highConfidence.modelId).toBe("zeta/tied"); + }); + + it("never lets preference override a real scoring difference", () => { + // cheapestDecoy is strictly cheaper, so preferring a tied model must not steal that tier. + const result = recommendModels({ + models: [tiedAlpha, tiedZeta, cheapestDecoy], + workspaceTokens: 5000, + goal: "debug", + preferredModelIds: ["zeta/tied"], + }); + + expect(result.cheapestSufficient.modelId).toBe("decoy/cheapest"); + }); + + it("ignores preferred ids that are not in the catalog", () => { + const options = { models: [tiedAlpha, tiedZeta], workspaceTokens: 5000, goal: "debug" as const }; + + expect(recommendModels({ ...options, preferredModelIds: ["nope/missing"] }).cheapestSufficient.modelId) + .toBe(recommendModels(options).cheapestSufficient.modelId); + }); + it("uses each tier's ranking when selecting a distinct fallback", () => { const result = recommendModels({ models: [cheapModel, weakModel, midModel, bigModel], diff --git a/packages/repo-map/src/budgetRepoMap.ts b/packages/repo-map/src/budgetRepoMap.ts index a93cfec..525366a 100644 --- a/packages/repo-map/src/budgetRepoMap.ts +++ b/packages/repo-map/src/budgetRepoMap.ts @@ -3,6 +3,9 @@ import { estimateTokens } from "@wma/tokenizers"; import { formatRepoMapMarkdown } from "./formatRepoMapMarkdown.js"; export function budgetRepoMap(repoMap: RepoMapResult, tokenBudget: number): RepoMapResult { + if (!Number.isFinite(tokenBudget) || tokenBudget <= 0) { + throw new RangeError(`tokenBudget must be a finite number > 0 (got ${tokenBudget})`); + } const result: RepoMapResult = { ...repoMap, importantFiles: [...repoMap.importantFiles], diff --git a/packages/repo-map/src/compactor/compactionLevels.ts b/packages/repo-map/src/compactor/compactionLevels.ts new file mode 100644 index 0000000..29c2bfe --- /dev/null +++ b/packages/repo-map/src/compactor/compactionLevels.ts @@ -0,0 +1,18 @@ +export type CompactionStage = 0 | 1 | 2 | 3; + +export interface CompactedCodeResult { + stage: CompactionStage; + originalChars: number; + compactedChars: number; + originalTokens: number; + compactedTokens: number; + reductionPercentage: number; + code: string; +} + +export interface CompactorOptions { + targetStage?: CompactionStage; + maxTargetTokens?: number; + language?: string; + charToTokenRatio?: number; +} diff --git a/packages/repo-map/src/compactor/progressiveCompactor.ts b/packages/repo-map/src/compactor/progressiveCompactor.ts new file mode 100644 index 0000000..408f559 --- /dev/null +++ b/packages/repo-map/src/compactor/progressiveCompactor.ts @@ -0,0 +1,258 @@ +import type { + CompactionStage, + CompactedCodeResult, + CompactorOptions, +} from "./compactionLevels.js"; + +const CONTROL_FLOW_KEYWORDS = new Set([ + "if", + "for", + "while", + "switch", + "catch", + "with", +]); + +/** + * Stage 1: Strip single-line, multi-line comments and docstrings. + */ +export function stripComments(code: string, language?: string): string { + const isPython = language?.toLowerCase().includes("python") || language?.toLowerCase() === "py"; + + let result = code; + if (!isPython) { + // Strip multi-line comments /* ... */ + result = result.replace(/\/\*[\s\S]*?\*\//g, ""); + // Strip single-line comments // ... + result = result.replace(/(^|[^\\])\/\/.*$/gm, "$1"); + } else { + // Python comments # ... + result = result.replace(/(^|[^\\])#.*$/gm, "$1"); + // Python triple-quoted docstrings """ ... """ + result = result.replace(/"""[\s\S]*?"""/g, '"""..."""'); + result = result.replace(/'''[\s\S]*?'''/g, "'''...'''"); + } + + // Remove empty comment lines and excessive consecutive blank lines + return result + .split("\n") + .map((line) => line.trimEnd()) + .filter((line, idx, arr) => line.length > 0 || (idx > 0 && arr[idx - 1].length > 0)) + .join("\n"); +} + +/** + * Stage 2: Collapse function and method implementation blocks { ... } to signatures. + */ +export function collapseFunctionBodies(code: string): string { + const lines = code.split("\n"); + const output: string[] = []; + let inBlock = false; + let braceDepth = 0; + let signatureHeader = ""; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const trimmed = line.trim(); + + // Check if line looks like a function, method, constructor, or arrow function + const isClassHeader = /^(export\s+)?(abstract\s+)?class\s+/.test(trimmed); + const firstWord = trimmed.match(/^(?:}\s*)?(?:else\s+)?([A-Za-z_$][A-Za-z0-9_$]*)/)?.[1]; + + const isGenericSignature = + !CONTROL_FLOW_KEYWORDS.has(firstWord ?? "") && + /\b[A-Za-z0-9_$]+\s*\([^)]*\)\s*(?::\s*[^;{]+)?\s*\{?$/.test(trimmed); + + const isSignature = + !isClassHeader && + (trimmed.startsWith("function ") || + trimmed.startsWith("export function ") || + trimmed.startsWith("async function ") || + trimmed.startsWith("export async function ") || + trimmed.startsWith("constructor") || + isGenericSignature || + /^(const|let|var)\s+[A-Za-z0-9_$]+\s*=\s*(async\s*)?\([^)]*\)\s*(=>)?\s*\{?$/.test(trimmed)); + + if (isSignature && !inBlock) { + if (line.includes("{")) { + const header = line.slice(0, line.indexOf("{") + 1); + output.push(`${header} /* collapsed */ }`); + // Count open vs close braces on this line + const opens = (line.match(/{/g) || []).length; + const closes = (line.match(/}/g) || []).length; + braceDepth = opens - closes; + if (braceDepth > 0) { + inBlock = true; + } + } else { + signatureHeader = line; + output.push(line); + } + continue; + } + + if (signatureHeader && !inBlock) { + if (line.includes("{")) { + output.push(" /* collapsed */\n}"); + inBlock = true; + braceDepth = 1; + signatureHeader = ""; + } + continue; + } + + if (inBlock) { + const opens = (line.match(/{/g) || []).length; + const closes = (line.match(/}/g) || []).length; + braceDepth += opens - closes; + if (braceDepth <= 0) { + inBlock = false; + braceDepth = 0; + } + continue; + } + + output.push(line); + } + + return output.join("\n"); +} + +/** + * Stage 3: Extract exported outline (interfaces, types, exports, signatures, imports). + */ +export function extractExportOutline(code: string): string { + const lines = code.split("\n"); + const outlineLines: string[] = []; + + for (const line of lines) { + const trimmed = line.trim(); + + if ( + trimmed.startsWith("import ") || + trimmed.startsWith("export ") || + trimmed.startsWith("interface ") || + trimmed.startsWith("type ") || + trimmed.startsWith("declare ") || + trimmed.startsWith("enum ") || + trimmed.startsWith("from ") || + trimmed === "}" || + trimmed.startsWith("export default") + ) { + outlineLines.push(line); + } + } + + return outlineLines.join("\n"); +} + +export function progressiveCompact( + code: string, + options: CompactorOptions = {}, +): CompactedCodeResult { + const ratio = options.charToTokenRatio ?? 3.8; + const originalChars = code.length; + const originalTokens = Math.max(1, Math.ceil(originalChars / ratio)); + + if (originalChars === 0) { + return { + stage: 0, + originalChars: 0, + compactedChars: 0, + originalTokens: 0, + compactedTokens: 0, + reductionPercentage: 0, + code: "", + }; + } + + // If specific targetStage is requested, apply directly + if (options.targetStage !== undefined) { + const stage = options.targetStage; + let compacted = code; + + if (stage >= 1) { + compacted = stripComments(compacted, options.language); + } + if (stage >= 2) { + compacted = collapseFunctionBodies(compacted); + } + if (stage >= 3) { + compacted = extractExportOutline(compacted); + } + + const compactedChars = compacted.length; + const compactedTokens = Math.max(1, Math.ceil(compactedChars / ratio)); + const reductionPercentage = Number( + (((originalTokens - compactedTokens) / originalTokens) * 100).toFixed(2), + ); + + return { + stage, + originalChars, + compactedChars, + originalTokens, + compactedTokens, + reductionPercentage: Math.max(0, reductionPercentage), + code: compacted, + }; + } + + // If maxTargetTokens is requested, find the lowest stage that satisfies it + const maxTokens = options.maxTargetTokens ?? originalTokens; + + if (originalTokens <= maxTokens) { + return { + stage: 0, + originalChars, + compactedChars: originalChars, + originalTokens, + compactedTokens: originalTokens, + reductionPercentage: 0, + code, + }; + } + + // Try Stage 1 + const stage1Code = stripComments(code, options.language); + const stage1Tokens = Math.max(1, Math.ceil(stage1Code.length / ratio)); + if (stage1Tokens <= maxTokens) { + return { + stage: 1, + originalChars, + compactedChars: stage1Code.length, + originalTokens, + compactedTokens: stage1Tokens, + reductionPercentage: Number((((originalTokens - stage1Tokens) / originalTokens) * 100).toFixed(2)), + code: stage1Code, + }; + } + + // Try Stage 2 + const stage2Code = collapseFunctionBodies(stage1Code); + const stage2Tokens = Math.max(1, Math.ceil(stage2Code.length / ratio)); + if (stage2Tokens <= maxTokens) { + return { + stage: 2, + originalChars, + compactedChars: stage2Code.length, + originalTokens, + compactedTokens: stage2Tokens, + reductionPercentage: Number((((originalTokens - stage2Tokens) / originalTokens) * 100).toFixed(2)), + code: stage2Code, + }; + } + + // Fallback to Stage 3 + const stage3Code = extractExportOutline(stage2Code); + const stage3Tokens = Math.max(1, Math.ceil(stage3Code.length / ratio)); + return { + stage: 3, + originalChars, + compactedChars: stage3Code.length, + originalTokens, + compactedTokens: stage3Tokens, + reductionPercentage: Number((((originalTokens - stage3Tokens) / originalTokens) * 100).toFixed(2)), + code: stage3Code, + }; +} diff --git a/packages/repo-map/src/index.ts b/packages/repo-map/src/index.ts index 1e93280..a0bc527 100644 --- a/packages/repo-map/src/index.ts +++ b/packages/repo-map/src/index.ts @@ -4,3 +4,5 @@ export { budgetRepoMap } from "./budgetRepoMap.js"; export { formatRepoMapMarkdown } from "./formatRepoMapMarkdown.js"; export { extractSymbols } from "./symbols/extractSymbols.js"; export { parseFileSymbols } from "./symbols/parseFileSymbols.js"; +export { progressiveCompact, stripComments, collapseFunctionBodies, extractExportOutline } from "./compactor/progressiveCompactor.js"; +export type * from "./compactor/compactionLevels.js"; diff --git a/packages/repo-map/src/selectImportantFiles.ts b/packages/repo-map/src/selectImportantFiles.ts index 29e793e..eed1a6f 100644 --- a/packages/repo-map/src/selectImportantFiles.ts +++ b/packages/repo-map/src/selectImportantFiles.ts @@ -93,6 +93,23 @@ function isGeneratedFile(relativePath: string, extension: string): boolean { return base.includes(".min.") || base.includes(".bundle."); } +/** + * The scanner's "generated" risk flag is authoritative; the path heuristic remains a fallback for + * scan results produced without it. + */ +function isGeneratedEntry(relativePath: string, extension: string, riskFlags: string[]): boolean { + return riskFlags.includes("generated") || isGeneratedFile(relativePath, extension); +} + +function dedupeByPath(files: RepoMapFile[]): RepoMapFile[] { + const seen = new Set(); + return files.filter((file) => { + if (seen.has(file.relativePath)) return false; + seen.add(file.relativePath); + return true; + }); +} + function matchPattern(relativePath: string, patterns: RegExp[]): boolean { return patterns.some((p) => p.test(relativePath)); } @@ -178,7 +195,7 @@ export function selectImportantFiles( priority, }; - if (isGeneratedFile(rp, file.extension)) { + if (isGeneratedEntry(rp, file.extension, file.riskFlags)) { generatedFiles.push(repoFile); } @@ -209,13 +226,20 @@ export function selectImportantFiles( for (const file of scanResult.files.filter((f) => !f.included)) { const rp = file.relativePath.replace(/\\/g, "/"); - excludedFiles.push({ + const repoFile: RepoMapFile = { relativePath: rp, language: getLanguage(file.extension), estimatedTokens: file.estimatedTokens, reason: file.excludedReason ?? "Excluded by configuration", priority: 0, - }); + }; + excludedFiles.push(repoFile); + + // Scanner exclusion is a token-budget decision. It must not erase the "do not edit this" + // guidance, so generated files stay listed here whether or not their tokens are counted. + if (isGeneratedEntry(rp, file.extension, file.riskFlags)) { + generatedFiles.push(repoFile); + } } const allImportant = [...entryPoints, ...configFiles, ...documentationFiles, ...testFiles, ...sourceFiles]; @@ -224,7 +248,8 @@ export function selectImportantFiles( .filter((f) => f.priority >= 60) .slice(0, options.maxFiles ?? 50); - const recommendedExclude = [...largeFiles, ...generatedFiles, ...excludedFiles, ...riskyFiles] + // These lists overlap by design, so collapse to one entry per file before recommending. + const recommendedExclude = dedupeByPath([...largeFiles, ...generatedFiles, ...excludedFiles, ...riskyFiles]) .filter((f) => f.estimatedTokens > 1000); return { @@ -251,7 +276,7 @@ function getPriority( const rp = relativePath.replace(/\\/g, "/"); if (matchPattern(rp, EXCLUDED_PATTERNS)) return 5; - if (isGeneratedFile(rp, extension)) return 10; + if (isGeneratedEntry(rp, extension, riskFlags)) return 10; if (riskFlags.includes("secret")) return 10; if (riskFlags.includes("lockfile")) return 15; if (riskFlags.includes("database-dump")) return 10; diff --git a/packages/repo-map/src/symbols/detectFrameworkRoutes.ts b/packages/repo-map/src/symbols/detectFrameworkRoutes.ts index 5d9a0a3..a58a805 100644 --- a/packages/repo-map/src/symbols/detectFrameworkRoutes.ts +++ b/packages/repo-map/src/symbols/detectFrameworkRoutes.ts @@ -1,65 +1,114 @@ import type { WorkspaceScanResult, RepoMapRoute, RepoMapFile } from "@wma/core"; import { readFileSync } from "node:fs"; +function hasNextProjectSignal( + root: string, + pathSet: Set, + fileByRelativePath: Map, +): boolean { + const prefix = root ? `${root}/` : ""; + if (["next.config.js", "next.config.mjs", "next.config.ts"].some((name) => pathSet.has(`${prefix}${name}`))) { + return true; + } + + const manifestPath = `${prefix}package.json`; + const manifest = fileByRelativePath.get(manifestPath); + if (!manifest) return false; + try { + const pkg = JSON.parse(readFileSync(manifest.path, "utf-8")) as { + dependencies?: Record; + devDependencies?: Record; + peerDependencies?: Record; + }; + return Boolean(pkg.dependencies?.next || pkg.devDependencies?.next || pkg.peerDependencies?.next); + } catch { + return false; + } +} + +function nextRouterRelativePath( + relativePath: string, + pathSet: Set, + fileByRelativePath: Map, +): { router: "app" | "pages"; path: string } | null { + const segments = relativePath.split("/"); + for (let index = 0; index < segments.length; index += 1) { + const isSrcRouter = segments[index] === "src" && (segments[index + 1] === "app" || segments[index + 1] === "pages"); + const isDirectRouter = segments[index] === "app" || segments[index] === "pages"; + if (!isSrcRouter && !isDirectRouter) continue; + + const routerIndex = isSrcRouter ? index + 1 : index; + const rootSegments = segments.slice(0, isSrcRouter ? index : routerIndex); + const root = rootSegments.join("/"); + if (!hasNextProjectSignal(root, pathSet, fileByRelativePath)) continue; + + return { + router: segments[routerIndex] as "app" | "pages", + path: segments.slice(routerIndex + 1).join("/"), + }; + } + return null; +} + +function normalizeDynamicSegments(route: string): string { + return route + .replace(/\[\[\.\.\.(\w+)\]\]/g, ":$1*?") + .replace(/\[\.\.\.(\w+)\]/g, ":$1*") + .replace(/\[(\w+)\]/g, ":$1"); +} + +function normalizeAppRoute(route: string): string | null { + const segments = route.split("/").filter(Boolean); + if (segments.some((segment) => segment.startsWith("_"))) return null; + return normalizeDynamicSegments(segments.filter((segment) => !/^\(.+\)$/.test(segment)).join("/")); +} + +function normalizePagesRoute(route: string): string | null { + const segments = route.split("/").filter(Boolean); + const leaf = segments.at(-1); + if (leaf && ["_app", "_document", "_error"].includes(leaf)) return null; + if (leaf === "index") segments.pop(); + return normalizeDynamicSegments(segments.join("/")); +} + export function detectFrameworkRoutes( scanResult: WorkspaceScanResult, files: RepoMapFile[], ): RepoMapRoute[] { const routes: RepoMapRoute[] = []; const pathSet = new Set(files.map((f) => f.relativePath)); + const fileByRelativePath = new Map(scanResult.files.map((file) => [file.relativePath.replace(/\\/g, "/"), file])); for (const file of scanResult.files) { if (!pathSet.has(file.relativePath)) continue; const rp = file.relativePath.replace(/\\/g, "/"); + const routerPath = nextRouterRelativePath(rp, pathSet, fileByRelativePath); + if (!routerPath) continue; - // Next.js App Router (matches both root app/page.tsx and nested app/some/page.tsx) - const appPageMatch = rp.match(/^app\/(?:(.*)\/)?page\.(?:js|jsx|ts|tsx)$/); - if (appPageMatch) { - const route = appPageMatch[1] ?? ""; + if (routerPath.router === "app") { + const match = routerPath.path.match(/^(?:(.*)\/)?(page|layout|route)\.(?:js|jsx|ts|tsx)$/); + if (!match) continue; + const route = normalizeAppRoute(match[1] ?? ""); + if (route === null) continue; routes.push({ relativePath: rp, - routePattern: `/${route.replace(/\/$/, "")}`, - framework: "nextjs", - reason: `Next.js App Router page: ${rp}`, - }); - continue; - } - - const appLayoutMatch = rp.match(/^app\/(?:(.*)\/)?layout\.(?:js|jsx|ts|tsx)$/); - if (appLayoutMatch) { - const route = appLayoutMatch[1] ?? ""; - routes.push({ - relativePath: rp, - routePattern: `/${route.replace(/\/$/, "")}`, - framework: "nextjs", - reason: `Next.js App Router layout: ${rp}`, - }); - continue; - } - - const appRouteMatch = rp.match(/^app\/(?:(.*)\/)?route\.(?:js|ts)$/); - if (appRouteMatch) { - const route = appRouteMatch[1] ?? ""; - routes.push({ - relativePath: rp, - routePattern: `/${route.replace(/\/$/, "")}`, + routePattern: `/${route}`, framework: "nextjs", - reason: `Next.js App Router API route: ${rp}`, + reason: `Next.js App Router ${match[2]}: ${rp}`, }); continue; } - // Next.js Pages Router - const pagesMatch = rp.match(/^pages\/(.*)(?:\.js|\.jsx|\.ts|\.tsx)$/); + const pagesMatch = routerPath.path.match(/^(.*)(?:\.js|\.jsx|\.ts|\.tsx)$/); if (pagesMatch) { - const route = pagesMatch[1].replace(/\/index$/, "").replace(/\[\.\.\.(\w+)\]/g, ":$1*").replace(/\[(\w+)\]/g, ":$1"); + const route = normalizePagesRoute(pagesMatch[1]); + if (route === null) continue; routes.push({ relativePath: rp, routePattern: `/${route}`, framework: "nextjs", reason: `Next.js Pages Router: ${rp}`, }); - continue; } } @@ -72,12 +121,7 @@ export function detectFrameworkRoutes( const routeRegex = /(?:app|router)\.(get|post|put|delete|patch|use)\s*\(\s*['"`]([^'"`]+)['"`]/g; let m: RegExpExecArray | null; while ((m = routeRegex.exec(content)) !== null) { - routes.push({ - relativePath: file.relativePath, - routePattern: m[2], - framework: "express", - reason: `${m[1].toUpperCase()} ${m[2]}`, - }); + routes.push({ relativePath: file.relativePath, routePattern: m[2], framework: "express", reason: `${m[1].toUpperCase()} ${m[2]}` }); } } catch { // Skip files that can't be read diff --git a/packages/repo-map/src/symbols/extractSymbols.ts b/packages/repo-map/src/symbols/extractSymbols.ts index 295fcba..f586fe2 100644 --- a/packages/repo-map/src/symbols/extractSymbols.ts +++ b/packages/repo-map/src/symbols/extractSymbols.ts @@ -1,69 +1,83 @@ -import type { WorkspaceScanResult, RepoMapSymbol, RepoMapImport, RepoMapRoute, RepoMapFile, RepoMapOptions } from "@wma/core"; -import { parseFileSymbols } from "./parseFileSymbols.js"; -import { detectFrameworkRoutes } from "./detectFrameworkRoutes.js"; -import { rankSymbols } from "./rankSymbols.js"; -import { resolveImports } from "./resolveImports.js"; - -export interface SymbolExtractionResult { - symbols: RepoMapSymbol[]; - imports: RepoMapImport[]; - routes: RepoMapRoute[]; - warnings: string[]; -} - -const SYMBOL_EXTS = new Set([".ts", ".tsx", ".js", ".jsx", ".py", ".java", ".kt", ".go", ".rs"]); - -export function extractSymbols( - scanResult: WorkspaceScanResult, - files: RepoMapFile[], - options: RepoMapOptions, -): SymbolExtractionResult { - const result: SymbolExtractionResult = { - symbols: [], - imports: [], - routes: [], - warnings: [], - }; - - if (options.enableSymbolExtraction === false) { - return result; - } - - const maxBytes = options.maxParseFileBytes ?? 256_000; - const maxSymbols = options.maxSymbols ?? 100; - const includeImports = options.includeImports !== false; - const includeRoutes = options.includeRoutes !== false; - - const fileSet = new Set(files.map((f) => f.relativePath)); - const eligible = scanResult.files.filter((f) => { - if (!fileSet.has(f.relativePath)) return false; - if (f.excludedReason) return false; - if (f.riskFlags.includes("generated")) return false; - if (f.riskFlags.includes("binary")) return false; - if (f.riskFlags.includes("secret")) return false; - if (f.riskFlags.includes("lockfile")) return false; - if (!SYMBOL_EXTS.has(f.extension)) return false; - if (f.bytes > maxBytes) return false; - return true; - }); - - for (const file of eligible) { - const fileSymbols = parseFileSymbols(file, includeImports); - result.symbols.push(...fileSymbols.symbols); - result.imports.push(...fileSymbols.imports); - - if (fileSymbols.error) { - result.warnings.push(`Failed to parse ${file.relativePath}: ${fileSymbols.error}`); - } - } - - if (includeRoutes) { - const routes = detectFrameworkRoutes(scanResult, files); - result.routes.push(...routes); - } - - result.symbols = rankSymbols(result.symbols, options.goal).slice(0, maxSymbols); - result.imports = resolveImports(result.imports, scanResult); - - return result; -} +import type { WorkspaceScanResult, RepoMapSymbol, RepoMapImport, RepoMapRoute, RepoMapFile, RepoMapOptions } from "@wma/core"; +import { parseFileSymbols } from "./parseFileSymbols.js"; +import { detectFrameworkRoutes } from "./detectFrameworkRoutes.js"; +import { rankSymbols } from "./rankSymbols.js"; +import { resolveImports } from "./resolveImports.js"; + +export interface SymbolExtractionResult { + symbols: RepoMapSymbol[]; + imports: RepoMapImport[]; + routes: RepoMapRoute[]; + warnings: string[]; +} + +const SYMBOL_EXTS = new Set([ + ".ts", + ".tsx", + ".mts", + ".cts", + ".js", + ".jsx", + ".mjs", + ".cjs", + ".py", + ".java", + ".kt", + ".go", + ".rs", +]); + +export function extractSymbols( + scanResult: WorkspaceScanResult, + files: RepoMapFile[], + options: RepoMapOptions, +): SymbolExtractionResult { + const result: SymbolExtractionResult = { + symbols: [], + imports: [], + routes: [], + warnings: [], + }; + + if (options.enableSymbolExtraction === false) { + return result; + } + + const maxBytes = options.maxParseFileBytes ?? 256_000; + const maxSymbols = options.maxSymbols ?? 100; + const includeImports = options.includeImports !== false; + const includeRoutes = options.includeRoutes !== false; + + const fileSet = new Set(files.map((f) => f.relativePath)); + const eligible = scanResult.files.filter((f) => { + if (!fileSet.has(f.relativePath)) return false; + if (f.excludedReason) return false; + if (f.riskFlags.includes("generated")) return false; + if (f.riskFlags.includes("binary")) return false; + if (f.riskFlags.includes("secret")) return false; + if (f.riskFlags.includes("lockfile")) return false; + if (!SYMBOL_EXTS.has(f.extension)) return false; + if (f.bytes > maxBytes) return false; + return true; + }); + + for (const file of eligible) { + const fileSymbols = parseFileSymbols(file, includeImports); + result.symbols.push(...fileSymbols.symbols); + result.imports.push(...fileSymbols.imports); + + if (fileSymbols.error) { + result.warnings.push(`Failed to parse ${file.relativePath}: ${fileSymbols.error}`); + } + } + + if (includeRoutes) { + const routes = detectFrameworkRoutes(scanResult, files); + result.routes.push(...routes); + } + + result.symbols = rankSymbols(result.symbols, options.goal).slice(0, maxSymbols); + result.imports = resolveImports(result.imports, scanResult); + + return result; +} diff --git a/packages/repo-map/src/symbols/parseFileSymbols.ts b/packages/repo-map/src/symbols/parseFileSymbols.ts index f35b17e..862debb 100644 --- a/packages/repo-map/src/symbols/parseFileSymbols.ts +++ b/packages/repo-map/src/symbols/parseFileSymbols.ts @@ -1,72 +1,82 @@ -import type { WorkspaceFileInfo, RepoMapSymbol, RepoMapImport } from "@wma/core"; -import { readFileSync } from "node:fs"; -import { extractTypeScriptImports, extractTypeScriptSymbols } from "./parsers/typescriptSymbols.js"; -import { extractJavaScriptSymbols } from "./parsers/javascriptSymbols.js"; -import { extractPythonSymbols } from "./parsers/pythonSymbols.js"; -import { extractCLikeSymbols } from "./parsers/cLikeSymbols.js"; - -export interface ParsedFileSymbols { - symbols: RepoMapSymbol[]; - imports: RepoMapImport[]; - error: string | null; -} - -export function parseFileSymbols(file: WorkspaceFileInfo, includeImports: boolean): ParsedFileSymbols { - try { - const content = readFileSync(file.path, "utf-8"); - const ext = file.extension; - - let symbols: RepoMapSymbol[] = []; - let imports: RepoMapImport[] = []; - let error: string | null = null; - - try { - if (ext === ".ts" || ext === ".tsx") { - symbols = extractTypeScriptSymbols(content, file.relativePath, ext); - if (includeImports) { - imports = extractTypeScriptImports(content, file.relativePath, ext); - } - } else if (ext === ".js" || ext === ".jsx") { - symbols = extractJavaScriptSymbols(content, file.relativePath, ext); - if (includeImports) { - imports = extractJavaScriptImports(content, file.relativePath, ext); - } - } else if (ext === ".py") { - symbols = extractPythonSymbols(content, file.relativePath); - if (includeImports) { - imports = extractPythonImports(content, file.relativePath); - } - } else if ([".java", ".kt", ".go", ".rs"].includes(ext)) { - symbols = extractCLikeSymbols(content, file.relativePath, ext); - } - } catch (parseErr) { - error = parseErr instanceof Error ? parseErr.message : String(parseErr); - return { symbols: [], imports: [], error }; - } - - return { symbols, imports, error: null }; - } catch (readErr: any) { - return { symbols: [], imports: [], error: readErr?.message ?? String(readErr) }; - } -} - -function extractJavaScriptImports(content: string, relativePath: string, ext: string): RepoMapImport[] { - return extractTypeScriptImports(content, relativePath, ext); -} - -function extractPythonImports(content: string, relativePath: string): RepoMapImport[] { - const imports: RepoMapImport[] = []; - - const importRegex = /^import\s+(\S+)/gm; - let m: RegExpExecArray | null; - while ((m = importRegex.exec(content)) !== null) { - imports.push({ source: m[1], relativePath, kind: "import" }); - } - - const fromImportRegex = /^from\s+(\S+)\s+import/gm; - while ((m = fromImportRegex.exec(content)) !== null) { - imports.push({ source: m[1], relativePath, kind: "import" }); - } - - return imports; -} +import type { WorkspaceFileInfo, RepoMapSymbol, RepoMapImport } from "@wma/core"; +import { readFileSync } from "node:fs"; +import { extractTypeScriptImports, extractTypeScriptSymbols } from "./parsers/typescriptSymbols.js"; +import { extractJavaScriptSymbols } from "./parsers/javascriptSymbols.js"; +import { extractPythonSymbols } from "./parsers/pythonSymbols.js"; +import { extractCLikeSymbols } from "./parsers/cLikeSymbols.js"; + +export interface ParsedFileSymbols { + symbols: RepoMapSymbol[]; + imports: RepoMapImport[]; + error: string | null; +} + +export function parseFileSymbols(file: WorkspaceFileInfo, includeImports: boolean): ParsedFileSymbols { + try { + const content = readFileSync(file.path, "utf-8"); + const ext = file.extension; + + let symbols: RepoMapSymbol[] = []; + let imports: RepoMapImport[] = []; + let error: string | null = null; + + try { + if ([".ts", ".tsx", ".mts", ".cts"].includes(ext)) { + symbols = extractTypeScriptSymbols(content, file.relativePath, ext); + if (includeImports) { + imports = extractTypeScriptImports(content, file.relativePath, ext); + } + } else if ([".js", ".jsx", ".mjs", ".cjs"].includes(ext)) { + symbols = extractJavaScriptSymbols(content, file.relativePath, ext); + if (includeImports) { + imports = extractJavaScriptImports(content, file.relativePath, ext); + } + } else if (ext === ".py") { + symbols = extractPythonSymbols(content, file.relativePath); + if (includeImports) { + imports = extractPythonImports(content, file.relativePath); + } + } else if ([".java", ".kt", ".go", ".rs"].includes(ext)) { + symbols = extractCLikeSymbols(content, file.relativePath, ext); + } + } catch (parseErr) { + error = parseErr instanceof Error ? parseErr.message : String(parseErr); + return { symbols: [], imports: [], error }; + } + + return { symbols, imports, error: null }; + } catch (readErr: any) { + return { symbols: [], imports: [], error: readErr?.message ?? String(readErr) }; + } +} + +function extractJavaScriptImports(content: string, relativePath: string, ext: string): RepoMapImport[] { + return extractTypeScriptImports(content, relativePath, ext); +} + +function extractPythonImports(content: string, relativePath: string): RepoMapImport[] { + const imports: RepoMapImport[] = []; + + const importRegex = /^import[ \t]+([^#\r\n]*?)[ \t]*(?:#.*)?$/gm; + let m: RegExpExecArray | null; + + while ((m = importRegex.exec(content)) !== null) { + const modules = m[1].split(",").map((item) => { + const moduleName = item.trim().split(/\s+as\s+/)[0]; + return moduleName; + }); + + for (const source of modules) { + if (source) { + imports.push({ source, relativePath, kind: "import" }); + } + } + } + + const fromImportRegex = /^from\s+(\S+)\s+import/gm; + while ((m = fromImportRegex.exec(content)) !== null) { + imports.push({ source: m[1], relativePath, kind: "import" }); + } + + return imports; +} diff --git a/packages/repo-map/tests/createRepoMap.test.ts b/packages/repo-map/tests/createRepoMap.test.ts index e4c4b56..aeaaf51 100644 --- a/packages/repo-map/tests/createRepoMap.test.ts +++ b/packages/repo-map/tests/createRepoMap.test.ts @@ -127,6 +127,37 @@ describe("createRepoMap", () => { expect(ts).toBeDefined(); }); + it("warns about generated files that the scanner excluded", () => { + const files: WorkspaceFileInfo[] = [ + makeFile({ relativePath: "src/index.ts", extension: ".ts", estimatedTokens: 200 }), + makeFile({ relativePath: "src/vendor.min.js", extension: ".js", estimatedTokens: 5000, included: false, excludedReason: "generated", riskFlags: ["generated"] }), + ]; + const scan: WorkspaceScanResult = { + rootPath: "/workspace", + scannedAt: new Date().toISOString(), + totalFiles: files.length, + includedFiles: files.filter((f) => f.included).length, + excludedFiles: files.filter((f) => !f.included).length, + totalBytes: files.reduce((s, f) => s + f.bytes, 0), + includedBytes: files.filter((f) => f.included).reduce((s, f) => s + f.bytes, 0), + totalEstimatedTokens: files.reduce((s, f) => s + f.estimatedTokens, 0), + includedTokens: files.filter((f) => f.included).reduce((s, f) => s + f.estimatedTokens, 0), + files, + folders: [], + languages: [], + warnings: [], + riskFiles: files.filter((f) => f.riskFlags.length > 0), + }; + const result = createRepoMap(scan, { tokenBudget: 8000 }); + + expect(result.generatedFiles.map((f) => f.relativePath)).toContain("src/vendor.min.js"); + expect(result.agentInstructions.some((line) => line.includes("generated file(s)"))).toBe(true); + + // The excluded file keeps its tokens out of the map's important set. + expect(result.importantFiles.map((f) => f.relativePath)).not.toContain("src/vendor.min.js"); + expect(result.recommendedExclude.filter((f) => f.relativePath === "src/vendor.min.js")).toHaveLength(1); + }); + it("handles empty scan result gracefully", () => { const scan: WorkspaceScanResult = { rootPath: "/empty", diff --git a/packages/repo-map/tests/progressiveCompactor.test.ts b/packages/repo-map/tests/progressiveCompactor.test.ts new file mode 100644 index 0000000..0296bfd --- /dev/null +++ b/packages/repo-map/tests/progressiveCompactor.test.ts @@ -0,0 +1,228 @@ +import { describe, it, expect } from "vitest"; +import { + stripComments, + collapseFunctionBodies, + extractExportOutline, + progressiveCompact, +} from "../src/index.js"; + +const SAMPLE_TS = ` +import { Database } from "./db.js"; + +/** + * User service handles registration and authentication. + * It coordinates with the SQL database. + */ +export class UserService { + // Database connection handle + private db: Database; + + constructor(db: Database) { + this.db = db; + } + + /* + * Registers a new user with email and hashed password. + */ + public async register(email: string, pass: string): Promise { + const valid = email.includes("@"); + if (!valid) return false; + await this.db.save({ email, pass }); + return true; + } +} + +export interface UserDTO { + id: string; + email: string; +} +`; + +describe("Progressive Compactor", () => { + it("strips single-line and multi-line comments in Stage 1", () => { + const stripped = stripComments(SAMPLE_TS); + + expect(stripped).not.toContain("User service handles registration"); + expect(stripped).not.toContain("Database connection handle"); + expect(stripped).not.toContain("Registers a new user"); + expect(stripped).toContain("export class UserService"); + expect(stripped).toContain("public async register"); + }); + + it("strips Python comments and docstrings", () => { + const pyCode = ` +# Module docstring +"""This is a service module""" +def process_data(items): + # Process items in loop + result = [] + for item in items: + result.append(item * 2) + return result +`; + const stripped = stripComments(pyCode, "python"); + + expect(stripped).not.toContain("# Module docstring"); + expect(stripped).not.toContain("# Process items in loop"); + expect(stripped).toContain("def process_data(items):"); + }); + + it("collapses function and method bodies in Stage 2", () => { + const collapsed = collapseFunctionBodies(stripComments(SAMPLE_TS)); + + expect(collapsed).toContain("/* collapsed */"); + expect(collapsed).toContain("export class UserService"); + expect(collapsed).not.toContain("this.db.save({ email, pass })"); + }); + it("preserves top-level control-flow blocks in Stage 2", () => { + const code = ` +if (ready) { + runTask(); +} else if (pending) { + queueTask(); +} + +for (const item of items) { + process(item); +} + +while (ready) { + wait(); +} + +switch (value) { + case 1: + handleOne(); + break; +} + +try { + risky(); +}catch (error) { + handleError(error); +} + +with (context) { + useContext(); +} +`; + + const result = collapseFunctionBodies(code); + + expect(result).toBe(code); + expect(result).toContain("runTask();"); + expect(result).toContain("queueTask();"); + expect(result).toContain("process(item);"); + expect(result).toContain("wait();"); + expect(result).toContain("handleOne();"); + expect(result).toContain("handleError(error);"); + expect(result).toContain("useContext();"); + }); + + it("extracts export and interface outline in Stage 3", () => { + const outline = extractExportOutline(SAMPLE_TS); + + expect(outline).toContain('import { Database } from "./db.js";'); + expect(outline).toContain("export class UserService"); + expect(outline).toContain("export interface UserDTO {"); + expect(outline).not.toContain("this.db = db;"); + }); + + it("returns stage 0 when code already fits under maxTargetTokens", () => { + const result = progressiveCompact("const x = 1;", { + maxTargetTokens: 1000, + }); + + expect(result.stage).toBe(0); + expect(result.reductionPercentage).toBe(0); + expect(result.code).toBe("const x = 1;"); + }); + + it("progressively steps from stage 0 to stage 1 when comment removal fits", () => { + const result = progressiveCompact(SAMPLE_TS, { maxTargetTokens: 110 }); + + expect(result.stage).toBe(1); + expect(result.compactedTokens).toBeLessThan(result.originalTokens); + expect(result.code).not.toContain("User service handles registration"); + }); + + it("progressively steps to stage 2 when body collapsing is needed", () => { + const result = progressiveCompact(SAMPLE_TS, { maxTargetTokens: 90 }); + + expect(result.stage).toBe(2); + expect(result.code).toContain("/* collapsed */"); + expect(result.reductionPercentage).toBeGreaterThan(15); + }); + + it("progressively steps to stage 3 for tight token limits", () => { + const result = progressiveCompact(SAMPLE_TS, { maxTargetTokens: 30 }); + + expect(result.stage).toBe(3); + expect(result.code).toContain("export interface UserDTO"); + expect(result.reductionPercentage).toBeGreaterThan(45); + }); + + it("handles empty input string safely", () => { + const result = progressiveCompact(""); + + expect(result.stage).toBe(0); + expect(result.originalTokens).toBe(0); + expect(result.compactedTokens).toBe(0); + expect(result.code).toBe(""); + }); + + it("directly applies targetStage when specified", () => { + const result = progressiveCompact(SAMPLE_TS, { targetStage: 3 }); + + expect(result.stage).toBe(3); + expect(result.code).toContain("export interface UserDTO"); + }); + it("preserves control-flow blocks when Stage 2 is selected by budget", () => { + const code = ` +if (ready) { + runTask(); +} else if (pending) { + queueTask(); +} + +try { + risky(); +}catch (error) { + handleError(error); +} + +for (const item of items) { + process(item); +} + +function largeFunction() { + const a = "this is a large function body"; + const b = "this is another line of content"; + const c = "more content to make stage 2 necessary"; + console.log(a, b, c); + return a + b + c; +} +`; + + const stage1 = stripComments(code); + const stage2 = collapseFunctionBodies(stage1); + + const stage1Tokens = Math.ceil(stage1.length / 3.8); + const stage2Tokens = Math.ceil(stage2.length / 3.8); + + const result = progressiveCompact(code, { + maxTargetTokens: stage2Tokens, + }); + + expect(stage1Tokens).toBeGreaterThan(stage2Tokens); + expect(result.stage).toBe(2); + + expect(result.code).toContain("} else if (pending) {\n queueTask();\n}"); + expect(result.code).toContain("}catch (error) {\n handleError(error);\n}"); + expect(result.code).toContain("runTask();"); + expect(result.code).toContain("queueTask();"); + expect(result.code).toContain("handleError(error);"); + expect(result.code).toContain("process(item);"); + expect(result.code).toContain("function largeFunction()"); + }); +}); diff --git a/packages/repo-map/tests/selectImportantFiles.test.ts b/packages/repo-map/tests/selectImportantFiles.test.ts index 01a272a..1e83203 100644 --- a/packages/repo-map/tests/selectImportantFiles.test.ts +++ b/packages/repo-map/tests/selectImportantFiles.test.ts @@ -118,6 +118,35 @@ describe("selectImportantFiles", () => { expect(selected.configFiles.length).toBeGreaterThanOrEqual(2); }); + it("keeps scanner-classified generated files listed after exclusion", () => { + const result = makeScanResult([ + makeFile({ relativePath: "src/vendor.min.js", extension: ".js", estimatedTokens: 5000, included: false, excludedReason: "generated", riskFlags: ["generated"] }), + makeFile({ relativePath: "src/index.ts", extension: ".ts" }), + ]); + const selected = selectImportantFiles(result, { tokenBudget: 8000 }); + + const generated = selected.generatedFiles.find((f) => f.relativePath === "src/vendor.min.js"); + expect(generated).toBeDefined(); + expect(selected.excludedFiles.map((f) => f.relativePath)).toContain("src/vendor.min.js"); + + // Exclusion is a token-budget decision, so the file must not be recommended for inclusion. + expect(selected.importantFiles.map((f) => f.relativePath)).not.toContain("src/vendor.min.js"); + expect(selected.recommendedInclude.map((f) => f.relativePath)).not.toContain("src/vendor.min.js"); + + const excludeCount = selected.recommendedExclude.filter((f) => f.relativePath === "src/vendor.min.js").length; + expect(excludeCount).toBe(1); + }); + + it("recommends each overlapping file for exclusion only once", () => { + const result = makeScanResult([ + makeFile({ relativePath: "dist/bundle.min.js", extension: ".js", estimatedTokens: 8000, riskFlags: ["generated"] }), + ]); + const selected = selectImportantFiles(result, { tokenBudget: 8000 }); + + const paths = selected.recommendedExclude.map((f) => f.relativePath); + expect(paths).toEqual([...new Set(paths)]); + }); + it("marks excluded files", () => { const result = makeScanResult([ makeFile({ relativePath: "node_modules/foo/index.js", extension: ".js", included: false, excludedReason: "In .gitignore" }), diff --git a/packages/repo-map/tests/symbols/detectFrameworkRoutes.test.ts b/packages/repo-map/tests/symbols/detectFrameworkRoutes.test.ts index e5e9efe..3794b8f 100644 --- a/packages/repo-map/tests/symbols/detectFrameworkRoutes.test.ts +++ b/packages/repo-map/tests/symbols/detectFrameworkRoutes.test.ts @@ -1,122 +1,76 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { detectFrameworkRoutes } from "../../src/symbols/detectFrameworkRoutes.js"; import type { WorkspaceScanResult, WorkspaceFileInfo, RepoMapFile } from "@wma/core"; -import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; let tempDir: string; -beforeAll(() => { - tempDir = mkdtempSync(join(tmpdir(), "wma-routes-test-")); -}); - -afterAll(() => { - rmSync(tempDir, { recursive: true, force: true }); -}); +beforeAll(() => { tempDir = mkdtempSync(join(tmpdir(), "wma-routes-test-")); }); +afterAll(() => { rmSync(tempDir, { recursive: true, force: true }); }); function makeFileInfo(relativePath: string, extension: string, content?: string): WorkspaceFileInfo { - return { - path: join(tempDir, relativePath.replace(/\//g, "\\")), - relativePath: relativePath.replace(/\\/g, "/"), - extension, - language: extension === ".ts" ? "TypeScript" : extension === ".tsx" ? "TypeScript React" : "Unknown", - bytes: content ? Buffer.byteLength(content, "utf-8") : 100, - estimatedTokens: 25, - included: true, - riskFlags: [], - }; -} - -function makeRepoMapFile(relativePath: string): RepoMapFile { - return { - relativePath: relativePath.replace(/\\/g, "/"), - estimatedTokens: 100, - reason: "test", - priority: 5, - }; + return { path: join(tempDir, relativePath.replace(/\//g, "\\")), relativePath: relativePath.replace(/\\/g, "/"), extension, language: extension === ".ts" ? "TypeScript" : extension === ".tsx" ? "TypeScript React" : "Unknown", bytes: content ? Buffer.byteLength(content, "utf-8") : 100, estimatedTokens: 25, included: true, riskFlags: [] }; } +function makeRepoMapFile(relativePath: string): RepoMapFile { return { relativePath: relativePath.replace(/\\/g, "/"), estimatedTokens: 100, reason: "test", priority: 5 }; } +function scan(files: WorkspaceFileInfo[]): WorkspaceScanResult { return { rootPath: tempDir, scannedAt: new Date().toISOString(), totalFiles: files.length, includedFiles: files.length, excludedFiles: 0, totalBytes: files.length * 100, includedBytes: files.length * 100, totalEstimatedTokens: files.length * 25, includedTokens: files.length * 25, files, folders: [], languages: [], warnings: [], riskFiles: [] }; } describe("detectFrameworkRoutes", () => { - it("detects Next.js app routes", () => { - const files = [makeFileInfo("app/page.jsx", ".jsx"), makeFileInfo("app/api/hello/route.js", ".js")]; - const important = files.map((f) => makeRepoMapFile(f.relativePath)); - const scanResult: WorkspaceScanResult = { - rootPath: tempDir, - scannedAt: new Date().toISOString(), - totalFiles: 2, - includedFiles: 2, - excludedFiles: 0, - totalBytes: 200, - includedBytes: 200, - totalEstimatedTokens: 50, - includedTokens: 50, - files, - folders: [], - languages: [], - warnings: [], - riskFiles: [], - }; - const routes = detectFrameworkRoutes(scanResult, important); - expect(routes.length).toBeGreaterThanOrEqual(2); - expect(routes.some((r) => r.relativePath === "app/page.jsx")).toBe(true); - expect(routes.some((r) => r.relativePath === "app/api/hello/route.js")).toBe(true); + it("detects root, src and package-nested Next.js app routes relative to the router root", () => { + const files = [ + makeFileInfo("app/page.jsx", ".jsx"), + makeFileInfo("src/app/users/[id]/page.tsx", ".tsx"), + makeFileInfo("apps/dashboard/app/docs/page.tsx", ".tsx"), + makeFileInfo("packages/admin/src/app/api/users/route.ts", ".ts"), + makeFileInfo("next.config.js", ".js"), + makeFileInfo("apps/dashboard/next.config.js", ".js"), + makeFileInfo("packages/admin/next.config.ts", ".ts"), + ]; + const routes = detectFrameworkRoutes(scan(files), files.map((f) => makeRepoMapFile(f.relativePath))); + expect(routes.find((r) => r.relativePath === "app/page.jsx")?.routePattern).toBe("/"); + expect(routes.find((r) => r.relativePath === "src/app/users/[id]/page.tsx")?.routePattern).toBe("/users/:id"); + expect(routes.find((r) => r.relativePath === "apps/dashboard/app/docs/page.tsx")?.routePattern).toBe("/docs"); + expect(routes.find((r) => r.relativePath === "packages/admin/src/app/api/users/route.ts")?.routePattern).toBe("/api/users"); + }); + + it("detects src and package-nested Pages Router paths", () => { + const files = [ + makeFileInfo("src/pages/index.tsx", ".tsx"), + makeFileInfo("apps/site/pages/blog/[slug].tsx", ".tsx"), + makeFileInfo("next.config.mjs", ".mjs"), + makeFileInfo("apps/site/next.config.js", ".js"), + ]; + const routes = detectFrameworkRoutes(scan(files), files.map((f) => makeRepoMapFile(f.relativePath))); + expect(routes.find((r) => r.relativePath === "src/pages/index.tsx")?.routePattern).toBe("/"); + expect(routes.find((r) => r.relativePath === "apps/site/pages/blog/[slug].tsx")?.routePattern).toBe("/blog/:slug"); + }); + + it("does not classify app routes in a non-Next package", () => { + const packagePath = join(tempDir, "packages", "admin", "package.json"); + const pagePath = join(tempDir, "packages", "admin", "app", "users", "page.tsx"); + mkdirSync(join(tempDir, "packages", "admin", "app", "users"), { recursive: true }); + writeFileSync(packagePath, JSON.stringify({ dependencies: { react: "^19.0.0" } }), "utf-8"); + writeFileSync(pagePath, "export default function Page() { return null; }", "utf-8"); + + const manifest = makeFileInfo("packages/admin/package.json", ".json"); + manifest.path = packagePath; + const page = makeFileInfo("packages/admin/app/users/page.tsx", ".tsx"); + page.path = pagePath; + const files = [manifest, page]; + expect(detectFrameworkRoutes(scan(files), files.map((f) => makeRepoMapFile(f.relativePath)))).toEqual([]); }); - it("detects Next.js pages routes", () => { - const files = [makeFileInfo("pages/index.tsx", ".tsx")]; - const important = files.map((f) => makeRepoMapFile(f.relativePath)); - const scanResult: WorkspaceScanResult = { - rootPath: tempDir, - scannedAt: new Date().toISOString(), - totalFiles: 1, - includedFiles: 1, - excludedFiles: 0, - totalBytes: 100, - includedBytes: 100, - totalEstimatedTokens: 25, - includedTokens: 25, - files, - folders: [], - languages: [], - warnings: [], - riskFiles: [], - }; - const routes = detectFrameworkRoutes(scanResult, important); - expect(routes.some((r) => r.framework === "nextjs")).toBe(true); + it("does not treat an unrelated app directory without Next.js route files as a route", () => { + const files = [makeFileInfo("lib/app/helpers.ts", ".ts")]; + expect(detectFrameworkRoutes(scan(files), files.map((f) => makeRepoMapFile(f.relativePath)))).toEqual([]); }); it("detects Express-style routes from file content", () => { - const content = ` -const app = express(); -app.get("/api/health", (req, res) => {}); -app.post("/api/data", (req, res) => {}); -const router = Router(); -router.get("/users", (req, res) => {}); -`; + const content = `const app = express();\napp.get("/api/health", (req, res) => {});\napp.post("/api/data", (req, res) => {});\nconst router = Router();\nrouter.get("/users", (req, res) => {});`; writeFileSync(join(tempDir, "server.ts"), content, "utf-8"); const files = [makeFileInfo("server.ts", ".ts", content)]; - const important = files.map((f) => makeRepoMapFile(f.relativePath)); - const scanResult: WorkspaceScanResult = { - rootPath: tempDir, - scannedAt: new Date().toISOString(), - totalFiles: 1, - includedFiles: 1, - excludedFiles: 0, - totalBytes: Buffer.byteLength(content, "utf-8"), - includedBytes: Buffer.byteLength(content, "utf-8"), - totalEstimatedTokens: 50, - includedTokens: 50, - files, - folders: [], - languages: [], - warnings: [], - riskFiles: [], - }; - const routes = detectFrameworkRoutes(scanResult, important); - const patterns = routes.map((r) => r.routePattern); - expect(patterns).toContain("/api/health"); - expect(patterns).toContain("/api/data"); - expect(patterns).toContain("/users"); + const patterns = detectFrameworkRoutes(scan(files), files.map((f) => makeRepoMapFile(f.relativePath))).map((r) => r.routePattern); + expect(patterns).toContain("/api/health"); expect(patterns).toContain("/api/data"); expect(patterns).toContain("/users"); }); }); diff --git a/packages/repo-map/tests/symbols/extractSymbols.test.ts b/packages/repo-map/tests/symbols/extractSymbols.test.ts index 447baa4..dbd0945 100644 --- a/packages/repo-map/tests/symbols/extractSymbols.test.ts +++ b/packages/repo-map/tests/symbols/extractSymbols.test.ts @@ -1,216 +1,321 @@ -import { describe, it, expect, beforeAll, afterAll } from "vitest"; -import { parseFileSymbols } from "../../src/symbols/parseFileSymbols.js"; -import type { WorkspaceFileInfo } from "@wma/core"; -import { mkdtempSync, writeFileSync, rmSync } from "node:fs"; -import { join } from "node:path"; -import { tmpdir } from "node:os"; - -let tempDir: string; - -beforeAll(() => { - tempDir = mkdtempSync(join(tmpdir(), "wma-symbols-test-")); -}); - -afterAll(() => { - rmSync(tempDir, { recursive: true, force: true }); -}); - -function makeFileInfo(relativePath: string, ext: string, name: string, bytes = 1000): WorkspaceFileInfo { - return { - path: join(tempDir, name), - relativePath, - extension: ext, - language: ext === ".py" ? "Python" : ext === ".tsx" || ext === ".jsx" ? "TypeScript React" : "TypeScript", - bytes, - estimatedTokens: Math.floor(bytes / 4), - included: true, - riskFlags: [], - }; -} - -describe("parseFileSymbols - TypeScript", () => { - it("extracts TS functions, classes, interfaces, types", () => { - const content = ` -export function hello(name: string): string { - return "hello " + name; -} - -class MyClass { - doStuff() {} -} - -interface MyInterface { - name: string; -} - -type MyType = string | number; - -enum MyEnum { - A, - B, -} - -const myConst = 42; -`; - const f = join(tempDir, "test.ts"); - writeFileSync(f, content, "utf-8"); - const file = makeFileInfo("src/test.ts", ".ts", "test.ts"); - const result = parseFileSymbols(file, true); - expect(result.error).toBeNull(); - const names = result.symbols.map((s) => s.name); - expect(names).toContain("hello"); - expect(names).toContain("MyClass"); - expect(names).toContain("MyInterface"); - expect(names).toContain("MyType"); - expect(names).toContain("MyEnum"); - expect(names).toContain("myConst"); - expect(names).toContain("MyClass.doStuff"); - - const hello = result.symbols.find((s) => s.name === "hello")!; - expect(hello.exported).toBe(true); - expect(hello.kind).toBe("function"); - }); - - it("detects async functions", () => { - const content = "export async function fetchData() { return null; }"; - const f = join(tempDir, "test2.ts"); - writeFileSync(f, content, "utf-8"); - const file = makeFileInfo("src/test.ts", ".ts", "test2.ts"); - const result = parseFileSymbols(file, true); - const fn = result.symbols.find((s) => s.name === "fetchData"); - expect(fn).toBeDefined(); - expect(fn!.async).toBe(true); - }); - - it("extracts TSX React component-like symbols", () => { - const content = ` -const MyButton: React.FC = () => { return