From 94c7ccbc6e87f88f20b3af528e54e784f0d0e821 Mon Sep 17 00:00:00 2001 From: Steve Worley Date: Tue, 11 Nov 2025 11:50:12 +1000 Subject: [PATCH 1/3] feat: Add visual regression testing tool. --- README.md | 61 +++++ package-lock.json | 96 ++++++- package.json | 7 +- src/commands/project.ts | 90 +++++++ src/commands/vrt.ts | 573 ++++++++++++++++++++++++++++++++++++++++ src/index.ts | 4 + src/utils/api.ts | 54 +++- src/utils/config.ts | 45 ++++ 8 files changed, 925 insertions(+), 5 deletions(-) create mode 100644 src/commands/project.ts create mode 100644 src/commands/vrt.ts diff --git a/README.md b/README.md index 27f02d9..266288a 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ Command-line interface for Quant Cloud Platform integration and management. - **Log Streaming** - Live log tailing with follow mode - **SSH Access** - Direct terminal access to cloud environments via AWS ECS Exec with interactive shells and one-shot commands - **Backup Management** - Create, list, download, and delete environment backups +- **Visual Regression Testing** - Automated visual comparison between Quant projects and remote URLs with Playwright - **Secure OAuth Authentication** - Browser-based login flow with PKCE security - **Non-Interactive Mode** - All commands support context overrides via CLI flags for automation @@ -116,6 +117,9 @@ qc env list # Still finds .quant.yml at project root - `qc app select [appId]` - Switch to application (auto-prompts for environment) - `qc app current` - Show current application context +### Projects +- `qc project list` - List Quant projects in current organization + ### Environments - `qc env list` - List environments in current application - `qc env select [envId]` - Switch to environment with searchable selection @@ -187,6 +191,63 @@ qc backup create --org=my-org --app=my-app --env=production --type=database --de qc backup download backup-123 --org=my-org --app=my-app --env=production --type=filesystem ``` +### Visual Regression Testing (VRT) + +Run automated visual regression testing to compare Quant projects against remote URLs using Playwright and Chromium. + +- `qc vrt` - Run VRT for all configured projects +- `qc vrt --project=project1,project2` - Run VRT for specific projects +- `qc vrt --threshold=0.05` - Set pixel difference threshold (0-1, default: 0.01) +- `qc vrt --max-pages=20` - Set maximum pages to crawl per project +- `qc vrt --max-depth=5` - Set maximum crawl depth +- `qc vrt --csv=report.csv` - Generate CSV report +- `qc vrt --output-dir=./screenshots` - Set screenshot output directory +- `qc vrt --quant-auth=user:pass` - Basic auth for Quant URLs +- `qc vrt --remote-auth=user:pass` - Basic auth for remote URLs + +**Configuration:** Create `~/.quant/vrt-config.json` with project mappings: + +```json +{ + "projects": { + "my-project": "https://example.com", + "another-project": "https://another-example.com" + }, + "threshold": 0.01, + "maxPages": 10, + "maxDepth": 3, + "quantAuth": "username:password", + "remoteAuth": "username:password" +} +``` + +**Note:** Auth credentials in config are optional. CLI flags (`--quant-auth`, `--remote-auth`) override config values. + +**Examples:** + +```bash +# Run VRT for all configured projects +qc vrt + +# Run for specific projects +qc vrt --project=my-project + +# Run with custom threshold and generate CSV +qc vrt --threshold=0.02 --csv=vrt-report.csv + +# Run with authentication +qc vrt --quant-auth=user:pass --remote-auth=user:pass + +# Run with custom limits +qc vrt --max-pages=50 --max-depth=5 --output-dir=./my-screenshots +``` + +**Output:** +- Console summary with pass/fail status per page +- Screenshot diffs saved to `./vrt-results/{project}/{date}/` +- Optional CSV report with detailed results +- Exit code 1 if any tests fail + ### Non-Interactive Mode All commands support context override flags for automation and CI/CD: diff --git a/package-lock.json b/package-lock.json index 5515bd6..9fe3ac2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@quantcdn/quant-cloud-cli", - "version": "0.2.1", + "version": "0.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@quantcdn/quant-cloud-cli", - "version": "0.2.1", + "version": "0.3.0", "license": "MIT", "dependencies": { "@quantcdn/quant-client": "^4.0.0", @@ -21,7 +21,10 @@ "inquirer-autocomplete-prompt": "^3.0.1", "js-yaml": "^4.1.0", "node-fetch": "^3.3.2", - "open": "^9.1.0" + "open": "^9.1.0", + "pixelmatch": "^6.0.0", + "playwright": "^1.40.0", + "pngjs": "^7.0.0" }, "bin": { "qc": "dist/index.js", @@ -36,6 +39,8 @@ "@types/jest": "^30.0.0", "@types/js-yaml": "^4.0.9", "@types/node": "^20.8.0", + "@types/pixelmatch": "^5.2.6", + "@types/pngjs": "^6.0.5", "@typescript-eslint/eslint-plugin": "^6.7.0", "@typescript-eslint/parser": "^6.7.0", "eslint": "^8.50.0", @@ -1953,6 +1958,26 @@ "undici-types": "~6.21.0" } }, + "node_modules/@types/pixelmatch": { + "version": "5.2.6", + "resolved": "https://registry.npmjs.org/@types/pixelmatch/-/pixelmatch-5.2.6.tgz", + "integrity": "sha512-wC83uexE5KGuUODn6zkm9gMzTwdY5L0chiK+VrKcDfEjzxh1uadlWTvOmAbCpnM9zx/Ww3f8uKlYQVnO/TrqVg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/pngjs": { + "version": "6.0.5", + "resolved": "https://registry.npmjs.org/@types/pngjs/-/pngjs-6.0.5.tgz", + "integrity": "sha512-0k5eKfrA83JOZPppLtS2C7OUtyNAl2wKNxfyYl9Q5g9lPkgBl/9hNyAu6HuEH2J4XmIv2znEpkDd0SaZVxW6iQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/qs": { "version": "6.14.0", "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.14.0.tgz", @@ -7011,6 +7036,18 @@ "node": ">= 6" } }, + "node_modules/pixelmatch": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-6.0.0.tgz", + "integrity": "sha512-FYpL4XiIWakTnIqLqvt3uN4L9B3TsuHIvhLILzTiJZMJUsGvmKNeL4H3b6I99LRyerK9W4IuOXw+N28AtRgK2g==", + "license": "ISC", + "dependencies": { + "pngjs": "^7.0.0" + }, + "bin": { + "pixelmatch": "bin/pixelmatch" + } + }, "node_modules/pkg-dir": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", @@ -7080,6 +7117,59 @@ "node": ">=8" } }, + "node_modules/playwright": { + "version": "1.56.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.56.1.tgz", + "integrity": "sha512-aFi5B0WovBHTEvpM3DzXTUaeN6eN0qWnTkKx4NQaH4Wvcmc153PdaY2UBdSYKaGYw+UyWXSVyxDUg5DoPEttjw==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.56.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.56.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.56.1.tgz", + "integrity": "sha512-hutraynyn31F+Bifme+Ps9Vq59hKuUCz7H1kDOcBs+2oGguKkWTU50bBWrtz34OUWmIwpBTWDxaRPXrIXkgvmQ==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/pngjs": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-7.0.0.tgz", + "integrity": "sha512-LKWqWJRhstyYo9pGvgor/ivk2w94eSjE3RGVuzLGlr3NmD8bf7RcYGze1mNdEHRP6TRP6rMuDHk5t44hnTRyow==", + "license": "MIT", + "engines": { + "node": ">=14.19.0" + } + }, "node_modules/prelude-ls": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", diff --git a/package.json b/package.json index 1a12fc1..1d08296 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,10 @@ "inquirer-autocomplete-prompt": "^3.0.1", "js-yaml": "^4.1.0", "node-fetch": "^3.3.2", - "open": "^9.1.0" + "open": "^9.1.0", + "pixelmatch": "^6.0.0", + "playwright": "^1.40.0", + "pngjs": "^7.0.0" }, "devDependencies": { "@types/express": "^4.17.21", @@ -68,6 +71,8 @@ "@types/jest": "^30.0.0", "@types/js-yaml": "^4.0.9", "@types/node": "^20.8.0", + "@types/pixelmatch": "^5.2.6", + "@types/pngjs": "^6.0.5", "@typescript-eslint/eslint-plugin": "^6.7.0", "@typescript-eslint/parser": "^6.7.0", "eslint": "^8.50.0", diff --git a/src/commands/project.ts b/src/commands/project.ts new file mode 100644 index 0000000..420eaad --- /dev/null +++ b/src/commands/project.ts @@ -0,0 +1,90 @@ +import { Command } from 'commander'; +import chalk from 'chalk'; +import { createSpinner } from '../utils/spinner.js'; +import { ApiClient } from '../utils/api.js'; +import { Logger } from '../utils/logger.js'; + +const logger = new Logger('Project'); + +export function projectCommand(program: Command) { + const project = program.command('project').description('Manage Quant projects'); + + project.command('list') + .description('List projects in the active organization') + .option('--org ', 'override organization') + .option('--platform ', 'platform to use (override active platform)') + .action(async (options) => { + await handleProjectList(options); + }); + + return project; +} + +interface ProjectOptions { + org?: string; + platform?: string; +} + +async function handleProjectList(options: ProjectOptions) { + const spinner = createSpinner('Loading projects...'); + + try { + const client = await ApiClient.create({ + org: options.org, + platform: options.platform + }); + const projects = await client.getProjects({ organizationId: options.org }); + + spinner.succeed(`Found ${projects.length} project${projects.length !== 1 ? 's' : ''}`); + + if (projects.length === 0) { + logger.info('No projects found in this organization.'); + logger.info(`${chalk.gray('Create your first project in the dashboard')}`); + return; + } + + logger.info('\nProjects:'); + projects.forEach((project: any, index) => { + const name = project.name || project.machine_name; + const machineName = project.machine_name; + const domain = project.domain || project.url || project.aws_cloudfront_domain_name; + + // Compact format: machine_name (name) - domain + let displayLine = ` ${chalk.cyan(machineName)}`; + + if (name !== machineName) { + displayLine += ` ${chalk.gray(`(${name})`)}`; + } + + if (domain) { + displayLine += ` - ${chalk.blue(domain)}`; + } + + if (project.region) { + displayLine += ` ${chalk.gray(`[${project.region}]`)}`; + } + + logger.info(displayLine); + }); + + // Show context + const orgInfo = options.org ? ` (org: ${options.org})` : ' (active organization)'; + logger.info(`\n${chalk.gray('Showing projects for')}${orgInfo}`); + + } catch (error: any) { + spinner.fail('Failed to load projects'); + + if (error.message?.includes('Not authenticated')) { + logger.info('Not authenticated. Run `quant-cloud login` to authenticate.'); + } else if (error.message?.includes('404') || error.message?.includes('not found')) { + logger.error('Organization not found or no access to projects.'); + logger.info(`${chalk.gray('Use')} ${chalk.cyan('quant-cloud org list')} ${chalk.gray('to see available organizations')}`); + } else if (error.message?.includes('403') || error.message?.includes('forbidden')) { + logger.error('Access denied. You may not have permission to view projects in this organization.'); + } else { + logger.error('Error:', error.message); + logger.debug('Full error:', error); + } + } +} + diff --git a/src/commands/vrt.ts b/src/commands/vrt.ts new file mode 100644 index 0000000..f06137c --- /dev/null +++ b/src/commands/vrt.ts @@ -0,0 +1,573 @@ +import { Command } from 'commander'; +import chalk from 'chalk'; +import { chromium, Browser, Page } from 'playwright'; +import { PNG } from 'pngjs'; +import pixelmatch from 'pixelmatch'; +import { promises as fs } from 'fs'; +import { join } from 'path'; +import { createSpinner } from '../utils/spinner.js'; +import { ApiClient } from '../utils/api.js'; +import { Logger } from '../utils/logger.js'; +import { loadVRTConfig, VRTConfig } from '../utils/config.js'; + +const logger = new Logger('VRT'); + +interface VRTOptions { + project?: string; + threshold?: string; + maxPages?: string; + maxDepth?: string; + csv?: string; + quantAuth?: string; + remoteAuth?: string; + outputDir?: string; +} + +interface VRTResult { + project: string; + status: 'PASS' | 'FAIL' | 'ERROR'; + quantUrl: string; + remoteUrl: string; + differencePercentage: number; + screenshot: string; + timestamp: string; + errorMessage?: string; + pagePath?: string; +} + +interface CrawledPage { + url: string; + path: string; +} + +export function vrtCommand(program: Command) { + const vrt = program + .command('vrt') + .description('Run visual regression testing against Quant projects') + .option('--project ', 'specific project to test (comma-separated for multiple)') + .option('--threshold ', 'pixel difference threshold (0-1)', '0.01') + .option('--max-pages ', 'maximum pages to crawl per project', '10') + .option('--max-depth ', 'maximum crawl depth', '3') + .option('--csv ', 'output CSV report file') + .option('--quant-auth ', 'basic auth for Quant URLs (user:pass)') + .option('--remote-auth ', 'basic auth for remote URLs (user:pass)') + .option('--output-dir ', 'output directory for screenshots', './vrt-results') + .action(async (options: VRTOptions) => { + await handleVRT(options); + }); + + return vrt; +} + +async function handleVRT(options: VRTOptions) { + try { + // Load VRT configuration + const config = await loadVRTConfig(); + if (!config || Object.keys(config.projects).length === 0) { + logger.error('No VRT projects configured.'); + logger.info(`Configure projects by creating ${chalk.cyan('~/.quant/vrt-config.json')}`); + logger.info('Example format:'); + logger.info(JSON.stringify({ + projects: { + 'my-project': 'https://example.com' + }, + threshold: 0.01, + maxPages: 10, + maxDepth: 3 + }, null, 2)); + process.exit(1); + } + + // Determine which projects to test + const projectsToTest = options.project + ? options.project.split(',').map(p => p.trim()) + : Object.keys(config.projects); + + // Validate projects exist in config + const invalidProjects = projectsToTest.filter(p => !config.projects[p]); + if (invalidProjects.length > 0) { + logger.error(`Invalid projects: ${invalidProjects.join(', ')}`); + logger.info('Available projects:'); + Object.keys(config.projects).forEach(p => logger.info(` - ${chalk.cyan(p)}`)); + process.exit(1); + } + + // Parse options (CLI flags override config file) + const threshold = parseFloat(options.threshold || config.threshold?.toString() || '0.01'); + const maxPages = parseInt(options.maxPages || config.maxPages?.toString() || '10', 10); + const maxDepth = parseInt(options.maxDepth || config.maxDepth?.toString() || '3', 10); + const outputDir = options.outputDir || './vrt-results'; + const quantAuth = options.quantAuth || config.quantAuth; + const remoteAuth = options.remoteAuth || config.remoteAuth; + + logger.info(`Running VRT for ${projectsToTest.length} project(s)...`); + logger.info(`Threshold: ${(threshold * 100).toFixed(2)}%, Max Pages: ${maxPages}, Max Depth: ${maxDepth}`); + + // Create API client + const client = await ApiClient.create(); + + // Run VRT for each project + const allResults: VRTResult[] = []; + let browser: Browser | null = null; + + try { + browser = await chromium.launch({ headless: true }); + + for (const projectName of projectsToTest) { + const remoteUrl = config.projects[projectName]; + logger.info(`\n${chalk.bold(`Testing project: ${chalk.cyan(projectName)}`)}`); + + const projectResults = await runProjectVRT( + browser, + client, + projectName, + remoteUrl, + { + threshold, + maxPages, + maxDepth, + outputDir, + quantAuth, + remoteAuth + } + ); + + allResults.push(...projectResults); + } + } finally { + if (browser) { + await browser.close(); + } + } + + // Display summary + displaySummary(allResults); + + // Generate CSV if requested + if (options.csv) { + await generateCSV(allResults, options.csv); + logger.info(`\nCSV report saved to: ${chalk.cyan(options.csv)}`); + } + + // Exit with error if any test failed + const hasFailures = allResults.some(r => r.status === 'FAIL' || r.status === 'ERROR'); + if (hasFailures) { + process.exit(1); + } + + } catch (error: any) { + logger.error('VRT failed:', error.message); + process.exit(1); + } +} + +async function runProjectVRT( + browser: Browser, + client: ApiClient, + projectName: string, + remoteUrl: string, + options: { + threshold: number; + maxPages: number; + maxDepth: number; + outputDir: string; + quantAuth?: string; + remoteAuth?: string; + } +): Promise { + const results: VRTResult[] = []; + const spinner = createSpinner(`Fetching Quant project details for ${projectName}...`); + + try { + // Fetch Quant project details + const projectDetails = await client.getProjectDetails(projectName); + + // Try various possible domain field names from the API response + // For cloud applications: aws_cloudfront_domain_name + // For standard projects: domain, url, domains, primary_domain, etc. + const quantUrl = projectDetails.aws_cloudfront_domain_name || + projectDetails.domain || + projectDetails.url || + projectDetails.domains?.[0] || + projectDetails.primary_domain || + projectDetails.quantDomain; + + if (!quantUrl) { + spinner.fail('Failed to get Quant URL'); + logger.debug('Project details:', JSON.stringify(projectDetails, null, 2)); + results.push({ + project: projectName, + status: 'ERROR', + quantUrl: '', + remoteUrl, + differencePercentage: 0, + screenshot: '', + timestamp: new Date().toISOString(), + errorMessage: `Could not retrieve Quant project domain. Available fields: ${Object.keys(projectDetails).join(', ')}` + }); + return results; + } + + spinner.succeed(`Quant URL: ${chalk.blue(quantUrl)}`); + logger.info(`Remote URL: ${chalk.blue(remoteUrl)}`); + + // Ensure URLs have protocol + const normalizedQuantUrl = quantUrl.startsWith('http') ? quantUrl : `https://${quantUrl}`; + const normalizedRemoteUrl = remoteUrl.startsWith('http') ? remoteUrl : `https://${remoteUrl}`; + + // Crawl pages + const crawlSpinner = createSpinner('Crawling pages...'); + const pages = await crawlPages(browser, normalizedRemoteUrl, options.maxPages, options.maxDepth, options.remoteAuth); + crawlSpinner.succeed(`Found ${pages.length} pages to test`); + + // Test each page + for (const page of pages) { + const pageSpinner = createSpinner(`Testing ${page.path}...`); + + try { + const result = await comparePages( + browser, + normalizedQuantUrl + page.path, + normalizedRemoteUrl + page.path, + projectName, + page.path, + options.threshold, + options.outputDir, + options.quantAuth, + options.remoteAuth + ); + + results.push(result); + + if (result.status === 'PASS') { + pageSpinner.succeed(`${page.path} ${chalk.green('PASS')} (${result.differencePercentage.toFixed(2)}%)`); + } else if (result.status === 'FAIL') { + pageSpinner.fail(`${page.path} ${chalk.red('FAIL')} (${result.differencePercentage.toFixed(2)}%)`); + } else { + pageSpinner.fail(`${page.path} ${chalk.red('ERROR')}`); + } + } catch (error: any) { + pageSpinner.fail(`${page.path} ${chalk.red('ERROR')}`); + results.push({ + project: projectName, + status: 'ERROR', + quantUrl: normalizedQuantUrl + page.path, + remoteUrl: normalizedRemoteUrl + page.path, + differencePercentage: 0, + screenshot: '', + timestamp: new Date().toISOString(), + errorMessage: error.message, + pagePath: page.path + }); + } + } + + } catch (error: any) { + spinner.fail(`Failed to test project ${projectName}`); + logger.error(`Error: ${error.message}`); + results.push({ + project: projectName, + status: 'ERROR', + quantUrl: '', + remoteUrl, + differencePercentage: 0, + screenshot: '', + timestamp: new Date().toISOString(), + errorMessage: error.message + }); + } + + return results; +} + +async function crawlPages( + browser: Browser, + startUrl: string, + maxPages: number, + maxDepth: number, + auth?: string +): Promise { + const visited = new Set(); + const queue: Array<{ url: string; depth: number }> = [{ url: startUrl, depth: 0 }]; + const pages: CrawledPage[] = []; + const baseUrl = new URL(startUrl); + + const context = await browser.newContext({ + ...(auth && { httpCredentials: parseBasicAuth(auth) }) + }); + const page = await context.newPage(); + + try { + while (queue.length > 0 && pages.length < maxPages) { + const { url, depth } = queue.shift()!; + + if (visited.has(url) || depth > maxDepth) { + continue; + } + + visited.add(url); + + try { + await page.goto(url, { waitUntil: 'networkidle', timeout: 30000 }); + + const currentUrl = new URL(page.url()); + const path = currentUrl.pathname + currentUrl.search; + + pages.push({ url: page.url(), path }); + + // Only crawl deeper if we haven't reached max depth + if (depth < maxDepth && pages.length < maxPages) { + // Find all links on the page + const links = await page.$$eval('a[href]', (anchors, base) => { + return anchors + .map(a => { + try { + const href = a.getAttribute('href'); + if (!href) return null; + return new URL(href, base).href; + } catch { + return null; + } + }) + .filter(Boolean); + }, baseUrl.origin); + + // Queue links from the same domain + for (const link of links) { + try { + const linkUrl = new URL(link as string); + if (linkUrl.origin === baseUrl.origin && !visited.has(link as string)) { + queue.push({ url: link as string, depth: depth + 1 }); + } + } catch { + // Invalid URL, skip + } + } + } + } catch (error: any) { + // Skip pages that fail to load + logger.debug(`Failed to crawl ${url}: ${error.message}`); + } + } + } finally { + await page.close(); + await context.close(); + } + + return pages; +} + +async function comparePages( + browser: Browser, + quantUrl: string, + remoteUrl: string, + projectName: string, + pagePath: string, + threshold: number, + outputDir: string, + quantAuth?: string, + remoteAuth?: string +): Promise { + const timestamp = new Date().toISOString(); + const sanitizedPath = pagePath.replace(/[^a-zA-Z0-9]/g, '_'); + const projectDir = join(outputDir, projectName, timestamp.split('T')[0]); + + // Ensure output directory exists + await fs.mkdir(projectDir, { recursive: true }); + + // Create browser contexts with auth if provided + const quantContext = await browser.newContext({ + ...(quantAuth && { httpCredentials: parseBasicAuth(quantAuth) }) + }); + const remoteContext = await browser.newContext({ + ...(remoteAuth && { httpCredentials: parseBasicAuth(remoteAuth) }) + }); + + let quantPage: Page | null = null; + let remotePage: Page | null = null; + + try { + // Capture screenshots + quantPage = await quantContext.newPage(); + remotePage = await remoteContext.newPage(); + + await Promise.all([ + quantPage.goto(quantUrl, { waitUntil: 'networkidle', timeout: 30000 }), + remotePage.goto(remoteUrl, { waitUntil: 'networkidle', timeout: 30000 }) + ]); + + const [quantScreenshot, remoteScreenshot] = await Promise.all([ + quantPage.screenshot({ fullPage: true }), + remotePage.screenshot({ fullPage: true }) + ]); + + // Save screenshots + const quantScreenshotPath = join(projectDir, `quant_${sanitizedPath}.png`); + const remoteScreenshotPath = join(projectDir, `remote_${sanitizedPath}.png`); + + await Promise.all([ + fs.writeFile(quantScreenshotPath, quantScreenshot), + fs.writeFile(remoteScreenshotPath, remoteScreenshot) + ]); + + // Compare images + const img1 = PNG.sync.read(quantScreenshot); + const img2 = PNG.sync.read(remoteScreenshot); + + // Ensure images are the same size (resize if needed) + const width = Math.max(img1.width, img2.width); + const height = Math.max(img1.height, img2.height); + + const resizedImg1 = resizeImage(img1, width, height); + const resizedImg2 = resizeImage(img2, width, height); + + const diff = new PNG({ width, height }); + const pixelsDiff = pixelmatch( + resizedImg1.data, + resizedImg2.data, + diff.data, + width, + height, + { threshold: 0.1 } + ); + + const totalPixels = width * height; + const differencePercentage = (pixelsDiff / totalPixels) * 100; + + // Save diff image + const diffPath = join(projectDir, `diff_${sanitizedPath}.png`); + await fs.writeFile(diffPath, PNG.sync.write(diff)); + + const status = differencePercentage <= (threshold * 100) ? 'PASS' : 'FAIL'; + + return { + project: projectName, + status, + quantUrl, + remoteUrl, + differencePercentage, + screenshot: diffPath, + timestamp, + pagePath + }; + + } finally { + if (quantPage) await quantPage.close(); + if (remotePage) await remotePage.close(); + await quantContext.close(); + await remoteContext.close(); + } +} + +function resizeImage(img: PNG, width: number, height: number): PNG { + if (img.width === width && img.height === height) { + return img; + } + + const resized = new PNG({ width, height }); + + // Fill with white background + for (let i = 0; i < resized.data.length; i += 4) { + resized.data[i] = 255; // R + resized.data[i + 1] = 255; // G + resized.data[i + 2] = 255; // B + resized.data[i + 3] = 255; // A + } + + // Copy original image data + for (let y = 0; y < Math.min(img.height, height); y++) { + for (let x = 0; x < Math.min(img.width, width); x++) { + const srcIdx = (y * img.width + x) * 4; + const dstIdx = (y * width + x) * 4; + resized.data[dstIdx] = img.data[srcIdx]; + resized.data[dstIdx + 1] = img.data[srcIdx + 1]; + resized.data[dstIdx + 2] = img.data[srcIdx + 2]; + resized.data[dstIdx + 3] = img.data[srcIdx + 3]; + } + } + + return resized; +} + +function parseBasicAuth(auth: string): { username: string; password: string } { + const [username, password] = auth.split(':'); + return { username, password }; +} + +function displaySummary(results: VRTResult[]) { + logger.info('\n' + chalk.bold('=== VRT Summary ===')); + + const passed = results.filter(r => r.status === 'PASS').length; + const failed = results.filter(r => r.status === 'FAIL').length; + const errors = results.filter(r => r.status === 'ERROR').length; + const total = results.length; + + logger.info(`Total Tests: ${total}`); + logger.info(`${chalk.green('Passed')}: ${passed}`); + logger.info(`${chalk.red('Failed')}: ${failed}`); + logger.info(`${chalk.yellow('Errors')}: ${errors}`); + + // Group by project + const byProject = results.reduce((acc, r) => { + if (!acc[r.project]) acc[r.project] = []; + acc[r.project].push(r); + return acc; + }, {} as Record); + + logger.info('\n' + chalk.bold('Results by Project:')); + for (const [project, projectResults] of Object.entries(byProject)) { + const projectPassed = projectResults.filter(r => r.status === 'PASS').length; + const projectFailed = projectResults.filter(r => r.status === 'FAIL').length; + const projectErrors = projectResults.filter(r => r.status === 'ERROR').length; + + logger.info(`\n${chalk.cyan(project)}:`); + logger.info(` ${chalk.green('✓')} ${projectPassed} passed`); + if (projectFailed > 0) logger.info(` ${chalk.red('✗')} ${projectFailed} failed`); + if (projectErrors > 0) { + logger.info(` ${chalk.yellow('⚠')} ${projectErrors} errors`); + + // Show error messages + const errorResults = projectResults.filter(r => r.status === 'ERROR'); + errorResults.forEach(r => { + if (r.errorMessage) { + const location = r.pagePath ? ` (${r.pagePath})` : ''; + logger.info(` ${chalk.gray('→')} ${chalk.red(r.errorMessage)}${location}`); + } + }); + } + } +} + +async function generateCSV(results: VRTResult[], filename: string): Promise { + const headers = [ + 'project', + 'status', + 'quant_url', + 'remote_url', + 'difference_percentage', + 'screenshot', + 'timestamp', + 'error_message', + 'page_path' + ]; + + const rows = results.map(r => [ + r.project, + r.status, + r.quantUrl, + r.remoteUrl, + r.differencePercentage.toFixed(4), + r.screenshot, + r.timestamp, + r.errorMessage || '', + r.pagePath || '/' + ]); + + const csv = [ + headers.join(','), + ...rows.map(row => row.map(cell => `"${cell}"`).join(',')) + ].join('\n'); + + await fs.writeFile(filename, csv, 'utf-8'); +} + diff --git a/src/index.ts b/src/index.ts index 9bd618d..ec5fa7e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,10 +9,12 @@ import { logoutCommand } from './commands/logout.js'; import { whoamiCommand } from './commands/whoami.js'; import { orgCommand } from './commands/org.js'; import { appCommand } from './commands/app.js'; +import { projectCommand } from './commands/project.js'; import { envCommand } from './commands/env.js'; import { sshCommand } from './commands/ssh.js'; import { platformCommand } from './commands/platform.js'; import { backupCommand } from './commands/backup.js'; +import { vrtCommand } from './commands/vrt.js'; import { getActivePlatformConfig } from './utils/config.js'; import { readFileSync } from 'fs'; import { join, dirname } from 'path'; @@ -107,10 +109,12 @@ async function main() { whoamiCommand(program); orgCommand(program); appCommand(program); + projectCommand(program); envCommand(program); platformCommand(program); program.addCommand(sshCommand); program.addCommand(backupCommand(program)); + vrtCommand(program); // Global error handling process.on('uncaughtException', (error) => { diff --git a/src/utils/api.ts b/src/utils/api.ts index 51cd078..c265ed6 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -1,7 +1,7 @@ import { getActivePlatformConfig } from './config.js'; import { resolveEffectiveContext, ContextOverrides, EffectiveContext } from './context.js'; import { Logger } from './logger.js'; -import { ApplicationsApi, EnvironmentsApi, SSHAccessApi, BackupManagementApi, Configuration } from '@quantcdn/quant-client'; +import { ApplicationsApi, EnvironmentsApi, SSHAccessApi, BackupManagementApi, ProjectsApi, Configuration } from '@quantcdn/quant-client'; const logger = new Logger('API'); @@ -16,6 +16,7 @@ export class ApiClient { public environmentsApi: EnvironmentsApi; public sshAccessApi: SSHAccessApi; public backupManagementApi: BackupManagementApi; + private projectsApi: ProjectsApi; public baseUrl: string; private defaultOrganizationId?: string; private defaultApplicationId?: string; @@ -46,6 +47,7 @@ export class ApiClient { this.environmentsApi = new EnvironmentsApi(config); this.sshAccessApi = new SSHAccessApi(config); this.backupManagementApi = new BackupManagementApi(config); + this.projectsApi = new ProjectsApi(config); this.baseUrl = baseUrl; this.token = token; @@ -189,4 +191,54 @@ export class ApiClient { } } } + + async getProjects(apiOptions?: ApiOptions): Promise { + const organizationId = apiOptions?.organizationId || this.defaultOrganizationId; + + if (!organizationId) { + throw new Error('Organization not found or no access to projects.\nUse quant-cloud org list to see available organizations'); + } + + try { + const response = await this.projectsApi.projectsList(organizationId); + return response.data; + } catch (error: any) { + // Provide friendly error messages + if (error.statusCode === 404 || error.response?.status === 404) { + throw new Error(`Organization '${organizationId}' not found or no access to projects.`); + } else if (error.statusCode === 403 || error.response?.status === 403) { + throw new Error(`Access denied to projects in organization '${organizationId}'.`); + } else if (error.statusCode === 401 || error.response?.status === 401) { + throw new Error('Authentication expired. Please run `qc login` to re-authenticate.'); + } else { + throw new Error(`Failed to fetch projects: ${error.message || 'Network error'}`); + } + } + } + + async getProjectDetails(projectName: string): Promise { + const organizationId = this.defaultOrganizationId; + + if (!organizationId) { + throw new Error('Organization not found. Use quant-cloud org list to see available organizations'); + } + + try { + // Fetch project details - for standard Quant projects, this will return the project domain + // withToken=false means we don't need the project token in the response + const response = await this.projectsApi.projectsRead(organizationId, projectName, false); + return response.data; + } catch (error: any) { + // Provide friendly error messages + if (error.statusCode === 404 || error.response?.status === 404) { + throw new Error(`Project '${projectName}' not found in organization '${organizationId}'.`); + } else if (error.statusCode === 403 || error.response?.status === 403) { + throw new Error(`Access denied to project '${projectName}'.`); + } else if (error.statusCode === 401 || error.response?.status === 401) { + throw new Error('Authentication expired. Please run `qc login` to re-authenticate.'); + } else { + throw new Error(`Failed to fetch project details: ${error.message || 'Network error'}`); + } + } + } } \ No newline at end of file diff --git a/src/utils/config.ts b/src/utils/config.ts index 1f2c751..4bba834 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -5,6 +5,20 @@ import { AuthConfig, MultiPlatformConfig, PlatformInfo } from '../types/auth.js' const CONFIG_DIR = join(homedir(), '.quant'); export const CONFIG_FILE = join(CONFIG_DIR, 'credentials'); +export const VRT_CONFIG_FILE = join(CONFIG_DIR, 'vrt-config.json'); + +export interface VRTProjectMapping { + [projectMachineName: string]: string; // machine name -> remote URL +} + +export interface VRTConfig { + projects: VRTProjectMapping; + threshold?: number; // Default threshold (0-1) + maxPages?: number; // Max pages to crawl per project + maxDepth?: number; // Max crawl depth + quantAuth?: string; // Basic auth for Quant URLs (username:password) + remoteAuth?: string; // Basic auth for remote URLs (username:password) +} export async function ensureConfigDir(): Promise { try { @@ -210,4 +224,35 @@ export async function saveActivePlatformConfig(updates: Partial): Pr // Backward compatibility - update existing functions to use active platform export async function loadAuthConfigCompat(): Promise { return await getActivePlatformConfig(); +} + +// VRT configuration management +export async function loadVRTConfig(): Promise { + try { + const data = await fs.readFile(VRT_CONFIG_FILE, 'utf-8'); + return JSON.parse(data); + } catch { + return null; + } +} + +export async function saveVRTConfig(config: VRTConfig): Promise { + await ensureConfigDir(); + await fs.writeFile(VRT_CONFIG_FILE, JSON.stringify(config, null, 2), 'utf-8'); +} + +export async function addVRTProject(machineName: string, remoteUrl: string): Promise { + const config = await loadVRTConfig() || { projects: {} }; + config.projects[machineName] = remoteUrl; + await saveVRTConfig(config); +} + +export async function removeVRTProject(machineName: string): Promise { + const config = await loadVRTConfig(); + if (!config || !config.projects[machineName]) { + return false; + } + delete config.projects[machineName]; + await saveVRTConfig(config); + return true; } \ No newline at end of file From 7cee10489389882743ed3eca2ad4a672529fb99c Mon Sep 17 00:00:00 2001 From: Steve Worley Date: Tue, 11 Nov 2025 12:23:32 +1000 Subject: [PATCH 2/3] fix: load max-pages from config. --- src/commands/vrt.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/commands/vrt.ts b/src/commands/vrt.ts index f06137c..0aa5957 100644 --- a/src/commands/vrt.ts +++ b/src/commands/vrt.ts @@ -45,13 +45,13 @@ export function vrtCommand(program: Command) { .command('vrt') .description('Run visual regression testing against Quant projects') .option('--project ', 'specific project to test (comma-separated for multiple)') - .option('--threshold ', 'pixel difference threshold (0-1)', '0.01') - .option('--max-pages ', 'maximum pages to crawl per project', '10') - .option('--max-depth ', 'maximum crawl depth', '3') + .option('--threshold ', 'pixel difference threshold (0-1)') + .option('--max-pages ', 'maximum pages to crawl per project') + .option('--max-depth ', 'maximum crawl depth') .option('--csv ', 'output CSV report file') .option('--quant-auth ', 'basic auth for Quant URLs (user:pass)') .option('--remote-auth ', 'basic auth for remote URLs (user:pass)') - .option('--output-dir ', 'output directory for screenshots', './vrt-results') + .option('--output-dir ', 'output directory for screenshots') .action(async (options: VRTOptions) => { await handleVRT(options); }); @@ -92,10 +92,10 @@ async function handleVRT(options: VRTOptions) { process.exit(1); } - // Parse options (CLI flags override config file) + // Parse options (CLI flags override config file) const threshold = parseFloat(options.threshold || config.threshold?.toString() || '0.01'); - const maxPages = parseInt(options.maxPages || config.maxPages?.toString() || '10', 10); - const maxDepth = parseInt(options.maxDepth || config.maxDepth?.toString() || '3', 10); + const maxPages = parseInt(options.maxPages || (config.maxPages !== undefined ? config.maxPages.toString() : '10'), 10); + const maxDepth = parseInt(options.maxDepth || (config.maxDepth !== undefined ? config.maxDepth.toString() : '3'), 10); const outputDir = options.outputDir || './vrt-results'; const quantAuth = options.quantAuth || config.quantAuth; const remoteAuth = options.remoteAuth || config.remoteAuth; From c5c24ea6097929615a0a74c727ed9bfbddf83b97 Mon Sep 17 00:00:00 2001 From: Steve Worley Date: Tue, 11 Nov 2025 13:23:56 +1000 Subject: [PATCH 3/3] feat: support per configuration authentication. --- README.md | 17 ++++++++++++----- src/commands/vrt.ts | 23 ++++++++++++++++++++--- src/utils/config.ts | 12 +++++++++--- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 266288a..4516f94 100644 --- a/README.md +++ b/README.md @@ -210,18 +210,25 @@ Run automated visual regression testing to compare Quant projects against remote ```json { "projects": { - "my-project": "https://example.com", - "another-project": "https://another-example.com" + "simple-project": "https://example.com", + "project-with-auth": { + "url": "https://example2.com", + "remoteAuth": "user:pass", + "quantAuth": "user:pass" + } }, "threshold": 0.01, "maxPages": 10, "maxDepth": 3, - "quantAuth": "username:password", - "remoteAuth": "username:password" + "quantAuth": "default-user:default-pass", + "remoteAuth": "default-user:default-pass" } ``` -**Note:** Auth credentials in config are optional. CLI flags (`--quant-auth`, `--remote-auth`) override config values. +**Note:** +- Projects can be defined as simple URLs (strings) or objects with per-project auth +- Global `quantAuth`/`remoteAuth` apply to all projects unless overridden at project level +- CLI flags (`--quant-auth`, `--remote-auth`) override all config values **Examples:** diff --git a/src/commands/vrt.ts b/src/commands/vrt.ts index 0aa5957..3157e0a 100644 --- a/src/commands/vrt.ts +++ b/src/commands/vrt.ts @@ -114,7 +114,24 @@ async function handleVRT(options: VRTOptions) { browser = await chromium.launch({ headless: true }); for (const projectName of projectsToTest) { - const remoteUrl = config.projects[projectName]; + const projectConfig = config.projects[projectName]; + + // Support both simple string and complex object format + let remoteUrl: string; + let projectQuantAuth: string | undefined; + let projectRemoteAuth: string | undefined; + + if (typeof projectConfig === 'string') { + remoteUrl = projectConfig; + projectQuantAuth = quantAuth; + projectRemoteAuth = remoteAuth; + } else { + remoteUrl = projectConfig.url; + // Project-specific auth overrides global auth + projectQuantAuth = projectConfig.quantAuth || quantAuth; + projectRemoteAuth = projectConfig.remoteAuth || remoteAuth; + } + logger.info(`\n${chalk.bold(`Testing project: ${chalk.cyan(projectName)}`)}`); const projectResults = await runProjectVRT( @@ -127,8 +144,8 @@ async function handleVRT(options: VRTOptions) { maxPages, maxDepth, outputDir, - quantAuth, - remoteAuth + quantAuth: projectQuantAuth, + remoteAuth: projectRemoteAuth } ); diff --git a/src/utils/config.ts b/src/utils/config.ts index 4bba834..4833699 100644 --- a/src/utils/config.ts +++ b/src/utils/config.ts @@ -7,8 +7,14 @@ const CONFIG_DIR = join(homedir(), '.quant'); export const CONFIG_FILE = join(CONFIG_DIR, 'credentials'); export const VRT_CONFIG_FILE = join(CONFIG_DIR, 'vrt-config.json'); +export interface VRTProjectConfig { + url: string; + remoteAuth?: string; + quantAuth?: string; +} + export interface VRTProjectMapping { - [projectMachineName: string]: string; // machine name -> remote URL + [projectMachineName: string]: string | VRTProjectConfig; // machine name -> remote URL or config object } export interface VRTConfig { @@ -16,8 +22,8 @@ export interface VRTConfig { threshold?: number; // Default threshold (0-1) maxPages?: number; // Max pages to crawl per project maxDepth?: number; // Max crawl depth - quantAuth?: string; // Basic auth for Quant URLs (username:password) - remoteAuth?: string; // Basic auth for remote URLs (username:password) + quantAuth?: string; // Default basic auth for Quant URLs (username:password) + remoteAuth?: string; // Default basic auth for remote URLs (username:password) } export async function ensureConfigDir(): Promise {