From 9c892bbad8ee315f5ad17ce000882bf3f4e25bf0 Mon Sep 17 00:00:00 2001 From: Steve Worley Date: Thu, 13 Nov 2025 17:57:21 +1000 Subject: [PATCH] Add Crawler and project commands. --- .gitignore | 4 +- src/commands/crawler.ts | 245 ++++++++++++++++++++++++++++++++++++++++ src/commands/project.ts | 150 ++++++++++++++++++++++++ src/index.ts | 12 +- src/types/auth.ts | 1 + src/utils/api.ts | 77 ++++++++++++- 6 files changed, 485 insertions(+), 4 deletions(-) create mode 100644 src/commands/crawler.ts diff --git a/.gitignore b/.gitignore index 25fbe92..fa8953e 100644 --- a/.gitignore +++ b/.gitignore @@ -77,4 +77,6 @@ logs .Spotlight-V100 .Trashes ehthumbs.db -Thumbs.db \ No newline at end of file +Thumbs.db + +vrt-results/ \ No newline at end of file diff --git a/src/commands/crawler.ts b/src/commands/crawler.ts new file mode 100644 index 0000000..959ac81 --- /dev/null +++ b/src/commands/crawler.ts @@ -0,0 +1,245 @@ +import { Command } from 'commander'; +import chalk from 'chalk'; +import inquirer from 'inquirer'; +import { createSpinner } from '../utils/spinner.js'; +import { ApiClient } from '../utils/api.js'; +import { Logger } from '../utils/logger.js'; +import { getActivePlatformConfig } from '../utils/config.js'; + +const logger = new Logger('Crawler'); + +export function crawlerCommand(program: Command) { + const crawler = program.command('crawler').description('Manage Quant crawlers'); + + crawler.command('run') + .description('Run a crawler') + .argument('[crawlerId]', 'crawler ID to run') + .option('--project ', 'project name (if not using active project)') + .option('--urls ', 'specific URLs to crawl (space-separated)') + .option('--org ', 'override organization') + .option('--platform ', 'platform to use (override active platform)') + .action(async (crawlerId, options) => { + await handleCrawlerRun(crawlerId, options); + }); + + crawler.command('list') + .description('List crawlers for a project') + .option('--project ', 'project name (if not using active project)') + .option('--org ', 'override organization') + .option('--platform ', 'platform to use (override active platform)') + .action(async (options) => { + await handleCrawlerList(options); + }); + + return crawler; +} + +interface CrawlerOptions { + project?: string; + urls?: string[]; + org?: string; + platform?: string; +} + +async function handleCrawlerRun(crawlerId?: string, options?: CrawlerOptions) { + try { + const auth = await getActivePlatformConfig(); + + if (!auth || !auth.token) { + logger.info('Not authenticated. Run `quant-cloud login` to authenticate.'); + return; + } + + // Create API client + const client = await ApiClient.create({ + org: options?.org, + platform: options?.platform + }); + + // Get project name - prefer option, fallback to active project + let projectName = options?.project || auth.activeProject; + + if (!projectName) { + // Try to get from active project context or prompt user + const projects = await client.getProjects(); + + if (projects.length === 0) { + logger.error('No projects found.'); + return; + } + + if (projects.length === 1) { + projectName = projects[0].name || projects[0].machine_name; + logger.info(`Using project: ${chalk.cyan(projectName)}`); + } else { + const { selectedProject } = await inquirer.prompt([ + { + type: 'list', + name: 'selectedProject', + message: 'Select a project:', + choices: projects.map((p: any) => ({ + name: `${p.name || p.machine_name} ${p.url ? chalk.gray(`(${p.url})`) : ''}`, + value: p.name || p.machine_name + })) + } + ]); + projectName = selectedProject; + } + } + + // Ensure we have a project name at this point + if (!projectName) { + logger.error('Project name is required'); + return; + } + + // Get crawler ID if not provided + if (!crawlerId) { + const spinner = createSpinner('Fetching crawlers...'); + const crawlers = await client.getCrawlers(projectName); + spinner.succeed(`Found ${crawlers.length} crawler(s)`); + + if (crawlers.length === 0) { + logger.error(`No crawlers found for project '${projectName}'.`); + return; + } + + if (crawlers.length === 1) { + crawlerId = crawlers[0].uuid || crawlers[0].machine_name; + logger.info(`Using crawler: ${chalk.cyan(crawlerId)}`); + } else { + const { selectedCrawler } = await inquirer.prompt([ + { + type: 'list', + name: 'selectedCrawler', + message: 'Select a crawler:', + choices: crawlers.map((c: any) => ({ + name: `${c.name || c.machine_name} ${c.description ? chalk.gray(`(${c.description})`) : ''}`, + value: c.uuid || c.machine_name + })) + } + ]); + crawlerId = selectedCrawler; + } + } + + // Ensure we have crawlerId at this point + if (!crawlerId) { + logger.error('Crawler ID is required'); + return; + } + + // Run the crawler + const spinner = createSpinner(`Running crawler '${crawlerId}'...`); + const result = await client.runCrawler(projectName, crawlerId, options?.urls); + + if (result.run_id) { + spinner.succeed(`Crawler run started successfully`); + logger.info(`Run ID: ${chalk.green(result.run_id)}`); + logger.info(`Project: ${chalk.cyan(projectName)}`); + logger.info(`Crawler: ${chalk.cyan(crawlerId)}`); + + if (options?.urls && options.urls.length > 0) { + logger.info(`URLs: ${chalk.gray(options.urls.join(', '))}`); + } + } else { + spinner.succeed(`Crawler run started`); + logger.info(`Project: ${chalk.cyan(projectName)}`); + logger.info(`Crawler: ${chalk.cyan(crawlerId)}`); + } + + } catch (error: any) { + logger.error('Failed to run crawler:', error.message); + process.exit(1); + } +} + +async function handleCrawlerList(options?: CrawlerOptions) { + try { + const auth = await getActivePlatformConfig(); + + if (!auth || !auth.token) { + logger.info('Not authenticated. Run `quant-cloud login` to authenticate.'); + return; + } + + // Create API client + const client = await ApiClient.create({ + org: options?.org, + platform: options?.platform + }); + + // Get project name - prefer option, fallback to active project + let projectName = options?.project || auth.activeProject; + + if (!projectName) { + const projects = await client.getProjects(); + + if (projects.length === 0) { + logger.error('No projects found.'); + return; + } + + if (projects.length === 1) { + projectName = projects[0].name || projects[0].machine_name; + logger.info(`Using project: ${chalk.cyan(projectName)}`); + } else { + const { selectedProject } = await inquirer.prompt([ + { + type: 'list', + name: 'selectedProject', + message: 'Select a project:', + choices: projects.map((p: any) => ({ + name: `${p.name || p.machine_name} ${p.url ? chalk.gray(`(${p.url})`) : ''}`, + value: p.name || p.machine_name + })) + } + ]); + projectName = selectedProject; + } + } + + // Ensure we have a project name at this point + if (!projectName) { + logger.error('Project name is required'); + return; + } + + // Get crawlers + const spinner = createSpinner('Fetching crawlers...'); + const crawlers = await client.getCrawlers(projectName); + spinner.succeed(`Found ${crawlers.length} crawler(s) for project '${projectName}'`); + + if (crawlers.length === 0) { + logger.info(`No crawlers found for project '${projectName}'.`); + return; + } + + // Display crawlers + logger.info(`\n${chalk.bold(`Crawlers for project: ${chalk.cyan(projectName)}`)}`); + crawlers.forEach((crawler: any) => { + const uuid = crawler.uuid || crawler.machine_name; + const name = crawler.name || uuid; + const description = crawler.description ? chalk.gray(` - ${crawler.description}`) : ''; + + logger.info(` ${chalk.green('•')} ${chalk.cyan(uuid)} ${name !== uuid ? chalk.white(`(${name})`) : ''}${description}`); + + if (crawler.url) { + logger.info(` ${chalk.gray('URL:')} ${crawler.url}`); + } + + if (crawler.status) { + logger.info(` ${chalk.gray('Status:')} ${crawler.status}`); + } + + if (crawler.schedule) { + logger.info(` ${chalk.gray('Schedule:')} ${crawler.schedule}`); + } + }); + + } catch (error: any) { + logger.error('Failed to list crawlers:', error.message); + process.exit(1); + } +} + diff --git a/src/commands/project.ts b/src/commands/project.ts index 420eaad..5118005 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -1,8 +1,10 @@ import { Command } from 'commander'; import chalk from 'chalk'; +import inquirer from 'inquirer'; import { createSpinner } from '../utils/spinner.js'; import { ApiClient } from '../utils/api.js'; import { Logger } from '../utils/logger.js'; +import { getActivePlatformConfig, saveActivePlatformConfig } from '../utils/config.js'; const logger = new Logger('Project'); @@ -17,6 +19,21 @@ export function projectCommand(program: Command) { await handleProjectList(options); }); + project.command('select') + .description('Select active project') + .argument('[projectName]', 'project name to select') + .option('--org ', 'override organization') + .option('--platform ', 'platform to use (override active platform)') + .action(async (projectName, options) => { + await handleProjectSelect(projectName, options); + }); + + project.command('current') + .description('Show current active project') + .action(async () => { + await handleProjectCurrent(); + }); + return project; } @@ -88,3 +105,136 @@ async function handleProjectList(options: ProjectOptions) { } } +async function handleProjectSelect(projectName?: string, options?: ProjectOptions) { + try { + const auth = await getActivePlatformConfig(); + + if (!auth || !auth.token) { + logger.info('Not authenticated. Run `quant-cloud login` to authenticate.'); + return; + } + + const spinner = createSpinner('Loading projects...'); + const client = await ApiClient.create({ + org: options?.org, + platform: options?.platform + }); + + const projects = await client.getProjects({ organizationId: options?.org }); + spinner.succeed(`Found ${projects.length} project${projects.length !== 1 ? 's' : ''}`); + + if (projects.length === 0) { + logger.info('No projects found in this organization.'); + return; + } + + let targetProjectName = projectName; + + // If no project name provided, show interactive selection + if (!targetProjectName) { + if (projects.length === 1) { + targetProjectName = projects[0].machine_name || projects[0].name; + logger.info(`Only one project available: ${chalk.cyan(targetProjectName)}`); + } else { + const { selectedProjectName } = await inquirer.prompt([ + { + type: 'list', + name: 'selectedProjectName', + message: 'Select project:', + choices: projects.map((proj: any) => ({ + name: `${chalk.cyan(proj.machine_name || proj.name)} ${proj.name && proj.name !== proj.machine_name ? chalk.gray(`(${proj.name})`) : ''} ${proj.domain ? `- ${chalk.blue(proj.domain)}` : ''}`, + value: proj.machine_name || proj.name + })) + } + ]); + targetProjectName = selectedProjectName; + } + } + + // Validate the project exists + const targetProject = projects.find((p: any) => + p.machine_name === targetProjectName || p.name === targetProjectName + ); + + if (!targetProject) { + logger.error(`Project '${targetProjectName}' not found.`); + logger.info('Available projects:'); + projects.forEach((proj: any) => { + logger.info(` ${chalk.cyan(proj.machine_name || proj.name)}`); + }); + return; + } + + // Save the selected project + await saveActivePlatformConfig({ + activeProject: targetProject.machine_name || targetProject.name + }); + + const displayName = targetProject.name || targetProject.machine_name; + logger.info(`Selected project: ${chalk.green(displayName)}`); + + if (targetProject.domain) { + logger.info(`URL: ${chalk.blue(targetProject.domain)}`); + } + + } catch (error: any) { + logger.error('Failed to select project:', error.message); + } +} + +async function handleProjectCurrent() { + try { + const auth = await getActivePlatformConfig(); + + if (!auth || !auth.token) { + logger.info('Not authenticated. Run `quant-cloud login` to authenticate.'); + return; + } + + if (!auth.activeProject) { + logger.info('No active project set.'); + logger.info(`${chalk.gray('Use')} ${chalk.cyan('quant-cloud project select')} ${chalk.gray('to select a project')}`); + return; + } + + try { + // Get fresh project data from API + const spinner = createSpinner('Loading project data...'); + const client = await ApiClient.create(); + + const projects = await client.getProjects(); + spinner.succeed('Loaded project data'); + + const activeProject = projects.find((p: any) => + p.machine_name === auth.activeProject || p.name === auth.activeProject + ); + + if (!activeProject) { + logger.info('Active project not found in current organization.'); + logger.info(`${chalk.gray('Use')} ${chalk.cyan('quant-cloud project list')} ${chalk.gray('to see available projects')}`); + return; + } + + logger.info(`Active project: ${chalk.green(activeProject.name || activeProject.machine_name)}`); + logger.info(`Machine name: ${chalk.gray(activeProject.machine_name)}`); + + if (activeProject.domain || activeProject.url || activeProject.aws_cloudfront_domain_name) { + const domain = activeProject.domain || activeProject.url || activeProject.aws_cloudfront_domain_name; + logger.info(`URL: ${chalk.blue(domain)}`); + } + + if (activeProject.region) { + logger.info(`Region: ${chalk.gray(activeProject.region)}`); + } + + } catch (error) { + // Fallback to showing just stored data + logger.info(`Active project: ${chalk.green(auth.activeProject)}`); + logger.info(`${chalk.gray('Use')} ${chalk.cyan('quant-cloud project list')} ${chalk.gray('to refresh project data')}`); + } + + } catch (error: any) { + logger.error('Failed to get current project:', error.message); + } +} + diff --git a/src/index.ts b/src/index.ts index ec5fa7e..2f42ad7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,6 +15,7 @@ import { sshCommand } from './commands/ssh.js'; import { platformCommand } from './commands/platform.js'; import { backupCommand } from './commands/backup.js'; import { vrtCommand } from './commands/vrt.js'; +import { crawlerCommand } from './commands/crawler.js'; import { getActivePlatformConfig } from './utils/config.js'; import { readFileSync } from 'fs'; import { join, dirname } from 'path'; @@ -70,9 +71,15 @@ async function displayContext() { } if (auth.activeEnvironment) { - console.log(chalk.gray('└─ Environment: ') + chalk.yellow(auth.activeEnvironment)); + console.log(chalk.gray('├─ Environment: ') + chalk.yellow(auth.activeEnvironment)); } else { - console.log(chalk.gray('└─ Environment: ') + chalk.red('None selected')); + console.log(chalk.gray('├─ Environment: ') + chalk.red('None selected')); + } + + if (auth.activeProject) { + console.log(chalk.gray('└─ Project: ') + chalk.blue(auth.activeProject)); + } else { + console.log(chalk.gray('└─ Project: ') + chalk.red('None selected')); } console.log(); // Empty line before help @@ -115,6 +122,7 @@ async function main() { program.addCommand(sshCommand); program.addCommand(backupCommand(program)); vrtCommand(program); + crawlerCommand(program); // Global error handling process.on('uncaughtException', (error) => { diff --git a/src/types/auth.ts b/src/types/auth.ts index b609877..66ea60c 100644 --- a/src/types/auth.ts +++ b/src/types/auth.ts @@ -8,6 +8,7 @@ export interface AuthConfig { activeOrganization?: string; activeApplication?: string; activeEnvironment?: string; + activeProject?: string; } export interface PlatformInfo { diff --git a/src/utils/api.ts b/src/utils/api.ts index c265ed6..c3a88fe 100644 --- a/src/utils/api.ts +++ b/src/utils/api.ts @@ -1,7 +1,7 @@ import { getActivePlatformConfig } from './config.js'; import { resolveEffectiveContext, ContextOverrides, EffectiveContext } from './context.js'; import { Logger } from './logger.js'; -import { ApplicationsApi, EnvironmentsApi, SSHAccessApi, BackupManagementApi, ProjectsApi, Configuration } from '@quantcdn/quant-client'; +import { ApplicationsApi, EnvironmentsApi, SSHAccessApi, BackupManagementApi, ProjectsApi, CrawlersApi, Configuration } from '@quantcdn/quant-client'; const logger = new Logger('API'); @@ -17,6 +17,7 @@ export class ApiClient { public sshAccessApi: SSHAccessApi; public backupManagementApi: BackupManagementApi; private projectsApi: ProjectsApi; + public crawlersApi: CrawlersApi; public baseUrl: string; private defaultOrganizationId?: string; private defaultApplicationId?: string; @@ -48,6 +49,7 @@ export class ApiClient { this.sshAccessApi = new SSHAccessApi(config); this.backupManagementApi = new BackupManagementApi(config); this.projectsApi = new ProjectsApi(config); + this.crawlersApi = new CrawlersApi(config); this.baseUrl = baseUrl; this.token = token; @@ -241,4 +243,77 @@ export class ApiClient { } } } + + async getCrawlers(projectName: string): Promise { + const organizationId = this.defaultOrganizationId; + + if (!organizationId) { + throw new Error('Organization not found. Use quant-cloud org list to see available organizations'); + } + + try { + const response = await this.crawlersApi.crawlersList(organizationId, projectName); + return response.data; + } catch (error: any) { + if (error.statusCode === 404 || error.response?.status === 404) { + throw new Error(`Project '${projectName}' not found or has no crawlers.`); + } else if (error.statusCode === 403 || error.response?.status === 403) { + throw new Error(`Access denied to crawlers for project '${projectName}'.`); + } else if (error.statusCode === 401 || error.response?.status === 401) { + throw new Error('Authentication expired. Please run `qc login` to re-authenticate.'); + } else { + throw new Error(`Failed to fetch crawlers: ${error.message || 'Network error'}`); + } + } + } + + async runCrawler(projectName: string, crawlerId: string, urls?: string[]): Promise { + const organizationId = this.defaultOrganizationId; + + if (!organizationId) { + throw new Error('Organization not found. Use quant-cloud org list to see available organizations'); + } + + try { + // NOTE: Using direct fetch() call as workaround + // The crawlersRun method is available in quant-ts-client commit fba13989a1f84e87fc16b431f77bb4d83b5970fa + // but that commit has build issues preventing npm install. + // Once the SDK is properly published with the crawlersRun method, replace this with: + // const response = await this.crawlersApi.crawlersRun(organizationId, projectName, crawlerId, payload); + const payload = urls ? { urls } : {}; + const response = await fetch( + `${this.baseUrl}/api/v2/organizations/${organizationId}/projects/${projectName}/crawlers/${crawlerId}/run`, + { + method: 'POST', + headers: { + 'Authorization': `Bearer ${this.token}`, + 'Accept': 'application/json', + 'Content-Type': 'application/json' + }, + body: JSON.stringify(payload) + } + ); + + if (!response.ok) { + if (response.status === 404) { + throw new Error(`Crawler '${crawlerId}' not found in project '${projectName}'.`); + } else if (response.status === 403) { + throw new Error(`Access denied to run crawler '${crawlerId}'.`); + } else if (response.status === 401) { + throw new Error('Authentication expired. Please run `qc login` to re-authenticate.'); + } else { + const errorText = await response.text(); + throw new Error(`Failed to run crawler: ${response.status} ${errorText}`); + } + } + + return await response.json(); + } catch (error: any) { + if (error.message?.includes('Crawler') || error.message?.includes('Authentication')) { + throw error; // Re-throw our friendly errors + } else { + throw new Error(`Failed to run crawler: ${error.message || 'Network error'}`); + } + } + } } \ No newline at end of file