run.ts

/**
 * AIR SDK Benchmark — Measures API response latency and computes
 * cost/speed comparisons against LLM DOM reasoning baselines.
 *
 * Run: npx tsx benchmark/run.ts
 * Requires: AIR_API_KEY env var or ~/.config/air/credentials.json
 */

import { resolveConfig, AIRHttpClient, CapabilityCache } from '../src/index';

// ---- Configuration ----

const DOMAINS = [
  'amazon.com',
  'youtube.com',
  'github.com',
  'linkedin.com',
  'kayak.com',
  'reddit.com',
  'twitter.com',
  'yelp.com',
];

const RUNS_PER_DOMAIN = 5;

// LLM baseline: published pricing for frontier models (Opus-class)
// Token estimates based on typical browser agent DOM reasoning sessions
const LLM_BASELINE = {
  inputTokensPerAction: 4000,
  outputTokensPerAction: 800,
  roundTripsPerAction: 2.5,
  inputPricePerMToken: 15.0,
  outputPricePerMToken: 75.0,
  latencyPerRoundTripMs: 2000,

  get costPerAction() {
    const inputCost = (this.inputTokensPerAction * this.roundTripsPerAction * this.inputPricePerMToken) / 1_000_000;
    const outputCost = (this.outputTokensPerAction * this.roundTripsPerAction * this.outputPricePerMToken) / 1_000_000;
    return inputCost + outputCost;
  },
  get latencyPerActionMs() {
    return this.latencyPerRoundTripMs * this.roundTripsPerAction;
  },
};

const AIR_SDK_COST = {
  free: 0,
  pro: 0.00196,
  scale: 0.000596,
};

// ---- Stats ----

interface Stats {
  mean: number;
  median: number;
  p5: number;
  p95: number;
  stdDev: number;
  min: number;
  max: number;
  n: number;
}

function computeStats(values: number[]): Stats {
  const sorted = [...values].sort((a, b) => a - b);
  const n = sorted.length;
  const mean = sorted.reduce((a, b) => a + b, 0) / n;
  const variance = sorted.reduce((sum, v) => sum + (v - mean) ** 2, 0) / (n - 1);
  return {
    mean,
    median: sorted[Math.floor(n / 2)],
    p5: sorted[Math.floor(n * 0.05)],
    p95: sorted[Math.floor(n * 0.95)],
    stdDev: Math.sqrt(variance),
    min: sorted[0],
    max: sorted[n - 1],
    n,
  };
}

// ---- Runner ----

interface BenchmarkResult {
  browseLatencies: number[];
  executeLatencies: number[];
  totalLatencies: number[];
  capabilitiesFound: number[];
  errors: string[];
}

async function runBenchmark(): Promise<BenchmarkResult> {
  const apiKey = process.env.AIR_API_KEY;
  if (!apiKey) {
    const fs = await import('fs');
    const os = await import('os');
    const path = await import('path');
    const credPath = path.join(os.homedir(), '.config', 'air', 'credentials.json');
    if (fs.existsSync(credPath)) {
      const creds = JSON.parse(fs.readFileSync(credPath, 'utf-8'));
      const key = creds.apiKey || creds.api_key;
      if (key) process.env.AIR_API_KEY = key;
    }
  }

  const resolvedKey = process.env.AIR_API_KEY;
  if (!resolvedKey) {
    throw new Error('AIR_API_KEY not set. Run: npx @arcede/air-sdk init');
  }

  const config = resolveConfig({ apiKey: resolvedKey, cacheEnabled: false });
  const httpClient = new AIRHttpClient(config);
  const cache = new CapabilityCache(config, httpClient);

  const result: BenchmarkResult = {
    browseLatencies: [],
    executeLatencies: [],
    totalLatencies: [],
    capabilitiesFound: [],
    errors: [],
  };

  const totalOps = DOMAINS.length * RUNS_PER_DOMAIN;
  let completed = 0;

  for (let run = 0; run < RUNS_PER_DOMAIN; run++) {
    for (const domain of DOMAINS) {
      completed++;
      try {
        cache.clear();
        const browseStart = performance.now();
        const capabilities = await cache.getCapabilities(domain);
        const browseMs = performance.now() - browseStart;

        cache.clear();
        const execStart = performance.now();
        await cache.getCapabilities(domain);
        const execMs = performance.now() - execStart;

        result.browseLatencies.push(browseMs);
        result.executeLatencies.push(execMs);
        result.totalLatencies.push(browseMs + execMs);
        result.capabilitiesFound.push(capabilities.length);

        process.stdout.write(
          `\r  [${completed}/${totalOps}] ${domain} run ${run + 1}: ` +
          `browse=${browseMs.toFixed(0)}ms exec=${execMs.toFixed(0)}ms caps=${capabilities.length}    `
        );
      } catch (err: any) {
        result.errors.push(`${domain} run ${run + 1}: ${err.message}`);
        process.stdout.write(`\r  [${completed}/${totalOps}] ${domain} run ${run + 1}: ERROR    `);
      }
      await new Promise(r => setTimeout(r, 200));
    }
  }

  console.log('\n');
  return result;
}

// ---- Report ----

function formatReport(result: BenchmarkResult): string {
  const browseStats = computeStats(result.browseLatencies);
  const executeStats = computeStats(result.executeLatencies);
  const totalStats = computeStats(result.totalLatencies);
  const capStats = computeStats(result.capabilitiesFound);

  const llmCost = LLM_BASELINE.costPerAction;
  const llmLatency = LLM_BASELINE.latencyPerActionMs;

  const lines: string[] = [];
  lines.push('');
  lines.push('  AIR SDK BENCHMARK RESULTS');
  lines.push('  ════════════════════════════════════════════════');
  lines.push('');
  lines.push(`  Domains tested:     ${DOMAINS.length}`);
  lines.push(`  Runs per domain:    ${RUNS_PER_DOMAIN}`);
  lines.push(`  Total measurements: ${result.browseLatencies.length}`);
  lines.push(`  Errors:             ${result.errors.length}`);
  lines.push('');
  lines.push('  API Latency');
  lines.push('  ────────────────────────────────────────────────');
  lines.push(`  browse_capabilities   median=${browseStats.median.toFixed(0)}ms  p95=${browseStats.p95.toFixed(0)}ms  min=${browseStats.min.toFixed(0)}ms`);
  lines.push(`  execute_capability    median=${executeStats.median.toFixed(0)}ms  p95=${executeStats.p95.toFixed(0)}ms  min=${executeStats.min.toFixed(0)}ms`);
  lines.push(`  combined              median=${totalStats.median.toFixed(0)}ms  p95=${totalStats.p95.toFixed(0)}ms  min=${totalStats.min.toFixed(0)}ms`);
  lines.push('');
  lines.push(`  Capabilities discovered: ${capStats.mean.toFixed(0)} avg/domain (${capStats.min}–${capStats.max})`);
  lines.push('');
  lines.push('  Comparison: AIR SDK vs LLM DOM Reasoning');
  lines.push('  ────────────────────────────────────────────────');
  lines.push(`  LLM baseline (1 action):   $${llmCost.toFixed(2)}/action  ${(llmLatency / 1000).toFixed(1)}s latency`);
  lines.push(`  AIR SDK (Scale):           $${AIR_SDK_COST.scale.toFixed(4)}/action  ${totalStats.median.toFixed(0)}ms latency`);
  lines.push('');
  lines.push(`  Single-action speedup:      ${(llmLatency / totalStats.median).toFixed(0)}x`);
  lines.push(`  Single-action cost reduction: ${(llmCost / AIR_SDK_COST.scale).toFixed(0)}x`);
  lines.push('');
  lines.push(`  10-action workflow:`);
  lines.push(`    LLM: ~$${(llmCost * 4 * 10 / 10).toFixed(2)} (context grows)  ~${((llmLatency * 2.5 * 10) / 1000 / 10).toFixed(0)}s reasoning`);
  lines.push(`    AIR: $${AIR_SDK_COST.scale.toFixed(4)} (1 API call)  ${totalStats.median.toFixed(0)}ms`);
  lines.push('');
  lines.push(`  Date: ${new Date().toISOString().split('T')[0]}`);
  lines.push('');

  if (result.errors.length > 0) {
    lines.push('  Errors:');
    for (const err of result.errors.slice(0, 5)) lines.push(`    ${err}`);
    lines.push('');
  }

  return lines.join('\n');
}

// ---- Main ----

async function main() {
  console.log('\n  AIR SDK Benchmark');
  console.log(`  Testing ${DOMAINS.length} domains × ${RUNS_PER_DOMAIN} runs = ${DOMAINS.length * RUNS_PER_DOMAIN} measurements\n`);

  const result = await runBenchmark();
  const report = formatReport(result);
  console.log(report);

  const fs = await import('fs');
  const outputPath = new URL('./results.txt', import.meta.url).pathname;
  fs.writeFileSync(outputPath, report);
  console.log(`  Results saved to: ${outputPath}\n`);
}

main().catch(err => {
  console.error('Benchmark failed:', err);
  process.exit(1);
});