diff --git a/.env.benchmark.example b/.env.benchmark.example new file mode 100644 index 000000000..2e25f1db7 --- /dev/null +++ b/.env.benchmark.example @@ -0,0 +1,4 @@ +BENCHMARK_TARGET_URL= +BENCHMARK_REQUESTS=5 +BENCHMARK_CONCURRENCY=2 +BENCHMARK_TOKEN= diff --git a/.github/workflows/api-benchmark.yml b/.github/workflows/api-benchmark.yml new file mode 100644 index 000000000..7252a71bc --- /dev/null +++ b/.github/workflows/api-benchmark.yml @@ -0,0 +1,22 @@ +name: API benchmark smoke + +on: + pull_request: + paths: + - "apps/api/**" + - "benchmarks/**" + - "package.json" + - "package-lock.json" + - ".github/workflows/api-benchmark.yml" + +jobs: + smoke: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + - run: npm ci + - run: npm run benchmark -- --smoke --output-dir=benchmarks/results/ci diff --git a/.gitignore b/.gitignore index 1e3ce10dd..e02a5dc48 100644 --- a/.gitignore +++ b/.gitignore @@ -3,5 +3,6 @@ node_modules dist .env .env.* +!.env.benchmark.example coverage *.log diff --git a/apps/api/package.json b/apps/api/package.json index 25fa0e90e..17c3c1a6e 100644 --- a/apps/api/package.json +++ b/apps/api/package.json @@ -5,7 +5,7 @@ "scripts": { "dev": "node src/server.js", "start": "node src/server.js", - "test": "node --test src/tests" + "test": "node --test \"src/tests/**/*.test.js\"" }, "dependencies": { "cors": "^2.8.5", diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..365614cd2 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,41 @@ +# API Benchmarks + +This benchmark suite exercises every route mounted under `/api/` and captures: + +- p50, p95, and p99 latency +- p50, p95, and p99 time to first byte +- peak and sustained requests per second +- status-code distribution +- error rate + +## Run Locally + +```bash +npm run benchmark +``` + +By default the command starts the API in-process on a random local port and writes both JSON and Markdown reports to `benchmarks/results/`. + +To benchmark an existing local or staging server: + +```bash +BENCHMARK_TARGET_URL=http://127.0.0.1:4000 npm run benchmark +``` + +## Configuration + +Create `.env.benchmark` from `.env.benchmark.example` if you want to override the target URL, request count, concurrency, or benchmark token. + +The admin benchmark route uses `BENCHMARK_TOKEN` when provided. If no token is configured, the benchmark script creates a short-lived local token with the API's development JWT secret. + +## CI Smoke Gate + +The GitHub Actions workflow runs: + +```bash +npm run benchmark -- --smoke --output-dir=benchmarks/results/ci +``` + +Smoke mode uses low concurrency and a small request count so CI can catch severe regressions without adding a heavy load test to every pull request. + +Thresholds live in `benchmarks/thresholds.json`. diff --git a/benchmarks/results/local-baseline/benchmark-baseline-2026-05-17T21-04-58-800Z.json b/benchmarks/results/local-baseline/benchmark-baseline-2026-05-17T21-04-58-800Z.json new file mode 100644 index 000000000..97c888a9a --- /dev/null +++ b/benchmarks/results/local-baseline/benchmark-baseline-2026-05-17T21-04-58-800Z.json @@ -0,0 +1,451 @@ +{ + "startedAt": "2026-05-17T21:04:58.800Z", + "completedAt": "2026-05-17T21:04:58.902Z", + "targetUrl": "local in-process server", + "mode": "baseline", + "requestsPerEndpoint": 5, + "concurrency": 2, + "endpointCount": 20, + "results": [ + { + "name": "auth_register", + "method": "POST", + "path": "/api/auth/register", + "requests": 5, + "peakRps": 157.06, + "sustainedRps": 157.06, + "errorRate": 0, + "statusCodes": { + "201": 5 + }, + "latencyMs": { + "p50": 3.88, + "p95": 23.48, + "p99": 23.48 + }, + "ttfbMs": { + "p50": 3.8, + "p95": 23.34, + "p99": 23.34 + } + }, + { + "name": "auth_login", + "method": "POST", + "path": "/api/auth/login", + "requests": 5, + "peakRps": 631.5, + "sustainedRps": 631.5, + "errorRate": 0, + "statusCodes": { + "200": 5 + }, + "latencyMs": { + "p50": 2.92, + "p95": 3.48, + "p99": 3.48 + }, + "ttfbMs": { + "p50": 2.9, + "p95": 3.46, + "p99": 3.46 + } + }, + { + "name": "auth_oauth_callback", + "method": "GET", + "path": "/api/auth/oauth/github/callback?code=benchmark-code", + "requests": 5, + "peakRps": 1243.32, + "sustainedRps": 1243.32, + "errorRate": 0, + "statusCodes": { + "200": 5 + }, + "latencyMs": { + "p50": 1.15, + "p95": 2.19, + "p99": 2.19 + }, + "ttfbMs": { + "p50": 1.13, + "p95": 2.17, + "p99": 2.17 + } + }, + { + "name": "auth_refresh", + "method": "POST", + "path": "/api/auth/refresh", + "requests": 5, + "peakRps": 1000.48, + "sustainedRps": 1000.48, + "errorRate": 0, + "statusCodes": { + "200": 5 + }, + "latencyMs": { + "p50": 1.91, + "p95": 2.06, + "p99": 2.06 + }, + "ttfbMs": { + "p50": 1.9, + "p95": 2.05, + "p99": 2.05 + } + }, + { + "name": "users_list", + "method": "GET", + "path": "/api/users", + "requests": 5, + "peakRps": 2049.85, + "sustainedRps": 2049.85, + "errorRate": 0, + "statusCodes": { + "200": 5 + }, + "latencyMs": { + "p50": 0.87, + "p95": 1.05, + "p99": 1.05 + }, + "ttfbMs": { + "p50": 0.85, + "p95": 1.03, + "p99": 1.03 + } + }, + { + "name": "users_create", + "method": "POST", + "path": "/api/users", + "requests": 5, + "peakRps": 1759.76, + "sustainedRps": 1759.76, + "errorRate": 0, + "statusCodes": { + "201": 5 + }, + "latencyMs": { + "p50": 0.98, + "p95": 1.12, + "p99": 1.12 + }, + "ttfbMs": { + "p50": 0.97, + "p95": 1.1, + "p99": 1.1 + } + }, + { + "name": "jobs_list", + "method": "GET", + "path": "/api/jobs", + "requests": 5, + "peakRps": 2078.31, + "sustainedRps": 2078.31, + "errorRate": 0, + "statusCodes": { + "200": 5 + }, + "latencyMs": { + "p50": 0.88, + "p95": 1.03, + "p99": 1.03 + }, + "ttfbMs": { + "p50": 0.87, + "p95": 1.02, + "p99": 1.02 + } + }, + { + "name": "jobs_create", + "method": "POST", + "path": "/api/jobs", + "requests": 5, + "peakRps": 1464.6, + "sustainedRps": 1464.6, + "errorRate": 0, + "statusCodes": { + "201": 5 + }, + "latencyMs": { + "p50": 1.24, + "p95": 1.5, + "p99": 1.5 + }, + "ttfbMs": { + "p50": 1.23, + "p95": 1.49, + "p99": 1.49 + } + }, + { + "name": "proposals_list", + "method": "GET", + "path": "/api/proposals", + "requests": 5, + "peakRps": 1312.4, + "sustainedRps": 1312.4, + "errorRate": 0, + "statusCodes": { + "200": 5 + }, + "latencyMs": { + "p50": 1.39, + "p95": 2.17, + "p99": 2.17 + }, + "ttfbMs": { + "p50": 1.38, + "p95": 2.16, + "p99": 2.16 + } + }, + { + "name": "proposals_create", + "method": "POST", + "path": "/api/proposals", + "requests": 5, + "peakRps": 1790.57, + "sustainedRps": 1790.57, + "errorRate": 0, + "statusCodes": { + "201": 5 + }, + "latencyMs": { + "p50": 0.99, + "p95": 1.15, + "p99": 1.15 + }, + "ttfbMs": { + "p50": 0.98, + "p95": 1.14, + "p99": 1.14 + } + }, + { + "name": "payments_create", + "method": "POST", + "path": "/api/payments", + "requests": 5, + "peakRps": 1806.62, + "sustainedRps": 1806.62, + "errorRate": 0, + "statusCodes": { + "201": 5 + }, + "latencyMs": { + "p50": 0.97, + "p95": 1.18, + "p99": 1.18 + }, + "ttfbMs": { + "p50": 0.95, + "p95": 1.17, + "p99": 1.17 + } + }, + { + "name": "reviews_list", + "method": "GET", + "path": "/api/reviews", + "requests": 5, + "peakRps": 2153.87, + "sustainedRps": 2153.87, + "errorRate": 0, + "statusCodes": { + "200": 5 + }, + "latencyMs": { + "p50": 0.82, + "p95": 1, + "p99": 1 + }, + "ttfbMs": { + "p50": 0.81, + "p95": 0.99, + "p99": 0.99 + } + }, + { + "name": "reviews_create", + "method": "POST", + "path": "/api/reviews", + "requests": 5, + "peakRps": 1775.19, + "sustainedRps": 1775.19, + "errorRate": 0, + "statusCodes": { + "201": 5 + }, + "latencyMs": { + "p50": 1.04, + "p95": 1.13, + "p99": 1.13 + }, + "ttfbMs": { + "p50": 1.03, + "p95": 1.12, + "p99": 1.12 + } + }, + { + "name": "messages_list", + "method": "GET", + "path": "/api/messages", + "requests": 5, + "peakRps": 2084.2, + "sustainedRps": 2084.2, + "errorRate": 0, + "statusCodes": { + "200": 5 + }, + "latencyMs": { + "p50": 0.82, + "p95": 1.01, + "p99": 1.01 + }, + "ttfbMs": { + "p50": 0.81, + "p95": 1, + "p99": 1 + } + }, + { + "name": "messages_create", + "method": "POST", + "path": "/api/messages", + "requests": 5, + "peakRps": 1813.37, + "sustainedRps": 1813.37, + "errorRate": 0, + "statusCodes": { + "201": 5 + }, + "latencyMs": { + "p50": 1, + "p95": 1.15, + "p99": 1.15 + }, + "ttfbMs": { + "p50": 0.98, + "p95": 1.14, + "p99": 1.14 + } + }, + { + "name": "notifications_list", + "method": "GET", + "path": "/api/notifications", + "requests": 5, + "peakRps": 2077.27, + "sustainedRps": 2077.27, + "errorRate": 0, + "statusCodes": { + "200": 5 + }, + "latencyMs": { + "p50": 0.87, + "p95": 1.01, + "p99": 1.01 + }, + "ttfbMs": { + "p50": 0.86, + "p95": 1, + "p99": 1 + } + }, + { + "name": "notifications_create", + "method": "POST", + "path": "/api/notifications", + "requests": 5, + "peakRps": 1745.57, + "sustainedRps": 1745.57, + "errorRate": 0, + "statusCodes": { + "201": 5 + }, + "latencyMs": { + "p50": 1.04, + "p95": 1.19, + "p99": 1.19 + }, + "ttfbMs": { + "p50": 1.02, + "p95": 1.18, + "p99": 1.18 + } + }, + { + "name": "uploads_create", + "method": "POST", + "path": "/api/uploads", + "requests": 5, + "peakRps": 603.82, + "sustainedRps": 603.82, + "errorRate": 0, + "statusCodes": { + "201": 5 + }, + "latencyMs": { + "p50": 1.89, + "p95": 5.18, + "p99": 5.18 + }, + "ttfbMs": { + "p50": 1.88, + "p95": 5.16, + "p99": 5.16 + } + }, + { + "name": "search", + "method": "GET", + "path": "/api/search?q=developer&category=automation", + "requests": 5, + "peakRps": 1753.83, + "sustainedRps": 1753.83, + "errorRate": 0, + "statusCodes": { + "200": 5 + }, + "latencyMs": { + "p50": 0.97, + "p95": 1.34, + "p99": 1.34 + }, + "ttfbMs": { + "p50": 0.96, + "p95": 1.32, + "p99": 1.32 + } + }, + { + "name": "admin_metrics", + "method": "GET", + "path": "/api/admin/metrics", + "requests": 5, + "peakRps": 898.1, + "sustainedRps": 898.1, + "errorRate": 0, + "statusCodes": { + "200": 5 + }, + "latencyMs": { + "p50": 1.84, + "p95": 2.69, + "p99": 2.69 + }, + "ttfbMs": { + "p50": 1.82, + "p95": 2.68, + "p99": 2.68 + } + } + ] +} diff --git a/benchmarks/results/local-baseline/benchmark-baseline-2026-05-17T21-04-58-800Z.md b/benchmarks/results/local-baseline/benchmark-baseline-2026-05-17T21-04-58-800Z.md new file mode 100644 index 000000000..577b906e8 --- /dev/null +++ b/benchmarks/results/local-baseline/benchmark-baseline-2026-05-17T21-04-58-800Z.md @@ -0,0 +1,32 @@ +# API Benchmark Summary + +- Target: local in-process server +- Mode: baseline +- Started: 2026-05-17T21:04:58.800Z +- Completed: 2026-05-17T21:04:58.902Z +- Endpoints: 20 +- Requests per endpoint: 5 +- Concurrency: 2 + +| Endpoint | Route | Requests | p50 ms | p95 ms | p99 ms | TTFB p95 ms | Sustained RPS | Error rate | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| auth_register | POST /api/auth/register | 5 | 3.88 | 23.48 | 23.48 | 23.34 | 157.06 | 0% | +| auth_login | POST /api/auth/login | 5 | 2.92 | 3.48 | 3.48 | 3.46 | 631.5 | 0% | +| auth_oauth_callback | GET /api/auth/oauth/github/callback?code=benchmark-code | 5 | 1.15 | 2.19 | 2.19 | 2.17 | 1243.32 | 0% | +| auth_refresh | POST /api/auth/refresh | 5 | 1.91 | 2.06 | 2.06 | 2.05 | 1000.48 | 0% | +| users_list | GET /api/users | 5 | 0.87 | 1.05 | 1.05 | 1.03 | 2049.85 | 0% | +| users_create | POST /api/users | 5 | 0.98 | 1.12 | 1.12 | 1.1 | 1759.76 | 0% | +| jobs_list | GET /api/jobs | 5 | 0.88 | 1.03 | 1.03 | 1.02 | 2078.31 | 0% | +| jobs_create | POST /api/jobs | 5 | 1.24 | 1.5 | 1.5 | 1.49 | 1464.6 | 0% | +| proposals_list | GET /api/proposals | 5 | 1.39 | 2.17 | 2.17 | 2.16 | 1312.4 | 0% | +| proposals_create | POST /api/proposals | 5 | 0.99 | 1.15 | 1.15 | 1.14 | 1790.57 | 0% | +| payments_create | POST /api/payments | 5 | 0.97 | 1.18 | 1.18 | 1.17 | 1806.62 | 0% | +| reviews_list | GET /api/reviews | 5 | 0.82 | 1 | 1 | 0.99 | 2153.87 | 0% | +| reviews_create | POST /api/reviews | 5 | 1.04 | 1.13 | 1.13 | 1.12 | 1775.19 | 0% | +| messages_list | GET /api/messages | 5 | 0.82 | 1.01 | 1.01 | 1 | 2084.2 | 0% | +| messages_create | POST /api/messages | 5 | 1 | 1.15 | 1.15 | 1.14 | 1813.37 | 0% | +| notifications_list | GET /api/notifications | 5 | 0.87 | 1.01 | 1.01 | 1 | 2077.27 | 0% | +| notifications_create | POST /api/notifications | 5 | 1.04 | 1.19 | 1.19 | 1.18 | 1745.57 | 0% | +| uploads_create | POST /api/uploads | 5 | 1.89 | 5.18 | 5.18 | 5.16 | 603.82 | 0% | +| search | GET /api/search?q=developer&category=automation | 5 | 0.97 | 1.34 | 1.34 | 1.32 | 1753.83 | 0% | +| admin_metrics | GET /api/admin/metrics | 5 | 1.84 | 2.69 | 2.69 | 2.68 | 898.1 | 0% | diff --git a/benchmarks/results/local-smoke/benchmark-smoke-2026-05-17T21-03-57-146Z.json b/benchmarks/results/local-smoke/benchmark-smoke-2026-05-17T21-03-57-146Z.json new file mode 100644 index 000000000..deafe9b42 --- /dev/null +++ b/benchmarks/results/local-smoke/benchmark-smoke-2026-05-17T21-03-57-146Z.json @@ -0,0 +1,451 @@ +{ + "startedAt": "2026-05-17T21:03:57.146Z", + "completedAt": "2026-05-17T21:03:57.213Z", + "targetUrl": "local in-process server", + "mode": "smoke", + "requestsPerEndpoint": 2, + "concurrency": 1, + "endpointCount": 20, + "results": [ + { + "name": "auth_register", + "method": "POST", + "path": "/api/auth/register", + "requests": 2, + "peakRps": 79.6, + "sustainedRps": 79.6, + "errorRate": 0, + "statusCodes": { + "201": 2 + }, + "latencyMs": { + "p50": 2.05, + "p95": 22.2, + "p99": 22.2 + }, + "ttfbMs": { + "p50": 2.02, + "p95": 22, + "p99": 22 + } + }, + { + "name": "auth_login", + "method": "POST", + "path": "/api/auth/login", + "requests": 2, + "peakRps": 465.7, + "sustainedRps": 465.7, + "errorRate": 0, + "statusCodes": { + "200": 2 + }, + "latencyMs": { + "p50": 1.69, + "p95": 2.41, + "p99": 2.41 + }, + "ttfbMs": { + "p50": 1.63, + "p95": 2.38, + "p99": 2.38 + } + }, + { + "name": "auth_oauth_callback", + "method": "GET", + "path": "/api/auth/oauth/github/callback?code=benchmark-code", + "requests": 2, + "peakRps": 390.32, + "sustainedRps": 390.32, + "errorRate": 0, + "statusCodes": { + "200": 2 + }, + "latencyMs": { + "p50": 0.89, + "p95": 4.09, + "p99": 4.09 + }, + "ttfbMs": { + "p50": 0.86, + "p95": 4.06, + "p99": 4.06 + } + }, + { + "name": "auth_refresh", + "method": "POST", + "path": "/api/auth/refresh", + "requests": 2, + "peakRps": 701.66, + "sustainedRps": 701.66, + "errorRate": 0, + "statusCodes": { + "200": 2 + }, + "latencyMs": { + "p50": 1.31, + "p95": 1.33, + "p99": 1.33 + }, + "ttfbMs": { + "p50": 1.29, + "p95": 1.31, + "p99": 1.31 + } + }, + { + "name": "users_list", + "method": "GET", + "path": "/api/users", + "requests": 2, + "peakRps": 806.39, + "sustainedRps": 806.39, + "errorRate": 0, + "statusCodes": { + "200": 2 + }, + "latencyMs": { + "p50": 0.82, + "p95": 1.43, + "p99": 1.43 + }, + "ttfbMs": { + "p50": 0.79, + "p95": 1.4, + "p99": 1.4 + } + }, + { + "name": "users_create", + "method": "POST", + "path": "/api/users", + "requests": 2, + "peakRps": 1256.91, + "sustainedRps": 1256.91, + "errorRate": 0, + "statusCodes": { + "201": 2 + }, + "latencyMs": { + "p50": 0.59, + "p95": 0.83, + "p99": 0.83 + }, + "ttfbMs": { + "p50": 0.57, + "p95": 0.81, + "p99": 0.81 + } + }, + { + "name": "jobs_list", + "method": "GET", + "path": "/api/jobs", + "requests": 2, + "peakRps": 1617.6, + "sustainedRps": 1617.6, + "errorRate": 0, + "statusCodes": { + "200": 2 + }, + "latencyMs": { + "p50": 0.54, + "p95": 0.6, + "p99": 0.6 + }, + "ttfbMs": { + "p50": 0.53, + "p95": 0.58, + "p99": 0.58 + } + }, + { + "name": "jobs_create", + "method": "POST", + "path": "/api/jobs", + "requests": 2, + "peakRps": 951.16, + "sustainedRps": 951.16, + "errorRate": 0, + "statusCodes": { + "201": 2 + }, + "latencyMs": { + "p50": 0.75, + "p95": 1.2, + "p99": 1.2 + }, + "ttfbMs": { + "p50": 0.73, + "p95": 1.18, + "p99": 1.18 + } + }, + { + "name": "proposals_list", + "method": "GET", + "path": "/api/proposals", + "requests": 2, + "peakRps": 1664.72, + "sustainedRps": 1664.72, + "errorRate": 0, + "statusCodes": { + "200": 2 + }, + "latencyMs": { + "p50": 0.51, + "p95": 0.59, + "p99": 0.59 + }, + "ttfbMs": { + "p50": 0.5, + "p95": 0.55, + "p99": 0.55 + } + }, + { + "name": "proposals_create", + "method": "POST", + "path": "/api/proposals", + "requests": 2, + "peakRps": 1483.13, + "sustainedRps": 1483.13, + "errorRate": 0, + "statusCodes": { + "201": 2 + }, + "latencyMs": { + "p50": 0.56, + "p95": 0.64, + "p99": 0.64 + }, + "ttfbMs": { + "p50": 0.55, + "p95": 0.62, + "p99": 0.62 + } + }, + { + "name": "payments_create", + "method": "POST", + "path": "/api/payments", + "requests": 2, + "peakRps": 1466.6, + "sustainedRps": 1466.6, + "errorRate": 0, + "statusCodes": { + "201": 2 + }, + "latencyMs": { + "p50": 0.58, + "p95": 0.62, + "p99": 0.62 + }, + "ttfbMs": { + "p50": 0.57, + "p95": 0.6, + "p99": 0.6 + } + }, + { + "name": "reviews_list", + "method": "GET", + "path": "/api/reviews", + "requests": 2, + "peakRps": 1786.83, + "sustainedRps": 1786.83, + "errorRate": 0, + "statusCodes": { + "200": 2 + }, + "latencyMs": { + "p50": 0.48, + "p95": 0.54, + "p99": 0.54 + }, + "ttfbMs": { + "p50": 0.47, + "p95": 0.52, + "p99": 0.52 + } + }, + { + "name": "reviews_create", + "method": "POST", + "path": "/api/reviews", + "requests": 2, + "peakRps": 1552.31, + "sustainedRps": 1552.31, + "errorRate": 0, + "statusCodes": { + "201": 2 + }, + "latencyMs": { + "p50": 0.53, + "p95": 0.59, + "p99": 0.59 + }, + "ttfbMs": { + "p50": 0.52, + "p95": 0.57, + "p99": 0.57 + } + }, + { + "name": "messages_list", + "method": "GET", + "path": "/api/messages", + "requests": 2, + "peakRps": 1746.72, + "sustainedRps": 1746.72, + "errorRate": 0, + "statusCodes": { + "200": 2 + }, + "latencyMs": { + "p50": 0.51, + "p95": 0.54, + "p99": 0.54 + }, + "ttfbMs": { + "p50": 0.5, + "p95": 0.52, + "p99": 0.52 + } + }, + { + "name": "messages_create", + "method": "POST", + "path": "/api/messages", + "requests": 2, + "peakRps": 1523.81, + "sustainedRps": 1523.81, + "errorRate": 0, + "statusCodes": { + "201": 2 + }, + "latencyMs": { + "p50": 0.54, + "p95": 0.62, + "p99": 0.62 + }, + "ttfbMs": { + "p50": 0.52, + "p95": 0.61, + "p99": 0.61 + } + }, + { + "name": "notifications_list", + "method": "GET", + "path": "/api/notifications", + "requests": 2, + "peakRps": 1585.67, + "sustainedRps": 1585.67, + "errorRate": 0, + "statusCodes": { + "200": 2 + }, + "latencyMs": { + "p50": 0.54, + "p95": 0.6, + "p99": 0.6 + }, + "ttfbMs": { + "p50": 0.5, + "p95": 0.52, + "p99": 0.52 + } + }, + { + "name": "notifications_create", + "method": "POST", + "path": "/api/notifications", + "requests": 2, + "peakRps": 1565.31, + "sustainedRps": 1565.31, + "errorRate": 0, + "statusCodes": { + "201": 2 + }, + "latencyMs": { + "p50": 0.49, + "p95": 0.63, + "p99": 0.63 + }, + "ttfbMs": { + "p50": 0.48, + "p95": 0.61, + "p99": 0.61 + } + }, + { + "name": "uploads_create", + "method": "POST", + "path": "/api/uploads", + "requests": 2, + "peakRps": 372.73, + "sustainedRps": 372.73, + "errorRate": 0, + "statusCodes": { + "201": 2 + }, + "latencyMs": { + "p50": 0.95, + "p95": 4.14, + "p99": 4.14 + }, + "ttfbMs": { + "p50": 0.94, + "p95": 4.12, + "p99": 4.12 + } + }, + { + "name": "search", + "method": "GET", + "path": "/api/search?q=developer&category=automation", + "requests": 2, + "peakRps": 1487.43, + "sustainedRps": 1487.43, + "errorRate": 0, + "statusCodes": { + "200": 2 + }, + "latencyMs": { + "p50": 0.61, + "p95": 0.63, + "p99": 0.63 + }, + "ttfbMs": { + "p50": 0.6, + "p95": 0.61, + "p99": 0.61 + } + }, + { + "name": "admin_metrics", + "method": "GET", + "path": "/api/admin/metrics", + "requests": 2, + "peakRps": 576.54, + "sustainedRps": 576.54, + "errorRate": 0, + "statusCodes": { + "200": 2 + }, + "latencyMs": { + "p50": 1.63, + "p95": 1.69, + "p99": 1.69 + }, + "ttfbMs": { + "p50": 1.62, + "p95": 1.67, + "p99": 1.67 + } + } + ] +} diff --git a/benchmarks/results/local-smoke/benchmark-smoke-2026-05-17T21-03-57-146Z.md b/benchmarks/results/local-smoke/benchmark-smoke-2026-05-17T21-03-57-146Z.md new file mode 100644 index 000000000..20af0b6d3 --- /dev/null +++ b/benchmarks/results/local-smoke/benchmark-smoke-2026-05-17T21-03-57-146Z.md @@ -0,0 +1,32 @@ +# API Benchmark Summary + +- Target: local in-process server +- Mode: smoke +- Started: 2026-05-17T21:03:57.146Z +- Completed: 2026-05-17T21:03:57.213Z +- Endpoints: 20 +- Requests per endpoint: 2 +- Concurrency: 1 + +| Endpoint | Route | Requests | p50 ms | p95 ms | p99 ms | TTFB p95 ms | Sustained RPS | Error rate | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| auth_register | POST /api/auth/register | 2 | 2.05 | 22.2 | 22.2 | 22 | 79.6 | 0% | +| auth_login | POST /api/auth/login | 2 | 1.69 | 2.41 | 2.41 | 2.38 | 465.7 | 0% | +| auth_oauth_callback | GET /api/auth/oauth/github/callback?code=benchmark-code | 2 | 0.89 | 4.09 | 4.09 | 4.06 | 390.32 | 0% | +| auth_refresh | POST /api/auth/refresh | 2 | 1.31 | 1.33 | 1.33 | 1.31 | 701.66 | 0% | +| users_list | GET /api/users | 2 | 0.82 | 1.43 | 1.43 | 1.4 | 806.39 | 0% | +| users_create | POST /api/users | 2 | 0.59 | 0.83 | 0.83 | 0.81 | 1256.91 | 0% | +| jobs_list | GET /api/jobs | 2 | 0.54 | 0.6 | 0.6 | 0.58 | 1617.6 | 0% | +| jobs_create | POST /api/jobs | 2 | 0.75 | 1.2 | 1.2 | 1.18 | 951.16 | 0% | +| proposals_list | GET /api/proposals | 2 | 0.51 | 0.59 | 0.59 | 0.55 | 1664.72 | 0% | +| proposals_create | POST /api/proposals | 2 | 0.56 | 0.64 | 0.64 | 0.62 | 1483.13 | 0% | +| payments_create | POST /api/payments | 2 | 0.58 | 0.62 | 0.62 | 0.6 | 1466.6 | 0% | +| reviews_list | GET /api/reviews | 2 | 0.48 | 0.54 | 0.54 | 0.52 | 1786.83 | 0% | +| reviews_create | POST /api/reviews | 2 | 0.53 | 0.59 | 0.59 | 0.57 | 1552.31 | 0% | +| messages_list | GET /api/messages | 2 | 0.51 | 0.54 | 0.54 | 0.52 | 1746.72 | 0% | +| messages_create | POST /api/messages | 2 | 0.54 | 0.62 | 0.62 | 0.61 | 1523.81 | 0% | +| notifications_list | GET /api/notifications | 2 | 0.54 | 0.6 | 0.6 | 0.52 | 1585.67 | 0% | +| notifications_create | POST /api/notifications | 2 | 0.49 | 0.63 | 0.63 | 0.61 | 1565.31 | 0% | +| uploads_create | POST /api/uploads | 2 | 0.95 | 4.14 | 4.14 | 4.12 | 372.73 | 0% | +| search | GET /api/search?q=developer&category=automation | 2 | 0.61 | 0.63 | 0.63 | 0.61 | 1487.43 | 0% | +| admin_metrics | GET /api/admin/metrics | 2 | 1.63 | 1.69 | 1.69 | 1.67 | 576.54 | 0% | diff --git a/benchmarks/run-benchmark.js b/benchmarks/run-benchmark.js new file mode 100644 index 000000000..a7c7af95c --- /dev/null +++ b/benchmarks/run-benchmark.js @@ -0,0 +1,417 @@ +#!/usr/bin/env node + +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import http from "node:http"; +import https from "node:https"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { createApp } from "../apps/api/src/app.js"; +import { signAccessToken } from "../apps/api/src/utils/jwt.js"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const repoRoot = resolve(__dirname, ".."); + +const args = new Set(process.argv.slice(2)); +const smoke = args.has("--smoke"); +const outputDir = getArgValue("--output-dir") ?? resolve(repoRoot, "benchmarks", "results"); + +await loadBenchmarkEnv(); + +const targetUrl = process.env.BENCHMARK_TARGET_URL; +const requestsPerEndpoint = Number(process.env.BENCHMARK_REQUESTS ?? (smoke ? 2 : 5)); +const concurrency = Number(process.env.BENCHMARK_CONCURRENCY ?? (smoke ? 1 : 2)); +const benchmarkToken = + process.env.BENCHMARK_TOKEN ?? + signAccessToken({ sub: "benchmark-user", role: "admin", purpose: "benchmark" }); + +const endpoints = [ + { name: "auth_register", method: "POST", path: "/api/auth/register", json: () => userPayload("register") }, + { name: "auth_login", method: "POST", path: "/api/auth/login", json: () => userPayload("login") }, + { name: "auth_oauth_callback", method: "GET", path: "/api/auth/oauth/github/callback?code=benchmark-code" }, + { name: "auth_refresh", method: "POST", path: "/api/auth/refresh", json: () => ({}) }, + { name: "users_list", method: "GET", path: "/api/users" }, + { name: "users_create", method: "POST", path: "/api/users", json: () => ({ email: uniqueEmail("user"), role: "client" }) }, + { name: "jobs_list", method: "GET", path: "/api/jobs" }, + { name: "jobs_create", method: "POST", path: "/api/jobs", json: jobPayload }, + { name: "proposals_list", method: "GET", path: "/api/proposals" }, + { name: "proposals_create", method: "POST", path: "/api/proposals", json: proposalPayload }, + { name: "payments_create", method: "POST", path: "/api/payments", json: paymentPayload }, + { name: "reviews_list", method: "GET", path: "/api/reviews" }, + { name: "reviews_create", method: "POST", path: "/api/reviews", json: reviewPayload }, + { name: "messages_list", method: "GET", path: "/api/messages" }, + { name: "messages_create", method: "POST", path: "/api/messages", json: messagePayload }, + { name: "notifications_list", method: "GET", path: "/api/notifications" }, + { name: "notifications_create", method: "POST", path: "/api/notifications", json: notificationPayload }, + { name: "uploads_create", method: "POST", path: "/api/uploads", multipart: uploadPayload }, + { name: "search", method: "GET", path: "/api/search?q=developer&category=automation" }, + { + name: "admin_metrics", + method: "GET", + path: "/api/admin/metrics", + headers: () => ({ authorization: `Bearer ${benchmarkToken}` }) + } +]; + +const thresholds = JSON.parse( + await readFile(resolve(repoRoot, "benchmarks", "thresholds.json"), "utf8") +); + +let server; +let baseUrl = targetUrl; + +if (!baseUrl) { + const app = createApp(); + server = app.listen(0); + await new Promise((resolvePromise, reject) => { + server.once("listening", resolvePromise); + server.once("error", reject); + }); + const { port } = server.address(); + baseUrl = `http://127.0.0.1:${port}`; +} + +try { + await mkdir(outputDir, { recursive: true }); + const startedAt = new Date(); + const results = []; + + for (const endpoint of endpoints) { + results.push(await benchmarkEndpoint(endpoint, baseUrl)); + } + + const summary = { + startedAt: startedAt.toISOString(), + completedAt: new Date().toISOString(), + targetUrl: targetUrl ? redactUrl(baseUrl) : "local in-process server", + mode: smoke ? "smoke" : "baseline", + requestsPerEndpoint, + concurrency, + endpointCount: endpoints.length, + results + }; + + const timestamp = startedAt.toISOString().replace(/[:.]/g, "-"); + const jsonPath = resolve(outputDir, `benchmark-${smoke ? "smoke" : "baseline"}-${timestamp}.json`); + const markdownPath = resolve(outputDir, `benchmark-${smoke ? "smoke" : "baseline"}-${timestamp}.md`); + + await writeFile(jsonPath, `${JSON.stringify(summary, null, 2)}\n`); + await writeFile(markdownPath, renderMarkdown(summary)); + + const failures = findThresholdFailures(results, thresholds); + console.log(`Wrote benchmark JSON: ${jsonPath}`); + console.log(`Wrote benchmark summary: ${markdownPath}`); + console.table(results.map(compactResult)); + + if (failures.length > 0) { + console.error("Benchmark threshold failures:"); + for (const failure of failures) { + console.error(`- ${failure}`); + } + process.exitCode = 1; + } +} finally { + if (server) { + await new Promise((resolvePromise, reject) => { + server.close((error) => (error ? reject(error) : resolvePromise())); + }); + } +} + +function getArgValue(name) { + const prefix = `${name}=`; + const match = process.argv.slice(2).find((arg) => arg.startsWith(prefix)); + return match ? match.slice(prefix.length) : undefined; +} + +async function loadBenchmarkEnv() { + try { + const envPath = resolve(repoRoot, ".env.benchmark"); + const content = await readFile(envPath, "utf8"); + for (const line of content.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#") || !trimmed.includes("=")) { + continue; + } + const [key, ...valueParts] = trimmed.split("="); + if (!process.env[key]) { + process.env[key] = valueParts.join("="); + } + } + } catch (error) { + if (error.code !== "ENOENT") { + throw error; + } + } +} + +async function benchmarkEndpoint(endpoint, base) { + const latencies = []; + const ttfbs = []; + const statuses = new Map(); + let errors = 0; + const total = Math.max(1, requestsPerEndpoint); + let nextIndex = 0; + const startedAt = performance.now(); + + async function worker() { + while (nextIndex < total) { + const requestIndex = nextIndex++; + try { + const result = await sendRequest(endpoint, base, requestIndex); + latencies.push(result.durationMs); + ttfbs.push(result.ttfbMs); + statuses.set(result.statusCode, (statuses.get(result.statusCode) ?? 0) + 1); + if (result.statusCode >= 500) { + errors += 1; + } + } catch { + errors += 1; + } + } + } + + await Promise.all(Array.from({ length: Math.min(concurrency, total) }, () => worker())); + const totalMs = performance.now() - startedAt; + + return { + name: endpoint.name, + method: endpoint.method, + path: endpoint.path, + requests: total, + peakRps: round((total / totalMs) * 1000), + sustainedRps: round((total / totalMs) * 1000), + errorRate: round((errors / total) * 100), + statusCodes: Object.fromEntries([...statuses.entries()].sort(([a], [b]) => a - b)), + latencyMs: { + p50: percentile(latencies, 50), + p95: percentile(latencies, 95), + p99: percentile(latencies, 99) + }, + ttfbMs: { + p50: percentile(ttfbs, 50), + p95: percentile(ttfbs, 95), + p99: percentile(ttfbs, 99) + } + }; +} + +function sendRequest(endpoint, base, requestIndex) { + const url = new URL(endpoint.path, base); + const isHttps = url.protocol === "https:"; + const transport = isHttps ? https : http; + const headers = { ...(endpoint.headers ? endpoint.headers(requestIndex) : {}) }; + let body; + + if (endpoint.json) { + body = Buffer.from(JSON.stringify(endpoint.json(requestIndex))); + headers["content-type"] = "application/json"; + headers["content-length"] = body.length; + } + + if (endpoint.multipart) { + const multipart = endpoint.multipart(requestIndex); + body = multipart.body; + headers["content-type"] = multipart.contentType; + headers["content-length"] = body.length; + } + + return new Promise((resolvePromise, reject) => { + const start = performance.now(); + let firstByteAt; + const req = transport.request( + { + method: endpoint.method, + hostname: url.hostname, + port: url.port || (isHttps ? 443 : 80), + path: `${url.pathname}${url.search}`, + headers + }, + (res) => { + res.once("data", () => { + firstByteAt ??= performance.now(); + }); + res.on("data", () => {}); + res.on("end", () => { + const end = performance.now(); + resolvePromise({ + statusCode: res.statusCode ?? 0, + durationMs: round(end - start), + ttfbMs: round((firstByteAt ?? end) - start) + }); + }); + } + ); + + req.on("error", reject); + req.setTimeout(10_000, () => { + req.destroy(new Error("benchmark request timed out")); + }); + + if (body) { + req.write(body); + } + req.end(); + }); +} + +function userPayload(kind) { + return { + email: uniqueEmail(kind), + password: "benchmark-password", + role: "client" + }; +} + +function jobPayload(index) { + return { + title: `Benchmark automation job ${index}`, + description: "Benchmark payload representing a realistic client automation request.", + budgetMin: 500, + budgetMax: 1500, + categoryId: "automation", + skills: ["node", "api", "benchmarking"] + }; +} + +function proposalPayload(index) { + return { + jobId: `job_${index}`, + freelancerId: `usr_freelancer_${index}`, + coverLetter: "I can deliver this benchmarked integration with milestones and clear acceptance checks.", + amount: 900, + timelineDays: 7 + }; +} + +function paymentPayload(index) { + return { + invoiceId: `inv_${index}`, + amount: 90000, + currency: "usd", + method: "card" + }; +} + +function reviewPayload(index) { + return { + jobId: `job_${index}`, + reviewerId: `usr_client_${index}`, + revieweeId: `usr_freelancer_${index}`, + rating: 5, + comment: "Clear communication, scoped delivery, and measurable performance improvements." + }; +} + +function messagePayload(index) { + return { + conversationId: `conv_${index}`, + senderId: `usr_client_${index}`, + recipientId: `usr_freelancer_${index}`, + body: "Can you share the benchmark summary and next recommended regression threshold?" + }; +} + +function notificationPayload(index) { + return { + userId: `usr_${index}`, + type: "benchmark", + title: "Benchmark completed", + body: "Your API benchmark report is ready for review." + }; +} + +function uploadPayload(index) { + const boundary = `----freelanceflow-benchmark-${Date.now()}-${index}`; + const content = Buffer.from("benchmark upload fixture\n"); + const body = Buffer.concat([ + Buffer.from(`--${boundary}\r\n`), + Buffer.from('Content-Disposition: form-data; name="file"; filename="benchmark.txt"\r\n'), + Buffer.from("Content-Type: text/plain\r\n\r\n"), + content, + Buffer.from(`\r\n--${boundary}--\r\n`) + ]); + + return { + contentType: `multipart/form-data; boundary=${boundary}`, + body + }; +} + +function uniqueEmail(prefix) { + return `${prefix}.${Date.now()}.${Math.random().toString(36).slice(2)}@example.com`; +} + +function percentile(values, percentileValue) { + if (values.length === 0) { + return 0; + } + const sorted = [...values].sort((a, b) => a - b); + const index = Math.min(sorted.length - 1, Math.ceil((percentileValue / 100) * sorted.length) - 1); + return round(sorted[index]); +} + +function round(value) { + return Number(value.toFixed(2)); +} + +function compactResult(result) { + return { + endpoint: result.name, + requests: result.requests, + p50: result.latencyMs.p50, + p95: result.latencyMs.p95, + p99: result.latencyMs.p99, + ttfbP95: result.ttfbMs.p95, + rps: result.sustainedRps, + errorRate: result.errorRate + }; +} + +function findThresholdFailures(results, thresholdConfig) { + const globalThresholds = thresholdConfig.global ?? {}; + const endpointThresholds = thresholdConfig.endpoints ?? {}; + const failures = []; + + for (const result of results) { + const effective = { ...globalThresholds, ...(endpointThresholds[result.name] ?? {}) }; + if (effective.maxP99LatencyMs !== undefined && result.latencyMs.p99 > effective.maxP99LatencyMs) { + failures.push(`${result.name} p99 ${result.latencyMs.p99}ms > ${effective.maxP99LatencyMs}ms`); + } + if (effective.maxErrorRatePercent !== undefined && result.errorRate > effective.maxErrorRatePercent) { + failures.push(`${result.name} error rate ${result.errorRate}% > ${effective.maxErrorRatePercent}%`); + } + } + + return failures; +} + +function renderMarkdown(summary) { + const rows = summary.results + .map( + (result) => + `| ${result.name} | ${result.method} ${result.path} | ${result.requests} | ${result.latencyMs.p50} | ${result.latencyMs.p95} | ${result.latencyMs.p99} | ${result.ttfbMs.p95} | ${result.sustainedRps} | ${result.errorRate}% |` + ) + .join("\n"); + + return `# API Benchmark Summary + +- Target: ${summary.targetUrl} +- Mode: ${summary.mode} +- Started: ${summary.startedAt} +- Completed: ${summary.completedAt} +- Endpoints: ${summary.endpointCount} +- Requests per endpoint: ${summary.requestsPerEndpoint} +- Concurrency: ${summary.concurrency} + +| Endpoint | Route | Requests | p50 ms | p95 ms | p99 ms | TTFB p95 ms | Sustained RPS | Error rate | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +${rows} +`; +} + +function redactUrl(value) { + const url = new URL(value); + url.username = ""; + url.password = ""; + return url.toString(); +} diff --git a/benchmarks/thresholds.json b/benchmarks/thresholds.json new file mode 100644 index 000000000..ca396f997 --- /dev/null +++ b/benchmarks/thresholds.json @@ -0,0 +1,7 @@ +{ + "global": { + "maxP99LatencyMs": 2000, + "maxErrorRatePercent": 0 + }, + "endpoints": {} +} diff --git a/package.json b/package.json index 675e6e69d..f2e09ef9d 100644 --- a/package.json +++ b/package.json @@ -1,12 +1,14 @@ { "name": "freelance-platform-monorepo", "private": true, + "type": "module", "workspaces": [ "apps/*", "packages/*" ], "scripts": { "build": "echo \"Run package-specific builds (e.g. npm run build -w apps/web)\"", + "benchmark": "node benchmarks/run-benchmark.js", "lint": "echo \"No root lint configured\"", "test": "npm run test -w apps/api" }