diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 9b7177bf..db17e9d8 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -378,7 +378,7 @@ stateDiagram-v2 - `/api/health` for health checks - `/api/metrics` for JSON snapshot - `/api/metrics?format=prometheus` for scrape-compatible format -- Ops UI consumes these APIs for dashboarding +- Ops UI renders the same in-process snapshot: `src/app/settings/page.tsx` (Ops tab, also reached via `/ops`) reads `getMetricsSnapshot()` on the server and hands it to `src/components/ops-dashboard.tsx`, so screen and scrape cannot disagree ## 9) 🚨 Failure Modes and Current Behavior diff --git a/README.md b/README.md index 53844b42..16f23d73 100644 --- a/README.md +++ b/README.md @@ -525,6 +525,7 @@ or `signedXdr` strings. - Health endpoint: `GET /api/health` β€” returns `blocklist` object with `configured`, `lastRefreshAt`, `domainCount`, `lastError`, plus `ready`, `checks`, and `failingChecks` (`production_config`, `storage`, `horizon`). The dashboard shows its pay, fund, and policy actions only while `ready` is `true`. - Metrics endpoint: `GET /api/metrics` + Prometheus format - `/ops` dashboard shows: + - allow / deny / rate-limit / submit-failure counters, read from the same in-process metrics snapshot that `GET /api/metrics` exports (zeroes when nothing has been recorded yet) - service health - total requests - error rate diff --git a/docs/REVIEWER_EVIDENCE_MATRIX.md b/docs/REVIEWER_EVIDENCE_MATRIX.md index 5e0c5620..742f143b 100644 --- a/docs/REVIEWER_EVIDENCE_MATRIX.md +++ b/docs/REVIEWER_EVIDENCE_MATRIX.md @@ -1,37 +1,38 @@ -# Reviewer Evidence Matrix - -This document maps Fortexa's core security and payment claims directly to the codebase, tests, and API routes. It is designed to allow an SCF or investor reviewer to verify all claims in under 15 minutes. - -## 15-Minute Reviewer Checklist - -- [ ] **Wallet-bound login**: Verify in `src/lib/auth/wallet-challenge.ts` and `src/lib/auth/freighter.ts`. -- [ ] **Role-based access control**: Verify `requireAuth` usage in `src/app/api/stellar/submit-signed/route.ts` and `src/lib/auth/require-auth.ts`. -- [ ] **Policy decision outcomes**: Verify logic in `src/lib/policy/engine.ts` and `src/lib/decision/engine.ts`. -- [ ] **Human approval gate**: Verify UI block in `src/components/decision-console.tsx` and logic in `src/lib/decision/simulate.ts`. -- [ ] **Signed-XDR only payment path**: Verify API only accepts signed XDR in `src/app/api/stellar/submit-signed/route.ts` and client builds unsigned in `src/lib/stellar/client.ts`. -- [ ] **Audit hash-chain integrity**: Verify hashing mechanism in `src/lib/audit/hash-chain.ts`. -- [ ] **Metrics and ops visibility**: Verify structured logging and metrics in `src/lib/observability/metrics.ts` and `src/lib/observability/logger.ts`. -- [ ] **Idempotent submit behavior**: Verify the key claim / canonical body hash / stored-result replay logic in `src/lib/storage/submit-idempotency-store.ts` and `src/app/api/stellar/submit-signed/route.ts`. -- [ ] **No server-side private-key custody**: Verify absence of backend signing (`Keypair.fromSecret()`) in `src/lib/stellar/client.ts`. - -## Detailed Evidence Mapping - -| Claim | Source File(s) / API Routes | Tests / Documentation | -| :--- | :--- | :--- | -| **Wallet-bound login** | [`src/lib/auth/wallet-challenge.ts`](../src/lib/auth/wallet-challenge.ts)
[`src/lib/auth/freighter.ts`](../src/lib/auth/freighter.ts) | `src/lib/auth/freighter.test.ts`
`src/lib/auth/wallet-challenge.test.ts` | -| **Role-based access control** | [`src/lib/auth/require-auth.ts`](../src/lib/auth/require-auth.ts)
[`src/lib/auth/wallet-role.ts`](../src/lib/auth/wallet-role.ts) | `src/lib/auth/login-lockout.test.ts` | -| **Policy decision outcomes** | [`src/lib/policy/engine.ts`](../src/lib/policy/engine.ts)
[`src/lib/decision/engine.ts`](../src/lib/decision/engine.ts) | `src/lib/policy/engine.test.ts`
`src/lib/decision/engine.test.ts` | -| **Human approval gate** | [`src/components/decision-console.tsx`](../src/components/decision-console.tsx)
[`src/lib/decision/simulate.ts`](../src/lib/decision/simulate.ts) | `src/lib/decision/simulate.test.ts` | -| **Signed-XDR only payment path** | [`src/app/api/stellar/submit-signed/route.ts`](../src/app/api/stellar/submit-signed/route.ts)
[`src/lib/stellar/client.ts`](../src/lib/stellar/client.ts) | `src/lib/stellar/verify-payment-quote.test.ts` | -| **Audit hash-chain integrity** | [`src/lib/audit/hash-chain.ts`](../src/lib/audit/hash-chain.ts) | `src/lib/audit/hash-chain.test.ts` | -| **Metrics and ops visibility** | [`src/lib/observability/metrics.ts`](../src/lib/observability/metrics.ts)
[`src/lib/observability/logger.ts`](../src/lib/observability/logger.ts) | `src/lib/observability/metrics.test.ts` | -| **Idempotent submit behavior** | [`src/app/api/stellar/submit-signed/route.ts`](../src/app/api/stellar/submit-signed/route.ts)
[`src/lib/storage/submit-idempotency-store.ts`](../src/lib/storage/submit-idempotency-store.ts) | `src/app/api/stellar/submit-signed-idempotency.test.ts`
`src/lib/storage/submit-idempotency-store.test.ts`
`src/lib/storage/submit-idempotency-cleanup.test.ts` | -| **No server-side private-key custody** | [`src/lib/stellar/client.ts`](../src/lib/stellar/client.ts) | | - -## Honest Limitations - -To maintain transparency for SCF reviewers and investors, we note the following current architectural constraints: - -1. **Audit Hash-Chain Storage**: The hash-chain is currently persisted in a standard database (Postgres) rather than being anchored to a decentralized public ledger or immutable storage network. -2. **Idempotency State**: Idempotency records live in Postgres (`fortexa_submit_idempotency`) with a local JSON-file fallback, so a record can be lost if the process restarts before the file store is written, and claims are per-instance unless all instances share the database. -3. **Hardware Wallet Integration**: The current wallet-bound login relies on browser extension wallets (like Freighter) and does not yet have direct native integrations with hardware signing devices. +# Reviewer Evidence Matrix + +This document maps Fortexa's core security and payment claims directly to the codebase, tests, and API routes. It is designed to allow an SCF or investor reviewer to verify all claims in under 15 minutes. + +## 15-Minute Reviewer Checklist + +- [ ] **Wallet-bound login**: Verify in `src/lib/auth/wallet-challenge.ts` and `src/lib/auth/freighter.ts`. +- [ ] **Role-based access control**: Verify `requireAuth` usage in `src/app/api/stellar/submit-signed/route.ts` and `src/lib/auth/require-auth.ts`. +- [ ] **Policy decision outcomes**: Verify logic in `src/lib/policy/engine.ts` and `src/lib/decision/engine.ts`. +- [ ] **Human approval gate**: Verify UI block in `src/components/decision-console.tsx` and logic in `src/lib/decision/simulate.ts`. +- [ ] **Signed-XDR only payment path**: Verify API only accepts signed XDR in `src/app/api/stellar/submit-signed/route.ts` and client builds unsigned in `src/lib/stellar/client.ts`. +- [ ] **Audit hash-chain integrity**: Verify hashing mechanism in `src/lib/audit/hash-chain.ts`. +- [ ] **Metrics and ops visibility**: Verify structured logging and metrics in `src/lib/observability/metrics.ts` and `src/lib/observability/logger.ts`. +- [ ] **Dashboard and scrape agree**: Verify the Ops tab loader (`src/app/settings/page.tsx`) passes `getMetricsSnapshot()` to `src/components/ops-dashboard.tsx`, the same object `GET /api/metrics` serialises. +- [ ] **Idempotent submit behavior**: Verify deduplication logic via `xdrHash` in `src/app/api/stellar/submit-signed/route.ts`. +- [ ] **No server-side private-key custody**: Verify absence of backend signing (`Keypair.fromSecret()`) in `src/lib/stellar/client.ts`. + +## Detailed Evidence Mapping + +| Claim | Source File(s) / API Routes | Tests / Documentation | +| :--- | :--- | :--- | +| **Wallet-bound login** | [`src/lib/auth/wallet-challenge.ts`](../src/lib/auth/wallet-challenge.ts)
[`src/lib/auth/freighter.ts`](../src/lib/auth/freighter.ts) | `src/lib/auth/freighter.test.ts`
`src/lib/auth/wallet-challenge.test.ts` | +| **Role-based access control** | [`src/lib/auth/require-auth.ts`](../src/lib/auth/require-auth.ts)
[`src/lib/auth/wallet-role.ts`](../src/lib/auth/wallet-role.ts) | `src/lib/auth/login-lockout.test.ts` | +| **Policy decision outcomes** | [`src/lib/policy/engine.ts`](../src/lib/policy/engine.ts)
[`src/lib/decision/engine.ts`](../src/lib/decision/engine.ts) | `src/lib/policy/engine.test.ts`
`src/lib/decision/engine.test.ts` | +| **Human approval gate** | [`src/components/decision-console.tsx`](../src/components/decision-console.tsx)
[`src/lib/decision/simulate.ts`](../src/lib/decision/simulate.ts) | `src/lib/decision/simulate.test.ts` | +| **Signed-XDR only payment path** | [`src/app/api/stellar/submit-signed/route.ts`](../src/app/api/stellar/submit-signed/route.ts)
[`src/lib/stellar/client.ts`](../src/lib/stellar/client.ts) | `src/lib/stellar/verify-payment-quote.test.ts` | +| **Audit hash-chain integrity** | [`src/lib/audit/hash-chain.ts`](../src/lib/audit/hash-chain.ts) | `src/lib/audit/hash-chain.test.ts` | +| **Metrics and ops visibility** | [`src/lib/observability/metrics.ts`](../src/lib/observability/metrics.ts)
[`src/lib/observability/logger.ts`](../src/lib/observability/logger.ts)
[`src/app/api/metrics/route.ts`](../src/app/api/metrics/route.ts)
[`src/components/ops-dashboard.tsx`](../src/components/ops-dashboard.tsx) | `src/lib/observability/metrics.test.ts`
`src/app/api/metrics/route.test.ts`
`src/components/ops-dashboard.snapshot.test.tsx` | +| **Idempotent submit behavior** | [`src/app/api/stellar/submit-signed/route.ts`](../src/app/api/stellar/submit-signed/route.ts) | | +| **No server-side private-key custody** | [`src/lib/stellar/client.ts`](../src/lib/stellar/client.ts) | | + +## Honest Limitations + +To maintain transparency for SCF reviewers and investors, we note the following current architectural constraints: + +1. **Audit Hash-Chain Storage**: The hash-chain is currently persisted in a standard database (Postgres) rather than being anchored to a decentralized public ledger or immutable storage network. +2. **Idempotency State**: Idempotency keys rely on local storage/Redis, which may be volatile depending on the infrastructure configuration. +3. **Hardware Wallet Integration**: The current wallet-bound login relies on browser extension wallets (like Freighter) and does not yet have direct native integrations with hardware signing devices. diff --git a/docs/observability.md b/docs/observability.md index 51359600..3927368e 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -1,294 +1,363 @@ -# Fortexa Observability Guide - -Fortexa exposes lightweight, Prometheus-compatible metrics for every API route. This guide covers how to scrape them, sample PromQL for common dashboards/alerts, and where to look inside the app. - -> Metrics implementation: [`src/lib/observability/metrics.ts`](../src/lib/observability/metrics.ts) - ---- - -## Endpoints - -| Endpoint | Purpose | Auth | -| --- | --- | --- | -| `GET /api/health` | Liveness check | None | -| `GET /api/metrics` | JSON snapshot of all route buckets | Operator session cookie | -| `GET /api/metrics?format=prometheus` | Prometheus text exposition (v0.0.4) | Operator session cookie | - -The `/ops` page in the app renders the same data for human operators. - -### Auth - -Both `/api/metrics` variants require an **operator** session. Fortexa uses an HMAC-signed cookie named `fortexa_session`, set when a wallet signs in via `POST /api/auth/login`. - -For local development (when `FORTEXA_OPERATOR_WALLETS` is unset), any valid-format Stellar public key resolves to the operator role. In production, set `FORTEXA_OPERATOR_WALLETS` to the comma-separated list of operator wallet keys. - ---- - -## Exported metrics - -All series are labeled by `route` (Next.js route path, e.g. `/api/decision`) and `method` (uppercase HTTP verb). - -| Metric | Type | Description | -| --- | --- | --- | -| `fortexa_requests_total` | counter | Total API requests by route/method | -| `fortexa_request_errors_total` | counter | Requests that returned HTTP >= 400 | -| `fortexa_request_duration_ms_p95` | gauge | Rolling p95 latency in milliseconds (last 500 samples per bucket) | -| `fortexa_decision_outcomes_total` | counter | Decision evaluations labelled by `outcome` (APPROVE \| WARN \| REQUIRE_APPROVAL \| BLOCK) | -| `fortexa_stellar_submit_results_total` | counter | Stellar submission attempts labelled by `result` (success \| horizon_failure \| validation_failure \| idempotency_replay \| idempotency_conflict \| idempotency_in_flight) | - -Sample output: - -``` -# HELP fortexa_requests_total Total API requests by route/method -# TYPE fortexa_requests_total counter -fortexa_requests_total{route="/api/health",method="GET"} 5 -fortexa_requests_total{route="/api/decision",method="POST"} 2 -# HELP fortexa_request_errors_total Total API errors by route/method -# TYPE fortexa_request_errors_total counter -fortexa_request_errors_total{route="/api/health",method="GET"} 0 -# HELP fortexa_request_duration_ms_p95 P95 request duration in milliseconds -# TYPE fortexa_request_duration_ms_p95 gauge -fortexa_request_duration_ms_p95{route="/api/health",method="GET"} 1.00 -``` - -> Note: `p95` is exported as a gauge computed from an in-memory ring buffer (last 500 observations per route), not as a Prometheus histogram. It is intended for at-a-glance dashboards, not high-fidelity SLO math. - ---- - -## Prometheus scrape config - -Fortexa's metrics endpoint is cookie-protected. Use Prometheus' `authorization` or `cookies` support via the `Cookie` header to authenticate. - -```yaml -scrape_configs: - - job_name: fortexa - metrics_path: /api/metrics - params: - format: [prometheus] - scheme: https - scrape_interval: 30s - scrape_timeout: 10s - static_configs: - - targets: - - fortexa.your-domain.example - # Operator session cookie minted from POST /api/auth/login - # Store the cookie value in a file and reference it here. - authorization: - type: "" - # Prometheus does not natively support cookie auth headers; use a - # request header instead: - # (Prometheus 2.26+ supports `http_headers`) - http_headers: - Cookie: - values: - - "fortexa_session=" -``` - -If your environment does not allow long-lived cookies, an alternative is to put a small auth-injecting proxy (e.g. nginx, Envoy, or a sidecar) in front of Fortexa that adds the `Cookie` header on each request. - -### Local verification - -```bash -# 1. Log in (any valid-format Stellar pubkey works when no operator allowlist is set) -curl -s -c /tmp/fortexa-cookies.txt \ - -H "Content-Type: application/json" \ - -d '{"publicKey":"GAIH3ULLFQ4DGSECF2AR555KZ4KNDGEKN4AFI4SU2M7B43MGK3QJZNSR"}' \ - http://localhost:3001/api/auth/login - -# 2. Scrape -curl -b /tmp/fortexa-cookies.txt \ - "http://localhost:3001/api/metrics?format=prometheus" -``` - ---- - -## Sample PromQL - -### Request rate per route (requests/sec, 5-minute window) - -```promql -sum by (route) (rate(fortexa_requests_total[5m])) -``` - -### Error rate per route (fraction, 5-minute window) - -```promql -sum by (route) (rate(fortexa_request_errors_total[5m])) - / -sum by (route) (rate(fortexa_requests_total[5m])) -``` - -### Overall error rate - -```promql -sum(rate(fortexa_request_errors_total[5m])) - / -sum(rate(fortexa_requests_total[5m])) -``` - -### p95 latency per route (ms) - -```promql -max by (route) (fortexa_request_duration_ms_p95) -``` - -### Top 5 noisiest routes by request count (last 1h) - -```promql -topk(5, sum by (route) (increase(fortexa_requests_total[1h]))) -``` - -### Decision outcome breakdown (rate over 5 minutes) - -```promql -sum by (outcome) (rate(fortexa_decision_outcomes_total[5m])) -``` - -### Fraction of decisions that resulted in BLOCK - -```promql -rate(fortexa_decision_outcomes_total{outcome="BLOCK"}[5m]) - / -sum(rate(fortexa_decision_outcomes_total[5m])) -``` - -### Stellar submission success rate (5-minute window) - -```promql -rate(fortexa_stellar_submit_results_total{result="success"}[5m]) - / -sum(rate(fortexa_stellar_submit_results_total[5m])) -``` - -### Stellar submission results breakdown - -```promql -sum by (result) (rate(fortexa_stellar_submit_results_total[5m])) -``` - -### Example alert β€” sustained elevated error rate - -```yaml -groups: - - name: fortexa - rules: - - alert: FortexaHighErrorRate - expr: | - ( - sum by (route) (rate(fortexa_request_errors_total[5m])) - / - sum by (route) (rate(fortexa_requests_total[5m])) - ) > 0.05 - for: 10m - labels: - severity: warning - annotations: - summary: "Fortexa error rate >5% on {{ $labels.route }}" -``` - ---- - -## Grafana - -A minimal importable dashboard with the three panels above lives at -[`docs/grafana/fortexa-minimal.json`](grafana/fortexa-minimal.json). - -To import: - -1. In Grafana, go to **Dashboards β†’ New β†’ Import**. -2. Upload `fortexa-minimal.json` (or paste its contents). -3. When prompted, select your Fortexa Prometheus datasource for the `DS_PROMETHEUS` variable. - -Panels: - -- **Request rate by route** β€” `sum by (route) (rate(fortexa_requests_total[5m]))` -- **Error rate by route** β€” error counter / request counter, with 5% / 10% threshold lines -- **p95 latency by route** β€” `max by (route) (fortexa_request_duration_ms_p95)` - ---- - -## In-app dashboard - -For operators who don't want to wire up Prometheus, the `/ops` page renders the same snapshot: - -- Service health (from `/api/health`) -- Total requests / error rate / signed tx count -- Top routes and rolling trend - -`/ops` is gated by operator session, same as `/api/metrics`. - ---- - -## Notes & caveats - -- **In-memory only.** All buckets live in the Node process. Restarting the server resets counters. For long-term retention, scrape into Prometheus. -- **Per-instance.** If Fortexa is horizontally scaled, each replica exposes its own counters. Aggregate with `sum by (route)` in PromQL. -- **No histogram.** p95 is a gauge derived from a 500-sample ring buffer, not a true Prometheus histogram. Don't use it for cross-instance percentile aggregation. -- **Local vs deployed auth.** Locally, no allowlist means any pubkey works. In production, set `FORTEXA_OPERATOR_WALLETS` to restrict. - - ---- - -## Reporting API Failures - -Every API response includes an `x-request-id` header whose value is a UUID v4. The same identifier appears in structured logs as the `requestId` field. - -When reporting a failure, include the `x-request-id` value so operators can correlate the issue with server-side logs. A complete report should contain: - -- **`x-request-id`** β€” the UUID from the response header or `requestId` from the log line -- **Route and method** β€” e.g. `POST /api/decision` -- **HTTP status code** β€” e.g. `500`, `403` -- **Timestamp** β€” the `ts` field from the log line, or the wall-clock time of the failure - -Example log line: - -```json -{"ts":"2026-06-29T12:34:56.789Z","level":"error","message":"Horizon submission failed","requestId":"a1b2c3d4-...","route":"/api/stellar/submit-signed","method":"POST","statusCode":502} -``` - -If an upstream proxy or load balancer already set an `x-request-id` header on the incoming request, that value is forwarded in both the response and the logs. Otherwise the server generates one automatically. - ---- - -## Audit Export Verifier - -Fortexa stores a tamper-evident hash chain in every audit entry (`entryHash` / `previousHash`). -The verifier CLI lets you confirm integrity of an exported audit file **without** a database, -wallet session, or running Next.js server. - -### Usage - - npx tsx scripts/verify-audit-export.ts - -Or via the package script: - - npm run verify:audit -- - -### Export formats supported - -| Format | Example shape | -|--------|--------------| -| Plain array | [{"id": "...", "entryHash": "...", ...}] | -| scope=mine | {"scope": "mine", "entries": [...]} | -| scope=all | {"scope": "all", "entriesByUser": {"userId": [...]}} | - -### Exit codes - -| Code | Meaning | -|------|---------| -| 0 | Chain valid | -| 1 | Chain invalid (tampered, deleted, or reordered entries) | -| 2 | File not found, unreadable, or unrecognized format | - -### Test fixtures - -Reference fixtures live in scripts/fixtures/: - -| File | Purpose | -|------|---------| -| valid-chain.json | 3-entry valid chain (plain array) | -| valid-mine-export.json | 2-entry valid chain (scope=mine) | -| valid-all-export.json | Valid multi-user export (scope=all) | -| tampered-field.json | Entry with modified explanation (tamper detection) | -| deleted-entry.json | Middle entry removed (gap detection) | -| reordered-entries.json | Timestamps swapped to disguise reordering | +# Fortexa Observability Guide + +Fortexa exposes lightweight, Prometheus-compatible metrics for every API route. This guide covers how to scrape them, sample PromQL for common dashboards/alerts, and where to look inside the app. + +> Metrics implementation: [`src/lib/observability/metrics.ts`](../src/lib/observability/metrics.ts) + +--- + +## Endpoints + +| Endpoint | Purpose | Auth | +| --- | --- | --- | +| `GET /api/health` | Liveness check | None | +| `GET /api/metrics` | JSON snapshot of all route buckets plus the `counters` block | Operator session cookie | +| `GET /api/metrics?format=prometheus` | Prometheus text exposition (v0.0.4) | Operator session cookie | + +The `/ops` page in the app renders the same snapshot for human operators. + +### Auth + +Both `/api/metrics` variants require an **operator** session. Fortexa uses an HMAC-signed cookie named `fortexa_session`, set when a wallet signs in via `POST /api/auth/login`. + +For local development (when `FORTEXA_OPERATOR_WALLETS` is unset), any valid-format Stellar public key resolves to the operator role. In production, set `FORTEXA_OPERATOR_WALLETS` to the comma-separated list of operator wallet keys. + +--- + +## Exported metrics + +All series are labeled by `route` (Next.js route path, e.g. `/api/decision`) and `method` (uppercase HTTP verb). + +| Metric | Type | Description | +| --- | --- | --- | +| `fortexa_requests_total` | counter | Total API requests by route/method | +| `fortexa_request_errors_total` | counter | Requests that returned HTTP >= 400 | +| `fortexa_request_duration_ms_p95` | gauge | Rolling p95 latency in milliseconds (last 500 samples per bucket) | +| `fortexa_decision_outcomes_total` | counter | Decision evaluations labelled by `outcome` (APPROVE \| WARN \| REQUIRE_APPROVAL \| BLOCK) | +| `fortexa_stellar_submit_results_total` | counter | Stellar submission attempts labelled by `result` (success \| horizon_failure \| validation_failure \| idempotency_replay \| idempotency_conflict) | +| `fortexa_decisions_allowed_total` | counter | Rolled up: decisions that let the action proceed (`APPROVE` + `WARN`) | +| `fortexa_decisions_denied_total` | counter | Rolled up: decisions that stopped the action (`REQUIRE_APPROVAL` + `BLOCK`) | +| `fortexa_rate_limit_rejections_total` | counter | Requests the rate limiter rejected with HTTP 429 | +| `fortexa_stellar_submit_failures_total` | counter | Rolled up: submissions that did not settle on-chain (everything except `success` and `idempotency_replay`) | + +The four rolled-up counters are the same numbers the ops dashboard renders. They are +derived from in-process counters only (never from a scan of the audit table) and are +always emitted, zero-initialised, so alert rules never break on a freshly restarted +process. + +Sample output: + +``` +# HELP fortexa_requests_total Total API requests by route/method +# TYPE fortexa_requests_total counter +fortexa_requests_total{route="/api/health",method="GET"} 5 +fortexa_requests_total{route="/api/decision",method="POST"} 2 +# HELP fortexa_request_errors_total Total API errors by route/method +# TYPE fortexa_request_errors_total counter +fortexa_request_errors_total{route="/api/health",method="GET"} 0 +# HELP fortexa_request_duration_ms_p95 P95 request duration in milliseconds +# TYPE fortexa_request_duration_ms_p95 gauge +fortexa_request_duration_ms_p95{route="/api/health",method="GET"} 1.00 +# HELP fortexa_decisions_allowed_total Total decisions that were allowed to proceed +# TYPE fortexa_decisions_allowed_total counter +fortexa_decisions_allowed_total 2 +# HELP fortexa_decisions_denied_total Total decisions that stopped the action +# TYPE fortexa_decisions_denied_total counter +fortexa_decisions_denied_total 1 +# HELP fortexa_rate_limit_rejections_total Total requests rejected by the rate limiter +# TYPE fortexa_rate_limit_rejections_total counter +fortexa_rate_limit_rejections_total 0 +# HELP fortexa_stellar_submit_failures_total Total Stellar submissions that did not settle on-chain +# TYPE fortexa_stellar_submit_failures_total counter +fortexa_stellar_submit_failures_total 0 +``` + +> Note: `p95` is exported as a gauge computed from an in-memory ring buffer (last 500 observations per route), not as a Prometheus histogram. It is intended for at-a-glance dashboards, not high-fidelity SLO math. + +### JSON snapshot shape + +```jsonc +{ + "service": "fortexa", + "timestamp": "2026-09-30T08:00:00.000Z", + "totals": { "totalCount": 12, "errorCount": 1, "errorRate": 0.083 }, + "counters": { "allow": 2, "deny": 1, "rateLimit": 0, "submitFailures": 0 }, + "routes": [ /* route/method buckets */ ] +} +``` + +`counters` is what the in-app dashboard renders, byte for byte: the server component +that loads the Ops tab calls `getMetricsSnapshot()` and passes the result to +`src/components/ops-dashboard.tsx`, so the screen and this body cannot drift. + +--- + +## Prometheus scrape config + +Fortexa's metrics endpoint is cookie-protected. Use Prometheus' `authorization` or `cookies` support via the `Cookie` header to authenticate. + +```yaml +scrape_configs: + - job_name: fortexa + metrics_path: /api/metrics + params: + format: [prometheus] + scheme: https + scrape_interval: 30s + scrape_timeout: 10s + static_configs: + - targets: + - fortexa.your-domain.example + # Operator session cookie minted from POST /api/auth/login + # Store the cookie value in a file and reference it here. + authorization: + type: "" + # Prometheus does not natively support cookie auth headers; use a + # request header instead: + # (Prometheus 2.26+ supports `http_headers`) + http_headers: + Cookie: + values: + - "fortexa_session=" +``` + +If your environment does not allow long-lived cookies, an alternative is to put a small auth-injecting proxy (e.g. nginx, Envoy, or a sidecar) in front of Fortexa that adds the `Cookie` header on each request. + +### Local verification + +```bash +# 1. Log in (any valid-format Stellar pubkey works when no operator allowlist is set) +curl -s -c /tmp/fortexa-cookies.txt \ + -H "Content-Type: application/json" \ + -d '{"publicKey":"GAIH3ULLFQ4DGSECF2AR555KZ4KNDGEKN4AFI4SU2M7B43MGK3QJZNSR"}' \ + http://localhost:3001/api/auth/login + +# 2. Scrape +curl -b /tmp/fortexa-cookies.txt \ + "http://localhost:3001/api/metrics?format=prometheus" +``` + +--- + +## Sample PromQL + +### Request rate per route (requests/sec, 5-minute window) + +```promql +sum by (route) (rate(fortexa_requests_total[5m])) +``` + +### Error rate per route (fraction, 5-minute window) + +```promql +sum by (route) (rate(fortexa_request_errors_total[5m])) + / +sum by (route) (rate(fortexa_requests_total[5m])) +``` + +### Overall error rate + +```promql +sum(rate(fortexa_request_errors_total[5m])) + / +sum(rate(fortexa_requests_total[5m])) +``` + +### p95 latency per route (ms) + +```promql +max by (route) (fortexa_request_duration_ms_p95) +``` + +### Top 5 noisiest routes by request count (last 1h) + +```promql +topk(5, sum by (route) (increase(fortexa_requests_total[1h]))) +``` + +### Decision outcome breakdown (rate over 5 minutes) + +```promql +sum by (outcome) (rate(fortexa_decision_outcomes_total[5m])) +``` + +### Fraction of decisions that resulted in BLOCK + +```promql +rate(fortexa_decision_outcomes_total{outcome="BLOCK"}[5m]) + / +sum(rate(fortexa_decision_outcomes_total[5m])) +``` + +### Stellar submission success rate (5-minute window) + +```promql +rate(fortexa_stellar_submit_results_total{result="success"}[5m]) + / +sum(rate(fortexa_stellar_submit_results_total[5m])) +``` + +### Stellar submission results breakdown + +```promql +sum by (result) (rate(fortexa_stellar_submit_results_total[5m])) +``` + +### Decision throughput: allowed vs denied (5-minute window) + +```promql +rate(fortexa_decisions_allowed_total[5m]) + + +rate(fortexa_decisions_denied_total[5m]) +``` + +### Share of decisions that were denied + +```promql +rate(fortexa_decisions_denied_total[5m]) + / +clamp_min( + rate(fortexa_decisions_allowed_total[5m]) + rate(fortexa_decisions_denied_total[5m]), + 1e-9 +) +``` + +### Rate-limit rejections per second + +```promql +rate(fortexa_rate_limit_rejections_total[5m]) +``` + +### Signed submission failures per second + +```promql +rate(fortexa_stellar_submit_failures_total[5m]) +``` + +### Example alert β€” sustained elevated error rate + +```yaml +groups: + - name: fortexa + rules: + - alert: FortexaHighErrorRate + expr: | + ( + sum by (route) (rate(fortexa_request_errors_total[5m])) + / + sum by (route) (rate(fortexa_requests_total[5m])) + ) > 0.05 + for: 10m + labels: + severity: warning + annotations: + summary: "Fortexa error rate >5% on {{ $labels.route }}" +``` + +--- + +## Grafana + +A minimal importable dashboard with the three panels above lives at +[`docs/grafana/fortexa-minimal.json`](grafana/fortexa-minimal.json). + +To import: + +1. In Grafana, go to **Dashboards β†’ New β†’ Import**. +2. Upload `fortexa-minimal.json` (or paste its contents). +3. When prompted, select your Fortexa Prometheus datasource for the `DS_PROMETHEUS` variable. + +Panels: + +- **Request rate by route** β€” `sum by (route) (rate(fortexa_requests_total[5m]))` +- **Error rate by route** β€” error counter / request counter, with 5% / 10% threshold lines +- **p95 latency by route** β€” `max by (route) (fortexa_request_duration_ms_p95)` + +--- + +## In-app dashboard + +For operators who don't want to wire up Prometheus, the `/ops` page renders the same snapshot: + +- Allowed / denied / rate-limited / submit-failure counters, read straight off the metrics snapshot +- Service health (from `/api/health`) +- Total requests / error rate / signed tx count +- Top routes and rolling trend + +`/ops` is gated by operator session, same as `/api/metrics`. + +### One snapshot, two views + +The Ops tab is loaded by a server component +([`src/app/settings/page.tsx`](../src/app/settings/page.tsx)), which calls +`getMetricsSnapshot()` in-process and hands the snapshot to +[`src/components/ops-dashboard.tsx`](../src/components/ops-dashboard.tsx). Consequences worth +knowing when reading the screen: + +- The four counters are **never** recomputed by scanning the audit table. They are the same + values the `/api/metrics` body and the Prometheus text export. +- An empty snapshot renders `0` in each tile instead of an endless spinner. +- The dashboard still polls `/api/metrics` every 8s for route/traffic refresh, so the numbers + keep moving without a reload; each poll returns the same snapshot shape, so both views stay + consistent. + +Tests: [`src/components/ops-dashboard.snapshot.test.tsx`](../src/components/ops-dashboard.snapshot.test.tsx) +(renders from a snapshot, never fetches over HTTP) and +[`src/app/api/metrics/route.test.ts`](../src/app/api/metrics/route.test.ts) (route body +parity). + +--- + +## Notes & caveats + +- **In-memory only.** All buckets live in the Node process. Restarting the server resets counters. For long-term retention, scrape into Prometheus. +- **Per-instance.** If Fortexa is horizontally scaled, each replica exposes its own counters. Aggregate with `sum by (route)` in PromQL. +- **No histogram.** p95 is a gauge derived from a 500-sample ring buffer, not a true Prometheus histogram. Don't use it for cross-instance percentile aggregation. +- **Local vs deployed auth.** Locally, no allowlist means any pubkey works. In production, set `FORTEXA_OPERATOR_WALLETS` to restrict. +- **Ops UI is single-instance.** The dashboard reads the snapshot of the instance that served the page, so a load-balanced setup shows the replica the operator happened to hit. Prometheus is the source of truth for fleet-wide numbers. + + +--- + +## Audit Export Verifier + +Fortexa stores a tamper-evident hash chain in every audit entry (`entryHash` / `previousHash`). +The verifier CLI lets you confirm integrity of an exported audit file **without** a database, +wallet session, or running Next.js server. + +### Usage + + npx tsx scripts/verify-audit-export.ts + +Or via the package script: + + npm run verify:audit -- + +### Export formats supported + +| Format | Example shape | +|--------|--------------| +| Plain array | [{"id": "...", "entryHash": "...", ...}] | +| scope=mine | {"scope": "mine", "entries": [...]} | +| scope=all | {"scope": "all", "entriesByUser": {"userId": [...]}} | + +### Exit codes + +| Code | Meaning | +|------|---------| +| 0 | Chain valid | +| 1 | Chain invalid (tampered, deleted, or reordered entries) | +| 2 | File not found, unreadable, or unrecognized format | + +### Test fixtures + +Reference fixtures live in scripts/fixtures/: + +| File | Purpose | +|------|---------| +| valid-chain.json | 3-entry valid chain (plain array) | +| valid-mine-export.json | 2-entry valid chain (scope=mine) | +| valid-all-export.json | Valid multi-user export (scope=all) | +| tampered-field.json | Entry with modified explanation (tamper detection) | +| deleted-entry.json | Middle entry removed (gap detection) | +| reordered-entries.json | Timestamps swapped to disguise reordering | diff --git a/src/app/api/metrics/route.test.ts b/src/app/api/metrics/route.test.ts index 0bdc013e..f8572439 100644 --- a/src/app/api/metrics/route.test.ts +++ b/src/app/api/metrics/route.test.ts @@ -1,213 +1,200 @@ -import { beforeEach, describe, expect, it } from "vitest"; -import { NextRequest } from "next/server"; - -import { AUTH_COOKIE_KEY, createSessionToken } from "@/lib/auth/session"; -import { - recordApiMetric, - recordDecisionOutcome, - recordStellarSubmitResult, - resetMetrics, -} from "@/lib/observability/metrics"; -import { GET } from "@/app/api/metrics/route"; - -function operatorCookie() { - process.env.FORTEXA_AUTH_SECRET = "metrics-test-secret"; - const token = createSessionToken({ - email: "ops@fortexa.local", - role: "operator", - userId: "metrics-operator-id", - expiresInSeconds: 120, - }); - return `${AUTH_COOKIE_KEY}=${token}`; -} - -describe("/api/metrics route", () => { - beforeEach(() => { - resetMetrics(); - }); - - it("returns 401 when unauthenticated", async () => { - const request = new NextRequest("http://localhost/api/metrics"); - const response = await GET(request); - expect(response.status).toBe(401); - - const body = (await response.json()) as { error: string }; - expect(body.error).toBe("Unauthorized. Login required."); - }); - - it("returns JSON shape with expected fields", async () => { - const request = new NextRequest("http://localhost/api/metrics", { - headers: { cookie: operatorCookie() }, - }); - - const response = await GET(request); - expect(response.status).toBe(200); - expect(response.headers.get("content-type")).toContain("application/json"); - - const body = (await response.json()) as { - service: string; - timestamp: string; - totals: { totalCount: number; errorCount: number; errorRate: number }; - routes: Array<{ - route: string; - method: string; - totalCount: number; - errorCount: number; - errorRate: number; - avgDurationMs: number; - p95DurationMs: number; - lastStatusCode: number; - lastSeenAt: string; - }>; - }; - - expect(body.service).toBe("fortexa"); - expect(typeof body.timestamp).toBe("string"); - expect(new Date(body.timestamp).toISOString()).toBe(body.timestamp); - - expect(body.totals).toHaveProperty("totalCount"); - expect(body.totals).toHaveProperty("errorCount"); - expect(body.totals).toHaveProperty("errorRate"); - expect(typeof body.totals.totalCount).toBe("number"); - - expect(Array.isArray(body.routes)).toBe(true); - - for (const route of body.routes) { - expect(route).toHaveProperty("route"); - expect(route).toHaveProperty("method"); - expect(route).toHaveProperty("totalCount"); - expect(route).toHaveProperty("errorCount"); - expect(route).toHaveProperty("errorRate"); - expect(route).toHaveProperty("p95DurationMs"); - expect(route).toHaveProperty("avgDurationMs"); - expect(route).toHaveProperty("lastStatusCode"); - expect(route).toHaveProperty("lastSeenAt"); - - expect(typeof route.route).toBe("string"); - expect(typeof route.method).toBe("string"); - expect(typeof route.totalCount).toBe("number"); - expect(typeof route.errorCount).toBe("number"); - expect(typeof route.errorRate).toBe("number"); - expect(typeof route.p95DurationMs).toBe("number"); - expect(typeof route.lastSeenAt).toBe("string"); - expect(new Date(route.lastSeenAt).toISOString()).toBe(route.lastSeenAt); - } - }); - - it("records the request itself in routes array", async () => { - const request = new NextRequest("http://localhost/api/metrics", { - headers: { cookie: operatorCookie() }, - }); - - const response = await GET(request); - const body = (await response.json()) as { - routes: Array<{ route: string; method: string; totalCount: number; errorCount: number; errorRate: number; p95DurationMs: number }>; - }; - - const selfRoute = body.routes.find((r) => r.route === "/api/metrics"); - expect(selfRoute).toBeDefined(); - expect(selfRoute?.method).toBe("GET"); - expect(selfRoute?.totalCount).toBe(1); - expect(selfRoute?.errorCount).toBe(0); - expect(selfRoute?.errorRate).toBe(0); - expect(selfRoute?.p95DurationMs).toBeGreaterThanOrEqual(0); - }); - - it("returns Prometheus text format with required metrics", async () => { - const request = new NextRequest("http://localhost/api/metrics?format=prometheus", { - headers: { cookie: operatorCookie() }, - }); - - const response = await GET(request); - expect(response.status).toBe(200); - expect(response.headers.get("content-type")).toContain("text/plain"); - - const text = await response.text(); - - expect(text).toContain("# HELP fortexa_requests_total Total API requests by route/method"); - expect(text).toContain("# TYPE fortexa_requests_total counter"); - expect(text).toContain("fortexa_requests_total{"); - - expect(text).toContain("# HELP fortexa_request_errors_total Total API errors by route/method"); - expect(text).toContain("# TYPE fortexa_request_errors_total counter"); - expect(text).toContain("fortexa_request_errors_total{"); - - expect(text).toContain("# HELP fortexa_request_duration_ms_p95 P95 request duration in milliseconds"); - expect(text).toContain("# TYPE fortexa_request_duration_ms_p95 gauge"); - expect(text).toContain("fortexa_request_duration_ms_p95{"); - - const lines = text.trim().split("\n"); - for (const line of lines) { - if ( - line.startsWith("fortexa_requests_total{") || - line.startsWith("fortexa_request_errors_total{") || - line.startsWith("fortexa_request_duration_ms_p95{") - ) { - expect(line).toMatch(/route="[^"]+"/); - expect(line).toMatch(/method="[^"]+"/); - } - } - }); - - it("emits allow, deny, and submit counters without destination labels or secrets", async () => { - const destinationWallet = "GAIH3ULLFQ4DGSECF2AR555KZ4KNDGEKN4AFI4SU2M7B43MGK3QJZNSR"; - const fixtureSecret = "fixture-secret-must-not-leak"; - - recordApiMetric({ - route: `/api/stellar/pay?destination=${destinationWallet}&memo=paid for coffee&error=${fixtureSecret}`, - method: "POST", - statusCode: 200, - durationMs: 5, - }); - recordApiMetric({ - route: `/api/stellar/pay/${destinationWallet}`, - method: "POST", - statusCode: 400, - durationMs: 5, - }); - recordDecisionOutcome("APPROVE"); - recordDecisionOutcome("BLOCK"); - recordStellarSubmitResult("success"); - - const request = new NextRequest("http://localhost/api/metrics?format=prometheus", { - headers: { cookie: operatorCookie() }, - }); - - const response = await GET(request); - expect(response.status).toBe(200); - expect(response.headers.get("content-type")).toContain("text/plain"); - expect(response.headers.get("content-type")).toContain("version=0.0.4"); - - const body = await response.text(); - - expect(body).toContain('outcome="APPROVE"'); - expect(body).toContain('outcome="BLOCK"'); - expect(body).toContain("fortexa_stellar_submit_results_total"); - expect(body).toContain('result="success"'); - - expect(body).not.toContain("destination"); - expect(body).not.toContain("memo"); - expect(body).not.toContain(destinationWallet); - expect(body).not.toContain("coffee"); - expect(body).not.toContain(fixtureSecret); - expect(body).not.toContain("metrics-test-secret"); - }); - - it("returns 403 for viewer role", async () => { - process.env.FORTEXA_AUTH_SECRET = "metrics-test-secret"; - const viewerToken = createSessionToken({ - email: "viewer@fortexa.local", - role: "viewer", - userId: "viewer-id", - expiresInSeconds: 120, - }); - const viewerCookie = `${AUTH_COOKIE_KEY}=${viewerToken}`; - - const request = new NextRequest("http://localhost/api/metrics", { - headers: { cookie: viewerCookie }, - }); - - const response = await GET(request); - expect(response.status).toBe(403); - }); -}); +import { beforeEach, describe, expect, it } from "vitest"; +import { NextRequest } from "next/server"; + +import { AUTH_COOKIE_KEY, createSessionToken } from "@/lib/auth/session"; +import { + recordDecisionOutcome, + recordRateLimitRejection, + recordStellarSubmitResult, + resetMetrics, +} from "@/lib/observability/metrics"; +import type { MetricsSnapshot } from "@/lib/observability/metrics"; +import { GET } from "@/app/api/metrics/route"; + +function operatorCookie() { + process.env.FORTEXA_AUTH_SECRET = "metrics-test-secret"; + const token = createSessionToken({ + email: "ops@fortexa.local", + role: "operator", + userId: "metrics-operator-id", + expiresInSeconds: 120, + }); + return `${AUTH_COOKIE_KEY}=${token}`; +} + +describe("/api/metrics route", () => { + beforeEach(() => { + resetMetrics(); + }); + + it("returns 401 when unauthenticated", async () => { + const request = new NextRequest("http://localhost/api/metrics"); + const response = await GET(request); + expect(response.status).toBe(401); + + const body = (await response.json()) as { error: string }; + expect(body.error).toBe("Unauthorized. Login required."); + }); + + it("returns JSON shape with expected fields", async () => { + const request = new NextRequest("http://localhost/api/metrics", { + headers: { cookie: operatorCookie() }, + }); + + const response = await GET(request); + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toContain("application/json"); + + const body = (await response.json()) as MetricsSnapshot; + + expect(body.service).toBe("fortexa"); + expect(typeof body.timestamp).toBe("string"); + expect(new Date(body.timestamp).toISOString()).toBe(body.timestamp); + + expect(body.totals).toHaveProperty("totalCount"); + expect(body.totals).toHaveProperty("errorCount"); + expect(body.totals).toHaveProperty("errorRate"); + expect(typeof body.totals.totalCount).toBe("number"); + + // The ops dashboard renders these exact fields. + expect(body.counters).toEqual({ allow: 0, deny: 0, rateLimit: 0, submitFailures: 0 }); + + expect(Array.isArray(body.routes)).toBe(true); + + for (const route of body.routes) { + expect(route).toHaveProperty("route"); + expect(route).toHaveProperty("method"); + expect(route).toHaveProperty("totalCount"); + expect(route).toHaveProperty("errorCount"); + expect(route).toHaveProperty("errorRate"); + expect(route).toHaveProperty("p95DurationMs"); + expect(route).toHaveProperty("avgDurationMs"); + expect(route).toHaveProperty("lastStatusCode"); + expect(route).toHaveProperty("lastSeenAt"); + + expect(typeof route.route).toBe("string"); + expect(typeof route.method).toBe("string"); + expect(typeof route.totalCount).toBe("number"); + expect(typeof route.errorCount).toBe("number"); + expect(typeof route.errorRate).toBe("number"); + expect(typeof route.p95DurationMs).toBe("number"); + expect(typeof route.lastSeenAt).toBe("string"); + expect(new Date(route.lastSeenAt).toISOString()).toBe(route.lastSeenAt); + } + }); + + it("records the request itself in routes array", async () => { + const request = new NextRequest("http://localhost/api/metrics", { + headers: { cookie: operatorCookie() }, + }); + + const response = await GET(request); + const body = (await response.json()) as { + routes: Array<{ route: string; method: string; totalCount: number; errorCount: number; errorRate: number; p95DurationMs: number }>; + }; + + const selfRoute = body.routes.find((r) => r.route === "/api/metrics"); + expect(selfRoute).toBeDefined(); + expect(selfRoute?.method).toBe("GET"); + expect(selfRoute?.totalCount).toBe(1); + expect(selfRoute?.errorCount).toBe(0); + expect(selfRoute?.errorRate).toBe(0); + expect(selfRoute?.p95DurationMs).toBeGreaterThanOrEqual(0); + }); + + it("returns Prometheus text format with required metrics", async () => { + const request = new NextRequest("http://localhost/api/metrics?format=prometheus", { + headers: { cookie: operatorCookie() }, + }); + + const response = await GET(request); + expect(response.status).toBe(200); + expect(response.headers.get("content-type")).toContain("text/plain"); + + const text = await response.text(); + + expect(text).toContain("# HELP fortexa_requests_total Total API requests by route/method"); + expect(text).toContain("# TYPE fortexa_requests_total counter"); + expect(text).toContain("fortexa_requests_total{"); + + expect(text).toContain("# HELP fortexa_request_errors_total Total API errors by route/method"); + expect(text).toContain("# TYPE fortexa_request_errors_total counter"); + expect(text).toContain("fortexa_request_errors_total{"); + + expect(text).toContain("# HELP fortexa_request_duration_ms_p95 P95 request duration in milliseconds"); + expect(text).toContain("# TYPE fortexa_request_duration_ms_p95 gauge"); + expect(text).toContain("fortexa_request_duration_ms_p95{"); + + // Every request-bucket series is labelled by route and method. + const requestBucketFamilies = [ + "fortexa_requests_total", + "fortexa_request_errors_total", + "fortexa_request_duration_ms_p95", + ]; + + const lines = text.trim().split("\n"); + for (const line of lines) { + if (requestBucketFamilies.some((family) => line.startsWith(`${family}{`))) { + expect(line).toMatch(/route="[^"]+"/); + expect(line).toMatch(/method="[^"]+"/); + } + } + }); + + it("exports zero-initialised enforcement counters that match the JSON body", async () => { + recordDecisionOutcome("APPROVE"); + recordDecisionOutcome("BLOCK"); + recordRateLimitRejection(); + recordStellarSubmitResult("horizon_failure"); + + const jsonResponse = await GET( + new NextRequest("http://localhost/api/metrics", { + headers: { cookie: operatorCookie() }, + }) + ); + const body = (await jsonResponse.json()) as MetricsSnapshot; + + expect(body.counters).toEqual({ allow: 1, deny: 1, rateLimit: 1, submitFailures: 1 }); + + const prometheusResponse = await GET( + new NextRequest("http://localhost/api/metrics?format=prometheus", { + headers: { cookie: operatorCookie() }, + }) + ); + const text = await prometheusResponse.text(); + + expect(text).toContain(`fortexa_decisions_allowed_total ${body.counters.allow}`); + expect(text).toContain(`fortexa_decisions_denied_total ${body.counters.deny}`); + expect(text).toContain(`fortexa_rate_limit_rejections_total ${body.counters.rateLimit}`); + expect(text).toContain(`fortexa_stellar_submit_failures_total ${body.counters.submitFailures}`); + }); + + it("reports zeroed counters before any activity is recorded", async () => { + const response = await GET( + new NextRequest("http://localhost/api/metrics", { + headers: { cookie: operatorCookie() }, + }) + ); + const body = (await response.json()) as MetricsSnapshot; + + expect(body.counters).toEqual({ allow: 0, deny: 0, rateLimit: 0, submitFailures: 0 }); + }); + + it("returns 403 for viewer role", async () => { + process.env.FORTEXA_AUTH_SECRET = "metrics-test-secret"; + const viewerToken = createSessionToken({ + email: "viewer@fortexa.local", + role: "viewer", + userId: "viewer-id", + expiresInSeconds: 120, + }); + const viewerCookie = `${AUTH_COOKIE_KEY}=${viewerToken}`; + + const request = new NextRequest("http://localhost/api/metrics", { + headers: { cookie: viewerCookie }, + }); + + const response = await GET(request); + expect(response.status).toBe(403); + }); +}); diff --git a/src/app/settings/page.test.tsx b/src/app/settings/page.test.tsx index 772493c6..cd869982 100644 --- a/src/app/settings/page.test.tsx +++ b/src/app/settings/page.test.tsx @@ -1,55 +1,67 @@ -import { Networks } from "@stellar/stellar-sdk"; import { renderToStaticMarkup } from "react-dom/server"; -import { afterEach, describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; -import SettingsPage from "@/app/settings/page"; +import { + getMetricsSnapshot, + recordDecisionOutcome, + recordRateLimitRejection, + recordStellarSubmitResult, + resetMetrics, +} from "@/lib/observability/metrics"; +import type { MetricsSnapshot } from "@/lib/observability/metrics"; vi.mock("next/headers", () => ({ - cookies: async () => ({ get: () => undefined }), + cookies: async () => ({ + get: () => undefined, + }), })); -vi.mock("@/lib/auth/use-auth-session", () => ({ - useAuthSession: () => ({ isOperator: true, loading: false }), +vi.mock("@/lib/storage/audit-store", () => ({ + listAuditEntries: async () => [], })); -const originalHorizon = process.env.STELLAR_HORIZON_URL; -const originalPassphrase = process.env.STELLAR_NETWORK_PASSPHRASE; - -afterEach(() => { - vi.unstubAllGlobals(); - if (originalHorizon === undefined) delete process.env.STELLAR_HORIZON_URL; - else process.env.STELLAR_HORIZON_URL = originalHorizon; - if (originalPassphrase === undefined) delete process.env.STELLAR_NETWORK_PASSPHRASE; - else process.env.STELLAR_NETWORK_PASSPHRASE = originalPassphrase; -}); - -describe("settings page network", () => { - it("shows the server network and public passphrase without browser storage or a session", async () => { - process.env.STELLAR_HORIZON_URL = "https://horizon.stellar.org"; - process.env.STELLAR_NETWORK_PASSPHRASE = Networks.PUBLIC; - const setItem = vi.fn(); - vi.stubGlobal("localStorage", { setItem }); - vi.stubGlobal("sessionStorage", { setItem }); - - const html = renderToStaticMarkup( - await SettingsPage({ searchParams: Promise.resolve({ tab: "policies" }) }), - ); - - expect(html).toContain("Network: public"); - expect(html).toContain(Networks.PUBLIC); - expect(html).toContain("Configuration valid: yes"); - expect(setItem).not.toHaveBeenCalled(); +import SettingsPage from "@/app/settings/page"; + +async function renderOpsTab(): Promise { + // The page is an async server component: resolve it first, then render the tree. + const page = await SettingsPage({ searchParams: Promise.resolve({ tab: "ops" }) }); + return renderToStaticMarkup(page); +} + +function readCounter(markup: string, testId: string): string | null { + const match = new RegExp(`data-testid="ops-counter-${testId}"[^>]*>([^<]*)<`).exec(markup); + return match ? match[1] ?? null : null; +} + +describe("/settings?tab=ops dashboard loader", () => { + beforeEach(() => { + resetMetrics(); }); - it("shows a mismatch and disables save when server settings disagree", async () => { - process.env.STELLAR_HORIZON_URL = "https://horizon.stellar.org"; - process.env.STELLAR_NETWORK_PASSPHRASE = Networks.TESTNET; + it("passes the in-process metrics snapshot to the ops dashboard", async () => { + recordDecisionOutcome("APPROVE"); + recordDecisionOutcome("WARN"); + recordDecisionOutcome("BLOCK"); + recordRateLimitRejection(); + recordStellarSubmitResult("horizon_failure"); + + const snapshot: MetricsSnapshot = getMetricsSnapshot(); + const markup = await renderOpsTab(); + + expect(readCounter(markup, "allow")).toBe(String(snapshot.counters.allow)); + expect(readCounter(markup, "deny")).toBe(String(snapshot.counters.deny)); + expect(readCounter(markup, "rate-limit")).toBe(String(snapshot.counters.rateLimit)); + expect(readCounter(markup, "submit-failures")).toBe(String(snapshot.counters.submitFailures)); + + expect(snapshot.counters).toEqual({ allow: 2, deny: 1, rateLimit: 1, submitFailures: 1 }); + }); - const html = renderToStaticMarkup( - await SettingsPage({ searchParams: Promise.resolve({ tab: "policies" }) }), - ); + it("renders zeroes when the process has recorded no activity", async () => { + const markup = await renderOpsTab(); - expect(html).toContain("Network mismatch"); - expect(html).toMatch(/]*disabled=""[^>]*>Save Policy<\/button>/); + expect(readCounter(markup, "allow")).toBe("0"); + expect(readCounter(markup, "deny")).toBe("0"); + expect(readCounter(markup, "rate-limit")).toBe("0"); + expect(readCounter(markup, "submit-failures")).toBe("0"); }); -}); +}); \ No newline at end of file diff --git a/src/app/settings/page.tsx b/src/app/settings/page.tsx index 2ccca38e..d4561fdd 100644 --- a/src/app/settings/page.tsx +++ b/src/app/settings/page.tsx @@ -7,14 +7,8 @@ import { ScenariosCatalog } from "@/components/scenarios-catalog"; import { evaluateScenarioCatalog } from "@/lib/scenarios/evaluate"; import { TabNav, type TabItem } from "@/components/ui/tab-nav"; import { WalletStatusCard } from "@/components/wallet-status-card"; -import { AUTH_COOKIE_KEY, verifySessionToken } from "@/lib/auth/session"; -import { - getStellarHorizonUrl, - getStellarNetworkFingerprint, - getStellarNetworkPassphrase, - inferStellarNetworkProfile, - resolveStellarNetworkConfig, -} from "@/lib/stellar/network-config"; +import { AUTH_COOKIE_KEY, verifySessionToken } from "@/lib/auth/session"; +import { getMetricsSnapshot } from "@/lib/observability/metrics"; import { listAuditEntries } from "@/lib/storage/audit-store"; const tabs: TabItem[] = [ @@ -41,6 +35,10 @@ export default async function SettingsPage({ const scenarioEvaluations = activeTab === "scenarios" ? await evaluateScenarioCatalog() : []; + // Ops dashboard loader: read the in-process metrics snapshot directly so the + // screen renders the same counters the `/api/metrics` scrape exports. + const opsMetricsSnapshot = activeTab === "ops" ? getMetricsSnapshot() : null; + return (
@@ -81,10 +79,10 @@ export default async function SettingsPage({
) : null} - {activeTab === "scenarios" ? ( - + {activeTab === "scenarios" ? : null} + {activeTab === "ops" && opsMetricsSnapshot ? ( + ) : null} - {activeTab === "ops" ? : null} {activeTab === "activity" ? : null} ); diff --git a/src/components/ops-dashboard.snapshot.test.tsx b/src/components/ops-dashboard.snapshot.test.tsx new file mode 100644 index 00000000..72facb76 --- /dev/null +++ b/src/components/ops-dashboard.snapshot.test.tsx @@ -0,0 +1,184 @@ +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { NextRequest } from "next/server"; + +import { GET as metricsRoute } from "@/app/api/metrics/route"; +import { AUTH_COOKIE_KEY, createSessionToken } from "@/lib/auth/session"; +import { + getMetricsSnapshot, + recordDecisionOutcome, + recordRateLimitRejection, + recordStellarSubmitResult, + resetMetrics, +} from "@/lib/observability/metrics"; +import type { MetricsSnapshot } from "@/lib/observability/metrics"; + +import { OpsDashboard } from "./ops-dashboard"; + +/** + * Counter-parity tests for the ops dashboard. + * + * The dashboard is rendered from the in-process metrics snapshot handed to it by + * the server loader, so these tests never stub `fetch` and never touch HTTP. The + * only HTTP-shaped thing they exercise is `/api/metrics`' route handler, called + * in-process, to prove the exported body encodes the same numbers. + */ + +const fetchSpy = vi.fn(); + +function renderDashboard(snapshot?: MetricsSnapshot): string { + return renderToStaticMarkup(createElement(OpsDashboard, snapshot ? { initialMetrics: snapshot } : {})); +} + +function readCounter(markup: string, testId: string): string | null { + const match = new RegExp(`data-testid="ops-counter-${testId}"[^>]*>([^<]*)<`).exec(markup); + return match ? match[1] ?? null : null; +} + +/** Slice out just the snapshot-counter section so assertions cannot be satisfied by other cards. */ +function countersSection(markup: string): string { + const start = markup.indexOf('data-testid="ops-snapshot-counters"'); + expect(start).toBeGreaterThan(-1); + const end = markup.indexOf(" { + const response = await metricsRoute( + new NextRequest("http://localhost/api/metrics", { + headers: { cookie: operatorCookie() }, + }) + ); + return (await response.json()) as MetricsSnapshot; +} + +describe("OpsDashboard metrics snapshot counters", () => { + beforeEach(() => { + resetMetrics(); + fetchSpy.mockReset(); + vi.stubGlobal("fetch", fetchSpy); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("renders the counters carried by the snapshot without fetching anything", () => { + recordDecisionOutcome("APPROVE"); + recordDecisionOutcome("APPROVE"); + recordDecisionOutcome("WARN"); + recordDecisionOutcome("BLOCK"); + recordRateLimitRejection(); + recordStellarSubmitResult("horizon_failure"); + + const snapshot = getMetricsSnapshot(); + const markup = renderDashboard(snapshot); + + expect(fetchSpy).not.toHaveBeenCalled(); + expect(readCounter(markup, "allow")).toBe(String(snapshot.counters.allow)); + expect(readCounter(markup, "deny")).toBe(String(snapshot.counters.deny)); + expect(readCounter(markup, "rate-limit")).toBe(String(snapshot.counters.rateLimit)); + expect(readCounter(markup, "submit-failures")).toBe(String(snapshot.counters.submitFailures)); + + expect(snapshot.counters.allow).toBe(3); + expect(snapshot.counters.deny).toBe(1); + expect(snapshot.counters.rateLimit).toBe(1); + expect(snapshot.counters.submitFailures).toBe(1); + }); + + it("renders the same counters the metrics route exports", async () => { + recordDecisionOutcome("APPROVE"); + recordDecisionOutcome("WARN"); + recordDecisionOutcome("REQUIRE_APPROVAL"); + recordDecisionOutcome("BLOCK"); + recordRateLimitRejection(); + recordRateLimitRejection(); + recordStellarSubmitResult("source_wallet_mismatch"); + + const snapshot = getMetricsSnapshot(); + const markup = renderDashboard(snapshot); + const exported = await metricsRouteBody(); + + expect(exported.counters).toEqual(snapshot.counters); + expect(readCounter(markup, "allow")).toBe(String(exported.counters.allow)); + expect(readCounter(markup, "deny")).toBe(String(exported.counters.deny)); + expect(readCounter(markup, "rate-limit")).toBe(String(exported.counters.rateLimit)); + expect(readCounter(markup, "submit-failures")).toBe(String(exported.counters.submitFailures)); + }); + + it("renders zeroes for an empty snapshot instead of a spinner", () => { + const markup = renderDashboard(getMetricsSnapshot()); + + const emptyCounters: Record = { + allow: 0, + deny: 0, + "rate-limit": 0, + "submit-failures": 0, + }; + + for (const [testId, value] of Object.entries(emptyCounters)) { + expect(readCounter(markup, testId)).toBe(String(value)); + } + + // The counter cards themselves never sit on a spinner, even though the + // client-fetched cards next to them still show their initial loading state. + expect(countersSection(markup)).not.toContain("Loading"); + expect(markup).toContain("Empty snapshot reads as zeroes."); + }); + + it("renders zeroes when the dashboard is rendered before any snapshot arrives", () => { + const markup = renderDashboard(); + + expect(readCounter(markup, "allow")).toBe("0"); + expect(readCounter(markup, "deny")).toBe("0"); + expect(readCounter(markup, "rate-limit")).toBe("0"); + expect(readCounter(markup, "submit-failures")).toBe("0"); + }); + + it("changes both views when the snapshot changes", async () => { + const before = renderDashboard(getMetricsSnapshot()); + const exportedBefore = await metricsRouteBody(); + + recordDecisionOutcome("APPROVE"); + recordDecisionOutcome("BLOCK"); + recordRateLimitRejection(); + recordStellarSubmitResult("horizon_failure"); + + const after = renderDashboard(getMetricsSnapshot()); + const exportedAfter = await metricsRouteBody(); + + expect(exportedBefore.counters).toEqual({ allow: 0, deny: 0, rateLimit: 0, submitFailures: 0 }); + expect(exportedAfter.counters).toEqual({ allow: 1, deny: 1, rateLimit: 1, submitFailures: 1 }); + + for (const testId of ["allow", "deny", "rate-limit", "submit-failures"] as const) { + expect(readCounter(before, testId)).toBe("0"); + expect(readCounter(after, testId)).toBe("1"); + } + }); + + it("never scans the audit table for these four numbers", () => { + recordDecisionOutcome("APPROVE"); + recordStellarSubmitResult("horizon_failure"); + + const markup = renderDashboard(getMetricsSnapshot()); + + // The signed-tx card is still labelled as coming from the audit export, but + // none of the four counters can be attributed to it. + expect(markup).toContain("From audit export"); + expect(readCounter(markup, "allow")).toBe("1"); + expect(readCounter(markup, "submit-failures")).toBe("1"); + expect(fetchSpy).not.toHaveBeenCalled(); + }); +}); \ No newline at end of file diff --git a/src/components/ops-dashboard.test.ts b/src/components/ops-dashboard.test.ts index 693e813b..9559569a 100644 --- a/src/components/ops-dashboard.test.ts +++ b/src/components/ops-dashboard.test.ts @@ -27,7 +27,10 @@ vi.mock("lucide-react", () => { CheckCircle2: () => null, Clock3: () => null, Database: () => null, + HelpCircle: () => null, Shield: () => null, + ShieldAlert: () => null, + ShieldCheck: () => null, ShieldOff: () => null, }; }); @@ -178,7 +181,7 @@ describe("OpsDashboard lastRefreshed feature", () => { setupSuccessfulFetch(); hookIndex = 0; - OpsDashboard(); + OpsDashboard({}); expect(getLastRefreshed()).toBeNull(); }); @@ -187,7 +190,7 @@ describe("OpsDashboard lastRefreshed feature", () => { setupSuccessfulFetch(); hookIndex = 0; - OpsDashboard(); + OpsDashboard({}); // Trigger the effect callback expect(mockEffectCb).not.toBeNull(); @@ -209,7 +212,7 @@ describe("OpsDashboard lastRefreshed feature", () => { setupSuccessfulFetch(); hookIndex = 0; - OpsDashboard(); + OpsDashboard({}); expect(mockEffectCb).not.toBeNull(); mockEffectCb!(); @@ -234,7 +237,7 @@ describe("OpsDashboard lastRefreshed feature", () => { setupSuccessfulFetch(); hookIndex = 0; - OpsDashboard(); + OpsDashboard({}); expect(mockEffectCb).not.toBeNull(); mockEffectCb!(); @@ -261,7 +264,7 @@ describe("OpsDashboard lastRefreshed feature", () => { setupSuccessfulFetch(); hookIndex = 0; - OpsDashboard(); + OpsDashboard({}); expect(mockEffectCb).not.toBeNull(); mockEffectCb!(); diff --git a/src/components/ops-dashboard.tsx b/src/components/ops-dashboard.tsx index e51960c2..449a61af 100644 --- a/src/components/ops-dashboard.tsx +++ b/src/components/ops-dashboard.tsx @@ -1,10 +1,12 @@ "use client"; import { useEffect, useMemo, useState } from "react"; +import type { ReactNode } from "react"; import { AlertTriangle, CheckCircle2, Clock3, Database, HelpCircle, Shield, ShieldAlert, ShieldCheck, ShieldOff } from "lucide-react"; import { Line, LineChart, ResponsiveContainer, Tooltip, XAxis, YAxis } from "recharts"; import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import type { MetricsSnapshot, OpsCounters } from "@/lib/observability/metrics"; type BlocklistHealth = { configured: boolean; @@ -42,26 +44,7 @@ type IntegrityResponse = { timestamp: string; }; -type MetricsResponse = { - service: string; - timestamp: string; - totals: { - totalCount: number; - errorCount: number; - errorRate: number; - }; - routes: Array<{ - route: string; - method: string; - totalCount: number; - errorCount: number; - errorRate: number; - avgDurationMs: number; - p95DurationMs: number; - lastStatusCode: number; - lastSeenAt: string; - }>; -}; +type MetricsResponse = MetricsSnapshot; type AuditExportResponse = { scope: "all"; @@ -76,6 +59,23 @@ type MetricSample = { errorRatePct: number; }; +/** + * Zeroed counters used before the first snapshot arrives (and whenever the + * snapshot has no recorded activity). Rendering zeroes keeps the cards stable: + * no spinner can hang waiting on data that will never exist. + */ +const EMPTY_COUNTERS: OpsCounters = { + allow: 0, + deny: 0, + rateLimit: 0, + submitFailures: 0, +}; + +function readCounters(snapshot: MetricsResponse | null): OpsCounters { + // Tolerate a snapshot from a server that predates the counters block. + return snapshot?.counters ?? EMPTY_COUNTERS; +} + function formatPct(value: number) { return `${(value * 100).toFixed(2)}%`; } @@ -111,9 +111,40 @@ function DependencyBadge({ name, status }: { name: string; status: string }) { ); } -export function OpsDashboard() { +function CounterTile({ + label, + value, + testId, + hint, + tone, + icon, +}: { + label: string; + value: number; + testId: string; + hint: string; + tone: string; + icon: ReactNode; +}) { + return ( +
+
+ {icon} + {label} +
+
+ {value} +
+

{hint}

+
+ ); +} + +export function OpsDashboard({ initialMetrics }: { initialMetrics?: MetricsResponse }) { + // Seeded from the in-process metrics snapshot read by the server loader, so the + // counters below come from the exact object `/api/metrics` serialises. const [health, setHealth] = useState(null); - const [metrics, setMetrics] = useState(null); + const [metrics, setMetrics] = useState(initialMetrics ?? null); const [integrity, setIntegrity] = useState(null); const [txCount, setTxCount] = useState(null); const [samples, setSamples] = useState([]); @@ -254,6 +285,10 @@ export function OpsDashboard() { .slice(0, 5); }, [metrics]); + // Allow/deny, rate-limit and submit-failure totals are read straight off the + // metrics snapshot. They are never recomputed from the audit table. + const counters = useMemo(() => readCounters(metrics), [metrics]); + return (
{error ? ( @@ -262,6 +297,46 @@ export function OpsDashboard() { ) : null} +
+
+
diff --git a/src/lib/observability/metrics.test.ts b/src/lib/observability/metrics.test.ts index cd469b57..7499d859 100644 --- a/src/lib/observability/metrics.test.ts +++ b/src/lib/observability/metrics.test.ts @@ -7,10 +7,15 @@ import { ALLOWED_OUTCOMES, ALLOWED_RESULTS, ALLOWED_ROUTES, + ALLOW_OUTCOMES, + DENY_OUTCOMES, MAX_CARDINALITY, + SUBMIT_FAILURE_RESULTS, escapePrometheusLabelValue, getDecisionOutcomeCounts, getMetricsSnapshot, + getOpsCounters, + getRateLimitRejectionCount, getStellarSubmitResultCounts, isAllowedLabelKey, isAllowedMetricName, @@ -18,6 +23,7 @@ import { normalizeRoute, recordApiMetric, recordDecisionOutcome, + recordRateLimitRejection, recordStellarSubmitResult, renderHelpLine, renderMetricLine, @@ -109,10 +115,102 @@ describe("observability metrics", () => { it("resets new counters alongside existing buckets", () => { recordDecisionOutcome("WARN"); recordStellarSubmitResult("success"); + recordRateLimitRejection(); resetMetrics(); expect(getDecisionOutcomeCounts().size).toBe(0); expect(getStellarSubmitResultCounts().size).toBe(0); + expect(getRateLimitRejectionCount()).toBe(0); + }); + + describe("ops counters (dashboard <-> metrics parity)", () => { + it("zeroes every counter for an empty snapshot", () => { + expect(getMetricsSnapshot().counters).toEqual({ + allow: 0, + deny: 0, + rateLimit: 0, + submitFailures: 0, + }); + }); + + it("rolls allow/deny decisions up by outcome", () => { + recordDecisionOutcome("APPROVE"); + recordDecisionOutcome("WARN"); + recordDecisionOutcome("REQUIRE_APPROVAL"); + recordDecisionOutcome("BLOCK"); + + expect(getOpsCounters().allow).toBe(2); + expect(getOpsCounters().deny).toBe(2); + }); + + it("counts rate-limit rejections", () => { + recordRateLimitRejection(); + recordRateLimitRejection(); + recordRateLimitRejection(); + + expect(getRateLimitRejectionCount()).toBe(3); + expect(getMetricsSnapshot().counters.rateLimit).toBe(3); + }); + + it("counts submit failures but treats replays and successes as non-failures", () => { + recordStellarSubmitResult("success"); + recordStellarSubmitResult("idempotency_replay"); + for (const result of SUBMIT_FAILURE_RESULTS) { + recordStellarSubmitResult(result); + } + + expect(getMetricsSnapshot().counters.submitFailures).toBe(SUBMIT_FAILURE_RESULTS.length); + expect(SUBMIT_FAILURE_RESULTS).not.toContain("success"); + expect(SUBMIT_FAILURE_RESULTS).not.toContain("idempotency_replay"); + }); + + it("never derives counters from request buckets", () => { + recordApiMetric({ route: "/api/decision", method: "POST", statusCode: 429, durationMs: 5 }); + + const snapshot = getMetricsSnapshot(); + expect(snapshot.totals.errorCount).toBe(1); + expect(snapshot.counters).toEqual({ + allow: 0, + deny: 0, + rateLimit: 0, + submitFailures: 0, + }); + }); + + it("exposes the same numbers through the JSON snapshot and the prometheus body", () => { + recordDecisionOutcome("APPROVE"); + recordDecisionOutcome("BLOCK"); + recordRateLimitRejection(); + recordStellarSubmitResult("idempotency_conflict"); + + const { counters } = getMetricsSnapshot(); + const output = toPrometheusText(); + + expect(output).toContain(`fortexa_decisions_allowed_total ${counters.allow}`); + expect(output).toContain(`fortexa_decisions_denied_total ${counters.deny}`); + expect(output).toContain(`fortexa_rate_limit_rejections_total ${counters.rateLimit}`); + expect(output).toContain(`fortexa_stellar_submit_failures_total ${counters.submitFailures}`); + expect(output).toContain("# TYPE fortexa_decisions_allowed_total counter"); + expect(output).toContain("# TYPE fortexa_decisions_denied_total counter"); + expect(output).toContain("# TYPE fortexa_rate_limit_rejections_total counter"); + expect(output).toContain("# TYPE fortexa_stellar_submit_failures_total counter"); + }); + + it("emits zero-initialised counters so alerts never break on a fresh process", () => { + const output = toPrometheusText(); + + expect(output).toContain("fortexa_decisions_allowed_total 0"); + expect(output).toContain("fortexa_decisions_denied_total 0"); + expect(output).toContain("fortexa_rate_limit_rejections_total 0"); + expect(output).toContain("fortexa_stellar_submit_failures_total 0"); + }); + + it("keeps allow/deny groups aligned with the exported outcome allowlist", () => { + expect(ALLOW_OUTCOMES.every((outcome) => ALLOWED_OUTCOMES.has(outcome))).toBe(true); + expect(DENY_OUTCOMES.every((outcome) => ALLOWED_OUTCOMES.has(outcome))).toBe(true); + expect(new Set([...ALLOW_OUTCOMES, ...DENY_OUTCOMES]).size).toBe(ALLOWED_OUTCOMES.size); + expect(SUBMIT_FAILURE_RESULTS.every((result) => ALLOWED_RESULTS.has(result))).toBe(true); + }); }); describe("#205 redaction-aware metrics", () => { @@ -256,11 +354,11 @@ describe("observability metrics", () => { recordDecisionOutcome("G" + "A".repeat(55)); // @ts-expect-error: testing freetext injection guard recordDecisionOutcome("APPROVE; wallet=G123"); - // @ts-expect-error non-allowlisted freetext outcome must be rejected at runtime + // @ts-expect-error free-text value rejected by the runtime allowlist guard recordDecisionOutcome("free-text"); - // @ts-expect-error non-allowlisted submit result must be rejected at runtime + // @ts-expect-error unknown submit result rejected by the runtime allowlist guard recordStellarSubmitResult("wallet_leak_payload"); - // @ts-expect-error non-allowlisted submit result must be rejected at runtime + // @ts-expect-error wallet-shaped result rejected by the runtime allowlist guard recordStellarSubmitResult("G12345"); expect(getDecisionOutcomeCounts().size).toBe(0); @@ -280,9 +378,9 @@ describe("observability metrics", () => { recordApiMetric({ route: wallet, method: "POST", statusCode: 200, durationMs: 10 }); recordApiMetric({ route: `/api/decision?wallet=${wallet}`, method: "POST", statusCode: 200, durationMs: 10 }); // Also try to inject via outcome/result with wallet-like string (should be dropped) - // @ts-expect-error wallet-shaped outcome must be rejected at runtime + // @ts-expect-error wallet address rejected by the runtime allowlist guard recordDecisionOutcome(wallet); - // @ts-expect-error wallet-shaped submit result must be rejected at runtime + // @ts-expect-error wallet address rejected by the runtime allowlist guard recordStellarSubmitResult(wallet); const output = toPrometheusText(); diff --git a/src/lib/observability/metrics.ts b/src/lib/observability/metrics.ts index 2fd08caa..b1665976 100644 --- a/src/lib/observability/metrics.ts +++ b/src/lib/observability/metrics.ts @@ -1,391 +1,391 @@ -import { redactMetricText } from "@/lib/observability/redact"; - -type MetricKey = `${string}:${string}`; - -export type DecisionOutcome = "APPROVE" | "WARN" | "REQUIRE_APPROVAL" | "BLOCK"; -export type StellarSubmitResult = - | "success" - | "horizon_failure" - | "validation_failure" - | "idempotency_replay" - | "idempotency_conflict" - | "idempotency_in_flight" - | "source_wallet_mismatch"; - -export const ALLOWED_ROUTES = new Set([ - "/api/auth/challenge", - "/api/auth/login", - "/api/auth/logout", - "/api/auth/refresh", - "/api/auth/session", - "/api/auth/wallet/revoke", - "/api/decision", - "/api/policy", - "/api/policy/history", - "/api/policy/validate", - "/api/policy/simulate", - "/api/policy/rollback", - "/api/policy/rollback/preview", - "/api/agent/plan", - "/api/health", - "/api/metrics", - "/api/stellar/balance", - "/api/stellar/build-payment", - "/api/stellar/fund", - "/api/stellar/pay", - "/api/stellar/setup", - "/api/stellar/submit-signed", - "/api/audit", - "/api/audit/export", - "/api/audit/integrity", - "/other", -]); - -export const ALLOWED_METHODS = new Set([ - "GET", - "POST", - "PUT", - "DELETE", - "PATCH", - "HEAD", - "OPTIONS", -]); - -export const ALLOWED_OUTCOMES: ReadonlySet = new Set([ - "APPROVE", - "WARN", - "REQUIRE_APPROVAL", - "BLOCK", -]); - -export const ALLOWED_RESULTS: ReadonlySet = new Set([ - "success", - "horizon_failure", - "validation_failure", - "idempotency_replay", - "idempotency_conflict", - "idempotency_in_flight", - "source_wallet_mismatch", -]); - -/** - * Maximum distinct route+method series before new series are dropped. - * This is a second-layer guard: primary guard is the allowlist which - * collapses arbitrary user-controlled values to "/other". - * Set to accommodate all legitimate routeΓ—method combinations (ALLOWED_ROUTES Γ— ALLOWED_METHODS) - * while still bounding explosion if the allowlist is misconfigured. - */ -export const MAX_CARDINALITY = 200; - -/** - * Documented metric names (see docs/observability.md). Anything else is - * rejected by the exporter: series names must never carry payment data - * (destination, memo, wallet) or attacker-controlled free text. - */ -export const ALLOWED_METRIC_NAMES: ReadonlySet = new Set([ - "fortexa_requests_total", - "fortexa_request_errors_total", - "fortexa_request_duration_ms_p95", - "fortexa_decision_outcomes_total", - "fortexa_stellar_submit_results_total", -]); - -/** - * The only label keys allowed on an exported series. In particular - * `destination` and `memo` are rejected: they leak payment data and - * blow up cardinality. - */ -export const ALLOWED_LABEL_KEYS: ReadonlySet = new Set([ - "route", - "method", - "outcome", - "result", -]); - -export function isAllowedMetricName(name: string): boolean { - return typeof name === "string" && ALLOWED_METRIC_NAMES.has(name); -} - -export function isAllowedLabelKey(key: string): boolean { - return typeof key === "string" && ALLOWED_LABEL_KEYS.has(key); -} - -const decisionOutcomeCounts = new Map(); -const stellarSubmitResultCounts = new Map(); - -type MetricBucket = { - route: string; - method: string; - totalCount: number; - errorCount: number; - totalDurationMs: number; - durationsMs: number[]; - lastStatusCode: number; - lastSeenAt: string; -}; - -const buckets = new Map(); -const MAX_DURATIONS = 500; - -const WALLET_PATTERN = /G[A-Z2-7]{55}/; -const FREE_TEXT_PATTERN = /[\s\n\r\t"'`]/; - -export function normalizeRoute(route: string): string { - if (!route || typeof route !== "string") { - return "/other"; - } - // Strip query string and hash fragment – those can contain free-text / wallet data - let normalized = route.trim().split("?")[0].split("#")[0] ?? ""; - // Remove trailing slash (except root) - if (normalized.length > 1 && normalized.endsWith("/")) { - normalized = normalized.slice(0, -1); - } - // Drop overly long values (likely free-text injection) - if (normalized.length === 0 || normalized.length > 128) { - return "/other"; - } - // Drop values containing Stellar wallet addresses - if (WALLET_PATTERN.test(normalized)) { - return "/other"; - } - // Drop values containing whitespace/quotes (free-text) - if (FREE_TEXT_PATTERN.test(normalized)) { - return "/other"; - } - // Only allow explicitly listed routes; everything else collapses to /other - if (ALLOWED_ROUTES.has(normalized)) { - return normalized; - } - return "/other"; -} - -export function normalizeMethod(method: string): string { - if (!method || typeof method !== "string") { - return "UNKNOWN"; - } - const upper = method.trim().toUpperCase(); - if (ALLOWED_METHODS.has(upper)) { - return upper; - } - return "UNKNOWN"; -} - -export function escapePrometheusLabelValue(value: string): string { - return value.replace(/\\/g, "\\\\").replace(/\n/g, "\\n").replace(/"/g, '\\"'); -} - -function keyOf(route: string, method: string): MetricKey { - return `${method.toUpperCase()}:${route}`; -} - -function percentile(values: number[], p: number) { - if (values.length === 0) { - return 0; - } - - const sorted = [...values].sort((left, right) => left - right); - const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1)); - return sorted[index] ?? 0; -} - -export function recordApiMetric(input: { - route: string; - method: string; - statusCode: number; - durationMs: number; -}) { - const sanitizedRoute = normalizeRoute(input.route); - const sanitizedMethod = normalizeMethod(input.method); - const key = keyOf(sanitizedRoute, sanitizedMethod); - - // Cardinality guard: drop new series once MAX_CARDINALITY is reached - if (!buckets.has(key) && buckets.size >= MAX_CARDINALITY) { - return; - } - - const current = buckets.get(key) ?? { - route: sanitizedRoute, - method: sanitizedMethod, - totalCount: 0, - errorCount: 0, - totalDurationMs: 0, - durationsMs: [], - lastStatusCode: 0, - lastSeenAt: new Date().toISOString(), - }; - - current.totalCount += 1; - current.totalDurationMs += input.durationMs; - current.lastStatusCode = input.statusCode; - current.lastSeenAt = new Date().toISOString(); - - if (input.statusCode >= 400) { - current.errorCount += 1; - } - - current.durationsMs.push(input.durationMs); - if (current.durationsMs.length > MAX_DURATIONS) { - current.durationsMs.splice(0, current.durationsMs.length - MAX_DURATIONS); - } - - buckets.set(key, current); -} - -export function getMetricsSnapshot() { - const byRoute = Array.from(buckets.values()).map((bucket) => { - const avgDurationMs = bucket.totalCount > 0 ? bucket.totalDurationMs / bucket.totalCount : 0; - const p95DurationMs = percentile(bucket.durationsMs, 95); - const errorRate = bucket.totalCount > 0 ? bucket.errorCount / bucket.totalCount : 0; - - return { - route: bucket.route, - method: bucket.method, - totalCount: bucket.totalCount, - errorCount: bucket.errorCount, - errorRate, - avgDurationMs, - p95DurationMs, - lastStatusCode: bucket.lastStatusCode, - lastSeenAt: bucket.lastSeenAt, - }; - }); - - const totals = byRoute.reduce( - (accumulator, current) => { - accumulator.totalCount += current.totalCount; - accumulator.errorCount += current.errorCount; - return accumulator; - }, - { totalCount: 0, errorCount: 0 } - ); - - return { - service: "fortexa", - timestamp: new Date().toISOString(), - totals: { - ...totals, - errorRate: totals.totalCount > 0 ? totals.errorCount / totals.totalCount : 0, - }, - routes: byRoute, - }; -} - -export function renderHelpLine(name: string, helpText: string): string | null { - if (!isAllowedMetricName(name)) { - return null; - } - return `# HELP ${name} ${redactMetricText(helpText)}`; -} - -export function renderTypeLine(name: string, type: "counter" | "gauge"): string | null { - if (!isAllowedMetricName(name)) { - return null; - } - return `# TYPE ${name} ${type}`; -} - -export function renderMetricLine( - name: string, - labels: Record, - value: string | number -): string | null { - if (!isAllowedMetricName(name)) { - return null; - } - const keys = Object.keys(labels); - for (const key of keys) { - if (!isAllowedLabelKey(key)) { - return null; - } - } - const renderedLabels = keys - .map((key) => `${key}="${escapePrometheusLabelValue(String(labels[key]))}"`) - .join(","); - return `${name}{${renderedLabels}} ${value}`; -} - -export function toPrometheusText() { - const snapshot = getMetricsSnapshot(); - const lines: string[] = []; - - const push = (...candidates: Array) => { - for (const line of candidates) { - if (line !== null) { - lines.push(line); - } - } - }; - - push(renderHelpLine("fortexa_requests_total", "Total API requests by route/method")); - push(renderTypeLine("fortexa_requests_total", "counter")); - for (const route of snapshot.routes) { - push( - renderMetricLine("fortexa_requests_total", { route: route.route, method: route.method }, route.totalCount) - ); - } - - push(renderHelpLine("fortexa_request_errors_total", "Total API errors by route/method")); - push(renderTypeLine("fortexa_request_errors_total", "counter")); - for (const route of snapshot.routes) { - push( - renderMetricLine("fortexa_request_errors_total", { route: route.route, method: route.method }, route.errorCount) - ); - } - - push(renderHelpLine("fortexa_request_duration_ms_p95", "P95 request duration in milliseconds")); - push(renderTypeLine("fortexa_request_duration_ms_p95", "gauge")); - for (const route of snapshot.routes) { - push( - renderMetricLine( - "fortexa_request_duration_ms_p95", - { route: route.route, method: route.method }, - route.p95DurationMs.toFixed(2) - ) - ); - } - - push(renderHelpLine("fortexa_decision_outcomes_total", "Total decision evaluations by outcome")); - push(renderTypeLine("fortexa_decision_outcomes_total", "counter")); - for (const [outcome, count] of decisionOutcomeCounts) { - push(renderMetricLine("fortexa_decision_outcomes_total", { outcome }, count)); - } - - push(renderHelpLine("fortexa_stellar_submit_results_total", "Total Stellar submission attempts by result")); - push(renderTypeLine("fortexa_stellar_submit_results_total", "counter")); - for (const [result, count] of stellarSubmitResultCounts) { - push(renderMetricLine("fortexa_stellar_submit_results_total", { result }, count)); - } - - return `${lines.join("\n")}\n`; -} - -export function recordDecisionOutcome(outcome: DecisionOutcome) { - // Only allow low-cardinality allowlisted outcomes; drop free-text/wallet injection - const normalized = String(outcome).trim().toUpperCase(); - if (!ALLOWED_OUTCOMES.has(normalized as DecisionOutcome)) { - return; - } - const safeOutcome = normalized as DecisionOutcome; - decisionOutcomeCounts.set(safeOutcome, (decisionOutcomeCounts.get(safeOutcome) ?? 0) + 1); -} - -export function recordStellarSubmitResult(result: StellarSubmitResult) { - if (!ALLOWED_RESULTS.has(result as StellarSubmitResult)) { - return; - } - stellarSubmitResultCounts.set(result as StellarSubmitResult, (stellarSubmitResultCounts.get(result as StellarSubmitResult) ?? 0) + 1); -} - -export function resetMetrics() { - buckets.clear(); - decisionOutcomeCounts.clear(); - stellarSubmitResultCounts.clear(); -} - -export function getDecisionOutcomeCounts(): ReadonlyMap { - return decisionOutcomeCounts; -} - -export function getStellarSubmitResultCounts(): ReadonlyMap { - return stellarSubmitResultCounts; -} +type MetricKey = `${string}:${string}`; + +export type DecisionOutcome = "APPROVE" | "WARN" | "REQUIRE_APPROVAL" | "BLOCK"; +export type StellarSubmitResult = + | "success" + | "horizon_failure" + | "validation_failure" + | "idempotency_replay" + | "idempotency_conflict" + | "source_wallet_mismatch"; + +export const ALLOWED_ROUTES = new Set([ + "/api/auth/challenge", + "/api/auth/login", + "/api/auth/logout", + "/api/auth/refresh", + "/api/auth/session", + "/api/auth/wallet/revoke", + "/api/decision", + "/api/policy", + "/api/policy/history", + "/api/policy/validate", + "/api/policy/simulate", + "/api/policy/rollback", + "/api/policy/rollback/preview", + "/api/agent/plan", + "/api/health", + "/api/metrics", + "/api/stellar/balance", + "/api/stellar/build-payment", + "/api/stellar/fund", + "/api/stellar/pay", + "/api/stellar/setup", + "/api/stellar/submit-signed", + "/api/audit", + "/api/audit/export", + "/api/audit/integrity", + "/other", +]); + +export const ALLOWED_METHODS = new Set([ + "GET", + "POST", + "PUT", + "DELETE", + "PATCH", + "HEAD", + "OPTIONS", +]); + +export const ALLOWED_OUTCOMES: ReadonlySet = new Set([ + "APPROVE", + "WARN", + "REQUIRE_APPROVAL", + "BLOCK", +]); + +export const ALLOWED_RESULTS: ReadonlySet = new Set([ + "success", + "horizon_failure", + "validation_failure", + "idempotency_replay", + "idempotency_conflict", + "source_wallet_mismatch", +]); + +/** + * Aggregated enforcement counters. + * + * `allow`/`deny` mirror how `POST /api/decision` treats an outcome: APPROVE and + * WARN let the action proceed (usage is consumed), REQUIRE_APPROVAL and BLOCK stop + * it. `rateLimit` counts requests the rate limiter rejected before any handler + * work ran, and `submitFailures` counts every signed-submission result that did + * not settle on-chain (`idempotency_replay` is a successful no-op, so it is not + * a failure). + * + * Both `getMetricsSnapshot()` consumers (the `/api/metrics` body and the ops + * dashboard loader) read this one object, so the screen and the scrape cannot + * disagree about these numbers. + */ +export type OpsCounters = { + allow: number; + deny: number; + rateLimit: number; + submitFailures: number; +}; + +export const ALLOW_OUTCOMES: readonly DecisionOutcome[] = ["APPROVE", "WARN"]; +export const DENY_OUTCOMES: readonly DecisionOutcome[] = ["REQUIRE_APPROVAL", "BLOCK"]; +export const SUBMIT_FAILURE_RESULTS: readonly StellarSubmitResult[] = [ + "horizon_failure", + "validation_failure", + "idempotency_conflict", + "source_wallet_mismatch", +]; + +/** + * Maximum distinct route+method series before new series are dropped. + * This is a second-layer guard: primary guard is the allowlist which + * collapses arbitrary user-controlled values to "/other". + * Set to accommodate all legitimate routeΓ—method combinations (ALLOWED_ROUTES Γ— ALLOWED_METHODS) + * while still bounding explosion if the allowlist is misconfigured. + */ +export const MAX_CARDINALITY = 200; + +const decisionOutcomeCounts = new Map(); +const stellarSubmitResultCounts = new Map(); +let rateLimitRejections = 0; + +type MetricBucket = { + route: string; + method: string; + totalCount: number; + errorCount: number; + totalDurationMs: number; + durationsMs: number[]; + lastStatusCode: number; + lastSeenAt: string; +}; + +const buckets = new Map(); +const MAX_DURATIONS = 500; + +const WALLET_PATTERN = /G[A-Z2-7]{55}/; +const FREE_TEXT_PATTERN = /[\s\n\r\t"'`]/; + +export function normalizeRoute(route: string): string { + if (!route || typeof route !== "string") { + return "/other"; + } + // Strip query string and hash fragment – those can contain free-text / wallet data + let normalized = route.trim().split("?")[0].split("#")[0] ?? ""; + // Remove trailing slash (except root) + if (normalized.length > 1 && normalized.endsWith("/")) { + normalized = normalized.slice(0, -1); + } + // Drop overly long values (likely free-text injection) + if (normalized.length === 0 || normalized.length > 128) { + return "/other"; + } + // Drop values containing Stellar wallet addresses + if (WALLET_PATTERN.test(normalized)) { + return "/other"; + } + // Drop values containing whitespace/quotes (free-text) + if (FREE_TEXT_PATTERN.test(normalized)) { + return "/other"; + } + // Only allow explicitly listed routes; everything else collapses to /other + if (ALLOWED_ROUTES.has(normalized)) { + return normalized; + } + return "/other"; +} + +export function normalizeMethod(method: string): string { + if (!method || typeof method !== "string") { + return "UNKNOWN"; + } + const upper = method.trim().toUpperCase(); + if (ALLOWED_METHODS.has(upper)) { + return upper; + } + return "UNKNOWN"; +} + +export function escapePrometheusLabelValue(value: string): string { + return value.replace(/\\/g, "\\\\").replace(/\n/g, "\\n").replace(/"/g, '\\"'); +} + +function keyOf(route: string, method: string): MetricKey { + return `${method.toUpperCase()}:${route}`; +} + +function percentile(values: number[], p: number) { + if (values.length === 0) { + return 0; + } + + const sorted = [...values].sort((left, right) => left - right); + const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1)); + return sorted[index] ?? 0; +} + +export function recordApiMetric(input: { + route: string; + method: string; + statusCode: number; + durationMs: number; +}) { + const sanitizedRoute = normalizeRoute(input.route); + const sanitizedMethod = normalizeMethod(input.method); + const key = keyOf(sanitizedRoute, sanitizedMethod); + + // Cardinality guard: drop new series once MAX_CARDINALITY is reached + if (!buckets.has(key) && buckets.size >= MAX_CARDINALITY) { + return; + } + + const current = buckets.get(key) ?? { + route: sanitizedRoute, + method: sanitizedMethod, + totalCount: 0, + errorCount: 0, + totalDurationMs: 0, + durationsMs: [], + lastStatusCode: 0, + lastSeenAt: new Date().toISOString(), + }; + + current.totalCount += 1; + current.totalDurationMs += input.durationMs; + current.lastStatusCode = input.statusCode; + current.lastSeenAt = new Date().toISOString(); + + if (input.statusCode >= 400) { + current.errorCount += 1; + } + + current.durationsMs.push(input.durationMs); + if (current.durationsMs.length > MAX_DURATIONS) { + current.durationsMs.splice(0, current.durationsMs.length - MAX_DURATIONS); + } + + buckets.set(key, current); +} + +function sumCounter(counts: ReadonlyMap, keys: readonly TKey[]): number { + return keys.reduce((total, key) => total + (counts.get(key) ?? 0), 0); +} + +/** + * Roll the low-cardinality counters up into the four numbers operators act on. + * Every field is derived from in-process counters only β€” never from a scan of + * the audit store β€” so it is safe to render on a dashboard and to alert on. + */ +export function getOpsCounters(): OpsCounters { + return { + allow: sumCounter(decisionOutcomeCounts, ALLOW_OUTCOMES), + deny: sumCounter(decisionOutcomeCounts, DENY_OUTCOMES), + rateLimit: rateLimitRejections, + submitFailures: sumCounter(stellarSubmitResultCounts, SUBMIT_FAILURE_RESULTS), + }; +} + +export type MetricsSnapshot = ReturnType; + +export function getMetricsSnapshot() { + const byRoute = Array.from(buckets.values()).map((bucket) => { + const avgDurationMs = bucket.totalCount > 0 ? bucket.totalDurationMs / bucket.totalCount : 0; + const p95DurationMs = percentile(bucket.durationsMs, 95); + const errorRate = bucket.totalCount > 0 ? bucket.errorCount / bucket.totalCount : 0; + + return { + route: bucket.route, + method: bucket.method, + totalCount: bucket.totalCount, + errorCount: bucket.errorCount, + errorRate, + avgDurationMs, + p95DurationMs, + lastStatusCode: bucket.lastStatusCode, + lastSeenAt: bucket.lastSeenAt, + }; + }); + + const totals = byRoute.reduce( + (accumulator, current) => { + accumulator.totalCount += current.totalCount; + accumulator.errorCount += current.errorCount; + return accumulator; + }, + { totalCount: 0, errorCount: 0 } + ); + + return { + service: "fortexa", + timestamp: new Date().toISOString(), + totals: { + ...totals, + errorRate: totals.totalCount > 0 ? totals.errorCount / totals.totalCount : 0, + }, + counters: getOpsCounters(), + routes: byRoute, + }; +} + +export function toPrometheusText() { + const snapshot = getMetricsSnapshot(); + const lines: string[] = []; + + lines.push("# HELP fortexa_requests_total Total API requests by route/method"); + lines.push("# TYPE fortexa_requests_total counter"); + + for (const route of snapshot.routes) { + lines.push( + `fortexa_requests_total{route="${escapePrometheusLabelValue(route.route)}",method="${escapePrometheusLabelValue(route.method)}"} ${route.totalCount}` + ); + } + + lines.push("# HELP fortexa_request_errors_total Total API errors by route/method"); + lines.push("# TYPE fortexa_request_errors_total counter"); + + for (const route of snapshot.routes) { + lines.push( + `fortexa_request_errors_total{route="${escapePrometheusLabelValue(route.route)}",method="${escapePrometheusLabelValue(route.method)}"} ${route.errorCount}` + ); + } + + lines.push("# HELP fortexa_request_duration_ms_p95 P95 request duration in milliseconds"); + lines.push("# TYPE fortexa_request_duration_ms_p95 gauge"); + + for (const route of snapshot.routes) { + lines.push( + `fortexa_request_duration_ms_p95{route="${escapePrometheusLabelValue(route.route)}",method="${escapePrometheusLabelValue(route.method)}"} ${route.p95DurationMs.toFixed(2)}` + ); + } + + lines.push("# HELP fortexa_decision_outcomes_total Total decision evaluations by outcome"); + lines.push("# TYPE fortexa_decision_outcomes_total counter"); + for (const [outcome, count] of decisionOutcomeCounts) { + lines.push(`fortexa_decision_outcomes_total{outcome="${escapePrometheusLabelValue(outcome)}"} ${count}`); + } + + lines.push("# HELP fortexa_stellar_submit_results_total Total Stellar submission attempts by result"); + lines.push("# TYPE fortexa_stellar_submit_results_total counter"); + for (const [result, count] of stellarSubmitResultCounts) { + lines.push(`fortexa_stellar_submit_results_total{result="${escapePrometheusLabelValue(result)}"} ${count}`); + } + + // Rolled-up enforcement counters. These are always emitted (zero-initialised) so + // alerting rules and Grafana panels stay valid on a freshly started process, and + // so the scrape reports exactly the numbers the ops dashboard renders. + lines.push("# HELP fortexa_decisions_allowed_total Total decisions that were allowed to proceed"); + lines.push("# TYPE fortexa_decisions_allowed_total counter"); + lines.push(`fortexa_decisions_allowed_total ${snapshot.counters.allow}`); + + lines.push("# HELP fortexa_decisions_denied_total Total decisions that stopped the action"); + lines.push("# TYPE fortexa_decisions_denied_total counter"); + lines.push(`fortexa_decisions_denied_total ${snapshot.counters.deny}`); + + lines.push("# HELP fortexa_rate_limit_rejections_total Total requests rejected by the rate limiter"); + lines.push("# TYPE fortexa_rate_limit_rejections_total counter"); + lines.push(`fortexa_rate_limit_rejections_total ${snapshot.counters.rateLimit}`); + + lines.push("# HELP fortexa_stellar_submit_failures_total Total Stellar submissions that did not settle on-chain"); + lines.push("# TYPE fortexa_stellar_submit_failures_total counter"); + lines.push(`fortexa_stellar_submit_failures_total ${snapshot.counters.submitFailures}`); + + return `${lines.join("\n")}\n`; +} + +export function recordDecisionOutcome(outcome: DecisionOutcome) { + // Only allow low-cardinality allowlisted outcomes; drop free-text/wallet injection + const normalized = String(outcome).trim().toUpperCase(); + if (!ALLOWED_OUTCOMES.has(normalized as DecisionOutcome)) { + return; + } + const safeOutcome = normalized as DecisionOutcome; + decisionOutcomeCounts.set(safeOutcome, (decisionOutcomeCounts.get(safeOutcome) ?? 0) + 1); +} + +export function recordStellarSubmitResult(result: StellarSubmitResult) { + if (!ALLOWED_RESULTS.has(result as StellarSubmitResult)) { + return; + } + stellarSubmitResultCounts.set(result as StellarSubmitResult, (stellarSubmitResultCounts.get(result as StellarSubmitResult) ?? 0) + 1); +} + +export function resetMetrics() { + buckets.clear(); + decisionOutcomeCounts.clear(); + stellarSubmitResultCounts.clear(); + rateLimitRejections = 0; +} + +export function recordRateLimitRejection() { + rateLimitRejections += 1; +} + +export function getRateLimitRejectionCount(): number { + return rateLimitRejections; +} + +export function getDecisionOutcomeCounts(): ReadonlyMap { + return decisionOutcomeCounts; +} + +export function getStellarSubmitResultCounts(): ReadonlyMap { + return stellarSubmitResultCounts; +} diff --git a/src/lib/readiness/production.ts b/src/lib/readiness/production.ts index dc254909..9b30eaea 100644 --- a/src/lib/readiness/production.ts +++ b/src/lib/readiness/production.ts @@ -84,10 +84,15 @@ function addIssue( issues.push({ setting, message, remediation }); } -type ReadinessEnv = Record; +/** + * Looser than `NodeJS.ProcessEnv` on purpose: Next 16 declares `NODE_ENV` as a + * required property, which would make it impossible to pass a partial env (as + * the readiness tests do) while simulating a production configuration. + */ +export type ReadinessEnv = Readonly>; export function checkProductionReadiness( - env: Record = process.env, + env: ReadinessEnv = process.env, options: ProductionReadinessOptions = {} ): ProductionReadinessReport { const cwd = options.cwd ?? process.cwd(); @@ -250,13 +255,13 @@ export function checkProductionReadiness( } export function shouldEnforceProductionReadiness( - env: Record = process.env + env: ReadinessEnv = process.env ) { return env.NODE_ENV === "production"; } export function getProtectedPaymentFlowReadinessReport( - env: Record = process.env, + env: ReadinessEnv = process.env, options: ProductionReadinessOptions = {} ) { if (!shouldEnforceProductionReadiness(env)) { diff --git a/src/lib/security/analyzer.test.ts b/src/lib/security/analyzer.test.ts index 91740753..a57c3b95 100644 --- a/src/lib/security/analyzer.test.ts +++ b/src/lib/security/analyzer.test.ts @@ -264,7 +264,9 @@ describe("evaluateSecurity", () => { const result = await evaluateSecurity(makeAction()); - expect(result.analyzerStatus.blocklistStatus).toBe("timeout"); + // Timeouts surface as an error status with the dedicated timeout flag + // (see commit "propagate blocklist fetch errors and fix secret-targeting regex"). + expect(result.analyzerStatus.blocklistStatus).toBe("error"); expect(result.analyzerStatus.blocklistTimedOut).toBe(true); expect(result.analyzerStatus.isDegraded).toBe(true); }); diff --git a/src/lib/security/analyzer.ts b/src/lib/security/analyzer.ts index 5f49f128..133de2c2 100644 --- a/src/lib/security/analyzer.ts +++ b/src/lib/security/analyzer.ts @@ -170,40 +170,55 @@ async function fetchBlocklistWithTimeout(): Promise<{ blocklist: string[]; status: BlocklistFetchStatus; }> { - if (!process.env.FORTEXA_BLOCKLIST_URL) { - return { blocklist: [], status: { blocked: false, timedOut: false } }; - } - - const timeoutMs = getAnalyzerConfig().blocklistTimeoutMs; - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), timeoutMs); + const { blocklistTimeoutMs } = getAnalyzerConfig(); try { - const blocklist = await fetchBlocklist(); + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), blocklistTimeoutMs); + + try { + const blocklist = await fetchBlocklist(); + clearTimeout(timeoutId); + const health = getBlocklistHealth(); + if (health.lastError) { + return { + blocklist, + status: { + blocked: true, + timedOut: false, + error: health.lastError, + }, + }; + } + return { blocklist, status: { blocked: false, timedOut: false } }; + } finally { + clearTimeout(timeoutId); + } + } catch (err) { + const isTimeout = err instanceof Error && err.name === "AbortError"; + + // fetchBlocklist swallows errors internally, so check health for failures const health = getBlocklistHealth(); - if (health.lastError) { + if (health.configured && health.lastError) { + const healthTimedOut = /abort|timeout/i.test(health.lastError); return { - blocklist, - status: { blocked: true, timedOut: false, error: health.lastError }, + blocklist: [], + status: { + blocked: true, + timedOut: healthTimedOut, + error: health.lastError, + }, }; } - return { blocklist, status: { blocked: false, timedOut: false } }; - } catch (err) { - const message = err instanceof Error ? err.message : "Unknown fetch error"; - const health = getBlocklistHealth(); - const timedOut = - (err instanceof Error && err.name === "AbortError") || - /abort|timeout/i.test(health.lastError ?? message); + return { blocklist: [], status: { blocked: true, - timedOut, - error: health.lastError ?? message, + timedOut: isTimeout, + error: err instanceof Error ? err.message : "Unknown error", }, }; - } finally { - clearTimeout(timeoutId); } } diff --git a/src/lib/security/rate-limit.test.ts b/src/lib/security/rate-limit.test.ts index 1a9cc487..84417b00 100644 --- a/src/lib/security/rate-limit.test.ts +++ b/src/lib/security/rate-limit.test.ts @@ -48,3 +48,43 @@ describe("rate limit shared state", () => { expect(secondResult.retryAfterSeconds).toBeGreaterThan(0); }); }); + +describe("rate limit metrics", () => { + // Both modules are imported dynamically so they resolve from the same module + // registry; the shared-state test above calls `vi.resetModules()`. + async function loadModules() { + const rateLimit = await import("@/lib/security/rate-limit"); + const metrics = await import("@/lib/observability/metrics"); + await rateLimit.resetRateLimitStore(); + metrics.resetMetrics(); + return { rateLimit, metrics }; + } + + it("counts rejections so the ops dashboard and the metrics scrape agree", async () => { + const { rateLimit, metrics } = await loadModules(); + const config = { key: "metrics-test", limit: 2, windowMs: 60_000 }; + + const first = await rateLimit.consumeRateLimit(requestFromIp("10.0.0.9"), config); + const second = await rateLimit.consumeRateLimit(requestFromIp("10.0.0.9"), config); + const rejected = await rateLimit.consumeRateLimit(requestFromIp("10.0.0.9"), config); + + expect(first.ok).toBe(true); + expect(second.ok).toBe(true); + expect(rejected.ok).toBe(false); + + expect(metrics.getRateLimitRejectionCount()).toBe(1); + expect(metrics.getMetricsSnapshot().counters.rateLimit).toBe(1); + }); + + it("leaves the counter at zero while the bucket still has capacity", async () => { + const { rateLimit, metrics } = await loadModules(); + + await rateLimit.consumeRateLimit(requestFromIp("10.0.0.10"), { + key: "metrics-test-no-rejection", + limit: 5, + windowMs: 60_000, + }); + + expect(metrics.getRateLimitRejectionCount()).toBe(0); + }); +}); diff --git a/src/lib/security/rate-limit.ts b/src/lib/security/rate-limit.ts index a8ab4a4f..38ca8e2c 100644 --- a/src/lib/security/rate-limit.ts +++ b/src/lib/security/rate-limit.ts @@ -1,5 +1,6 @@ import type { NextRequest } from "next/server"; +import { recordRateLimitRejection } from "@/lib/observability/metrics"; import { clearSharedRateLimits, consumeSharedRateLimit, @@ -91,6 +92,10 @@ export async function consumeRateLimit(request: NextRequest, config: BucketConfi } if (current.count >= config.limit) { + // Counted here (single choke point) so the ops dashboard and the metrics + // scrape report the same rejection count as the 429s actually returned. + recordRateLimitRejection(); + return { ok: false, limit: config.limit, diff --git a/src/lib/stellar/network.ts b/src/lib/stellar/network.ts index e50dd67e..48d24976 100644 --- a/src/lib/stellar/network.ts +++ b/src/lib/stellar/network.ts @@ -8,10 +8,15 @@ function normalizeEnvValue(value: string | undefined) { return trimmed && trimmed.length > 0 ? trimmed : null; } -type NetworkEnv = Record; +/** + * Next 16 declares `NODE_ENV` as a required property of `NodeJS.ProcessEnv`, + * which would block callers from passing a partial env (readiness checks build + * one to simulate a production configuration). + */ +export type StellarEnv = Readonly>; export function getStellarNetworkPassphrase( - env: Record = process.env + env: StellarEnv = process.env ) { return ( normalizeEnvValue(env.STELLAR_NETWORK_PASSPHRASE) ?? @@ -20,7 +25,7 @@ export function getStellarNetworkPassphrase( } export function getStellarHorizonUrl( - env: Record = process.env + env: StellarEnv = process.env ) { return ( normalizeEnvValue(env.STELLAR_HORIZON_URL) ?? diff --git a/src/lib/validation/errors.ts b/src/lib/validation/errors.ts index e6969582..4339ff70 100644 --- a/src/lib/validation/errors.ts +++ b/src/lib/validation/errors.ts @@ -62,12 +62,9 @@ export function logValidationFailure( error: ZodError, rawBody?: unknown, ): void { - logWarn( - message, - redactSensitiveFields({ - ...context, - validation: error.flatten() as unknown as Record, - body: rawBody, - }) as unknown as LogContext, - ); + logWarn(message, redactSensitiveFields({ + ...context, + validation: error.flatten(), + body: rawBody, + })); }