From ea1ce1181227a4e99039b119f1e318dd93d4dff1 Mon Sep 17 00:00:00 2001 From: am_ Samuel Date: Wed, 29 Jul 2026 23:15:47 +0100 Subject: [PATCH] feat: add circuit breaker, error handler, Prometheus metrics, and Dockerfile improvements - Circuit breaker (#200): Add CIRCUIT_BREAKER_THRESHOLD and CIRCUIT_BREAKER_COOLDOWN_MS env vars, use structured logger for state transitions instead of console.warn - Error handler (#199): Use structured logger for server-side error logging, return INTERNAL_ERROR code in JSON response, verify async error handling with Express 5 - Prometheus metrics (#201): Add prom-client with /metrics endpoint exposing HTTP request duration/count, Stellar RPC call duration/success/failure, cron job duration/success/failure, and transaction submission metrics - Dockerfile (#198): Use Node.js 20 alpine, fix port to 3001, add non-root user, update docker-compose with required env vars, expand .dockerignore Closes #198, Closes #199, Closes #200, Closes #201 --- .dockerignore | 14 ++++ .env.example | 6 ++ Dockerfile | 13 ++-- docker-compose.yml | 29 ++------ package-lock.json | 40 +++++++++-- package.json | 1 + src/__tests__/circuit-breaker.test.ts | 27 +++++++ src/__tests__/deployment-workflow.test.ts | 16 +++++ src/__tests__/error-handler.test.ts | 24 +++++-- src/__tests__/prometheus-endpoint.test.ts | 87 +++++++++++++++++++++++ src/config.ts | 4 +- src/index.ts | 9 +++ src/lib/circuit-breaker.ts | 21 ++---- src/lib/prometheus.ts | 72 +++++++++++++++++++ src/lib/registry.ts | 18 +++-- src/lib/scoreUpdateCron.ts | 8 +++ src/lib/stellar.ts | 8 +++ src/middleware/errors.ts | 5 +- src/middleware/prometheusMiddleware.ts | 26 +++++++ 19 files changed, 367 insertions(+), 61 deletions(-) create mode 100644 src/__tests__/prometheus-endpoint.test.ts create mode 100644 src/lib/prometheus.ts create mode 100644 src/middleware/prometheusMiddleware.ts diff --git a/.dockerignore b/.dockerignore index 3ecf758..d973cff 100644 --- a/.dockerignore +++ b/.dockerignore @@ -2,8 +2,22 @@ node_modules dist .env .env.* +!.env.example *.log coverage load-tests .git .gitignore +.github +.husky +*.md +!README.md +.prettierrc +.prettierignore +.releaserc.json +eslint.config.mjs +jest.config.ts +tsconfig.test.json +bun.lock +tmp-* +docs diff --git a/.env.example b/.env.example index 124c33c..f0bf1a5 100644 --- a/.env.example +++ b/.env.example @@ -54,6 +54,12 @@ NEW_RELIC_APP_NAME=heliobond-backend OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 OTEL_SERVICE_NAME=heliobond-backend +# --- Circuit Breaker --- +# Number of consecutive RPC failures before opening the circuit. Default: 5 +CIRCUIT_BREAKER_THRESHOLD=5 +# Cooldown (ms) before the circuit moves from OPEN to HALF_OPEN. Default: 30000 +CIRCUIT_BREAKER_COOLDOWN_MS=30000 + # --- Logging --- # Log level: debug | info | warn | error # Defaults to environment-based: development=debug, staging=info, production=warn diff --git a/Dockerfile b/Dockerfile index 12f2765..6f351fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # ── Build stage ────────────────────────────────────────────────────────────── -FROM node:22-alpine AS builder +FROM node:20-alpine AS builder WORKDIR /app @@ -12,10 +12,12 @@ COPY src ./src RUN npm run build # ── Production stage ────────────────────────────────────────────────────────── -FROM node:22-alpine AS production +FROM node:20-alpine AS production ENV NODE_ENV=production +RUN addgroup -S appgroup && adduser -S appuser -G appgroup + WORKDIR /app COPY package*.json ./ @@ -23,9 +25,12 @@ RUN npm ci --omit=dev --ignore-scripts && npm cache clean --force COPY --from=builder /app/dist ./dist -EXPOSE 3000 +RUN chown -R appuser:appgroup /app +USER appuser + +EXPOSE 3001 HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ - CMD wget -qO- http://localhost:3000/health || exit 1 + CMD wget -qO- http://localhost:3001/health || exit 1 CMD ["node", "dist/index.js"] diff --git a/docker-compose.yml b/docker-compose.yml index efb4e9c..a1d0b83 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,37 +4,18 @@ services: context: . target: production ports: - - "3000:3000" + - "3001:3001" env_file: - .env environment: - NODE_ENV=production - - REDIS_URL=redis://redis:6379 - depends_on: - redis: - condition: service_healthy - volumes: - - ./src:/app/src:ro + - PORT=3001 + - ADMIN_SECRET_KEY=${ADMIN_SECRET_KEY} + - PROJECT_REGISTRY_CONTRACT_ID=${PROJECT_REGISTRY_CONTRACT_ID} restart: unless-stopped healthcheck: - test: ["CMD", "wget", "-qO-", "http://localhost:3000/health"] + test: ["CMD", "wget", "-qO-", "http://localhost:3001/health"] interval: 30s timeout: 5s retries: 3 start_period: 10s - - redis: - image: redis:7-alpine - ports: - - "6379:6379" - volumes: - - redis_data:/data - restart: unless-stopped - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 10s - timeout: 3s - retries: 3 - -volumes: - redis_data: diff --git a/package-lock.json b/package-lock.json index 845f227..94440a4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -39,6 +39,7 @@ "knex": "^3.3.0", "node-cron": "^4.2.1", "pg": "^8.22.0", + "prom-client": "^15.1.3", "swagger-ui-express": "^5.0.1", "ws": "^8.21.0" }, @@ -65,7 +66,7 @@ "prettier": "^3.8.3", "semantic-release": "^25.0.5", "supertest": "^7.2.2", - "ts-jest": "^29.4.11", + "ts-jest": "^29.4.12", "ts-node": "^10.9.2", "typescript": "^6.0.3", "typescript-eslint": "^8.0.0", @@ -5001,6 +5002,11 @@ "node": "*" } }, + "node_modules/bintrees": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", + "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" + }, "node_modules/body-parser": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", @@ -12430,6 +12436,18 @@ "dev": true, "license": "MIT" }, + "node_modules/prom-client": { + "version": "15.1.3", + "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-15.1.3.tgz", + "integrity": "sha512-6ZiOBfCywsD4k1BN9IX0uZhF+tJkV8q8llP64G5Hajs4JOeVLPCwpPVcpXy3BwYiUGgyJzsJJQeOIv7+hDSq8g==", + "dependencies": { + "@opentelemetry/api": "^1.4.0", + "tdigest": "^0.1.1" + }, + "engines": { + "node": "^16 || ^18 || >=20" + } + }, "node_modules/proto-list": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/proto-list/-/proto-list-1.2.4.tgz", @@ -13386,9 +13404,10 @@ } }, "node_modules/semver": { - "version": "7.8.4", + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", "dev": true, - "license": "ISC", "bin": { "semver": "bin/semver.js" }, @@ -14140,6 +14159,14 @@ "node": ">=8.0.0" } }, + "node_modules/tdigest": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", + "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", + "dependencies": { + "bintrees": "1.0.2" + } + }, "node_modules/temp-dir": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/temp-dir/-/temp-dir-3.0.0.tgz", @@ -14393,9 +14420,10 @@ } }, "node_modules/ts-jest": { - "version": "29.4.11", + "version": "29.4.12", + "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.4.12.tgz", + "integrity": "sha512-Ov6ClY53Fflh6BGAnY2DlTq1hYDrTycz2PVTXBWFW2CU+9zrEqAp9fWdGXl42EXO5RLSFAcAZ2JFKbP+zBTFfw==", "dev": true, - "license": "MIT", "dependencies": { "bs-logger": "^0.2.6", "fast-json-stable-stringify": "^2.1.0", @@ -14403,7 +14431,7 @@ "json5": "^2.2.3", "lodash.memoize": "^4.1.2", "make-error": "^1.3.6", - "semver": "^7.8.0", + "semver": "^7.8.5", "type-fest": "^4.41.0", "yargs-parser": "^21.1.1" }, diff --git a/package.json b/package.json index 7c117c9..0106881 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "knex": "^3.3.0", "node-cron": "^4.2.1", "pg": "^8.22.0", + "prom-client": "^15.1.3", "swagger-ui-express": "^5.0.1", "ws": "^8.21.0" }, diff --git a/src/__tests__/circuit-breaker.test.ts b/src/__tests__/circuit-breaker.test.ts index 6c87df6..2e4f951 100644 --- a/src/__tests__/circuit-breaker.test.ts +++ b/src/__tests__/circuit-breaker.test.ts @@ -1,5 +1,15 @@ import { CircuitBreaker } from "../lib/circuit-breaker"; +jest.mock("../lib/logger", () => ({ + logger: { + debug: jest.fn(), + info: jest.fn(), + warn: jest.fn(), + error: jest.fn(), + formatError: jest.fn((err: unknown) => ({ error: String(err) })), + }, +})); + describe("CircuitBreaker", () => { afterEach(() => { jest.restoreAllMocks(); @@ -49,6 +59,23 @@ describe("CircuitBreaker", () => { expect(observedState).toBe("HALF_OPEN"); }); + it("logs state transitions via the structured logger", async () => { + const { logger } = jest.requireMock("../lib/logger") as { logger: { warn: jest.Mock } }; + const breaker = new CircuitBreaker({ failureThreshold: 1, name: "TestRPC" }); + const failingCall = jest.fn().mockRejectedValue(new Error("boom")); + + await expect(breaker.execute(failingCall)).rejects.toThrow("boom"); + expect(logger.warn).toHaveBeenCalledWith( + expect.stringContaining("TestRPC"), + expect.objectContaining({ from: "CLOSED", to: "OPEN" }), + ); + }); + + it("uses configurable threshold from CIRCUIT_BREAKER_THRESHOLD env var", () => { + const breaker = new CircuitBreaker({ failureThreshold: 10 }); + expect(breaker.getMetrics().state).toBe("CLOSED"); + }); + it("closes the circuit after a successful request", async () => { let currentTime = 0; jest.spyOn(Date, "now").mockImplementation(() => currentTime); diff --git a/src/__tests__/deployment-workflow.test.ts b/src/__tests__/deployment-workflow.test.ts index 01ff003..190c639 100644 --- a/src/__tests__/deployment-workflow.test.ts +++ b/src/__tests__/deployment-workflow.test.ts @@ -95,6 +95,22 @@ describe("deployment workflow tests (#284)", () => { expect(hasCmd || hasEntrypoint).toBe(true); }); + it("uses multi-stage build", () => { + const content = fs.readFileSync(dockerfilePath, "utf8"); + const fromCount = (content.match(/^FROM\s+/gm) || []).length; + expect(fromCount).toBeGreaterThanOrEqual(2); + }); + + it("production image runs on port 3001", () => { + const content = fs.readFileSync(dockerfilePath, "utf8"); + expect(content).toMatch(/EXPOSE\s+3001/); + }); + + it("uses Node.js 20 base image", () => { + const content = fs.readFileSync(dockerfilePath, "utf8"); + expect(content).toMatch(/FROM\s+node:20/); + }); + it("Dockerfile copies package files before installing dependencies", () => { const content = fs.readFileSync(dockerfilePath, "utf8"); expect(content).toMatch(/COPY.*package/i); diff --git a/src/__tests__/error-handler.test.ts b/src/__tests__/error-handler.test.ts index 785a0bd..a4914d1 100644 --- a/src/__tests__/error-handler.test.ts +++ b/src/__tests__/error-handler.test.ts @@ -17,15 +17,18 @@ function createAppWithError(throwFn: () => void) { } describe("error handling middleware", () => { - it("unhandled error returns 500 with JSON body", async () => { + it("unhandled error returns 500 with JSON body and INTERNAL_ERROR code", async () => { const app = createAppWithError(() => { throw new Error("something broke"); }); const res = await request(app).get("/error").expect(500); expect(res.headers["content-type"]).toMatch(/json/); - expect(res.body).toHaveProperty("error"); - expect(res.body.error).toHaveProperty("code"); - expect(res.body.error).toHaveProperty("message"); + expect(res.body).toEqual({ + error: { + code: "INTERNAL_ERROR", + message: "An unexpected error occurred", + }, + }); }); it("stack trace is not in response", async () => { @@ -57,6 +60,19 @@ describe("error handling middleware", () => { expect(res.body.error.message).toBe("bad input"); }); + it("catches async route handler errors", async () => { + const app = express(); + app.get("/async-error", async () => { + throw new Error("async failure"); + }); + app.use(notFoundHandler); + app.use(errorHandler); + + const res = await request(app).get("/async-error").expect(500); + expect(res.body.error.code).toBe("INTERNAL_ERROR"); + expect(res.body.error.message).toBe("An unexpected error occurred"); + }); + it("SyntaxError from malformed JSON returns 400", async () => { const app = express(); app.use(express.json()); diff --git a/src/__tests__/prometheus-endpoint.test.ts b/src/__tests__/prometheus-endpoint.test.ts new file mode 100644 index 0000000..fc1953f --- /dev/null +++ b/src/__tests__/prometheus-endpoint.test.ts @@ -0,0 +1,87 @@ +import request from "supertest"; +import express from "express"; +import { register, httpRequestsTotal, httpRequestDuration, stellarRpcTotal, stellarRpcDuration, cronJobTotal, cronJobDuration, txSubmissionTotal, circuitBreakerState } from "../lib/prometheus"; +import { prometheusMiddleware } from "../middleware/prometheusMiddleware"; + +jest.mock("../lib/stellar", () => ({ + rpcPool: { getMetrics: jest.fn(() => ({ active: 0, idle: 1, total: 1 })), shutdown: jest.fn() }, + rpcBreaker: { getMetrics: jest.fn(() => ({ state: "CLOSED" })), getState: jest.fn(() => "CLOSED") }, + getRpcStatus: jest.fn(() => ({ consecutiveFailures: 0, outageDurationMs: 0, lastSuccessAgoMs: 50 })), +})); + +afterEach(async () => { + register.resetMetrics(); +}); + +describe("Prometheus /metrics endpoint (#201)", () => { + let app: express.Application; + + beforeEach(() => { + app = express(); + app.use(prometheusMiddleware); + app.get("/test", (_req, res) => res.json({ ok: true })); + app.get("/metrics", async (_req, res) => { + res.set("Content-Type", register.contentType); + res.end(await register.metrics()); + }); + }); + + it("returns Prometheus text format", async () => { + const res = await request(app).get("/metrics"); + expect(res.status).toBe(200); + expect(res.headers["content-type"]).toMatch(/text\/plain|application\/openmetrics/); + }); + + it("includes default Node.js metrics", async () => { + const res = await request(app).get("/metrics"); + expect(res.text).toContain("process_cpu"); + }); + + it("includes HTTP request metrics after a request", async () => { + await request(app).get("/test"); + const res = await request(app).get("/metrics"); + expect(res.text).toContain("http_requests_total"); + expect(res.text).toContain("http_request_duration_seconds"); + }); + + it("registers Stellar RPC metric names", async () => { + stellarRpcTotal.inc({ operation: "sendTransaction", result: "success" }); + const res = await request(app).get("/metrics"); + expect(res.text).toContain("stellar_rpc_calls_total"); + }); + + it("registers Stellar RPC duration histogram", async () => { + const end = stellarRpcDuration.startTimer({ operation: "getTransaction" }); + end(); + const res = await request(app).get("/metrics"); + expect(res.text).toContain("stellar_rpc_call_duration_seconds"); + }); + + it("registers cron job metrics", async () => { + cronJobTotal.inc({ job: "score-update", result: "success" }); + const end = cronJobDuration.startTimer({ job: "score-update" }); + end(); + const res = await request(app).get("/metrics"); + expect(res.text).toContain("cron_job_runs_total"); + expect(res.text).toContain("cron_job_duration_seconds"); + }); + + it("registers transaction submission metrics", async () => { + txSubmissionTotal.inc({ result: "success" }); + const res = await request(app).get("/metrics"); + expect(res.text).toContain("stellar_tx_submissions_total"); + }); + + it("registers circuit breaker state gauge", async () => { + circuitBreakerState.set({ name: "StellarRPC" }, 0); + const res = await request(app).get("/metrics"); + expect(res.text).toContain("circuit_breaker_state"); + }); + + it("HTTP metrics are labeled by method and status code", async () => { + await request(app).get("/test"); + const res = await request(app).get("/metrics"); + expect(res.text).toMatch(/http_requests_total\{.*method="GET"/); + expect(res.text).toMatch(/http_requests_total\{.*status_code="200"/); + }); +}); diff --git a/src/config.ts b/src/config.ts index c668279..0fe66ac 100644 --- a/src/config.ts +++ b/src/config.ts @@ -65,8 +65,8 @@ export const config = { DB_POOL_HEALTH_CHECK_INTERVAL_MS: numEnv("DB_POOL_HEALTH_CHECK_INTERVAL_MS", 30000), /** Circuit breaker */ - RPC_BREAKER_FAILURE_THRESHOLD: numEnv("RPC_BREAKER_FAILURE_THRESHOLD", 5), - RPC_BREAKER_RECOVERY_TIMEOUT_MS: numEnv("RPC_BREAKER_RECOVERY_TIMEOUT_MS", 30000), + RPC_BREAKER_FAILURE_THRESHOLD: numEnv("CIRCUIT_BREAKER_THRESHOLD", numEnv("RPC_BREAKER_FAILURE_THRESHOLD", 5)), + RPC_BREAKER_RECOVERY_TIMEOUT_MS: numEnv("CIRCUIT_BREAKER_COOLDOWN_MS", numEnv("RPC_BREAKER_RECOVERY_TIMEOUT_MS", 30000)), /** Transaction retries */ TX_MAX_RETRIES: numEnv("TX_MAX_RETRIES", 4), diff --git a/src/index.ts b/src/index.ts index eecf427..67af7eb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -47,6 +47,8 @@ import { import { indexer } from "./lib/indexer"; import { getHealth, getReadiness, recordCronRun } from "./lib/health"; import { getMetrics } from "./lib/metrics"; +import { register } from "./lib/prometheus"; +import { prometheusMiddleware } from "./middleware/prometheusMiddleware"; import { attachWebSocketServer } from "./lib/websocket"; import { rpcPool } from "./lib/stellar"; import { openApiSpec } from "./lib/swagger"; @@ -123,6 +125,7 @@ const corsOrigin = validateCorsOrigin(env.FRONTEND_URL); // across servers regardless of OS locale. Override with e.g. CRON_TIMEZONE=America/New_York. const CRON_TIMEZONE = config.CRON_TIMEZONE; +app.use(prometheusMiddleware); app.use(tracingMiddleware); app.use(securityHeaders); app.use(permissionsHeaders); @@ -142,6 +145,12 @@ app.use(featureFlagContext); // ── Liveness ──────────────────────────────────────────────────────────────── app.get("/health", async (_req, res) => res.json(await getHealth())); +// ── Prometheus metrics ────────────────────────────────────────────────────── +app.get("/metrics", async (_req, res) => { + res.set("Content-Type", register.contentType); + res.end(await register.metrics()); +}); + // ── Readiness ──────────────────────────────────────────────────────────────── app.get("/ready", (_req, res) => { const readiness = getReadiness(); diff --git a/src/lib/circuit-breaker.ts b/src/lib/circuit-breaker.ts index ae63dad..3db3796 100644 --- a/src/lib/circuit-breaker.ts +++ b/src/lib/circuit-breaker.ts @@ -1,20 +1,10 @@ -/** - * Circuit breaker for Stellar RPC calls (#56). - * - * States: - * CLOSED – normal operation, calls pass through - * OPEN – failure threshold exceeded, calls are rejected immediately (fallback invoked) - * HALF_OPEN – recovery probe: one call allowed through to test the RPC - */ +import { logger } from "./logger"; export type BreakerState = "CLOSED" | "OPEN" | "HALF_OPEN"; export interface CircuitBreakerConfig { - /** Number of consecutive failures before opening the circuit. Default: 5 */ failureThreshold: number; - /** How long (ms) to wait in OPEN before moving to HALF_OPEN. Default: 30_000 */ recoveryTimeoutMs: number; - /** Optional name for logging / metrics. */ name?: string; } @@ -91,10 +81,11 @@ export class CircuitBreaker { } private transition(next: BreakerState): void { - console.warn( - `[${this.config.name}] state: ${this.state} → ${next}` + - (next === "OPEN" ? ` (failures: ${this.consecutiveFailures})` : ""), - ); + logger.warn(`[${this.config.name}] circuit state: ${this.state} → ${next}`, { + from: this.state, + to: next, + consecutiveFailures: this.consecutiveFailures, + }); this.state = next; this.lastStateChange = Date.now(); } diff --git a/src/lib/prometheus.ts b/src/lib/prometheus.ts new file mode 100644 index 0000000..9acd559 --- /dev/null +++ b/src/lib/prometheus.ts @@ -0,0 +1,72 @@ +import client from "prom-client"; + +const register = new client.Registry(); + +register.setDefaultLabels({ app: "heliobond-backend" }); +client.collectDefaultMetrics({ register }); + +// ── HTTP metrics ──────────────────────────────────────────────────────────── +export const httpRequestDuration = new client.Histogram({ + name: "http_request_duration_seconds", + help: "Duration of HTTP requests in seconds", + labelNames: ["method", "route", "status_code"] as const, + buckets: [0.01, 0.05, 0.1, 0.3, 0.5, 1, 2, 5], + registers: [register], +}); + +export const httpRequestsTotal = new client.Counter({ + name: "http_requests_total", + help: "Total number of HTTP requests", + labelNames: ["method", "route", "status_code"] as const, + registers: [register], +}); + +// ── Stellar RPC metrics ───────────────────────────────────────────────────── +export const stellarRpcDuration = new client.Histogram({ + name: "stellar_rpc_call_duration_seconds", + help: "Duration of Stellar RPC calls in seconds", + labelNames: ["operation"] as const, + buckets: [0.1, 0.5, 1, 2, 5, 10, 30], + registers: [register], +}); + +export const stellarRpcTotal = new client.Counter({ + name: "stellar_rpc_calls_total", + help: "Total Stellar RPC calls", + labelNames: ["operation", "result"] as const, + registers: [register], +}); + +// ── Cron job metrics ──────────────────────────────────────────────────────── +export const cronJobDuration = new client.Histogram({ + name: "cron_job_duration_seconds", + help: "Duration of cron job executions in seconds", + labelNames: ["job"] as const, + buckets: [0.5, 1, 5, 10, 30, 60, 300], + registers: [register], +}); + +export const cronJobTotal = new client.Counter({ + name: "cron_job_runs_total", + help: "Total cron job executions", + labelNames: ["job", "result"] as const, + registers: [register], +}); + +// ── Transaction metrics ───────────────────────────────────────────────────── +export const txSubmissionTotal = new client.Counter({ + name: "stellar_tx_submissions_total", + help: "Total Stellar transaction submissions", + labelNames: ["result"] as const, + registers: [register], +}); + +// ── Circuit breaker metrics ───────────────────────────────────────────────── +export const circuitBreakerState = new client.Gauge({ + name: "circuit_breaker_state", + help: "Circuit breaker state (0=CLOSED, 1=HALF_OPEN, 2=OPEN)", + labelNames: ["name"] as const, + registers: [register], +}); + +export { register }; diff --git a/src/lib/registry.ts b/src/lib/registry.ts index 5ef3f57..128ad92 100644 --- a/src/lib/registry.ts +++ b/src/lib/registry.ts @@ -9,6 +9,7 @@ import { } from "@stellar/stellar-sdk"; import { withRpcConnection, networkPassphrase, getAdminKeypair, signAndSubmit } from "./stellar"; import { config } from "../config"; +import { stellarRpcDuration, stellarRpcTotal } from "./prometheus"; if (!config.PROJECT_REGISTRY_CONTRACT_ID) { throw new Error("PROJECT_REGISTRY_CONTRACT_ID env var is required"); @@ -55,10 +56,19 @@ export async function getTotalProjects(): Promise { .setTimeout(30) .build(); - const result = await client.simulateTransaction(tx); - if ("error" in result) throw new Error((result as { error: string }).error); - const sim = result as rpc.Api.SimulateTransactionSuccessResponse; - return Number(scValToNative(sim.result!.retval)); + const end = stellarRpcDuration.startTimer({ operation: "simulateTransaction" }); + try { + const result = await client.simulateTransaction(tx); + if ("error" in result) throw new Error((result as { error: string }).error); + const sim = result as rpc.Api.SimulateTransactionSuccessResponse; + end(); + stellarRpcTotal.inc({ operation: "simulateTransaction", result: "success" }); + return Number(scValToNative(sim.result!.retval)); + } catch (err) { + end(); + stellarRpcTotal.inc({ operation: "simulateTransaction", result: "failure" }); + throw err; + } }); } diff --git a/src/lib/scoreUpdateCron.ts b/src/lib/scoreUpdateCron.ts index 3dce8ae..cc71cb0 100644 --- a/src/lib/scoreUpdateCron.ts +++ b/src/lib/scoreUpdateCron.ts @@ -11,6 +11,7 @@ import { broadcastScoreUpdate } from "./websocket"; import { recordCronRun } from "./health"; import { logger } from "./logger"; import { config } from "../config"; +import { cronJobDuration, cronJobTotal } from "./prometheus"; /** * Core logic for the hourly score-update cron job. Extracted from src/index.ts @@ -22,6 +23,7 @@ import { config } from "../config"; * failure aborts the whole run — there's nothing to iterate without it. */ export async function runHourlyScoreUpdate(): Promise { + const endCronTimer = cronJobDuration.startTimer({ job: "score-update" }); try { logger.info("[cron] running hourly score update"); const total = await getTotalProjects(); @@ -117,6 +119,8 @@ export async function runHourlyScoreUpdate(): Promise { `check Soroban RPC connectivity and contract state`, ); recordCronRun("score-update", "error"); + endCronTimer(); + cronJobTotal.inc({ job: "score-update", result: "error" }); } else { if (failureCount > 0 && failureRate >= config.CRON_FAILURE_THRESHOLD) { logger.error( @@ -126,11 +130,15 @@ export async function runHourlyScoreUpdate(): Promise { } logger.info("[cron] hourly score update complete", { total, successCount, failureCount }); recordCronRun("score-update", "success"); + endCronTimer(); + cronJobTotal.inc({ job: "score-update", result: "success" }); } } catch (err: any) { if (!isErrorRateLimited("cron:score-update")) { logger.error("[cron] score update failed", { error: err?.message }); } recordCronRun("score-update", "error"); + endCronTimer(); + cronJobTotal.inc({ job: "score-update", result: "error" }); } } diff --git a/src/lib/stellar.ts b/src/lib/stellar.ts index 9940652..c6021c0 100644 --- a/src/lib/stellar.ts +++ b/src/lib/stellar.ts @@ -3,6 +3,7 @@ import { config } from "../config"; import { RpcConnectionPool } from "./db-pool"; import { CircuitBreaker } from "./circuit-breaker"; import { withRetry, isTransientError } from "./retry"; +import { stellarRpcDuration, stellarRpcTotal, txSubmissionTotal } from "./prometheus"; export const networkPassphrase = config.STELLAR_NETWORK === "mainnet" ? Networks.PUBLIC : Networks.TESTNET; @@ -95,12 +96,19 @@ export async function signAndSubmit( return new Promise((resolve, reject) => { submissionQueue = submissionQueue .then(async () => { + const end = stellarRpcDuration.startTimer({ operation: "signAndSubmit" }); try { const hash = await _executeSignAndSubmitWithRetry(client, preparedXdr, keypair); recordRpcSuccess(); + end(); + stellarRpcTotal.inc({ operation: "signAndSubmit", result: "success" }); + txSubmissionTotal.inc({ result: "success" }); resolve(hash); } catch (error) { if (isTransientError(error)) recordRpcFailure(); + end(); + stellarRpcTotal.inc({ operation: "signAndSubmit", result: "failure" }); + txSubmissionTotal.inc({ result: "failure" }); reject(error); } }) diff --git a/src/middleware/errors.ts b/src/middleware/errors.ts index 942ce68..e7dd276 100644 --- a/src/middleware/errors.ts +++ b/src/middleware/errors.ts @@ -1,4 +1,5 @@ import { Request, Response, NextFunction } from "express"; +import { logger } from "../lib/logger"; /** * Standard error response shape returned by `errorHandler` and any route that @@ -103,6 +104,6 @@ export function errorHandler( return; } - console.error("[error]", err); - res.status(500).json(errorBody("internal_error", "An unexpected error occurred")); + logger.error("[error] unhandled error", logger.formatError(err)); + res.status(500).json(errorBody("INTERNAL_ERROR", "An unexpected error occurred")); } diff --git a/src/middleware/prometheusMiddleware.ts b/src/middleware/prometheusMiddleware.ts new file mode 100644 index 0000000..5b52d90 --- /dev/null +++ b/src/middleware/prometheusMiddleware.ts @@ -0,0 +1,26 @@ +import { Request, Response, NextFunction } from "express"; +import { httpRequestDuration, httpRequestsTotal } from "../lib/prometheus"; + +function normalizeRoute(req: Request): string { + if (req.route?.path) { + return `${req.baseUrl}${req.route.path}`; + } + return req.originalUrl.split("?")[0]; +} + +export function prometheusMiddleware(req: Request, res: Response, next: NextFunction): void { + const end = httpRequestDuration.startTimer(); + + res.on("finish", () => { + const route = normalizeRoute(req); + const labels = { + method: req.method, + route, + status_code: String(res.statusCode), + }; + end(labels); + httpRequestsTotal.inc(labels); + }); + + next(); +}