feat(observability): implement distributed tracing, metrics, log aggr…

.env.example

@@ -54,6 +54,12 @@ THROTTLE_LIMIT=60
 SENTRY_DSN=your_sentry_dsn_here
 GIT_COMMIT_SHA=
 
+# Observability (OpenTelemetry + Loki)
+# Set OTEL_EXPORTER_OTLP_ENDPOINT to enable distributed tracing (e.g. http://localhost:4318/v1/traces)
+OTEL_EXPORTER_OTLP_ENDPOINT=
+# Set LOKI_URL to enable log aggregation (e.g. http://localhost:3100)
+LOKI_URL=
+
 # Frontend
 NEXT_PUBLIC_API_URL=http://localhost:3000
 NEXT_PUBLIC_STELLAR_NETWORK=testnet

apps/backend/package.json

@@ -45,7 +45,20 @@
     "prom-client": "^15.1.0",
     "typeorm": "^0.3.0",
     "class-sanitizer": "^1.0.1",
-    "sanitize-html": "^2.13.0"
+    "sanitize-html": "^2.13.0",
+    "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/auto-instrumentations-node": "^0.51.0",
+    "@opentelemetry/exporter-prometheus": "^0.55.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.55.0",
+    "@opentelemetry/resources": "^1.28.0",
+    "@opentelemetry/sdk-node": "^0.55.0",
+    "@opentelemetry/semantic-conventions": "^1.28.0",
+    "winston-loki": "^6.1.3",
+    "nest-winston": "^1.10.0",
+    "winston": "^3.14.0",
+    "@nestjs/terminus": "^10.2.3",
+    "@nestjs/axios": "^3.0.3",
+    "axios": "^1.7.9"
   },
   "devDependencies": {
     "@eslint/js": "^8.56.0",

apps/backend/src/app.module.ts

@@ -14,6 +14,7 @@ import { NotificationsModule } from './notifications/notifications.module';
 import { LoggerModule } from './common/logger';
 import { HealthModule } from './health/health.module';
 import { MetricsModule } from './metrics/metrics.module';
+import { TracingModule } from './tracing';
 import * as redisStore from 'cache-manager-redis-store';
 import { ThrottlerStorageRedisService } from '@nest-lab/throttler-storage-redis';
 import configuration from './config/configuration';
@@ -75,6 +76,7 @@ import { validationSchema } from './config/validation.schema';
     NotificationsModule,
     HealthModule,
     MetricsModule,
+    TracingModule,
   ],
   providers: [{ provide: APP_GUARD, useClass: ThrottlerGuard }],
 })

apps/backend/src/common/logger/logger.module.ts

@@ -3,47 +3,72 @@ import { WinstonModule } from 'nest-winston';
 import { ConfigService } from '@nestjs/config';
 import * as winston from 'winston';
 
+// Loki transport for log aggregation (optional, loaded dynamically)
+function createLokiTransport(lokiUrl: string, nodeEnv: string) {
+  try {
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const LokiTransport = require('winston-loki');
+    return new LokiTransport({
+      host: lokiUrl,
+      labels: { app: 'brain-storm-backend', env: nodeEnv },
+      json: true,
+      batching: true,
+      interval: 5,
+      onConnectionError: (err: Error) => console.error('Loki connection error:', err.message),
+    });
+  } catch {
+    return null;
+  }
+}
+
 @Module({
   imports: [
     WinstonModule.forRootAsync({
       inject: [ConfigService],
       useFactory: (configService: ConfigService) => {
         const logLevel = configService.get<string>('LOG_LEVEL', 'info');
         const nodeEnv = configService.get<string>('NODE_ENV', 'development');
-
-        // Define log format based on environment
-        const logFormat = nodeEnv === 'production' 
-          ? winston.format.combine(
-              winston.format.timestamp(),
-              winston.format.errors({ stack: true }),
-              winston.format.json()
-            )
-          : winston.format.combine(
-              winston.format.timestamp(),
-              winston.format.errors({ stack: true }),
-              winston.format.colorize(),
-              winston.format.printf(({ timestamp, level, message, context, ...meta }) => {
-                const contextStr = context ? `[${context}] ` : '';
-                const metaStr = Object.keys(meta).length ? ` ${JSON.stringify(meta)}` : '';
-                return `${timestamp} ${level}: ${contextStr}${message}${metaStr}`;
-              })
-            );
+        const lokiUrl = configService.get<string>('LOKI_URL');
+
+        const logFormat =
+          nodeEnv === 'production'
+            ? winston.format.combine(
+                winston.format.timestamp(),
+                winston.format.errors({ stack: true }),
+                winston.format.json(),
+              )
+            : winston.format.combine(
+                winston.format.timestamp(),
+                winston.format.errors({ stack: true }),
+                winston.format.colorize(),
+                winston.format.printf(({ timestamp, level, message, context, ...meta }) => {
+                  const contextStr = context ? `[${context}] ` : '';
+                  const metaStr = Object.keys(meta).length ? ` ${JSON.stringify(meta)}` : '';
+                  return `${timestamp} ${level}: ${contextStr}${message}${metaStr}`;
+                }),
+              );
+
+        const transports: winston.transport[] = [
+          new winston.transports.Console({
+            handleExceptions: true,
+            handleRejections: true,
+          }),
+        ];
+
+        if (lokiUrl) {
+          const lokiTransport = createLokiTransport(lokiUrl, nodeEnv);
+          if (lokiTransport) transports.push(lokiTransport);
+        }
 
         return {
           level: logLevel,
           format: logFormat,
-          transports: [
-            // Console transport - logs to stdout for container orchestrators
-            new winston.transports.Console({
-              handleExceptions: true,
-              handleRejections: true,
-            }),
-          ],
+          transports,
           exitOnError: false,
         };
       },
     }),
   ],
   exports: [WinstonModule],
 })
-export class LoggerModule {}
+export class LoggerModule {}

apps/backend/src/main.ts

@@ -1,3 +1,4 @@
+import './tracing/otel';
 import './instrument';
 import { NestFactory } from '@nestjs/core';
 import { AppModule } from './app.module';

apps/backend/src/metrics/metrics.interceptor.ts

@@ -15,14 +15,14 @@ export class MetricsInterceptor implements NestInterceptor {
   intercept(context: ExecutionContext, next: CallHandler): Observable<any> {
     const request = context.switchToHttp().getRequest();
     const response = context.switchToHttp().getResponse();
+    const startTime = Date.now();
 
     return next.handle().pipe(
       tap(() => {
-        this.metricsService.incrementHttpRequests(
-          request.method,
-          request.route?.path || request.url,
-          response.statusCode,
-        );
+        const route = request.route?.path || request.url;
+        const durationSeconds = (Date.now() - startTime) / 1000;
+        this.metricsService.incrementHttpRequests(request.method, route, response.statusCode);
+        this.metricsService.observeHttpDuration(request.method, route, response.statusCode, durationSeconds);
       }),
     );
   }

apps/backend/src/metrics/metrics.service.ts

@@ -1,12 +1,17 @@
-import { Injectable } from '@nestjs/common';
-import { Counter, Histogram, register } from 'prom-client';
+import { Injectable, OnModuleInit } from '@nestjs/common';
+import { Counter, Histogram, Gauge, register } from 'prom-client';
 
 @Injectable()
-export class MetricsService {
+export class MetricsService implements OnModuleInit {
   private readonly httpRequestsTotal: Counter;
+  private readonly httpRequestDuration: Histogram;
   private readonly credentialIssuedTotal: Counter;
   private readonly bstMintedTotal: Counter;
   private readonly stellarRpcLatency: Histogram;
+  private readonly activeConnections: Gauge;
+  private readonly enrollmentsTotal: Counter;
+  private readonly courseCompletionsTotal: Counter;
+  private readonly authAttemptsTotal: Counter;
 
   constructor() {
     this.httpRequestsTotal = new Counter({
@@ -16,6 +21,14 @@ export class MetricsService {
       registers: [register],
     });
 
+    this.httpRequestDuration = new Histogram({
+      name: 'http_request_duration_seconds',
+      help: 'HTTP request duration in seconds',
+      labelNames: ['method', 'route', 'status_code'],
+      buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5],
+      registers: [register],
+    });
+
     this.credentialIssuedTotal = new Counter({
       name: 'credential_issued_total',
       help: 'Total number of credentials issued',
@@ -37,14 +50,45 @@ export class MetricsService {
       buckets: [0.1, 0.5, 1, 2, 5],
       registers: [register],
     });
+
+    this.activeConnections = new Gauge({
+      name: 'active_connections',
+      help: 'Number of active HTTP connections',
+      registers: [register],
+    });
+
+    this.enrollmentsTotal = new Counter({
+      name: 'enrollments_total',
+      help: 'Total number of course enrollments',
+      labelNames: ['course_id'],
+      registers: [register],
+    });
+
+    this.courseCompletionsTotal = new Counter({
+      name: 'course_completions_total',
+      help: 'Total number of course completions',
+      labelNames: ['course_id'],
+      registers: [register],
+    });
+
+    this.authAttemptsTotal = new Counter({
+      name: 'auth_attempts_total',
+      help: 'Total number of authentication attempts',
+      labelNames: ['type', 'status'],
+      registers: [register],
+    });
+  }
+
+  onModuleInit() {
+    // Metrics are registered in constructor; nothing extra needed
   }
 
   incrementHttpRequests(method: string, route: string, statusCode: number) {
-    this.httpRequestsTotal.inc({
-      method,
-      route,
-      status_code: statusCode.toString(),
-    });
+    this.httpRequestsTotal.inc({ method, route, status_code: statusCode.toString() });
+  }
+
+  observeHttpDuration(method: string, route: string, statusCode: number, durationSeconds: number) {
+    this.httpRequestDuration.observe({ method, route, status_code: statusCode.toString() }, durationSeconds);
   }
 
   incrementCredentialIssued(credentialType: string) {
@@ -58,4 +102,20 @@ export class MetricsService {
   observeStellarRpcLatency(method: string, status: string, durationSeconds: number) {
     this.stellarRpcLatency.observe({ method, status }, durationSeconds);
   }
+
+  setActiveConnections(count: number) {
+    this.activeConnections.set(count);
+  }
+
+  incrementEnrollments(courseId: string) {
+    this.enrollmentsTotal.inc({ course_id: courseId });
+  }
+
+  incrementCourseCompletions(courseId: string) {
+    this.courseCompletionsTotal.inc({ course_id: courseId });
+  }
+
+  incrementAuthAttempts(type: 'login' | 'register' | 'refresh', status: 'success' | 'failure') {
+    this.authAttemptsTotal.inc({ type, status });
+  }
 }

apps/backend/src/tracing/index.ts

@@ -0,0 +1,2 @@
+export { TracingModule } from './tracing.module';
+export { TracingService } from './tracing.service';

apps/backend/src/tracing/otel.ts

@@ -0,0 +1,34 @@
+import { NodeSDK } from '@opentelemetry/sdk-node';
+import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node';
+import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
+import { Resource } from '@opentelemetry/resources';
+import { SEMRESATTRS_SERVICE_NAME, SEMRESATTRS_SERVICE_VERSION, SEMRESATTRS_DEPLOYMENT_ENVIRONMENT } from '@opentelemetry/semantic-conventions';
+
+const traceExporter = process.env.OTEL_EXPORTER_OTLP_ENDPOINT
+  ? new OTLPTraceExporter({ url: process.env.OTEL_EXPORTER_OTLP_ENDPOINT })
+  : undefined;
+
+const sdk = new NodeSDK({
+  resource: new Resource({
+    [SEMRESATTRS_SERVICE_NAME]: 'brain-storm-backend',
+    [SEMRESATTRS_SERVICE_VERSION]: process.env.npm_package_version || '1.0.0',
+    [SEMRESATTRS_DEPLOYMENT_ENVIRONMENT]: process.env.NODE_ENV || 'development',
+  }),
+  traceExporter,
+  instrumentations: [
+    getNodeAutoInstrumentations({
+      '@opentelemetry/instrumentation-http': { enabled: true },
+      '@opentelemetry/instrumentation-express': { enabled: true },
+      '@opentelemetry/instrumentation-pg': { enabled: true },
+      '@opentelemetry/instrumentation-redis': { enabled: true },
+      '@opentelemetry/instrumentation-fs': { enabled: false },
+    }),
+  ],
+});
+
+sdk.start();
+
+process.on('SIGTERM', () => {
+  sdk.shutdown().finally(() => process.exit(0));
+});

apps/backend/src/tracing/tracing.module.ts

@@ -0,0 +1,9 @@
+import { Module, Global } from '@nestjs/common';
+import { TracingService } from './tracing.service';
+
+@Global()
+@Module({
+  providers: [TracingService],
+  exports: [TracingService],
+})
+export class TracingModule {}

apps/backend/src/tracing/tracing.service.ts

@@ -0,0 +1,50 @@
+import { Injectable } from '@nestjs/common';
+import { trace, context, SpanStatusCode, Span, Tracer } from '@opentelemetry/api';
+
+@Injectable()
+export class TracingService {
+  private readonly tracer: Tracer;
+
+  constructor() {
+    this.tracer = trace.getTracer('brain-storm-backend', '1.0.0');
+  }
+
+  startSpan(name: string, attributes?: Record<string, string | number | boolean>): Span {
+    const span = this.tracer.startSpan(name);
+    if (attributes) {
+      span.setAttributes(attributes);
+    }
+    return span;
+  }
+
+  async withSpan<T>(
+    name: string,
+    fn: (span: Span) => Promise<T>,
+    attributes?: Record<string, string | number | boolean>,
+  ): Promise<T> {
+    return this.tracer.startActiveSpan(name, { attributes }, async (span) => {
+      try {
+        const result = await fn(span);
+        span.setStatus({ code: SpanStatusCode.OK });
+        return result;
+      } catch (error) {
+        span.setStatus({ code: SpanStatusCode.ERROR, message: error.message });
+        span.recordException(error);
+        throw error;
+      } finally {
+        span.end();
+      }
+    });
+  }
+
+  getCurrentSpan(): Span | undefined {
+    return trace.getActiveSpan();
+  }
+
+  addSpanAttributes(attributes: Record<string, string | number | boolean>): void {
+    const span = this.getCurrentSpan();
+    if (span) {
+      span.setAttributes(attributes);
+    }
+  }
+}