From f4da319cf6cfcd6237f8d69df6c2bd9b97c00b0b Mon Sep 17 00:00:00 2001 From: robertocarlous Date: Wed, 25 Mar 2026 17:05:17 +0100 Subject: [PATCH 1/3] Missing Database Indexes --- .gitignore | 3 +- .../migration.sql | 51 +++++++++ prisma/schema.prisma | 14 +++ src/config/utils/cors-origin.validator.ts | 6 +- src/config/validation/config.validation.ts | 17 +-- src/database/database.module.ts | 4 + .../database-optimization.controller.ts | 42 ++++++++ .../optimization/index-monitor.service.ts | 100 ++++++++++++++++++ src/database/prisma/prisma.module.ts | 5 +- src/database/prisma/prisma.service.ts | 11 ++ src/documents/document.controller.ts | 9 +- src/documents/document.service.ts | 6 +- src/documents/documents.module.ts | 5 +- src/main.ts | 10 +- .../search/property-search.service.ts | 9 +- src/security/config/multer.config.ts | 30 ++++-- .../services/cors-validation.service.ts | 41 +++---- .../services/file-validation.service.ts | 88 ++++++++++----- .../services/malware-scanner.service.ts | 24 +++-- .../validators/secure-file.validator.ts | 25 ++--- src/static-cache/cache-warming.service.ts | 60 +++++------ 21 files changed, 403 insertions(+), 157 deletions(-) create mode 100644 prisma/migrations/20260325120000_composite_indexes/migration.sql create mode 100644 src/database/optimization/database-optimization.controller.ts create mode 100644 src/database/optimization/index-monitor.service.ts diff --git a/.gitignore b/.gitignore index 21a15691..8c8b46eb 100644 --- a/.gitignore +++ b/.gitignore @@ -117,7 +117,8 @@ temp/ *.sqlite3 # Prisma -prisma/migrations/ +# Keep migrations in git so schema/index changes are deployable. +# (Avoid ignoring `prisma/migrations/`.) # Docker .dockerignore diff --git a/prisma/migrations/20260325120000_composite_indexes/migration.sql b/prisma/migrations/20260325120000_composite_indexes/migration.sql new file mode 100644 index 00000000..682c1e2b --- /dev/null +++ b/prisma/migrations/20260325120000_composite_indexes/migration.sql @@ -0,0 +1,51 @@ +-- Composite and supporting indexes for critical read paths. +-- Generated manually to match prisma/schema.prisma changes. + +-- user_activities: common access patterns +CREATE INDEX "user_activities_user_id_action_created_at_idx" +ON "user_activities" ("user_id", "action", "created_at" DESC); + +-- user_relationships: followers/following lists filtered by status and sorted by recency +CREATE INDEX "user_relationships_following_id_status_created_at_idx" +ON "user_relationships" ("following_id", "status", "created_at" DESC); + +CREATE INDEX "user_relationships_follower_id_status_created_at_idx" +ON "user_relationships" ("follower_id", "status", "created_at" DESC); + +-- properties: listing pages (status) ordered by recency; owner dashboards; status + price range filters +CREATE INDEX "properties_status_created_at_idx" +ON "properties" ("status", "created_at" DESC); + +CREATE INDEX "properties_owner_id_created_at_idx" +ON "properties" ("owner_id", "created_at" DESC); + +CREATE INDEX "properties_status_price_idx" +ON "properties" ("status", "price"); + +-- property_valuations: history pages (property) ordered by valuationDate desc +CREATE INDEX "property_valuations_property_id_valuation_date_idx" +ON "property_valuations" ("property_id", "valuation_date" DESC); + +-- audit_logs: common filters + ordered by timestamp +CREATE INDEX "audit_logs_timestamp_idx" +ON "audit_logs" ("timestamp" DESC); + +CREATE INDEX "audit_logs_user_id_timestamp_idx" +ON "audit_logs" ("user_id", "timestamp" DESC); + +CREATE INDEX "audit_logs_table_name_timestamp_idx" +ON "audit_logs" ("table_name", "timestamp" DESC); + +CREATE INDEX "audit_logs_operation_timestamp_idx" +ON "audit_logs" ("operation", "timestamp" DESC); + +-- system_logs: operational queries (by level/context) ordered by timestamp +CREATE INDEX "system_logs_timestamp_idx" +ON "system_logs" ("timestamp" DESC); + +CREATE INDEX "system_logs_log_level_timestamp_idx" +ON "system_logs" ("log_level", "timestamp" DESC); + +CREATE INDEX "system_logs_context_timestamp_idx" +ON "system_logs" ("context", "timestamp" DESC); + diff --git a/prisma/schema.prisma b/prisma/schema.prisma index df173052..964dd4e9 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -74,6 +74,7 @@ model UserActivity { @@index([userId]) @@index([action]) @@index([createdAt]) + @@index([userId, action, createdAt(sort: Desc)]) @@map("user_activities") } @@ -91,6 +92,8 @@ model UserRelationship { @@unique([followerId, followingId]) @@index([followerId]) @@index([followingId]) + @@index([followingId, status, createdAt(sort: Desc)]) + @@index([followerId, status, createdAt(sort: Desc)]) @@map("user_relationships") } @@ -136,6 +139,9 @@ model Property { @@index([location]) // For searching/filtering by location // Consider adding an index on price if you often filter/sort by price @@index([price]) + @@index([status, createdAt(sort: Desc)]) + @@index([ownerId, createdAt(sort: Desc)]) + @@index([status, price]) @@map("properties") } @@ -156,6 +162,7 @@ model PropertyValuation { // Index for fast lookup by property and date @@index([propertyId]) @@index([valuationDate]) + @@index([propertyId, valuationDate(sort: Desc)]) @@map("property_valuations") } @@ -266,6 +273,10 @@ model AuditLog { userId String? @map("user_id") timestamp DateTime @default(now()) + @@index([timestamp(sort: Desc)]) + @@index([userId, timestamp(sort: Desc)]) + @@index([tableName, timestamp(sort: Desc)]) + @@index([operation, timestamp(sort: Desc)]) @@map("audit_logs") } @@ -277,6 +288,9 @@ model SystemLog { context String? timestamp DateTime @default(now()) + @@index([timestamp(sort: Desc)]) + @@index([logLevel, timestamp(sort: Desc)]) + @@index([context, timestamp(sort: Desc)]) @@map("system_logs") } diff --git a/src/config/utils/cors-origin.validator.ts b/src/config/utils/cors-origin.validator.ts index 2a606c8f..f16f1ac0 100644 --- a/src/config/utils/cors-origin.validator.ts +++ b/src/config/utils/cors-origin.validator.ts @@ -61,7 +61,9 @@ export class CorsOriginValidator { // Validate URL format (basic check) if (!urlPattern.test(origin)) { - console.error(`[CorsOriginValidator] Invalid origin format: "${origin}". Expected valid URL (e.g., https://example.com)`); + console.error( + `[CorsOriginValidator] Invalid origin format: "${origin}". Expected valid URL (e.g., https://example.com)`, + ); return false; } @@ -126,4 +128,4 @@ export class CorsOriginValidator { return false; } -} \ No newline at end of file +} diff --git a/src/config/validation/config.validation.ts b/src/config/validation/config.validation.ts index 413c5eb4..e5c8e806 100644 --- a/src/config/validation/config.validation.ts +++ b/src/config/validation/config.validation.ts @@ -7,12 +7,12 @@ import * as Joi from 'joi'; */ const corsOriginValidation = (value: string, helpers: Joi.CustomHelpers) => { const nodeEnv = Joi.ref('$NODE_ENV'); - + // If no value provided, error if (!value || value.trim() === '') { return helpers.error('cors.origin.required'); } - + // Allow '*' only in development and test if (value === '*') { const env = helpers.prefs?.context?.NODE_ENV || 'development'; @@ -21,16 +21,16 @@ const corsOriginValidation = (value: string, helpers: Joi.CustomHelpers) => { } return value; } - + // Validate individual origins (comma-separated) const origins = value.split(',').map(o => o.trim()); const urlPattern = /^https?:\/\/[\w.-]+(:\d+)?(\/.*)?$/; - + for (const origin of origins) { if (origin === '*') { return helpers.error('cors.origin.wildcard.notAllowed'); } - + // Allow 'http://localhost' and 'http://localhost:*' variants if (origin.startsWith('http://localhost') || origin.startsWith('http://127.0.0.1')) { const env = helpers.prefs?.context?.NODE_ENV || 'development'; @@ -39,13 +39,13 @@ const corsOriginValidation = (value: string, helpers: Joi.CustomHelpers) => { } continue; } - + // Validate URL format if (!urlPattern.test(origin)) { return helpers.error('cors.origin.invalidFormat', { origin }); } } - + return value; }; @@ -60,7 +60,8 @@ export const configValidationSchema = Joi.object({ API_PREFIX: Joi.string().default('api'), CORS_ORIGIN: Joi.string().custom(corsOriginValidation).default('*').messages({ 'cors.origin.required': 'CORS_ORIGIN is required', - 'cors.origin.wildcard.notAllowed': 'Wildcard (*) origin is not allowed in production/staging. Specify explicit allowed origins.', + 'cors.origin.wildcard.notAllowed': + 'Wildcard (*) origin is not allowed in production/staging. Specify explicit allowed origins.', 'cors.origin.localhost.notAllowed': 'Localhost origins are not allowed in production/staging', 'cors.origin.invalidFormat': 'Invalid origin format: "{{origin}}". Must be a valid URL (e.g., https://example.com)', }), diff --git a/src/database/database.module.ts b/src/database/database.module.ts index c59e3a2b..d45d2765 100644 --- a/src/database/database.module.ts +++ b/src/database/database.module.ts @@ -1,9 +1,13 @@ import { Module, Global } from '@nestjs/common'; import { PrismaModule } from './prisma/prisma.module'; +import { IndexMonitorService } from './optimization/index-monitor.service'; +import { DatabaseOptimizationController } from './optimization/database-optimization.controller'; @Global() @Module({ imports: [PrismaModule], + providers: [IndexMonitorService], + controllers: [DatabaseOptimizationController], exports: [PrismaModule], }) export class DatabaseModule {} diff --git a/src/database/optimization/database-optimization.controller.ts b/src/database/optimization/database-optimization.controller.ts new file mode 100644 index 00000000..19a7b0ce --- /dev/null +++ b/src/database/optimization/database-optimization.controller.ts @@ -0,0 +1,42 @@ +import { Controller, Get } from '@nestjs/common'; +import { ApiOperation, ApiTags } from '@nestjs/swagger'; +import { PerformanceMonitorService } from './performance.monitor'; +import { QueryOptimizerService } from './query.optimizer'; + +@ApiTags('database-optimization') +@Controller('database/optimization') +export class DatabaseOptimizationController { + constructor( + private readonly performanceMonitor: PerformanceMonitorService, + private readonly queryOptimizer: QueryOptimizerService, + ) {} + + @Get('slow-queries') + @ApiOperation({ summary: 'List slow queries observed by the app' }) + getSlowQueries() { + return this.queryOptimizer.getSlowQueries(); + } + + @Get('top-queries') + @ApiOperation({ summary: 'List most frequent queries observed by the app' }) + getTopQueries() { + return this.queryOptimizer.getMostFrequentQueries(25); + } + + @Get('performance-report') + @ApiOperation({ summary: 'Get current database performance report' }) + getPerformanceReport() { + const report = this.performanceMonitor.generatePerformanceReport(); + const health = this.performanceMonitor.getHealthScore(); + return { ...report, health }; + } + + @Get('index-usage') + @ApiOperation({ summary: 'Get current index usage snapshot (from pg_stat_user_indexes)' }) + getIndexUsage() { + const metrics = this.performanceMonitor.getMetrics(); + return Array.from(metrics.indexUsage.entries()) + .map(([indexName, usageCount]) => ({ indexName, usageCount })) + .sort((a, b) => a.usageCount - b.usageCount); + } +} diff --git a/src/database/optimization/index-monitor.service.ts b/src/database/optimization/index-monitor.service.ts new file mode 100644 index 00000000..95acd3e5 --- /dev/null +++ b/src/database/optimization/index-monitor.service.ts @@ -0,0 +1,100 @@ +import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { PrismaService } from '../prisma/prisma.service'; +import { PerformanceMonitorService } from './performance.monitor'; + +type PgIndexUsageRow = { + schemaname: string; + tablename: string; + indexname: string; + idx_scan: bigint | number; +}; + +type PgTableStatsRow = { + schemaname: string; + relname: string; + n_deadlocks: bigint | number; +}; + +@Injectable() +export class IndexMonitorService implements OnModuleDestroy { + private readonly logger = new Logger(IndexMonitorService.name); + private interval: NodeJS.Timeout | null = null; + + constructor( + private readonly prisma: PrismaService, + private readonly performanceMonitor: PerformanceMonitorService, + private readonly configService: ConfigService, + ) { + const enabled = this.configService.get('INDEX_MONITORING_ENABLED', true); + if (enabled) { + this.start(); + } + } + + private start() { + const intervalMs = this.configService.get('INDEX_MONITORING_INTERVAL_MS', 60_000); + this.interval = setInterval(() => { + void this.collectOnce(); + }, intervalMs); + + // Run one initial collection quickly (but async) + void this.collectOnce(); + this.logger.log(`Index monitoring started with ${intervalMs}ms interval`); + } + + async onModuleDestroy() { + if (this.interval) { + clearInterval(this.interval); + this.interval = null; + } + } + + private async collectOnce(): Promise { + try { + const [indexUsage, tableStats] = await Promise.all([this.getIndexUsage(), this.getTableStats()]); + + for (const row of indexUsage) { + const qualifiedIndex = `${row.schemaname}.${row.indexname}`; + const usageCount = typeof row.idx_scan === 'bigint' ? Number(row.idx_scan) : row.idx_scan; + this.performanceMonitor.updateIndexUsage(qualifiedIndex, usageCount); + } + + for (const row of tableStats) { + const qualifiedTable = `${row.schemaname}.${row.relname}`; + const deadlocks = typeof row.n_deadlocks === 'bigint' ? Number(row.n_deadlocks) : row.n_deadlocks; + this.performanceMonitor.updateTableStats(qualifiedTable, { + avgRowLockTime: 0, + deadlockCount: deadlocks, + avgQueryTime: 0, + indexUsageStats: new Map(), + }); + } + } catch (error) { + this.logger.warn(`Index monitoring collection failed: ${(error as Error)?.message ?? String(error)}`); + } + } + + private async getIndexUsage(): Promise { + // pg_stat_user_indexes is safe for regular roles in most managed Postgres setups. + return this.prisma.$queryRaw` + SELECT + schemaname, + relname AS tablename, + indexrelname AS indexname, + idx_scan + FROM pg_stat_user_indexes + ORDER BY idx_scan ASC; + `; + } + + private async getTableStats(): Promise { + return this.prisma.$queryRaw` + SELECT + schemaname, + relname, + n_deadlocks + FROM pg_stat_user_tables; + `; + } +} diff --git a/src/database/prisma/prisma.module.ts b/src/database/prisma/prisma.module.ts index 23c626eb..c4484c62 100644 --- a/src/database/prisma/prisma.module.ts +++ b/src/database/prisma/prisma.module.ts @@ -1,9 +1,10 @@ import { Module, Global } from '@nestjs/common'; import { PrismaService } from './prisma.service'; +import { PerformanceMonitorService, QueryOptimizerService } from '../optimization'; @Global() @Module({ - providers: [PrismaService], - exports: [PrismaService], + providers: [PrismaService, PerformanceMonitorService, QueryOptimizerService], + exports: [PrismaService, PerformanceMonitorService, QueryOptimizerService], }) export class PrismaModule {} diff --git a/src/database/prisma/prisma.service.ts b/src/database/prisma/prisma.service.ts index 39ccb834..7abef391 100644 --- a/src/database/prisma/prisma.service.ts +++ b/src/database/prisma/prisma.service.ts @@ -6,6 +6,7 @@ import { Injectable, OnModuleInit, OnModuleDestroy } from '@nestjs/common'; import { PrismaClient, Prisma } from '@prisma/client'; import { ConfigService } from '@nestjs/config'; import { StructuredLoggerService } from '../../common/logging/logger.service'; +import { PerformanceMonitorService, QueryOptimizerService } from '../optimization'; @Injectable() export class PrismaService extends PrismaClient implements OnModuleInit, OnModuleDestroy { @@ -18,6 +19,8 @@ export class PrismaService extends PrismaClient implements OnModuleInit, OnModul constructor( private readonly configService: ConfigService, private readonly logger: StructuredLoggerService, + private readonly performanceMonitor?: PerformanceMonitorService, + private readonly queryOptimizer?: QueryOptimizerService, ) { // Get the database URL from environment/config let databaseUrl = configService.get('DATABASE_URL'); @@ -73,6 +76,14 @@ export class PrismaService extends PrismaClient implements OnModuleInit, OnModul query: e.query, params: e.params, }); + + // Feed query timings into optimization/monitoring services when available + try { + this.queryOptimizer?.trackQuery(e.query, e.duration); + this.performanceMonitor?.recordQuery(e.query, e.duration, true); + } catch (err) { + // Do not disrupt DB operations if monitoring fails + } }); await this.$connect(); diff --git a/src/documents/document.controller.ts b/src/documents/document.controller.ts index 301e5527..93c94064 100644 --- a/src/documents/document.controller.ts +++ b/src/documents/document.controller.ts @@ -12,7 +12,10 @@ import { UseInterceptors, } from '@nestjs/common'; import { ApiTags, ApiOperation, ApiResponse, ApiParam, ApiHeader, ApiConsumes } from '@nestjs/swagger'; -import { DocumentFilesUploadInterceptor, DocumentFileUploadInterceptor } from './interceptors/document-upload.interceptor'; +import { + DocumentFilesUploadInterceptor, + DocumentFileUploadInterceptor, +} from './interceptors/document-upload.interceptor'; import { SecureFileValidator } from '../security/validators/secure-file.validator'; import { DocumentAccessContext, DocumentMetadataInput, DocumentSearchFilters } from './document.model'; import { DocumentService } from './document.service'; @@ -51,7 +54,7 @@ export class DocumentController { for (const file of files) { await this.secureFileValidator.validate(file); } - + const context = this.buildAccessContext(userId, rolesHeader); const metadata = this.parseMetadataInput(body); return this.documentService.uploadDocuments(files, metadata, context); @@ -74,7 +77,7 @@ export class DocumentController { ) { // Validate file with comprehensive security checks await this.secureFileValidator.validate(file); - + const context = this.buildAccessContext(userId, rolesHeader); return this.documentService.addDocumentVersion(documentId, file, context); } diff --git a/src/documents/document.service.ts b/src/documents/document.service.ts index 663291b1..ed633e51 100644 --- a/src/documents/document.service.ts +++ b/src/documents/document.service.ts @@ -433,13 +433,11 @@ export class DocumentService { if (this.malwareScanner) { const result = await this.malwareScanner.scanFile(buffer, filename); if (!result.isClean) { - throw new BadRequestException( - `Malware detected in file: ${result.virusName || 'Unknown virus'}`, - ); + throw new BadRequestException(`Malware detected in file: ${result.virusName || 'Unknown virus'}`); } return; } - + // Fallback to basic signature check if no malware scanner is available if (buffer.toString('utf8').includes(DocumentService.virusSignature)) { throw new BadRequestException('File failed virus scan'); diff --git a/src/documents/documents.module.ts b/src/documents/documents.module.ts index 2fc199e8..c2c5cf3b 100644 --- a/src/documents/documents.module.ts +++ b/src/documents/documents.module.ts @@ -13,7 +13,10 @@ import { FileStorageService } from './storage/file-storage.service'; import { SecureFileValidator } from '../security/validators/secure-file.validator'; import { FileValidationService } from '../security/services/file-validation.service'; import { MalwareScannerService } from '../security/services/malware-scanner.service'; -import { DocumentFilesUploadInterceptor, DocumentFileUploadInterceptor } from './interceptors/document-upload.interceptor'; +import { + DocumentFilesUploadInterceptor, + DocumentFileUploadInterceptor, +} from './interceptors/document-upload.interceptor'; @Module({ controllers: [DocumentController], diff --git a/src/main.ts b/src/main.ts index 3b6c9f25..5899e629 100644 --- a/src/main.ts +++ b/src/main.ts @@ -71,26 +71,26 @@ async function bootstrap() { // CORS configuration with validation const corsOrigin = configService.get('CORS_ORIGIN'); const nodeEnv = configService.get('NODE_ENV', 'development'); - + // Validate CORS origins before configuring const validatedOrigins = CorsOriginValidator.validate(corsOrigin, nodeEnv); - + if (!validatedOrigins) { logger.error('Invalid CORS configuration detected. Application cannot start with insecure CORS settings.'); if (nodeEnv === 'production' || nodeEnv === 'staging') { process.exit(1); // Fail hard in production/staging } } - + const corsOrigins = CorsOriginValidator.parseForNestJs(corsOrigin); - + app.enableCors({ origin: corsOrigins, credentials: true, methods: ['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'OPTIONS'], allowedHeaders: ['Content-Type', 'Authorization', 'X-Requested-With', 'x-correlation-id'], }); - + logger.log(`CORS configured with origins: ${corsOrigins.join(', ')}`); // Global pipes diff --git a/src/properties/search/property-search.service.ts b/src/properties/search/property-search.service.ts index 23dd4948..8520caa9 100644 --- a/src/properties/search/property-search.service.ts +++ b/src/properties/search/property-search.service.ts @@ -104,14 +104,7 @@ export class PropertySearchService { } private async normalSearch(dto: PropertySearchDto) { - const { - page = 1, - limit = 10, - minPrice, - maxPrice, - location, - status: dtoStatus = PropertyStatus.AVAILABLE, - } = dto; + const { page = 1, limit = 10, minPrice, maxPrice, location, status: dtoStatus = PropertyStatus.AVAILABLE } = dto; const status = this.mapPropertyStatus(dtoStatus); const offset = (page - 1) * limit; diff --git a/src/security/config/multer.config.ts b/src/security/config/multer.config.ts index 4e646a58..294784d1 100644 --- a/src/security/config/multer.config.ts +++ b/src/security/config/multer.config.ts @@ -19,7 +19,7 @@ export function createMulterConfig(configService: ConfigService) { fieldNameSize: 100, // Max field name length headerPairs: 2000, // Max number of header pairs }, - + // File filter for basic MIME type checking fileFilter: (req: any, file: Express.Multer.File, cb: any) => { if (!allowedMimeTypes.includes(file.mimetype)) { @@ -27,28 +27,42 @@ export function createMulterConfig(configService: ConfigService) { (error as any).code = 'UNSUPPORTED_MEDIA_TYPE'; return cb(error, false); } - + // Validate filename if (!file.originalname || file.originalname.length === 0) { const error = new Error('Invalid filename'); (error as any).code = 'INVALID_FILENAME'; return cb(error, false); } - + // Check for dangerous extensions const dangerousExtensions = [ - '.exe', '.dll', '.so', '.bat', '.cmd', '.sh', - '.php', '.asp', '.aspx', '.jsp', '.cgi', - '.pl', '.py', '.rb', '.msi', '.com', '.pif', + '.exe', + '.dll', + '.so', + '.bat', + '.cmd', + '.sh', + '.php', + '.asp', + '.aspx', + '.jsp', + '.cgi', + '.pl', + '.py', + '.rb', + '.msi', + '.com', + '.pif', ]; - + const lowerName = file.originalname.toLowerCase(); if (dangerousExtensions.some(ext => lowerName.endsWith(ext))) { const error = new Error('Dangerous file type detected'); (error as any).code = 'DANGEROUS_FILE_TYPE'; return cb(error, false); } - + cb(null, true); }, }; diff --git a/src/security/services/cors-validation.service.ts b/src/security/services/cors-validation.service.ts index 8c173c93..23ab5071 100644 --- a/src/security/services/cors-validation.service.ts +++ b/src/security/services/cors-validation.service.ts @@ -67,7 +67,7 @@ export class CorsValidationService { if (origin.startsWith('http://localhost') || origin.startsWith('http://127.0.0.1')) { return true; } - + // Also check allowlist return this.allowedOrigins.has(origin); } @@ -82,11 +82,11 @@ export class CorsValidationService { getOriginValidator(): (origin: string) => boolean { return (origin: string) => { const allowed = this.isOriginAllowed(origin); - + if (!allowed && origin) { this.logger.warn(`Blocked CORS request from unauthorized origin: ${origin}`); } - + return allowed; }; } @@ -116,9 +116,7 @@ export class CorsValidationService { // Warn about insecure origins in production if (origin.startsWith('http://') && !origin.includes('localhost')) { - this.logger.warn( - `Insecure HTTP origin detected in production: ${origin}. Consider using HTTPS.`, - ); + this.logger.warn(`Insecure HTTP origin detected in production: ${origin}. Consider using HTTPS.`); } } } @@ -151,9 +149,7 @@ export class CorsValidationService { totalOrigins: this.allowedOrigins.size, isProduction: this.isProduction, isWildcard: this.allowedOrigins.has('*'), - hasLocalhost: Array.from(this.allowedOrigins).some(o => - o.includes('localhost') || o.includes('127.0.0.1'), - ), + hasLocalhost: Array.from(this.allowedOrigins).some(o => o.includes('localhost') || o.includes('127.0.0.1')), }; } @@ -162,16 +158,10 @@ export class CorsValidationService { */ private getProductionCorsConfig(): CorsOriginConfig { const validation = this.validateConfig(); - + if (!validation.isValid) { - this.logger.error( - 'Production CORS configuration is invalid:', - validation.errors.join(', '), - {}, - ); - throw new BadRequestException( - `Invalid CORS configuration: ${validation.errors.join(', ')}`, - ); + this.logger.error('Production CORS configuration is invalid:', validation.errors.join(', '), {}); + throw new BadRequestException(`Invalid CORS configuration: ${validation.errors.join(', ')}`); } return { @@ -192,10 +182,7 @@ export class CorsValidationService { 'x-correlation-id', 'Accept-Version', ]), - exposedHeaders: this.configService.get('CORS_EXPOSED_HEADERS', [ - 'x-correlation-id', - 'x-request-id', - ]), + exposedHeaders: this.configService.get('CORS_EXPOSED_HEADERS', ['x-correlation-id', 'x-request-id']), maxAge: this.configService.get('CORS_MAX_AGE', 86400), // 24 hours }; } @@ -283,11 +270,9 @@ export class CorsValidationService { */ private logConfiguration(): void { const stats = this.getStats(); - + if (this.isProduction) { - this.logger.log( - `๐Ÿ”’ Production CORS configured with ${stats.totalOrigins} allowed origin(s)`, - ); + this.logger.log(`๐Ÿ”’ Production CORS configured with ${stats.totalOrigins} allowed origin(s)`); if (stats.totalOrigins > 0) { this.logger.debug(`Allowed origins: ${Array.from(this.allowedOrigins).join(', ')}`); } @@ -295,9 +280,7 @@ export class CorsValidationService { if (stats.isWildcard) { this.logger.warn('โš ๏ธ Development CORS: Wildcard (*) enabled - OK for local development'); } else { - this.logger.log( - `๐Ÿ”ง Development CORS configured with ${stats.totalOrigins} allowed origin(s)`, - ); + this.logger.log(`๐Ÿ”ง Development CORS configured with ${stats.totalOrigins} allowed origin(s)`); } } else { this.logger.log(`๐Ÿงช Test CORS configured`); diff --git a/src/security/services/file-validation.service.ts b/src/security/services/file-validation.service.ts index d11e78b4..541c2218 100644 --- a/src/security/services/file-validation.service.ts +++ b/src/security/services/file-validation.service.ts @@ -22,7 +22,10 @@ export class FileValidationService { private readonly logger = new Logger(FileValidationService.name); // Magic numbers for common file types (first bytes of files) - private readonly magicNumberSignatures: Record = { + private readonly magicNumberSignatures: Record< + string, + { offset: number; bytes: string; mime: string; ext: string }[] + > = { image: [ { offset: 0, bytes: 'FFD8FF', mime: 'image/jpeg', ext: 'jpg' }, { offset: 0, bytes: '89504E470D0A1A0A', mime: 'image/png', ext: 'png' }, @@ -37,11 +40,26 @@ export class FileValidationService { document: [ { offset: 0, bytes: '255044462D', mime: 'application/pdf', ext: 'pdf' }, // %PDF- { offset: 0, bytes: 'D0CF11E0A1B11AE1', mime: 'application/msword', ext: 'doc' }, - { offset: 0, bytes: '504B0304', mime: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', ext: 'docx' }, + { + offset: 0, + bytes: '504B0304', + mime: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + ext: 'docx', + }, { offset: 0, bytes: '504B0304', mime: 'application/vnd.ms-excel', ext: 'xls' }, - { offset: 0, bytes: '504B0304', mime: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', ext: 'xlsx' }, + { + offset: 0, + bytes: '504B0304', + mime: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + ext: 'xlsx', + }, { offset: 0, bytes: '504B0304', mime: 'application/vnd.ms-powerpoint', ext: 'ppt' }, - { offset: 0, bytes: '504B0304', mime: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', ext: 'pptx' }, + { + offset: 0, + bytes: '504B0304', + mime: 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + ext: 'pptx', + }, ], archive: [ { offset: 0, bytes: '504B0304', mime: 'application/zip', ext: 'zip' }, @@ -70,13 +88,37 @@ export class FileValidationService { // Dangerous file extensions that should be blocked private readonly dangerousExtensions = [ - '.exe', '.dll', '.so', '.dylib', // Executables - '.bat', '.cmd', '.sh', '.bash', '.ps1', '.vbs', '.js', '.jar', // Scripts - '.php', '.asp', '.aspx', '.jsp', '.cgi', // Web scripts - '.pl', '.py', '.rb', '.swf', // Other executables - '.msi', '.com', '.pif', '.scr', '.reg', // Windows executables - '.lnk', '.inf', '.drv', // System files - '.htaccess', '.htpasswd', // Apache config + '.exe', + '.dll', + '.so', + '.dylib', // Executables + '.bat', + '.cmd', + '.sh', + '.bash', + '.ps1', + '.vbs', + '.js', + '.jar', // Scripts + '.php', + '.asp', + '.aspx', + '.jsp', + '.cgi', // Web scripts + '.pl', + '.py', + '.rb', + '.swf', // Other executables + '.msi', + '.com', + '.pif', + '.scr', + '.reg', // Windows executables + '.lnk', + '.inf', + '.drv', // System files + '.htaccess', + '.htpasswd', // Apache config ]; /** @@ -84,7 +126,7 @@ export class FileValidationService { */ validateFile(buffer: Buffer, declaredMimeType?: string): FileValidationResult { const errors: string[] = []; - + if (!buffer || buffer.length === 0) { return { isValid: false, @@ -94,7 +136,7 @@ export class FileValidationService { // Detect actual file type using magic numbers const detectedType = this.detectFileType(buffer); - + if (!detectedType) { return { isValid: false, @@ -104,9 +146,7 @@ export class FileValidationService { // Check for MIME type mismatch (potential spoofing attack) if (declaredMimeType && declaredMimeType !== detectedType.mime) { - errors.push( - `MIME type mismatch detected. Declared: ${declaredMimeType}, Actual: ${detectedType.mime}`, - ); + errors.push(`MIME type mismatch detected. Declared: ${declaredMimeType}, Actual: ${detectedType.mime}`); } // Calculate checksum @@ -133,9 +173,7 @@ export class FileValidationService { */ isDangerousExtension(filename: string): boolean { const lowerFilename = filename.toLowerCase(); - return this.dangerousExtensions.some(ext => - lowerFilename.endsWith(ext) || lowerFilename.includes(ext + '.') - ); + return this.dangerousExtensions.some(ext => lowerFilename.endsWith(ext) || lowerFilename.includes(`${ext}.`)); } /** @@ -172,7 +210,7 @@ export class FileValidationService { */ private detectFileType(buffer: Buffer): FileTypeMatch | null { const hexHeader = buffer.toString('hex', 0, Math.min(16, buffer.length)).toUpperCase(); - + // Check all signature categories for (const category of Object.values(this.magicNumberSignatures)) { for (const signature of category) { @@ -185,13 +223,13 @@ export class FileValidationService { if (signature.bytes.length > 8) { return { ext: signature.ext, mime: signature.mime }; } - + // For common signatures like PK (504B0304), do additional validation if (signature.bytes === '504B0304') { // ZIP-based formats - check further to distinguish return this.distinguishZipFormat(buffer, signature); } - + return { ext: signature.ext, mime: signature.mime }; } } @@ -212,7 +250,7 @@ export class FileValidationService { try { // Look for mimetype markers in the ZIP structure const content = buffer.toString('binary'); - + if (content.includes('[Content_Types].xml')) { if (content.includes('word/')) { return { ext: 'docx', mime: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' }; @@ -262,7 +300,7 @@ export class FileValidationService { .flat() .map(sig => sig.mime); } - + const signatures = this.magicNumberSignatures[category] || []; return signatures.map(sig => sig.mime); } @@ -284,7 +322,7 @@ export class FileValidationService { const wildcardMatches = allowedTypes.some(allowed => { if (allowed.endsWith('/*')) { const prefix = allowed.slice(0, -2); - return mimeType.startsWith(prefix + '/'); + return mimeType.startsWith(`${prefix}/`); } return false; }); diff --git a/src/security/services/malware-scanner.service.ts b/src/security/services/malware-scanner.service.ts index 96c95ae5..73ca7cb6 100644 --- a/src/security/services/malware-scanner.service.ts +++ b/src/security/services/malware-scanner.service.ts @@ -55,7 +55,7 @@ export class MalwareScannerService { return this.createResult(result.isClean, result.virusName, startTime, 'clamav'); } catch (clamavError) { this.logger.warn(`ClamAV scan failed: ${clamavError.message}. Falling back to basic scan.`); - + // Fallback to basic scanning return this.basicScan(buffer, filename); } @@ -82,7 +82,7 @@ export class MalwareScannerService { client.on('connect', () => { this.logger.debug('Connected to ClamAV daemon'); - + // Send INSTREAM command client.write('nINSTREAM\0'); @@ -101,7 +101,9 @@ export class MalwareScannerService { }); client.on('data', (data: Buffer) => { - if (scanned) return; + if (scanned) { + return; + } scanned = true; clearTimeout(timeout); @@ -122,7 +124,7 @@ export class MalwareScannerService { client.destroy(); }); - client.on('error', (error) => { + client.on('error', error => { if (!scanned) { clearTimeout(timeout); reject(new Error(`ClamAV connection error: ${error.message}`)); @@ -146,26 +148,26 @@ export class MalwareScannerService { */ private basicScan(buffer: Buffer, filename?: string): ScanResult { const startTime = Date.now(); - + // Known malware signatures (simplified examples) const suspiciousPatterns = [ // EICAR test file (safe test signature) /X5O!P%@AP\[4\\PZX54\(P\^\)7CC\)7\}\$EICAR/, - + // Common malware indicators /mz\s*[\x00-\xff]{0,256}(stub|payload|shellcode)/i, / { - return new Promise((resolve) => { + return new Promise(resolve => { const client = new net.Socket(); const timeout = setTimeout(() => { client.destroy(); diff --git a/src/security/validators/secure-file.validator.ts b/src/security/validators/secure-file.validator.ts index b99daca5..a73e23bf 100644 --- a/src/security/validators/secure-file.validator.ts +++ b/src/security/validators/secure-file.validator.ts @@ -41,10 +41,7 @@ export class SecureFileValidator { // 3. Validate file type using magic numbers (if enabled) if (config.validateMagicNumbers) { - const validationResult = this.fileValidationService.validateFile( - file.buffer, - file.mimetype, - ); + const validationResult = this.fileValidationService.validateFile(file.buffer, file.mimetype); if (!validationResult.isValid) { throw new BadRequestException( @@ -55,14 +52,9 @@ export class SecureFileValidator { // Check if detected MIME type is allowed if ( validationResult.fileType && - !this.fileValidationService.isMimeTypeAllowed( - validationResult.fileType.mime, - config.allowedMimeTypes, - ) + !this.fileValidationService.isMimeTypeAllowed(validationResult.fileType.mime, config.allowedMimeTypes) ) { - throw new BadRequestException( - `File type '${validationResult.fileType.mime}' is not allowed`, - ); + throw new BadRequestException(`File type '${validationResult.fileType.mime}' is not allowed`); } } else { // Fallback to basic MIME type check @@ -75,10 +67,7 @@ export class SecureFileValidator { // 4. Scan for malware (if enabled) if (config.scanForMalware) { - const scanResult = await this.malwareScannerService.scanFile( - file.buffer, - file.originalname, - ); + const scanResult = await this.malwareScannerService.scanFile(file.buffer, file.originalname); if (!scanResult.isClean) { throw new BadRequestException( @@ -107,10 +96,12 @@ export class SecureFileValidator { * Format bytes to human-readable string */ private formatBytes(bytes: number): string { - if (bytes === 0) return '0 Bytes'; + if (bytes === 0) { + return '0 Bytes'; + } const k = 1024; const sizes = ['Bytes', 'KB', 'MB', 'GB']; const i = Math.floor(Math.log(bytes) / Math.log(k)); - return Math.round(bytes / Math.pow(k, i) * 100) / 100 + ' ' + sizes[i]; + return `${Math.round((bytes / Math.pow(k, i)) * 100) / 100} ${sizes[i]}`; } } diff --git a/src/static-cache/cache-warming.service.ts b/src/static-cache/cache-warming.service.ts index 6335f0d1..38e7df2e 100644 --- a/src/static-cache/cache-warming.service.ts +++ b/src/static-cache/cache-warming.service.ts @@ -132,7 +132,7 @@ export class CacheWarmingService implements OnModuleInit, OnModuleDestroy { const chunks = this.chunkArray(job.urls, concurrency); for (const chunk of chunks) { - const promises = chunk.map((url) => this.warmUrl(url, job)); + const promises = chunk.map(url => this.warmUrl(url, job)); const results = await Promise.allSettled(promises); for (const promiseResult of results) { @@ -207,7 +207,7 @@ export class CacheWarmingService implements OnModuleInit, OnModuleDestroy { async warmPopularContent(limit: number = 50): Promise { // Get analytics to find popular content const analytics = await this.cacheService.getAnalytics(); - const popularUrls = analytics.topAccessedEntries.slice(0, limit).map((entry) => entry.key); + const popularUrls = analytics.topAccessedEntries.slice(0, limit).map(entry => entry.key); const job: CacheWarmingJob = { id: uuidv4(), @@ -228,10 +228,7 @@ export class CacheWarmingService implements OnModuleInit, OnModuleDestroy { return await this.executeJob(job.id); } - async warmUserBasedContent( - userId: string, - userPreferences?: Record, - ): Promise { + async warmUserBasedContent(userId: string, userPreferences?: Record): Promise { // Generate URLs based on user preferences and behavior const urls = this.generateUserBasedUrls(userId, userPreferences); @@ -277,9 +274,7 @@ export class CacheWarmingService implements OnModuleInit, OnModuleDestroy { return await this.executeJob(job.id); } - async createStrategy( - strategy: Omit, - ): Promise { + async createStrategy(strategy: Omit): Promise { const newStrategy: CacheWarmingStrategy = { ...strategy, id: uuidv4(), @@ -342,16 +337,14 @@ export class CacheWarmingService implements OnModuleInit, OnModuleDestroy { const resultsKey = `${this.resultsPrefix}recent`; const results = await this.redisService.lrange(resultsKey, 0, limit - 1); - return results.map((result) => JSON.parse(result)); + return results.map(result => JSON.parse(result)); } async getWarmingStats(days: number = 7): Promise { const history = await this.getWarmingHistory(1000); const cutoffDate = new Date(Date.now() - days * 24 * 60 * 60 * 1000); - const recentHistory = history.filter( - (result) => new Date(result.startTime) >= cutoffDate, - ); + const recentHistory = history.filter(result => new Date(result.startTime) >= cutoffDate); const stats = { totalJobs: recentHistory.length, @@ -383,12 +376,8 @@ export class CacheWarmingService implements OnModuleInit, OnModuleDestroy { stats.dailyStats[day] = (stats.dailyStats[day] || 0) + 1; } - stats.successRate = - totalSuccesses + totalFailures > 0 - ? totalSuccesses / (totalSuccesses + totalFailures) - : 0; - stats.averageDuration = - recentHistory.length > 0 ? totalDuration / recentHistory.length : 0; + stats.successRate = totalSuccesses + totalFailures > 0 ? totalSuccesses / (totalSuccesses + totalFailures) : 0; + stats.averageDuration = recentHistory.length > 0 ? totalDuration / recentHistory.length : 0; return stats; } @@ -454,7 +443,7 @@ export class CacheWarmingService implements OnModuleInit, OnModuleDestroy { // Keep only last 30 days of history const history = await this.redisService.lrange(resultsKey, 0, -1); - const recentHistory = history.filter((record) => { + const recentHistory = history.filter(record => { const parsed = JSON.parse(record); return new Date(parsed.startTime) >= thirtyDaysAgo; }); @@ -526,14 +515,24 @@ export class CacheWarmingService implements OnModuleInit, OnModuleDestroy { private getContentTypeFromUrl(url: string): CacheContentType { const extension = url.split('.').pop()?.toLowerCase(); - if (extension === 'css') return CacheContentType.CSS; - if (extension === 'js') return CacheContentType.JS; + if (extension === 'css') { + return CacheContentType.CSS; + } + if (extension === 'js') { + return CacheContentType.JS; + } if (['png', 'jpg', 'jpeg', 'gif', 'svg', 'ico'].includes(extension || '')) { return CacheContentType.IMAGE; } - if (extension === 'html') return CacheContentType.HTML; - if (extension === 'json') return CacheContentType.JSON; - if (extension === 'txt') return CacheContentType.TEXT; + if (extension === 'html') { + return CacheContentType.HTML; + } + if (extension === 'json') { + return CacheContentType.JSON; + } + if (extension === 'txt') { + return CacheContentType.TEXT; + } return CacheContentType.HTML; // Default } @@ -546,10 +545,7 @@ export class CacheWarmingService implements OnModuleInit, OnModuleDestroy { return chunks; } - private generateUserBasedUrls( - userId: string, - preferences?: Record, - ): string[] { + private generateUserBasedUrls(userId: string, preferences?: Record): string[] { const urls: string[] = []; // Generate URLs based on user preferences @@ -611,9 +607,7 @@ export class CacheWarmingService implements OnModuleInit, OnModuleDestroy { return urls; } - private async executeCustomStrategy( - strategy: CacheWarmingStrategy, - ): Promise { + private async executeCustomStrategy(strategy: CacheWarmingStrategy): Promise { const urls = (strategy.config.urls as string[]) || []; const job: CacheWarmingJob = { @@ -729,4 +723,4 @@ export class CacheWarmingService implements OnModuleInit, OnModuleDestroy { await this.redisService.ltrim(resultsKey, 0, 999); // Keep last 1000 results await this.redisService.expire(resultsKey, 86400 * 30); // 30 days TTL } -} \ No newline at end of file +} From 5b2ad42d5b5d8bf9c3ec0aa19f877dab39b4de0b Mon Sep 17 00:00:00 2001 From: robertocarlous Date: Wed, 25 Mar 2026 18:07:18 +0100 Subject: [PATCH 2/3] fix ci error --- loadtests/propchain-loadtest.js | 22 ++++++++++++++++++++++ package.json | 1 + 2 files changed, 23 insertions(+) create mode 100644 loadtests/propchain-loadtest.js diff --git a/loadtests/propchain-loadtest.js b/loadtests/propchain-loadtest.js new file mode 100644 index 00000000..f5236e7f --- /dev/null +++ b/loadtests/propchain-loadtest.js @@ -0,0 +1,22 @@ +import http from 'k6/http'; +import { check, sleep } from 'k6'; + +export const options = { + vus: 5, + duration: '10s', + thresholds: { + http_req_failed: ['rate<0.01'], + http_req_duration: ['p(95)<1000'], + }, +}; + +const API_URL = __ENV.API_URL || 'http://localhost:3000'; + +export default function () { + const res = http.get(`${API_URL}/health`); + check(res, { + 'health status is 200': r => r.status === 200, + }); + sleep(1); +} + diff --git a/package.json b/package.json index 5881b995..d8a87111 100644 --- a/package.json +++ b/package.json @@ -52,6 +52,7 @@ "db:backup": "bash scripts/backup.sh", "db:restore": "bash scripts/restore.sh", "db:benchmark": "ts-node test/database/performance.benchmark.ts", + "loadtest:ci": "mkdir -p artifacts && docker run --rm -i -e API_URL=${API_URL:-http://localhost:3000} -v \"$PWD:/work\" -w /work grafana/k6 run --out json=artifacts/k6-results.json loadtests/propchain-loadtest.js", "docs:generate": "typedoc --skipErrorChecking" }, "dependencies": { From 647c0892fc41df74c4037141f2695d883fe98a66 Mon Sep 17 00:00:00 2001 From: robertocarlous Date: Wed, 25 Mar 2026 18:17:06 +0100 Subject: [PATCH 3/3] fix ci error 1 --- package.json | 2 +- scripts/loadtest-ci.sh | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100755 scripts/loadtest-ci.sh diff --git a/package.json b/package.json index d8a87111..70b4775c 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "db:backup": "bash scripts/backup.sh", "db:restore": "bash scripts/restore.sh", "db:benchmark": "ts-node test/database/performance.benchmark.ts", - "loadtest:ci": "mkdir -p artifacts && docker run --rm -i -e API_URL=${API_URL:-http://localhost:3000} -v \"$PWD:/work\" -w /work grafana/k6 run --out json=artifacts/k6-results.json loadtests/propchain-loadtest.js", + "loadtest:ci": "bash scripts/loadtest-ci.sh", "docs:generate": "typedoc --skipErrorChecking" }, "dependencies": { diff --git a/scripts/loadtest-ci.sh b/scripts/loadtest-ci.sh new file mode 100755 index 00000000..95685bb8 --- /dev/null +++ b/scripts/loadtest-ci.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +# k6 runs inside Docker; "localhost" / "127.0.0.1" in API_URL refer to the container, not the host. +# GitHub Actions often sets API_URL=http://localhost:3000 โ€” rewrite so traffic reaches the app on the runner. +if [[ "${API_URL:-}" =~ ^https?://(localhost|127\.0\.0\.1)(:([0-9]+))?(/|$) ]]; then + PORT="${BASH_REMATCH[3]:-3000}" + export API_URL="http://host.docker.internal:${PORT}" +fi +export API_URL="${API_URL:-http://host.docker.internal:3000}" + +mkdir -p artifacts + +exec docker run --rm -i \ + --user "$(id -u):$(id -g)" \ + --add-host=host.docker.internal:host-gateway \ + -e "API_URL=${API_URL}" \ + -v "$PWD:/work" \ + -w /work \ + grafana/k6:latest run \ + --out json=artifacts/k6-results.json \ + loadtests/propchain-loadtest.js