diff --git a/Dockerfile b/Dockerfile index 1a978182..b9710139 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM node:18-alpine AS base # Install dependencies only when needed FROM base AS deps -RUN apk add --no-cache libc6-compat +RUN apk add --no-cache libc6-compat postgresql-client bash WORKDIR /app # Install dependencies based on the preferred package manager @@ -34,6 +34,10 @@ COPY --from=builder --chown=nestjs:nodejs /app/dist ./dist COPY --from=builder --chown=nestjs:nodejs /app/node_modules ./node_modules COPY --from=builder --chown=nestjs:nodejs /app/package.json ./package.json +# Copy scripts for backups +COPY --from=builder --chown=nestjs:nodejs /app/scripts ./scripts +RUN chmod +x scripts/*.sh + # Create logs directory RUN mkdir -p logs && chown nestjs:nodejs logs diff --git a/build_output.txt b/build_output.txt new file mode 100644 index 00000000..ff308fea Binary files /dev/null and b/build_output.txt differ diff --git a/build_output_2.txt b/build_output_2.txt new file mode 100644 index 00000000..e69069a3 Binary files /dev/null and b/build_output_2.txt differ diff --git a/scripts/backup.sh b/scripts/backup.sh index d9020854..567e997a 100644 --- a/scripts/backup.sh +++ b/scripts/backup.sh @@ -83,6 +83,15 @@ pg_dump \ echo "Calculating checksum..." sha256sum "$BACKUP_DIR/$BACKUP_NAME.dump" > "$BACKUP_DIR/$BACKUP_NAME.sha256" +# Validate backup +echo "Validating backup..." +if pg_restore -l "$BACKUP_DIR/$BACKUP_NAME.dump" > /dev/null 2>&1; then + echo "Backup validation passed: Dump is readable." +else + echo "ERROR: Backup validation failed: Dump is corrupted or unreadable!" + exit 1 +fi + # Compress backup echo "Compressing backup..." gzip -k "$BACKUP_DIR/$BACKUP_NAME.dump" diff --git a/scripts/test-restore.sh b/scripts/test-restore.sh new file mode 100644 index 00000000..a13b3c4b --- /dev/null +++ b/scripts/test-restore.sh @@ -0,0 +1,115 @@ +#!/bin/bash + +# PropChain Database Restore Testing Script +# Usage: ./test-restore.sh + +set -e + +# Configuration +DATABASE_URL="${DATABASE_URL:-postgresql://postgres:password@localhost:5432/propchain}" +TEST_DB_NAME="propchain_test_restore_$(date +%s)" + +# Parse DATABASE_URL +parse_db_url() { + local url="$1" + # Remove protocol + url="${url#postgresql://}" + + # Extract user:password + local auth="${url%%@*}" + DB_USER="${auth%%:*}" + DB_PASSWORD="${auth#*:}" + + # Extract host:port/database + url="${url#*@}" + local host_port="${url%%/*}" + DB_HOST="${host_port%%:*}" + DB_PORT="${host_port#*:}" +} + +# Check arguments +if [ -z "$1" ]; then + echo "Usage: $0 " + exit 1 +fi + +BACKUP_FILE="$1" + +# Check if backup file exists +if [ ! -f "$BACKUP_FILE" ]; then + echo "Error: Backup file not found: $BACKUP_FILE" + exit 1 +fi + +# Parse database URL +parse_db_url "$DATABASE_URL" + +echo "=== PropChain Backup Restoration Test ===" +echo "Backup File: $BACKUP_FILE" +echo "Test Database: $TEST_DB_NAME" +echo "" + +# Set password for psql/pg_restore +export PGPASSWORD="$DB_PASSWORD" + +# Decompress if necessary +RESTORE_FILE="$BACKUP_FILE" +TEMP_FILE="" +if [[ "$BACKUP_FILE" == *.gz ]]; then + echo "Decompressing backup..." + RESTORE_FILE="$(mktemp --suffix=.dump)" + TEMP_FILE="$RESTORE_FILE" + gunzip -c "$BACKUP_FILE" > "$RESTORE_FILE" +fi + +# Create test database +echo "Creating test database $TEST_DB_NAME..." +psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d postgres -c "CREATE DATABASE $TEST_DB_NAME;" + +# Function to clean up +cleanup() { + echo "Cleaning up..." + psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d postgres -c "DROP DATABASE IF EXISTS $TEST_DB_NAME;" + if [ -n "$TEMP_FILE" ] && [ -f "$TEMP_FILE" ]; then + rm "$TEMP_FILE" + fi +} + +trap cleanup EXIT + +# Restore backup to test database +echo "Restoring backup to test database..." +pg_restore \ + -h "$DB_HOST" \ + -p "$DB_PORT" \ + -U "$DB_USER" \ + -d "$TEST_DB_NAME" \ + --no-owner \ + --no-privileges \ + "$RESTORE_FILE" > /dev/null 2>&1 + +# Run verification checks +echo "Running verification checks..." + +# Check 1: Can we connect? +psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$TEST_DB_NAME" -c "SELECT 1;" > /dev/null + +# Check 2: Check for essential tables (e.g., users) +TABLE_COUNT=$(psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$TEST_DB_NAME" -t -c "SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public';") +echo "Found $TABLE_COUNT tables in restored database." + +if [ "$TABLE_COUNT" -lt 5 ]; then + echo "ERROR: Too few tables found ($TABLE_COUNT). Restore might have failed." + exit 1 +fi + +# Check 3: Check data in key tables +# We use -t to get only the value +USER_COUNT=$(psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$TEST_DB_NAME" -t -c "SELECT count(*) FROM \"users\";" | xargs) +PROPERTY_COUNT=$(psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$TEST_DB_NAME" -t -c "SELECT count(*) FROM \"properties\";" | xargs) + +echo "Restored User Count: $USER_COUNT" +echo "Restored Property Count: $PROPERTY_COUNT" + +echo "" +echo "=== Restoration Test Successful ===" diff --git a/src/app.module.ts b/src/app.module.ts index 895dfc0b..8e219a0f 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -41,6 +41,7 @@ import { ValuationModule } from './valuation/valuation.module'; import { ApiKeysModule } from './api-keys/api-keys.module'; import { DocumentsModule } from './documents/documents.module'; import { SecurityModule } from './security/security.module'; +import { BackupRecoveryModule } from './backup-recovery/backup-recovery.module'; // Compliance & Security Modules import { AuditModule } from './common/audit/audit.module'; @@ -126,6 +127,7 @@ import { ObservabilityModule } from './observability/observability.module'; AuditModule, RbacModule, ObservabilityModule, + BackupRecoveryModule, ], controllers: [ AuditController, // Add the audit controller diff --git a/src/auth/auth.controller.ts b/src/auth/auth.controller.ts index 00853275..5f0bb63b 100644 --- a/src/auth/auth.controller.ts +++ b/src/auth/auth.controller.ts @@ -102,10 +102,13 @@ export class AuthController { @ApiStandardErrorResponse([400, 401]) @HttpCode(HttpStatus.OK) async login(@Body() loginDto: LoginDto, @Req() req: Request) { - return this.authService.login({ - email: loginDto.email, - password: loginDto.password, - }, this.getRequestMeta(req)); + return this.authService.login( + { + email: loginDto.email, + password: loginDto.password, + }, + this.getRequestMeta(req), + ); } /** @@ -137,10 +140,13 @@ export class AuthController { @ApiStandardErrorResponse([401]) @HttpCode(HttpStatus.OK) async web3Login(@Body() loginDto: LoginWeb3Dto, @Req() req: Request) { - return this.authService.login({ - walletAddress: loginDto.walletAddress, - signature: loginDto.signature, - }, this.getRequestMeta(req)); + return this.authService.login( + { + walletAddress: loginDto.walletAddress, + signature: loginDto.signature, + }, + this.getRequestMeta(req), + ); } /** diff --git a/src/auth/auth.service.ts b/src/auth/auth.service.ts index 71ce4f44..fa6b1067 100644 --- a/src/auth/auth.service.ts +++ b/src/auth/auth.service.ts @@ -362,7 +362,11 @@ export class AuthService { } const ttl = this.getSessionExpiry(session.lastActivity || session.createdAt); if (ttl > 0) { - await this.redisService.setex(tokenRevocationRedisKeys.accessRevoked(session.jti), Math.ceil(ttl / 1000), userId); + await this.redisService.setex( + tokenRevocationRedisKeys.accessRevoked(session.jti), + Math.ceil(ttl / 1000), + userId, + ); } } await this.redisService.del(tokenRevocationRedisKeys.activeSession(userId, sessionId)); @@ -474,11 +478,7 @@ export class AuthService { fingerprint: sessionMeta.fingerprint, }), ); - await this.redisService.setex( - tokenRevocationRedisKeys.userRefreshSession(user.id), - refreshTtl, - refreshSessionId, - ); + await this.redisService.setex(tokenRevocationRedisKeys.userRefreshSession(user.id), refreshTtl, refreshSessionId); const sessionExpiry = this.configService.get('SESSION_TIMEOUT', 3600); await this.redisService.setex(tokenRevocationRedisKeys.accessSession(jti), sessionExpiry, sessionId); @@ -563,7 +563,11 @@ export class AuthService { sessionExpiry, JSON.stringify(session), ); - await this.redisService.setex(tokenRevocationRedisKeys.accessSession(session.jti), sessionExpiry, session.sessionId); + await this.redisService.setex( + tokenRevocationRedisKeys.accessSession(session.jti), + sessionExpiry, + session.sessionId, + ); } private buildFingerprint(requestMeta?: { ip?: string; userAgent?: string }): string { diff --git a/src/backup-recovery/backup-verification.service.ts b/src/backup-recovery/backup-verification.service.ts index 8200eda2..2c0a0457 100644 --- a/src/backup-recovery/backup-verification.service.ts +++ b/src/backup-recovery/backup-verification.service.ts @@ -111,6 +111,11 @@ export class BackupVerificationService { // Verify content structure await this.verifyBackupStructure(backupPath, check); + // Full restoration test (automated testing) + if (check.accessible && check.tableIntegrity.errors.length === 0) { + await this.verifyRestoration(backupPath, check); + } + // Check restorability check.restorable = check.tableIntegrity.errors.length === 0 && check.accessible; @@ -212,6 +217,49 @@ export class BackupVerificationService { } } + /** + * Perform full restoration test + */ + private async verifyRestoration(filePath: string, check: BackupIntegrityCheck): Promise { + if (!filePath.endsWith('.dump') && !filePath.endsWith('.dump.gz')) { + return; // Only test restoration for postgres dumps + } + + this.logger.log(`Starting restoration test for ${filePath}`); + const execAsync = promisify(exec); + const scriptPath = path.join(process.cwd(), 'scripts', 'test-restore.sh'); + + try { + // Check if script exists and is executable + if (!fsSync.existsSync(scriptPath)) { + this.logger.warn(`Restoration test script not found at ${scriptPath}`); + return; + } + + await fs.chmod(scriptPath, 0o755); + + const databaseUrl = this.configService.get('DATABASE_URL'); + const { stdout } = await execAsync(`bash "${scriptPath}" "${filePath}"`, { + env: { ...process.env, DATABASE_URL: databaseUrl }, + }); + + this.logger.log(`Restoration test output: ${stdout}`); + + // Look for User and Property counts in output + const userCountMatch = stdout.match(/Restored User Count: (\d+)/); + const propertyCountMatch = stdout.match(/Restored Property Count: (\d+)/); + + if (userCountMatch && propertyCountMatch) { + this.logger.log(`Restoration test PASSED: Users=${userCountMatch[1]}, Properties=${propertyCountMatch[1]}`); + } else { + this.logger.warn('Restoration test completed but count regex failed'); + } + } catch (error) { + this.logger.error(`Restoration test failed: ${error.message}`); + check.tableIntegrity.errors.push(`Restoration test failed: ${error.message}`); + } + } + /** * Verify tar archive backup */ diff --git a/src/backup-recovery/database-backup.service.ts b/src/backup-recovery/database-backup.service.ts index 64c44024..b11670f0 100644 --- a/src/backup-recovery/database-backup.service.ts +++ b/src/backup-recovery/database-backup.service.ts @@ -16,6 +16,7 @@ import { BackupConfiguration, PointInTimeRecovery, } from './backup.types'; +import { BackupVerificationService } from './backup-verification.service'; const execAsync = promisify(exec); @@ -34,6 +35,7 @@ export class DatabaseBackupService { constructor( private readonly prisma: PrismaService, private readonly configService: ConfigService, + private readonly verificationService: BackupVerificationService, ) { this.backupDir = path.join(process.cwd(), this.configService.get('BACKUP_DIR') || 'backups/database'); this.initializeBackupDirectory(); @@ -142,6 +144,13 @@ export class DatabaseBackupService { this.logger.log(`Full backup completed: ${backupId} (${metadata.size} bytes in ${metadata.duration}ms)`); + // Trigger immediate verification + try { + await this.verificationService.verifyBackup(backupId); + } catch (verifyError) { + this.logger.warn(`Immediate verification failed for backup ${backupId}: ${verifyError.message}`); + } + return metadata; } catch (error) { this.logger.error(`Full backup failed: ${error.message}`, error.stack); diff --git a/src/communication/email/email.queue.ts b/src/communication/email/email.queue.ts index bcf8e6d1..1d395683 100644 --- a/src/communication/email/email.queue.ts +++ b/src/communication/email/email.queue.ts @@ -446,11 +446,7 @@ export class EmailQueueService implements OnModuleDestroy { * Set up queue event listeners */ private setupEventListeners(): void { - const registerListener = ( - queue: Bull.Queue, - event: string, - handler: (...args: any[]) => void, - ) => { + const registerListener = (queue: Bull.Queue, event: string, handler: (...args: any[]) => void) => { queue.on(event, handler); this.listenerDisposers.push(() => queue.removeListener(event, handler)); }; @@ -523,20 +519,23 @@ export class EmailQueueService implements OnModuleDestroy { } private startQueueCleanupMonitoring(): void { - this.queueCleanupMonitor = setInterval(async () => { - try { - await Promise.all([ - this.emailQueue.clean(24 * 60 * 60 * 1000, 'completed'), - this.emailQueue.clean(7 * 24 * 60 * 60 * 1000, 'failed'), - this.priorityQueue.clean(24 * 60 * 60 * 1000, 'completed'), - this.priorityQueue.clean(7 * 24 * 60 * 60 * 1000, 'failed'), - this.batchQueue.clean(24 * 60 * 60 * 1000, 'completed'), - this.batchQueue.clean(7 * 24 * 60 * 60 * 1000, 'failed'), - ]); - } catch (error) { - this.logger.error('Failed to clean queue history', error); - } - }, this.configService.get('EMAIL_QUEUE_CLEANUP_INTERVAL_MS', 300000)); + this.queueCleanupMonitor = setInterval( + async () => { + try { + await Promise.all([ + this.emailQueue.clean(24 * 60 * 60 * 1000, 'completed'), + this.emailQueue.clean(7 * 24 * 60 * 60 * 1000, 'failed'), + this.priorityQueue.clean(24 * 60 * 60 * 1000, 'completed'), + this.priorityQueue.clean(7 * 24 * 60 * 60 * 1000, 'failed'), + this.batchQueue.clean(24 * 60 * 60 * 1000, 'completed'), + this.batchQueue.clean(7 * 24 * 60 * 60 * 1000, 'failed'), + ]); + } catch (error) { + this.logger.error('Failed to clean queue history', error); + } + }, + this.configService.get('EMAIL_QUEUE_CLEANUP_INTERVAL_MS', 300000), + ); this.queueCleanupMonitor.unref?.(); } @@ -573,7 +572,9 @@ export class EmailQueueService implements OnModuleDestroy { delete job.data.text; } } catch (error) { - this.logger.warn(`Failed to cleanup job resources for ${job?.id}: ${error instanceof Error ? error.message : error}`); + this.logger.warn( + `Failed to cleanup job resources for ${job?.id}: ${error instanceof Error ? error.message : error}`, + ); } } diff --git a/src/models/api-key.entity.ts b/src/models/api-key.entity.ts index 8cfdf1b8..1fceba22 100644 --- a/src/models/api-key.entity.ts +++ b/src/models/api-key.entity.ts @@ -10,6 +10,9 @@ export class ApiKey implements PrismaApiKey { lastUsedAt: Date | null; isActive: boolean; rateLimit: number | null; + keyVersion: number; + lastRotatedAt: Date | null; + rotationDueAt: Date | null; createdAt: Date; updatedAt: Date; } diff --git a/src/models/user.entity.ts b/src/models/user.entity.ts index a79c8dcc..a4cdcae3 100644 --- a/src/models/user.entity.ts +++ b/src/models/user.entity.ts @@ -7,9 +7,6 @@ export class User implements PrismaUser { email: string; password: string | null; - firstName: string | null; - lastName: string | null; - walletAddress: string | null; isVerified: boolean; @@ -64,8 +61,6 @@ export class UserRelationship { export type CreateUserInput = { email: string; password?: string; - firstName?: string; - lastName?: string; walletAddress?: string; role?: UserRole; roleId?: string; diff --git a/src/security/services/input-sanitization.service.ts b/src/security/services/input-sanitization.service.ts index 00c9b4ec..9c79bdde 100644 --- a/src/security/services/input-sanitization.service.ts +++ b/src/security/services/input-sanitization.service.ts @@ -76,7 +76,6 @@ export class InputSanitizationService { if (this.matchesPattern(value, SQL_INJECTION_PATTERNS)) { throw new BadRequestException(`Potential SQL injection detected in ${path}`); } - } private containsIllegalControlCharacters(value: string): boolean { diff --git a/src/users/user.controller.ts b/src/users/user.controller.ts index cdb503cf..4ad9feb5 100644 --- a/src/users/user.controller.ts +++ b/src/users/user.controller.ts @@ -1,13 +1,11 @@ import { Controller, Get, Post, Body, Patch, Param, Delete, UseGuards } from '@nestjs/common'; import { UserService } from './user.service'; import { CreateUserDto } from './dto/create-user.dto'; -import { ApiTags, ApiOperation, ApiResponse } from '@nestjs/swagger'; - import { - ApiResponse, ApiTags, ApiOperation, - ApiBody, + ApiResponse, + ApiBearerAuth, ApiParam, ApiExtraModels, ApiOkResponse, @@ -15,15 +13,13 @@ import { ApiNotFoundResponse, ApiConflictResponse, ApiCreatedResponse, - ApiBearerAuth, - ApiDeprecated, + ApiBody, ApiQuery, ApiConsumes, ApiProduces, ApiProperty, ApiPropertyOptional, ApiResponseOptions, - ApiVersion, } from '@nestjs/swagger'; import { UserResponseDto } from './dto/user-response.dto'; import { UpdateUserDto } from './dto/update-user.dto'; @@ -42,15 +38,11 @@ export class UserController { @ApiCreatedResponse({ description: 'User created successfully.', type: UserResponseDto, - examples: { - success: { - value: { - id: 'user_abc123', - email: 'john.doe@example.com', - firstName: 'John', - lastName: 'Doe', - isEmailVerified: false, - }, + schema: { + example: { + id: 'user_abc123', + email: 'john.doe@example.com', + isEmailVerified: false, }, }, }) diff --git a/src/users/user.service.ts b/src/users/user.service.ts index b0ca1194..9d6d30f2 100644 --- a/src/users/user.service.ts +++ b/src/users/user.service.ts @@ -486,7 +486,8 @@ export class UserService { return this.cacheService.wrap( `user:followers:${userId}:${limit}`, - () => this.monitorQuery('users.getFollowers', { userId, limit }, () => this.prisma.userRelationship.findMany(query)), + () => + this.monitorQuery('users.getFollowers', { userId, limit }, () => this.prisma.userRelationship.findMany(query)), { l1Ttl: 30, l2Ttl: 120, tags: ['user', `user:${userId}`] }, ); } @@ -520,7 +521,8 @@ export class UserService { return this.cacheService.wrap( `user:following:${userId}:${limit}`, - () => this.monitorQuery('users.getFollowing', { userId, limit }, () => this.prisma.userRelationship.findMany(query)), + () => + this.monitorQuery('users.getFollowing', { userId, limit }, () => this.prisma.userRelationship.findMany(query)), { l1Ttl: 30, l2Ttl: 120, tags: ['user', `user:${userId}`] }, ); }