Implement automated database backup process

Dockerfile

@@ -3,7 +3,7 @@ FROM node:18-alpine AS base
 
 # Install dependencies only when needed
 FROM base AS deps
-RUN apk add --no-cache libc6-compat
+RUN apk add --no-cache libc6-compat postgresql-client bash
 WORKDIR /app
 
 # Install dependencies based on the preferred package manager
@@ -34,6 +34,10 @@ COPY --from=builder --chown=nestjs:nodejs /app/dist ./dist
 COPY --from=builder --chown=nestjs:nodejs /app/node_modules ./node_modules
 COPY --from=builder --chown=nestjs:nodejs /app/package.json ./package.json
 
+# Copy scripts for backups
+COPY --from=builder --chown=nestjs:nodejs /app/scripts ./scripts
+RUN chmod +x scripts/*.sh
+
 # Create logs directory
 RUN mkdir -p logs && chown nestjs:nodejs logs
 

scripts/backup.sh

@@ -83,6 +83,15 @@ pg_dump \
 echo "Calculating checksum..."
 sha256sum "$BACKUP_DIR/$BACKUP_NAME.dump" > "$BACKUP_DIR/$BACKUP_NAME.sha256"
 
+# Validate backup
+echo "Validating backup..."
+if pg_restore -l "$BACKUP_DIR/$BACKUP_NAME.dump" > /dev/null 2>&1; then
+    echo "Backup validation passed: Dump is readable."
+else
+    echo "ERROR: Backup validation failed: Dump is corrupted or unreadable!"
+    exit 1
+fi
+
 # Compress backup
 echo "Compressing backup..."
 gzip -k "$BACKUP_DIR/$BACKUP_NAME.dump"

scripts/test-restore.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+
+# PropChain Database Restore Testing Script
+# Usage: ./test-restore.sh <backup_file>
+
+set -e
+
+# Configuration
+DATABASE_URL="${DATABASE_URL:-postgresql://postgres:password@localhost:5432/propchain}"
+TEST_DB_NAME="propchain_test_restore_$(date +%s)"
+
+# Parse DATABASE_URL
+parse_db_url() {
+    local url="$1"
+    # Remove protocol
+    url="${url#postgresql://}"
+
+    # Extract user:password
+    local auth="${url%%@*}"
+    DB_USER="${auth%%:*}"
+    DB_PASSWORD="${auth#*:}"
+
+    # Extract host:port/database
+    url="${url#*@}"
+    local host_port="${url%%/*}"
+    DB_HOST="${host_port%%:*}"
+    DB_PORT="${host_port#*:}"
+}
+
+# Check arguments
+if [ -z "$1" ]; then
+    echo "Usage: $0 <backup_file>"
+    exit 1
+fi
+
+BACKUP_FILE="$1"
+
+# Check if backup file exists
+if [ ! -f "$BACKUP_FILE" ]; then
+    echo "Error: Backup file not found: $BACKUP_FILE"
+    exit 1
+fi
+
+# Parse database URL
+parse_db_url "$DATABASE_URL"
+
+echo "=== PropChain Backup Restoration Test ==="
+echo "Backup File: $BACKUP_FILE"
+echo "Test Database: $TEST_DB_NAME"
+echo ""
+
+# Set password for psql/pg_restore
+export PGPASSWORD="$DB_PASSWORD"
+
+# Decompress if necessary
+RESTORE_FILE="$BACKUP_FILE"
+TEMP_FILE=""
+if [[ "$BACKUP_FILE" == *.gz ]]; then
+    echo "Decompressing backup..."
+    RESTORE_FILE="$(mktemp --suffix=.dump)"
+    TEMP_FILE="$RESTORE_FILE"
+    gunzip -c "$BACKUP_FILE" > "$RESTORE_FILE"
+fi
+
+# Create test database
+echo "Creating test database $TEST_DB_NAME..."
+psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d postgres -c "CREATE DATABASE $TEST_DB_NAME;"
+
+# Function to clean up
+cleanup() {
+    echo "Cleaning up..."
+    psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d postgres -c "DROP DATABASE IF EXISTS $TEST_DB_NAME;"
+    if [ -n "$TEMP_FILE" ] && [ -f "$TEMP_FILE" ]; then
+        rm "$TEMP_FILE"
+    fi
+}
+
+trap cleanup EXIT
+
+# Restore backup to test database
+echo "Restoring backup to test database..."
+pg_restore \
+    -h "$DB_HOST" \
+    -p "$DB_PORT" \
+    -U "$DB_USER" \
+    -d "$TEST_DB_NAME" \
+    --no-owner \
+    --no-privileges \
+    "$RESTORE_FILE" > /dev/null 2>&1
+
+# Run verification checks
+echo "Running verification checks..."
+
+# Check 1: Can we connect?
+psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$TEST_DB_NAME" -c "SELECT 1;" > /dev/null
+
+# Check 2: Check for essential tables (e.g., users)
+TABLE_COUNT=$(psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$TEST_DB_NAME" -t -c "SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public';")
+echo "Found $TABLE_COUNT tables in restored database."
+
+if [ "$TABLE_COUNT" -lt 5 ]; then
+    echo "ERROR: Too few tables found ($TABLE_COUNT). Restore might have failed."
+    exit 1
+fi
+
+# Check 3: Check data in key tables
+# We use -t to get only the value
+USER_COUNT=$(psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$TEST_DB_NAME" -t -c "SELECT count(*) FROM \"users\";" | xargs)
+PROPERTY_COUNT=$(psql -h "$DB_HOST" -p "$DB_PORT" -U "$DB_USER" -d "$TEST_DB_NAME" -t -c "SELECT count(*) FROM \"properties\";" | xargs)
+
+echo "Restored User Count: $USER_COUNT"
+echo "Restored Property Count: $PROPERTY_COUNT"
+
+echo ""
+echo "=== Restoration Test Successful ==="

src/app.module.ts

@@ -41,6 +41,7 @@ import { ValuationModule } from './valuation/valuation.module';
 import { ApiKeysModule } from './api-keys/api-keys.module';
 import { DocumentsModule } from './documents/documents.module';
 import { SecurityModule } from './security/security.module';
+import { BackupRecoveryModule } from './backup-recovery/backup-recovery.module';
 
 // Compliance & Security Modules
 import { AuditModule } from './common/audit/audit.module';
@@ -126,6 +127,7 @@ import { ObservabilityModule } from './observability/observability.module';
     AuditModule,
     RbacModule,
     ObservabilityModule,
+    BackupRecoveryModule,
   ],
   controllers: [
     AuditController, // Add the audit controller

src/auth/auth.controller.ts

@@ -102,10 +102,13 @@ export class AuthController {
   @ApiStandardErrorResponse([400, 401])
   @HttpCode(HttpStatus.OK)
   async login(@Body() loginDto: LoginDto, @Req() req: Request) {
-    return this.authService.login({
-      email: loginDto.email,
-      password: loginDto.password,
-    }, this.getRequestMeta(req));
+    return this.authService.login(
+      {
+        email: loginDto.email,
+        password: loginDto.password,
+      },
+      this.getRequestMeta(req),
+    );
   }
 
   /**
@@ -137,10 +140,13 @@ export class AuthController {
   @ApiStandardErrorResponse([401])
   @HttpCode(HttpStatus.OK)
   async web3Login(@Body() loginDto: LoginWeb3Dto, @Req() req: Request) {
-    return this.authService.login({
-      walletAddress: loginDto.walletAddress,
-      signature: loginDto.signature,
-    }, this.getRequestMeta(req));
+    return this.authService.login(
+      {
+        walletAddress: loginDto.walletAddress,
+        signature: loginDto.signature,
+      },
+      this.getRequestMeta(req),
+    );
   }
 
   /**

src/auth/auth.service.ts

@@ -362,7 +362,11 @@ export class AuthService {
       }
       const ttl = this.getSessionExpiry(session.lastActivity || session.createdAt);
       if (ttl > 0) {
-        await this.redisService.setex(tokenRevocationRedisKeys.accessRevoked(session.jti), Math.ceil(ttl / 1000), userId);
+        await this.redisService.setex(
+          tokenRevocationRedisKeys.accessRevoked(session.jti),
+          Math.ceil(ttl / 1000),
+          userId,
+        );
       }
     }
     await this.redisService.del(tokenRevocationRedisKeys.activeSession(userId, sessionId));
@@ -474,11 +478,7 @@ export class AuthService {
         fingerprint: sessionMeta.fingerprint,
       }),
     );
-    await this.redisService.setex(
-      tokenRevocationRedisKeys.userRefreshSession(user.id),
-      refreshTtl,
-      refreshSessionId,
-    );
+    await this.redisService.setex(tokenRevocationRedisKeys.userRefreshSession(user.id), refreshTtl, refreshSessionId);
 
     const sessionExpiry = this.configService.get<number>('SESSION_TIMEOUT', 3600);
     await this.redisService.setex(tokenRevocationRedisKeys.accessSession(jti), sessionExpiry, sessionId);
@@ -563,7 +563,11 @@ export class AuthService {
       sessionExpiry,
       JSON.stringify(session),
     );
-    await this.redisService.setex(tokenRevocationRedisKeys.accessSession(session.jti), sessionExpiry, session.sessionId);
+    await this.redisService.setex(
+      tokenRevocationRedisKeys.accessSession(session.jti),
+      sessionExpiry,
+      session.sessionId,
+    );
   }
 
   private buildFingerprint(requestMeta?: { ip?: string; userAgent?: string }): string {

src/backup-recovery/backup-verification.service.ts

@@ -111,6 +111,11 @@ export class BackupVerificationService {
       // Verify content structure
       await this.verifyBackupStructure(backupPath, check);
 
+      // Full restoration test (automated testing)
+      if (check.accessible && check.tableIntegrity.errors.length === 0) {
+        await this.verifyRestoration(backupPath, check);
+      }
+
       // Check restorability
       check.restorable = check.tableIntegrity.errors.length === 0 && check.accessible;
 
@@ -212,6 +217,49 @@ export class BackupVerificationService {
     }
   }
 
+  /**
+   * Perform full restoration test
+   */
+  private async verifyRestoration(filePath: string, check: BackupIntegrityCheck): Promise<void> {
+    if (!filePath.endsWith('.dump') && !filePath.endsWith('.dump.gz')) {
+      return; // Only test restoration for postgres dumps
+    }
+
+    this.logger.log(`Starting restoration test for ${filePath}`);
+    const execAsync = promisify(exec);
+    const scriptPath = path.join(process.cwd(), 'scripts', 'test-restore.sh');
+
+    try {
+      // Check if script exists and is executable
+      if (!fsSync.existsSync(scriptPath)) {
+        this.logger.warn(`Restoration test script not found at ${scriptPath}`);
+        return;
+      }
+
+      await fs.chmod(scriptPath, 0o755);
+
+      const databaseUrl = this.configService.get('DATABASE_URL');
+      const { stdout } = await execAsync(`bash "${scriptPath}" "${filePath}"`, {
+        env: { ...process.env, DATABASE_URL: databaseUrl },
+      });
+
+      this.logger.log(`Restoration test output: ${stdout}`);
+
+      // Look for User and Property counts in output
+      const userCountMatch = stdout.match(/Restored User Count: (\d+)/);
+      const propertyCountMatch = stdout.match(/Restored Property Count: (\d+)/);
+
+      if (userCountMatch && propertyCountMatch) {
+        this.logger.log(`Restoration test PASSED: Users=${userCountMatch[1]}, Properties=${propertyCountMatch[1]}`);
+      } else {
+        this.logger.warn('Restoration test completed but count regex failed');
+      }
+    } catch (error) {
+      this.logger.error(`Restoration test failed: ${error.message}`);
+      check.tableIntegrity.errors.push(`Restoration test failed: ${error.message}`);
+    }
+  }
+
   /**
    * Verify tar archive backup
    */

src/backup-recovery/database-backup.service.ts

@@ -16,6 +16,7 @@ import {
   BackupConfiguration,
   PointInTimeRecovery,
 } from './backup.types';
+import { BackupVerificationService } from './backup-verification.service';
 
 const execAsync = promisify(exec);
 
@@ -34,6 +35,7 @@ export class DatabaseBackupService {
   constructor(
     private readonly prisma: PrismaService,
     private readonly configService: ConfigService,
+    private readonly verificationService: BackupVerificationService,
   ) {
     this.backupDir = path.join(process.cwd(), this.configService.get('BACKUP_DIR') || 'backups/database');
     this.initializeBackupDirectory();
@@ -142,6 +144,13 @@ export class DatabaseBackupService {
 
       this.logger.log(`Full backup completed: ${backupId} (${metadata.size} bytes in ${metadata.duration}ms)`);
 
+      // Trigger immediate verification
+      try {
+        await this.verificationService.verifyBackup(backupId);
+      } catch (verifyError) {
+        this.logger.warn(`Immediate verification failed for backup ${backupId}: ${verifyError.message}`);
+      }
+
       return metadata;
     } catch (error) {
       this.logger.error(`Full backup failed: ${error.message}`, error.stack);

src/communication/email/email.queue.ts

@@ -446,11 +446,7 @@ export class EmailQueueService implements OnModuleDestroy {
    * Set up queue event listeners
    */
   private setupEventListeners(): void {
-    const registerListener = (
-      queue: Bull.Queue,
-      event: string,
-      handler: (...args: any[]) => void,
-    ) => {
+    const registerListener = (queue: Bull.Queue, event: string, handler: (...args: any[]) => void) => {
       queue.on(event, handler);
       this.listenerDisposers.push(() => queue.removeListener(event, handler));
     };
@@ -523,20 +519,23 @@ export class EmailQueueService implements OnModuleDestroy {
   }
 
   private startQueueCleanupMonitoring(): void {
-    this.queueCleanupMonitor = setInterval(async () => {
-      try {
-        await Promise.all([
-          this.emailQueue.clean(24 * 60 * 60 * 1000, 'completed'),
-          this.emailQueue.clean(7 * 24 * 60 * 60 * 1000, 'failed'),
-          this.priorityQueue.clean(24 * 60 * 60 * 1000, 'completed'),
-          this.priorityQueue.clean(7 * 24 * 60 * 60 * 1000, 'failed'),
-          this.batchQueue.clean(24 * 60 * 60 * 1000, 'completed'),
-          this.batchQueue.clean(7 * 24 * 60 * 60 * 1000, 'failed'),
-        ]);
-      } catch (error) {
-        this.logger.error('Failed to clean queue history', error);
-      }
-    }, this.configService.get<number>('EMAIL_QUEUE_CLEANUP_INTERVAL_MS', 300000));
+    this.queueCleanupMonitor = setInterval(
+      async () => {
+        try {
+          await Promise.all([
+            this.emailQueue.clean(24 * 60 * 60 * 1000, 'completed'),
+            this.emailQueue.clean(7 * 24 * 60 * 60 * 1000, 'failed'),
+            this.priorityQueue.clean(24 * 60 * 60 * 1000, 'completed'),
+            this.priorityQueue.clean(7 * 24 * 60 * 60 * 1000, 'failed'),
+            this.batchQueue.clean(24 * 60 * 60 * 1000, 'completed'),
+            this.batchQueue.clean(7 * 24 * 60 * 60 * 1000, 'failed'),
+          ]);
+        } catch (error) {
+          this.logger.error('Failed to clean queue history', error);
+        }
+      },
+      this.configService.get<number>('EMAIL_QUEUE_CLEANUP_INTERVAL_MS', 300000),
+    );
     this.queueCleanupMonitor.unref?.();
   }
 
@@ -573,7 +572,9 @@ export class EmailQueueService implements OnModuleDestroy {
         delete job.data.text;
       }
     } catch (error) {
-      this.logger.warn(`Failed to cleanup job resources for ${job?.id}: ${error instanceof Error ? error.message : error}`);
+      this.logger.warn(
+        `Failed to cleanup job resources for ${job?.id}: ${error instanceof Error ? error.message : error}`,
+      );
     }
   }