From 6f686c9d170e671944388a500d8470210d84fdf3 Mon Sep 17 00:00:00 2001 From: Giorgi Zankaidze Date: Thu, 11 Dec 2025 20:56:49 +0400 Subject: [PATCH 01/13] minio client module --- PRD-MINIO-STORAGE.md | 773 ++++++++++++++++++++ apps/api/src/app/app.module.ts | 2 + apps/api/src/app/storage/storage.module.ts | 9 + apps/api/src/app/storage/storage.service.ts | 64 ++ 4 files changed, 848 insertions(+) create mode 100644 PRD-MINIO-STORAGE.md create mode 100644 apps/api/src/app/storage/storage.module.ts create mode 100644 apps/api/src/app/storage/storage.service.ts diff --git a/PRD-MINIO-STORAGE.md b/PRD-MINIO-STORAGE.md new file mode 100644 index 0000000..b5b80fc --- /dev/null +++ b/PRD-MINIO-STORAGE.md @@ -0,0 +1,773 @@ +# Product Requirements Document: MinIO Storage Service + +## Executive Summary + +This document defines the requirements for implementing a MinIO Storage Service module for the StreamForge video transcoding platform. The storage service will provide streaming upload and download capabilities for video files, enabling efficient handling of large video files without RAM buffering. This service is a critical component that bridges the API layer with object storage, supporting the overall video upload and transcoding pipeline. + +**Timeline**: Day 5-6 (4-6 hours) +**Effort Estimate**: 4-6 hours +**Priority**: High (Blocking Day 7 streaming upload endpoint) + +--- + +## Background & Context + +### Why MinIO Storage is Needed + +StreamForge requires object storage to handle large video files efficiently. Storing videos in PostgreSQL would be impractical due to: +- Database size limitations +- Performance degradation with large binary data +- Inability to stream content efficiently + +MinIO provides S3-compatible object storage that: +- Handles large files (videos can be hundreds of MBs to GBs) +- Supports streaming uploads/downloads without RAM buffering +- Scales horizontally +- Provides a familiar S3 API interface + +### Integration with Existing System + +The storage service integrates with: +- **Video Entity** (`apps/api/src/app/videos/entities/video.entity.ts`): The `s3Key` field stores the object key/path in MinIO +- **Configuration** (`apps/api/src/config/configuration.ts`): MinIO connection settings are already defined +- **Docker Infrastructure** (`docker-compose.yml`): MinIO container is already configured and running +- **Future Upload Endpoint** (Day 7): The storage service will be used by the video upload controller + +### Architecture Context + +The storage service follows the StreamForge architecture principle: **Object Storage for Blobs**. Videos are stored in MinIO (S3-compatible), while only metadata (filename, status, s3Key) is stored in PostgreSQL. This separation enables: +- Efficient streaming uploads from HTTP requests directly to MinIO +- Scalable storage that can grow independently +- Support for future CDN integration + +--- + +## Objectives & Goals + +### Primary Objectives + +1. **Streaming Capabilities**: Enable pass-through streaming upload/download without RAM buffering +2. **Bucket Management**: Automatically create and verify required buckets on module initialization +3. **Reliable Connection**: Establish and maintain connection to MinIO with proper error handling +4. **Service Integration**: Provide a clean, injectable service following NestJS patterns + +### Success Criteria + +- ✅ Storage service successfully connects to MinIO on module initialization +- ✅ Required buckets (`raw-videos`, `processed-videos`) are created if they don't exist +- ✅ Files can be uploaded via streaming without loading entire file into memory +- ✅ Files can be downloaded as readable streams +- ✅ Object existence checks and deletion operations work correctly +- ✅ Service integrates seamlessly with existing NestJS application structure +- ✅ Test file upload succeeds and appears in MinIO console + +### Key Deliverables + +- `apps/api/src/app/storage/storage.module.ts` - NestJS module +- `apps/api/src/app/storage/storage.service.ts` - Core storage service with streaming methods +- Working connection to MinIO with bucket management +- Test script or manual verification of file upload + +--- + +## Functional Requirements + +### FR1: Module Initialization + +**Requirement**: The storage module must initialize the MinIO client connection when the module is loaded. + +**Details**: +- Connection should be established using configuration from `ConfigService` +- Connection parameters: endpoint, port, accessKey, secretKey, useSSL +- Connection should be validated on initialization +- Connection errors should be logged and handled gracefully + +### FR2: Bucket Management + +**Requirement**: The service must ensure required buckets exist before operations. + +**Details**: +- On module initialization, check if buckets exist +- Create buckets if they don't exist: + - `raw-videos` - for uploaded raw video files + - `processed-videos` - for transcoded HLS segments and playlists +- Bucket creation should be idempotent (safe to call multiple times) +- Bucket names should be configurable via environment variables +- Log bucket creation/verification status + +### FR3: Streaming Upload + +**Requirement**: `uploadStream(stream, objectName, metadata)` method must upload files via streaming. + +**Method Signature**: +```typescript +uploadStream( + stream: Readable, + objectName: string, + metadata?: Record +): Promise +``` + +**Details**: +- Accept a Node.js `Readable` stream as input +- Stream data directly to MinIO without buffering entire file in memory +- Support optional metadata (key-value pairs) for object metadata +- Return the object name/key upon successful upload +- Handle stream errors and MinIO upload errors +- Support large files (hundreds of MBs to GBs) +- Log upload progress/status + +**Parameters**: +- `stream`: Node.js Readable stream (from HTTP request, file system, etc.) +- `objectName`: Full object key/path (e.g., `raw-videos/{videoId}.mp4`) +- `metadata`: Optional object metadata (e.g., `{ 'Content-Type': 'video/mp4', 'original-filename': 'video.mp4' }`) + +**Returns**: Promise resolving to the object name/key + +**Errors**: Should throw descriptive errors for: +- Stream errors +- MinIO connection errors +- Upload failures +- Invalid parameters + +### FR4: Streaming Download + +**Requirement**: `downloadStream(objectName)` method must return a readable stream for downloading. + +**Method Signature**: +```typescript +downloadStream(objectName: string): Promise +``` + +**Details**: +- Accept object name/key as parameter +- Return a Node.js `Readable` stream +- Stream data directly from MinIO without buffering entire file +- Handle object not found errors +- Support large files efficiently + +**Parameters**: +- `objectName`: Full object key/path (e.g., `raw-videos/{videoId}.mp4`) + +**Returns**: Promise resolving to a Readable stream + +**Errors**: Should throw if: +- Object doesn't exist +- MinIO connection fails +- Invalid object name + +### FR5: Object Existence Check + +**Requirement**: `objectExists(objectName)` method must check if an object exists. + +**Method Signature**: +```typescript +objectExists(objectName: string): Promise +``` + +**Details**: +- Check if object exists in MinIO +- Return boolean (true if exists, false if not) +- Handle errors gracefully (return false or throw based on error type) +- Efficient operation (should not download object) + +**Parameters**: +- `objectName`: Full object key/path + +**Returns**: Promise resolving to boolean + +### FR6: Object Deletion + +**Requirement**: `deleteObject(objectName)` method must delete objects from MinIO. + +**Method Signature**: +```typescript +deleteObject(objectName: string): Promise +``` + +**Details**: +- Delete object from MinIO +- Handle object not found gracefully (should not throw if already deleted) +- Log deletion operations +- Return void on success + +**Parameters**: +- `objectName`: Full object key/path + +**Returns**: Promise resolving to void + +**Errors**: Should handle gracefully: +- Object not found (should not throw) +- MinIO connection errors (should throw) + +### FR7: Error Handling + +**Requirement**: All methods must handle errors appropriately. + +**Details**: +- Catch and log MinIO client errors +- Provide descriptive error messages +- Distinguish between connection errors, object errors, and validation errors +- Use NestJS Logger for consistent logging +- Throw appropriate NestJS exceptions where applicable + +### FR8: Logging + +**Requirement**: Service must log important operations and errors. + +**Details**: +- Use NestJS Logger (`@nestjs/common`) +- Log connection establishment +- Log bucket creation/verification +- Log upload/download operations (with object names, not full data) +- Log errors with context +- Use appropriate log levels (log, warn, error) + +--- + +## Technical Specifications + +### Technology Stack + +- **MinIO Client Library**: `minio` package (v8.0.6, already installed) +- **Node.js Streams**: Native `stream` module for Readable streams +- **NestJS**: Module and service patterns +- **TypeScript**: Full type safety + +### Configuration Requirements + +The service will use configuration from `apps/api/src/config/configuration.ts`: + +```typescript +minio: { + endpoint: string, // e.g., 'localhost' + port: number, // e.g., 9000 + useSSL: boolean, // false for local development + accessKey: string, // e.g., 'minioadmin' + secretKey: string, // e.g., 'minioadmin' + buckets: { + raw: string, // 'raw-videos' + processed: string, // 'processed-videos' + }, +} +``` + +### Bucket Naming Conventions + +- **Raw Videos Bucket**: `raw-videos` (configurable via `MINIO_BUCKET_RAW`) +- **Processed Videos Bucket**: `processed-videos` (configurable via `MINIO_BUCKET_PROCESSED`) +- Buckets should be created with default settings (no versioning, no lifecycle policies initially) + +### Object Naming Patterns + +Objects will follow these patterns: +- **Raw Videos**: `raw-videos/{videoId}.{extension}` (e.g., `raw-videos/123e4567-e89b-12d3-a456-426614174000.mp4`) +- **Processed Videos**: `processed-videos/{videoId}/{resolution}/{filename}` (e.g., `processed-videos/123e4567-e89b-12d3-a456-426614174000/720p/720p.m3u8`) + +The `objectName` parameter in methods should include the full path including bucket prefix or bucket name, depending on MinIO client API requirements. + +### Streaming Implementation Details + +**Upload Streaming**: +- Use MinIO client's `putObject()` method with stream +- Pass Readable stream directly to MinIO client +- Do NOT use `Buffer.concat()` or load entire stream into memory +- Handle backpressure automatically via Node.js streams + +**Download Streaming**: +- Use MinIO client's `getObject()` method +- Return the stream directly from MinIO client +- Stream is readable and can be piped to response or file system + +### Metadata Handling + +- Metadata is optional key-value pairs (string to string) +- Common metadata keys: + - `Content-Type`: MIME type (e.g., `video/mp4`) + - `original-filename`: Original uploaded filename + - `video-id`: Associated video entity ID +- Metadata is stored as object metadata in MinIO +- Metadata can be retrieved later via MinIO API if needed + +### MinIO Client Initialization + +```typescript +import * as MinIO from 'minio'; + +const minioClient = new MinIO.Client({ + endPoint: config.endpoint, + port: config.port, + useSSL: config.useSSL, + accessKey: config.accessKey, + secretKey: config.secretKey, +}); +``` + +--- + +## Architecture & Design + +### Module Structure + +``` +apps/api/src/app/storage/ +├── storage.module.ts # NestJS module definition +└── storage.service.ts # Core storage service implementation +``` + +### Module Definition (`storage.module.ts`) + +```typescript +@Module({ + imports: [ConfigModule], // Already global, but explicit for clarity + providers: [StorageService], + exports: [StorageService], // Export for use in other modules +}) +export class StorageModule {} +``` + +### Service Lifecycle + +1. **Module Initialization**: StorageModule is imported into AppModule +2. **Service Construction**: StorageService is instantiated +3. **OnModuleInit**: Connection to MinIO is established +4. **Bucket Verification**: Required buckets are checked and created if needed +5. **Ready State**: Service is ready to handle upload/download operations + +### Integration Points + +**With AppModule**: +- StorageModule will be imported into `apps/api/src/app/app.module.ts` +- Service will be available for injection in other modules + +**With VideosModule** (Future - Day 7): +- VideosService will inject StorageService +- Upload endpoint will use `uploadStream()` method +- Video entity `s3Key` field will store the object name returned from upload + +**With Configuration**: +- Uses ConfigService to access MinIO configuration +- Configuration is already loaded globally via ConfigModule + +### Dependency Injection + +```typescript +@Injectable() +export class StorageService implements OnModuleInit { + private minioClient: MinIO.Client; + + constructor( + private configService: ConfigService, + private logger: Logger, + ) {} + + async onModuleInit() { + // Initialize connection and buckets + } +} +``` + +### Error Handling Strategy + +**Connection Errors**: +- Log error with context +- Throw descriptive exception +- Application startup may fail if MinIO is unavailable (fail-fast approach) + +**Bucket Creation Errors**: +- Log warning if bucket already exists (expected) +- Log error if creation fails for other reasons +- Consider retry logic for transient errors + +**Operation Errors**: +- Catch MinIO client errors +- Transform to descriptive error messages +- Log errors with context (object name, operation type) +- Throw appropriate exceptions for callers to handle + +**Error Types**: +- `StorageConnectionError`: MinIO connection failures +- `StorageOperationError`: Upload/download/delete failures +- `StorageNotFoundError`: Object not found errors + +--- + +## Non-Functional Requirements + +### Performance + +- **Streaming**: All uploads/downloads must use streaming (no RAM buffering) +- **Memory Efficiency**: Service should handle files of any size without memory issues +- **Connection Pooling**: MinIO client handles connection pooling internally +- **Latency**: Operations should complete within reasonable time (depends on network/storage) + +### Reliability + +- **Connection Retry**: Consider retry logic for transient connection errors +- **Error Recovery**: Service should recover from temporary MinIO unavailability +- **Idempotency**: Bucket creation should be idempotent +- **Graceful Degradation**: Service should fail gracefully if MinIO is unavailable + +### Security + +- **Credentials Management**: Credentials stored in environment variables, not hardcoded +- **Access Control**: MinIO access keys should have appropriate permissions +- **SSL Support**: Support for SSL connections (configurable) +- **Input Validation**: Validate object names and parameters + +### Scalability + +- **Horizontal Scaling**: Service can be used by multiple API instances +- **Concurrent Operations**: MinIO client supports concurrent operations +- **Storage Growth**: MinIO can scale storage independently +- **Future CDN**: Design allows for future CDN integration + +--- + +## Dependencies + +### External Dependencies + +- **minio** (v8.0.6): Already installed in `package.json` + - Provides MinIO client library + - Supports streaming operations + - S3-compatible API + +### Internal Dependencies + +- **@nestjs/config**: Already installed and configured + - Provides ConfigService for accessing configuration + - ConfigModule is already global in AppModule + +- **@nestjs/common**: Already installed + - Provides Logger, Injectable, OnModuleInit decorators + - Provides exception classes + +### Infrastructure Dependencies + +- **MinIO Docker Container**: Already configured in `docker-compose.yml` + - Running on `localhost:9000` (API port) + - Console on `localhost:9001` + - Default credentials: `minioadmin`/`minioadmin` + - Must be running before API starts (or handle connection failures gracefully) + +--- + +## Testing Requirements + +### Unit Testing + +**StorageService Tests**: +- Mock MinIO client +- Test connection initialization +- Test bucket creation logic +- Test each method (uploadStream, downloadStream, objectExists, deleteObject) +- Test error handling scenarios +- Test with various stream types + +**Test File**: `apps/api/src/app/storage/storage.service.spec.ts` + +### Integration Testing + +**MinIO Integration Tests**: +- Use test MinIO instance or Docker container +- Test actual file upload/download +- Test bucket creation +- Test object existence checks +- Test deletion +- Verify files appear in MinIO + +**Test Approach**: +- Create test file (small video or text file) +- Upload via `uploadStream()` +- Verify in MinIO console +- Download via `downloadStream()` +- Compare file contents +- Clean up test objects + +### Manual Verification Steps + +1. **Start Infrastructure**: + ```bash + docker-compose up -d + ``` + +2. **Verify MinIO Running**: + - Check `http://localhost:9001` (MinIO Console) + - Login with default credentials + - Verify buckets are created + +3. **Test Upload**: + - Create test Node.js script or use API endpoint (Day 7) + - Upload a test file + - Verify file appears in MinIO console under `raw-videos` bucket + +4. **Test Download**: + - Download the uploaded file + - Verify file contents match original + +### Test Data Requirements + +- Small test file (1-10 MB) for initial testing +- Various file types (video/mp4, video/quicktime, etc.) +- Test with different object names/paths +- Test error scenarios (invalid object names, non-existent objects) + +--- + +## Implementation Details + +### File Structure + +``` +apps/api/src/app/storage/ +├── storage.module.ts # Module definition +├── storage.service.ts # Service implementation +└── storage.service.spec.ts # Unit tests (optional for Day 5-6) +``` + +### Code Organization + +**storage.module.ts**: +- Module decorator +- Imports ConfigModule +- Provides and exports StorageService + +**storage.service.ts**: +- Injectable service class +- Implements OnModuleInit +- Private MinIO client instance +- Public methods: uploadStream, downloadStream, objectExists, deleteObject +- Private helper methods: initializeClient, ensureBucketsExist + +### Key Implementation Patterns + +**Connection Initialization**: +```typescript +async onModuleInit() { + await this.initializeClient(); + await this.ensureBucketsExist(); +} +``` + +**Bucket Creation**: +```typescript +private async ensureBucketsExist() { + const rawBucket = this.configService.get('minio.buckets.raw'); + const processedBucket = this.configService.get('minio.buckets.processed'); + + // Check and create buckets + const bucketsExist = await this.minioClient.bucketExists(rawBucket); + if (!bucketsExist) { + await this.minioClient.makeBucket(rawBucket); + } + // Repeat for processed bucket +} +``` + +**Streaming Upload**: +```typescript +async uploadStream( + stream: Readable, + objectName: string, + metadata?: Record +): Promise { + // Extract bucket and object key from objectName if needed + // Or pass full objectName if MinIO client handles it + await this.minioClient.putObject(bucketName, objectKey, stream, metadata); + return objectName; +} +``` + +**Streaming Download**: +```typescript +async downloadStream(objectName: string): Promise { + // Extract bucket and object key + const stream = await this.minioClient.getObject(bucketName, objectKey); + return stream; +} +``` + +### Configuration Usage + +Access configuration via ConfigService: +```typescript +const endpoint = this.configService.get('minio.endpoint'); +const port = this.configService.get('minio.port'); +const accessKey = this.configService.get('minio.accessKey'); +const secretKey = this.configService.get('minio.secretKey'); +const useSSL = this.configService.get('minio.useSSL'); +const rawBucket = this.configService.get('minio.buckets.raw'); +const processedBucket = this.configService.get('minio.buckets.processed'); +``` + +--- + +## Risks & Considerations + +### Connection Failures + +**Risk**: MinIO container may not be running when API starts. + +**Mitigation**: +- Handle connection errors gracefully +- Log clear error messages +- Consider health check endpoint +- Document requirement for MinIO to be running + +### Bucket Creation Failures + +**Risk**: Bucket creation may fail due to permissions or MinIO issues. + +**Mitigation**: +- Check if bucket exists before creating (idempotent) +- Handle "bucket already exists" errors gracefully +- Log warnings for expected errors +- Throw exceptions for unexpected errors + +### Streaming Edge Cases + +**Risk**: Stream errors, backpressure, or incomplete uploads. + +**Mitigation**: +- Handle stream errors in uploadStream +- Let Node.js streams handle backpressure automatically +- Consider stream error events +- Validate stream is readable before upload + +### Error Scenarios + +**Risk**: Various error types from MinIO client. + +**Mitigation**: +- Catch and categorize MinIO errors +- Provide descriptive error messages +- Log errors with context +- Use appropriate exception types + +### Object Naming + +**Risk**: Object names may include invalid characters or paths. + +**Mitigation**: +- Validate object names +- Sanitize if necessary +- Document naming conventions +- Handle path separators correctly + +### Large File Handling + +**Risk**: Very large files may cause timeouts or issues. + +**Mitigation**: +- Streaming handles large files efficiently +- Consider timeout configurations +- Monitor upload/download progress +- Document file size limits if any + +--- + +## Acceptance Criteria + +### Definition of Done + +The MinIO Storage Service implementation is complete when: + +1. ✅ **Module Created**: `storage.module.ts` exists and is properly structured +2. ✅ **Service Created**: `storage.service.ts` implements all required methods +3. ✅ **Connection Established**: Service connects to MinIO on module initialization +4. ✅ **Buckets Created**: Required buckets are created if they don't exist +5. ✅ **Upload Works**: `uploadStream()` successfully uploads files via streaming +6. ✅ **Download Works**: `downloadStream()` returns readable stream +7. ✅ **Existence Check Works**: `objectExists()` correctly identifies object existence +8. ✅ **Deletion Works**: `deleteObject()` successfully deletes objects +9. ✅ **Error Handling**: All methods handle errors appropriately +10. ✅ **Logging**: Important operations are logged +11. ✅ **Integration**: Module is imported into AppModule +12. ✅ **Testing**: Test file upload succeeds and appears in MinIO console + +### Verification Checklist + +**Setup Verification**: +- [ ] MinIO Docker container is running (`docker-compose up -d`) +- [ ] MinIO console accessible at `http://localhost:9001` +- [ ] API application starts without errors +- [ ] Storage module loads successfully + +**Bucket Verification**: +- [ ] `raw-videos` bucket exists in MinIO console +- [ ] `processed-videos` bucket exists in MinIO console +- [ ] Buckets are created automatically if they don't exist +- [ ] Bucket creation is idempotent (safe to call multiple times) + +**Functionality Verification**: +- [ ] Test file can be uploaded via `uploadStream()` +- [ ] Uploaded file appears in MinIO console under correct bucket +- [ ] File can be downloaded via `downloadStream()` +- [ ] Downloaded file contents match original +- [ ] `objectExists()` returns true for existing objects +- [ ] `objectExists()` returns false for non-existent objects +- [ ] `deleteObject()` successfully deletes objects +- [ ] Deleted objects no longer appear in MinIO console + +**Error Handling Verification**: +- [ ] Connection errors are logged appropriately +- [ ] Invalid object names are handled gracefully +- [ ] Non-existent object downloads throw appropriate errors +- [ ] Service handles MinIO unavailability gracefully + +**Integration Verification**: +- [ ] StorageModule is imported in AppModule +- [ ] StorageService can be injected in other services +- [ ] Configuration is accessed correctly from ConfigService +- [ ] Service follows NestJS patterns and conventions + +**Performance Verification**: +- [ ] Large files (100MB+) can be uploaded without memory issues +- [ ] Streaming works without buffering entire file in RAM +- [ ] Multiple concurrent operations work correctly + +--- + +## Architecture Diagram + +```mermaid +graph TB + AppModule[AppModule] --> StorageModule[StorageModule] + StorageModule --> StorageService[StorageService] + StorageService --> ConfigService[ConfigService] + StorageService --> MinIOClient[MinIO Client] + MinIOClient --> MinIOContainer[MinIO Container
localhost:9000] + + StorageService --> RawBucket[raw-videos Bucket] + StorageService --> ProcessedBucket[processed-videos Bucket] + + VideosModule[VideosModule
Future Day 7] -.->|Uses| StorageService + + style StorageService fill:#e1f5ff + style MinIOContainer fill:#fff4e1 + style RawBucket fill:#e8f5e9 + style ProcessedBucket fill:#e8f5e9 +``` + +--- + +## Next Steps (Day 7 Integration) + +After completing this PRD implementation, the storage service will be used in Day 7 for: + +1. **Streaming Upload Endpoint**: `POST /videos/upload` will use `uploadStream()` +2. **Video Record Creation**: Uploaded file's `s3Key` will be stored in Video entity +3. **Future Transcoding**: Worker will use `downloadStream()` to fetch videos for processing + +The storage service is designed to be a foundational component that supports the entire video processing pipeline. + +--- + +## Document Version + +**Version**: 1.0 +**Date**: 2024 +**Author**: StreamForge Development Team +**Status**: Draft for Review + diff --git a/apps/api/src/app/app.module.ts b/apps/api/src/app/app.module.ts index 370b773..d42bc2c 100644 --- a/apps/api/src/app/app.module.ts +++ b/apps/api/src/app/app.module.ts @@ -4,6 +4,7 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import configuration from '../config/configuration'; import { AppController } from './app.controller'; import { AppService } from './app.service'; +import { StorageModule } from './storage/storage.module'; import { VideosModule } from './videos/videos.module'; @Module({ @@ -29,6 +30,7 @@ import { VideosModule } from './videos/videos.module'; inject: [ConfigService], }), VideosModule, + StorageModule, ], controllers: [AppController], providers: [AppService], diff --git a/apps/api/src/app/storage/storage.module.ts b/apps/api/src/app/storage/storage.module.ts new file mode 100644 index 0000000..ed554be --- /dev/null +++ b/apps/api/src/app/storage/storage.module.ts @@ -0,0 +1,9 @@ +import { Module } from '@nestjs/common'; +import { StorageService } from './storage.service'; + +@Module({ + providers: [StorageService], + exports: [StorageService], // Export for use in other modules +}) +export class StorageModule {} + diff --git a/apps/api/src/app/storage/storage.service.ts b/apps/api/src/app/storage/storage.service.ts new file mode 100644 index 0000000..e4a3038 --- /dev/null +++ b/apps/api/src/app/storage/storage.service.ts @@ -0,0 +1,64 @@ +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import * as MinIO from 'minio'; + +@Injectable() +export class StorageService implements OnModuleInit { + private readonly logger = new Logger(StorageService.name); + private minioClient: MinIO.Client; + + constructor(private configService: ConfigService) {} + + async onModuleInit() { + await this.initializeClient(); + } + + private async initializeClient(): Promise { + try { + const endpoint = this.configService.get('minio.endpoint'); + const port = this.configService.get('minio.port'); + const useSSL = this.configService.get('minio.useSSL'); + const accessKey = this.configService.get('minio.accessKey'); + const secretKey = this.configService.get('minio.secretKey'); + + this.logger.log( + `Initializing MinIO client: ${endpoint}:${port} (SSL: ${useSSL})`, + ); + + this.minioClient = new MinIO.Client({ + endPoint: endpoint, + port: port, + useSSL: useSSL, + accessKey: accessKey, + secretKey: secretKey, + }); + + // Validate connection by listing buckets (lightweight operation) + await this.validateConnection(); + + this.logger.log('MinIO client initialized successfully'); + } catch (error) { + this.logger.error( + `Failed to initialize MinIO client: ${error.message}`, + error.stack, + ); + throw new Error( + `MinIO connection failed: ${error.message}. Ensure MinIO is running.`, + ); + } + } + + private async validateConnection(): Promise { + try { + // List buckets to validate connection + await this.minioClient.listBuckets(); + this.logger.log('MinIO connection validated successfully'); + } catch (error) { + this.logger.error( + `MinIO connection validation failed: ${error.message}`, + ); + throw error; + } + } +} + From 566b0d62b229638aac2acabdc7f08b0ffc994a61 Mon Sep 17 00:00:00 2001 From: Giorgi Zankaidze Date: Thu, 11 Dec 2025 23:42:05 +0400 Subject: [PATCH 02/13] feat: add controle --- .vscode/tasks.json | 22 +++++++++ apps/api/src/app/storage/storage.service.ts | 49 +++++++++++++++++++++ 2 files changed, 71 insertions(+) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index c763755..5666e14 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -21,5 +21,27 @@ } } }, + { + "label": "🔄 Restart Nx Daemon", + "type": "shell", + "command": "powershell", + "args": [ + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-Command", + "npx nx daemon --stop; npx nx reset; npx nx daemon --start --verbose" + ], + "group": "build", + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "shared", + "showReuseMessage": true, + "clear": false + }, + "problemMatcher": [] + }, ] } \ No newline at end of file diff --git a/apps/api/src/app/storage/storage.service.ts b/apps/api/src/app/storage/storage.service.ts index e4a3038..c881bb9 100644 --- a/apps/api/src/app/storage/storage.service.ts +++ b/apps/api/src/app/storage/storage.service.ts @@ -5,12 +5,14 @@ import * as MinIO from 'minio'; @Injectable() export class StorageService implements OnModuleInit { private readonly logger = new Logger(StorageService.name); + private minioClient: MinIO.Client; constructor(private configService: ConfigService) {} async onModuleInit() { await this.initializeClient(); + await this.ensureBucketsExist(); } private async initializeClient(): Promise { @@ -60,5 +62,52 @@ export class StorageService implements OnModuleInit { throw error; } } + + private async ensureBucketsExist(): Promise { + const rawBucket = this.configService.get('minio.buckets.raw'); + const processedBucket = this.configService.get( + 'minio.buckets.processed', + ); + + this.logger.log( + `Ensuring buckets exist: ${rawBucket}, ${processedBucket}`, + ); + + await this.ensureBucketExists(rawBucket); + await this.ensureBucketExists(processedBucket); + + this.logger.log('Bucket verification completed'); + } + + private async ensureBucketExists(bucketName: string): Promise { + try { + const exists = await this.minioClient.bucketExists(bucketName); + + if (exists) { + this.logger.log(`Bucket '${bucketName}' already exists`); + return; + } + + this.logger.log(`Creating bucket '${bucketName}'...`); + await this.minioClient.makeBucket(bucketName); + this.logger.log(`Bucket '${bucketName}' created successfully`); + } catch (error) { + // Handle "bucket already exists" error gracefully (idempotent) + if (error.code === 'BucketAlreadyOwnedByYou') { + this.logger.warn( + `Bucket '${bucketName}' already exists (race condition handled)`, + ); + return; + } + + this.logger.error( + `Failed to create bucket '${bucketName}': ${error.message}`, + error.stack, + ); + throw new Error( + `Bucket creation failed for '${bucketName}': ${error.message}`, + ); + } + } } From 61a9a384c049c6aeb26abcf860df1dbd76ede734 Mon Sep 17 00:00:00 2001 From: Giorgi Zankaidze Date: Fri, 12 Dec 2025 00:15:19 +0400 Subject: [PATCH 03/13] add --- apps/api/src/app/storage/storage.service.ts | 293 ++++++++++++++++++++ 1 file changed, 293 insertions(+) diff --git a/apps/api/src/app/storage/storage.service.ts b/apps/api/src/app/storage/storage.service.ts index c881bb9..0916ac8 100644 --- a/apps/api/src/app/storage/storage.service.ts +++ b/apps/api/src/app/storage/storage.service.ts @@ -1,6 +1,7 @@ import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import * as MinIO from 'minio'; +import { Readable } from 'stream'; @Injectable() export class StorageService implements OnModuleInit { @@ -109,5 +110,297 @@ export class StorageService implements OnModuleInit { ); } } + + /** + * Upload a file via streaming to MinIO. + * Streams data directly without buffering entire file in memory. + * + * @param stream - Node.js Readable stream (from HTTP request, file system, etc.) + * @param objectName - Full object key/path (e.g., 'raw-videos/{videoId}.mp4') + * @param metadata - Optional object metadata (e.g., { 'Content-Type': 'video/mp4' }) + * @returns Promise resolving to the object name/key + * @throws Error for stream errors, MinIO connection errors, upload failures, or invalid parameters + */ + async uploadStream( + stream: Readable, + objectName: string, + metadata?: Record, + ): Promise { + // Validate parameters + if (!stream) { + throw new Error('Stream parameter is required'); + } + + if (!objectName || typeof objectName !== 'string' || objectName.trim() === '') { + throw new Error('Object name must be a non-empty string'); + } + + // Validate stream is readable + if (!stream.readable) { + throw new Error('Stream is not readable'); + } + + // Parse objectName to extract bucket and object key + const { bucketName, objectKey } = this.parseObjectName(objectName); + + this.logger.log( + `Starting stream upload: bucket='${bucketName}', key='${objectKey}'`, + ); + return new Promise((resolve, reject) => { + // Handle stream errors + const streamErrorHandler = (error: Error) => { + this.logger.error( + `Stream error during upload for '${objectName}': ${error.message}`, + error.stack, + ); + reject(new Error(`Stream error: ${error.message}`)); + }; + + stream.once('error', streamErrorHandler); + + // Upload to MinIO + this.minioClient + .putObject(bucketName, objectKey, stream, stream.readableLength, metadata) + .then(() => { + // Remove error handler since upload succeeded + stream.removeListener('error', streamErrorHandler); + + this.logger.log( + `Successfully uploaded stream to '${objectName}'`, + ); + resolve(objectName); + }) + .catch((error) => { + // Remove error handler + stream.removeListener('error', streamErrorHandler); + + this.logger.error( + `MinIO upload failed for '${objectName}': ${error.message}`, + error.stack, + ); + + // Provide descriptive error messages + if (error.code === 'ENOTFOUND' || error.code === 'ECONNREFUSED') { + reject( + new Error( + `MinIO connection error: ${error.message}. Ensure MinIO is running and accessible.`, + ), + ); + } else if (error.code === 'NoSuchBucket') { + reject( + new Error( + `Bucket '${bucketName}' does not exist. Ensure buckets are created.`, + ), + ); + } else { + reject( + new Error(`Upload failed for '${objectName}': ${error.message}`), + ); + } + }); + }); + } + + /** + * Parse objectName to extract bucket name and object key. + * Object name format: '{bucket-name}/{object-key}' (e.g., 'raw-videos/video.mp4') + * + * @param objectName - Full object path including bucket prefix + * @returns Object with bucketName and objectKey + * @throws Error if objectName format is invalid or bucket doesn't match configured buckets + */ + private parseObjectName(objectName: string): { + bucketName: string; + objectKey: string; + } { + const trimmedName = objectName.trim(); + + // Find the first '/' to separate bucket from key + const firstSlashIndex = trimmedName.indexOf('/'); + + if (firstSlashIndex === -1 || firstSlashIndex === 0) { + throw new Error( + `Invalid object name format: '${objectName}'. Expected format: '{bucket-name}/{object-key}'`, + ); + } + + const bucketName = trimmedName.substring(0, firstSlashIndex); + const objectKey = trimmedName.substring(firstSlashIndex + 1); + + if (!objectKey || objectKey.trim() === '') { + throw new Error( + `Invalid object name: object key cannot be empty in '${objectName}'`, + ); + } + + // Validate bucket name matches configured buckets + const rawBucket = this.configService.get('minio.buckets.raw'); + const processedBucket = this.configService.get( + 'minio.buckets.processed', + ); + + if (bucketName !== rawBucket && bucketName !== processedBucket) { + throw new Error( + `Invalid bucket name: '${bucketName}'. Must be one of: '${rawBucket}', '${processedBucket}'`, + ); + } + return { bucketName, objectKey }; + } + + /** + * Download a file as a readable stream from MinIO. + * Streams data directly without buffering entire file in memory. + * + * @param objectName - Full object key/path (e.g., 'raw-videos/{videoId}.mp4') + * @returns Promise resolving to a Readable stream + * @throws Error if object doesn't exist, MinIO connection fails, or object name is invalid + */ + async downloadStream(objectName: string): Promise { + // Validate parameters + if (!objectName || typeof objectName !== 'string' || objectName.trim() === '') { + throw new Error('Object name must be a non-empty string'); + } + + // Parse objectName to extract bucket and object key + const { bucketName, objectKey } = this.parseObjectName(objectName); + + this.logger.log( + `Starting stream download: bucket='${bucketName}', key='${objectKey}'`, + ); + + try { + const stream = await this.minioClient.getObject(bucketName, objectKey); + this.logger.log(`Successfully initiated download stream for '${objectName}'`); + return stream; + } catch (error) { + this.logger.error( + `MinIO download failed for '${objectName}': ${error.message}`, + error.stack, + ); + + // Provide descriptive error messages + if (error.code === 'ENOTFOUND' || error.code === 'ECONNREFUSED') { + throw new Error( + `MinIO connection error: ${error.message}. Ensure MinIO is running and accessible.`, + ); + } else if (error.code === 'NoSuchKey' || error.code === 'NotFound') { + throw new Error(`Object '${objectName}' does not exist`); + } else if (error.code === 'NoSuchBucket') { + throw new Error( + `Bucket '${bucketName}' does not exist. Ensure buckets are created.`, + ); + } else { + throw new Error( + `Download failed for '${objectName}': ${error.message}`, + ); + } + } + } + + /** + * Check if an object exists in MinIO. + * Efficient operation that does not download the object. + * + * @param objectName - Full object key/path + * @returns Promise resolving to boolean (true if exists, false if not) + */ + async objectExists(objectName: string): Promise { + // Validate parameters + if (!objectName || typeof objectName !== 'string' || objectName.trim() === '') { + throw new Error('Object name must be a non-empty string'); + } + + // Parse objectName to extract bucket and object key + const { bucketName, objectKey } = this.parseObjectName(objectName); + + try { + await this.minioClient.statObject(bucketName, objectKey); + this.logger.debug(`Object '${objectName}' exists`); + return true; + } catch (error) { + // Object doesn't exist if statObject throws NoSuchKey or NotFound + if (error.code === 'NoSuchKey' || error.code === 'NotFound') { + this.logger.debug(`Object '${objectName}' does not exist`); + return false; + } + + // For other errors (connection, bucket issues), log and return false + // or rethrow based on error type + if (error.code === 'ENOTFOUND' || error.code === 'ECONNREFUSED') { + this.logger.error( + `MinIO connection error while checking object existence: ${error.message}`, + ); + throw new Error( + `MinIO connection error: ${error.message}. Ensure MinIO is running and accessible.`, + ); + } else if (error.code === 'NoSuchBucket') { + this.logger.error( + `Bucket '${bucketName}' does not exist while checking object '${objectName}'`, + ); + return false; // Bucket doesn't exist, so object doesn't exist + } + + // For unexpected errors, log and return false (graceful degradation) + this.logger.warn( + `Unexpected error while checking object existence for '${objectName}': ${error.message}`, + ); + return false; + } + } + + /** + * Delete an object from MinIO. + * Handles object not found gracefully (does not throw if already deleted). + * + * @param objectName - Full object key/path + * @returns Promise resolving to void + * @throws Error for MinIO connection errors or invalid object names + */ + async deleteObject(objectName: string): Promise { + // Validate parameters + if (!objectName || typeof objectName !== 'string' || objectName.trim() === '') { + throw new Error('Object name must be a non-empty string'); + } + + // Parse objectName to extract bucket and object key + const { bucketName, objectKey } = this.parseObjectName(objectName); + + this.logger.log( + `Deleting object: bucket='${bucketName}', key='${objectKey}'`, + ); + + try { + await this.minioClient.removeObject(bucketName, objectKey); + this.logger.log(`Successfully deleted object '${objectName}'`); + } catch (error) { + // Handle object not found gracefully (idempotent operation) + if (error.code === 'NoSuchKey' || error.code === 'NotFound') { + this.logger.warn( + `Object '${objectName}' not found during deletion (already deleted)`, + ); + return; // Don't throw - object already doesn't exist + } + + this.logger.error( + `MinIO deletion failed for '${objectName}': ${error.message}`, + error.stack, + ); + + // Provide descriptive error messages for other errors + if (error.code === 'ENOTFOUND' || error.code === 'ECONNREFUSED') { + throw new Error( + `MinIO connection error: ${error.message}. Ensure MinIO is running and accessible.`, + ); + } else if (error.code === 'NoSuchBucket') { + throw new Error( + `Bucket '${bucketName}' does not exist. Ensure buckets are created.`, + ); + } else { + throw new Error( + `Deletion failed for '${objectName}': ${error.message}`, + ); + } + } + } } From 8a4e3be6ab58412c87aeafb1c71f1c4133d843be Mon Sep 17 00:00:00 2001 From: Giorgi Zankaidze Date: Fri, 12 Dec 2025 00:27:36 +0400 Subject: [PATCH 04/13] delete --- .vscode/tasks.json | 32 +++---- apps/api/src/app/app.controller.ts | 139 +++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+), 15 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 5666e14..3a9b451 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -3,23 +3,25 @@ "version": "2.0.0", "tasks": [ { - "label": "API", + "label": "🌐 API Gateway", "type": "shell", - "command": "npx nx serve api --no-tui", - "isBackground": true, - "problemMatcher": { - "pattern": { - "regexp": "^$", - "file": 1, - "location": 2, - "message": 3 - }, - "background": { - "activeOnStart": true, - "beginsPattern": ".*", - "endsPattern": ".*Application is running on.*" + "command": "npx", + "args": ["nx", "serve", "api", "--no-tui"], + "group": "build", + "presentation": { + "echo": true, + "reveal": "always", + "focus": false, + "panel": "dedicated", + "showReuseMessage": true, + "clear": false + }, + "options": { + "env": { + "NODE_ENV": "development" } - } + }, + "problemMatcher": [] }, { "label": "🔄 Restart Nx Daemon", diff --git a/apps/api/src/app/app.controller.ts b/apps/api/src/app/app.controller.ts index ab9a01d..63a0634 100644 --- a/apps/api/src/app/app.controller.ts +++ b/apps/api/src/app/app.controller.ts @@ -4,6 +4,7 @@ import { HttpCode, HttpStatus, Logger, + Param, Post, UploadedFile, UseInterceptors, @@ -15,10 +16,12 @@ import { ApiCreatedResponse, ApiOkResponse, ApiOperation, + ApiParam, ApiTags, } from '@nestjs/swagger'; import { v4 as uuidv4 } from 'uuid'; import { AppService } from './app.service'; +import { StorageService } from './storage/storage.service'; import { UploadVideoDto } from './videos/dto/upload-video.dto'; import { Video, VideoStatus } from './videos/entities/video.entity'; import { VideosService } from './videos/videos.service'; @@ -40,6 +43,7 @@ export class AppController { constructor( private readonly appService: AppService, private readonly videosService: VideosService, + private readonly storageService: StorageService, ) { this.logger.log(`AppController constructor`); } @@ -130,4 +134,139 @@ export class AppController { async testGetVideos(): Promise { return await this.videosService.findAll(); } + + @Post('test-storage/delete/:objectName') + @ApiOperation({ + summary: 'Delete an object from MinIO storage', + description: + 'Deletes an object from MinIO storage. Handles object not found gracefully (idempotent operation).', + }) + @ApiParam({ + name: 'objectName', + description: 'Full object key/path (e.g., raw-videos/video.mp4)', + type: 'string', + example: 'raw-videos/123e4567-e89b-12d3-a456-426614174000.mp4', + }) + @ApiOkResponse({ + description: 'Object deleted successfully or already deleted', + schema: { + type: 'object', + properties: { + success: { type: 'boolean' }, + message: { type: 'string' }, + objectName: { type: 'string' }, + }, + }, + }) + @HttpCode(HttpStatus.OK) + async testDeleteObject( + @Param('objectName') objectName: string, + ): Promise<{ success: boolean; message: string; objectName: string }> { + this.logger.log(`Test delete object request: objectName=${objectName}`); + + try { + // Check if object exists before deletion + const exists = await this.storageService.objectExists(objectName); + this.logger.log( + `Object '${objectName}' exists before deletion: ${exists}`, + ); + + // Delete the object + await this.storageService.deleteObject(objectName); + + // Verify deletion + const stillExists = await this.storageService.objectExists(objectName); + this.logger.log( + `Object '${objectName}' exists after deletion: ${stillExists}`, + ); + return { + success: true, + message: exists + ? 'Object deleted successfully' + : 'Object was already deleted (idempotent operation)', + objectName, + }; + } catch (error) { + this.logger.error( + `Failed to delete object '${objectName}': ${error.message}`, + ); + throw error; + } + } + + @Post('test-storage/delete-video/:id') + @ApiOperation({ + summary: 'Delete a video object by video ID', + description: + 'Deletes a video file from MinIO storage using the video ID. Retrieves the s3Key from the database and deletes the object.', + }) + @ApiParam({ + name: 'id', + description: 'Video ID (UUID)', + type: 'string', + example: '123e4567-e89b-12d3-a456-426614174000', + }) + @ApiOkResponse({ + description: 'Video object deleted successfully', + schema: { + type: 'object', + properties: { + success: { type: 'boolean' }, + message: { type: 'string' }, + videoId: { type: 'string' }, + s3Key: { type: 'string' }, + }, + }, + }) + @HttpCode(HttpStatus.OK) + async testDeleteVideoObject( + @Param('id') id: string, + ): Promise<{ + success: boolean; + message: string; + videoId: string; + s3Key: string; + }> { + this.logger.log(`Test delete video object request: videoId=${id}`); + + // Get video record from database + const video = await this.videosService.findOne(id); + if (!video) { + throw new Error(`Video with ID ${id} not found`); + } + + this.logger.log( + `Found video: id=${video.id}, s3Key=${video.s3Key}`, + ); + + try { + // Check if object exists + const exists = await this.storageService.objectExists(video.s3Key); + this.logger.log( + `Video object '${video.s3Key}' exists before deletion: ${exists}`, + ); + + // Delete the object + await this.storageService.deleteObject(video.s3Key); + + // Verify deletion + const stillExists = await this.storageService.objectExists(video.s3Key); + this.logger.log( + `Video object '${video.s3Key}' exists after deletion: ${stillExists}`, + ); + return { + success: true, + message: exists + ? 'Video object deleted successfully from MinIO' + : 'Video object was already deleted (idempotent operation)', + videoId: video.id, + s3Key: video.s3Key, + }; + } catch (error) { + this.logger.error( + `Failed to delete video object for video ${id}: ${error.message}`, + ); + throw error; + } + } } From 35a3b2d15cf286b668c17e0b0161e2f84827fd2d Mon Sep 17 00:00:00 2001 From: Giorgi Zankaidze Date: Fri, 12 Dec 2025 00:47:36 +0400 Subject: [PATCH 05/13] refactor: replace generic error handling with specific storage exceptions in StorageService --- .../storage/exceptions/storage.exceptions.ts | 69 ++++++++++++ apps/api/src/app/storage/storage.service.ts | 102 +++++++++++++----- 2 files changed, 142 insertions(+), 29 deletions(-) create mode 100644 apps/api/src/app/storage/exceptions/storage.exceptions.ts diff --git a/apps/api/src/app/storage/exceptions/storage.exceptions.ts b/apps/api/src/app/storage/exceptions/storage.exceptions.ts new file mode 100644 index 0000000..105fe32 --- /dev/null +++ b/apps/api/src/app/storage/exceptions/storage.exceptions.ts @@ -0,0 +1,69 @@ +import { + BadRequestException, + InternalServerErrorException, + NotFoundException, + ServiceUnavailableException, +} from '@nestjs/common'; + +/** + * Custom exception for MinIO connection errors + */ +export class StorageConnectionError extends ServiceUnavailableException { + constructor(message: string, cause?: Error) { + super({ + statusCode: 503, + message: `Storage connection error: ${message}`, + error: 'Service Unavailable', + cause: cause?.message, + }); + this.name = 'StorageConnectionError'; + } +} + +/** + * Custom exception for storage operation errors (upload, download, delete) + */ +export class StorageOperationError extends InternalServerErrorException { + constructor(operation: string, message: string, cause?: Error) { + super({ + statusCode: 500, + message: `Storage ${operation} failed: ${message}`, + error: 'Internal Server Error', + operation, + cause: cause?.message, + }); + this.name = 'StorageOperationError'; + } +} + +/** + * Custom exception for object not found errors + */ +export class StorageNotFoundError extends NotFoundException { + constructor(objectName: string, cause?: Error) { + super({ + statusCode: 404, + message: `Storage object not found: '${objectName}'`, + error: 'Not Found', + objectName, + cause: cause?.message, + }); + this.name = 'StorageNotFoundError'; + } +} + +/** + * Custom exception for validation errors (invalid parameters) + */ +export class StorageValidationError extends BadRequestException { + constructor(message: string, field?: string) { + super({ + statusCode: 400, + message: `Storage validation error: ${message}`, + error: 'Bad Request', + field, + }); + this.name = 'StorageValidationError'; + } +} + diff --git a/apps/api/src/app/storage/storage.service.ts b/apps/api/src/app/storage/storage.service.ts index 0916ac8..57cbde6 100644 --- a/apps/api/src/app/storage/storage.service.ts +++ b/apps/api/src/app/storage/storage.service.ts @@ -2,6 +2,12 @@ import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import * as MinIO from 'minio'; import { Readable } from 'stream'; +import { + StorageConnectionError, + StorageNotFoundError, + StorageOperationError, + StorageValidationError, +} from './exceptions/storage.exceptions'; @Injectable() export class StorageService implements OnModuleInit { @@ -45,8 +51,9 @@ export class StorageService implements OnModuleInit { `Failed to initialize MinIO client: ${error.message}`, error.stack, ); - throw new Error( - `MinIO connection failed: ${error.message}. Ensure MinIO is running.`, + throw new StorageConnectionError( + `${error.message}. Ensure MinIO is running.`, + error, ); } } @@ -105,8 +112,10 @@ export class StorageService implements OnModuleInit { `Failed to create bucket '${bucketName}': ${error.message}`, error.stack, ); - throw new Error( - `Bucket creation failed for '${bucketName}': ${error.message}`, + throw new StorageOperationError( + 'bucket creation', + `Failed to create bucket '${bucketName}': ${error.message}`, + error, ); } } @@ -128,16 +137,19 @@ export class StorageService implements OnModuleInit { ): Promise { // Validate parameters if (!stream) { - throw new Error('Stream parameter is required'); + throw new StorageValidationError('Stream parameter is required', 'stream'); } if (!objectName || typeof objectName !== 'string' || objectName.trim() === '') { - throw new Error('Object name must be a non-empty string'); + throw new StorageValidationError( + 'Object name must be a non-empty string', + 'objectName', + ); } // Validate stream is readable if (!stream.readable) { - throw new Error('Stream is not readable'); + throw new StorageValidationError('Stream is not readable', 'stream'); } // Parse objectName to extract bucket and object key @@ -153,7 +165,9 @@ export class StorageService implements OnModuleInit { `Stream error during upload for '${objectName}': ${error.message}`, error.stack, ); - reject(new Error(`Stream error: ${error.message}`)); + reject( + new StorageOperationError('upload', `Stream error: ${error.message}`, error), + ); }; stream.once('error', streamErrorHandler); @@ -182,19 +196,26 @@ export class StorageService implements OnModuleInit { // Provide descriptive error messages if (error.code === 'ENOTFOUND' || error.code === 'ECONNREFUSED') { reject( - new Error( - `MinIO connection error: ${error.message}. Ensure MinIO is running and accessible.`, + new StorageConnectionError( + `${error.message}. Ensure MinIO is running and accessible.`, + error, ), ); } else if (error.code === 'NoSuchBucket') { reject( - new Error( + new StorageOperationError( + 'upload', `Bucket '${bucketName}' does not exist. Ensure buckets are created.`, + error, ), ); } else { reject( - new Error(`Upload failed for '${objectName}': ${error.message}`), + new StorageOperationError( + 'upload', + `Upload failed for '${objectName}': ${error.message}`, + error, + ), ); } }); @@ -219,8 +240,9 @@ export class StorageService implements OnModuleInit { const firstSlashIndex = trimmedName.indexOf('/'); if (firstSlashIndex === -1 || firstSlashIndex === 0) { - throw new Error( + throw new StorageValidationError( `Invalid object name format: '${objectName}'. Expected format: '{bucket-name}/{object-key}'`, + 'objectName', ); } @@ -228,8 +250,9 @@ export class StorageService implements OnModuleInit { const objectKey = trimmedName.substring(firstSlashIndex + 1); if (!objectKey || objectKey.trim() === '') { - throw new Error( + throw new StorageValidationError( `Invalid object name: object key cannot be empty in '${objectName}'`, + 'objectName', ); } @@ -240,8 +263,9 @@ export class StorageService implements OnModuleInit { ); if (bucketName !== rawBucket && bucketName !== processedBucket) { - throw new Error( + throw new StorageValidationError( `Invalid bucket name: '${bucketName}'. Must be one of: '${rawBucket}', '${processedBucket}'`, + 'objectName', ); } return { bucketName, objectKey }; @@ -258,7 +282,10 @@ export class StorageService implements OnModuleInit { async downloadStream(objectName: string): Promise { // Validate parameters if (!objectName || typeof objectName !== 'string' || objectName.trim() === '') { - throw new Error('Object name must be a non-empty string'); + throw new StorageValidationError( + 'Object name must be a non-empty string', + 'objectName', + ); } // Parse objectName to extract bucket and object key @@ -280,18 +307,23 @@ export class StorageService implements OnModuleInit { // Provide descriptive error messages if (error.code === 'ENOTFOUND' || error.code === 'ECONNREFUSED') { - throw new Error( - `MinIO connection error: ${error.message}. Ensure MinIO is running and accessible.`, + throw new StorageConnectionError( + `${error.message}. Ensure MinIO is running and accessible.`, + error, ); } else if (error.code === 'NoSuchKey' || error.code === 'NotFound') { - throw new Error(`Object '${objectName}' does not exist`); + throw new StorageNotFoundError(objectName, error); } else if (error.code === 'NoSuchBucket') { - throw new Error( + throw new StorageOperationError( + 'download', `Bucket '${bucketName}' does not exist. Ensure buckets are created.`, + error, ); } else { - throw new Error( + throw new StorageOperationError( + 'download', `Download failed for '${objectName}': ${error.message}`, + error, ); } } @@ -307,7 +339,10 @@ export class StorageService implements OnModuleInit { async objectExists(objectName: string): Promise { // Validate parameters if (!objectName || typeof objectName !== 'string' || objectName.trim() === '') { - throw new Error('Object name must be a non-empty string'); + throw new StorageValidationError( + 'Object name must be a non-empty string', + 'objectName', + ); } // Parse objectName to extract bucket and object key @@ -330,8 +365,9 @@ export class StorageService implements OnModuleInit { this.logger.error( `MinIO connection error while checking object existence: ${error.message}`, ); - throw new Error( - `MinIO connection error: ${error.message}. Ensure MinIO is running and accessible.`, + throw new StorageConnectionError( + `${error.message}. Ensure MinIO is running and accessible.`, + error, ); } else if (error.code === 'NoSuchBucket') { this.logger.error( @@ -359,7 +395,10 @@ export class StorageService implements OnModuleInit { async deleteObject(objectName: string): Promise { // Validate parameters if (!objectName || typeof objectName !== 'string' || objectName.trim() === '') { - throw new Error('Object name must be a non-empty string'); + throw new StorageValidationError( + 'Object name must be a non-empty string', + 'objectName', + ); } // Parse objectName to extract bucket and object key @@ -388,16 +427,21 @@ export class StorageService implements OnModuleInit { // Provide descriptive error messages for other errors if (error.code === 'ENOTFOUND' || error.code === 'ECONNREFUSED') { - throw new Error( - `MinIO connection error: ${error.message}. Ensure MinIO is running and accessible.`, + throw new StorageConnectionError( + `${error.message}. Ensure MinIO is running and accessible.`, + error, ); } else if (error.code === 'NoSuchBucket') { - throw new Error( + throw new StorageOperationError( + 'delete', `Bucket '${bucketName}' does not exist. Ensure buckets are created.`, + error, ); } else { - throw new Error( + throw new StorageOperationError( + 'delete', `Deletion failed for '${objectName}': ${error.message}`, + error, ); } } From 6e89025dc873ecab23236988aea3329f33b6eba1 Mon Sep 17 00:00:00 2001 From: Giorgi Zankaidze Date: Fri, 12 Dec 2025 00:55:47 +0400 Subject: [PATCH 06/13] feat: enhance logging and error handling in StorageService initialization and operations --- apps/api/src/app/storage/storage.service.ts | 99 +++++++++++++++++---- 1 file changed, 80 insertions(+), 19 deletions(-) diff --git a/apps/api/src/app/storage/storage.service.ts b/apps/api/src/app/storage/storage.service.ts index 57cbde6..c7aae8c 100644 --- a/apps/api/src/app/storage/storage.service.ts +++ b/apps/api/src/app/storage/storage.service.ts @@ -18,8 +18,23 @@ export class StorageService implements OnModuleInit { constructor(private configService: ConfigService) {} async onModuleInit() { - await this.initializeClient(); - await this.ensureBucketsExist(); + this.logger.log('StorageService module initialization started'); + const startTime = Date.now(); + try { + await this.initializeClient(); + await this.ensureBucketsExist(); + const duration = Date.now() - startTime; + this.logger.log( + `StorageService module initialization completed successfully in ${duration}ms`, + ); + } catch (error) { + const duration = Date.now() - startTime; + this.logger.error( + `StorageService module initialization failed after ${duration}ms: ${error.message}`, + error.stack, + ); + throw error; + } } private async initializeClient(): Promise { @@ -59,13 +74,20 @@ export class StorageService implements OnModuleInit { } private async validateConnection(): Promise { + this.logger.debug('Validating MinIO connection...'); + const startTime = Date.now(); try { // List buckets to validate connection - await this.minioClient.listBuckets(); - this.logger.log('MinIO connection validated successfully'); + const buckets = await this.minioClient.listBuckets(); + const duration = Date.now() - startTime; + this.logger.log( + `MinIO connection validated successfully in ${duration}ms (found ${buckets.length} bucket(s))`, + ); } catch (error) { + const duration = Date.now() - startTime; this.logger.error( - `MinIO connection validation failed: ${error.message}`, + `MinIO connection validation failed after ${duration}ms: ${error.message}`, + error.stack, ); throw error; } @@ -88,17 +110,25 @@ export class StorageService implements OnModuleInit { } private async ensureBucketExists(bucketName: string): Promise { + const startTime = Date.now(); try { const exists = await this.minioClient.bucketExists(bucketName); if (exists) { - this.logger.log(`Bucket '${bucketName}' already exists`); + const duration = Date.now() - startTime; + this.logger.log( + `Bucket '${bucketName}' already exists (verified in ${duration}ms)`, + ); return; } this.logger.log(`Creating bucket '${bucketName}'...`); + const createStartTime = Date.now(); await this.minioClient.makeBucket(bucketName); - this.logger.log(`Bucket '${bucketName}' created successfully`); + const createDuration = Date.now() - createStartTime; + this.logger.log( + `Bucket '${bucketName}' created successfully in ${createDuration}ms`, + ); } catch (error) { // Handle "bucket already exists" error gracefully (idempotent) if (error.code === 'BucketAlreadyOwnedByYou') { @@ -155,14 +185,21 @@ export class StorageService implements OnModuleInit { // Parse objectName to extract bucket and object key const { bucketName, objectKey } = this.parseObjectName(objectName); + const startTime = Date.now(); + const streamSize = stream.readableLength || 'unknown'; + const metadataInfo = metadata + ? `with metadata: ${Object.keys(metadata).join(', ')}` + : 'without metadata'; + this.logger.log( - `Starting stream upload: bucket='${bucketName}', key='${objectKey}'`, + `Starting stream upload: bucket='${bucketName}', key='${objectKey}', size=${streamSize} bytes, ${metadataInfo}`, ); return new Promise((resolve, reject) => { // Handle stream errors const streamErrorHandler = (error: Error) => { + const streamErrorDuration = Date.now() - startTime; this.logger.error( - `Stream error during upload for '${objectName}': ${error.message}`, + `Stream error during upload for '${objectName}' after ${streamErrorDuration}ms: ${error.message}`, error.stack, ); reject( @@ -179,8 +216,9 @@ export class StorageService implements OnModuleInit { // Remove error handler since upload succeeded stream.removeListener('error', streamErrorHandler); + const duration = Date.now() - startTime; this.logger.log( - `Successfully uploaded stream to '${objectName}'`, + `Successfully uploaded stream to '${objectName}' in ${duration}ms (bucket='${bucketName}', key='${objectKey}')`, ); resolve(objectName); }) @@ -188,8 +226,9 @@ export class StorageService implements OnModuleInit { // Remove error handler stream.removeListener('error', streamErrorHandler); + const uploadDuration = Date.now() - startTime; this.logger.error( - `MinIO upload failed for '${objectName}': ${error.message}`, + `MinIO upload failed for '${objectName}' after ${uploadDuration}ms: ${error.message}`, error.stack, ); @@ -291,17 +330,22 @@ export class StorageService implements OnModuleInit { // Parse objectName to extract bucket and object key const { bucketName, objectKey } = this.parseObjectName(objectName); + const startTime = Date.now(); this.logger.log( `Starting stream download: bucket='${bucketName}', key='${objectKey}'`, ); try { const stream = await this.minioClient.getObject(bucketName, objectKey); - this.logger.log(`Successfully initiated download stream for '${objectName}'`); + const duration = Date.now() - startTime; + this.logger.log( + `Successfully initiated download stream for '${objectName}' in ${duration}ms (bucket='${bucketName}', key='${objectKey}')`, + ); return stream; } catch (error) { + const duration = Date.now() - startTime; this.logger.error( - `MinIO download failed for '${objectName}': ${error.message}`, + `MinIO download failed for '${objectName}' after ${duration}ms: ${error.message}`, error.stack, ); @@ -348,14 +392,25 @@ export class StorageService implements OnModuleInit { // Parse objectName to extract bucket and object key const { bucketName, objectKey } = this.parseObjectName(objectName); + const startTime = Date.now(); + this.logger.debug( + `Checking object existence: bucket='${bucketName}', key='${objectKey}'`, + ); + try { - await this.minioClient.statObject(bucketName, objectKey); - this.logger.debug(`Object '${objectName}' exists`); + const stats = await this.minioClient.statObject(bucketName, objectKey); + const duration = Date.now() - startTime; + this.logger.debug( + `Object '${objectName}' exists (size: ${stats.size} bytes, checked in ${duration}ms)`, + ); return true; } catch (error) { // Object doesn't exist if statObject throws NoSuchKey or NotFound if (error.code === 'NoSuchKey' || error.code === 'NotFound') { - this.logger.debug(`Object '${objectName}' does not exist`); + const duration = Date.now() - startTime; + this.logger.debug( + `Object '${objectName}' does not exist (checked in ${duration}ms)`, + ); return false; } @@ -404,24 +459,30 @@ export class StorageService implements OnModuleInit { // Parse objectName to extract bucket and object key const { bucketName, objectKey } = this.parseObjectName(objectName); + const startTime = Date.now(); this.logger.log( `Deleting object: bucket='${bucketName}', key='${objectKey}'`, ); try { await this.minioClient.removeObject(bucketName, objectKey); - this.logger.log(`Successfully deleted object '${objectName}'`); + const duration = Date.now() - startTime; + this.logger.log( + `Successfully deleted object '${objectName}' in ${duration}ms (bucket='${bucketName}', key='${objectKey}')`, + ); } catch (error) { // Handle object not found gracefully (idempotent operation) if (error.code === 'NoSuchKey' || error.code === 'NotFound') { + const duration = Date.now() - startTime; this.logger.warn( - `Object '${objectName}' not found during deletion (already deleted)`, + `Object '${objectName}' not found during deletion (already deleted, checked in ${duration}ms)`, ); return; // Don't throw - object already doesn't exist } + const duration = Date.now() - startTime; this.logger.error( - `MinIO deletion failed for '${objectName}': ${error.message}`, + `MinIO deletion failed for '${objectName}' after ${duration}ms: ${error.message}`, error.stack, ); From d92cc46e9e3639db669e34eb83e1370ebc817088 Mon Sep 17 00:00:00 2001 From: Giorgi Zankaidze Date: Fri, 12 Dec 2025 00:56:23 +0400 Subject: [PATCH 07/13] docs: add manual testing points for bucket auto-creation, streaming workflow, and error handling in MinIO storage service --- PRD-MINIO-STORAGE.md | 89 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/PRD-MINIO-STORAGE.md b/PRD-MINIO-STORAGE.md index b5b80fc..7ac1667 100644 --- a/PRD-MINIO-STORAGE.md +++ b/PRD-MINIO-STORAGE.md @@ -771,3 +771,92 @@ The storage service is designed to be a foundational component that supports the **Author**: StreamForge Development Team **Status**: Draft for Review +--- + +## Manual Testing Points + +### Test Point 1: Bucket Auto-Creation and Module Initialization + +**Objective**: Verify that buckets are automatically created when the API starts and the storage module initializes correctly. + +**Steps**: +1. Ensure MinIO container is running: `docker-compose up -d` +2. Delete both `raw-videos` and `processed-videos` buckets from MinIO Console (if they exist) +3. Start the NestJS API application: `npm run start:dev` (or equivalent) +4. Check the application logs for bucket creation messages +5. Open MinIO Console at `http://localhost:9001` and verify: + - Both `raw-videos` and `processed-videos` buckets exist + - Buckets are empty (no objects) +6. Restart the API application and verify no errors occur (idempotent bucket creation) + +**Expected Result**: +- API starts successfully without errors +- Both buckets are created automatically +- Logs show bucket creation/verification messages +- Restarting the API does not cause errors (buckets already exist) + +--- + +### Test Point 2: Streaming Upload and Download Workflow + +**Objective**: Verify that files can be uploaded and downloaded using streaming without memory buffering. + +**Steps**: +1. Create a test file (e.g., `test-video.mp4` - 5-10 MB) in a temporary location +2. Create a simple test script that: + - Injects `StorageService` into a temporary NestJS context + - Uses `fs.createReadStream()` to create a readable stream from the test file + - Calls `storageService.uploadStream(stream, 'raw-videos/test-upload.mp4', { 'Content-Type': 'video/mp4' })` + - Verifies the upload completes successfully + - Calls `storageService.objectExists('raw-videos/test-upload.mp4')` and verifies it returns `true` + - Calls `storageService.downloadStream('raw-videos/test-upload.mp4')` to get a readable stream + - Pipes the download stream to a new file (e.g., `test-download.mp4`) + - Compares the original and downloaded files (checksum or file size) +3. Run the test script +4. Verify in MinIO Console that the file appears in the `raw-videos` bucket +5. Monitor system memory usage during upload/download to ensure no significant RAM spike + +**Expected Result**: +- Upload completes successfully and returns the object name +- File appears in MinIO Console under `raw-videos` bucket +- `objectExists()` returns `true` for the uploaded file +- Download stream works and produces an identical file +- Memory usage remains stable (no large buffering) +- File metadata (Content-Type) is preserved + +--- + +### Test Point 3: Error Handling and Edge Cases + +**Objective**: Verify that the storage service handles errors gracefully and validates inputs correctly. + +**Steps**: +1. **Test Non-Existent Object Download**: + - Call `storageService.downloadStream('raw-videos/non-existent-file.mp4')` + - Verify an appropriate error is thrown (e.g., `StorageNotFoundError` or similar) + - Verify error message is descriptive + +2. **Test Object Existence Check**: + - Call `storageService.objectExists('raw-videos/non-existent-file.mp4')` and verify it returns `false` + - Upload a test file, then call `objectExists()` with the correct path and verify it returns `true` + +3. **Test Object Deletion**: + - Upload a test file via `uploadStream()` + - Verify the file exists in MinIO Console + - Call `storageService.deleteObject('raw-videos/test-delete.mp4')` + - Verify the deletion completes without errors + - Call `objectExists()` and verify it returns `false` + - Verify the file no longer appears in MinIO Console + - Call `deleteObject()` again on the same path and verify it doesn't throw an error (graceful handling) + +4. **Test Invalid Object Names**: + - Attempt to upload with an invalid object name (e.g., empty string, null, or invalid characters) + - Verify appropriate validation errors are thrown + +**Expected Result**: +- Non-existent object downloads throw descriptive errors +- `objectExists()` correctly returns `true`/`false` based on actual object existence +- `deleteObject()` successfully removes objects and handles already-deleted objects gracefully +- Invalid inputs are validated and appropriate errors are thrown +- All errors are logged with context + From 5ea9a4c9170800d7050ad97e43bcc16c728a233a Mon Sep 17 00:00:00 2001 From: Giorgi Zankaidze Date: Fri, 12 Dec 2025 23:45:41 +0400 Subject: [PATCH 08/13] docs: update manual testing points for error handling and streaming workflow in MinIO storage service --- MINIO-SIGNATURE-ISSUE-REPORT.md | 248 ++++++++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 MINIO-SIGNATURE-ISSUE-REPORT.md diff --git a/MINIO-SIGNATURE-ISSUE-REPORT.md b/MINIO-SIGNATURE-ISSUE-REPORT.md new file mode 100644 index 0000000..92c99a5 --- /dev/null +++ b/MINIO-SIGNATURE-ISSUE-REPORT.md @@ -0,0 +1,248 @@ +# MinIO Signature Mismatch Issue Report + +**Date**: December 12, 2025 +**Status**: Unresolved +**Priority**: High (Blocking video upload functionality) + +## Problem Summary + +When attempting to upload video files to MinIO using the `putObject` method, the operation fails with the following error: + +``` +S3Error: The request signature we calculated does not match the signature you provided. Check your key and signing method. +``` + +This error occurs consistently during file uploads, preventing videos from being stored in MinIO. + +## Error Details + +### Error Stack Trace +``` +S3Error: The request signature we calculated does not match the signature you provided. Check your key and signing method. + at parseError (node_modules/minio/dist/main/internal/xml-parser.ts:49:13) + at Object.parseResponseError (node_modules/minio/dist/main/internal/xml-parser.ts:107:11) + at Client.makeRequestStreamAsync (node_modules/minio/dist/main/internal/client.ts:740:19) + at Client.uploadBuffer (node_modules/minio/dist/main/internal/client.ts:1687:17) +``` + +### When It Occurs +- During `putObject` calls when uploading files +- Both with and without metadata +- With Buffer and Readable stream inputs +- Connection to MinIO is successful (buckets are created/verified) + +## Configuration + +### MinIO Client Setup +```typescript +this.minioClient = new MinIO.Client({ + endPoint: 'localhost', + port: 9000, + useSSL: false, + accessKey: 'minioadmin', + secretKey: 'minioadmin', +}); +``` + +### Current Implementation +- **Method**: `uploadStream(stream: Readable, objectName: string, metadata?: Record)` +- **Input**: Readable stream from file buffer +- **Size**: Using `stream.readableLength` (may be undefined) +- **Metadata**: Passed directly to `putObject` + +### Upload Call +```typescript +this.minioClient.putObject(bucketName, objectKey, stream, stream.readableLength, metadata) +``` + +## Attempted Solutions + +### 1. Metadata Formatting +**Attempted**: Separating `Content-Type` from custom metadata and prefixing custom metadata with `x-amz-meta-` + +**Result**: Still failed with signature mismatch + +**Code Tried**: +```typescript +const contentType = metadata?.['Content-Type']; +const customMetadata: Record = {}; +// Format custom metadata with x-amz-meta- prefix +Object.keys(metadata).forEach((key) => { + if (key !== 'Content-Type') { + customMetadata[`x-amz-meta-${key}`] = metadata[key]; + } +}); +``` + +### 2. Size Parameter +**Attempted**: Explicitly passing file size parameter + +**Result**: Still failed + +**Code Tried**: +```typescript +await this.storageService.uploadStream(fileStream, s3Key, metadata, file.size); +``` + +### 3. Buffer vs Stream +**Attempted**: +- Passing Buffer directly +- Converting Buffer to Readable stream +- Using Buffer without metadata + +**Result**: All approaches failed with the same signature error + +### 4. Metadata Removal +**Attempted**: Uploading without any metadata + +**Result**: Still failed (though not fully tested) + +### 5. Region Configuration +**Attempted**: Adding explicit region `'us-east-1'` to client configuration + +**Result**: Reverted by user, issue persists + +## Root Cause Analysis + +### Possible Causes + +1. **Size Parameter Issue** + - `stream.readableLength` may be `undefined` for streams created from buffers + - MinIO signature calculation requires accurate size + - **Likelihood**: High + +2. **Metadata in Signature Calculation** + - Metadata headers may not be included correctly in signature + - Content-Type handling may differ between MinIO versions + - **Likelihood**: Medium + +3. **Stream Type Detection** + - MinIO may detect buffer-based streams differently + - Internal `uploadBuffer` method may have different signature requirements + - **Likelihood**: Medium + +4. **Client Library Version** + - MinIO client library version: `minio@8.0.6` + - Possible incompatibility or bug in this version + - **Likelihood**: Low + +5. **Credentials/Configuration** + - Access key and secret key are correct (connection works) + - SSL is disabled (correct for local setup) + - **Likelihood**: Low + +## Current Code State + +### Storage Service (`apps/api/src/app/storage/storage.service.ts`) +```typescript +async uploadStream( + stream: Readable, + objectName: string, + metadata?: Record, +): Promise { + // ... validation ... + + this.minioClient + .putObject(bucketName, objectKey, stream, stream.readableLength, metadata) + .then(() => { /* success */ }) + .catch((error) => { /* signature mismatch error */ }); +} +``` + +### Controller (`apps/api/src/app/app.controller.ts`) +```typescript +const fileStream = Readable.from(file.buffer); +await this.storageService.uploadStream(fileStream, s3Key, metadata); +``` + +## Recommendations + +### Immediate Next Steps + +1. **Fix Size Parameter** + ```typescript + // In controller, pass size explicitly + await this.storageService.uploadStream( + fileStream, + s3Key, + metadata, + file.size // Explicit size + ); + + // In storage service, use provided size + .putObject(bucketName, objectKey, stream, size || stream.readableLength, metadata) + ``` + +2. **Test Without Metadata** + - Temporarily remove all metadata to isolate the issue + - If upload succeeds, add metadata back incrementally + +3. **Use Buffer Directly** + - Try passing buffer directly instead of converting to stream + - MinIO's `putObject` accepts Buffer, Readable, or string + +4. **Check MinIO Version** + - Verify MinIO server version compatibility with client library + - Consider updating MinIO client library to latest version + +5. **Alternative: Use fPutObject** + - For testing, try saving file to disk first + - Use `fPutObject(bucketName, objectName, filePath, metadata)` + - This bypasses stream handling issues + +### Long-term Solutions + +1. **Upgrade MinIO Client** + - Check for newer versions of `minio` package + - Review changelog for signature-related fixes + +2. **Implement Retry Logic** + - Add retry mechanism for transient signature errors + - Implement exponential backoff + +3. **Add Comprehensive Logging** + - Log exact request parameters before upload + - Log MinIO client configuration + - Enable MinIO client debug mode if available + +4. **Consider Alternative Libraries** + - AWS SDK for JavaScript v3 (S3-compatible) + - May have better error messages and debugging + +## Testing Checklist + +- [ ] Test upload with explicit size parameter +- [ ] Test upload without any metadata +- [ ] Test upload with only custom metadata (no Content-Type) +- [ ] Test upload using Buffer directly (not stream) +- [ ] Test upload using `fPutObject` with file path +- [ ] Verify MinIO server version +- [ ] Check MinIO server logs for additional error details +- [ ] Test with different file sizes (small vs large) +- [ ] Test with different file types + +## Related Files + +- `apps/api/src/app/storage/storage.service.ts` - Storage service implementation +- `apps/api/src/app/app.controller.ts` - Upload endpoint +- `apps/api/src/config/configuration.ts` - MinIO configuration +- `docker-compose.yml` - MinIO container setup + +## References + +- [MinIO JavaScript Client Documentation](https://docs.min.io/docs/javascript-client-quickstart-guide.html) +- [AWS Signature Version 4 Documentation](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html) +- [MinIO GitHub Issues - Signature Mismatch](https://github.com/minio/minio/issues?q=signature+mismatch) + +## Notes + +- MinIO connection and bucket operations work correctly +- The issue is specific to `putObject` uploads +- Error occurs in MinIO client's internal `uploadBuffer` method +- File size: ~4MB (4221114 bytes) in test cases +- MinIO server is running in Docker container + +--- + +**Next Action**: Implement explicit size parameter and test without metadata to isolate the root cause. + From cbbf7560d111e4fe6f8d320f4c4b7b3285518b87 Mon Sep 17 00:00:00 2001 From: Giorgi Zankaidze Date: Sun, 14 Dec 2025 16:23:23 +0400 Subject: [PATCH 09/13] feat: implement video upload and record creation workflow in VideosService, enhance StorageService with size parameter for MinIO uploads --- .cursor/debug.log | 6 ++ MINIO-SIGNATURE-ISSUE-REPORT.md | 2 + apps/api/src/app/app.controller.ts | 19 +++--- apps/api/src/app/storage/storage.service.ts | 23 ++++++- apps/api/src/app/videos/videos.module.ts | 6 +- apps/api/src/app/videos/videos.service.ts | 70 ++++++++++++++++++++- 6 files changed, 112 insertions(+), 14 deletions(-) create mode 100644 .cursor/debug.log diff --git a/.cursor/debug.log b/.cursor/debug.log new file mode 100644 index 0000000..314039c --- /dev/null +++ b/.cursor/debug.log @@ -0,0 +1,6 @@ +{"location":"videos.service.ts:55","message":"calling uploadStream","data":{"s3Key":"raw-videos/aaf44fef-cac7-49ed-8172-d7ae7600f1c9.mp4","fileSize":4221114,"hasMetadata":true,"metadataKeys":["Content-Type","original-filename"]},"timestamp":1765705891060,"sessionId":"debug-session","runId":"run1","hypothesisId":"A"} +{"location":"storage.service.ts:190","message":"uploadStream entry","data":{"objectName":"raw-videos/aaf44fef-cac7-49ed-8172-d7ae7600f1c9.mp4","hasMetadata":true,"metadataKeys":["Content-Type","original-filename"],"sizeParam":4221114,"readableLength":0,"isBuffer":false},"timestamp":1765705891067,"sessionId":"debug-session","runId":"run1","hypothesisId":"A"} +{"location":"storage.service.ts:200","message":"size calculation","data":{"sizeParam":4221114,"readableLength":0,"actualSize":4221114,"streamSize":4221114},"timestamp":1765705891069,"sessionId":"debug-session","runId":"run1","hypothesisId":"A"} +{"location":"storage.service.ts:218","message":"putObject call","data":{"bucketName":"raw-videos","objectKey":"aaf44fef-cac7-49ed-8172-d7ae7600f1c9.mp4","sizeParam":4221114,"readableLength":0,"hasMetadata":true,"metadataKeys":["Content-Type","original-filename"]},"timestamp":1765705891072,"sessionId":"debug-session","runId":"run1","hypothesisId":"B"} +{"location":"storage.service.ts:225","message":"upload success","data":{"objectName":"raw-videos/aaf44fef-cac7-49ed-8172-d7ae7600f1c9.mp4","duration":66,"sizeUsed":4221114},"timestamp":1765705891135,"sessionId":"debug-session","runId":"run1","hypothesisId":"A"} +{"location":"videos.service.ts:58","message":"uploadStream completed","data":{"s3Key":"raw-videos/aaf44fef-cac7-49ed-8172-d7ae7600f1c9.mp4"},"timestamp":1765705891137,"sessionId":"debug-session","runId":"run1","hypothesisId":"A"} diff --git a/MINIO-SIGNATURE-ISSUE-REPORT.md b/MINIO-SIGNATURE-ISSUE-REPORT.md index 92c99a5..9203ae1 100644 --- a/MINIO-SIGNATURE-ISSUE-REPORT.md +++ b/MINIO-SIGNATURE-ISSUE-REPORT.md @@ -246,3 +246,5 @@ await this.storageService.uploadStream(fileStream, s3Key, metadata); **Next Action**: Implement explicit size parameter and test without metadata to isolate the root cause. + + diff --git a/apps/api/src/app/app.controller.ts b/apps/api/src/app/app.controller.ts index 63a0634..a12494d 100644 --- a/apps/api/src/app/app.controller.ts +++ b/apps/api/src/app/app.controller.ts @@ -19,11 +19,11 @@ import { ApiParam, ApiTags, } from '@nestjs/swagger'; -import { v4 as uuidv4 } from 'uuid'; +import { Readable } from 'stream'; import { AppService } from './app.service'; import { StorageService } from './storage/storage.service'; import { UploadVideoDto } from './videos/dto/upload-video.dto'; -import { Video, VideoStatus } from './videos/entities/video.entity'; +import { Video } from './videos/entities/video.entity'; import { VideosService } from './videos/videos.service'; interface UploadedFileInterface { @@ -106,19 +106,18 @@ export class AppController { `Received file upload: filename=${file.originalname}, mimetype=${file.mimetype}, size=${file.size}`, ); - // Generate a unique S3 key (placeholder until MinIO is implemented) - const fileExtension = file.originalname.split('.').pop(); - const s3Key = `raw-videos/${uuidv4()}.${fileExtension}`; + // Convert file buffer to stream and delegate to videos service + const fileStream = Readable.from(file.buffer); - // Create video record - const video = await this.videosService.create( + // Videos service handles MinIO upload and database record creation + const video = await this.videosService.uploadAndCreate( + fileStream, file.originalname, - s3Key, file.mimetype, - VideoStatus.UPLOADED, + file.size, ); - this.logger.log(`Video record created: id=${video.id}`); + this.logger.log(`Video uploaded and record created: id=${video.id}`); return video; } diff --git a/apps/api/src/app/storage/storage.service.ts b/apps/api/src/app/storage/storage.service.ts index c7aae8c..338d9b9 100644 --- a/apps/api/src/app/storage/storage.service.ts +++ b/apps/api/src/app/storage/storage.service.ts @@ -49,12 +49,16 @@ export class StorageService implements OnModuleInit { `Initializing MinIO client: ${endpoint}:${port} (SSL: ${useSSL})`, ); + // Initialize MinIO client with configuration values + // Note: Do not hardcode values - use config to ensure consistency + // Region is not needed for local MinIO instances this.minioClient = new MinIO.Client({ endPoint: endpoint, port: port, useSSL: useSSL, accessKey: accessKey, secretKey: secretKey, + // Do not set region for local MinIO - it can cause signature issues }); // Validate connection by listing buckets (lightweight operation) @@ -157,6 +161,7 @@ export class StorageService implements OnModuleInit { * @param stream - Node.js Readable stream (from HTTP request, file system, etc.) * @param objectName - Full object key/path (e.g., 'raw-videos/{videoId}.mp4') * @param metadata - Optional object metadata (e.g., { 'Content-Type': 'video/mp4' }) + * @param size - Optional file size in bytes (required for accurate signature calculation when stream.readableLength is unavailable) * @returns Promise resolving to the object name/key * @throws Error for stream errors, MinIO connection errors, upload failures, or invalid parameters */ @@ -164,6 +169,7 @@ export class StorageService implements OnModuleInit { stream: Readable, objectName: string, metadata?: Record, + size?: number, ): Promise { // Validate parameters if (!stream) { @@ -186,7 +192,19 @@ export class StorageService implements OnModuleInit { const { bucketName, objectKey } = this.parseObjectName(objectName); const startTime = Date.now(); - const streamSize = stream.readableLength || 'unknown'; + // Use provided size parameter, fallback to stream.readableLength + // CRITICAL: MinIO signature calculation requires a valid number, not undefined + const actualSize = size ?? stream.readableLength; + + // Validate that we have a size for signature calculation + if (actualSize === undefined || actualSize === null) { + throw new StorageValidationError( + 'Size parameter is required for accurate MinIO signature calculation. Provide file size explicitly.', + 'size', + ); + } + + const streamSize = actualSize; const metadataInfo = metadata ? `with metadata: ${Object.keys(metadata).join(', ')}` : 'without metadata'; @@ -210,8 +228,9 @@ export class StorageService implements OnModuleInit { stream.once('error', streamErrorHandler); // Upload to MinIO + // Size parameter is critical for accurate signature calculation this.minioClient - .putObject(bucketName, objectKey, stream, stream.readableLength, metadata) + .putObject(bucketName, objectKey, stream, actualSize, metadata) .then(() => { // Remove error handler since upload succeeded stream.removeListener('error', streamErrorHandler); diff --git a/apps/api/src/app/videos/videos.module.ts b/apps/api/src/app/videos/videos.module.ts index a6d8233..dedcef5 100644 --- a/apps/api/src/app/videos/videos.module.ts +++ b/apps/api/src/app/videos/videos.module.ts @@ -1,10 +1,14 @@ import { Module } from '@nestjs/common'; import { TypeOrmModule } from '@nestjs/typeorm'; +import { StorageModule } from '../storage/storage.module'; import { Video } from './entities/video.entity'; import { VideosService } from './videos.service'; @Module({ - imports: [TypeOrmModule.forFeature([Video])], + imports: [ + TypeOrmModule.forFeature([Video]), + StorageModule, // Import StorageModule to use StorageService + ], providers: [VideosService], exports: [VideosService], }) diff --git a/apps/api/src/app/videos/videos.service.ts b/apps/api/src/app/videos/videos.service.ts index 405d092..410f529 100644 --- a/apps/api/src/app/videos/videos.service.ts +++ b/apps/api/src/app/videos/videos.service.ts @@ -1,6 +1,9 @@ import { Injectable, Logger } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; +import { Readable } from 'stream'; import { Repository } from 'typeorm'; +import { v4 as uuidv4 } from 'uuid'; +import { StorageService } from '../storage/storage.service'; import { Video, VideoStatus } from './entities/video.entity'; @Injectable() @@ -10,10 +13,75 @@ export class VideosService { constructor( @InjectRepository(Video) private videoRepository: Repository