diff --git a/src/routes/images.ts b/src/routes/images.ts index c4bcfce..c1082ad 100644 --- a/src/routes/images.ts +++ b/src/routes/images.ts @@ -24,14 +24,21 @@ imagesRouter.get( const storageService = StorageFactory.getInstance(); try { - // Stream the file directly through our API (more reliable than presigned URL redirects) - const fileBuffer = await storageService.downloadFile( + // Check if file exists first (fast check) + const exists = await storageService.fileExists( orgId, projectId, category as 'uploads' | 'generated' | 'references', filename ); + if (!exists) { + return res.status(404).json({ + success: false, + message: 'File not found' + }); + } + // Determine content type from filename const ext = filename.toLowerCase().split('.').pop(); const contentType = { @@ -43,11 +50,38 @@ imagesRouter.get( 'svg': 'image/svg+xml' }[ext || ''] || 'application/octet-stream'; + // Set headers for optimal caching and performance res.setHeader('Content-Type', contentType); - res.setHeader('Cache-Control', 'public, max-age=86400'); // 24 hours - res.setHeader('Content-Length', fileBuffer.length); + res.setHeader('Cache-Control', 'public, max-age=86400, immutable'); // 24 hours + immutable + res.setHeader('ETag', `"${orgId}-${projectId}-${filename}"`); // Simple ETag - return res.send(fileBuffer); + // Handle conditional requests (304 Not Modified) + const ifNoneMatch = req.headers['if-none-match']; + if (ifNoneMatch === `"${orgId}-${projectId}-${filename}"`) { + return res.status(304).end(); // Not Modified + } + + // Stream the file directly through our API (memory efficient) + const fileStream = await storageService.streamFile( + orgId, + projectId, + category as 'uploads' | 'generated' | 'references', + filename + ); + + // Handle stream errors + fileStream.on('error', (streamError) => { + console.error('Stream error:', streamError); + if (!res.headersSent) { + res.status(500).json({ + success: false, + message: 'Error streaming file' + }); + } + }); + + // Stream the file without loading into memory + fileStream.pipe(res); } catch (error) { console.error('Failed to stream file:', error); diff --git a/src/services/MinioStorageService.ts b/src/services/MinioStorageService.ts index 07b925d..d64824f 100644 --- a/src/services/MinioStorageService.ts +++ b/src/services/MinioStorageService.ts @@ -48,18 +48,63 @@ export class MinioStorageService implements StorageService { } private generateUniqueFilename(originalFilename: string): string { + // Sanitize filename first + const sanitized = this.sanitizeFilename(originalFilename); + const timestamp = Date.now(); const random = Math.random().toString(36).substring(2, 8); - const ext = originalFilename.includes('.') - ? originalFilename.substring(originalFilename.lastIndexOf('.')) + const ext = sanitized.includes('.') + ? sanitized.substring(sanitized.lastIndexOf('.')) : ''; - const name = originalFilename.includes('.') - ? originalFilename.substring(0, originalFilename.lastIndexOf('.')) - : originalFilename; + const name = sanitized.includes('.') + ? sanitized.substring(0, sanitized.lastIndexOf('.')) + : sanitized; return `${name}-${timestamp}-${random}${ext}`; } + private sanitizeFilename(filename: string): string { + // Remove dangerous characters and path traversal attempts + return filename + .replace(/[<>:"/\\|?*\x00-\x1f]/g, '') // Remove dangerous chars + .replace(/\.\./g, '') // Remove path traversal + .replace(/^\.+/, '') // Remove leading dots + .trim() + .substring(0, 255); // Limit length + } + + private validateFilePath(orgId: string, projectId: string, category: string, filename: string): void { + // Validate orgId + if (!orgId || !/^[a-zA-Z0-9_-]+$/.test(orgId) || orgId.length > 50) { + throw new Error('Invalid organization ID: must be alphanumeric with dashes/underscores, max 50 chars'); + } + + // Validate projectId + if (!projectId || !/^[a-zA-Z0-9_-]+$/.test(projectId) || projectId.length > 50) { + throw new Error('Invalid project ID: must be alphanumeric with dashes/underscores, max 50 chars'); + } + + // Validate category + if (!['uploads', 'generated', 'references'].includes(category)) { + throw new Error('Invalid category: must be uploads, generated, or references'); + } + + // Validate filename + if (!filename || filename.length === 0 || filename.length > 255) { + throw new Error('Invalid filename: must be 1-255 characters'); + } + + // Check for path traversal and dangerous patterns + if (filename.includes('..') || filename.includes('/') || filename.includes('\\')) { + throw new Error('Invalid characters in filename: path traversal not allowed'); + } + + // Prevent null bytes and control characters + if (/[\x00-\x1f]/.test(filename)) { + throw new Error('Invalid filename: control characters not allowed'); + } + } + async createBucket(): Promise { const exists = await this.client.bucketExists(this.bucketName); if (!exists) { @@ -83,6 +128,17 @@ export class MinioStorageService implements StorageService { buffer: Buffer, contentType: string ): Promise { + // Validate inputs first + this.validateFilePath(orgId, projectId, category, filename); + + if (!buffer || buffer.length === 0) { + throw new Error('Buffer cannot be empty'); + } + + if (!contentType || contentType.trim().length === 0) { + throw new Error('Content type is required'); + } + // Ensure bucket exists await this.createBucket(); @@ -129,6 +185,7 @@ export class MinioStorageService implements StorageService { category: 'uploads' | 'generated' | 'references', filename: string ): Promise { + this.validateFilePath(orgId, projectId, category, filename); const filePath = this.getFilePath(orgId, projectId, category, filename); const stream = await this.client.getObject(this.bucketName, filePath); @@ -141,12 +198,26 @@ export class MinioStorageService implements StorageService { }); } + async streamFile( + orgId: string, + projectId: string, + category: 'uploads' | 'generated' | 'references', + filename: string + ): Promise { + this.validateFilePath(orgId, projectId, category, filename); + const filePath = this.getFilePath(orgId, projectId, category, filename); + + // Return the stream directly without buffering - memory efficient! + return await this.client.getObject(this.bucketName, filePath); + } + async deleteFile( orgId: string, projectId: string, category: 'uploads' | 'generated' | 'references', filename: string ): Promise { + this.validateFilePath(orgId, projectId, category, filename); const filePath = this.getFilePath(orgId, projectId, category, filename); await this.client.removeObject(this.bucketName, filePath); } @@ -157,6 +228,7 @@ export class MinioStorageService implements StorageService { category: 'uploads' | 'generated' | 'references', filename: string ): string { + this.validateFilePath(orgId, projectId, category, filename); // Production-ready: Return API URL for presigned URL access const apiBaseUrl = process.env['API_BASE_URL'] || 'http://localhost:3000'; return `${apiBaseUrl}/api/images/${orgId}/${projectId}/${category}/${filename}`; @@ -167,8 +239,15 @@ export class MinioStorageService implements StorageService { projectId: string, category: 'uploads' | 'generated' | 'references', filename: string, - expirySeconds: number = 3600 + expirySeconds: number, + contentType: string ): Promise { + this.validateFilePath(orgId, projectId, category, filename); + + if (!contentType || contentType.trim().length === 0) { + throw new Error('Content type is required for presigned upload URL'); + } + const filePath = this.getFilePath(orgId, projectId, category, filename); return await this.client.presignedPutObject(this.bucketName, filePath, expirySeconds); } @@ -180,6 +259,7 @@ export class MinioStorageService implements StorageService { filename: string, expirySeconds: number = 86400 // 24 hours default ): Promise { + this.validateFilePath(orgId, projectId, category, filename); const filePath = this.getFilePath(orgId, projectId, category, filename); const presignedUrl = await this.client.presignedGetObject(this.bucketName, filePath, expirySeconds); @@ -271,4 +351,67 @@ export class MinioStorageService implements StorageService { return null; } } + + // MISSING METHODS FROM INTERFACE + + async fileExists( + orgId: string, + projectId: string, + category: 'uploads' | 'generated' | 'references', + filename: string + ): Promise { + try { + this.validateFilePath(orgId, projectId, category, filename); + const filePath = this.getFilePath(orgId, projectId, category, filename); + await this.client.statObject(this.bucketName, filePath); + return true; + } catch (error) { + return false; + } + } + + async listFiles( + orgId: string, + projectId: string, + category: 'uploads' | 'generated' | 'references', + prefix?: string + ): Promise { + this.validateFilePath(orgId, projectId, category, 'dummy.txt'); // Validate path components + + const basePath = `${orgId}/${projectId}/${category}/`; + const searchPrefix = prefix ? `${basePath}${prefix}` : basePath; + + const files: FileMetadata[] = []; + + return new Promise((resolve, reject) => { + const stream = this.client.listObjects(this.bucketName, searchPrefix, true); + + stream.on('data', async (obj) => { + if (!obj.name || !obj.size) return; + + try { + const pathParts = obj.name.split('/'); + const filename = pathParts[pathParts.length - 1]; + + if (!filename) return; + + const metadata = await this.client.statObject(this.bucketName, obj.name); + + files.push({ + filename, + size: obj.size, + contentType: metadata.metaData?.['content-type'] || 'application/octet-stream', + lastModified: obj.lastModified || new Date(), + etag: metadata.etag, + path: obj.name + }); + } catch (error) { + console.error(`Error processing file ${obj.name}:`, error); + } + }); + + stream.on('end', () => resolve(files)); + stream.on('error', reject); + }); + } } \ No newline at end of file diff --git a/src/services/StorageFactory.ts b/src/services/StorageFactory.ts index 35710b4..244f3b5 100644 --- a/src/services/StorageFactory.ts +++ b/src/services/StorageFactory.ts @@ -3,55 +3,129 @@ import { MinioStorageService } from './MinioStorageService'; export class StorageFactory { private static instance: StorageService | null = null; + private static initializationPromise: Promise | null = null; - static getInstance(): StorageService { + static async getInstance(): Promise { + if (this.instance) { + return this.instance; + } + + if (this.initializationPromise) { + return await this.initializationPromise; + } + + this.initializationPromise = this.createStorageServiceWithRetry(); + + try { + this.instance = await this.initializationPromise; + return this.instance; + } catch (error) { + this.initializationPromise = null; + throw error; + } + } + + // Synchronous version for backward compatibility (with graceful degradation) + static getInstanceSync(): StorageService { if (!this.instance) { - this.instance = this.createStorageService(); + try { + this.instance = this.createStorageService(); + } catch (error) { + console.error('Failed to create storage service:', error); + throw new Error('Storage service unavailable. Please check MinIO configuration.'); + } } return this.instance; } + + private static async createStorageServiceWithRetry(): Promise { + const maxRetries = 3; + const baseDelay = 1000; // 1 second + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + console.log(`Attempting to create storage service (attempt ${attempt}/${maxRetries})`); + + const service = this.createStorageService(); + + // Test the connection by checking if bucket exists + await service.bucketExists(); + + console.log('Storage service created successfully'); + return service; + + } catch (error) { + console.error(`Storage service creation attempt ${attempt} failed:`, error); + + if (attempt === maxRetries) { + throw new Error( + `Failed to initialize storage service after ${maxRetries} attempts. ` + + `Last error: ${error instanceof Error ? error.message : 'Unknown error'}` + ); + } + + // Exponential backoff + const delay = baseDelay * Math.pow(2, attempt - 1); + console.log(`Waiting ${delay}ms before retry...`); + await this.sleep(delay); + } + } + + throw new Error('Unexpected error in storage service creation'); + } + + private static sleep(ms: number): Promise { + return new Promise(resolve => setTimeout(resolve, ms)); + } + private static createStorageService(): StorageService { const storageType = process.env['STORAGE_TYPE'] || 'minio'; - switch (storageType.toLowerCase()) { - case 'minio': { - const endpoint = process.env['MINIO_ENDPOINT']; - const accessKey = process.env['MINIO_ACCESS_KEY']; - const secretKey = process.env['MINIO_SECRET_KEY']; - const useSSL = process.env['MINIO_USE_SSL'] === 'true'; - const bucketName = process.env['MINIO_BUCKET_NAME'] || 'banatie'; - const publicUrl = process.env['MINIO_PUBLIC_URL']; + try { + switch (storageType.toLowerCase()) { + case 'minio': { + const endpoint = process.env['MINIO_ENDPOINT']; + const accessKey = process.env['MINIO_ACCESS_KEY']; + const secretKey = process.env['MINIO_SECRET_KEY']; + const useSSL = process.env['MINIO_USE_SSL'] === 'true'; + const bucketName = process.env['MINIO_BUCKET_NAME'] || 'banatie'; + const publicUrl = process.env['MINIO_PUBLIC_URL']; - if (!endpoint || !accessKey || !secretKey) { - throw new Error( - 'MinIO configuration missing. Required: MINIO_ENDPOINT, MINIO_ACCESS_KEY, MINIO_SECRET_KEY' + if (!endpoint || !accessKey || !secretKey) { + throw new Error( + 'MinIO configuration missing. Required: MINIO_ENDPOINT, MINIO_ACCESS_KEY, MINIO_SECRET_KEY' + ); + } + + console.log(`Initializing MinIO Storage Service:`); + console.log(` Endpoint: ${endpoint}`); + console.log(` Bucket: ${bucketName}`); + console.log(` SSL: ${useSSL}`); + console.log(` Public URL: ${publicUrl}`); + + return new MinioStorageService( + endpoint, + accessKey, + secretKey, + useSSL, + bucketName, + publicUrl ); } - console.log(`Initializing MinIO Storage Service:`); - console.log(` Endpoint: ${endpoint}`); - console.log(` Bucket: ${bucketName}`); - console.log(` SSL: ${useSSL}`); - console.log(` Public URL: ${publicUrl}`); - - return new MinioStorageService( - endpoint, - accessKey, - secretKey, - useSSL, - bucketName, - publicUrl - ); + default: + throw new Error(`Unsupported storage type: ${storageType}`); } - - default: - throw new Error(`Unsupported storage type: ${storageType}`); + } catch (error) { + console.error('Error creating storage service:', error); + throw error; } } // Reset instance for testing static resetInstance(): void { this.instance = null; + this.initializationPromise = null; } } \ No newline at end of file diff --git a/src/services/StorageService.ts b/src/services/StorageService.ts index a69f21d..6d22a00 100644 --- a/src/services/StorageService.ts +++ b/src/services/StorageService.ts @@ -1,3 +1,5 @@ +import { Readable } from 'stream'; + export interface FileMetadata { filename: string; size: number; @@ -60,6 +62,20 @@ export interface StorageService { filename: string ): Promise; + /** + * Stream a file from storage (memory efficient) + * @param orgId Organization ID + * @param projectId Project ID + * @param category File category + * @param filename Filename to stream + */ + streamFile( + orgId: string, + projectId: string, + category: 'uploads' | 'generated' | 'references', + filename: string + ): Promise; + /** * Generate a presigned URL for downloading a file * @param orgId Organization ID