478 lines
15 KiB
TypeScript
478 lines
15 KiB
TypeScript
import { Client as MinioClient } from 'minio';
|
||
import { StorageService, FileMetadata, UploadResult } from './StorageService';
|
||
|
||
export class MinioStorageService implements StorageService {
|
||
private client: MinioClient;
|
||
private bucketName: string;
|
||
private publicUrl: string;
|
||
|
||
constructor(
|
||
endpoint: string,
|
||
accessKey: string,
|
||
secretKey: string,
|
||
useSSL: boolean = false,
|
||
bucketName: string = 'banatie',
|
||
publicUrl?: string,
|
||
) {
|
||
// Parse endpoint to separate hostname and port
|
||
const cleanEndpoint = endpoint.replace(/^https?:\/\//, '');
|
||
const [hostname, portStr] = cleanEndpoint.split(':');
|
||
const port = portStr ? parseInt(portStr, 10) : useSSL ? 443 : 9000;
|
||
|
||
if (!hostname) {
|
||
throw new Error(`Invalid MinIO endpoint: ${endpoint}`);
|
||
}
|
||
|
||
this.client = new MinioClient({
|
||
endPoint: hostname,
|
||
port,
|
||
useSSL,
|
||
accessKey,
|
||
secretKey,
|
||
});
|
||
this.bucketName = bucketName;
|
||
this.publicUrl = publicUrl || `${useSSL ? 'https' : 'http'}://${endpoint}`;
|
||
}
|
||
|
||
private getFilePath(
|
||
orgId: string,
|
||
projectId: string,
|
||
category: 'uploads' | 'generated' | 'references',
|
||
filename: string,
|
||
): string {
|
||
// Simplified path without date folder for now
|
||
return `${orgId}/${projectId}/${category}/${filename}`;
|
||
}
|
||
|
||
private generateUniqueFilename(originalFilename: string): string {
|
||
// Sanitize filename first
|
||
const sanitized = this.sanitizeFilename(originalFilename);
|
||
|
||
const timestamp = Date.now();
|
||
const random = Math.random().toString(36).substring(2, 8);
|
||
const ext = sanitized.includes('.') ? sanitized.substring(sanitized.lastIndexOf('.')) : '';
|
||
const name = sanitized.includes('.')
|
||
? sanitized.substring(0, sanitized.lastIndexOf('.'))
|
||
: sanitized;
|
||
|
||
return `${name}-${timestamp}-${random}${ext}`;
|
||
}
|
||
|
||
private sanitizeFilename(filename: string): string {
|
||
// Remove path traversal attempts FIRST from entire filename
|
||
let cleaned = filename.replace(/\.\./g, '').trim();
|
||
|
||
// Split filename and extension
|
||
const lastDotIndex = cleaned.lastIndexOf('.');
|
||
let baseName = lastDotIndex > 0 ? cleaned.substring(0, lastDotIndex) : cleaned;
|
||
const extension = lastDotIndex > 0 ? cleaned.substring(lastDotIndex) : '';
|
||
|
||
// Remove dangerous characters from base name
|
||
baseName = baseName
|
||
.replace(/[<>:"/\\|?*\x00-\x1f]/g, '') // Remove dangerous chars
|
||
.trim();
|
||
|
||
// Replace non-ASCII characters with ASCII equivalents or remove them
|
||
// This prevents S3 signature mismatches with MinIO
|
||
baseName = baseName
|
||
.normalize('NFD') // Decompose combined characters (é -> e + ´)
|
||
.replace(/[\u0300-\u036f]/g, '') // Remove diacritical marks
|
||
.replace(/[^\x20-\x7E]/g, '_') // Replace any remaining non-ASCII with underscore
|
||
.replace(/[^\w\s\-_.]/g, '_') // Replace special chars (except word chars, space, dash, underscore, dot) with underscore
|
||
.replace(/\s+/g, '_') // Replace spaces with underscores
|
||
.replace(/_{2,}/g, '_') // Collapse multiple underscores
|
||
.replace(/^_+|_+$/g, ''); // Remove leading/trailing underscores
|
||
|
||
// Ensure we still have a valid base name
|
||
if (baseName.length === 0) {
|
||
baseName = 'file';
|
||
}
|
||
|
||
// Sanitize extension (remove only dangerous chars, keep the dot)
|
||
let sanitizedExt = extension
|
||
.replace(/[<>:"/\\|?*\x00-\x1f]/g, '')
|
||
.replace(/[^\x20-\x7E]/g, '')
|
||
.toLowerCase();
|
||
|
||
// Ensure extension starts with a dot and is reasonable
|
||
if (sanitizedExt && !sanitizedExt.startsWith('.')) {
|
||
sanitizedExt = '.' + sanitizedExt;
|
||
}
|
||
if (sanitizedExt.length > 10) {
|
||
sanitizedExt = sanitizedExt.substring(0, 10);
|
||
}
|
||
|
||
const result = baseName + sanitizedExt;
|
||
return result.substring(0, 255); // Limit total length
|
||
}
|
||
|
||
private validateFilePath(
|
||
orgId: string,
|
||
projectId: string,
|
||
category: string,
|
||
filename: string,
|
||
): void {
|
||
// Validate orgId
|
||
if (!orgId || !/^[a-zA-Z0-9_-]+$/.test(orgId) || orgId.length > 50) {
|
||
throw new Error(
|
||
'Invalid organization ID: must be alphanumeric with dashes/underscores, max 50 chars',
|
||
);
|
||
}
|
||
|
||
// Validate projectId
|
||
if (!projectId || !/^[a-zA-Z0-9_-]+$/.test(projectId) || projectId.length > 50) {
|
||
throw new Error(
|
||
'Invalid project ID: must be alphanumeric with dashes/underscores, max 50 chars',
|
||
);
|
||
}
|
||
|
||
// Validate category
|
||
if (!['uploads', 'generated', 'references'].includes(category)) {
|
||
throw new Error('Invalid category: must be uploads, generated, or references');
|
||
}
|
||
|
||
// Validate filename
|
||
if (!filename || filename.length === 0 || filename.length > 255) {
|
||
throw new Error('Invalid filename: must be 1-255 characters');
|
||
}
|
||
|
||
// Check for path traversal and dangerous patterns
|
||
if (filename.includes('..') || filename.includes('/') || filename.includes('\\')) {
|
||
throw new Error('Invalid characters in filename: path traversal not allowed');
|
||
}
|
||
|
||
// Prevent null bytes and control characters
|
||
if (/[\x00-\x1f]/.test(filename)) {
|
||
throw new Error('Invalid filename: control characters not allowed');
|
||
}
|
||
}
|
||
|
||
async createBucket(): Promise<void> {
|
||
const exists = await this.client.bucketExists(this.bucketName);
|
||
if (!exists) {
|
||
await this.client.makeBucket(this.bucketName, 'us-east-1');
|
||
console.log(`Created bucket: ${this.bucketName}`);
|
||
}
|
||
|
||
// Note: With SNMD and presigned URLs, we don't need bucket policies
|
||
console.log(`Bucket ${this.bucketName} ready for presigned URL access`);
|
||
}
|
||
|
||
async bucketExists(): Promise<boolean> {
|
||
return await this.client.bucketExists(this.bucketName);
|
||
}
|
||
|
||
async uploadFile(
|
||
orgId: string,
|
||
projectId: string,
|
||
category: 'uploads' | 'generated' | 'references',
|
||
filename: string,
|
||
buffer: Buffer,
|
||
contentType: string,
|
||
): Promise<UploadResult> {
|
||
// Validate inputs first
|
||
this.validateFilePath(orgId, projectId, category, filename);
|
||
|
||
if (!buffer || buffer.length === 0) {
|
||
throw new Error('Buffer cannot be empty');
|
||
}
|
||
|
||
if (!contentType || contentType.trim().length === 0) {
|
||
throw new Error('Content type is required');
|
||
}
|
||
|
||
// Ensure bucket exists
|
||
await this.createBucket();
|
||
|
||
// Generate unique filename to avoid conflicts
|
||
const uniqueFilename = this.generateUniqueFilename(filename);
|
||
const filePath = this.getFilePath(orgId, projectId, category, uniqueFilename);
|
||
|
||
// Encode original filename to Base64 to safely store non-ASCII characters in metadata
|
||
const originalNameEncoded = Buffer.from(filename, 'utf-8').toString('base64');
|
||
|
||
const metadata = {
|
||
'Content-Type': contentType,
|
||
'X-Amz-Meta-Original-Name': originalNameEncoded,
|
||
'X-Amz-Meta-Original-Name-Encoding': 'base64',
|
||
'X-Amz-Meta-Category': category,
|
||
'X-Amz-Meta-Project': projectId,
|
||
'X-Amz-Meta-Organization': orgId,
|
||
'X-Amz-Meta-Upload-Time': new Date().toISOString(),
|
||
};
|
||
|
||
console.log(`Uploading file to: ${this.bucketName}/${filePath}`);
|
||
|
||
const result = await this.client.putObject(
|
||
this.bucketName,
|
||
filePath,
|
||
buffer,
|
||
buffer.length,
|
||
metadata,
|
||
);
|
||
|
||
const url = this.getPublicUrl(orgId, projectId, category, uniqueFilename);
|
||
|
||
console.log(`Generated API URL: ${url}`);
|
||
|
||
return {
|
||
success: true,
|
||
filename: uniqueFilename,
|
||
path: filePath,
|
||
url,
|
||
size: buffer.length,
|
||
contentType,
|
||
};
|
||
}
|
||
|
||
async downloadFile(
|
||
orgId: string,
|
||
projectId: string,
|
||
category: 'uploads' | 'generated' | 'references',
|
||
filename: string,
|
||
): Promise<Buffer> {
|
||
this.validateFilePath(orgId, projectId, category, filename);
|
||
const filePath = this.getFilePath(orgId, projectId, category, filename);
|
||
|
||
const stream = await this.client.getObject(this.bucketName, filePath);
|
||
|
||
return new Promise((resolve, reject) => {
|
||
const chunks: Buffer[] = [];
|
||
stream.on('data', (chunk) => chunks.push(chunk));
|
||
stream.on('end', () => resolve(Buffer.concat(chunks)));
|
||
stream.on('error', reject);
|
||
});
|
||
}
|
||
|
||
async streamFile(
|
||
orgId: string,
|
||
projectId: string,
|
||
category: 'uploads' | 'generated' | 'references',
|
||
filename: string,
|
||
): Promise<import('stream').Readable> {
|
||
this.validateFilePath(orgId, projectId, category, filename);
|
||
const filePath = this.getFilePath(orgId, projectId, category, filename);
|
||
|
||
// Return the stream directly without buffering - memory efficient!
|
||
return await this.client.getObject(this.bucketName, filePath);
|
||
}
|
||
|
||
async deleteFile(
|
||
orgId: string,
|
||
projectId: string,
|
||
category: 'uploads' | 'generated' | 'references',
|
||
filename: string,
|
||
): Promise<void> {
|
||
this.validateFilePath(orgId, projectId, category, filename);
|
||
const filePath = this.getFilePath(orgId, projectId, category, filename);
|
||
await this.client.removeObject(this.bucketName, filePath);
|
||
}
|
||
|
||
getPublicUrl(
|
||
orgId: string,
|
||
projectId: string,
|
||
category: 'uploads' | 'generated' | 'references',
|
||
filename: string,
|
||
): string {
|
||
this.validateFilePath(orgId, projectId, category, filename);
|
||
// Production-ready: Return API URL for presigned URL access
|
||
const apiBaseUrl = process.env['API_BASE_URL'] || 'http://localhost:3000';
|
||
return `${apiBaseUrl}/api/images/${orgId}/${projectId}/${category}/${filename}`;
|
||
}
|
||
|
||
async getPresignedUploadUrl(
|
||
orgId: string,
|
||
projectId: string,
|
||
category: 'uploads' | 'generated' | 'references',
|
||
filename: string,
|
||
expirySeconds: number,
|
||
contentType: string,
|
||
): Promise<string> {
|
||
this.validateFilePath(orgId, projectId, category, filename);
|
||
|
||
if (!contentType || contentType.trim().length === 0) {
|
||
throw new Error('Content type is required for presigned upload URL');
|
||
}
|
||
|
||
const filePath = this.getFilePath(orgId, projectId, category, filename);
|
||
return await this.client.presignedPutObject(this.bucketName, filePath, expirySeconds);
|
||
}
|
||
|
||
async getPresignedDownloadUrl(
|
||
orgId: string,
|
||
projectId: string,
|
||
category: 'uploads' | 'generated' | 'references',
|
||
filename: string,
|
||
expirySeconds: number = 86400, // 24 hours default
|
||
): Promise<string> {
|
||
this.validateFilePath(orgId, projectId, category, filename);
|
||
const filePath = this.getFilePath(orgId, projectId, category, filename);
|
||
const presignedUrl = await this.client.presignedGetObject(
|
||
this.bucketName,
|
||
filePath,
|
||
expirySeconds,
|
||
);
|
||
|
||
// Replace internal Docker hostname with public URL if configured
|
||
if (this.publicUrl) {
|
||
const clientEndpoint = this.client.host + (this.client.port ? `:${this.client.port}` : '');
|
||
const publicEndpoint = this.publicUrl.replace(/^https?:\/\//, '');
|
||
|
||
return presignedUrl.replace(`${this.client.protocol}//${clientEndpoint}`, this.publicUrl);
|
||
}
|
||
|
||
return presignedUrl;
|
||
}
|
||
|
||
async listProjectFiles(
|
||
orgId: string,
|
||
projectId: string,
|
||
category?: 'uploads' | 'generated' | 'references',
|
||
): Promise<FileMetadata[]> {
|
||
const prefix = category ? `${orgId}/${projectId}/${category}/` : `${orgId}/${projectId}/`;
|
||
|
||
const files: FileMetadata[] = [];
|
||
|
||
return new Promise((resolve, reject) => {
|
||
const stream = this.client.listObjects(this.bucketName, prefix, true);
|
||
|
||
stream.on('data', async (obj) => {
|
||
try {
|
||
if (!obj.name) return;
|
||
|
||
const metadata = await this.client.statObject(this.bucketName, obj.name);
|
||
|
||
const pathParts = obj.name.split('/');
|
||
const filename = pathParts[pathParts.length - 1];
|
||
const categoryFromPath = pathParts[2] as 'uploads' | 'generated' | 'references';
|
||
|
||
if (!filename || !categoryFromPath) {
|
||
return;
|
||
}
|
||
|
||
files.push({
|
||
key: `${this.bucketName}/${obj.name}`,
|
||
filename,
|
||
contentType: metadata.metaData?.['content-type'] || 'application/octet-stream',
|
||
size: obj.size || 0,
|
||
url: this.getPublicUrl(orgId, projectId, categoryFromPath, filename),
|
||
createdAt: obj.lastModified || new Date(),
|
||
});
|
||
} catch (error) {}
|
||
});
|
||
|
||
stream.on('end', () => resolve(files));
|
||
stream.on('error', reject);
|
||
});
|
||
}
|
||
|
||
parseKey(key: string): {
|
||
orgId: string;
|
||
projectId: string;
|
||
category: 'uploads' | 'generated' | 'references';
|
||
filename: string;
|
||
} | null {
|
||
try {
|
||
const match = key.match(
|
||
/^banatie\/([^/]+)\/([^/]+)\/(uploads|generated|references)\/[^/]+\/(.+)$/,
|
||
);
|
||
|
||
if (!match) {
|
||
return null;
|
||
}
|
||
|
||
const [, orgId, projectId, category, filename] = match;
|
||
|
||
if (!orgId || !projectId || !category || !filename) {
|
||
return null;
|
||
}
|
||
|
||
return {
|
||
orgId,
|
||
projectId,
|
||
category: category as 'uploads' | 'generated' | 'references',
|
||
filename,
|
||
};
|
||
} catch {
|
||
return null;
|
||
}
|
||
}
|
||
|
||
async fileExists(
|
||
orgId: string,
|
||
projectId: string,
|
||
category: 'uploads' | 'generated' | 'references',
|
||
filename: string,
|
||
): Promise<boolean> {
|
||
try {
|
||
this.validateFilePath(orgId, projectId, category, filename);
|
||
const filePath = this.getFilePath(orgId, projectId, category, filename);
|
||
await this.client.statObject(this.bucketName, filePath);
|
||
return true;
|
||
} catch (error) {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
async listFiles(
|
||
orgId: string,
|
||
projectId: string,
|
||
category: 'uploads' | 'generated' | 'references',
|
||
prefix?: string,
|
||
): Promise<FileMetadata[]> {
|
||
this.validateFilePath(orgId, projectId, category, 'dummy.txt');
|
||
|
||
const basePath = `${orgId}/${projectId}/${category}/`;
|
||
const searchPrefix = prefix ? `${basePath}${prefix}` : basePath;
|
||
|
||
const files: FileMetadata[] = [];
|
||
|
||
return new Promise((resolve, reject) => {
|
||
const stream = this.client.listObjects(this.bucketName, searchPrefix, true);
|
||
|
||
stream.on('data', (obj) => {
|
||
if (!obj.name || !obj.size) return;
|
||
|
||
try {
|
||
const pathParts = obj.name.split('/');
|
||
const filename = pathParts[pathParts.length - 1];
|
||
|
||
if (!filename) return;
|
||
|
||
// Infer content type from file extension (more efficient than statObject)
|
||
const ext = filename.toLowerCase().split('.').pop();
|
||
const contentType =
|
||
{
|
||
png: 'image/png',
|
||
jpg: 'image/jpeg',
|
||
jpeg: 'image/jpeg',
|
||
gif: 'image/gif',
|
||
webp: 'image/webp',
|
||
svg: 'image/svg+xml',
|
||
}[ext || ''] || 'application/octet-stream';
|
||
|
||
files.push({
|
||
filename,
|
||
size: obj.size,
|
||
contentType,
|
||
lastModified: obj.lastModified || new Date(),
|
||
etag: obj.etag,
|
||
path: obj.name,
|
||
});
|
||
} catch (error) {
|
||
console.error('[MinIO listFiles] Error processing file:', obj.name, error);
|
||
}
|
||
});
|
||
|
||
stream.on('end', () => {
|
||
resolve(files);
|
||
});
|
||
|
||
stream.on('error', (error) => {
|
||
console.error('[MinIO listFiles] Stream error:', error);
|
||
reject(error);
|
||
});
|
||
});
|
||
}
|
||
}
|