sh-storage

sh-storage

A unified NestJS storage library with S3 support, streaming upload, and configurable file retention.

Features

• 🚀 S3 Storage Provider: Full AWS S3 support with optimized performance
• 📁 Streaming Upload: Efficient streaming upload for large files (primary method)
• 🔄 Internal Retry: Built-in retry mechanism with exponential backoff
• 📊 Progress Tracking: Upload progress callbacks
• 🔗 Presigned URLs: Generate presigned URLs for direct uploads
• 🏷️ Prefix Support: Configurable file prefix (default: 'mountseaapi')
• 📦 TypeScript Support: Full TypeScript support with type definitions
• 🎯 Simplified API: Streamlined interface focusing on core functionality

Quick Example

import { StorageModule, StorageService } from 'sh-storage';

// 1. Configure in your module
@Module({
  imports: [
    StorageModule.forRoot({
      config: {
        provider: 's3',
        s3: {
          accessKeyId: process.env.AWS_ACCESS_KEY_ID,
          secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
          region: process.env.AWS_REGION,
          bucket: process.env.AWS_BUCKET,
        },
      },
    }),
  ],
})
export class AppModule {}

// 2. Use in your service
@Injectable()
export class FileService {
  constructor(private storage: StorageService) {}

  async uploadFile(file: Express.Multer.File) {
    return await this.storage.uploadBuffer(file.buffer, {
      key: `uploads/${file.originalname}`,
      contentType: file.mimetype,
    });
  }
}

Installation

# Using npm
npm install sh-storage

# Using yarn
yarn add sh-storage

# Using pnpm
pnpm add sh-storage

Quick Start

Prerequisites

• Node.js 16+
• NestJS 10+ or 11+
• AWS S3 bucket (for S3 provider)

1. S3 Configuration

import { StorageModule } from 'sh-storage';

@Module({
  imports: [
    StorageModule.forRoot({
      config: {
        provider: 's3',
        s3: {
          accessKeyId: 'your-access-key',
          secretAccessKey: 'your-secret-key',
          region: 'us-east-1',
          bucket: 'your-bucket-name',
        },
        prefix: 'mountseaapi', // 文件前缀，默认为 'mountseaapi'
        retryConfig: {
          maxRetries: 3, // 最大重试次数，默认3
          retryDelay: 1000, // 重试间隔基数（毫秒），默认1000
        },
      },
    }),
  ],
})
export class AppModule {}

2. Using the Service

import { Injectable } from '@nestjs/common';
import { StorageService } from 'sh-storage';

@Injectable()
export class FileService {
  constructor(private readonly storageService: StorageService) {}

  async uploadFile(file: Express.Multer.File) {
    const result = await this.storageService.uploadBuffer(file.buffer, {
      key: `uploads/${Date.now()}-${file.originalname}`,
      contentType: file.mimetype,
    });
    
    return result;
  }

  async uploadStream(stream: NodeJS.ReadableStream, filename: string) {
    const result = await this.storageService.uploadStream(stream, {
      key: `videos/${Date.now()}-${filename}`,
      contentType: 'video/mp4',
      onProgress: (progress) => {
        console.log(`Upload progress: ${progress.loaded}/${progress.total}`);
      },
    });
    
    return result;
  }

  async downloadFile(key: string) {
    return await this.storageService.download({ key });
  }

  async deleteFile(key: string) {
    return await this.storageService.delete({ keys: [key] });
  }
}

API Reference

StorageService

Upload Methods

• uploadStream(stream, options) - Upload a readable stream
• uploadBuffer(buffer, options) - Upload a buffer
• uploadString(content, options) - Upload a string
• downloadAndUpload(url, options, maxRetries?) - Download from URL and upload
• downloadAndUploadBatch(urls, keyPrefix, options, maxRetries?) - Batch download and upload

Download Methods

• download(options) - Download as stream
• downloadString(key) - Download as string

Management Methods

• delete(options) - Delete files
• list(options) - List files
• getFileInfo(key) - Get file information
• exists(key) - Check if file exists
• cleanupExpiredFiles(prefix?) - Clean up expired files

URL Methods

• generatePresignedUrl(options) - Generate presigned URL

Upload Options

interface UploadOptions {
  key: string;                    // File key/path
  contentType?: string;           // MIME type
  metadata?: Record<string, string>; // Custom metadata
  tags?: Record<string, string>;     // Tags for organization
}

Stream Upload Options

interface StreamUploadOptions extends UploadOptions {
  onProgress?: (progress: { loaded: number; total?: number }) => void;
}

Advanced Usage

Async Configuration

import { ConfigService } from '@nestjs/config';
import { StorageModule } from 'sh-storage';

@Module({
  imports: [
    StorageModule.forRootAsync({
      useFactory: (configService: ConfigService) => ({
        provider: 's3',
        s3: {
          accessKeyId: configService.get('AWS_ACCESS_KEY_ID'),
          secretAccessKey: configService.get('AWS_SECRET_ACCESS_KEY'),
          region: configService.get('AWS_REGION'),
          bucket: configService.get('AWS_BUCKET'),
        },
      }),
      inject: [ConfigService],
    }),
  ],
})
export class AppModule {}

Prefix and Retry Configuration

// 自定义前缀和重试配置
const config = {
  provider: 's3',
  s3: { /* ... */ },
  prefix: 'myapp', // 自定义前缀
  retryConfig: {
    maxRetries: 5, // 最大重试5次
    retryDelay: 2000, // 重试间隔基数2秒
  },
};

// 使用默认配置
const config = {
  provider: 's3',
  s3: { /* ... */ },
  // prefix 默认为 'mountseaapi'
  // retryConfig 默认为 { maxRetries: 3, retryDelay: 1000 }
};

Progress Tracking

async uploadWithProgress(file: Express.Multer.File) {
  const stream = Readable.from(file.buffer);
  
  return await this.storageService.uploadStream(stream, {
    key: `uploads/${file.originalname}`,
    contentType: file.mimetype,
    onProgress: (progress) => {
      const percentage = progress.total 
        ? Math.round((progress.loaded / progress.total) * 100)
        : 0;
      console.log(`Upload progress: ${percentage}%`);
    },
  });
}

Batch Operations

async processMultipleFiles(urls: string[]) {
  const results = await this.storageService.downloadAndUploadBatch(
    urls,
    'batch-uploads',
    {
      contentType: 'application/octet-stream',
    }
  );
  
  return results;
}

Error Handling

The library provides comprehensive error handling with detailed error messages:

try {
  const result = await this.storageService.uploadBuffer(buffer, options);
} catch (error) {
  if (error.message.includes('S3 upload failed')) {
    // Handle S3 specific errors
  } else if (error.message.includes('Local upload failed')) {
    // Handle local storage errors
  }
  // Handle other errors
}

Environment Variables

For S3 configuration, you can use these environment variables:

AWS_ACCESS_KEY_ID=your-access-key
AWS_SECRET_ACCESS_KEY=your-secret-key
AWS_REGION=us-east-1
AWS_BUCKET=your-bucket-name

Version

Current version: 1.0.1

Real-world Examples

File Upload Controller

import { Controller, Post, UploadedFile, UseInterceptors } from '@nestjs/common';
import { FileInterceptor } from '@nestjs/platform-express';
import { StorageService } from 'sh-storage';

@Controller('files')
export class FileController {
  constructor(private readonly storageService: StorageService) {}

  @Post('upload')
  @UseInterceptors(FileInterceptor('file'))
  async uploadFile(@UploadedFile() file: Express.Multer.File) {
    const result = await this.storageService.uploadBuffer(file.buffer, {
      key: `uploads/${Date.now()}-${file.originalname}`,
      contentType: file.mimetype,
    });
    
    return {
      success: true,
      key: result.key,
      url: result.url,
      size: file.size,
    };
  }
}

Video Processing Service

import { Injectable } from '@nestjs/common';
import { StorageService } from 'sh-storage';
import { Readable } from 'stream';

@Injectable()
export class VideoService {
  constructor(private readonly storageService: StorageService) {}

  async processVideo(videoBuffer: Buffer, filename: string) {
    const stream = Readable.from(videoBuffer);
    
    return await this.storageService.uploadStream(stream, {
      key: `videos/processed/${Date.now()}-${filename}`,
      contentType: 'video/mp4',
      onProgress: (progress) => {
        console.log(`Video upload: ${progress.loaded}/${progress.total || 'unknown'}`);
      },
    });
  }
}

License

MIT

Contributing

• Fork the repository
• Create your feature branch (git checkout -b feature/amazing-feature)
• Commit your changes (git commit -m 'Add some amazing feature')
• Push to the branch (git push origin feature/amazing-feature)
• Open a Pull Request