Merge pull request #170 from internxt/feat/pb-1698-generate-thumbnails-on-upload

larryrider · web-flow · commit c06a0b448e19 · 2025-01-17T19:03:07.000+01:00
[PB-1698]: feat/generate-thumbnails-on-upload
diff --git a/package.json b/package.json
@@ -62,6 +62,7 @@
     "selfsigned": "2.4.1",
     "sequelize": "6.37.5",
     "sequelize-typescript": "2.1.6",
+    "sharp": "0.33.5",
     "sqlite3": "5.1.7",
     "tty-table": "4.2.3",
     "winston": "3.17.0"
@@ -75,13 +76,13 @@
     "@types/cli-progress": "3.11.6",
     "@types/express": "5.0.0",
     "@types/mime-types": "2.1.4",
-    "@types/node": "22.10.6",
+    "@types/node": "22.10.7",
     "@types/range-parser": "1.2.7",
     "@vitest/coverage-istanbul": "2.1.8",
     "@vitest/spy": "2.1.8",
     "eslint": "9.18.0",
     "husky": "9.1.7",
-    "lint-staged": "15.3.0",
+    "lint-staged": "15.4.0",
     "nock": "13.5.6",
     "nodemon": "3.1.9",
     "oclif": "4.17.13",
diff --git a/src/commands/upload-file.ts b/src/commands/upload-file.ts
@@ -15,6 +15,10 @@ import { ErrorUtils } from '../utils/errors.utils';
 import { MissingCredentialsError, NotValidDirectoryError, NotValidFolderUuidError } from '../types/command.types';
 import { ValidationService } from '../services/validation.service';
 import { EncryptionVersion } from '@internxt/sdk/dist/drive/storage/types';
+import { ThumbnailService } from '../services/thumbnail.service';
+import { BufferStream } from '../utils/stream.utils';
+import { isFileThumbnailable } from '../utils/thumbnail.utils';
+import { Readable } from 'node:stream';
 
 export default class UploadFile extends Command {
   static readonly args = {};
@@ -52,6 +56,9 @@ export default class UploadFile extends Command {
       throw new Error('The file is empty. Uploading empty files is not allowed.');
     }
 
+    const fileInfo = path.parse(filePath);
+    const fileType = fileInfo.ext.replaceAll('.', '');
+
     let destinationFolderUuid = await this.getDestinationFolderUuid(flags['destination'], nonInteractive);
     if (destinationFolderUuid.trim().length === 0) {
       // destinationFolderUuid is empty from flags&prompt, which means we should use RootFolderUuid
@@ -75,45 +82,34 @@ export default class UploadFile extends Command {
     CLIUtils.done();
 
     // 2. Upload file to the Network
-    const fileStream = createReadStream(filePath);
+    const readStream = createReadStream(filePath);
     const timer = CLIUtils.timer();
     const progressBar = CLIUtils.progress({
       format: 'Uploading file [{bar}] {percentage}%',
       linewrap: true,
     });
     progressBar.start(100, 0);
 
-    const minimumMultipartThreshold = 100 * 1024 * 1024;
-    const useMultipart = stats.size > minimumMultipartThreshold;
-    const partSize = 30 * 1024 * 1024;
-    const parts = Math.ceil(stats.size / partSize);
-
-    let uploadOperation: Promise<
-      [
-        Promise<{
-          fileId: string;
-          hash: Buffer;
-        }>,
-        AbortController,
-      ]
-    >;
-
-    if (useMultipart) {
-      uploadOperation = networkFacade.uploadMultipartFromStream(user.bucket, user.mnemonic, stats.size, fileStream, {
-        parts,
-        progressCallback: (progress) => {
-          progressBar.update(progress * 0.99);
-        },
-      });
-    } else {
-      uploadOperation = networkFacade.uploadFromStream(user.bucket, user.mnemonic, stats.size, fileStream, {
-        progressCallback: (progress) => {
-          progressBar.update(progress * 0.99);
-        },
-      });
+    let bufferStream: BufferStream | undefined;
+    let fileStream: Readable = readStream;
+    const isThumbnailable = isFileThumbnailable(fileType);
+    if (isThumbnailable) {
+      bufferStream = new BufferStream();
+      fileStream = readStream.pipe(bufferStream);
     }
 
-    const [uploadPromise, abortable] = await uploadOperation;
+    const progressCallback = (progress: number) => {
+      progressBar.update(progress * 0.99);
+    };
+
+    const [uploadPromise, abortable] = await UploadService.instance.uploadFileStream(
+      fileStream,
+      user.bucket,
+      user.mnemonic,
+      stats.size,
+      networkFacade,
+      progressCallback,
+    );
 
     process.on('SIGINT', () => {
       abortable.abort('SIGINT received');
@@ -123,10 +119,9 @@ export default class UploadFile extends Command {
     const uploadResult = await uploadPromise;
 
     // 3. Create the file in Drive
-    const fileInfo = path.parse(filePath);
     const createdDriveFile = await DriveFileService.instance.createFile({
       plain_name: fileInfo.name,
-      type: fileInfo.ext.replaceAll('.', ''),
+      type: fileType,
       size: stats.size,
       folder_id: destinationFolderUuid,
       id: uploadResult.fileId,
@@ -135,6 +130,25 @@ export default class UploadFile extends Command {
       name: '',
     });
 
+    try {
+      if (isThumbnailable && bufferStream) {
+        const thumbnailBuffer = bufferStream.getBuffer();
+
+        if (thumbnailBuffer) {
+          await ThumbnailService.instance.uploadThumbnail(
+            thumbnailBuffer,
+            fileType,
+            user.bucket,
+            user.mnemonic,
+            createdDriveFile.id,
+            networkFacade,
+          );
+        }
+      }
+    } catch (error) {
+      ErrorUtils.report(this.error.bind(this), error, { command: this.id });
+    }
+
     progressBar.update(100);
     progressBar.stop();
 
diff --git a/src/services/drive/drive-file.service.ts b/src/services/drive/drive-file.service.ts
@@ -51,4 +51,9 @@ export class DriveFileService {
     const fileMetadata = await storageClient.getFileByPath(encodeURIComponent(path));
     return DriveUtils.driveFileMetaToItem(fileMetadata);
   };
+
+  public createThumbnail = (payload: StorageTypes.ThumbnailEntry): Promise<StorageTypes.Thumbnail> => {
+    const storageClient = SdkManager.instance.getStorage(false);
+    return storageClient.createThumbnailEntry(payload);
+  };
 }
diff --git a/src/services/network/network-facade.service.ts b/src/services/network/network-facade.service.ts
@@ -165,7 +165,7 @@ export class NetworkFacade {
     };
 
     const uploadFile: UploadFileFunction = async (url) => {
-      await this.uploadService.uploadFile(url, encryptionTransform, {
+      await this.uploadService.uploadFileToNetwork(url, encryptionTransform, {
         abortController: abortable,
         progressCallback: onProgress,
       });
@@ -244,7 +244,7 @@ export class NetworkFacade {
       const limitConcurrency = 6;
 
       const uploadPart = async (upload: UploadTask) => {
-        const { etag } = await this.uploadService.uploadFile(upload.urlToUpload, upload.contentToUpload, {
+        const { etag } = await this.uploadService.uploadFileToNetwork(upload.urlToUpload, upload.contentToUpload, {
           abortController: abortable,
           progressCallback: (loadedBytes: number) => {
             onProgress(upload.index, loadedBytes);
diff --git a/src/services/network/upload.service.ts b/src/services/network/upload.service.ts
@@ -1,11 +1,16 @@
 import { Readable } from 'node:stream';
 import axios from 'axios';
 import { UploadOptions } from '../../types/network.types';
+import { NetworkFacade } from './network-facade.service';
 
 export class UploadService {
   public static readonly instance: UploadService = new UploadService();
 
-  async uploadFile(url: string, from: Readable | Buffer, options: UploadOptions): Promise<{ etag: string }> {
+  public uploadFileToNetwork = async (
+    url: string,
+    from: Readable | Buffer,
+    options: UploadOptions,
+  ): Promise<{ etag: string }> => {
     const response = await axios.put(url, from, {
       signal: options.abortController?.signal,
       onUploadProgress: (progressEvent) => {
@@ -20,5 +25,44 @@ export class UploadService {
       throw new Error('Missing Etag in response when uploading file');
     }
     return { etag };
-  }
+  };
+
+  public uploadFileStream = async (
+    fileStream: Readable,
+    userBucket: string,
+    userMnemonic: string,
+    fileSize: number,
+    networkFacade: NetworkFacade,
+    progressCallback?: (progress: number) => void,
+  ) => {
+    const minimumMultipartThreshold = 100 * 1024 * 1024;
+    const useMultipart = fileSize > minimumMultipartThreshold;
+    const partSize = 30 * 1024 * 1024;
+    const parts = Math.ceil(fileSize / partSize);
+
+    let uploadOperation: Promise<
+      [
+        Promise<{
+          fileId: string;
+          hash: Buffer;
+        }>,
+        AbortController,
+      ]
+    >;
+
+    if (useMultipart) {
+      uploadOperation = networkFacade.uploadMultipartFromStream(userBucket, userMnemonic, fileSize, fileStream, {
+        parts,
+        progressCallback,
+      });
+    } else {
+      uploadOperation = networkFacade.uploadFromStream(userBucket, userMnemonic, fileSize, fileStream, {
+        progressCallback,
+      });
+    }
+
+    const uploadFileOperation = await uploadOperation;
+
+    return uploadFileOperation;
+  };
 }
diff --git a/src/services/thumbnail.service.ts b/src/services/thumbnail.service.ts
@@ -0,0 +1,63 @@
+import { Readable } from 'node:stream';
+import { DriveFileService } from './drive/drive-file.service';
+import { StorageTypes } from '@internxt/sdk/dist/drive';
+import { NetworkFacade } from './network/network-facade.service';
+import { UploadService } from './network/upload.service';
+import { isImageThumbnailable, ThumbnailConfig } from '../utils/thumbnail.utils';
+import sharp from 'sharp';
+
+export class ThumbnailService {
+  public static readonly instance: ThumbnailService = new ThumbnailService();
+
+  public uploadThumbnail = async (
+    fileContent: Buffer,
+    fileType: string,
+    userBucket: string,
+    userMnemonic: string,
+    file_id: number,
+    networkFacade: NetworkFacade,
+  ): Promise<StorageTypes.Thumbnail | undefined> => {
+    let thumbnailBuffer: Buffer | undefined;
+    if (isImageThumbnailable(fileType)) {
+      thumbnailBuffer = await this.getThumbnailFromImageBuffer(fileContent);
+    }
+    if (thumbnailBuffer) {
+      const size = thumbnailBuffer.length;
+      const [thumbnailPromise] = await UploadService.instance.uploadFileStream(
+        Readable.from(thumbnailBuffer),
+        userBucket,
+        userMnemonic,
+        size,
+        networkFacade,
+        () => {},
+      );
+
+      const thumbnailUploadResult = await thumbnailPromise;
+
+      const createdThumbnailFile = await DriveFileService.instance.createThumbnail({
+        file_id: file_id,
+        max_width: ThumbnailConfig.MaxWidth,
+        max_height: ThumbnailConfig.MaxHeight,
+        type: ThumbnailConfig.Type,
+        size: size,
+        bucket_id: userBucket,
+        bucket_file: thumbnailUploadResult.fileId,
+        encrypt_version: StorageTypes.EncryptionVersion.Aes03,
+      });
+      return createdThumbnailFile;
+    }
+  };
+
+  private getThumbnailFromImageBuffer = (buffer: Buffer): Promise<Buffer> => {
+    return sharp(buffer)
+      .resize({
+        height: ThumbnailConfig.MaxHeight,
+        width: ThumbnailConfig.MaxWidth,
+        fit: 'inside',
+      })
+      .png({
+        quality: ThumbnailConfig.Quality,
+      })
+      .toBuffer();
+  };
+}
diff --git a/src/types/network.types.ts b/src/types/network.types.ts
@@ -6,7 +6,7 @@ export interface NetworkCredentials {
 export type DownloadProgressCallback = (downloadedBytes: number) => void;
 export type UploadProgressCallback = (uploadedBytes: number) => void;
 export interface NetworkOperationBaseOptions {
-  progressCallback: UploadProgressCallback;
+  progressCallback?: UploadProgressCallback;
   abortController?: AbortController;
 }
 
diff --git a/src/utils/stream.utils.ts b/src/utils/stream.utils.ts
@@ -1,5 +1,5 @@
 import { ReadStream, WriteStream } from 'node:fs';
-import { Readable, Transform, TransformCallback } from 'node:stream';
+import { Readable, Transform, TransformCallback, TransformOptions } from 'node:stream';
 
 export class StreamUtils {
   static readStreamToReadableStream(readStream: ReadStream): ReadableStream<Uint8Array> {
@@ -130,3 +130,30 @@ export class ProgressTransform extends Transform {
     callback(null);
   }
 }
+
+export class BufferStream extends Transform {
+  public buffer: Buffer | null;
+
+  constructor(opts?: TransformOptions) {
+    super(opts);
+    this.buffer = null;
+  }
+
+  _transform(chunk: Buffer, _: BufferEncoding, callback: TransformCallback) {
+    const currentBuffer = this.buffer ?? Buffer.alloc(0);
+    this.buffer = Buffer.concat([currentBuffer, chunk]);
+    callback(null, chunk);
+  }
+
+  _flush(callback: TransformCallback) {
+    callback();
+  }
+
+  reset() {
+    this.buffer = null;
+  }
+
+  getBuffer(): Buffer | null {
+    return this.buffer;
+  }
+}
diff --git a/src/utils/thumbnail.utils.ts b/src/utils/thumbnail.utils.ts
@@ -0,0 +1,43 @@
+export const ThumbnailConfig = {
+  MaxWidth: 300,
+  MaxHeight: 300,
+  Quality: 100,
+  Type: 'png',
+} as const;
+
+type FileExtensionMap = Record<string, string[]>;
+const imageExtensions: FileExtensionMap = {
+  tiff: ['tif', 'tiff'],
+  bmp: ['bmp'],
+  heic: ['heic'],
+  jpg: ['jpg', 'jpeg'],
+  gif: ['gif'],
+  png: ['png'],
+  eps: ['eps'],
+  raw: ['raw', 'cr2', 'nef', 'orf', 'sr2'],
+  webp: ['webp'],
+};
+const pdfExtensions: FileExtensionMap = {
+  pdf: ['pdf'],
+};
+const thumbnailableImageExtension: string[] = [
+  ...imageExtensions['jpg'],
+  ...imageExtensions['png'],
+  ...imageExtensions['webp'],
+  ...imageExtensions['gif'],
+  ...imageExtensions['tiff'],
+];
+const thumbnailablePdfExtension: string[] = pdfExtensions['pdf'];
+const thumbnailableExtension: string[] = [...thumbnailableImageExtension];
+
+export const isFileThumbnailable = (fileType: string) => {
+  return fileType.trim().length > 0 && thumbnailableExtension.includes(fileType);
+};
+
+export const isPDFThumbnailable = (fileType: string) => {
+  return fileType.trim().length > 0 && thumbnailablePdfExtension.includes(fileType);
+};
+
+export const isImageThumbnailable = (fileType: string) => {
+  return fileType.trim().length > 0 && thumbnailableImageExtension.includes(fileType);
+};
diff --git a/src/webdav/handlers/PUT.handler.ts b/src/webdav/handlers/PUT.handler.ts
diff --git a/test/services/network/upload.service.test.ts b/test/services/network/upload.service.test.ts
diff --git a/yarn.lock b/yarn.lock

Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@ export interface NetworkCredentials {`
`6`	`6`	`export type DownloadProgressCallback = (downloadedBytes: number) => void;`
`7`	`7`	`export type UploadProgressCallback = (uploadedBytes: number) => void;`
`8`	`8`	`export interface NetworkOperationBaseOptions {`
`9`		`- progressCallback: UploadProgressCallback;`
	`9`	`+ progressCallback?: UploadProgressCallback;`
`10`	`10`	`abortController?: AbortController;`
`11`	`11`	`}`
`12`	`12`