Merge pull request #153 from internxt/feat/improve-progress-reporting

larryrider · web-flow · commit 470d6bd063cc · 2025-01-15T10:36:37.000+01:00
[_]: Feat/improve-progress-reporting
diff --git a/package.json b/package.json
@@ -54,7 +54,6 @@
     "express-basic-auth": "1.2.1",
     "fast-xml-parser": "4.5.1",
     "mime-types": "2.1.35",
-    "node-fetch": "2.7.0",
     "openpgp": "5.11.2",
     "pm2": "5.4.3",
     "range-parser": "^1.2.1",
@@ -75,8 +74,7 @@
     "@types/express": "5.0.0",
     "@types/mime-types": "2.1.4",
     "@types/node": "22.10.6",
-    "@types/node-fetch": "2.6.12",
-    "@types/range-parser": "^1.2.7",
+    "@types/range-parser": "1.2.7",
     "@vitest/coverage-istanbul": "2.1.8",
     "@vitest/spy": "2.1.8",
     "eslint": "9.17.0",
diff --git a/src/commands/download-file.ts b/src/commands/download-file.ts
@@ -79,12 +79,13 @@ export default class DownloadFile extends Command {
       user.bucket,
       user.mnemonic,
       driveFile.fileId,
+      driveFile.size,
       StreamUtils.writeStreamToWritableStream(fileWriteStream),
       undefined,
       {
         abortController: new AbortController(),
         progressCallback: (progress) => {
-          progressBar.update(progress);
+          progressBar.update(progress * 0.99);
         },
       },
     );
diff --git a/src/commands/upload-file.ts b/src/commands/upload-file.ts
@@ -74,22 +74,22 @@ export default class UploadFile extends Command {
 
     CLIUtils.done();
 
-    const timer = CLIUtils.timer();
     // 2. Upload file to the Network
     const fileStream = createReadStream(filePath);
+    const timer = CLIUtils.timer();
     const progressBar = CLIUtils.progress({
       format: 'Uploading file [{bar}] {percentage}%',
       linewrap: true,
     });
-    progressBar.start(1, 0);
+    progressBar.start(100, 0);
     const [uploadPromise, abortable] = await networkFacade.uploadFromStream(
       user.bucket,
       user.mnemonic,
       stats.size,
       fileStream,
       {
         progressCallback: (progress) => {
-          progressBar.update(progress);
+          progressBar.update(progress * 0.99);
         },
       },
     );
@@ -100,7 +100,6 @@ export default class UploadFile extends Command {
     });
 
     const uploadResult = await uploadPromise;
-    progressBar.stop();
 
     // 3. Create the file in Drive
     const fileInfo = path.parse(filePath);
@@ -115,6 +114,9 @@ export default class UploadFile extends Command {
       name: '',
     });
 
+    progressBar.update(100);
+    progressBar.stop();
+
     const uploadTime = timer.stop();
     this.log('\n');
     // eslint-disable-next-line max-len
diff --git a/src/services/crypto.service.ts b/src/services/crypto.service.ts
@@ -166,11 +166,11 @@ export class CryptoService {
     return decryptedStream;
   }
 
-  public async getEncryptionTransform(key: Buffer, iv: Buffer): Promise<Transform> {
+  public getEncryptionTransform = (key: Buffer, iv: Buffer): Transform => {
     const cipher = createCipheriv('aes-256-ctr', key, iv);
-
     return cipher;
-  }
+  };
+
   /**
    * Generates the key and the iv by transforming a secret and a salt.
    * It will generate the same key and iv if the same secret and salt is used.
diff --git a/src/services/network/download.service.ts b/src/services/network/download.service.ts
@@ -1,23 +1,22 @@
 import axios from 'axios';
+import { DownloadProgressCallback } from '../../types/network.types';
 
 export class DownloadService {
   static readonly instance = new DownloadService();
 
   async downloadFile(
     url: string,
     options: {
-      progressCallback?: (progress: number) => void;
+      progressCallback?: DownloadProgressCallback;
       abortController?: AbortController;
       rangeHeader?: string;
     },
   ): Promise<ReadableStream<Uint8Array>> {
     const response = await axios.get(url, {
       responseType: 'stream',
       onDownloadProgress(progressEvent) {
-        if (options.progressCallback && progressEvent.total) {
-          const reportedProgress = progressEvent.loaded / progressEvent.total;
-
-          options.progressCallback(reportedProgress);
+        if (options.progressCallback && progressEvent.loaded) {
+          options.progressCallback(progressEvent.loaded);
         }
       },
       headers: {
diff --git a/src/services/network/network-facade.service.ts b/src/services/network/network-facade.service.ts
@@ -10,13 +10,12 @@ import {
 import { Environment } from '@internxt/inxt-js';
 import { randomBytes } from 'node:crypto';
 import { Readable, Transform } from 'node:stream';
-import { DownloadOptions, UploadOptions, UploadProgressCallback } from '../../types/network.types';
+import { DownloadOptions, UploadOptions, UploadProgressCallback, DownloadProgressCallback } from '../../types/network.types';
 import { CryptoService } from '../crypto.service';
 import { UploadService } from './upload.service';
 import { DownloadService } from './download.service';
 import { ValidationService } from '../validation.service';
 import { HashStream } from '../../utils/hash.utils';
-import { ProgressTransform } from '../../utils/stream.utils';
 import { RangeOptions } from '../../utils/network.utils';
 
 export class NetworkFacade {
@@ -54,6 +53,7 @@ export class NetworkFacade {
     bucketId: string,
     mnemonic: string,
     fileId: string,
+    size: number,
     to: WritableStream,
     rangeOptions?: RangeOptions,
     options?: DownloadOptions,
@@ -62,13 +62,10 @@ export class NetworkFacade {
     let fileStream: ReadableStream<Uint8Array>;
     const abortable = options?.abortController ?? new AbortController();
 
-    const onProgress: UploadProgressCallback = (progress: number) => {
+    const onProgress: DownloadProgressCallback = (loadedBytes: number) => {
       if (!options?.progressCallback) return;
-      options.progressCallback(progress);
-    };
-
-    const onDownloadProgress = (progress: number) => {
-      onProgress(progress);
+      const reportedProgress = Math.round((loadedBytes / size) * 100);
+      options.progressCallback(reportedProgress);
     };
 
     const decryptFile: DecryptFileFunction = async (_, key, iv) => {
@@ -97,7 +94,7 @@ export class NetworkFacade {
         }
 
         const encryptedContentStream = await this.downloadService.downloadFile(downloadable.url, {
-          progressCallback: onDownloadProgress,
+          progressCallback: onProgress,
           abortController: options?.abortController,
           rangeHeader: rangeOptions?.range,
         });
@@ -142,38 +139,31 @@ export class NetworkFacade {
     const hashStream = new HashStream();
     const abortable = options?.abortController ?? new AbortController();
     let encryptionTransform: Transform;
-    const progressTransform = new ProgressTransform({ totalBytes: size }, (progress) => {
-      if (options?.progressCallback) {
-        options.progressCallback(progress * 0.95);
-      }
-    });
+    let hash: Buffer;
 
-    const onProgress: UploadProgressCallback = (progress: number) => {
+    const onProgress: UploadProgressCallback = (loadedBytes: number) => {
       if (!options?.progressCallback) return;
-      options.progressCallback(progress);
+      const reportedProgress = Math.round((loadedBytes / size) * 100);
+      options.progressCallback(reportedProgress);
     };
 
     const encryptFile: EncryptFileFunction = async (_, key, iv) => {
-      encryptionTransform = from
-        .pipe(
-          await this.cryptoService.getEncryptionTransform(
-            Buffer.from(key as ArrayBuffer),
-            Buffer.from(iv as ArrayBuffer),
-          ),
-        )
-        .pipe(hashStream);
+      const encryptionCipher = this.cryptoService.getEncryptionTransform(
+        Buffer.from(key as ArrayBuffer),
+        Buffer.from(iv as ArrayBuffer),
+      );
+      encryptionTransform = from.pipe(encryptionCipher).pipe(hashStream);
     };
 
     const uploadFile: UploadFileFunction = async (url) => {
-      await this.uploadService.uploadFile(url, encryptionTransform.pipe(progressTransform), {
+      await this.uploadService.uploadFile(url, encryptionTransform, {
         abortController: abortable,
-        progressCallback: () => {
-          // No progress here, we are using the progressTransform
-        },
+        progressCallback: onProgress,
       });
-
-      return hashStream.getHash().toString('hex');
+      hash = hashStream.getHash();
+      return hash.toString('hex');
     };
+
     const uploadOperation = async () => {
       const uploadResult = await NetworkUpload.uploadFile(
         this.network,
@@ -184,12 +174,10 @@ export class NetworkFacade {
         encryptFile,
         uploadFile,
       );
-      const fileHash: Buffer = Buffer.from('');
 
-      onProgress(1);
       return {
         fileId: uploadResult,
-        hash: fileHash,
+        hash: hash,
       };
     };
 
diff --git a/src/services/network/upload.service.ts b/src/services/network/upload.service.ts
@@ -1,20 +1,21 @@
 import { Readable } from 'node:stream';
-import fetch from 'node-fetch';
-import { AbortSignal } from 'node-fetch/externals';
+import axios from 'axios';
 import { UploadOptions } from '../../types/network.types';
 
 export class UploadService {
   public static readonly instance: UploadService = new UploadService();
 
   async uploadFile(url: string, from: Readable, options: UploadOptions): Promise<{ etag: string }> {
-    const response = await fetch(url, {
-      method: 'PUT',
-      body: from,
-      signal: options.abortController?.signal as AbortSignal,
+    const response = await axios.put(url, from, {
+      signal: options.abortController?.signal,
+      onUploadProgress: (progressEvent) => {
+        if (options.progressCallback && progressEvent.loaded) {
+          options.progressCallback(progressEvent.loaded);
+        }
+      },
     });
 
-    const etag = response.headers.get('etag');
-    options.progressCallback(1);
+    const etag = response.headers['etag'];
     if (!etag) {
       throw new Error('Missing Etag in response when uploading file');
     }
diff --git a/src/types/network.types.ts b/src/types/network.types.ts
@@ -3,7 +3,8 @@ export interface NetworkCredentials {
   pass: string;
 }
 
-export type UploadProgressCallback = (progress: number) => void;
+export type DownloadProgressCallback = (downloadedBytes: number) => void;
+export type UploadProgressCallback = (uploadedBytes: number) => void;
 export interface NetworkOperationBaseOptions {
   progressCallback: UploadProgressCallback;
   abortController?: AbortController;
diff --git a/src/webdav/handlers/GET.handler.ts b/src/webdav/handlers/GET.handler.ts
@@ -43,15 +43,6 @@ export class GETRequestHandler implements WebDavMethodHandler {
     const { user } = await authService.getAuthDetails();
     webdavLogger.info(`[GET] [${driveFile.uuid}] Network ready for download`);
 
-    const writable = new WritableStream({
-      write(chunk) {
-        res.write(chunk);
-      },
-      close() {
-        res.end();
-      },
-    });
-
     const range = req.headers['range'];
     const rangeOptions = NetworkUtils.parseRangeHeader({
       range,
@@ -66,10 +57,20 @@ export class GETRequestHandler implements WebDavMethodHandler {
     res.header('Content-Type', 'application/octet-stream');
     res.header('Content-length', contentLength.toString());
 
+    const writable = new WritableStream({
+      write(chunk) {
+        res.write(chunk);
+      },
+      close() {
+        res.end();
+      },
+    });
+
     const [executeDownload] = await networkFacade.downloadToStream(
       driveFile.bucket,
       user.mnemonic,
       driveFile.fileId,
+      contentLength,
       writable,
       rangeOptions,
     );
diff --git a/src/webdav/handlers/MKCOL.handler.ts b/src/webdav/handlers/MKCOL.handler.ts
@@ -7,7 +7,6 @@ import { webdavLogger } from '../../utils/logger.utils';
 import { XMLUtils } from '../../utils/xml.utils';
 import { AsyncUtils } from '../../utils/async.utils';
 import { DriveFolderItem } from '../../types/drive.types';
-import { MethodNotAllowed } from '../../utils/errors.utils';
 
 export class MKCOLRequestHandler implements WebDavMethodHandler {
   constructor(
@@ -21,8 +20,6 @@ export class MKCOLRequestHandler implements WebDavMethodHandler {
     const { driveDatabaseManager, driveFolderService } = this.dependencies;
     const resource = await WebDavUtils.getRequestedResource(req);
 
-    if (resource.type === 'file') throw new MethodNotAllowed('Files cannot be created with MKCOL. Use PUT instead.');
-
     webdavLogger.info(`[MKCOL] Request received for ${resource.type} at ${resource.url}`);
 
     const parentResource = await WebDavUtils.getRequestedResource(resource.parentPath);
@@ -34,7 +31,7 @@ export class MKCOLRequestHandler implements WebDavMethodHandler {
     })) as DriveFolderItem;
 
     const [createFolder] = driveFolderService.createFolder({
-      plainName: resource.name,
+      plainName: resource.path.base,
       parentFolderUuid: parentFolderItem.uuid,
     });
 
diff --git a/src/webdav/webdav-server.ts b/src/webdav/webdav-server.ts
@@ -171,7 +171,8 @@ export class WebDavServer {
 
     server.listen(configs.port, () => {
       webdavLogger.info(
-        `Internxt WebDav server listening at ${configs.protocol}://${ConfigService.WEBDAV_LOCAL_URL}:${configs.port}`,
+        `Internxt ${SdkManager.getAppDetails().clientVersion} WebDav server ` +
+          `listening at ${configs.protocol}://${ConfigService.WEBDAV_LOCAL_URL}:${configs.port}`,
       );
     });
   };
diff --git a/test/services/network/download.service.test.ts b/test/services/network/download.service.test.ts
@@ -48,6 +48,6 @@ describe('Download Service', () => {
 
     await sut.downloadFile('https://example.com/file', options);
 
-    expect(options.progressCallback).toHaveBeenCalledWith(1);
+    expect(options.progressCallback).toHaveBeenCalledWith(100);
   });
 });
diff --git a/test/services/network/network-facade.service.test.ts b/test/services/network/network-facade.service.test.ts
diff --git a/test/services/network/upload.service.test.ts b/test/services/network/upload.service.test.ts
diff --git a/test/webdav/handlers/GET.handler.test.ts b/test/webdav/handlers/GET.handler.test.ts
diff --git a/yarn.lock b/yarn.lock

Original file line number	Diff line number	Diff line change
`@@ -3,7 +3,8 @@ export interface NetworkCredentials {`
`3`	`3`	`pass: string;`
`4`	`4`	`}`
`5`	`5`
`6`		`-export type UploadProgressCallback = (progress: number) => void;`
	`6`	`+export type DownloadProgressCallback = (downloadedBytes: number) => void;`
	`7`	`+export type UploadProgressCallback = (uploadedBytes: number) => void;`
`7`	`8`	`export interface NetworkOperationBaseOptions {`
`8`	`9`	`progressCallback: UploadProgressCallback;`
`9`	`10`	`abortController?: AbortController;`