improved progress reporting and migrated upload from node-fetch to axios

Author: larryrider

package.json

@@ -54,7 +54,6 @@
     "express-basic-auth": "1.2.1",
     "fast-xml-parser": "4.5.1",
     "mime-types": "2.1.35",
-    "node-fetch": "2.7.0",
     "openpgp": "5.11.2",
     "pm2": "5.4.3",
     "range-parser": "^1.2.1",
@@ -75,8 +74,7 @@
     "@types/express": "5.0.0",
     "@types/mime-types": "2.1.4",
     "@types/node": "22.10.2",
-    "@types/node-fetch": "2.6.12",
-    "@types/range-parser": "^1.2.7",
+    "@types/range-parser": "1.2.7",
     "@vitest/coverage-istanbul": "2.1.8",
     "@vitest/spy": "2.1.8",
     "eslint": "9.17.0",

src/commands/download-file.ts

@@ -79,12 +79,13 @@ export default class DownloadFile extends Command {
       user.bucket,
       user.mnemonic,
       driveFile.fileId,
+      driveFile.size,
       StreamUtils.writeStreamToWritableStream(fileWriteStream),
       undefined,
       {
         abortController: new AbortController(),
         progressCallback: (progress) => {
-          progressBar.update(progress);
+          progressBar.update(progress * 0.99);
         },
       },
     );

src/commands/upload-file.ts

@@ -74,22 +74,22 @@ export default class UploadFile extends Command {
 
     CLIUtils.done();
 
-    const timer = CLIUtils.timer();
     // 2. Upload file to the Network
     const fileStream = createReadStream(filePath);
+    const timer = CLIUtils.timer();
     const progressBar = CLIUtils.progress({
       format: 'Uploading file [{bar}] {percentage}%',
       linewrap: true,
     });
-    progressBar.start(1, 0);
+    progressBar.start(100, 0);
     const [uploadPromise, abortable] = await networkFacade.uploadFromStream(
       user.bucket,
       user.mnemonic,
       stats.size,
       fileStream,
       {
         progressCallback: (progress) => {
-          progressBar.update(progress);
+          progressBar.update(progress * 0.99);
         },
       },
     );
@@ -100,7 +100,6 @@ export default class UploadFile extends Command {
     });
 
     const uploadResult = await uploadPromise;
-    progressBar.stop();
 
     // 3. Create the file in Drive
     const fileInfo = path.parse(filePath);
@@ -115,6 +114,9 @@ export default class UploadFile extends Command {
       name: '',
     });
 
+    progressBar.update(100);
+    progressBar.stop();
+
     const uploadTime = timer.stop();
     this.log('\n');
     // eslint-disable-next-line max-len

src/services/network/download.service.ts

@@ -5,6 +5,7 @@ export class DownloadService {
 
   async downloadFile(
     url: string,
+    size: number,
     options: {
       progressCallback?: (progress: number) => void;
       abortController?: AbortController;
@@ -14,9 +15,8 @@ export class DownloadService {
     const response = await axios.get(url, {
       responseType: 'stream',
       onDownloadProgress(progressEvent) {
-        if (options.progressCallback && progressEvent.total) {
-          const reportedProgress = progressEvent.loaded / progressEvent.total;
-
+        if (options.progressCallback && progressEvent.loaded) {
+          const reportedProgress = Math.round((progressEvent.loaded / size) * 100);
           options.progressCallback(reportedProgress);
         }
       },

src/services/network/network-facade.service.ts

@@ -16,7 +16,6 @@ import { UploadService } from './upload.service';
 import { DownloadService } from './download.service';
 import { ValidationService } from '../validation.service';
 import { HashStream } from '../../utils/hash.utils';
-import { ProgressTransform } from '../../utils/stream.utils';
 import { RangeOptions } from '../../utils/network.utils';
 
 export class NetworkFacade {
@@ -54,6 +53,7 @@ export class NetworkFacade {
     bucketId: string,
     mnemonic: string,
     fileId: string,
+    size: number,
     to: WritableStream,
     rangeOptions?: RangeOptions,
     options?: DownloadOptions,
@@ -67,10 +67,6 @@ export class NetworkFacade {
       options.progressCallback(progress);
     };
 
-    const onDownloadProgress = (progress: number) => {
-      onProgress(progress);
-    };
-
     const decryptFile: DecryptFileFunction = async (_, key, iv) => {
       let startOffsetByte;
       if (rangeOptions) {
@@ -96,8 +92,8 @@ export class NetworkFacade {
           throw new Error('Download aborted');
         }
 
-        const encryptedContentStream = await this.downloadService.downloadFile(downloadable.url, {
-          progressCallback: onDownloadProgress,
+        const encryptedContentStream = await this.downloadService.downloadFile(downloadable.url, size, {
+          progressCallback: onProgress,
           abortController: options?.abortController,
           rangeHeader: rangeOptions?.range,
         });
@@ -142,11 +138,7 @@ export class NetworkFacade {
     const hashStream = new HashStream();
     const abortable = options?.abortController ?? new AbortController();
     let encryptionTransform: Transform;
-    const progressTransform = new ProgressTransform({ totalBytes: size }, (progress) => {
-      if (options?.progressCallback) {
-        options.progressCallback(progress * 0.95);
-      }
-    });
+    let hash: Buffer;
 
     const onProgress: UploadProgressCallback = (progress: number) => {
       if (!options?.progressCallback) return;
@@ -165,15 +157,14 @@ export class NetworkFacade {
     };
 
     const uploadFile: UploadFileFunction = async (url) => {
-      await this.uploadService.uploadFile(url, encryptionTransform.pipe(progressTransform), {
+      await this.uploadService.uploadFile(url, size, encryptionTransform, {
         abortController: abortable,
-        progressCallback: () => {
-          // No progress here, we are using the progressTransform
-        },
+        progressCallback: onProgress,
       });
-
-      return hashStream.getHash().toString('hex');
+      hash = hashStream.getHash();
+      return hash.toString('hex');
     };
+
     const uploadOperation = async () => {
       const uploadResult = await NetworkUpload.uploadFile(
         this.network,
@@ -184,12 +175,10 @@ export class NetworkFacade {
         encryptFile,
         uploadFile,
       );
-      const fileHash: Buffer = Buffer.from('');
 
-      onProgress(1);
       return {
         fileId: uploadResult,
-        hash: fileHash,
+        hash: hash,
       };
     };
 

src/services/network/upload.service.ts

@@ -1,20 +1,22 @@
 import { Readable } from 'node:stream';
-import fetch from 'node-fetch';
-import { AbortSignal } from 'node-fetch/externals';
+import axios from 'axios';
 import { UploadOptions } from '../../types/network.types';
 
 export class UploadService {
   public static readonly instance: UploadService = new UploadService();
 
-  async uploadFile(url: string, from: Readable, options: UploadOptions): Promise<{ etag: string }> {
-    const response = await fetch(url, {
-      method: 'PUT',
-      body: from,
-      signal: options.abortController?.signal as AbortSignal,
+  async uploadFile(url: string, size: number, from: Readable, options: UploadOptions): Promise<{ etag: string }> {
+    const response = await axios.put(url, from, {
+      signal: options.abortController?.signal,
+      onUploadProgress: (progressEvent) => {
+        if (options.progressCallback && progressEvent.loaded) {
+          const reportedProgress = Math.round((progressEvent.loaded / size) * 100);
+          options.progressCallback(reportedProgress);
+        }
+      },
     });
 
-    const etag = response.headers.get('etag');
-    options.progressCallback(1);
+    const etag = response.headers['etag'];
     if (!etag) {
       throw new Error('Missing Etag in response when uploading file');
     }

src/webdav/handlers/GET.handler.ts

@@ -43,15 +43,6 @@ export class GETRequestHandler implements WebDavMethodHandler {
     const { user } = await authService.getAuthDetails();
     webdavLogger.info(`[GET] [${driveFile.uuid}] Network ready for download`);
 
-    const writable = new WritableStream({
-      write(chunk) {
-        res.write(chunk);
-      },
-      close() {
-        res.end();
-      },
-    });
-
     const range = req.headers['range'];
     const rangeOptions = NetworkUtils.parseRangeHeader({
       range,
@@ -66,10 +57,20 @@ export class GETRequestHandler implements WebDavMethodHandler {
     res.header('Content-Type', 'application/octet-stream');
     res.header('Content-length', contentLength.toString());
 
+    const writable = new WritableStream({
+      write(chunk) {
+        res.write(chunk);
+      },
+      close() {
+        res.end();
+      },
+    });
+
     const [executeDownload] = await networkFacade.downloadToStream(
       driveFile.bucket,
       user.mnemonic,
       driveFile.fileId,
+      contentLength,
       writable,
       rangeOptions,
     );

yarn.lock

@@ -2937,14 +2937,6 @@
   dependencies:
     "@types/node" "*"
 
-"@types/node-fetch@2.6.12":
-  version "2.6.12"
-  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.12.tgz#8ab5c3ef8330f13100a7479e2cd56d3386830a03"
-  integrity sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==
-  dependencies:
-    "@types/node" "*"
-    form-data "^4.0.0"
-
 "@types/node-forge@^1.3.0":
   version "1.3.11"
   resolved "https://registry.yarnpkg.com/@types/node-forge/-/node-forge-1.3.11.tgz#0972ea538ddb0f4d9c2fa0ec5db5724773a604da"
@@ -2971,7 +2963,7 @@
   resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.17.tgz#fc560f60946d0aeff2f914eb41679659d3310e1a"
   integrity sha512-rX4/bPcfmvxHDv0XjfJELTTr+iB+tn032nPILqHm5wbthUUUuVtNGGqzhya9XUxjTP8Fpr0qYgSZZKxGY++svQ==
 
-"@types/range-parser@*", "@types/range-parser@^1.2.7":
+"@types/range-parser@*", "@types/range-parser@1.2.7":
   version "1.2.7"
   resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.7.tgz#50ae4353eaaddc04044279812f52c8c65857dbcb"
   integrity sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==
@@ -6540,13 +6532,6 @@ node-addon-api@^7.0.0:
   resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-7.1.1.tgz#1aba6693b0f255258a049d621329329322aad558"
   integrity sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==
 
-node-fetch@2.7.0:
-  version "2.7.0"
-  resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d"
-  integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==
-  dependencies:
-    whatwg-url "^5.0.0"
-
 node-forge@^1:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-1.3.1.tgz#be8da2af243b2417d5f646a770663a92b7e9ded3"
@@ -8143,11 +8128,6 @@ touch@^3.1.0:
   resolved "https://registry.yarnpkg.com/touch/-/touch-3.1.1.tgz#097a23d7b161476435e5c1344a95c0f75b4a5694"
   integrity sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==
 
-tr46@~0.0.3:
-  version "0.0.3"
-  resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
-  integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==
-
 triple-beam@^1.3.0:
   version "1.4.1"
   resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.4.1.tgz#6fde70271dc6e5d73ca0c3b24e2d92afb7441984"
@@ -8530,19 +8510,6 @@ web-streams-polyfill@~3.0.3:
   resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.0.3.tgz#f49e487eedeca47a207c1aee41ee5578f884b42f"
   integrity sha512-d2H/t0eqRNM4w2WvmTdoeIvzAUSpK7JmATB8Nr2lb7nQ9BTIJVjbQ/TRFVEh2gUH1HwclPdoPtfMoFfetXaZnA==
 
-webidl-conversions@^3.0.0:
-  version "3.0.1"
-  resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"
-  integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==
-
-whatwg-url@^5.0.0:
-  version "5.0.0"
-  resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"
-  integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==
-  dependencies:
-    tr46 "~0.0.3"
-    webidl-conversions "^3.0.0"
-
 which-boxed-primitive@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz#13757bc89b209b049fe5d86430e21cf40a89a8e6"