feat(graasp-import): export and import the item thumbnails (#1868)

* feat(graasp-import): export and import the item thumbnails

* fix(graasp-import): pull request review fixes

* fix(graasp-import): correct the test fixture

* fix(graasp-import): correct test

* refactor(graasp-import): refactor the thumbnail function

* fix(graasp-import): correct the graasp import test

* fix(s3-file): check the fetch response before starting the file stream

Author: MatveyK

src/services/file/file.service.test.ts

@@ -145,7 +145,7 @@ describe('FileService', () => {
       const downloadMock = jest
         .spyOn(s3Repository, 'getFile')
         .mockImplementation(async () => returnValue);
-      expect(await s3FileService.getFile(member, downloadPayload)).toBeTruthy();
+      expect(await s3FileService.getFile(downloadPayload)).toBeTruthy();
       expect(downloadMock).toHaveBeenCalled();
     });
 
@@ -154,7 +154,7 @@ describe('FileService', () => {
       const downloadMock = jest
         .spyOn(s3Repository, 'getFile')
         .mockImplementation(async () => returnValue);
-      expect(await s3FileService.getFile(undefined, downloadPayload)).toBeTruthy();
+      expect(await s3FileService.getFile(downloadPayload)).toBeTruthy();
       expect(downloadMock).toHaveBeenCalled();
     });
   });

src/services/file/file.service.ts

@@ -104,15 +104,10 @@ class FileService {
     return file;
   }
 
-  async getFile(_actor: MaybeUser, data: { id?: string; path?: string }): Promise<Readable> {
-    const { id, path: filepath } = data;
-    if (!filepath || !id) {
-      throw new DownloadFileInvalidParameterError();
-    }
-
+  async getFile({ id, path }: { id: string; path: string }): Promise<Readable> {
     return this.repository.getFile(
       {
-        filepath,
+        filepath: path,
         id,
       },
       this.logger,

src/services/file/repositories/s3.ts

@@ -188,6 +188,11 @@ export class S3FileRepository implements FileRepository {
       // return readstream of the file saved at given filepath
       // fetch and save file in temporary path
       const res = await fetch(url);
+
+      if (!res.ok) {
+        throw new S3FileNotFound();
+      }
+
       const fileStream = fs.createWriteStream(filepath);
       await new Promise<void>((resolve, reject) => {
         res.body.pipe(fileStream);

src/services/item/plugins/file/itemFile.service.ts

@@ -208,7 +208,7 @@ class FileItemService extends ItemService {
       item,
     );
     const extraData = item.extra[ItemType.FILE] as FileItemProperties;
-    const result = await this.fileService.getFile(actor, {
+    const result = await this.fileService.getFile({
       id: itemId,
       ...extraData,
     });

src/services/item/plugins/importExport/constants.ts

@@ -10,6 +10,8 @@ export const ROOT_PATH = './';
 
 export const GRAASP_MANIFEST_FILENAME = 'graasp-manifest.json';
 
+export const GRAASP_ARCHIVE_THUMBNAIL_SUFFIX = '-thumbnail';
+
 export const DESCRIPTION_EXTENSION = '.description.html';
 
 export const GRAASP_DOCUMENT_EXTENSION = '.graasp';

src/services/item/plugins/importExport/service.ts

@@ -1,4 +1,5 @@
 import fs, { existsSync } from 'fs';
+import { createReadStream, exists } from 'fs-extra';
 import { readFile } from 'fs/promises';
 import mimetics from 'mimetics';
 import fetch from 'node-fetch';
@@ -14,21 +15,22 @@ import {
   ItemSettings,
   ItemType,
   ItemTypeUnion,
+  ThumbnailSize,
   getMimetype,
 } from '@graasp/sdk';
 
 import { type DBConnection } from '../../../../drizzle/db';
 import { type ItemRaw } from '../../../../drizzle/types';
 import { BaseLogger } from '../../../../logger';
 import { MaybeUser, MinimalMember } from '../../../../types';
-import FileService from '../../../file/file.service';
 import { UploadEmptyFileError } from '../../../file/utils/errors';
 import { BasicItemService } from '../../basic.service';
 import { isItemType } from '../../discrimination';
 import { ItemService } from '../../item.service';
 import { EtherpadItemService } from '../etherpad/etherpad.service';
 import FileItemService from '../file/itemFile.service';
 import { H5PService } from '../html/h5p/h5p.service';
+import { ItemThumbnailService } from '../thumbnail/itemThumbnail.service';
 import {
   DESCRIPTION_EXTENSION,
   GRAASP_DOCUMENT_EXTENSION,
@@ -39,7 +41,7 @@ import {
   URL_PREFIX,
 } from './constants';
 import { GraaspExportInvalidFileError, UnexpectedExportError } from './errors';
-import { buildTextContent, getFilenameFromItem } from './utils';
+import { buildTextContent, generateThumbnailFilename, getFilenameFromItem } from './utils';
 
 /**
  * Defines the properties of an individual item in the graasp export format.
@@ -60,28 +62,28 @@ export type GraaspExportItem = {
 
 @singleton()
 export class ImportExportService {
-  private readonly fileService: FileService;
   private readonly fileItemService: FileItemService;
   private readonly h5pService: H5PService;
   private readonly itemService: ItemService;
   private readonly basicItemService: BasicItemService;
   private readonly etherpadService: EtherpadItemService;
+  private readonly itemThumbnailService: ItemThumbnailService;
   private readonly log: BaseLogger;
 
   constructor(
-    fileService: FileService,
     fileItemService: FileItemService,
     itemService: ItemService,
     h5pService: H5PService,
     etherpadService: EtherpadItemService,
+    itemThumbnailService: ItemThumbnailService,
     basicItemService: BasicItemService,
     log: BaseLogger,
   ) {
-    this.fileService = fileService;
     this.fileItemService = fileItemService;
     this.h5pService = h5pService;
     this.itemService = itemService;
     this.etherpadService = etherpadService;
+    this.itemThumbnailService = itemThumbnailService;
     this.basicItemService = basicItemService;
     this.log = log;
   }
@@ -292,6 +294,13 @@ export class ImportExportService {
           extra = { [ItemType.DOCUMENT]: { content: sanitizedContent } };
         }
 
+        // Find and upload the thumbnail
+        let thumbnail: Readable | undefined = undefined;
+        const itemThumbnailPath = path.join(folderPath, generateThumbnailFilename(item.id));
+        if (await exists(itemThumbnailPath)) {
+          thumbnail = createReadStream(itemThumbnailPath);
+        }
+
         // Handle the file upload
         if (item.type === ItemType.FILE) {
           if (!item.mimetype) {
@@ -312,7 +321,7 @@ export class ImportExportService {
 
         const augmentedItem = { ...item, description: sanitizedDescription, extra };
 
-        return { item: augmentedItem, thumbnail: undefined };
+        return { item: augmentedItem, thumbnail };
       }),
     );
 
@@ -411,6 +420,15 @@ export class ImportExportService {
     const exportItemId = v4();
     const itemPath = path.join(path.dirname('./'), exportItemId);
 
+    // add the thumbnail to export, if present
+    const thumbnailFilename = await this.getAndWriteThumbnail(
+      dbConnection,
+      actor,
+      item.id,
+      exportItemId,
+      archive,
+    );
+
     // TODO EXPORT treat the shortcut items correctly
     // ignore the shortcuts for now
     if (isItemType(item, ItemType.SHORTCUT)) {
@@ -436,6 +454,7 @@ export class ImportExportService {
         type: item.type,
         settings: item.settings,
         extra: item.extra,
+        thumbnailFilename,
         children: childrenManifest,
       });
       return itemManifest;
@@ -451,12 +470,39 @@ export class ImportExportService {
       type: item.type,
       settings: item.settings,
       extra: item.extra,
+      thumbnailFilename,
       mimetype,
     });
     archive.addReadStream(stream, itemPath);
     return itemManifest;
   }
 
+  /**
+   * Try and get the item thumbnail and write it to the zip archive.
+   * @returns Thumbnail filename, undefined if the thumbnail was not found.
+   */
+  private async getAndWriteThumbnail(
+    dbConnection: DBConnection,
+    actor: MaybeUser,
+    itemId: string,
+    exportItemId: string,
+    archive: ZipFile,
+  ) {
+    const filename = generateThumbnailFilename(exportItemId);
+    const itemThumbnailPath = path.join(path.dirname('./'), filename);
+    try {
+      const thumbnailStream = await this.itemThumbnailService.getFile(dbConnection, actor, {
+        size: ThumbnailSize.Original,
+        itemId,
+      });
+
+      archive.addReadStream(thumbnailStream, itemThumbnailPath);
+      return filename;
+    } catch (_err) {
+      this.log.debug(`Thumbnail not found for item ${itemId}`);
+    }
+  }
+
   /**
    * Export the items recursively
    * @param item The root item

src/services/item/plugins/importExport/specifications.md

@@ -13,11 +13,15 @@ The export operation produces a single zip file. This file is a simple zip file
 - description (item description in HTML)
 - settings (item settings)
 - extra (item extras)
-- thumbnailFilename (item thumbnail in the original size, if present) - COMING SOON
+- thumbnailFilename (item thumbnail in the original size, if present)
 - children (item children, in case of a folder item)
 - mimetype (item file mimetype, in case there's a file attached to the item)
 ```
 
+### Files and thumbnails
+
+Files are stored in the top level of the zip. The filename is the same as the item ID in the manifest file. The thumbnails are also stored on the top-level and are named using the `{ID}-thumbnail` convention.
+
 ## Import
 
 Upon the import, the uploaded ZIP file is scanned for the presence of a `graasp-manifest.json` file. If the file is present, it is scanned and then the items and their children are recursively imported, respecting the item order in the manifest file.

src/services/item/plugins/importExport/test/fixtures/graasp-archive.zip

@@ -9,7 +9,7 @@ import waitForExpect from 'wait-for-expect';
 
 import { FastifyInstance } from 'fastify';
 
-import { FileItemProperties, HttpMethod, ItemType, MimeTypes } from '@graasp/sdk';
+import { FileItemProperties, HttpMethod, ItemType, MimeTypes, ThumbnailSize } from '@graasp/sdk';
 
 import build, {
   clearDatabase,
@@ -21,6 +21,7 @@ import { db } from '../../../../../drizzle/db';
 import { isDescendantOrSelf, isDirectChild } from '../../../../../drizzle/operations';
 import { itemsRawTable } from '../../../../../drizzle/schema';
 import { assertIsDefined } from '../../../../../utils/assertions';
+import { ITEMS_ROUTE_PREFIX, THUMBNAILS_ROUTE_PREFIX } from '../../../../../utils/config';
 import { LocalFileRepository } from '../../../../file/repositories/local';
 import { GRAASP_MANIFEST_FILENAME } from '../constants';
 import { GraaspExportItem } from '../service';
@@ -451,7 +452,7 @@ describe('ZIP routes tests', () => {
             isDescendantOrSelf(itemsRawTable.path, parentItem.path),
             ne(itemsRawTable.id, parentItem.id),
           ),
-          orderBy: asc(itemsRawTable.order),
+          orderBy: [asc(itemsRawTable.order), asc(itemsRawTable.path)],
         });
         const folderItem = itemsInDB[0];
         const documentItem = itemsInDB[1];
@@ -472,17 +473,22 @@ describe('ZIP routes tests', () => {
         expect(documentItem.extra[ItemType.DOCUMENT].content).toEqual(documentContent);
 
         // Check that all the item properties have been assigned for the file type
-        let fileItemProperties: FileItemProperties;
-        if (fileItem.extra[ItemType.FILE]) {
-          fileItemProperties = fileItem.extra[ItemType.FILE] as FileItemProperties;
-        } else {
-          fileItemProperties = fileItem.extra[ItemType.LOCAL_FILE] as FileItemProperties;
-        }
+        const fileItemProperties = fileItem.extra[ItemType.FILE] as FileItemProperties;
         expect(fileItemProperties).toBeDefined();
         expect(fileItemProperties.name).toEqual(pdfName);
         expect(fileItemProperties.path).toBeDefined();
         expect(fileItemProperties.mimetype).toEqual(MimeTypes.PDF);
         expect(fileItemProperties.content).toEqual(pdfContent);
+
+        // Check the the file item thumbnail has been imported correctly
+        expect(fileItem.settings.hasThumbnail).toBeTruthy();
+
+        const thumbnailURLResponse = await app.inject({
+          method: HttpMethod.Get,
+          url: `${ITEMS_ROUTE_PREFIX}/${fileItem.id}${THUMBNAILS_ROUTE_PREFIX}/${ThumbnailSize.Original}`,
+        });
+
+        expect(thumbnailURLResponse.statusCode).toBe(StatusCodes.OK);
       });
     });
   });

src/services/item/plugins/importExport/test/index.test.ts

@@ -99,3 +99,7 @@ export const getFilenameFromItem = (item: ItemRaw): string => {
       return item.name;
   }
 };
+
+export const generateThumbnailFilename = (id: string): string => {
+  return `${id}-thumbnail`;
+};