feat(graasp-export): export in graasp format (#1796)

* feat(graasp-export): export in graasp format

* feat(graasp-export): add comments

Author: MatveyK

src/services/item/plugins/importExport/constants.ts

@@ -8,6 +8,8 @@ export const TMP_IMPORT_ZIP_FOLDER_PATH = path.join(TMP_FOLDER, 'zip-import');
 
 export const ROOT_PATH = './';
 
+export const GRAASP_MANIFEST_FILENAME = 'graasp-manifest.json';
+
 export const DESCRIPTION_EXTENSION = '.description.html';
 
 export const GRAASP_DOCUMENT_EXTENSION = '.graasp';

src/services/item/plugins/importExport/index.ts

@@ -87,7 +87,7 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => {
     },
   );
 
-  // download item
+  // export item as a zip containing raw files
   fastify.get(
     '/:itemId/export',
     {
@@ -132,15 +132,41 @@ const plugin: FastifyPluginAsyncTypebox = async (fastify) => {
       }
 
       // generate archive stream
-      const archiveStream = await importExportService.export(
-        member,
-        repositories,
-        {
-          item,
-          reply,
-        },
-        log,
-      );
+      const archiveStream = await importExportService.exportRaw(member, repositories, item);
+
+      try {
+        reply.raw.setHeader('Content-Disposition', `filename="${encodeFilename(item.name)}.zip"`);
+      } catch (e) {
+        // TODO: send sentry error
+        log?.error(e);
+        reply.raw.setHeader('Content-Disposition', 'filename="download.zip"');
+      }
+      reply.type('application/octet-stream');
+      return archiveStream.outputStream;
+    },
+  );
+
+  // export item in graasp format
+  fastify.get(
+    '/:itemId/graasp-export',
+    {
+      schema: zipExport,
+      preHandler: optionalIsAuthenticated,
+    },
+    async (request, reply) => {
+      const {
+        user,
+        params: { itemId },
+      } = request;
+      const member = user?.account;
+      const repositories = buildRepositories();
+      const item = await itemService.get(member, repositories, itemId);
+
+      // allow browser to access content disposition
+      reply.header('Access-Control-Expose-Headers', 'Content-Disposition');
+
+      // generate archive stream
+      const archiveStream = await importExportService.exportGraasp(member, repositories, item);
 
       try {
         reply.raw.setHeader('Content-Disposition', `filename="${encodeFilename(item.name)}.zip"`);

src/services/item/plugins/importExport/service.test.ts

@@ -1,12 +1,11 @@
 import * as nodeFetch from 'node-fetch';
 import { ZipFile } from 'yazl';
 
-import { FastifyInstance, FastifyReply } from 'fastify';
+import { FastifyInstance } from 'fastify';
 
 import { ItemType, ItemVisibilityType } from '@graasp/sdk';
 
 import build, {
-  MOCK_LOGGER,
   clearDatabase,
   mockAuthenticate,
   unmockAuthenticate,
@@ -70,8 +69,7 @@ describe('ZIP routes tests', () => {
         resolveDependency(BaseLogger),
       );
       const repositories = buildRepositories();
-      const reply = {} as unknown as FastifyReply;
-      await importExportService.export(actor, repositories, { item, reply }, MOCK_LOGGER);
+      await importExportService.exportRaw(actor, repositories, item);
 
       // called for parent and one child
       expect(mock).toHaveBeenCalledTimes(2);

src/services/item/plugins/importExport/service.ts

@@ -7,11 +7,10 @@ import sanitize from 'sanitize-html';
 import { Readable } from 'stream';
 import { DataSource } from 'typeorm';
 import util from 'util';
+import { v4 } from 'uuid';
 import { ZipFile } from 'yazl';
 
-import { FastifyReply } from 'fastify';
-
-import { ItemType, getMimetype } from '@graasp/sdk';
+import { ItemSettings, ItemType, ItemTypeUnion, getMimetype } from '@graasp/sdk';
 
 import { BaseLogger } from '../../../../logger';
 import { Repositories, buildRepositories } from '../../../../utils/repositories';
@@ -25,6 +24,7 @@ import { H5PService } from '../html/h5p/service';
 import {
   DESCRIPTION_EXTENSION,
   GRAASP_DOCUMENT_EXTENSION,
+  GRAASP_MANIFEST_FILENAME,
   HTML_EXTENSION,
   LINK_EXTENSION,
   TXT_EXTENSION,
@@ -33,6 +33,22 @@ import {
 import { UnexpectedExportError } from './errors';
 import { buildTextContent, getFilenameFromItem } from './utils';
 
+/**
+ * Defines the properties of an individual item in the graasp export format.
+ * @property children Children items, if the item if of type FOLDER.
+ * @property mimetype Mimetype of the item. Present if the item is not of type FOLDER.
+ */
+export type GraaspExportItem = {
+  id: string;
+  name: string;
+  type: ItemTypeUnion;
+  description: string | null;
+  settings: ItemSettings;
+  thumbnailFilename?: string;
+  children?: GraaspExportItem[];
+  mimetype?: string;
+};
+
 const magic = new Magic(MAGIC_MIME_TYPE);
 const asyncDetectFile = util.promisify(magic.detectFile.bind(magic));
 
@@ -281,14 +297,12 @@ export class ImportExportService {
     actor: Actor,
     repositories: Repositories,
     args: {
-      reply;
       item: Item;
       archiveRootPath: string;
       archive: ZipFile;
     },
-    logger: BaseLogger,
   ) {
-    const { item, archiveRootPath, archive, reply } = args;
+    const { item, archiveRootPath, archive } = args;
 
     // save description in file
     if (item.description) {
@@ -304,17 +318,11 @@ export class ImportExportService {
       const children = await this.itemService.getChildren(actor, repositories, item.id);
       const result = await Promise.all(
         children.map((child) =>
-          this._addItemToZip(
-            actor,
-            repositories,
-            {
-              item: child,
-              archiveRootPath: folderPath,
-              archive,
-              reply,
-            },
-            logger,
-          ),
+          this._addItemToZip(actor, repositories, {
+            item: child,
+            archiveRootPath: folderPath,
+            archive,
+          }),
         ),
       );
       // add empty folder
@@ -329,40 +337,133 @@ export class ImportExportService {
     return archive.addReadStream(stream, path.join(archiveRootPath, name));
   }
 
-  async export(
+  /**
+   * Recursively add items to the Graasp export file.
+   * Note that the shortcut items are excluded for now, they will be included in a later release.
+   * @param args item - the item to add
+   *             archive - reference to the zip file to which the files will be written
+   *             itemManifest - reference to the item manifest list
+   * @returns A full manifest promise for the given item
+   */
+  private async addItemToGraaspExport(
     actor: Actor,
     repositories: Repositories,
-    { item, reply }: { item: Item; reply: FastifyReply },
-    logger: BaseLogger,
+    args: {
+      item: Item;
+      archive: ZipFile;
+      itemManifest: GraaspExportItem[];
+    },
   ) {
+    const { item, archive, itemManifest } = args;
+
+    // assign the uuid to the exported items
+    const exportItemId = v4();
+    const itemPath = path.join(path.dirname('./'), exportItemId);
+
+    // TODO EXPORT treat the shortcut items correctly
+    // ignore the shortcuts
+    if (isItemType(item, ItemType.SHORTCUT)) {
+      return itemManifest;
+    }
+
+    // treat folder items recursively
+    const childrenManifest: GraaspExportItem[] = [];
+    if (isItemType(item, ItemType.FOLDER)) {
+      const childrenItems = await this.itemService.getChildren(actor, repositories, item.id, {
+        ordered: true,
+      });
+      for (const child of childrenItems) {
+        await this.addItemToGraaspExport(actor, repositories, {
+          item: child,
+          archive,
+          itemManifest: childrenManifest,
+        });
+      }
+
+      itemManifest.push({
+        id: exportItemId,
+        name: item.name,
+        description: item.description,
+        type: item.type,
+        settings: item.settings,
+        children: childrenManifest,
+      });
+      return itemManifest;
+    }
+
+    // treat single items
+    const { stream, name, mimetype } = await this.fetchItemData(actor, repositories, item);
+
+    itemManifest.push({
+      id: exportItemId,
+      name,
+      description: item.description,
+      type: item.type,
+      settings: item.settings,
+      mimetype,
+    });
+    archive.addReadStream(stream, itemPath);
+    return itemManifest;
+  }
+
+  /**
+   * Export the items recursively
+   * @param item The root item
+   * @returns A zip file promise
+   */
+  async exportRaw(actor: Actor, repositories: Repositories, item: Item) {
     // init archive
     const archive = new ZipFile();
     archive.outputStream.on('error', function (err) {
       throw new UnexpectedExportError(err);
     });
-
     // path used to index files in archive
     const rootPath = path.dirname('./');
 
     // import items in zip recursively
-    await this._addItemToZip(
-      actor,
-      repositories,
-      {
-        item,
-        reply,
-        archiveRootPath: rootPath,
-        archive,
-      },
-      logger,
-    ).catch((error) => {
+    await this._addItemToZip(actor, repositories, {
+      item,
+      archiveRootPath: rootPath,
+      archive,
+    }).catch((error) => {
       throw new UnexpectedExportError(error);
     });
 
     archive.end();
     return archive;
   }
 
+  /**
+   * Export the items recursively in the Graasp export format
+   * @param item The root item
+   * @returns A zip file promise
+   */
+  async exportGraasp(actor: Actor, repositories: Repositories, item: Item) {
+    // init archive
+    const archive = new ZipFile();
+    archive.outputStream.on('error', function (err) {
+      throw new UnexpectedExportError(err);
+    });
+    // path used to index files in archive
+    const rootPath = path.dirname('./');
+
+    const manifest = await this.addItemToGraaspExport(actor, repositories, {
+      item,
+      archive,
+      itemManifest: [],
+    }).catch((error) => {
+      throw new UnexpectedExportError(error);
+    });
+
+    archive.addReadStream(
+      Readable.from(JSON.stringify(manifest)),
+      path.join(rootPath, GRAASP_MANIFEST_FILENAME),
+    );
+
+    archive.end();
+    return archive;
+  }
+
   /**
    * Util recursive function that create graasp item given folder content
    * @param actor