Fix http upload

.github/workflows/ci.yml

@@ -292,6 +292,7 @@ jobs:
           MAX_CONNECTIONS_PER_MINUTE: 320
           DOCKER_COMPUTE_ENVIRONMENTS: '[{"socketPath":"/var/run/docker.sock","environments":[{"storageExpiry":604800,"maxJobDuration":3600,"minJobDuration":60,"resources":[{"id":"cpu","total":4,"max":4,"min":1,"type":"cpu"},{"id":"ram","total":10,"max":10,"min":1,"type":"ram"},{"id":"disk","total":10,"max":10,"min":0,"type":"disk"}],"fees":{"8996":[{"prices":[{"id":"cpu","price":1}]}]},"free":{"maxJobDuration":60,"maxJobs":3,"resources":[{"id":"cpu","max":1},{"id":"ram","max":1},{"id":"disk","max":1}]}}]}]'
           DOCKER_REGISTRY_AUTHS: ${{ env.DOCKER_REGISTRY_AUTHS }}
+          PERSISTENT_STORAGE: '{"enabled": true, "type": "localfs", "options": {"folder": "/tmp/ocean-persistent-storage"}}'
       - name: Check Ocean Node is running
         run: |
           for i in $(seq 1 90); do

src/components/httpRoutes/provider.ts

@@ -36,35 +36,39 @@ providerRoutes.post(`${SERVICES_API_BASE_PATH}/decrypt`, async (req, res) => {
   }
 })
 
-providerRoutes.post(`${SERVICES_API_BASE_PATH}/encrypt`, async (req, res) => {
-  try {
-    const data = req.body.toString()
-    if (!data) {
-      res.status(400).send('Missing required body')
-      return
-    }
-    const result = await new EncryptHandler(req.oceanNode).handle({
-      blob: data,
-      encoding: 'string',
-      encryptionType: EncryptMethod.ECIES,
-      command: PROTOCOL_COMMANDS.ENCRYPT,
-      caller: req.caller,
-      nonce: req.query.nonce as string,
-      consumerAddress: req.query.consumerAddress as string,
-      signature: req.query.signature as string
-    })
-    if (result.stream) {
-      const encryptedData = await streamToString(result.stream as Readable)
-      res.header('Content-Type', 'application/octet-stream')
-      res.status(200).send(encryptedData)
-    } else {
-      res.status(result.status.httpStatus).send(result.status.error)
+providerRoutes.post(
+  `${SERVICES_API_BASE_PATH}/encrypt`,
+  express.raw({ limit: '25mb' }),
+  async (req, res) => {
+    try {
+      const data = req.body.toString()
+      if (!data) {
+        res.status(400).send('Missing required body')
+        return
+      }
+      const result = await new EncryptHandler(req.oceanNode).handle({
+        blob: data,
+        encoding: 'string',
+        encryptionType: EncryptMethod.ECIES,
+        command: PROTOCOL_COMMANDS.ENCRYPT,
+        caller: req.caller,
+        nonce: req.query.nonce as string,
+        consumerAddress: req.query.consumerAddress as string,
+        signature: req.query.signature as string
+      })
+      if (result.stream) {
+        const encryptedData = await streamToString(result.stream as Readable)
+        res.header('Content-Type', 'application/octet-stream')
+        res.status(200).send(encryptedData)
+      } else {
+        res.status(result.status.httpStatus).send(result.status.error)
+      }
+    } catch (error) {
+      HTTP_LOGGER.log(LOG_LEVELS_STR.LEVEL_ERROR, `Error: ${error}`)
+      res.status(500).send('Internal Server Error')
     }
-  } catch (error) {
-    HTTP_LOGGER.log(LOG_LEVELS_STR.LEVEL_ERROR, `Error: ${error}`)
-    res.status(500).send('Internal Server Error')
   }
-})
+)
 
 // There are two ways of encrypting a file:
 
@@ -81,77 +85,84 @@ providerRoutes.post(`${SERVICES_API_BASE_PATH}/encrypt`, async (req, res) => {
 // Headers
 // X-Encrypted-By: our_node_id
 // X-Encrypted-Method: aes or ecies
-providerRoutes.post(`${SERVICES_API_BASE_PATH}/encryptFile`, async (req, res) => {
-  const writeResponse = async (
-    result: P2PCommandResponse,
-    encryptMethod: EncryptMethod
-  ) => {
-    if (result.stream) {
-      const encryptedData = await streamToString(result.stream as Readable)
-      res.set(result.status.headers)
-      res.status(200).send(encryptedData)
-    } else {
-      res.status(result.status.httpStatus).send(result.status.error)
+providerRoutes.post(
+  `${SERVICES_API_BASE_PATH}/encryptFile`,
+  express.raw({ limit: '25mb', type: 'application/octet-stream' }),
+  express.json(),
+  async (req, res) => {
+    const writeResponse = async (
+      result: P2PCommandResponse,
+      encryptMethod: EncryptMethod
+    ) => {
+      if (result.stream) {
+        const encryptedData = await streamToString(result.stream as Readable)
+        res.set(result.status.headers)
+        res.status(200).send(encryptedData)
+      } else {
+        res.status(result.status.httpStatus).send(result.status.error)
+      }
     }
-  }
 
-  const getEncryptedData = async (
-    encryptMethod: EncryptMethod.AES | EncryptMethod.ECIES,
-    input: Buffer
-  ) => {
-    const result = await new EncryptFileHandler(req.oceanNode).handle({
-      rawData: input,
-      encryptionType: encryptMethod,
-      command: PROTOCOL_COMMANDS.ENCRYPT_FILE,
-      caller: req.caller,
-      nonce: req.query.nonce as string,
-      consumerAddress: req.query.consumerAddress as string,
-      signature: req.query.signature as string
-    })
-    return result
-  }
-
-  try {
-    const encryptMethod: EncryptMethod = getEncryptMethodFromString(
-      req.query.encryptMethod as string
-    )
-    let result: P2PCommandResponse
-    if (req.is('application/json')) {
-      // body as fileObject
-      result = await new EncryptFileHandler(req.oceanNode).handle({
-        files: req.body as StorageObject,
+    const getEncryptedData = async (
+      encryptMethod: EncryptMethod.AES | EncryptMethod.ECIES,
+      input: Buffer
+    ) => {
+      const result = await new EncryptFileHandler(req.oceanNode).handle({
+        rawData: input,
         encryptionType: encryptMethod,
         command: PROTOCOL_COMMANDS.ENCRYPT_FILE,
         caller: req.caller,
         nonce: req.query.nonce as string,
         consumerAddress: req.query.consumerAddress as string,
         signature: req.query.signature as string
       })
-      return await writeResponse(result, encryptMethod)
-      // raw data on body
-    } else if (req.is('application/octet-stream') || req.is('multipart/form-data')) {
-      if (req.is('application/octet-stream')) {
-        result = await getEncryptedData(encryptMethod, req.body)
-        return await writeResponse(result, encryptMethod)
-      } else {
-        // multipart/form-data
-        const data: Buffer[] = []
-        req.on('data', function (chunk) {
-          data.push(chunk)
+      return result
+    }
+
+    try {
+      const encryptMethod: EncryptMethod = getEncryptMethodFromString(
+        req.query.encryptMethod as string
+      )
+      let result: P2PCommandResponse
+      if (req.is('application/json')) {
+        // body as fileObject
+        result = await new EncryptFileHandler(req.oceanNode).handle({
+          files: req.body as StorageObject,
+          encryptionType: encryptMethod,
+          command: PROTOCOL_COMMANDS.ENCRYPT_FILE,
+          caller: req.caller,
+          nonce: req.query.nonce as string,
+          consumerAddress: req.query.consumerAddress as string,
+          signature: req.query.signature as string
         })
-        req.on('end', async function () {
-          result = await getEncryptedData(encryptMethod, Buffer.concat(data))
+        return await writeResponse(result, encryptMethod)
+        // raw data on body
+      } else if (req.is('application/octet-stream') || req.is('multipart/form-data')) {
+        if (req.is('application/octet-stream')) {
+          result = await getEncryptedData(encryptMethod, req.body)
           return await writeResponse(result, encryptMethod)
-        })
+        } else {
+          // multipart/form-data
+          const data: Buffer[] = []
+          req.on('data', function (chunk) {
+            data.push(chunk)
+          })
+          req.on('end', async function () {
+            result = await getEncryptedData(encryptMethod, Buffer.concat(data))
+            return await writeResponse(result, encryptMethod)
+          })
+        }
+      } else {
+        res
+          .status(400)
+          .send('Invalid request (missing body data or invalid content-type)')
       }
-    } else {
-      res.status(400).send('Invalid request (missing body data or invalid content-type)')
+    } catch (error) {
+      HTTP_LOGGER.log(LOG_LEVELS_STR.LEVEL_ERROR, `Error: ${error}`)
+      res.status(500).send('Internal Server Error')
     }
-  } catch (error) {
-    HTTP_LOGGER.log(LOG_LEVELS_STR.LEVEL_ERROR, `Error: ${error}`)
-    res.status(500).send('Internal Server Error')
   }
-})
+)
 
 providerRoutes.get(`${SERVICES_API_BASE_PATH}/initialize`, async (req, res) => {
   try {

src/index.ts

@@ -131,8 +131,6 @@ function removeExtraSlashes(req: any, res: any, next: any) {
 }
 
 if (config.hasHttp) {
-  // allow up to 25Mb file upload
-  app.use(express.raw({ limit: '25mb' }))
   app.use(cors())
   app.use(requestValidator, (req, res, next) => {
     req.caller = req.headers['x-forwarded-for'] || req.socket.remoteAddress