fix: use file object headers when accessing file (#1192)

* fix: use file object headers when accessing file

* fix: review

* fix: build issue & prettier

* fix: code review

Author: ndrpp

src/components/P2P/index.ts

@@ -914,7 +914,7 @@ export class OceanP2P extends EventEmitter {
     const peersFound = []
     try {
       // @ts-ignore ignore the type mismatch
-      const f = await this._libp2p.contentRouting.findProviders(cid, {
+      const f = this._libp2p.contentRouting.findProviders(cid, {
         queryFuncTimeout: timeout || 20000 // 20 seconds
         // on timeout the query ends with an abort signal => CodeError: Query aborted
       })

src/components/core/compute/utils.ts

@@ -52,8 +52,14 @@ export async function getAlgoChecksums(
             : file.type === 'ipfs'
               ? urlJoin(config.ipfsGateway, (file as IpfsFileObject).hash)
               : null
+      const headers = file.type === 'url' ? (file as UrlFileObject).headers : undefined
 
-      const { contentChecksum } = await fetchFileMetadata(url, 'get', false)
+      const { contentChecksum } = await fetchFileMetadata(
+        url,
+        'get',
+        false,
+        headers ? headers[0] : undefined
+      )
       checksums.files = checksums.files.concat(contentChecksum)
     }
 

src/components/core/handler/fileInfoHandler.ts

@@ -20,6 +20,7 @@ import {
 } from '../../httpRoutes/validateCommands.js'
 import { getFile } from '../../../utils/file.js'
 import { getConfiguration } from '../../../utils/index.js'
+
 async function formatMetadata(
   file: ArweaveFileObject | IpfsFileObject | UrlFileObject,
   config: OceanNodeConfig
@@ -32,11 +33,13 @@ async function formatMetadata(
         : file.type === 'ipfs'
           ? urlJoin(config.ipfsGateway, (file as IpfsFileObject).hash)
           : null
+  const headers = file.type === 'url' ? (file as UrlFileObject).headers : undefined
 
   const { contentLength, contentType, contentChecksum } = await fetchFileMetadata(
     url,
     'get',
-    false
+    false,
+    headers ? headers[0] : undefined
   )
   CORE_LOGGER.logMessage(`Metadata for file: ${contentLength} ${contentType}`)
 

src/components/storage/index.ts

@@ -148,6 +148,24 @@ export class UrlStorage extends Storage {
     }
   }
 
+  async getReadableStream(): Promise<StorageReadable> {
+    const input = this.getDownloadUrl()
+    const file = this.getFile()
+    const response = await axios({
+      method: 'get',
+      url: input,
+      headers: file.headers ? file.headers[0] : undefined,
+      responseType: 'stream',
+      timeout: 30000
+    })
+
+    return {
+      httpStatus: response.status,
+      stream: response.data,
+      headers: response.headers as any
+    }
+  }
+
   validate(): [boolean, string] {
     const file: UrlFileObject = this.getFile() as UrlFileObject
     if (!file.url || !file.method) {
@@ -194,11 +212,12 @@ export class UrlStorage extends Storage {
     fileObject: UrlFileObject,
     forceChecksum: boolean
   ): Promise<FileInfoResponse> {
-    const { url, method } = fileObject
+    const { url, method, headers } = fileObject
     const { contentLength, contentType, contentChecksum } = await fetchFileMetadata(
       url,
       method,
-      forceChecksum
+      forceChecksum,
+      headers ? headers[0] : undefined
     )
     return {
       valid: true,

src/test/unit/storage.test.ts

@@ -171,6 +171,18 @@ describe('URL Storage tests', () => {
     const stream = await storage.getReadableStream()
     expect(stream).not.to.eql(null)
   })
+
+  it('Gets readable stream with headers', async () => {
+    file = {
+      type: 'url',
+      url: 'https://stock-api.oceanprotocol.com/stock/stock.json',
+      method: 'get',
+      headers: [{ 'X-Test-Header': 'test' }]
+    }
+    const storage = Storage.getStorageClass(file, config)
+    const stream = await storage.getReadableStream()
+    expect(stream).not.to.eql(null)
+  })
 })
 
 describe('Unsafe URL tests', () => {

src/utils/asset.ts

@@ -9,6 +9,7 @@ import { KNOWN_CONFIDENTIAL_EVMS } from './address.js'
 import ERC20Template from '@oceanprotocol/contracts/artifacts/contracts/interfaces/IERC20Template.sol/IERC20Template.json' with { type: 'json' }
 import ERC20Template4 from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC20Template4.sol/ERC20Template4.json' with { type: 'json' }
 import { getContractAddress, getNFTFactory } from '../components/Indexer/utils.js'
+import { HeadersObject } from '../@types/fileObject.js'
 
 // Notes:
 // Asset as per asset.py on provider, is a class there, while on ocean.Js we only have a type
@@ -44,7 +45,8 @@ export const AssetUtils = {
 export async function fetchFileMetadata(
   url: string,
   method: string,
-  forceChecksum: boolean
+  forceChecksum: boolean,
+  headers?: HeadersObject
 ): Promise<{ contentLength: string; contentType: string; contentChecksum: string }> {
   let contentType: string = ''
   let contentLength: number = 0
@@ -56,6 +58,7 @@ export async function fetchFileMetadata(
     const response = await axios({
       url,
       method: method || 'get',
+      headers,
       responseType: 'stream',
       timeout: 30000
     })
@@ -143,7 +146,7 @@ export async function isERC20Template4Active(
 ): Promise<boolean> {
   try {
     const nftFactoryAddress = getContractAddress(network, 'ERC721Factory')
-    const factoryERC721 = await getNFTFactory(owner, nftFactoryAddress)
+    const factoryERC721 = getNFTFactory(owner, nftFactoryAddress)
     const currentTokenCount = await factoryERC721.getCurrentTemplateCount()
 
     for (let i = 1; i <= currentTokenCount; i++) {

src/utils/logging/Logger.ts

@@ -198,7 +198,7 @@ const alignedWithColorsAndTime: winston.Logform.Format = winston.format.combine(
   winston.format.timestamp(),
   winston.format.align(),
   winston.format.printf(
-    (info) => `${info.timestamp} ${info.level}: ${info.message.trim()}`
+    (info) => `${info.timestamp} ${info.level}: ${(info.message as string).trim()}`
   )
 )
 const consoleColorFormatting: winston.Logform.Format | Record<string, any> = {