Skip to content

Commit de4b81b

Browse files
ndrppalexcos20
ndrpp
and
alexcos20
authored
Fix/1184 file headers not sent (#1195)
* fix: use file object headers when accessing file * fix: review * fix: build issue & prettier * fix: code review * fix: simplify file headers usage * fix: formatting * fix ci --------- Co-authored-by: alexcos20 <alex.coseru@gmail.com>
1 parent 7d9453b commit de4b81b

7 files changed

Lines changed: 66 additions & 55 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -122,9 +122,9 @@ jobs:
122122
if: ${{ env.DOCKERHUB_PASSWORD && env.DOCKERHUB_USERNAME }}
123123
run: |
124124
echo "Login to Docker Hub";echo "$DOCKERHUB_PASSWORD" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
125-
env:
126-
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
127-
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
125+
env:
126+
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
127+
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
128128
- name: Run Barge
129129
working-directory: ${{ github.workspace }}/barge
130130
run: |
@@ -145,6 +145,19 @@ jobs:
145145
- name: docker logs
146146
run: docker logs ocean-ocean-contracts-1 && docker logs ocean-typesense-1
147147
if: ${{ failure() }}
148+
- name: Set DOCKER_REGISTRY_AUTHS from Docker Hub secrets
149+
if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_PASSWORD
150+
run: |
151+
DOCKER_REGISTRY_AUTHS=$(jq -n \
152+
--arg username "$DOCKERHUB_USERNAME" \
153+
--arg password "$DOCKERHUB_PASSWORD" \
154+
'{ "https://registry-1.docker.io": { "username": $username, "password": $password } }')
155+
echo "DOCKER_REGISTRY_AUTHS<<EOF" >> $GITHUB_ENV
156+
echo "$DOCKER_REGISTRY_AUTHS" >> $GITHUB_ENV
157+
echo "EOF" >> $GITHUB_ENV
158+
env:
159+
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
160+
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
148161
- name: integration tests
149162
run: npm run test:integration:cover
150163
env:
@@ -162,6 +175,7 @@ jobs:
162175
FEE_AMOUNT: '{ "amount": 1, "unit": "MB" }'
163176
ASSET_PURGATORY_URL: 'https://raw.githubusercontent.com/oceanprotocol/list-purgatory/main/list-assets.json'
164177
ACCOUNT_PURGATORY_URL: 'https://raw.githubusercontent.com/oceanprotocol/list-purgatory/main/list-accounts.json'
178+
DOCKER_REGISTRY_AUTHS: ${{ env.DOCKER_REGISTRY_AUTHS }}
165179
- name: docker logs
166180
run: docker logs ocean-ocean-contracts-1 && docker logs ocean-typesense-1
167181
if: ${{ failure() }}
@@ -237,7 +251,19 @@ jobs:
237251
repository: 'oceanprotocol/ocean-node'
238252
path: 'ocean-node'
239253
ref: ${{ github.event_name == 'pull_request' && github.head_ref || 'main' }}
240-
254+
- name: Set DOCKER_REGISTRY_AUTHS from Docker Hub secrets
255+
if: env.DOCKERHUB_USERNAME && env.DOCKERHUB_PASSWORD
256+
run: |
257+
DOCKER_REGISTRY_AUTHS=$(jq -n \
258+
--arg username "$DOCKERHUB_USERNAME" \
259+
--arg password "$DOCKERHUB_PASSWORD" \
260+
'{ "https://registry-1.docker.io": { "username": $username, "password": $password } }')
261+
echo "DOCKER_REGISTRY_AUTHS<<EOF" >> $GITHUB_ENV
262+
echo "$DOCKER_REGISTRY_AUTHS" >> $GITHUB_ENV
263+
echo "EOF" >> $GITHUB_ENV
264+
env:
265+
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
266+
DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
241267
- name: Start Ocean Node
242268
working-directory: ${{ github.workspace }}/ocean-node
243269
run: |
@@ -265,6 +291,7 @@ jobs:
265291
MAX_REQ_PER_MINUTE: 320
266292
MAX_CONNECTIONS_PER_MINUTE: 320
267293
DOCKER_COMPUTE_ENVIRONMENTS: '[{"socketPath":"/var/run/docker.sock","resources":[{"id":"disk","total":10}],"storageExpiry":604800,"maxJobDuration":3600,"minJobDuration": 60,"fees":{"8996":[{"prices":[{"id":"cpu","price":1}]}]},"free":{"maxJobDuration":60,"minJobDuration": 10,"maxJobs":3,"resources":[{"id":"cpu","max":1},{"id":"ram","max":1},{"id":"disk","max":1}]}}]'
294+
DOCKER_REGISTRY_AUTHS: ${{ env.DOCKER_REGISTRY_AUTHS }}
268295
- name: Check Ocean Node is running
269296
run: |
270297
for i in $(seq 1 90); do

‎.github/workflows/docker.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ jobs:
3939
- name: Login to Docker Hub
4040
uses: docker/login-action@v3
4141
with:
42-
username: ${{ secrets.DOCKERHUB_USERNAME }}
42+
username: ${{ secrets.DOCKERHUB_PUSH_USERNAME }}
4343
password: ${{ secrets.DOCKER_PUSH_TOKEN }}
4444

4545
- name: Set Docker metadata
@@ -111,7 +111,7 @@ jobs:
111111
- name: Login to Docker Hub
112112
uses: docker/login-action@v3
113113
with:
114-
username: ${{ secrets.DOCKERHUB_USERNAME }}
114+
username: ${{ secrets.DOCKERHUB_PUSH_USERNAME }}
115115
password: ${{ secrets.DOCKER_PUSH_TOKEN }}
116116
- name: Create manifest list and push
117117
working-directory: /tmp/digests

‎src/@types/fileObject.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ export interface BaseFileObject {
1919
export interface UrlFileObject extends BaseFileObject {
2020
url: string
2121
method: string
22-
headers?: [HeadersObject]
22+
headers?: HeadersObject
2323
}
2424

2525
export interface IpfsFileObject extends BaseFileObject {

‎src/components/core/compute/utils.ts‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -54,12 +54,7 @@ export async function getAlgoChecksums(
5454
: null
5555
const headers = file.type === 'url' ? (file as UrlFileObject).headers : undefined
5656

57-
const { contentChecksum } = await fetchFileMetadata(
58-
url,
59-
'get',
60-
false,
61-
headers ? headers[0] : undefined
62-
)
57+
const { contentChecksum } = await fetchFileMetadata(url, 'get', false, headers)
6358
checksums.files = checksums.files.concat(contentChecksum)
6459
}
6560

‎src/components/core/handler/fileInfoHandler.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ async function formatMetadata(
3939
url,
4040
'get',
4141
false,
42-
headers ? headers[0] : undefined
42+
headers
4343
)
4444
CORE_LOGGER.logMessage(`Metadata for file: ${contentLength} ${contentType}`)
4545

‎src/components/storage/index.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -151,10 +151,11 @@ export class UrlStorage extends Storage {
151151
async getReadableStream(): Promise<StorageReadable> {
152152
const input = this.getDownloadUrl()
153153
const file = this.getFile()
154+
const { headers } = file
154155
const response = await axios({
155156
method: 'get',
156157
url: input,
157-
headers: file.headers ? file.headers[0] : undefined,
158+
headers,
158159
responseType: 'stream',
159160
timeout: 30000
160161
})
@@ -215,9 +216,9 @@ export class UrlStorage extends Storage {
215216
const { url, method, headers } = fileObject
216217
const { contentLength, contentType, contentChecksum } = await fetchFileMetadata(
217218
url,
218-
method,
219+
method || 'get',
219220
forceChecksum,
220-
headers ? headers[0] : undefined
221+
headers
221222
)
222223
return {
223224
valid: true,

‎src/test/unit/storage.test.ts‎

Lines changed: 26 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -29,12 +29,10 @@ describe('URL Storage tests', () => {
2929
type: 'url',
3030
url: 'http://someUrl.com/file.json',
3131
method: 'get',
32-
headers: [
33-
{
34-
'Content-Type': 'application/json',
35-
Authorization: 'Bearer auth_token_X'
36-
}
37-
],
32+
headers: {
33+
'Content-Type': 'application/json',
34+
Authorization: 'Bearer auth_token_X'
35+
},
3836
encryptedBy: nodeId,
3937
encryptMethod: EncryptMethod.AES
4038
}
@@ -70,12 +68,10 @@ describe('URL Storage tests', () => {
7068
file = {
7169
type: 'url',
7270
method: 'get',
73-
headers: [
74-
{
75-
'Content-Type': 'application/json',
76-
Authorization: 'Bearer auth_token_X'
77-
}
78-
]
71+
headers: {
72+
'Content-Type': 'application/json',
73+
Authorization: 'Bearer auth_token_X'
74+
}
7975
}
8076
try {
8177
Storage.getStorageClass(file, config)
@@ -88,12 +84,10 @@ describe('URL Storage tests', () => {
8884
file = {
8985
type: 'url',
9086
url: 'http://someUrl.com/file.json',
91-
headers: [
92-
{
93-
'Content-Type': 'application/json',
94-
Authorization: 'Bearer auth_token_X'
95-
}
96-
]
87+
headers: {
88+
'Content-Type': 'application/json',
89+
Authorization: 'Bearer auth_token_X'
90+
}
9791
}
9892
try {
9993
Storage.getStorageClass(file, config)
@@ -109,12 +103,10 @@ describe('URL Storage tests', () => {
109103
type: 'url',
110104
url: 'http://someUrl.com/file.json',
111105
method: 'put',
112-
headers: [
113-
{
114-
'Content-Type': 'application/json',
115-
Authorization: 'Bearer auth_token_X'
116-
}
117-
]
106+
headers: {
107+
'Content-Type': 'application/json',
108+
Authorization: 'Bearer auth_token_X'
109+
}
118110
}
119111
try {
120112
Storage.getStorageClass(file, config)
@@ -129,12 +121,10 @@ describe('URL Storage tests', () => {
129121
type: 'url',
130122
url: './../dir/file.json',
131123
method: 'get',
132-
headers: [
133-
{
134-
'Content-Type': 'application/json',
135-
Authorization: 'Bearer auth_token_X'
136-
}
137-
]
124+
headers: {
125+
'Content-Type': 'application/json',
126+
Authorization: 'Bearer auth_token_X'
127+
}
138128
}
139129
try {
140130
Storage.getStorageClass(file, config)
@@ -150,12 +140,10 @@ describe('URL Storage tests', () => {
150140
type: 'url',
151141
url: 'http://someUrl.com/file.json',
152142
method: 'get',
153-
headers: [
154-
{
155-
'Content-Type': 'application/json',
156-
Authorization: 'Bearer auth_token_X'
157-
}
158-
]
143+
headers: {
144+
'Content-Type': 'application/json',
145+
Authorization: 'Bearer auth_token_X'
146+
}
159147
}
160148
storage = Storage.getStorageClass(file, config)
161149
expect(storage.getDownloadUrl()).to.eql('http://someUrl.com/file.json')
@@ -172,12 +160,12 @@ describe('URL Storage tests', () => {
172160
expect(stream).not.to.eql(null)
173161
})
174162

175-
it('Gets readable stream with headers', async () => {
163+
it('Gets readable stream with headers as plain object', async () => {
176164
file = {
177165
type: 'url',
178166
url: 'https://stock-api.oceanprotocol.com/stock/stock.json',
179167
method: 'get',
180-
headers: [{ 'X-Test-Header': 'test' }]
168+
headers: { 'X-Test-Header': 'test' }
181169
}
182170
const storage = Storage.getStorageClass(file, config)
183171
const stream = await storage.getReadableStream()

0 commit comments

Comments
 (0)