Skip to content

Commit 670089b

Browse files
ferhatelmasferhatelmas
committed
fix: more tests and edge case handling
Signed-off-by: ferhat elmas <elmas.ferhat@gmail.com>
1 parent 29d3129 commit 670089b

7 files changed

Lines changed: 413 additions & 31 deletions

File tree

‎src/storage/object.ts‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -864,13 +864,14 @@ const CONTINUATION_TOKEN_PART_MAP: Record<string, keyof ContinuationToken> = {
864864
}
865865

866866
function encodeContinuationToken(tokenInfo: ContinuationToken) {
867-
let result = ''
867+
const result: string[] = []
868868
for (const [k, v] of Object.entries(CONTINUATION_TOKEN_PART_MAP)) {
869-
if (tokenInfo[v]) {
870-
result += `${k}:${tokenInfo[v]}\n`
869+
const value = tokenInfo[v]
870+
if (value) {
871+
result.push(`${k}:${encodeURIComponent(value)}`)
871872
}
872873
}
873-
return Buffer.from(result.slice(0, -1)).toString('base64')
874+
return Buffer.from(result.join('\n')).toString('base64')
874875
}
875876

876877
function decodeContinuationToken(token: string): ContinuationToken {
@@ -884,7 +885,13 @@ function decodeContinuationToken(token: string): ContinuationToken {
884885
if (!partMatch || partMatch.length !== 3 || !(partMatch[1] in CONTINUATION_TOKEN_PART_MAP)) {
885886
throw new Error('Invalid continuation token')
886887
}
887-
result[CONTINUATION_TOKEN_PART_MAP[partMatch[1]]] = partMatch[2]
888+
let value = partMatch[2]
889+
try {
890+
value = decodeURIComponent(value)
891+
} catch {
892+
// Backward compatibility: previously cursor values were stored unescaped.
893+
}
894+
result[CONTINUATION_TOKEN_PART_MAP[partMatch[1]]] = value
888895
}
889896
return result
890897
}

‎src/storage/protocols/s3/s3-handler.ts‎

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1406,45 +1406,55 @@ function parseCopySource(copySource: string): {
14061406
sourceVersion?: string
14071407
} {
14081408
const normalizedCopySource = copySource.startsWith('/') ? copySource.slice(1) : copySource
1409-
const [encodedPath, queryParams = ''] = normalizedCopySource.split('?')
1410-
const [encodedBucketName, ...encodedObjectKeyParts] = encodedPath.split('/')
1409+
const [encodedPath, ...queryParts] = normalizedCopySource.split('?')
1410+
const queryParams = queryParts.join('?')
14111411

1412-
if (!encodedBucketName) {
1413-
throw ERRORS.InvalidBucketName('')
1412+
let decodedPath = ''
1413+
try {
1414+
decodedPath = decodeURIComponent(encodedPath)
1415+
} catch {
1416+
throw ERRORS.InvalidParameter('CopySource')
14141417
}
14151418

1416-
if (!encodedObjectKeyParts.length) {
1419+
const separatorIdx = decodedPath.indexOf('/')
1420+
if (separatorIdx <= 0) {
14171421
throw ERRORS.MissingParameter('CopySource')
14181422
}
14191423

1420-
try {
1421-
const searchParams = new URLSearchParams(queryParams)
1422-
const sourceVersion = searchParams.get('versionId') || undefined
1424+
const bucketName = decodedPath.slice(0, separatorIdx)
1425+
const objectKey = decodedPath.slice(separatorIdx + 1)
1426+
if (!objectKey) {
1427+
throw ERRORS.MissingParameter('CopySource')
1428+
}
14231429

1424-
if (searchParams.has('versionId') && !sourceVersion) {
1425-
throw ERRORS.InvalidParameter('CopySource')
1426-
}
1430+
const searchParams = new URLSearchParams(queryParams)
1431+
const sourceVersion = searchParams.get('versionId') || undefined
14271432

1428-
return {
1429-
bucketName: decodeURIComponent(encodedBucketName),
1430-
objectKey: decodeURIComponent(encodedObjectKeyParts.join('/')),
1431-
sourceVersion,
1432-
}
1433-
} catch {
1433+
if (searchParams.has('versionId') && !sourceVersion) {
14341434
throw ERRORS.InvalidParameter('CopySource')
14351435
}
1436+
1437+
return {
1438+
bucketName,
1439+
objectKey,
1440+
sourceVersion,
1441+
}
14361442
}
14371443

14381444
function encodeContinuationToken(name: string) {
14391445
return Buffer.from(`l:${name}`).toString('base64')
14401446
}
14411447

14421448
function decodeContinuationToken(token: string) {
1443-
const decoded = Buffer.from(token, 'base64').toString().split(':')
1449+
const decoded = Buffer.from(token, 'base64').toString()
1450+
if (!decoded.startsWith('l:')) {
1451+
throw new Error('Invalid continuation token')
1452+
}
14441453

1445-
if (decoded.length === 0) {
1454+
const value = decoded.slice(2)
1455+
if (!value) {
14461456
throw new Error('Invalid continuation token')
14471457
}
14481458

1449-
return decoded[1]
1459+
return value
14501460
}

‎src/test/object-list-v2.test.ts‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -578,6 +578,7 @@ describe('objects - list v2 sorting tests', () => {
578578
})
579579

580580
const LIST_V2_WILDCARD_BUCKET = `list-v2-wildcard-${randomUUID()}`
581+
const LIST_V2_CURSOR_BUCKET = `list-v2-cursor-${randomUUID()}`
581582

582583
describe('objects - list v2 prefix wildcard handling', () => {
583584
beforeAll(async () => {
@@ -700,3 +701,98 @@ describe('objects - list v2 prefix wildcard handling', () => {
700701
expect(data.objects.map((obj) => obj.name)).toEqual([literalMatch])
701702
})
702703
})
704+
705+
describe('objects - list v2 cursor encoding', () => {
706+
beforeAll(async () => {
707+
appInstance = app()
708+
await appInstance.inject({
709+
method: 'POST',
710+
url: `/bucket`,
711+
headers: {
712+
authorization: `Bearer ${serviceKey}`,
713+
},
714+
payload: {
715+
name: LIST_V2_CURSOR_BUCKET,
716+
},
717+
})
718+
await appInstance.close()
719+
})
720+
721+
afterAll(async () => {
722+
appInstance = app()
723+
await appInstance.inject({
724+
method: 'POST',
725+
url: `/bucket/${LIST_V2_CURSOR_BUCKET}/empty`,
726+
headers: {
727+
authorization: `Bearer ${serviceKey}`,
728+
},
729+
})
730+
731+
await appInstance.inject({
732+
method: 'DELETE',
733+
url: `/bucket/${LIST_V2_CURSOR_BUCKET}`,
734+
headers: {
735+
authorization: `Bearer ${serviceKey}`,
736+
},
737+
})
738+
739+
await appInstance.close()
740+
})
741+
742+
test('paginates when keys contain newline and percent characters', async () => {
743+
const runId = randomUUID()
744+
const prefix = `cursor-${runId}-`
745+
const keys = [`${prefix}first-\n-🙂-%.txt`, `${prefix}second-\n-일이삼-:.txt`]
746+
747+
for (const key of keys) {
748+
const uploadResponse = await appInstance.inject({
749+
method: 'POST',
750+
url: `/object/${LIST_V2_CURSOR_BUCKET}/${encodeURIComponent(key)}`,
751+
payload: createUpload('utf8.txt', 'cursor test'),
752+
headers: {
753+
authorization: `Bearer ${serviceKey}`,
754+
},
755+
})
756+
expect(uploadResponse.statusCode).toBe(200)
757+
}
758+
759+
const page1Response = await appInstance.inject({
760+
method: 'POST',
761+
url: `/object/list-v2/${LIST_V2_CURSOR_BUCKET}`,
762+
payload: {
763+
with_delimiter: false,
764+
prefix,
765+
limit: 1,
766+
},
767+
headers: {
768+
authorization: `Bearer ${serviceKey}`,
769+
},
770+
})
771+
expect(page1Response.statusCode).toBe(200)
772+
const page1 = page1Response.json<ListObjectsV2Result>()
773+
expect(page1.objects).toHaveLength(1)
774+
expect(page1.hasNext).toBe(true)
775+
expect(page1.nextCursor).toBeTruthy()
776+
777+
const page2Response = await appInstance.inject({
778+
method: 'POST',
779+
url: `/object/list-v2/${LIST_V2_CURSOR_BUCKET}`,
780+
payload: {
781+
with_delimiter: false,
782+
prefix,
783+
limit: 1,
784+
cursor: page1.nextCursor,
785+
},
786+
headers: {
787+
authorization: `Bearer ${serviceKey}`,
788+
},
789+
})
790+
expect(page2Response.statusCode).toBe(200)
791+
const page2 = page2Response.json<ListObjectsV2Result>()
792+
expect(page2.objects).toHaveLength(1)
793+
expect(page2.hasNext).toBe(false)
794+
795+
const listed = [page1.objects[0]?.name, page2.objects[0]?.name].filter(Boolean).sort()
796+
expect(listed).toEqual([...keys].sort())
797+
})
798+
})

‎src/test/object.test.ts‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1849,7 +1849,7 @@ describe('testing uploading with generated signed upload URL', () => {
18491849
})
18501850

18511851
const BUCKET_ID = 'bucket2'
1852-
const OBJECT_NAME = 'public/sadcat-upload1.png'
1852+
const OBJECT_NAME = `public/sadcat-upload-${randomUUID()}.png`
18531853
const urlToSign = `${BUCKET_ID}/${OBJECT_NAME}`
18541854
const owner = '317eadce-631a-4429-a0bb-f19a7a517b4a'
18551855

@@ -1893,10 +1893,11 @@ describe('testing uploading with generated signed upload URL', () => {
18931893
const headers = Object.assign({}, form.getHeaders(), {
18941894
'content-type': 'image/jpeg',
18951895
})
1896+
const objectName = `public/sadcat-upload-${randomUUID()}.png`
18961897

18971898
const response = await appInstance.inject({
18981899
method: 'PUT',
1899-
url: `/object/upload/sign/bucket2/public/sadcat-upload1.png`,
1900+
url: `/object/upload/sign/bucket2/${objectName}`,
19001901
headers,
19011902
payload: form,
19021903
})
@@ -1910,10 +1911,11 @@ describe('testing uploading with generated signed upload URL', () => {
19101911
const headers = Object.assign({}, form.getHeaders(), {
19111912
'content-type': 'image/jpeg',
19121913
})
1914+
const objectName = `public/sadcat-upload-${randomUUID()}.png`
19131915

19141916
const response = await appInstance.inject({
19151917
method: 'PUT',
1916-
url: `/object/upload/sign/bucket2/public/sadcat-upload1.png?token=xxx`,
1918+
url: `/object/upload/sign/bucket2/${objectName}?token=xxx`,
19171919
headers,
19181920
payload: form,
19191921
})
@@ -1929,7 +1931,7 @@ describe('testing uploading with generated signed upload URL', () => {
19291931
})
19301932

19311933
const BUCKET_ID = 'bucket2'
1932-
const OBJECT_NAME = 'public/sadcat-upload1.png'
1934+
const OBJECT_NAME = `public/sadcat-upload-${randomUUID()}.png`
19331935
const urlToSign = `${BUCKET_ID}/${OBJECT_NAME}`
19341936
const owner = '317eadce-631a-4429-a0bb-f19a7a517b4a'
19351937

@@ -1952,7 +1954,7 @@ describe('testing uploading with generated signed upload URL', () => {
19521954
}
19531955

19541956
const BUCKET_ID = 'bucket2'
1955-
const OBJECT_NAME = 'signed/sadcat-upload-signed-2.png'
1957+
const OBJECT_NAME = `signed/sadcat-upload-signed-${randomUUID()}.png`
19561958
const urlToSign = `${BUCKET_ID}/${OBJECT_NAME}`
19571959

19581960
// Upload a file first
@@ -1997,7 +1999,7 @@ describe('testing uploading with generated signed upload URL', () => {
19971999
}
19982000

19992001
const BUCKET_ID = 'bucket2'
2000-
const OBJECT_NAME = 'signed/sadcat-upload-signed-3.png'
2002+
const OBJECT_NAME = `signed/sadcat-upload-signed-${randomUUID()}.png`
20012003
const urlToSign = `${BUCKET_ID}/${OBJECT_NAME}`
20022004
const owner = '317eadce-631a-4429-a0bb-f19a7a517b4a'
20032005

0 commit comments

Comments
 (0)