|
| 1 | +import { randomUUID } from 'node:crypto' |
| 2 | +import { mkdtemp, rm } from 'node:fs/promises' |
| 3 | +import { tmpdir } from 'node:os' |
| 4 | +import { join } from 'node:path' |
| 5 | +import { ListPartsCommand, S3Client } from '@aws-sdk/client-s3' |
| 6 | +import { KnexMetastore } from '@storage/protocols/iceberg/knex' |
| 7 | +import { FastifyInstance } from 'fastify' |
| 8 | +import { getConfig } from '../config' |
| 9 | +import { useStorage } from './utils/storage' |
| 10 | + |
| 11 | +const { |
| 12 | + icebergBucketDetectionSuffix, |
| 13 | + s3ProtocolAccessKeyId, |
| 14 | + s3ProtocolAccessKeySecret, |
| 15 | + storageS3Region, |
| 16 | +} = getConfig() |
| 17 | + |
| 18 | +async function createFileBackedApp(fileBackendPath: string) { |
| 19 | + jest.resetModules() |
| 20 | + |
| 21 | + const configModule = await import('../config') |
| 22 | + |
| 23 | + configModule.getConfig({ reload: true }) |
| 24 | + configModule.mergeConfig({ |
| 25 | + storageBackendType: 'file', |
| 26 | + storageFilePath: fileBackendPath, |
| 27 | + }) |
| 28 | + |
| 29 | + return (await import('../app')).default() |
| 30 | +} |
| 31 | + |
| 32 | +describe('S3 protocol error code', () => { |
| 33 | + const t = useStorage() |
| 34 | + |
| 35 | + let testApp: FastifyInstance |
| 36 | + let client: S3Client |
| 37 | + let icebergMetastore: KnexMetastore |
| 38 | + let fileBackendPath: string |
| 39 | + |
| 40 | + beforeAll(async () => { |
| 41 | + fileBackendPath = await mkdtemp(join(tmpdir(), 'storage-file-backend-')) |
| 42 | + testApp = await createFileBackedApp(fileBackendPath) |
| 43 | + icebergMetastore = new KnexMetastore(t.database.connection.pool.acquire(), { |
| 44 | + multiTenant: false, |
| 45 | + schema: 'storage', |
| 46 | + }) |
| 47 | + |
| 48 | + const listener = await testApp.listen() |
| 49 | + |
| 50 | + client = new S3Client({ |
| 51 | + endpoint: `${listener.replace('[::1]', 'localhost')}/s3`, |
| 52 | + forcePathStyle: true, |
| 53 | + region: storageS3Region, |
| 54 | + credentials: { |
| 55 | + accessKeyId: s3ProtocolAccessKeyId!, |
| 56 | + secretAccessKey: s3ProtocolAccessKeySecret!, |
| 57 | + }, |
| 58 | + }) |
| 59 | + }) |
| 60 | + |
| 61 | + afterAll(async () => { |
| 62 | + client?.destroy() |
| 63 | + await testApp?.close() |
| 64 | + |
| 65 | + jest.resetModules() |
| 66 | + await rm(fileBackendPath, { recursive: true, force: true }) |
| 67 | + }) |
| 68 | + |
| 69 | + it('returns NotSupported for iceberg list-parts on non-S3 backends', async () => { |
| 70 | + const nonce = randomUUID().replaceAll('-', '') |
| 71 | + const analyticsBucket = await t.storage.createIcebergBucket({ |
| 72 | + name: `ice_bucket_${nonce}`, |
| 73 | + }) |
| 74 | + const namespaceName = `namespace${nonce}` |
| 75 | + const tableName = `table${nonce}` |
| 76 | + const internalBucketName = `internal-${nonce}${icebergBucketDetectionSuffix}` |
| 77 | + |
| 78 | + const namespace = await icebergMetastore.createNamespace({ |
| 79 | + name: namespaceName, |
| 80 | + bucketName: analyticsBucket.name, |
| 81 | + bucketId: analyticsBucket.id, |
| 82 | + tenantId: '', |
| 83 | + metadata: {}, |
| 84 | + }) |
| 85 | + |
| 86 | + await icebergMetastore.createTable({ |
| 87 | + name: tableName, |
| 88 | + bucketName: analyticsBucket.name, |
| 89 | + bucketId: analyticsBucket.id, |
| 90 | + location: `s3://${internalBucketName}`, |
| 91 | + namespaceId: namespace.id, |
| 92 | + }) |
| 93 | + |
| 94 | + await expect( |
| 95 | + client.send( |
| 96 | + new ListPartsCommand({ |
| 97 | + Bucket: internalBucketName, |
| 98 | + Key: `${namespaceName}/${tableName}/data.parquet`, |
| 99 | + UploadId: 'upload-id', |
| 100 | + }) |
| 101 | + ) |
| 102 | + ).rejects.toMatchObject({ |
| 103 | + $metadata: expect.objectContaining({ httpStatusCode: 409 }), |
| 104 | + name: 'NotSupported', |
| 105 | + }) |
| 106 | + }) |
| 107 | +}) |
0 commit comments