From 1cdc90cb25424e3985d983882d3b2b95af1a0323 Mon Sep 17 00:00:00 2001 From: Lyubov Voloshko Date: Thu, 7 May 2026 13:11:53 +0000 Subject: [PATCH] feat: extend S3 widget with multi-provider support Adds support for AWS, DigitalOcean Spaces, Backblaze B2, Wasabi, and Cloudflare R2 in the S3 widget. The endpoint URL is derived on the backend per provider rules (region-based for most; account_id for R2) so users only pick a provider rather than supplying a raw endpoint. Renames widget DTOs/params to be provider-agnostic; widget_params JSON keys aws_access_key_id_secret_name/aws_secret_access_key_secret_name become access_key_id_secret_name/secret_access_key_secret_name. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../data-structures/bucket-provider.enum.ts | 7 + .../data-structures/s3-operation.ds.ts | 8 +- .../data-structures/s3-widget-params.ds.ts | 12 +- .../entities/s3-widget/s3-helper.service.ts | 59 +++++- .../s3-widget/s3-widget.controller.ts | 6 +- .../use-cases/get-s3-file-url.use.case.ts | 27 ++- .../use-cases/get-s3-upload-url.use.case.ts | 24 ++- .../use-cases/s3-use-cases.interface.ts | 12 +- .../utils/validate-create-widgets-ds.ts | 18 +- .../non-saas-s3-widget-e2e.test.ts | 191 +++++++++++++++++- .../db-table-widgets.component.ts | 26 ++- .../s3/s3.component.spec.ts | 15 +- .../record-edit-fields/s3/s3.component.ts | 12 +- .../s3/s3.component.spec.ts | 8 +- .../record-view-fields/s3/s3.component.ts | 12 +- .../s3/s3.component.spec.ts | 4 +- .../table-display-fields/s3/s3.component.ts | 12 +- frontend/src/app/services/s3.service.ts | 14 +- 18 files changed, 380 insertions(+), 87 deletions(-) create mode 100644 backend/src/entities/s3-widget/application/data-structures/bucket-provider.enum.ts diff --git a/backend/src/entities/s3-widget/application/data-structures/bucket-provider.enum.ts b/backend/src/entities/s3-widget/application/data-structures/bucket-provider.enum.ts new file mode 100644 index 000000000..c8047f2cb --- /dev/null +++ b/backend/src/entities/s3-widget/application/data-structures/bucket-provider.enum.ts @@ -0,0 +1,7 @@ +export enum BucketProviderEnum { + AWS = 'aws', + DigitalOceanSpaces = 'digitalocean', + BackblazeB2 = 'backblaze', + Wasabi = 'wasabi', + CloudflareR2 = 'cloudflare-r2', +} diff --git a/backend/src/entities/s3-widget/application/data-structures/s3-operation.ds.ts b/backend/src/entities/s3-widget/application/data-structures/s3-operation.ds.ts index 9a6e516b5..174c52b0a 100644 --- a/backend/src/entities/s3-widget/application/data-structures/s3-operation.ds.ts +++ b/backend/src/entities/s3-widget/application/data-structures/s3-operation.ds.ts @@ -1,4 +1,4 @@ -export class S3GetFileUrlDs { +export class GetBucketFileUrlDs { connectionId: string; tableName: string; fieldName: string; @@ -7,7 +7,7 @@ export class S3GetFileUrlDs { masterPwd: string; } -export class S3GetUploadUrlDs { +export class GetBucketUploadUrlDs { connectionId: string; tableName: string; fieldName: string; @@ -17,13 +17,13 @@ export class S3GetUploadUrlDs { contentType: string; } -export class S3FileUrlResponseDs { +export class BucketFileUrlResponseDs { url: string; key: string; expiresIn: number; } -export class S3UploadUrlResponseDs { +export class BucketUploadUrlResponseDs { uploadUrl: string; key: string; expiresIn: number; diff --git a/backend/src/entities/s3-widget/application/data-structures/s3-widget-params.ds.ts b/backend/src/entities/s3-widget/application/data-structures/s3-widget-params.ds.ts index 571b380c4..933542fad 100644 --- a/backend/src/entities/s3-widget/application/data-structures/s3-widget-params.ds.ts +++ b/backend/src/entities/s3-widget/application/data-structures/s3-widget-params.ds.ts @@ -1,8 +1,12 @@ -export interface S3WidgetParams { +import { BucketProviderEnum } from './bucket-provider.enum.js'; + +export interface BucketWidgetParams { + provider?: BucketProviderEnum; bucket: string; prefix?: string; region?: string; - aws_access_key_id_secret_name: string; - aws_secret_access_key_secret_name: string; - type?: 'file' | 'image'; // 'file' (default) - accepts all files, 'image' - accepts only images + account_id?: string; + access_key_id_secret_name: string; + secret_access_key_secret_name: string; + type?: 'file' | 'image'; } diff --git a/backend/src/entities/s3-widget/s3-helper.service.ts b/backend/src/entities/s3-widget/s3-helper.service.ts index 29f8292b0..1956ee662 100644 --- a/backend/src/entities/s3-widget/s3-helper.service.ts +++ b/backend/src/entities/s3-widget/s3-helper.service.ts @@ -1,16 +1,30 @@ import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; -import { Injectable } from '@nestjs/common'; +import { HttpException, HttpStatus, Injectable } from '@nestjs/common'; import { nanoid } from 'nanoid'; +import { BucketProviderEnum } from './application/data-structures/bucket-provider.enum.js'; + +export interface BucketClientConfig { + accessKeyId: string; + secretAccessKey: string; + provider?: BucketProviderEnum; + region?: string; + accountId?: string; +} @Injectable() export class S3HelperService { - public createS3Client(accessKeyId: string, secretAccessKey: string, region: string = 'us-east-1'): S3Client { + public createS3Client(config: BucketClientConfig): S3Client { + const provider = config.provider || BucketProviderEnum.AWS; + const region = this._resolveRegion(provider, config.region); + const endpoint = this._resolveEndpoint(provider, region, config.accountId); + return new S3Client({ region, + endpoint, credentials: { - accessKeyId, - secretAccessKey, + accessKeyId: config.accessKeyId, + secretAccessKey: config.secretAccessKey, }, }); } @@ -52,6 +66,43 @@ export class S3HelperService { return key; } + private _resolveRegion(provider: BucketProviderEnum, region: string | undefined): string { + if (region) { + return region; + } + if (provider === BucketProviderEnum.CloudflareR2) { + return 'auto'; + } + return 'us-east-1'; + } + + private _resolveEndpoint( + provider: BucketProviderEnum, + region: string, + accountId: string | undefined, + ): string | undefined { + switch (provider) { + case BucketProviderEnum.AWS: + return undefined; + case BucketProviderEnum.DigitalOceanSpaces: + return `https://${region}.digitaloceanspaces.com`; + case BucketProviderEnum.BackblazeB2: + return `https://s3.${region}.backblazeb2.com`; + case BucketProviderEnum.Wasabi: + return `https://s3.${region}.wasabisys.com`; + case BucketProviderEnum.CloudflareR2: + if (!accountId) { + throw new HttpException( + { message: 'Cloudflare R2 requires account_id in widget params' }, + HttpStatus.BAD_REQUEST, + ); + } + return `https://${accountId}.r2.cloudflarestorage.com`; + default: + return undefined; + } + } + private _extractFileExtension(filename: string): string { const lastDotIndex = filename.lastIndexOf('.'); if (lastDotIndex === -1 || lastDotIndex === 0) { diff --git a/backend/src/entities/s3-widget/s3-widget.controller.ts b/backend/src/entities/s3-widget/s3-widget.controller.ts index 50a754593..6b2d4cbc5 100644 --- a/backend/src/entities/s3-widget/s3-widget.controller.ts +++ b/backend/src/entities/s3-widget/s3-widget.controller.ts @@ -23,7 +23,7 @@ import { Messages } from '../../exceptions/text/messages.js'; import { ConnectionEditGuard } from '../../guards/connection-edit.guard.js'; import { ConnectionReadGuard } from '../../guards/connection-read.guard.js'; import { SentryInterceptor } from '../../interceptors/sentry.interceptor.js'; -import { S3FileUrlResponseDs, S3UploadUrlResponseDs } from './application/data-structures/s3-operation.ds.js'; +import { BucketFileUrlResponseDs, BucketUploadUrlResponseDs } from './application/data-structures/s3-operation.ds.js'; import { IGetS3FileUrl, IGetS3UploadUrl } from './use-cases/s3-use-cases.interface.js'; @UseInterceptors(SentryInterceptor) @@ -61,7 +61,7 @@ export class S3WidgetController { @QueryTableName() tableName: string, @Query('fieldName') fieldName: string, @Query('rowPrimaryKey') rowPrimaryKeyStr: string, - ): Promise { + ): Promise { if (!connectionId) { throw new HttpException({ message: Messages.CONNECTION_ID_MISSING }, HttpStatus.BAD_REQUEST); } @@ -118,7 +118,7 @@ export class S3WidgetController { @QueryTableName() tableName: string, @Query('fieldName') fieldName: string, @Body() body: { filename: string; contentType: string }, - ): Promise { + ): Promise { if (!connectionId) { throw new HttpException({ message: Messages.CONNECTION_ID_MISSING }, HttpStatus.BAD_REQUEST); } diff --git a/backend/src/entities/s3-widget/use-cases/get-s3-file-url.use.case.ts b/backend/src/entities/s3-widget/use-cases/get-s3-file-url.use.case.ts index b2947fff5..6229836dd 100644 --- a/backend/src/entities/s3-widget/use-cases/get-s3-file-url.use.case.ts +++ b/backend/src/entities/s3-widget/use-cases/get-s3-file-url.use.case.ts @@ -10,13 +10,16 @@ import { WidgetTypeEnum } from '../../../enums/widget-type.enum.js'; import { Messages } from '../../../exceptions/text/messages.js'; import { Encryptor } from '../../../helpers/encryption/encryptor.js'; import { isConnectionTypeAgent } from '../../../helpers/is-connection-entity-agent.js'; -import { S3FileUrlResponseDs, S3GetFileUrlDs } from '../application/data-structures/s3-operation.ds.js'; -import { S3WidgetParams } from '../application/data-structures/s3-widget-params.ds.js'; +import { BucketFileUrlResponseDs, GetBucketFileUrlDs } from '../application/data-structures/s3-operation.ds.js'; +import { BucketWidgetParams } from '../application/data-structures/s3-widget-params.ds.js'; import { S3HelperService } from '../s3-helper.service.js'; import { IGetS3FileUrl } from './s3-use-cases.interface.js'; @Injectable() -export class GetS3FileUrlUseCase extends AbstractUseCase implements IGetS3FileUrl { +export class GetS3FileUrlUseCase + extends AbstractUseCase + implements IGetS3FileUrl +{ constructor( @Inject(BaseType.GLOBAL_DB_CONTEXT) protected _dbContext: IGlobalDatabaseContext, @@ -25,7 +28,7 @@ export class GetS3FileUrlUseCase extends AbstractUseCase { + protected async implementation(inputData: GetBucketFileUrlDs): Promise { const { connectionId, tableName, fieldName, rowPrimaryKey, userId, masterPwd } = inputData; const user = await this._dbContext.userRepository.findOneUserByIdWithCompany(userId); @@ -45,7 +48,7 @@ export class GetS3FileUrlUseCase extends AbstractUseCase + extends AbstractUseCase implements IGetS3UploadUrl { constructor( @@ -25,7 +25,7 @@ export class GetS3UploadUrlUseCase super(); } - protected async implementation(inputData: S3GetUploadUrlDs): Promise { + protected async implementation(inputData: GetBucketUploadUrlDs): Promise { const { connectionId, tableName, fieldName, userId, masterPwd, filename, contentType } = inputData; const user = await this._dbContext.userRepository.findOneUserByIdWithCompany(userId); @@ -40,21 +40,21 @@ export class GetS3UploadUrlUseCase throw new HttpException({ message: 'S3 widget not configured for this field' }, HttpStatus.BAD_REQUEST); } - const params: S3WidgetParams = + const params: BucketWidgetParams = typeof widget.widget_params === 'string' ? JSON5.parse(widget.widget_params) : widget.widget_params; const accessKeySecret = await this._dbContext.userSecretRepository.findSecretBySlugAndCompanyId( - params.aws_access_key_id_secret_name, + params.access_key_id_secret_name, user.company.id, ); const secretKeySecret = await this._dbContext.userSecretRepository.findSecretBySlugAndCompanyId( - params.aws_secret_access_key_secret_name, + params.secret_access_key_secret_name, user.company.id, ); if (!accessKeySecret || !secretKeySecret) { - throw new HttpException({ message: 'AWS credentials secrets not found' }, HttpStatus.NOT_FOUND); + throw new HttpException({ message: 'Bucket credentials secrets not found' }, HttpStatus.NOT_FOUND); } let accessKeyId = Encryptor.decryptData(accessKeySecret.encryptedValue); @@ -67,7 +67,13 @@ export class GetS3UploadUrlUseCase secretAccessKey = Encryptor.decryptDataMasterPwd(secretAccessKey, masterPwd); } - const client = this.s3Helper.createS3Client(accessKeyId, secretAccessKey, params.region || 'us-east-1'); + const client = this.s3Helper.createS3Client({ + accessKeyId, + secretAccessKey, + provider: params.provider, + region: params.region, + accountId: params.account_id, + }); const key = this.s3Helper.generateFileKey(params.prefix, filename); const expiresIn = 3600; diff --git a/backend/src/entities/s3-widget/use-cases/s3-use-cases.interface.ts b/backend/src/entities/s3-widget/use-cases/s3-use-cases.interface.ts index f9ef13688..ee399503c 100644 --- a/backend/src/entities/s3-widget/use-cases/s3-use-cases.interface.ts +++ b/backend/src/entities/s3-widget/use-cases/s3-use-cases.interface.ts @@ -1,15 +1,15 @@ import { InTransactionEnum } from '../../../enums/in-transaction.enum.js'; import { - S3FileUrlResponseDs, - S3GetFileUrlDs, - S3GetUploadUrlDs, - S3UploadUrlResponseDs, + BucketFileUrlResponseDs, + BucketUploadUrlResponseDs, + GetBucketFileUrlDs, + GetBucketUploadUrlDs, } from '../application/data-structures/s3-operation.ds.js'; export interface IGetS3FileUrl { - execute(inputData: S3GetFileUrlDs, inTransaction: InTransactionEnum): Promise; + execute(inputData: GetBucketFileUrlDs, inTransaction: InTransactionEnum): Promise; } export interface IGetS3UploadUrl { - execute(inputData: S3GetUploadUrlDs, inTransaction: InTransactionEnum): Promise; + execute(inputData: GetBucketUploadUrlDs, inTransaction: InTransactionEnum): Promise; } diff --git a/backend/src/entities/widget/utils/validate-create-widgets-ds.ts b/backend/src/entities/widget/utils/validate-create-widgets-ds.ts index 13cd045f1..5239abd65 100644 --- a/backend/src/entities/widget/utils/validate-create-widgets-ds.ts +++ b/backend/src/entities/widget/utils/validate-create-widgets-ds.ts @@ -1,4 +1,5 @@ import JSON5 from 'json5'; +import { BucketProviderEnum } from '../../s3-widget/application/data-structures/bucket-provider.enum.js'; import { EncryptionAlgorithmEnum } from '../../../enums/encryption-algorithm.enum.js'; import { WidgetTypeEnum } from '../../../enums/widget-type.enum.js'; import { Messages } from '../../../exceptions/text/messages.js'; @@ -102,11 +103,20 @@ export async function validateCreateWidgetsDs( if (!widget_params.bucket) { errors.push(Messages.WIDGET_REQUIRED_PARAMETER_MISSING('bucket')); } - if (!widget_params.aws_access_key_id_secret_name) { - errors.push(Messages.WIDGET_REQUIRED_PARAMETER_MISSING('aws_access_key_id_secret_name')); + if (!widget_params.access_key_id_secret_name) { + errors.push(Messages.WIDGET_REQUIRED_PARAMETER_MISSING('access_key_id_secret_name')); } - if (!widget_params.aws_secret_access_key_secret_name) { - errors.push(Messages.WIDGET_REQUIRED_PARAMETER_MISSING('aws_secret_access_key_secret_name')); + if (!widget_params.secret_access_key_secret_name) { + errors.push(Messages.WIDGET_REQUIRED_PARAMETER_MISSING('secret_access_key_secret_name')); + } + if ( + widget_params.provider && + !Object.values(BucketProviderEnum).includes(widget_params.provider as BucketProviderEnum) + ) { + errors.push(Messages.WIDGET_PARAMETER_UNSUPPORTED('provider', widget_type)); + } + if (widget_params.provider === BucketProviderEnum.CloudflareR2 && !widget_params.account_id) { + errors.push(Messages.WIDGET_REQUIRED_PARAMETER_MISSING('account_id')); } } diff --git a/backend/test/ava-tests/non-saas-tests/non-saas-s3-widget-e2e.test.ts b/backend/test/ava-tests/non-saas-tests/non-saas-s3-widget-e2e.test.ts index 211932928..ae8c64588 100644 --- a/backend/test/ava-tests/non-saas-tests/non-saas-s3-widget-e2e.test.ts +++ b/backend/test/ava-tests/non-saas-tests/non-saas-s3-widget-e2e.test.ts @@ -87,11 +87,12 @@ async function setupS3WidgetTestEnvironment( // Create S3 widget for the field if (createWidget) { const s3WidgetParams = JSON.stringify({ + provider: 'aws', bucket: 'test-bucket', prefix: 'uploads', region: 'us-east-1', - aws_access_key_id_secret_name: 'test-aws-access-key', - aws_secret_access_key_secret_name: 'test-aws-secret-key', + access_key_id_secret_name: 'test-aws-access-key', + secret_access_key_secret_name: 'test-aws-secret-key', }); const widgetDto = { @@ -338,7 +339,7 @@ test.serial(`${currentTest} - should return 404 when AWS credentials secrets are t.is(response.status, 404); const responseBody = JSON.parse(response.text); - t.is(responseBody.message, 'AWS credentials secrets not found'); + t.is(responseBody.message, 'Bucket credentials secrets not found'); }); // ==================== POST /s3/upload-url/:connectionId Tests ==================== @@ -477,7 +478,7 @@ test.serial(`${currentTest} - should return 404 when AWS credentials secrets are t.is(response.status, 404); const responseBody = JSON.parse(response.text); - t.is(responseBody.message, 'AWS credentials secrets not found'); + t.is(responseBody.message, 'Bucket credentials secrets not found'); }); // ==================== Authorization Tests - Multiple Users ==================== @@ -564,3 +565,185 @@ test.serial(`${currentTest} - unauthenticated user cannot access S3 endpoints`, t.is(uploadUrlResponse.status, 401); }); + +// ==================== Bucket Provider Validation Tests ==================== + +currentTest = 'Bucket Provider Validation'; + +async function createConnectionAndTableWithS3Field( + s3FieldName: string = 'file_key', +): Promise<{ token: string; connectionId: string; testTableName: string }> { + const { token } = await registerUserAndReturnUserInfo(app); + const connectionToPostgres = getTestData(mockFactory).connectionToPostgres; + const { testTableName } = await createTestTableWithS3Field(connectionToPostgres, s3FieldName); + + const createdConnection = await request(app.getHttpServer()) + .post('/connection') + .send(connectionToPostgres) + .set('Cookie', token) + .set('masterpwd', 'ahalaimahalai') + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + const connectionId = JSON.parse(createdConnection.text).id; + return { token, connectionId, testTableName }; +} + +test.serial(`${currentTest} - widget creation succeeds with provider 'digitalocean'`, async (t) => { + const { token, connectionId, testTableName } = await createConnectionAndTableWithS3Field(); + + const widgetParams = JSON.stringify({ + provider: 'digitalocean', + bucket: 'test-bucket', + region: 'fra1', + access_key_id_secret_name: 'do-key', + secret_access_key_secret_name: 'do-secret', + }); + + const response = await request(app.getHttpServer()) + .post(`/widget/${connectionId}?tableName=${testTableName}`) + .send({ + widgets: [ + { + field_name: 'file_key', + widget_type: 'S3', + widget_params: widgetParams, + name: 'DO Spaces Widget', + description: 'DigitalOcean Spaces', + }, + ], + }) + .set('Cookie', token) + .set('masterpwd', 'ahalaimahalai') + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + t.is(response.status, 201); +}); + +test.serial(`${currentTest} - widget creation rejected with unsupported provider`, async (t) => { + const { token, connectionId, testTableName } = await createConnectionAndTableWithS3Field(); + + const widgetParams = JSON.stringify({ + provider: 'not-a-real-provider', + bucket: 'test-bucket', + region: 'us-east-1', + access_key_id_secret_name: 'k', + secret_access_key_secret_name: 's', + }); + + const response = await request(app.getHttpServer()) + .post(`/widget/${connectionId}?tableName=${testTableName}`) + .send({ + widgets: [ + { + field_name: 'file_key', + widget_type: 'S3', + widget_params: widgetParams, + name: 'Bad Provider Widget', + description: '', + }, + ], + }) + .set('Cookie', token) + .set('masterpwd', 'ahalaimahalai') + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + t.is(response.status, 400); +}); + +test.serial(`${currentTest} - widget creation rejected when cloudflare-r2 missing account_id`, async (t) => { + const { token, connectionId, testTableName } = await createConnectionAndTableWithS3Field(); + + const widgetParams = JSON.stringify({ + provider: 'cloudflare-r2', + bucket: 'test-bucket', + region: 'auto', + access_key_id_secret_name: 'k', + secret_access_key_secret_name: 's', + }); + + const response = await request(app.getHttpServer()) + .post(`/widget/${connectionId}?tableName=${testTableName}`) + .send({ + widgets: [ + { + field_name: 'file_key', + widget_type: 'S3', + widget_params: widgetParams, + name: 'R2 Widget', + description: '', + }, + ], + }) + .set('Cookie', token) + .set('masterpwd', 'ahalaimahalai') + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + t.is(response.status, 400); +}); + +test.serial(`${currentTest} - widget creation succeeds with cloudflare-r2 and account_id`, async (t) => { + const { token, connectionId, testTableName } = await createConnectionAndTableWithS3Field(); + + const widgetParams = JSON.stringify({ + provider: 'cloudflare-r2', + bucket: 'test-bucket', + region: 'auto', + account_id: 'abc123def456', + access_key_id_secret_name: 'k', + secret_access_key_secret_name: 's', + }); + + const response = await request(app.getHttpServer()) + .post(`/widget/${connectionId}?tableName=${testTableName}`) + .send({ + widgets: [ + { + field_name: 'file_key', + widget_type: 'S3', + widget_params: widgetParams, + name: 'R2 Widget', + description: '', + }, + ], + }) + .set('Cookie', token) + .set('masterpwd', 'ahalaimahalai') + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + t.is(response.status, 201); +}); + +test.serial(`${currentTest} - widget creation rejected when access_key_id_secret_name missing`, async (t) => { + const { token, connectionId, testTableName } = await createConnectionAndTableWithS3Field(); + + const widgetParams = JSON.stringify({ + bucket: 'test-bucket', + region: 'us-east-1', + secret_access_key_secret_name: 's', + }); + + const response = await request(app.getHttpServer()) + .post(`/widget/${connectionId}?tableName=${testTableName}`) + .send({ + widgets: [ + { + field_name: 'file_key', + widget_type: 'S3', + widget_params: widgetParams, + name: 'Missing Key Widget', + description: '', + }, + ], + }) + .set('Cookie', token) + .set('masterpwd', 'ahalaimahalai') + .set('Content-Type', 'application/json') + .set('Accept', 'application/json'); + + t.is(response.status, 400); +}); diff --git a/frontend/src/app/components/dashboard/db-table-view/db-table-widgets/db-table-widgets.component.ts b/frontend/src/app/components/dashboard/db-table-view/db-table-widgets/db-table-widgets.component.ts index 9511bf2f1..6a2944162 100644 --- a/frontend/src/app/components/dashboard/db-table-view/db-table-widgets/db-table-widgets.component.ts +++ b/frontend/src/app/components/dashboard/db-table-view/db-table-widgets/db-table-widgets.component.ts @@ -269,24 +269,32 @@ export class DbTableWidgetsComponent implements OnInit { } `, Email: `// No settings required`, - S3: `// Configure AWS S3 widget for file storage -// bucket: S3 bucket name (required) + S3: `// Configure S3-compatible bucket widget for file storage +// provider: Cloud provider. Supported values: +// "aws" - AWS S3 (default) +// "digitalocean" - DigitalOcean Spaces +// "backblaze" - Backblaze B2 +// "wasabi" - Wasabi +// "cloudflare-r2" - Cloudflare R2 (requires "account_id") +// bucket: bucket name (required) // prefix: Optional path prefix for uploaded files -// region: AWS region (default: us-east-1) +// region: Region for the bucket (default: us-east-1; for cloudflare-r2 use "auto") +// account_id: Cloudflare account ID. Required when provider is "cloudflare-r2". // type: "file" (default) - accepts all file types, "image" - accepts only images -// aws_access_key_id_secret_name: Unique identifier of the secret containing AWS Access Key ID -// aws_secret_access_key_secret_name: Unique identifier of the secret containing AWS Secret Access Key +// access_key_id_secret_name: Unique identifier of the secret containing the Access Key ID +// secret_access_key_secret_name: Unique identifier of the secret containing the Secret Access Key // -// ⚠️ IMPORTANT: DO NOT INCLUDE AWS SECRETS DIRECTLY IN WIDGET SETTINGS! -// Store your AWS credentials as secrets in Rocketadmin and reference them here by their unique identifiers to ensure security and prevent exposure of sensitive information. +// ⚠️ IMPORTANT: DO NOT INCLUDE BUCKET SECRETS DIRECTLY IN WIDGET SETTINGS! +// Store your credentials as secrets in Rocketadmin and reference them here by their unique identifiers to ensure security and prevent exposure of sensitive information. { + "provider": "aws", "bucket": "your-bucket-name", "prefix": "uploads/", "region": "us-east-1", "type": "file", - "aws_access_key_id_secret_name": "aws-access-key-id", - "aws_secret_access_key_secret_name": "aws-secret-access-key" + "access_key_id_secret_name": "bucket-access-key-id", + "secret_access_key_secret_name": "bucket-secret-access-key" } `, }; diff --git a/frontend/src/app/components/ui-components/record-edit-fields/s3/s3.component.spec.ts b/frontend/src/app/components/ui-components/record-edit-fields/s3/s3.component.spec.ts index 39841937b..34b146344 100644 --- a/frontend/src/app/components/ui-components/record-edit-fields/s3/s3.component.spec.ts +++ b/frontend/src/app/components/ui-components/record-edit-fields/s3/s3.component.spec.ts @@ -18,11 +18,12 @@ describe('S3EditComponent', () => { field_name: 'document', widget_type: 'S3', widget_params: { + provider: 'aws', bucket: 'test-bucket', prefix: 'uploads/', region: 'us-east-1', - aws_access_key_id_secret_name: 'aws-key', - aws_secret_access_key_secret_name: 'aws-secret', + access_key_id_secret_name: 'bucket-key', + secret_access_key_secret_name: 'bucket-secret', }, name: 'Document Upload', description: 'Upload documents to S3', @@ -32,11 +33,12 @@ describe('S3EditComponent', () => { field_name: 'document', widget_type: 'S3', widget_params: JSON.stringify({ + provider: 'aws', bucket: 'test-bucket', prefix: 'uploads/', region: 'us-east-1', - aws_access_key_id_secret_name: 'aws-key', - aws_secret_access_key_secret_name: 'aws-secret', + access_key_id_secret_name: 'bucket-key', + secret_access_key_secret_name: 'bucket-secret', }) as any, name: 'Document Upload', description: 'Upload documents to S3', @@ -107,11 +109,12 @@ describe('S3EditComponent', () => { fixture.detectChanges(); expect(component.params()).toEqual({ + provider: 'aws', bucket: 'test-bucket', prefix: 'uploads/', region: 'us-east-1', - aws_access_key_id_secret_name: 'aws-key', - aws_secret_access_key_secret_name: 'aws-secret', + access_key_id_secret_name: 'bucket-key', + secret_access_key_secret_name: 'bucket-secret', }); }); diff --git a/frontend/src/app/components/ui-components/record-edit-fields/s3/s3.component.ts b/frontend/src/app/components/ui-components/record-edit-fields/s3/s3.component.ts index b1bb53888..6e07b9461 100644 --- a/frontend/src/app/components/ui-components/record-edit-fields/s3/s3.component.ts +++ b/frontend/src/app/components/ui-components/record-edit-fields/s3/s3.component.ts @@ -12,12 +12,16 @@ import { ConnectionsService } from 'src/app/services/connections.service'; import { S3Service } from 'src/app/services/s3.service'; import { TablesService } from 'src/app/services/tables.service'; -interface S3WidgetParams { +type BucketProvider = 'aws' | 'digitalocean' | 'backblaze' | 'wasabi' | 'cloudflare-r2'; + +interface BucketWidgetParams { + provider?: BucketProvider; bucket: string; prefix?: string; region?: string; - aws_access_key_id_secret_name: string; - aws_secret_access_key_secret_name: string; + account_id?: string; + access_key_id_secret_name: string; + secret_access_key_secret_name: string; type?: 'file' | 'image'; } @@ -59,7 +63,7 @@ export class S3EditComponent implements OnInit { // Computed signals readonly normalizedLabel = computed(() => normalizeFieldName(this.label() || '')); - readonly params = computed(() => { + readonly params = computed(() => { const ws = this.widgetStructure(); if (!ws?.widget_params) return null; try { diff --git a/frontend/src/app/components/ui-components/record-view-fields/s3/s3.component.spec.ts b/frontend/src/app/components/ui-components/record-view-fields/s3/s3.component.spec.ts index 089b3ccb6..909dc13c0 100644 --- a/frontend/src/app/components/ui-components/record-view-fields/s3/s3.component.spec.ts +++ b/frontend/src/app/components/ui-components/record-view-fields/s3/s3.component.spec.ts @@ -44,8 +44,8 @@ describe('S3RecordViewComponent', () => { widget_params: { bucket: 'my-bucket', type: 'file', - aws_access_key_id_secret_name: 'key', - aws_secret_access_key_secret_name: 'secret', + access_key_id_secret_name: 'key', + secret_access_key_secret_name: 'secret', }, }); component.ngOnInit(); @@ -58,8 +58,8 @@ describe('S3RecordViewComponent', () => { widget_params: { bucket: 'my-bucket', type: 'image', - aws_access_key_id_secret_name: 'key', - aws_secret_access_key_secret_name: 'secret', + access_key_id_secret_name: 'key', + secret_access_key_secret_name: 'secret', }, }); component.ngOnInit(); diff --git a/frontend/src/app/components/ui-components/record-view-fields/s3/s3.component.ts b/frontend/src/app/components/ui-components/record-view-fields/s3/s3.component.ts index 9682f30f6..f2520e565 100644 --- a/frontend/src/app/components/ui-components/record-view-fields/s3/s3.component.ts +++ b/frontend/src/app/components/ui-components/record-view-fields/s3/s3.component.ts @@ -5,12 +5,16 @@ import { S3Service } from 'src/app/services/s3.service'; import { TablesService } from 'src/app/services/tables.service'; import { BaseRecordViewFieldComponent } from '../base-record-view-field/base-record-view-field.component'; -interface S3WidgetParams { +type BucketProvider = 'aws' | 'digitalocean' | 'backblaze' | 'wasabi' | 'cloudflare-r2'; + +interface BucketWidgetParams { + provider?: BucketProvider; bucket: string; prefix?: string; region?: string; - aws_access_key_id_secret_name: string; - aws_secret_access_key_secret_name: string; + account_id?: string; + access_key_id_secret_name: string; + secret_access_key_secret_name: string; type?: 'file' | 'image'; } @@ -21,7 +25,7 @@ interface S3WidgetParams { imports: [MatProgressSpinnerModule], }) export class S3RecordViewComponent extends BaseRecordViewFieldComponent implements OnInit { - public params: S3WidgetParams; + public params: BucketWidgetParams; public previewUrl: string | null = null; public isLoading: boolean = false; diff --git a/frontend/src/app/components/ui-components/table-display-fields/s3/s3.component.spec.ts b/frontend/src/app/components/ui-components/table-display-fields/s3/s3.component.spec.ts index 4ebc1c424..bb1a93dfe 100644 --- a/frontend/src/app/components/ui-components/table-display-fields/s3/s3.component.spec.ts +++ b/frontend/src/app/components/ui-components/table-display-fields/s3/s3.component.spec.ts @@ -46,8 +46,8 @@ describe('S3DisplayComponent', () => { fixture.componentRef.setInput('widgetStructure', { widget_params: { bucket: 'my-bucket', - aws_access_key_id_secret_name: 'key', - aws_secret_access_key_secret_name: 'secret', + access_key_id_secret_name: 'key', + secret_access_key_secret_name: 'secret', type: 'image', }, }); diff --git a/frontend/src/app/components/ui-components/table-display-fields/s3/s3.component.ts b/frontend/src/app/components/ui-components/table-display-fields/s3/s3.component.ts index 95630e55d..474735b6c 100644 --- a/frontend/src/app/components/ui-components/table-display-fields/s3/s3.component.ts +++ b/frontend/src/app/components/ui-components/table-display-fields/s3/s3.component.ts @@ -9,12 +9,16 @@ import { S3Service } from 'src/app/services/s3.service'; import { TablesService } from 'src/app/services/tables.service'; import { BaseTableDisplayFieldComponent } from '../base-table-display-field/base-table-display-field.component'; -interface S3WidgetParams { +type BucketProvider = 'aws' | 'digitalocean' | 'backblaze' | 'wasabi' | 'cloudflare-r2'; + +interface BucketWidgetParams { + provider?: BucketProvider; bucket: string; prefix?: string; region?: string; - aws_access_key_id_secret_name: string; - aws_secret_access_key_secret_name: string; + account_id?: string; + access_key_id_secret_name: string; + secret_access_key_secret_name: string; type?: 'file' | 'image'; } @@ -25,7 +29,7 @@ interface S3WidgetParams { imports: [ClipboardModule, MatButtonModule, MatIconModule, MatProgressSpinnerModule, MatTooltipModule], }) export class S3DisplayComponent extends BaseTableDisplayFieldComponent implements OnInit { - public params: S3WidgetParams; + public params: BucketWidgetParams; public previewUrl: string | null = null; public isLoading: boolean = false; diff --git a/frontend/src/app/services/s3.service.ts b/frontend/src/app/services/s3.service.ts index 1f21cdcf4..459d921a6 100644 --- a/frontend/src/app/services/s3.service.ts +++ b/frontend/src/app/services/s3.service.ts @@ -4,13 +4,13 @@ import { firstValueFrom } from 'rxjs'; import { AlertActionType, AlertType } from '../models/alert'; import { NotificationsService } from './notifications.service'; -interface S3FileUrlResponse { +interface BucketFileUrlResponse { url: string; key: string; expiresIn: number; } -interface S3UploadUrlResponse { +interface BucketUploadUrlResponse { uploadUrl: string; key: string; expiresIn: number; @@ -29,10 +29,10 @@ export class S3Service { tableName: string, fieldName: string, rowPrimaryKey: Record, - ): Promise { + ): Promise { try { return await firstValueFrom( - this.http.get(`/s3/file/${connectionId}`, { + this.http.get(`/s3/file/${connectionId}`, { params: { tableName, fieldName, @@ -52,10 +52,10 @@ export class S3Service { fieldName: string, filename: string, contentType: string, - ): Promise { + ): Promise { try { return await firstValueFrom( - this.http.post( + this.http.post( `/s3/upload-url/${connectionId}`, { filename, contentType }, { params: { tableName, fieldName } }, @@ -93,7 +93,7 @@ export class S3Service { ): Promise<{ key: string; previewUrl: string } | null> { try { const response = await firstValueFrom( - this.http.post( + this.http.post( `/s3/upload-url/${connectionId}`, { filename: file.name, contentType: file.type }, { params: { tableName, fieldName } },