Skip to content

Commit b08fd0b

Browse files
ArtuomkaArtuomka
authored
Merge pull request #1760 from rocket-admin/s3-bucket-providers
feat: extend S3 widget with multi-provider support
2 parents 53d5873 + 1cdc90c commit b08fd0b

18 files changed

Lines changed: 380 additions & 87 deletions

File tree

‎backend/src/entities/s3-widget/application/data-structures/bucket-provider.enum.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
export enum BucketProviderEnum {
2+
AWS = 'aws',
3+
DigitalOceanSpaces = 'digitalocean',
4+
BackblazeB2 = 'backblaze',
5+
Wasabi = 'wasabi',
6+
CloudflareR2 = 'cloudflare-r2',
7+
}

‎backend/src/entities/s3-widget/application/data-structures/s3-operation.ds.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
export class S3GetFileUrlDs {
1+
export class GetBucketFileUrlDs {
22
connectionId: string;
33
tableName: string;
44
fieldName: string;
@@ -7,7 +7,7 @@ export class S3GetFileUrlDs {
77
masterPwd: string;
88
}
99

10-
export class S3GetUploadUrlDs {
10+
export class GetBucketUploadUrlDs {
1111
connectionId: string;
1212
tableName: string;
1313
fieldName: string;
@@ -17,13 +17,13 @@ export class S3GetUploadUrlDs {
1717
contentType: string;
1818
}
1919

20-
export class S3FileUrlResponseDs {
20+
export class BucketFileUrlResponseDs {
2121
url: string;
2222
key: string;
2323
expiresIn: number;
2424
}
2525

26-
export class S3UploadUrlResponseDs {
26+
export class BucketUploadUrlResponseDs {
2727
uploadUrl: string;
2828
key: string;
2929
expiresIn: number;

‎backend/src/entities/s3-widget/application/data-structures/s3-widget-params.ds.ts‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
1-
export interface S3WidgetParams {
1+
import { BucketProviderEnum } from './bucket-provider.enum.js';
2+
3+
export interface BucketWidgetParams {
4+
provider?: BucketProviderEnum;
25
bucket: string;
36
prefix?: string;
47
region?: string;
5-
aws_access_key_id_secret_name: string;
6-
aws_secret_access_key_secret_name: string;
7-
type?: 'file' | 'image'; // 'file' (default) - accepts all files, 'image' - accepts only images
8+
account_id?: string;
9+
access_key_id_secret_name: string;
10+
secret_access_key_secret_name: string;
11+
type?: 'file' | 'image';
812
}

‎backend/src/entities/s3-widget/s3-helper.service.ts‎

Lines changed: 55 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,30 @@
11
import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
22
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
3-
import { Injectable } from '@nestjs/common';
3+
import { HttpException, HttpStatus, Injectable } from '@nestjs/common';
44
import { nanoid } from 'nanoid';
5+
import { BucketProviderEnum } from './application/data-structures/bucket-provider.enum.js';
6+
7+
export interface BucketClientConfig {
8+
accessKeyId: string;
9+
secretAccessKey: string;
10+
provider?: BucketProviderEnum;
11+
region?: string;
12+
accountId?: string;
13+
}
514

615
@Injectable()
716
export class S3HelperService {
8-
public createS3Client(accessKeyId: string, secretAccessKey: string, region: string = 'us-east-1'): S3Client {
17+
public createS3Client(config: BucketClientConfig): S3Client {
18+
const provider = config.provider || BucketProviderEnum.AWS;
19+
const region = this._resolveRegion(provider, config.region);
20+
const endpoint = this._resolveEndpoint(provider, region, config.accountId);
21+
922
return new S3Client({
1023
region,
24+
endpoint,
1125
credentials: {
12-
accessKeyId,
13-
secretAccessKey,
26+
accessKeyId: config.accessKeyId,
27+
secretAccessKey: config.secretAccessKey,
1428
},
1529
});
1630
}
@@ -52,6 +66,43 @@ export class S3HelperService {
5266
return key;
5367
}
5468

69+
private _resolveRegion(provider: BucketProviderEnum, region: string | undefined): string {
70+
if (region) {
71+
return region;
72+
}
73+
if (provider === BucketProviderEnum.CloudflareR2) {
74+
return 'auto';
75+
}
76+
return 'us-east-1';
77+
}
78+
79+
private _resolveEndpoint(
80+
provider: BucketProviderEnum,
81+
region: string,
82+
accountId: string | undefined,
83+
): string | undefined {
84+
switch (provider) {
85+
case BucketProviderEnum.AWS:
86+
return undefined;
87+
case BucketProviderEnum.DigitalOceanSpaces:
88+
return `https://${region}.digitaloceanspaces.com`;
89+
case BucketProviderEnum.BackblazeB2:
90+
return `https://s3.${region}.backblazeb2.com`;
91+
case BucketProviderEnum.Wasabi:
92+
return `https://s3.${region}.wasabisys.com`;
93+
case BucketProviderEnum.CloudflareR2:
94+
if (!accountId) {
95+
throw new HttpException(
96+
{ message: 'Cloudflare R2 requires account_id in widget params' },
97+
HttpStatus.BAD_REQUEST,
98+
);
99+
}
100+
return `https://${accountId}.r2.cloudflarestorage.com`;
101+
default:
102+
return undefined;
103+
}
104+
}
105+
55106
private _extractFileExtension(filename: string): string {
56107
const lastDotIndex = filename.lastIndexOf('.');
57108
if (lastDotIndex === -1 || lastDotIndex === 0) {

‎backend/src/entities/s3-widget/s3-widget.controller.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import { Messages } from '../../exceptions/text/messages.js';
2323
import { ConnectionEditGuard } from '../../guards/connection-edit.guard.js';
2424
import { ConnectionReadGuard } from '../../guards/connection-read.guard.js';
2525
import { SentryInterceptor } from '../../interceptors/sentry.interceptor.js';
26-
import { S3FileUrlResponseDs, S3UploadUrlResponseDs } from './application/data-structures/s3-operation.ds.js';
26+
import { BucketFileUrlResponseDs, BucketUploadUrlResponseDs } from './application/data-structures/s3-operation.ds.js';
2727
import { IGetS3FileUrl, IGetS3UploadUrl } from './use-cases/s3-use-cases.interface.js';
2828

2929
@UseInterceptors(SentryInterceptor)
@@ -61,7 +61,7 @@ export class S3WidgetController {
6161
@QueryTableName() tableName: string,
6262
@Query('fieldName') fieldName: string,
6363
@Query('rowPrimaryKey') rowPrimaryKeyStr: string,
64-
): Promise<S3FileUrlResponseDs> {
64+
): Promise<BucketFileUrlResponseDs> {
6565
if (!connectionId) {
6666
throw new HttpException({ message: Messages.CONNECTION_ID_MISSING }, HttpStatus.BAD_REQUEST);
6767
}
@@ -118,7 +118,7 @@ export class S3WidgetController {
118118
@QueryTableName() tableName: string,
119119
@Query('fieldName') fieldName: string,
120120
@Body() body: { filename: string; contentType: string },
121-
): Promise<S3UploadUrlResponseDs> {
121+
): Promise<BucketUploadUrlResponseDs> {
122122
if (!connectionId) {
123123
throw new HttpException({ message: Messages.CONNECTION_ID_MISSING }, HttpStatus.BAD_REQUEST);
124124
}

‎backend/src/entities/s3-widget/use-cases/get-s3-file-url.use.case.ts‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,16 @@ import { WidgetTypeEnum } from '../../../enums/widget-type.enum.js';
1010
import { Messages } from '../../../exceptions/text/messages.js';
1111
import { Encryptor } from '../../../helpers/encryption/encryptor.js';
1212
import { isConnectionTypeAgent } from '../../../helpers/is-connection-entity-agent.js';
13-
import { S3FileUrlResponseDs, S3GetFileUrlDs } from '../application/data-structures/s3-operation.ds.js';
14-
import { S3WidgetParams } from '../application/data-structures/s3-widget-params.ds.js';
13+
import { BucketFileUrlResponseDs, GetBucketFileUrlDs } from '../application/data-structures/s3-operation.ds.js';
14+
import { BucketWidgetParams } from '../application/data-structures/s3-widget-params.ds.js';
1515
import { S3HelperService } from '../s3-helper.service.js';
1616
import { IGetS3FileUrl } from './s3-use-cases.interface.js';
1717

1818
@Injectable()
19-
export class GetS3FileUrlUseCase extends AbstractUseCase<S3GetFileUrlDs, S3FileUrlResponseDs> implements IGetS3FileUrl {
19+
export class GetS3FileUrlUseCase
20+
extends AbstractUseCase<GetBucketFileUrlDs, BucketFileUrlResponseDs>
21+
implements IGetS3FileUrl
22+
{
2023
constructor(
2124
@Inject(BaseType.GLOBAL_DB_CONTEXT)
2225
protected _dbContext: IGlobalDatabaseContext,
@@ -25,7 +28,7 @@ export class GetS3FileUrlUseCase extends AbstractUseCase<S3GetFileUrlDs, S3FileU
2528
super();
2629
}
2730

28-
protected async implementation(inputData: S3GetFileUrlDs): Promise<S3FileUrlResponseDs> {
31+
protected async implementation(inputData: GetBucketFileUrlDs): Promise<BucketFileUrlResponseDs> {
2932
const { connectionId, tableName, fieldName, rowPrimaryKey, userId, masterPwd } = inputData;
3033

3134
const user = await this._dbContext.userRepository.findOneUserByIdWithCompany(userId);
@@ -45,7 +48,7 @@ export class GetS3FileUrlUseCase extends AbstractUseCase<S3GetFileUrlDs, S3FileU
4548
throw new HttpException({ message: 'S3 widget not configured for this field' }, HttpStatus.BAD_REQUEST);
4649
}
4750

48-
const params: S3WidgetParams =
51+
const params: BucketWidgetParams =
4952
typeof widget.widget_params === 'string' ? JSON5.parse(widget.widget_params) : widget.widget_params;
5053

5154
// Fetch the row from database to get the actual file key
@@ -78,17 +81,17 @@ export class GetS3FileUrlUseCase extends AbstractUseCase<S3GetFileUrlDs, S3FileU
7881
}
7982

8083
const accessKeySecret = await this._dbContext.userSecretRepository.findSecretBySlugAndCompanyId(
81-
params.aws_access_key_id_secret_name,
84+
params.access_key_id_secret_name,
8285
user.company.id,
8386
);
8487

8588
const secretKeySecret = await this._dbContext.userSecretRepository.findSecretBySlugAndCompanyId(
86-
params.aws_secret_access_key_secret_name,
89+
params.secret_access_key_secret_name,
8790
user.company.id,
8891
);
8992

9093
if (!accessKeySecret || !secretKeySecret) {
91-
throw new HttpException({ message: 'AWS credentials secrets not found' }, HttpStatus.NOT_FOUND);
94+
throw new HttpException({ message: 'Bucket credentials secrets not found' }, HttpStatus.NOT_FOUND);
9295
}
9396

9497
let accessKeyId = Encryptor.decryptData(accessKeySecret.encryptedValue);
@@ -101,7 +104,13 @@ export class GetS3FileUrlUseCase extends AbstractUseCase<S3GetFileUrlDs, S3FileU
101104
secretAccessKey = Encryptor.decryptDataMasterPwd(secretAccessKey, masterPwd);
102105
}
103106

104-
const client = this.s3Helper.createS3Client(accessKeyId, secretAccessKey, params.region || 'us-east-1');
107+
const client = this.s3Helper.createS3Client({
108+
accessKeyId,
109+
secretAccessKey,
110+
provider: params.provider,
111+
region: params.region,
112+
accountId: params.account_id,
113+
});
105114

106115
const expiresIn = 3600;
107116
const url = await this.s3Helper.getSignedGetUrl(client, params.bucket, fileKey, expiresIn);

‎backend/src/entities/s3-widget/use-cases/get-s3-upload-url.use.case.ts‎

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,14 @@ import { BaseType } from '../../../common/data-injection.tokens.js';
77
import { WidgetTypeEnum } from '../../../enums/widget-type.enum.js';
88
import { Messages } from '../../../exceptions/text/messages.js';
99
import { Encryptor } from '../../../helpers/encryption/encryptor.js';
10-
import { S3GetUploadUrlDs, S3UploadUrlResponseDs } from '../application/data-structures/s3-operation.ds.js';
11-
import { S3WidgetParams } from '../application/data-structures/s3-widget-params.ds.js';
10+
import { BucketUploadUrlResponseDs, GetBucketUploadUrlDs } from '../application/data-structures/s3-operation.ds.js';
11+
import { BucketWidgetParams } from '../application/data-structures/s3-widget-params.ds.js';
1212
import { S3HelperService } from '../s3-helper.service.js';
1313
import { IGetS3UploadUrl } from './s3-use-cases.interface.js';
1414

1515
@Injectable()
1616
export class GetS3UploadUrlUseCase
17-
extends AbstractUseCase<S3GetUploadUrlDs, S3UploadUrlResponseDs>
17+
extends AbstractUseCase<GetBucketUploadUrlDs, BucketUploadUrlResponseDs>
1818
implements IGetS3UploadUrl
1919
{
2020
constructor(
@@ -25,7 +25,7 @@ export class GetS3UploadUrlUseCase
2525
super();
2626
}
2727

28-
protected async implementation(inputData: S3GetUploadUrlDs): Promise<S3UploadUrlResponseDs> {
28+
protected async implementation(inputData: GetBucketUploadUrlDs): Promise<BucketUploadUrlResponseDs> {
2929
const { connectionId, tableName, fieldName, userId, masterPwd, filename, contentType } = inputData;
3030

3131
const user = await this._dbContext.userRepository.findOneUserByIdWithCompany(userId);
@@ -40,21 +40,21 @@ export class GetS3UploadUrlUseCase
4040
throw new HttpException({ message: 'S3 widget not configured for this field' }, HttpStatus.BAD_REQUEST);
4141
}
4242

43-
const params: S3WidgetParams =
43+
const params: BucketWidgetParams =
4444
typeof widget.widget_params === 'string' ? JSON5.parse(widget.widget_params) : widget.widget_params;
4545

4646
const accessKeySecret = await this._dbContext.userSecretRepository.findSecretBySlugAndCompanyId(
47-
params.aws_access_key_id_secret_name,
47+
params.access_key_id_secret_name,
4848
user.company.id,
4949
);
5050

5151
const secretKeySecret = await this._dbContext.userSecretRepository.findSecretBySlugAndCompanyId(
52-
params.aws_secret_access_key_secret_name,
52+
params.secret_access_key_secret_name,
5353
user.company.id,
5454
);
5555

5656
if (!accessKeySecret || !secretKeySecret) {
57-
throw new HttpException({ message: 'AWS credentials secrets not found' }, HttpStatus.NOT_FOUND);
57+
throw new HttpException({ message: 'Bucket credentials secrets not found' }, HttpStatus.NOT_FOUND);
5858
}
5959

6060
let accessKeyId = Encryptor.decryptData(accessKeySecret.encryptedValue);
@@ -67,7 +67,13 @@ export class GetS3UploadUrlUseCase
6767
secretAccessKey = Encryptor.decryptDataMasterPwd(secretAccessKey, masterPwd);
6868
}
6969

70-
const client = this.s3Helper.createS3Client(accessKeyId, secretAccessKey, params.region || 'us-east-1');
70+
const client = this.s3Helper.createS3Client({
71+
accessKeyId,
72+
secretAccessKey,
73+
provider: params.provider,
74+
region: params.region,
75+
accountId: params.account_id,
76+
});
7177

7278
const key = this.s3Helper.generateFileKey(params.prefix, filename);
7379
const expiresIn = 3600;

‎backend/src/entities/s3-widget/use-cases/s3-use-cases.interface.ts‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
import { InTransactionEnum } from '../../../enums/in-transaction.enum.js';
22
import {
3-
S3FileUrlResponseDs,
4-
S3GetFileUrlDs,
5-
S3GetUploadUrlDs,
6-
S3UploadUrlResponseDs,
3+
BucketFileUrlResponseDs,
4+
BucketUploadUrlResponseDs,
5+
GetBucketFileUrlDs,
6+
GetBucketUploadUrlDs,
77
} from '../application/data-structures/s3-operation.ds.js';
88

99
export interface IGetS3FileUrl {
10-
execute(inputData: S3GetFileUrlDs, inTransaction: InTransactionEnum): Promise<S3FileUrlResponseDs>;
10+
execute(inputData: GetBucketFileUrlDs, inTransaction: InTransactionEnum): Promise<BucketFileUrlResponseDs>;
1111
}
1212

1313
export interface IGetS3UploadUrl {
14-
execute(inputData: S3GetUploadUrlDs, inTransaction: InTransactionEnum): Promise<S3UploadUrlResponseDs>;
14+
execute(inputData: GetBucketUploadUrlDs, inTransaction: InTransactionEnum): Promise<BucketUploadUrlResponseDs>;
1515
}

‎backend/src/entities/widget/utils/validate-create-widgets-ds.ts‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import JSON5 from 'json5';
2+
import { BucketProviderEnum } from '../../s3-widget/application/data-structures/bucket-provider.enum.js';
23
import { EncryptionAlgorithmEnum } from '../../../enums/encryption-algorithm.enum.js';
34
import { WidgetTypeEnum } from '../../../enums/widget-type.enum.js';
45
import { Messages } from '../../../exceptions/text/messages.js';
@@ -102,11 +103,20 @@ export async function validateCreateWidgetsDs(
102103
if (!widget_params.bucket) {
103104
errors.push(Messages.WIDGET_REQUIRED_PARAMETER_MISSING('bucket'));
104105
}
105-
if (!widget_params.aws_access_key_id_secret_name) {
106-
errors.push(Messages.WIDGET_REQUIRED_PARAMETER_MISSING('aws_access_key_id_secret_name'));
106+
if (!widget_params.access_key_id_secret_name) {
107+
errors.push(Messages.WIDGET_REQUIRED_PARAMETER_MISSING('access_key_id_secret_name'));
107108
}
108-
if (!widget_params.aws_secret_access_key_secret_name) {
109-
errors.push(Messages.WIDGET_REQUIRED_PARAMETER_MISSING('aws_secret_access_key_secret_name'));
109+
if (!widget_params.secret_access_key_secret_name) {
110+
errors.push(Messages.WIDGET_REQUIRED_PARAMETER_MISSING('secret_access_key_secret_name'));
111+
}
112+
if (
113+
widget_params.provider &&
114+
!Object.values(BucketProviderEnum).includes(widget_params.provider as BucketProviderEnum)
115+
) {
116+
errors.push(Messages.WIDGET_PARAMETER_UNSUPPORTED('provider', widget_type));
117+
}
118+
if (widget_params.provider === BucketProviderEnum.CloudflareR2 && !widget_params.account_id) {
119+
errors.push(Messages.WIDGET_REQUIRED_PARAMETER_MISSING('account_id'));
110120
}
111121
}
112122

0 commit comments

Comments
 (0)