Skip to content

Commit 2b6d22f

Browse files
mish-ellen1ru4l
andauthored
feat: support for S3 AWS IAM auth (#8079)
Co-authored-by: Laurin Quast <laurinquast@googlemail.com>
1 parent a5d454d commit 2b6d22f

15 files changed

Lines changed: 1175 additions & 347 deletions

File tree

.changeset/jolly-chefs-behave.md

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
---
2+
'hive': minor
3+
---
4+
5+
Added opt-in AWS IAM authentication for S3 connections. When IAM is enabled, services authenticate
6+
to S3 using short-lived SigV4 pre-signed tokens instead of static passwords, since S3 connections
7+
are HTTP requests a new token will be generate for each call.
8+
9+
### New environment variables
10+
11+
| Variable | Service | Description |
12+
| ----------------------------------- | ------- | --------------------------------------------------------- |
13+
| `S3_AWS_IAM_AUTH_ENABLED` | server | Set to `1` to enable IAM authentication for S3. |
14+
| `S3_MIRROR_AWS_IAM_AUTH_ENABLED` | server | Set to `1` to enable IAM authentication for S3 Mirror. |
15+
| `S3_AUDIT_LOG_AWS_IAM_AUTH_ENABLED` | server | Set to `1` to enable IAM authentication for S3 Audit Log. |
16+
17+
### To enable
18+
19+
- `S3_*_AWS_IAM_AUTH_ENABLED=1`.
20+
- `S3_BUCKET_NAME` set to the AWS S3 bucket.
21+
- `S3_ENDPOINT` set with the S3 endpoint with the AWS Region (i.e. https://s3.us-east-1.amazonaws.com)
22+
23+
When `CDN_API=1` is set on the server, the CDN artifact handler also uses IAM-authenticated S3 clients.
24+
Adds support for S3 Audit Logs exported to AWS S3.

packages/services/api/package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
"exports": {
1010
"./modules/auth/lib/supertokens-at-home/crypto": "./src/modules/auth/lib/supertokens-at-home/crypto.ts",
1111
"./modules/auth/providers/supertokens-store": "./src/modules/auth/providers/supertokens-store.ts",
12+
"./modules/cdn/providers/aws": "./src/modules/cdn/providers/aws.ts",
1213
"./modules/shared/providers/logger": "./src/modules/shared/providers/logger.ts",
1314
"./modules/shared/providers/redis": "./src/modules/shared/providers/redis.ts",
1415
"./modules/schema/providers/schema-version-store": "./src/modules/schema/providers/schema-version-store.ts"
@@ -19,6 +20,7 @@
1920
},
2021
"devDependencies": {
2122
"@aws-sdk/client-s3": "3.1035.0",
23+
"@aws-sdk/credential-providers": "3.1035.0",
2224
"@aws-sdk/s3-request-presigner": "3.1035.0",
2325
"@bentocache/plugin-prometheus": "0.2.0",
2426
"@date-fns/utc": "2.1.1",

packages/services/api/src/create.ts

Lines changed: 9 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import { authModule } from './modules/auth';
1313
import { Session } from './modules/auth/lib/authz';
1414
import { cdnModule } from './modules/cdn';
1515
import { AwsClient } from './modules/cdn/providers/aws';
16+
import type { S3CredentialProvider } from './modules/cdn/providers/aws';
1617
import { CDN_CONFIG, CDNConfig } from './modules/cdn/providers/tokens';
1718
import { collectionModule } from './modules/collection';
1819
import { commerceModule } from './modules/commerce';
@@ -137,23 +138,17 @@ export function createRegistry({
137138
s3: {
138139
bucketName: string;
139140
endpoint: string;
140-
accessKeyId: string;
141-
secretAccessKeyId: string;
142-
sessionToken?: string;
141+
credentialProvider: S3CredentialProvider;
143142
};
144143
s3Mirror: {
145144
bucketName: string;
146145
endpoint: string;
147-
accessKeyId: string;
148-
secretAccessKeyId: string;
149-
sessionToken?: string;
146+
credentialProvider: S3CredentialProvider;
150147
} | null;
151148
s3AuditLogs: {
152149
bucketName: string;
153150
endpoint: string;
154-
accessKeyId: string;
155-
secretAccessKeyId: string;
156-
sessionToken?: string;
151+
credentialProvider: S3CredentialProvider;
157152
} | null;
158153
encryptionSecret: string;
159154
app: {
@@ -177,9 +172,7 @@ export function createRegistry({
177172
const s3Config: S3Config = [
178173
{
179174
client: new AwsClient({
180-
accessKeyId: s3.accessKeyId,
181-
secretAccessKey: s3.secretAccessKeyId,
182-
sessionToken: s3.sessionToken,
175+
credentialProvider: s3.credentialProvider,
183176
service: 's3',
184177
}),
185178
bucket: s3.bucketName,
@@ -190,9 +183,7 @@ export function createRegistry({
190183
if (s3Mirror) {
191184
s3Config.push({
192185
client: new AwsClient({
193-
accessKeyId: s3Mirror.accessKeyId,
194-
secretAccessKey: s3Mirror.secretAccessKeyId,
195-
sessionToken: s3Mirror.sessionToken,
186+
credentialProvider: s3Mirror.credentialProvider,
196187
service: 's3',
197188
}),
198189
bucket: s3Mirror.bucketName,
@@ -205,13 +196,11 @@ export function createRegistry({
205196
const auditLogS3Config = s3AuditLogs
206197
? new AuditLogS3Config(
207198
new AwsClient({
208-
accessKeyId: s3AuditLogs.accessKeyId,
209-
secretAccessKey: s3AuditLogs.secretAccessKeyId,
210-
sessionToken: s3AuditLogs.sessionToken,
199+
credentialProvider: s3AuditLogs.credentialProvider,
211200
service: 's3',
212201
}),
213-
s3.endpoint,
214-
s3.bucketName,
202+
s3AuditLogs.endpoint,
203+
s3AuditLogs.bucketName,
215204
)
216205
: new AuditLogS3Config(s3Config[0].client, s3Config[0].endpoint, s3Config[0].bucket);
217206

packages/services/api/src/modules/app-deployments/worker/persisted-documents-worker.ts

Lines changed: 5 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { type MessagePort } from 'node:worker_threads';
22
import { AwsClient } from '../../cdn/providers/aws';
3+
import type { S3CredentialProvider } from '../../cdn/providers/aws';
34
import { ClickHouse } from '../../operations/providers/clickhouse-client';
45
import { HttpClient } from '../../shared/providers/http-client';
56
import { Logger } from '../../shared/providers/logger';
@@ -21,20 +22,12 @@ export function createWorker(
2122
s3: {
2223
readonly bucketName: string;
2324
readonly endpoint: string;
24-
readonly credentials: {
25-
readonly accessKeyId: string;
26-
readonly secretAccessKey: string;
27-
readonly sessionToken: string | undefined;
28-
};
25+
readonly credentialProvider: S3CredentialProvider;
2926
};
3027
s3Mirror: {
3128
readonly bucketName: string;
3229
readonly endpoint: string;
33-
readonly credentials: {
34-
readonly accessKeyId: string;
35-
readonly secretAccessKey: string;
36-
readonly sessionToken: string | undefined;
37-
};
30+
readonly credentialProvider: S3CredentialProvider;
3831
} | null;
3932
clickhouse: {
4033
readonly host: string;
@@ -48,9 +41,7 @@ export function createWorker(
4841
const s3Config: S3Config = [
4942
{
5043
client: new AwsClient({
51-
accessKeyId: env.s3.credentials.accessKeyId,
52-
secretAccessKey: env.s3.credentials.secretAccessKey,
53-
sessionToken: env.s3.credentials.sessionToken,
44+
credentialProvider: env.s3.credentialProvider,
5445
service: 's3',
5546
}),
5647
bucket: env.s3.bucketName,
@@ -61,9 +52,7 @@ export function createWorker(
6152
if (env.s3Mirror) {
6253
s3Config.push({
6354
client: new AwsClient({
64-
accessKeyId: env.s3Mirror.credentials.accessKeyId,
65-
secretAccessKey: env.s3Mirror.credentials.secretAccessKey,
66-
sessionToken: env.s3Mirror.credentials.sessionToken,
55+
credentialProvider: env.s3Mirror.credentialProvider,
6756
service: 's3',
6857
}),
6958
bucket: env.s3Mirror.bucketName,
Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
import { describe, expect, it, vi } from 'vitest';
2+
import { AwsClient, type S3CredentialProvider } from '../../cdn/providers/aws';
3+
import { AuditLogManager, AuditLogS3Config } from './audit-logs-manager';
4+
5+
vi.mock('@hive/workflows/kit', () => ({ TaskScheduler: class {} }));
6+
vi.mock('@hive/workflows/tasks/audit-log-export', () => ({ AuditLogExportTask: {} }));
7+
vi.mock('@sentry/node', () => ({ captureException: vi.fn() }));
8+
vi.mock('../../operations/providers/clickhouse-client', () => ({
9+
ClickHouse: class {},
10+
sql: (strings: TemplateStringsArray, ...values: any[]) => ({ strings, values }),
11+
}));
12+
vi.mock('./audit-log-recorder', () => ({
13+
formatToClickhouseDateTime: (d: Date) => d.toISOString(),
14+
}));
15+
vi.mock('./audit-logs-types', () => ({
16+
AuditLogClickhouseArrayModel: { parse: (v: any) => v },
17+
}));
18+
19+
function fakeCredentialProvider(keyId: string): S3CredentialProvider {
20+
return {
21+
getCredentials: async () => ({ accessKeyId: keyId, secretAccessKey: `${keyId}-secret` }),
22+
};
23+
}
24+
25+
function createAuditLogManager(fetchImpl: (...args: any[]) => any) {
26+
const credentialProvider = fakeCredentialProvider('test-key');
27+
const client = new AwsClient({ credentialProvider, service: 's3' });
28+
const fetchSpy = vi.spyOn(client, 'fetch').mockImplementation(fetchImpl as any);
29+
30+
const s3Config = new AuditLogS3Config(client, 'https://s3.example.com', 'audit-bucket');
31+
32+
const mockSession = {
33+
assertPerformAction: vi.fn().mockResolvedValue(undefined),
34+
getViewer: vi.fn().mockResolvedValue({ email: 'user@example.com' }),
35+
};
36+
const mockClickHouse = {
37+
query: vi.fn().mockResolvedValue({
38+
data: [
39+
{
40+
id: '1',
41+
timestamp: '2026-01-15',
42+
organizationId: 'org-1',
43+
eventAction: 'login',
44+
userId: 'u1',
45+
userEmail: 'u@x.com',
46+
accessTokenId: null,
47+
metadata: '{}',
48+
},
49+
],
50+
}),
51+
};
52+
const mockLogger = { child: () => mockLogger, info: vi.fn(), error: vi.fn() } as any;
53+
const mockTaskScheduler = { scheduleTask: vi.fn().mockResolvedValue(undefined) };
54+
const mockStorage = {
55+
getOrganization: vi.fn().mockResolvedValue({ name: 'TestOrg', id: 'org-1' }),
56+
};
57+
58+
return {
59+
manager: new AuditLogManager(
60+
mockLogger,
61+
mockClickHouse as any,
62+
s3Config,
63+
mockTaskScheduler as any,
64+
mockSession as any,
65+
mockStorage as any,
66+
),
67+
fetchSpy,
68+
};
69+
}
70+
71+
/**
72+
* Guards against regression where the S3 PUT upload was missing
73+
* `aws: { signQuery: true }`, causing the `got` library to receive
74+
* auth headers it can't handle - resulting in silent upload failures.
75+
*/
76+
describe('AuditLogManager S3 upload uses signQuery', () => {
77+
it('PUT upload includes aws.signQuery: true', async () => {
78+
const { manager, fetchSpy } = createAuditLogManager(
79+
vi.fn().mockResolvedValue({ ok: true, url: 'https://s3.example.com/key' }),
80+
);
81+
82+
await manager.exportAndSendEmail('org-1', {
83+
startDate: new Date('2026-01-01'),
84+
endDate: new Date('2026-01-31'),
85+
});
86+
87+
const putCall = fetchSpy.mock.calls[0];
88+
expect(putCall[1]).toEqual(
89+
expect.objectContaining({
90+
method: 'PUT',
91+
aws: expect.objectContaining({ signQuery: true }),
92+
}),
93+
);
94+
});
95+
96+
it('GET presigned URL includes aws.signQuery: true', async () => {
97+
const { manager, fetchSpy } = createAuditLogManager(
98+
vi.fn().mockResolvedValue({ ok: true, url: 'https://s3.example.com/key' }),
99+
);
100+
101+
await manager.exportAndSendEmail('org-1', {
102+
startDate: new Date('2026-01-01'),
103+
endDate: new Date('2026-01-31'),
104+
});
105+
106+
const getCall = fetchSpy.mock.calls[1];
107+
expect(getCall[1]).toEqual(
108+
expect.objectContaining({
109+
method: 'GET',
110+
aws: expect.objectContaining({ signQuery: true }),
111+
}),
112+
);
113+
});
114+
});

packages/services/api/src/modules/audit-logs/providers/audit-logs-manager.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,9 @@ export class AuditLogManager {
171171
'Content-Type': 'text/csv',
172172
},
173173
body: csvData,
174+
aws: {
175+
signQuery: true,
176+
},
174177
});
175178

176179
if (!uploadResult.ok) {

0 commit comments

Comments
 (0)