Skip to content

Commit 0d15ab7

Browse files
authored
Merge pull request #4056 from yugalkaushik/memory-fix
Enforce asset cap and surface upload-limit messaging
2 parents b05550a + 9561723 commit 0d15ab7

File tree

5 files changed

+126
-85
lines changed

5 files changed

+126
-85
lines changed

client/modules/IDE/actions/uploader.js

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ import { apiClient } from '../../../utils/apiClient';
33
import { getConfig } from '../../../utils/getConfig';
44
import { isTestEnvironment } from '../../../utils/checkTestEnv';
55
import { handleCreateFile } from './files';
6+
import { showErrorModal } from './ide';
67

78
const s3BucketUrlBase = getConfig('S3_BUCKET_URL_BASE');
89
const awsRegion = getConfig('AWS_REGION');
@@ -22,7 +23,7 @@ function isS3Upload(file) {
2223
return !TEXT_FILE_REGEX.test(file.name) || file.size >= MAX_LOCAL_FILE_SIZE;
2324
}
2425

25-
export async function dropzoneAcceptCallback(userId, file, done) {
26+
export async function dropzoneAcceptCallback(userId, file, done, dispatch) {
2627
// if a user would want to edit this file as text, local interceptor
2728
if (!isS3Upload(file)) {
2829
try {
@@ -51,6 +52,13 @@ export async function dropzoneAcceptCallback(userId, file, done) {
5152
file.postData = response.data;
5253
done();
5354
} catch (error) {
55+
if (error?.response?.status === 403) {
56+
if (dispatch) {
57+
dispatch(showErrorModal('uploadLimit'));
58+
}
59+
done('Upload limit reached.');
60+
return;
61+
}
5462
done(
5563
error?.response?.data?.responseText?.message ||
5664
error?.message ||

client/modules/IDE/components/ErrorModal.jsx

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@ import PropTypes from 'prop-types';
22
import React from 'react';
33
import { Link } from 'react-router-dom';
44
import { useTranslation } from 'react-i18next';
5+
import prettyBytes from 'pretty-bytes';
6+
import { getConfig } from '../../../utils/getConfig';
7+
import { parseNumber } from '../../../utils/parseStringToType';
8+
9+
const uploadLimit = parseNumber(getConfig('UPLOAD_LIMIT')) || 250000000;
10+
const uploadLimitText = prettyBytes(uploadLimit);
511

612
const ErrorModal = ({ type, service, closeModal }) => {
713
const { t } = useTranslation();
@@ -51,6 +57,18 @@ const ErrorModal = ({ type, service, closeModal }) => {
5157
return <p>{t('ErrorModal.SavedDifferentWindow')}</p>;
5258
}
5359

60+
function uploadLimitReached() {
61+
return (
62+
<p>
63+
{t('UploadFileModal.SizeLimitError', { sizeLimit: uploadLimitText })}
64+
<Link to="/assets" onClick={closeModal}>
65+
assets
66+
</Link>
67+
.
68+
</p>
69+
);
70+
}
71+
5472
return (
5573
<div className="error-modal__content">
5674
{(() => { // eslint-disable-line
@@ -60,6 +78,8 @@ const ErrorModal = ({ type, service, closeModal }) => {
6078
return staleSession();
6179
} else if (type === 'staleProject') {
6280
return staleProject();
81+
} else if (type === 'uploadLimit') {
82+
return uploadLimitReached();
6383
} else if (type === 'oauthError') {
6484
return oauthError();
6585
}
@@ -73,7 +93,8 @@ ErrorModal.propTypes = {
7393
'forceAuthentication',
7494
'staleSession',
7595
'staleProject',
76-
'oauthError'
96+
'oauthError',
97+
'uploadLimit'
7798
]).isRequired,
7899
closeModal: PropTypes.func.isRequired,
79100
service: PropTypes.oneOf(['google', 'github'])

client/modules/IDE/components/FileUploader.jsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ function FileUploader() {
6969
acceptedFiles: fileExtensionsAndMimeTypes,
7070
dictDefaultMessage: t('FileUploader.DictDefaultMessage'),
7171
accept: (file, done) => {
72-
dropzoneAcceptCallback(userId, file, done);
72+
dropzoneAcceptCallback(userId, file, done, dispatch);
7373
},
7474
sending: dropzoneSendingCallback
7575
});

client/modules/IDE/selectors/users.js

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,11 @@ export const getCanUploadMedia = createSelector(
1313
getAuthenticated,
1414
getTotalSize,
1515
(authenticated, totalSize) => {
16+
const currentSize = totalSize || 0;
1617
if (!authenticated) return false;
1718
// eventually do the same thing for verified when
1819
// email verification actually works
19-
if (totalSize > limit) return false;
20+
if (currentSize >= limit) return false;
2021
return true;
2122
}
2223
);
@@ -25,8 +26,8 @@ export const getreachedTotalSizeLimit = createSelector(
2526
getTotalSize,
2627
getAssetsTotalSize,
2728
(totalSize, assetsTotalSize) => {
28-
const currentSize = totalSize || assetsTotalSize;
29-
if (currentSize && currentSize > limit) return true;
29+
const currentSize = totalSize || assetsTotalSize || 0;
30+
if (currentSize >= limit) return true;
3031
// if (totalSize > 1000) return true;
3132
return false;
3233
}

server/controllers/aws.controller.js

Lines changed: 90 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -78,28 +78,97 @@ export async function deleteObjectFromS3(req, res) {
7878
}
7979
}
8080

81-
export function signS3(req, res) {
82-
const limit = process.env.UPLOAD_LIMIT || 250000000;
83-
if (req.user.totalSize > limit) {
84-
res
85-
.status(403)
86-
.send({ message: 'user has uploaded the maximum size of assets.' });
87-
return;
81+
export async function listObjectsInS3ForUser(userId) {
82+
try {
83+
let assets = [];
84+
const params = {
85+
Bucket: process.env.S3_BUCKET,
86+
Prefix: `${userId}/`
87+
};
88+
89+
const data = await s3Client.send(new ListObjectsCommand(params));
90+
91+
assets = data.Contents?.map((object) => ({
92+
key: object.Key,
93+
size: object.Size
94+
}));
95+
96+
const projects = await Project.getProjectsForUserId(userId);
97+
const projectAssets = [];
98+
let totalSize = 0;
99+
100+
assets?.forEach((asset) => {
101+
const name = asset.key.split('/').pop();
102+
const foundAsset = {
103+
key: asset.key,
104+
name,
105+
size: asset.size,
106+
url: `${process.env.S3_BUCKET_URL_BASE}${asset.key}`
107+
};
108+
totalSize += asset.size;
109+
110+
const wasMatched = projects.some((project) =>
111+
project.files.some((file) => {
112+
if (!file.url) return false;
113+
if (file.url.includes(asset.key)) {
114+
foundAsset.name = file.name;
115+
foundAsset.sketchName = project.name;
116+
foundAsset.sketchId = project.id;
117+
foundAsset.url = file.url;
118+
return true;
119+
}
120+
return false;
121+
})
122+
);
123+
124+
if (wasMatched) {
125+
projectAssets.push(foundAsset);
126+
}
127+
});
128+
129+
return { assets: projectAssets, totalSize };
130+
} catch (error) {
131+
if (error instanceof TypeError) {
132+
return null;
133+
}
134+
console.error('Got an error: ', error);
135+
throw error;
136+
}
137+
}
138+
139+
export async function signS3(req, res) {
140+
const limit = Number(process.env.UPLOAD_LIMIT) || 250000000;
141+
142+
try {
143+
const objects = await listObjectsInS3ForUser(req.user.id);
144+
const currentSize = Number(objects?.totalSize ?? req.user.totalSize) || 0;
145+
const incomingSize = Number(req.body.size) || 0;
146+
147+
if (currentSize >= limit || currentSize + incomingSize > limit) {
148+
res
149+
.status(403)
150+
.send({ message: 'user has uploaded the maximum size of assets.' });
151+
return;
152+
}
153+
154+
const fileExtension = getExtension(req.body.name);
155+
const filename = uuidv4() + fileExtension;
156+
const acl = 'public-read';
157+
const policy = S3Policy.generate({
158+
acl,
159+
key: `${req.body.userId}/${filename}`,
160+
bucket: process.env.S3_BUCKET,
161+
contentType: req.body.type,
162+
region: process.env.AWS_REGION,
163+
accessKey: process.env.AWS_ACCESS_KEY,
164+
secretKey: process.env.AWS_SECRET_KEY,
165+
metadata: []
166+
});
167+
res.json(policy);
168+
} catch (error) {
169+
console.error('Error signing upload policy:', error);
170+
res.status(500).json({ error: 'Failed to sign upload policy' });
88171
}
89-
const fileExtension = getExtension(req.body.name);
90-
const filename = uuidv4() + fileExtension;
91-
const acl = 'public-read';
92-
const policy = S3Policy.generate({
93-
acl,
94-
key: `${req.body.userId}/${filename}`,
95-
bucket: process.env.S3_BUCKET,
96-
contentType: req.body.type,
97-
region: process.env.AWS_REGION,
98-
accessKey: process.env.AWS_ACCESS_KEY,
99-
secretKey: process.env.AWS_SECRET_KEY,
100-
metadata: []
101-
});
102-
res.json(policy);
103172
}
104173

105174
export async function copyObjectInS3(url, userId) {
@@ -182,64 +251,6 @@ export async function moveObjectToUserInS3(url, userId) {
182251
return `${s3Bucket}${userId}/${newFilename}`;
183252
}
184253

185-
export async function listObjectsInS3ForUser(userId) {
186-
try {
187-
let assets = [];
188-
const params = {
189-
Bucket: process.env.S3_BUCKET,
190-
Prefix: `${userId}/`
191-
};
192-
193-
const data = await s3Client.send(new ListObjectsCommand(params));
194-
195-
assets = data.Contents?.map((object) => ({
196-
key: object.Key,
197-
size: object.Size
198-
}));
199-
200-
const projects = await Project.getProjectsForUserId(userId);
201-
const projectAssets = [];
202-
let totalSize = 0;
203-
204-
assets?.forEach((asset) => {
205-
const name = asset.key.split('/').pop();
206-
const foundAsset = {
207-
key: asset.key,
208-
name,
209-
size: asset.size,
210-
url: `${process.env.S3_BUCKET_URL_BASE}${asset.key}`
211-
};
212-
totalSize += asset.size;
213-
214-
const wasMatched = projects.some((project) =>
215-
project.files.some((file) => {
216-
if (!file.url) return false;
217-
if (file.url.includes(asset.key)) {
218-
foundAsset.name = file.name;
219-
foundAsset.sketchName = project.name;
220-
foundAsset.sketchId = project.id;
221-
foundAsset.url = file.url;
222-
return true;
223-
}
224-
return false;
225-
})
226-
);
227-
228-
if (wasMatched) {
229-
projectAssets.push(foundAsset);
230-
}
231-
});
232-
233-
return { assets: projectAssets, totalSize };
234-
} catch (error) {
235-
if (error instanceof TypeError) {
236-
return null;
237-
}
238-
console.error('Got an error: ', error);
239-
throw error;
240-
}
241-
}
242-
243254
export async function listObjectsInS3ForUserRequestHandler(req, res) {
244255
const { username } = req.user;
245256

0 commit comments

Comments
 (0)