Skip to content

Commit acb7186

Browse files
author
Rajat
committed
Db and AWS S3 migrations
1 parent c5ae231 commit acb7186

22 files changed

+427
-94
lines changed

‎.migrations/0005-migrate-to-dual-bucket-architecture.js‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,4 @@
1-
// TODO: rename "public-read" to "public"
1+
db.media.updateMany(
2+
{ accessControl: "public-read" },
3+
{ $set: { accessControl: "public" } },
4+
);

‎.prettierignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
node_modules
22
dist
33
**/.next
4-
apps/docs/.source
4+
apps/docs/.source
5+
apps/docs/out

‎apps/api/__tests__/media/utils/get-public-urls.test.ts‎

Lines changed: 42 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import { describe, test, beforeEach, afterEach } from "node:test";
22
import assert from "node:assert";
33
import { Constants, Media } from "@medialit/models";
4-
import { PATH_KEY } from "@/media/utils/generate-key";
54

65
// Helper to clear module cache and re-import
76
const clearModuleCache = () => {
@@ -61,7 +60,7 @@ describe("get-public-urls", () => {
6160
clearModuleCache();
6261
});
6362

64-
describe("getMainFileUrl", () => {
63+
describe("getPublicFileUrl", () => {
6564
test("should use CDN_ENDPOINT when provided (takes precedence)", () => {
6665
process.env.CDN_ENDPOINT = "https://cdn.example.com";
6766
process.env.CLOUD_ENDPOINT = "https://private.s3.amazonaws.com";
@@ -70,16 +69,18 @@ describe("get-public-urls", () => {
7069
process.env.PATH_PREFIX = "";
7170
clearModuleCache();
7271

73-
const { getMainFileUrl } = require("@/media/utils/get-public-urls");
72+
const {
73+
getPublicFileUrl,
74+
} = require("@/media/utils/get-public-urls");
7475
const media = createMockMedia({
7576
accessControl: Constants.AccessControl.PUBLIC,
7677
fileName: "main.jpg",
7778
});
7879

79-
const url = getMainFileUrl(media);
80+
const url = getPublicFileUrl(media);
8081
assert.strictEqual(
8182
url,
82-
`https://cdn.example.com/${PATH_KEY.PUBLIC}/test-media-id-123/main.jpg`,
83+
`https://cdn.example.com/${Constants.PathKey.PUBLIC}/test-media-id-123/main.jpg`,
8384
);
8485
});
8586

@@ -91,16 +92,18 @@ describe("get-public-urls", () => {
9192
process.env.PATH_PREFIX = "";
9293
clearModuleCache();
9394

94-
const { getMainFileUrl } = require("@/media/utils/get-public-urls");
95+
const {
96+
getPublicFileUrl,
97+
} = require("@/media/utils/get-public-urls");
9598
const media = createMockMedia({
9699
accessControl: Constants.AccessControl.PUBLIC,
97100
fileName: "main.png",
98101
});
99102

100-
const url = getMainFileUrl(media);
103+
const url = getPublicFileUrl(media);
101104
assert.strictEqual(
102105
url,
103-
`https://public.s3.amazonaws.com/${PATH_KEY.PUBLIC}/test-media-id-123/main.png`,
106+
`https://public.s3.amazonaws.com/${Constants.PathKey.PUBLIC}/test-media-id-123/main.png`,
104107
);
105108
});
106109

@@ -111,16 +114,18 @@ describe("get-public-urls", () => {
111114
process.env.PATH_PREFIX = "";
112115
clearModuleCache();
113116

114-
const { getMainFileUrl } = require("@/media/utils/get-public-urls");
117+
const {
118+
getPublicFileUrl,
119+
} = require("@/media/utils/get-public-urls");
115120
const media = createMockMedia({
116121
accessControl: Constants.AccessControl.PUBLIC,
117122
fileName: "main.webp",
118123
});
119124

120-
const url = getMainFileUrl(media);
125+
const url = getPublicFileUrl(media);
121126
assert.strictEqual(
122127
url,
123-
`https://private.s3.amazonaws.com/${PATH_KEY.PUBLIC}/test-media-id-123/main.webp`,
128+
`https://private.s3.amazonaws.com/${Constants.PathKey.PUBLIC}/test-media-id-123/main.webp`,
124129
);
125130
});
126131

@@ -132,16 +137,18 @@ describe("get-public-urls", () => {
132137
process.env.PATH_PREFIX = "";
133138
clearModuleCache();
134139

135-
const { getMainFileUrl } = require("@/media/utils/get-public-urls");
140+
const {
141+
getPublicFileUrl,
142+
} = require("@/media/utils/get-public-urls");
136143
const media = createMockMedia({
137144
accessControl: Constants.AccessControl.PRIVATE,
138145
fileName: "main.jpg",
139146
});
140147

141-
const url = getMainFileUrl(media);
148+
const url = getPublicFileUrl(media);
142149
assert.strictEqual(
143150
url,
144-
`https://private.s3.amazonaws.com/${PATH_KEY.PUBLIC}/test-media-id-123/main.jpg`,
151+
`https://private.s3.amazonaws.com/${Constants.PathKey.PUBLIC}/test-media-id-123/main.jpg`,
145152
);
146153
});
147154

@@ -150,16 +157,18 @@ describe("get-public-urls", () => {
150157
process.env.PATH_PREFIX = "tenant-123";
151158
clearModuleCache();
152159

153-
const { getMainFileUrl } = require("@/media/utils/get-public-urls");
160+
const {
161+
getPublicFileUrl,
162+
} = require("@/media/utils/get-public-urls");
154163
const media = createMockMedia({
155164
accessControl: Constants.AccessControl.PUBLIC,
156165
fileName: "main.jpg",
157166
});
158167

159-
const url = getMainFileUrl(media);
168+
const url = getPublicFileUrl(media);
160169
assert.strictEqual(
161170
url,
162-
`https://cdn.example.com/tenant-123/${PATH_KEY.PUBLIC}/test-media-id-123/main.jpg`,
171+
`https://cdn.example.com/tenant-123/${Constants.PathKey.PUBLIC}/test-media-id-123/main.jpg`,
163172
);
164173
});
165174
});
@@ -181,7 +190,7 @@ describe("get-public-urls", () => {
181190
const url = getThumbnailUrl(media);
182191
assert.strictEqual(
183192
url,
184-
`https://cdn.example.com/${PATH_KEY.PUBLIC}/test-media-id-123/thumb.webp`,
193+
`https://cdn.example.com/${Constants.PathKey.PUBLIC}/test-media-id-123/thumb.webp`,
185194
);
186195
});
187196

@@ -201,7 +210,7 @@ describe("get-public-urls", () => {
201210
const url = getThumbnailUrl(media);
202211
assert.strictEqual(
203212
url,
204-
`https://public.s3.amazonaws.com/${PATH_KEY.PUBLIC}/test-media-id-123/thumb.webp`,
213+
`https://public.s3.amazonaws.com/${Constants.PathKey.PUBLIC}/test-media-id-123/thumb.webp`,
205214
);
206215
});
207216

@@ -221,7 +230,7 @@ describe("get-public-urls", () => {
221230
const url = getThumbnailUrl(media);
222231
assert.strictEqual(
223232
url,
224-
`https://public.s3.amazonaws.com/${PATH_KEY.PUBLIC}/test-media-id-123/thumb.webp`,
233+
`https://public.s3.amazonaws.com/${Constants.PathKey.PUBLIC}/test-media-id-123/thumb.webp`,
225234
);
226235
});
227236

@@ -238,7 +247,7 @@ describe("get-public-urls", () => {
238247
const url = getThumbnailUrl(media);
239248
assert.strictEqual(
240249
url,
241-
`https://cdn.example.com/tenant-456/${PATH_KEY.PUBLIC}/test-media-id-123/thumb.webp`,
250+
`https://cdn.example.com/tenant-456/${Constants.PathKey.PUBLIC}/test-media-id-123/thumb.webp`,
242251
);
243252
});
244253
});
@@ -254,23 +263,23 @@ describe("get-public-urls", () => {
254263
clearModuleCache();
255264

256265
const {
257-
getMainFileUrl,
266+
getPublicFileUrl,
258267
getThumbnailUrl,
259268
} = require("@/media/utils/get-public-urls");
260269
const media = createMockMedia({
261270
accessControl: Constants.AccessControl.PUBLIC,
262271
});
263272

264-
const mainUrl = getMainFileUrl(media);
273+
const mainUrl = getPublicFileUrl(media);
265274
const thumbUrl = getThumbnailUrl(media);
266275

267276
assert.strictEqual(
268277
mainUrl,
269-
`https://public.r2.cloudflarestorage.com/${PATH_KEY.PUBLIC}/test-media-id-123/main.jpg`,
278+
`https://public.r2.cloudflarestorage.com/${Constants.PathKey.PUBLIC}/test-media-id-123/main.jpg`,
270279
);
271280
assert.strictEqual(
272281
thumbUrl,
273-
`https://public.r2.cloudflarestorage.com/${PATH_KEY.PUBLIC}/test-media-id-123/thumb.webp`,
282+
`https://public.r2.cloudflarestorage.com/${Constants.PathKey.PUBLIC}/test-media-id-123/thumb.webp`,
274283
);
275284
});
276285

@@ -284,24 +293,24 @@ describe("get-public-urls", () => {
284293
clearModuleCache();
285294

286295
const {
287-
getMainFileUrl,
296+
getPublicFileUrl,
288297
getThumbnailUrl,
289298
} = require("@/media/utils/get-public-urls");
290299
const media = createMockMedia({
291300
accessControl: Constants.AccessControl.PUBLIC,
292301
});
293302

294-
const mainUrl = getMainFileUrl(media);
303+
const mainUrl = getPublicFileUrl(media);
295304
const thumbUrl = getThumbnailUrl(media);
296305

297306
// CDN should take precedence
298307
assert.strictEqual(
299308
mainUrl,
300-
`https://cdn.medialit.cloud/${PATH_KEY.PUBLIC}/test-media-id-123/main.jpg`,
309+
`https://cdn.medialit.cloud/${Constants.PathKey.PUBLIC}/test-media-id-123/main.jpg`,
301310
);
302311
assert.strictEqual(
303312
thumbUrl,
304-
`https://cdn.medialit.cloud/${PATH_KEY.PUBLIC}/test-media-id-123/thumb.webp`,
313+
`https://cdn.medialit.cloud/${Constants.PathKey.PUBLIC}/test-media-id-123/thumb.webp`,
305314
);
306315
});
307316

@@ -314,25 +323,25 @@ describe("get-public-urls", () => {
314323
clearModuleCache();
315324

316325
const {
317-
getMainFileUrl,
326+
getPublicFileUrl,
318327
getThumbnailUrl,
319328
} = require("@/media/utils/get-public-urls");
320329
const media = createMockMedia({
321330
accessControl: Constants.AccessControl.PUBLIC,
322331
});
323332

324-
const mainUrl = getMainFileUrl(media);
333+
const mainUrl = getPublicFileUrl(media);
325334
const thumbUrl = getThumbnailUrl(media);
326335

327336
// Main files use CLOUD_ENDPOINT_PUBLIC for public media
328337
assert.strictEqual(
329338
mainUrl,
330-
`https://public.s3.amazonaws.com/${PATH_KEY.PUBLIC}/test-media-id-123/main.jpg`,
339+
`https://public.s3.amazonaws.com/${Constants.PathKey.PUBLIC}/test-media-id-123/main.jpg`,
331340
);
332341
// Thumbnails always use CLOUD_ENDPOINT_PUBLIC
333342
assert.strictEqual(
334343
thumbUrl,
335-
`https://public.s3.amazonaws.com/${PATH_KEY.PUBLIC}/test-media-id-123/thumb.webp`,
344+
`https://public.s3.amazonaws.com/${Constants.PathKey.PUBLIC}/test-media-id-123/thumb.webp`,
336345
);
337346
});
338347
});

‎apps/api/src/media/cleanup.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import {
66
TEMP_MEDIA_EXPIRATION_HOURS,
77
cloudBucket,
88
} from "../config/constants";
9-
import { PATH_KEY } from "./utils/generate-key";
9+
import { Constants } from "@medialit/models";
1010

1111
export async function cleanupExpiredTempUploads(): Promise<void> {
1212
const cutoff = new Date(
@@ -33,7 +33,7 @@ export async function cleanupExpiredTempUploads(): Promise<void> {
3333
for (const media of expired) {
3434
try {
3535
// Delete S3 objects from private bucket (using private path prefix)
36-
const tmpPrefix = `${PATH_PREFIX ? `${PATH_PREFIX}/` : ""}${PATH_KEY.PRIVATE}/${media.mediaId}/`;
36+
const tmpPrefix = `${PATH_PREFIX ? `${PATH_PREFIX}/` : ""}${Constants.PathKey.PRIVATE}/${media.mediaId}/`;
3737
await deleteFolder(tmpPrefix, cloudBucket);
3838

3939
// Delete media record

‎apps/api/src/media/service.ts‎

Lines changed: 15 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ import {
2727
UploadParams,
2828
} from "../services/s3";
2929
import logger from "../services/log";
30-
import generateKey, { PATH_KEY } from "./utils/generate-key";
30+
import generateKey from "./utils/generate-key";
3131
import { getMediaSettings } from "../media-settings/queries";
3232
import generateFileName from "./utils/generate-file-name";
3333
import mongoose from "mongoose";
@@ -41,7 +41,7 @@ import {
4141
import MediaModel from "./model";
4242
import * as presignedUrlService from "../signature/service";
4343
import getTags from "./utils/get-tags";
44-
import { getMainFileUrl, getThumbnailUrl } from "./utils/get-public-urls";
44+
import { getPublicFileUrl, getThumbnailUrl } from "./utils/get-public-urls";
4545
import { AccessControl, Constants, MediaWithUserId } from "@medialit/models";
4646

4747
const generateAndUploadThumbnail = async ({
@@ -129,7 +129,7 @@ async function upload({
129129
const uploadParams: UploadParams = {
130130
Key: generateKey({
131131
mediaId: fileName.name,
132-
path: PATH_KEY.PRIVATE,
132+
path: Constants.PathKey.PRIVATE,
133133
filename: `main.${fileExtension}`,
134134
}),
135135
Body: createReadStream(mainFilePath),
@@ -149,7 +149,7 @@ async function upload({
149149
originalFilePath: mainFilePath,
150150
key: generateKey({
151151
mediaId: fileName.name,
152-
path: PATH_KEY.PRIVATE,
152+
path: Constants.PathKey.PRIVATE,
153153
filename: "thumb.webp",
154154
}),
155155
tags,
@@ -259,7 +259,7 @@ async function getMediaDetails({
259259
fileUrl = await getPrivateFileUrl(media);
260260
} else {
261261
// Public sealed files: use direct URL from public bucket
262-
fileUrl = getMainFileUrl(media);
262+
fileUrl = getPublicFileUrl(media);
263263
}
264264

265265
// Determine thumbnail URL
@@ -297,7 +297,7 @@ async function getPrivateFileUrl(media: MediaWithUserId, thumb?: boolean) {
297297

298298
const key = generateKey({
299299
mediaId: media.mediaId,
300-
path: PATH_KEY.PRIVATE,
300+
path: Constants.PathKey.PRIVATE,
301301
filename,
302302
});
303303

@@ -329,8 +329,8 @@ async function deleteMedia({
329329

330330
const mainPath =
331331
media.temp || media.accessControl === Constants.AccessControl.PRIVATE
332-
? PATH_KEY.PRIVATE
333-
: PATH_KEY.PUBLIC;
332+
? Constants.PathKey.PRIVATE
333+
: Constants.PathKey.PUBLIC;
334334

335335
const fileExtension = path.extname(media.fileName).replace(".", "");
336336
const key = generateKey({
@@ -343,7 +343,9 @@ async function deleteMedia({
343343
if (media.thumbnailGenerated) {
344344
// Thumbnails are in public bucket if sealed, private bucket if temp
345345
const thumbBucket = media.temp ? cloudBucket : cloudPublicBucket;
346-
const thumbPath = media.temp ? PATH_KEY.PRIVATE : PATH_KEY.PUBLIC;
346+
const thumbPath = media.temp
347+
? Constants.PathKey.PRIVATE
348+
: Constants.PathKey.PUBLIC;
347349
const thumbKey = generateKey({
348350
mediaId,
349351
path: thumbPath,
@@ -378,7 +380,7 @@ async function sealMedia({
378380
// Get tags from source object (in private bucket)
379381
const tmpMainKey = generateKey({
380382
mediaId,
381-
path: PATH_KEY.PRIVATE,
383+
path: Constants.PathKey.PRIVATE,
382384
filename: `main.${fileExtension}`,
383385
});
384386
let tags: string | undefined;
@@ -405,7 +407,7 @@ async function sealMedia({
405407
if (isPublic) {
406408
const finalMainKey = generateKey({
407409
mediaId,
408-
path: PATH_KEY.PUBLIC,
410+
path: Constants.PathKey.PUBLIC,
409411
filename: `main.${fileExtension}`,
410412
});
411413

@@ -423,12 +425,12 @@ async function sealMedia({
423425
if (media.thumbnailGenerated) {
424426
const tmpThumbKey = generateKey({
425427
mediaId,
426-
path: PATH_KEY.PRIVATE,
428+
path: Constants.PathKey.PRIVATE,
427429
filename: "thumb.webp",
428430
});
429431
const finalThumbKey = generateKey({
430432
mediaId,
431-
path: PATH_KEY.PUBLIC,
433+
path: Constants.PathKey.PUBLIC,
432434
filename: "thumb.webp",
433435
});
434436

0 commit comments

Comments
 (0)