Skip to content

Commit 821756a

Browse files
author
Rajat Saxena
committed
Uploads are temporary by default
1 parent 2729e8a commit 821756a

File tree

33 files changed

+1496
-949
lines changed

33 files changed

+1496
-949
lines changed

‎apps/api/__tests__/services/s3.test.ts‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
import { describe, test } from "node:test";
2+
import assert from "node:assert";
3+
4+
describe("S3 Client Configuration", () => {
5+
test("should include endpoint and forcePathStyle when cloudEndpoint is set", () => {
6+
// Set cloudEndpoint in environment
7+
const originalEnv = process.env.CLOUD_ENDPOINT;
8+
process.env.CLOUD_ENDPOINT = "http://localhost:9000";
9+
process.env.CLOUD_REGION = "us-east-1";
10+
process.env.CLOUD_KEY = "test-key";
11+
process.env.CLOUD_SECRET = "test-secret";
12+
13+
// Clear module cache to force re-evaluation with new env
14+
const modulePath = require.resolve("../../src/services/s3");
15+
const constantsPath = require.resolve("../../src/config/constants");
16+
delete require.cache[modulePath];
17+
delete require.cache[constantsPath];
18+
19+
// Re-import to get fresh config
20+
const { s3ClientConfig } = require("../../src/services/s3");
21+
22+
// Verify config structure
23+
assert.ok(
24+
s3ClientConfig !== undefined,
25+
"s3ClientConfig should be defined",
26+
);
27+
assert.strictEqual(
28+
s3ClientConfig.endpoint,
29+
"http://localhost:9000",
30+
"endpoint should be set when cloudEndpoint is provided",
31+
);
32+
assert.strictEqual(
33+
s3ClientConfig.forcePathStyle,
34+
true,
35+
"forcePathStyle should be true when cloudEndpoint is provided",
36+
);
37+
38+
// Restore original env
39+
if (originalEnv !== undefined) {
40+
process.env.CLOUD_ENDPOINT = originalEnv;
41+
} else {
42+
delete process.env.CLOUD_ENDPOINT;
43+
}
44+
});
45+
46+
test("should not include endpoint or forcePathStyle when cloudEndpoint is not set", () => {
47+
// Unset cloudEndpoint
48+
const originalEnv = process.env.CLOUD_ENDPOINT;
49+
delete process.env.CLOUD_ENDPOINT;
50+
process.env.CLOUD_REGION = "us-east-1";
51+
process.env.CLOUD_KEY = "test-key";
52+
process.env.CLOUD_SECRET = "test-secret";
53+
54+
// Clear module cache
55+
const modulePath = require.resolve("../../src/services/s3");
56+
const constantsPath = require.resolve("../../src/config/constants");
57+
delete require.cache[modulePath];
58+
delete require.cache[constantsPath];
59+
60+
// Re-import to get fresh config
61+
const { s3ClientConfig } = require("../../src/services/s3");
62+
63+
// Verify config structure
64+
assert.ok(
65+
s3ClientConfig !== undefined,
66+
"s3ClientConfig should be defined",
67+
);
68+
assert.strictEqual(
69+
s3ClientConfig.endpoint,
70+
undefined,
71+
"endpoint should not be set when cloudEndpoint is not provided",
72+
);
73+
assert.strictEqual(
74+
s3ClientConfig.forcePathStyle,
75+
undefined,
76+
"forcePathStyle should not be set when cloudEndpoint is not provided",
77+
);
78+
79+
// Restore original env
80+
if (originalEnv !== undefined) {
81+
process.env.CLOUD_ENDPOINT = originalEnv;
82+
}
83+
});
84+
});

‎apps/api/package.json‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@
3131
"test": "node --import tsx --test '**/*.test.ts'"
3232
},
3333
"dependencies": {
34-
"@aws-sdk/client-s3": "^3.55.0",
34+
"@aws-sdk/client-s3": "^3.922.0",
3535
"@aws-sdk/cloudfront-signer": "^3.572.0",
36-
"@aws-sdk/s3-request-presigner": "^3.55.0",
36+
"@aws-sdk/s3-request-presigner": "^3.922.0",
3737
"@medialit/images": "workspace:*",
3838
"@medialit/models": "workspace:*",
3939
"@medialit/thumbnail": "workspace:*",
@@ -42,14 +42,14 @@
4242
"@tus/server": "^2.3.0",
4343
"aws-sdk": "^2.1692.0",
4444
"cors": "^2.8.5",
45-
"dotenv": "^16.4.7",
45+
"dotenv": "^17.2.3",
4646
"express": "^4.2.0",
47-
"express-fileupload": "^1.3.1",
47+
"express-fileupload": "^1.5.2",
4848
"joi": "^17.6.0",
49-
"mongoose": "^8.0.1",
50-
"passport": "^0.5.2",
51-
"passport-jwt": "^4.0.0",
52-
"pino": "^7.9.1"
49+
"mongoose": "^8.19.3",
50+
"passport": "^0.7.0",
51+
"passport-jwt": "^4.0.1",
52+
"pino": "^10.1.0"
5353
},
5454
"devDependencies": {
5555
"@types/cors": "^2.8.12",
@@ -62,9 +62,9 @@
6262
"@typescript-eslint/eslint-plugin": "^5.17.0",
6363
"@typescript-eslint/parser": "^5.17.0",
6464
"eslint": "^8.12.0",
65-
"nodemon": "^3.0.3",
66-
"ts-node": "^10.7.0",
67-
"tsx": "^4.7.0",
68-
"typescript": "^5.2.2"
65+
"nodemon": "^3.1.10",
66+
"ts-node": "^10.9.2",
67+
"tsx": "^4.20.6",
68+
"typescript": "^5.9.3"
6969
}
7070
}

‎apps/api/src/config/constants.ts‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@ export const cloudKey = process.env.CLOUD_KEY || "";
5151
export const cloudSecret = process.env.CLOUD_SECRET || "";
5252
export const cloudBucket = process.env.CLOUD_BUCKET_NAME || "";
5353
export const CLOUD_PREFIX = process.env.CLOUD_PREFIX || "";
54-
export const S3_ENDPOINT = process.env.S3_ENDPOINT || "";
54+
export const PUBLIC_ENDPOINT = process.env.PUBLIC_ENDPOINT || "";
55+
export const HOUR_IN_SECONDS = 1000 * 60 * 60;
5556

5657
// Cloudfront config
5758
export const USE_CLOUDFRONT = process.env.USE_CLOUDFRONT === "true";
@@ -60,7 +61,11 @@ export const CLOUDFRONT_KEY_PAIR_ID = process.env.CLOUDFRONT_KEY_PAIR_ID || "";
6061
export const CLOUDFRONT_PRIVATE_KEY = process.env.CLOUDFRONT_PRIVATE_KEY || "";
6162
export const CDN_MAX_AGE = process.env.CDN_MAX_AGE
6263
? +process.env.CDN_MAX_AGE
63-
: 1000 * 60 * 60; // one hour
64+
: HOUR_IN_SECONDS; // one hour
6465

65-
export const ENDPOINT = USE_CLOUDFRONT ? CLOUDFRONT_ENDPOINT : S3_ENDPOINT;
66-
export const HOSTNAME_OVERRIDE = process.env.HOSTNAME_OVERRIDE || ""; // Useful for hosting via Docker
66+
export const ENDPOINT = USE_CLOUDFRONT ? CLOUDFRONT_ENDPOINT : PUBLIC_ENDPOINT;
67+
export const TEMP_MEDIA_EXPIRATION_HOURS = process.env
68+
.TEMP_MEDIA_EXPIRATION_HOURS
69+
? +process.env.TEMP_MEDIA_EXPIRATION_HOURS
70+
: 24; // 24 hours
71+
export const DISABLE_TAGGING = process.env.DISABLE_TAGGING === "true";

‎apps/api/src/index.ts‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@ import { createUser, findByEmail } from "./user/queries";
1313
import { Apikey, User } from "@medialit/models";
1414
import { createApiKey } from "./apikey/queries";
1515
import { spawn } from "child_process";
16-
import { Cleanup } from "./tus/cleanup";
16+
import { cleanupTUSUploads } from "./tus/cleanup";
17+
import { cleanupExpiredTempUploads } from "./media/cleanup";
18+
import { HOUR_IN_SECONDS } from "./config/constants";
1719

1820
connectToDatabase();
1921
const app = express();
@@ -34,6 +36,19 @@ app.use("/media/signature", signatureRoutes);
3436
app.use("/media", tusRoutes);
3537
app.use("/media", mediaRoutes);
3638

39+
app.get("/cleanup/temp", async (req, res) => {
40+
await cleanupExpiredTempUploads();
41+
res.status(200).json({
42+
message: "Expired temp uploads cleaned up",
43+
});
44+
});
45+
app.get("/cleanup/tus", async (req, res) => {
46+
await cleanupTUSUploads();
47+
res.status(200).json({
48+
message: "Expired tus uploads cleaned up",
49+
});
50+
});
51+
3752
const port = process.env.PORT || 80;
3853

3954
if (process.env.EMAIL) {
@@ -48,9 +63,17 @@ checkDependencies().then(() => {
4863
// Setup background cleanup job for expired tus uploads
4964
setInterval(
5065
async () => {
51-
await Cleanup();
66+
await cleanupTUSUploads();
67+
},
68+
HOUR_IN_SECONDS, // 1 hour
69+
);
70+
71+
// Setup background cleanup job for expired temp uploads
72+
setInterval(
73+
async () => {
74+
await cleanupExpiredTempUploads();
5275
},
53-
1000 * 60 * 60, // 1 hours
76+
HOUR_IN_SECONDS, // 1 hour
5477
);
5578
});
5679

‎apps/api/src/media/GetPageProps.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
1+
import { AccessControl } from "@medialit/models";
12
import mongoose from "mongoose";
23

34
export default interface GetPageProps {
45
userId: mongoose.Types.ObjectId;
56
apikey: string;
6-
access: "public-read" | "private";
7+
access: AccessControl;
78
page: number;
89
recordsPerPage: number;
910
group?: string;

‎apps/api/src/media/cleanup.ts‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
import logger from "../services/log";
2+
import MediaModel from "./model";
3+
import { deleteFolder } from "../services/s3";
4+
import { CLOUD_PREFIX, TEMP_MEDIA_EXPIRATION_HOURS } from "../config/constants";
5+
6+
export async function cleanupExpiredTempUploads(): Promise<void> {
7+
const cutoff = new Date(
8+
Date.now() - TEMP_MEDIA_EXPIRATION_HOURS * 1000 * 60 * 60,
9+
);
10+
11+
try {
12+
const expired = await MediaModel.find({
13+
temp: true,
14+
createdAt: { $lt: cutoff },
15+
}).lean();
16+
17+
if (expired.length === 0) {
18+
logger.info("No expired temp uploads found to cleanup");
19+
return;
20+
}
21+
22+
logger.info(
23+
{ count: expired.length },
24+
"Found expired temp uploads to cleanup",
25+
);
26+
27+
let count = 0;
28+
for (const media of expired) {
29+
try {
30+
// Delete S3 objects in tmp folder
31+
const tmpPrefix = `${CLOUD_PREFIX ? `${CLOUD_PREFIX}/` : ""}tmp/${media.mediaId}/`;
32+
await deleteFolder(tmpPrefix);
33+
34+
// Delete media record
35+
await MediaModel.deleteOne({ _id: media._id });
36+
count++;
37+
} catch (err: any) {
38+
logger.error(
39+
{ err, mediaId: media.mediaId },
40+
"Error cleaning up expired temp upload",
41+
);
42+
}
43+
}
44+
logger.info({ count }, "Cleaned up expired temp uploads");
45+
} catch (err: any) {
46+
logger.error({ err }, "Error in cleanupExpiredTempUploads");
47+
}
48+
}
49+
50+
export default cleanupExpiredTempUploads;

‎apps/api/src/media/handlers.ts‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ export async function uploadMedia(
3232
res: any,
3333
next: (...args: any[]) => void,
3434
) {
35-
req.socket.setTimeout(10 * 60 * 1000);
35+
req.socket.setTimeout(10 * 60 * 1000); // 10 minutes
3636

3737
if (!req.files || !req.files.file) {
3838
return res.status(400).json({ error: FILE_IS_REQUIRED });
@@ -180,3 +180,31 @@ export async function deleteMedia(req: any, res: any) {
180180
return res.status(500).json(err.message);
181181
}
182182
}
183+
184+
export async function sealMedia(req: any, res: any) {
185+
const { mediaId } = req.params;
186+
187+
try {
188+
const media = await mediaService.sealMedia({
189+
userId: req.user.id,
190+
apikey: req.apikey,
191+
mediaId,
192+
});
193+
194+
const mediaDetails = await mediaService.getMediaDetails({
195+
userId: req.user.id,
196+
apikey: req.apikey,
197+
mediaId: media.mediaId,
198+
});
199+
200+
return res.status(200).json(mediaDetails);
201+
} catch (err: any) {
202+
logger.error({ err }, err.message);
203+
const statusCode =
204+
err.message === "Media not found" ||
205+
err.message === "Media is already sealed"
206+
? 404
207+
: 500;
208+
return res.status(statusCode).json({ error: err.message });
209+
}
210+
}

‎apps/api/src/media/model.ts‎

Lines changed: 1 addition & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,4 @@
1-
import { Media } from "@medialit/models";
1+
import { MediaSchema } from "@medialit/models";
22
import mongoose from "mongoose";
33

4-
export type MediaWithUserId = Media & { userId: mongoose.Types.ObjectId };
5-
6-
const MediaSchema = new mongoose.Schema<MediaWithUserId>(
7-
{
8-
fileName: { type: String, required: true },
9-
mediaId: { type: String, required: true },
10-
userId: { type: mongoose.Schema.Types.ObjectId, required: true },
11-
apikey: { type: String, required: true },
12-
originalFileName: { type: String, required: true },
13-
mimeType: { type: String, required: true },
14-
size: { type: Number, required: true },
15-
thumbnailGenerated: { type: Boolean, required: true, default: false },
16-
accessControl: { type: String, required: true, default: "private" },
17-
group: { type: String },
18-
caption: { type: String },
19-
},
20-
{
21-
timestamps: true,
22-
},
23-
);
24-
25-
MediaSchema.index({
26-
originalFileName: "text",
27-
caption: "text",
28-
});
29-
304
export default mongoose.models.Media || mongoose.model("Media", MediaSchema);

0 commit comments

Comments
 (0)