Skip to content

Commit ee7ca4c

Browse files
mattyateamattyatea
committed
ChunkedUpload
1 parent 811048d commit ee7ca4c

5 files changed

Lines changed: 795 additions & 75 deletions

File tree

‎packages/backend/src/server/api/endpoint-list.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,9 @@ export * as 'drive/files/find-by-hash' from './endpoints/drive/files/find-by-has
188188
export * as 'drive/files/show' from './endpoints/drive/files/show.js';
189189
export * as 'drive/files/update' from './endpoints/drive/files/update.js';
190190
export * as 'drive/files/move-bulk' from './endpoints/drive/files/move-bulk.js';
191+
export * as 'drive/files/upload-init' from './endpoints/drive/files/upload-init.js';
192+
export * as 'drive/files/upload-chunk' from './endpoints/drive/files/upload-chunk.js';
193+
export * as 'drive/files/upload-commit' from './endpoints/drive/files/upload-commit.js';
191194
export * as 'drive/files/upload-from-url' from './endpoints/drive/files/upload-from-url.js';
192195
export * as 'drive/folders' from './endpoints/drive/folders.js';
193196
export * as 'drive/folders/create' from './endpoints/drive/folders/create.js';

‎packages/backend/src/server/api/endpoints/drive/files/upload-chunk.ts‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
/*
2+
* SPDX-FileCopyrightText: syuilo and misskey-project
3+
* SPDX-License-Identifier: AGPL-3.0-only
4+
*/
5+
6+
import * as fs from 'node:fs';
7+
import { promises as fsp } from 'node:fs';
8+
import { pipeline } from 'node:stream/promises';
9+
import { Inject, Injectable } from '@nestjs/common';
10+
import * as Redis from 'ioredis';
11+
import { Endpoint } from '@/server/api/endpoint-base.js';
12+
import { DI } from '@/di-symbols.js';
13+
import { ApiError } from '../../../error.js';
14+
15+
const chunkSize = 5 * 1024 * 1024;
16+
const sessionTtl = 60 * 30;
17+
18+
type UploadSession = {
19+
sessionId: string;
20+
userId: string;
21+
name: string;
22+
size: number;
23+
folderId: string | null;
24+
isSensitive: boolean;
25+
comment: string | null;
26+
force: boolean;
27+
totalChunks: number;
28+
tempDir: string;
29+
tempFilePath: string;
30+
};
31+
32+
function getSessionKey(sessionId: string): string {
33+
return `drive:chunk-upload:session:${sessionId}`;
34+
}
35+
36+
function getChunksKey(sessionId: string): string {
37+
return `drive:chunk-upload:session:${sessionId}:chunks`;
38+
}
39+
40+
export const meta = {
41+
tags: ['drive'],
42+
43+
requireCredential: true,
44+
45+
requireFile: true,
46+
47+
prohibitMoved: true,
48+
49+
kind: 'write:drive',
50+
51+
description: 'Upload a chunk for a chunked drive file upload session.',
52+
53+
res: {
54+
type: 'object',
55+
optional: false, nullable: false,
56+
properties: {
57+
uploaded: { type: 'number' },
58+
},
59+
required: ['uploaded'],
60+
},
61+
62+
errors: {
63+
sessionNotFound: {
64+
message: 'Upload session not found.',
65+
code: 'SESSION_NOT_FOUND',
66+
id: '4e53158b-2687-43cf-a607-f9fddeafddf9',
67+
},
68+
invalidChunkIndex: {
69+
message: 'Invalid chunk index.',
70+
code: 'INVALID_CHUNK_INDEX',
71+
id: 'a9d3439d-6228-4e5e-8f56-e9b50f0e0d26',
72+
},
73+
invalidChunkSize: {
74+
message: 'Invalid chunk size.',
75+
code: 'INVALID_CHUNK_SIZE',
76+
id: 'a67226fc-ec7d-4118-91be-a21cf7b20a45',
77+
},
78+
},
79+
} as const;
80+
81+
export const paramDef = {
82+
type: 'object',
83+
properties: {
84+
sessionId: { type: 'string' },
85+
index: { type: 'number', minimum: 0 },
86+
},
87+
required: ['sessionId', 'index'],
88+
} as const;
89+
90+
@Injectable()
91+
export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
92+
constructor(
93+
@Inject(DI.redis)
94+
private redisClient: Redis.Redis,
95+
) {
96+
super(meta, paramDef, async (ps, me, _, file, cleanup) => {
97+
if (file == null) {
98+
throw new ApiError();
99+
}
100+
101+
try {
102+
const sessionJson = await this.redisClient.get(getSessionKey(ps.sessionId));
103+
if (sessionJson == null) {
104+
throw new ApiError(meta.errors.sessionNotFound);
105+
}
106+
107+
const session = JSON.parse(sessionJson) as UploadSession;
108+
if (session.userId !== me.id) {
109+
throw new ApiError(meta.errors.sessionNotFound);
110+
}
111+
112+
if (!Number.isInteger(ps.index) || ps.index < 0 || ps.index >= session.totalChunks) {
113+
throw new ApiError(meta.errors.invalidChunkIndex);
114+
}
115+
116+
const stats = await fsp.stat(file.path);
117+
const actualChunkSize = stats.size;
118+
const expectedChunkSize = Math.min(chunkSize, session.size - (ps.index * chunkSize));
119+
120+
if (actualChunkSize > chunkSize || actualChunkSize !== expectedChunkSize) {
121+
throw new ApiError(meta.errors.invalidChunkSize);
122+
}
123+
124+
await pipeline(
125+
fs.createReadStream(file.path),
126+
fs.createWriteStream(session.tempFilePath, {
127+
flags: 'r+',
128+
start: ps.index * chunkSize,
129+
}),
130+
);
131+
132+
const uploadedResult = await this.redisClient.multi()
133+
.sadd(getChunksKey(ps.sessionId), String(ps.index))
134+
.scard(getChunksKey(ps.sessionId))
135+
.expire(getSessionKey(ps.sessionId), sessionTtl)
136+
.expire(getChunksKey(ps.sessionId), sessionTtl)
137+
.exec();
138+
139+
const uploaded = Number(uploadedResult?.[1]?.[1] ?? 0);
140+
141+
return {
142+
uploaded,
143+
};
144+
} finally {
145+
cleanup?.();
146+
}
147+
});
148+
}
149+
}

‎packages/backend/src/server/api/endpoints/drive/files/upload-commit.ts‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
/*
2+
* SPDX-FileCopyrightText: syuilo and misskey-project
3+
* SPDX-License-Identifier: AGPL-3.0-only
4+
*/
5+
6+
import * as fs from 'node:fs/promises';
7+
import { Inject, Injectable } from '@nestjs/common';
8+
import * as Redis from 'ioredis';
9+
import { IdentifiableError } from '@/misc/identifiable-error.js';
10+
import { Endpoint } from '@/server/api/endpoint-base.js';
11+
import { DriveFileEntityService } from '@/core/entities/DriveFileEntityService.js';
12+
import { DriveService } from '@/core/DriveService.js';
13+
import { MiMeta } from '@/models/_.js';
14+
import { DI } from '@/di-symbols.js';
15+
import { ApiError } from '../../../error.js';
16+
17+
type UploadSession = {
18+
sessionId: string;
19+
userId: string;
20+
name: string;
21+
size: number;
22+
folderId: string | null;
23+
isSensitive: boolean;
24+
comment: string | null;
25+
force: boolean;
26+
totalChunks: number;
27+
tempDir: string;
28+
tempFilePath: string;
29+
};
30+
31+
function getSessionKey(sessionId: string): string {
32+
return `drive:chunk-upload:session:${sessionId}`;
33+
}
34+
35+
function getChunksKey(sessionId: string): string {
36+
return `drive:chunk-upload:session:${sessionId}:chunks`;
37+
}
38+
39+
async function cleanupUpload(redisClient: Redis.Redis, session: UploadSession): Promise<void> {
40+
await redisClient.del(getSessionKey(session.sessionId), getChunksKey(session.sessionId));
41+
await fs.rm(session.tempDir, { recursive: true, force: true }).catch(() => {});
42+
}
43+
44+
export const meta = {
45+
tags: ['drive'],
46+
47+
requireCredential: true,
48+
49+
prohibitMoved: true,
50+
51+
kind: 'write:drive',
52+
53+
description: 'Commit a chunked drive file upload session.',
54+
55+
res: {
56+
type: 'object',
57+
optional: false, nullable: false,
58+
ref: 'DriveFile',
59+
},
60+
61+
errors: {
62+
sessionNotFound: {
63+
message: 'Upload session not found.',
64+
code: 'SESSION_NOT_FOUND',
65+
id: '4df76536-4f72-4c80-a4f8-4be45d09d5f7',
66+
},
67+
68+
missingChunks: {
69+
message: 'Some chunks are missing.',
70+
code: 'MISSING_CHUNKS',
71+
id: '15c5a42d-b790-4e7c-b4b6-6a0bf8713910',
72+
},
73+
74+
inappropriate: {
75+
message: 'Cannot upload the file because it has been determined that it possibly contains inappropriate content.',
76+
code: 'INAPPROPRIATE',
77+
id: 'bec5bd69-fba3-43c9-b4fb-2894b66ad5d2',
78+
},
79+
80+
noFreeSpace: {
81+
message: 'Cannot upload the file because you have no free space of drive.',
82+
code: 'NO_FREE_SPACE',
83+
id: 'd08dbc37-a6a9-463a-8c47-96c32ab5f064',
84+
},
85+
86+
maxFileSizeExceeded: {
87+
message: 'Cannot upload the file because it exceeds the maximum file size.',
88+
code: 'MAX_FILE_SIZE_EXCEEDED',
89+
id: 'b9d8c348-33f0-4673-b9a9-5d4da058977a',
90+
httpStatusCode: 413,
91+
},
92+
93+
unallowedFileType: {
94+
message: 'Cannot upload the file because it is an unallowed file type.',
95+
code: 'UNALLOWED_FILE_TYPE',
96+
id: '4becd248-7f2c-48c4-a9f0-75edc4f9a1ea',
97+
},
98+
},
99+
} as const;
100+
101+
export const paramDef = {
102+
type: 'object',
103+
properties: {
104+
sessionId: { type: 'string' },
105+
},
106+
required: ['sessionId'],
107+
} as const;
108+
109+
@Injectable()
110+
export default class extends Endpoint<typeof meta, typeof paramDef> { // eslint-disable-line import/no-default-export
111+
constructor(
112+
@Inject(DI.meta)
113+
private serverSettings: MiMeta,
114+
115+
@Inject(DI.redis)
116+
private redisClient: Redis.Redis,
117+
118+
private driveFileEntityService: DriveFileEntityService,
119+
private driveService: DriveService,
120+
) {
121+
super(meta, paramDef, async (ps, me, _, _1, _2, ip, headers) => {
122+
const sessionJson = await this.redisClient.get(getSessionKey(ps.sessionId));
123+
if (sessionJson == null) {
124+
throw new ApiError(meta.errors.sessionNotFound);
125+
}
126+
127+
const session = JSON.parse(sessionJson) as UploadSession;
128+
if (session.userId !== me.id) {
129+
throw new ApiError(meta.errors.sessionNotFound);
130+
}
131+
132+
const uploadedChunks = new Set(await this.redisClient.smembers(getChunksKey(ps.sessionId)));
133+
for (let index = 0; index < session.totalChunks; index++) {
134+
if (!uploadedChunks.has(String(index))) {
135+
throw new ApiError(meta.errors.missingChunks);
136+
}
137+
}
138+
139+
const stats = await fs.stat(session.tempFilePath);
140+
if (stats.size !== session.size) {
141+
throw new ApiError(meta.errors.missingChunks);
142+
}
143+
144+
try {
145+
const driveFile = await this.driveService.addFile({
146+
user: me,
147+
path: session.tempFilePath,
148+
name: session.name,
149+
comment: session.comment,
150+
folderId: session.folderId,
151+
force: session.force,
152+
sensitive: session.isSensitive,
153+
requestIp: this.serverSettings.enableIpLogging ? ip : null,
154+
requestHeaders: this.serverSettings.enableIpLogging ? headers : null,
155+
});
156+
157+
await cleanupUpload(this.redisClient, session);
158+
159+
return await this.driveFileEntityService.pack(driveFile, { self: true });
160+
} catch (err) {
161+
await cleanupUpload(this.redisClient, session);
162+
if (err instanceof Error || typeof err === 'string') {
163+
console.error(err);
164+
}
165+
if (err instanceof IdentifiableError) {
166+
if (err.id === '282f77bf-5816-4f72-9264-aa14d8261a21') throw new ApiError(meta.errors.inappropriate);
167+
if (err.id === 'c6244ed2-a39a-4e1c-bf93-f0fbd7764fa6') throw new ApiError(meta.errors.noFreeSpace);
168+
if (err.id === 'f9e4e5f3-4df4-40b5-b400-f236945f7073') throw new ApiError(meta.errors.maxFileSizeExceeded);
169+
if (err.id === 'bd71c601-f9b0-4808-9137-a330647ced9b') throw new ApiError(meta.errors.unallowedFileType);
170+
}
171+
throw new ApiError();
172+
}
173+
});
174+
}
175+
}

0 commit comments

Comments
 (0)