diff --git a/scripts/storage-emulator-integration/conformance/gcs.endpoints.test.ts b/scripts/storage-emulator-integration/conformance/gcs.endpoints.test.ts index ad29d3971be..c0c5b47fcc2 100644 --- a/scripts/storage-emulator-integration/conformance/gcs.endpoints.test.ts +++ b/scripts/storage-emulator-integration/conformance/gcs.endpoints.test.ts @@ -329,6 +329,59 @@ describe("GCS endpoint conformance tests", () => { expect(returnedMetadata.contentType).to.equal("text/plain"); }); + + emulatorOnly.it( + "should handle signed post policy uploads via the emulator endpoint", + async () => { + const boundary = "signed-post-policy-boundary"; + const startBuffer = Buffer.from(`--${boundary}\r +Content-Disposition: form-data; name="key"\r +\r +${TEST_FILE_NAME}\r +--${boundary}\r +Content-Disposition: form-data; name="Content-Type"\r +\r +text/plain\r +--${boundary}\r +Content-Disposition: form-data; name="Cache-Control"\r +\r +public, max-age=60\r +--${boundary}\r +Content-Disposition: form-data; name="Content-Disposition"\r +\r +inline\r +--${boundary}\r +Content-Disposition: form-data; name="x-goog-meta-color"\r +\r +blue\r +--${boundary}\r +Content-Disposition: form-data; name="file"; filename="testFile"\r +Content-Type: text/plain\r +\r +`); + const endBuffer = Buffer.from(`\r +--${boundary}--\r +`); + const body = Buffer.concat([startBuffer, Buffer.from("hello world"), endBuffer]); + + await supertest(storageHost) + .post(`/${storageBucket}`) + .set("content-type", `multipart/form-data; boundary=${boundary}`) + .send(body) + .expect(204); + + const metadata = await supertest(storageHost) + .get(`/storage/v1/b/${storageBucket}/o/${ENCODED_TEST_FILE_NAME}`) + .expect(200) + .then((res) => res.body); + + expect(metadata.name).to.equal(TEST_FILE_NAME); + expect(metadata.contentType).to.equal("text/plain"); + expect(metadata.cacheControl).to.equal("public, max-age=60"); + expect(metadata.contentDisposition).to.equal("inline"); + expect(metadata.metadata).to.deep.equal({ color: "blue" }); + }, + ); }); }); diff --git a/src/emulator/storage/apis/gcloud.ts b/src/emulator/storage/apis/gcloud.ts index 99d9fd45766..68e23889d23 100644 --- a/src/emulator/storage/apis/gcloud.ts +++ b/src/emulator/storage/apis/gcloud.ts @@ -12,7 +12,7 @@ import { StorageEmulator } from "../index"; import { EmulatorLogger } from "../../emulatorLogger"; import { GetObjectResponse, ListObjectsResponse } from "../files"; import type { Request, Response } from "express"; -import { parseObjectUploadMultipartRequest } from "../multipart"; +import { parseFormDataMultipartRequest, parseObjectUploadMultipartRequest } from "../multipart"; import { Upload, UploadNotActiveError } from "../upload"; import { ForbiddenError, NotFoundError } from "../errors"; import { reqBodyToBuffer } from "../../shared/request"; @@ -368,6 +368,77 @@ export function createCloudEndpoints(emulator: StorageEmulator): Router { return sendFileBytes(getObjectResponse.metadata, getObjectResponse.data, req, res); }); + gcloudStorageAPI.post( + "/:bucketId", + (req, res, next) => { + adminStorageLayer.createBucket(req.params.bucketId); + next(); + }, + async (req, res) => { + const contentTypeHeader = req.header("content-type"); + if (!contentTypeHeader?.includes("multipart/form-data")) { + return res.status(400).send("Content-Type must be multipart/form-data"); + } + + try { + const bodyBuffer = await reqBodyToBuffer(req); + + const formData = parseFormDataMultipartRequest(contentTypeHeader, bodyBuffer); + + const keyPart = formData.find((p) => p.name === "key"); + const filePart = formData.find((p) => p.type === "file"); + + if (keyPart?.type !== "field" || filePart?.type !== "file") { + return res.status(400).send("Missing 'key' or file."); + } + + const metadata: IncomingMetadata = { + contentType: filePart.contentType, + metadata: {}, + }; + + const HEADER_MAP: Record = { + "content-type": "contentType", + "cache-control": "cacheControl", + "content-disposition": "contentDisposition", + "content-encoding": "contentEncoding", + "content-language": "contentLanguage", + }; + for (const part of formData) { + if (part.type === "file" || part.name === "key") continue; + const key = part.name.toLowerCase(); + if (key.startsWith("x-goog-meta-")) { + metadata.metadata![key.substring(12)] = part.value; + } else if (HEADER_MAP[key]) { + (metadata[HEADER_MAP[key]] as string) = part.value.trim(); + } + } + + const upload = uploadService.multipartUpload({ + bucketId: req.params.bucketId, + objectId: keyPart.value, + dataRaw: filePart.data, + metadata: metadata, + authorization: req.header("authorization"), + }); + + await adminStorageLayer.uploadObject(upload); + return res.sendStatus(204); + } catch (err) { + if (err instanceof ForbiddenError) { + return res.sendStatus(403); + } + if (err instanceof NotFoundError) { + return res.sendStatus(404); + } + if (err instanceof Error) { + return res.status(400).send(err.message); + } + throw err; + } + }, + ); + gcloudStorageAPI.post( "/b/:bucketId/o/:objectId/:method(rewriteTo|copyTo)/b/:destBucketId/o/:destObjectId", (req, res, next) => { diff --git a/src/emulator/storage/multipart.spec.ts b/src/emulator/storage/multipart.spec.ts index 7131a184261..ff074eeb93b 100644 --- a/src/emulator/storage/multipart.spec.ts +++ b/src/emulator/storage/multipart.spec.ts @@ -1,5 +1,10 @@ import { expect } from "chai"; -import { parseObjectUploadMultipartRequest } from "./multipart"; +import { + parseObjectUploadMultipartRequest, + parseFormDataMultipartRequest, + MultipartFile, + MultipartField, +} from "./multipart"; import { randomBytes } from "crypto"; describe("Storage Multipart Request Parser", () => { @@ -142,4 +147,90 @@ hello there! ); }); }); + + describe("#parseFormDataMultipartRequest()", () => { + const boundary = "b1d5b2e3-1845-4338-9400-6ac07ce53c1e"; + const CONTENT_TYPE_HEADER = `multipart/form-data; boundary=${boundary}`; + + const pngSignature = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + const imageContent = Buffer.concat([pngSignature, randomBytes(100)]); + const startBuffer = Buffer.from(`--${boundary}\r +Content-Disposition: form-data; name="key"\r +\r +path/image.png\r +--${boundary}\r +Content-Disposition: form-data; name="content-type"\r +\r +image/png\r +--${boundary}\r +Content-Disposition: form-data; name="x-goog-meta-color"\r +\r +blue\r +--${boundary}\r +Content-Disposition: form-data; name="file"; filename="image.png"\r +Content-Type: image/png\r +\r +`); + const endBuffer = Buffer.from(`\r +--${boundary}--\r +`); + const BODY = Buffer.concat([startBuffer, imageContent, endBuffer]); + + it("parses a form data multipart request with file and fields successfully", () => { + const parts = parseFormDataMultipartRequest(CONTENT_TYPE_HEADER, BODY); + + expect(parts.length).to.equal(4); + + const keyPart = parts.find((p) => p.name === "key") as MultipartField; + expect(keyPart.value).to.equal("path/image.png"); + + const contentTypePart = parts.find((p) => p.name === "content-type") as MultipartField; + expect(contentTypePart.value).to.equal("image/png"); + + const metadataPart = parts.find((p) => p.name === "x-goog-meta-color") as MultipartField; + expect(metadataPart.value).to.equal("blue"); + + const filePart = parts.find((p) => p.name === "file") as MultipartFile; + expect(filePart.filename).to.equal("image.png"); + expect(filePart.contentType).to.equal("image/png"); + expect(filePart.data.byteLength).to.equal(imageContent.byteLength); + }); + + it("fails to parse a form data multipart request when file part is missing name", () => { + const invalidBody = Buffer.from(`--${boundary}\r +Content-Disposition: form-data; name="key"\r +\r +path/image.png\r +--${boundary}\r +Content-Disposition: form-data; filename="image.png"\r +Content-Type: image/png\r +\r +hello there!\r +--${boundary}--\r +`); + + expect(() => parseFormDataMultipartRequest(CONTENT_TYPE_HEADER, invalidBody)).to.throw( + "Missing 'name' in Content-Disposition header.", + ); + }); + + it("fails to parse a form data multipart request when part body is missing trailing line separator", () => { + const invalidStartBuffer = Buffer.from(`--${boundary}\r +Content-Disposition: form-data; name="key"\r +\r +path/image.png\r +--${boundary}\r +Content-Disposition: form-data; name="file"; filename="image.png"\r +Content-Type: image/png\r +\r +`); + const invalidEndBuffer = Buffer.from(`--${boundary}--\r +`); + const invalidBody = Buffer.concat([invalidStartBuffer, imageContent, invalidEndBuffer]); + + expect(() => parseFormDataMultipartRequest(CONTENT_TYPE_HEADER, invalidBody)).to.throw( + "Missing trailing line separator.", + ); + }); + }); }); diff --git a/src/emulator/storage/multipart.ts b/src/emulator/storage/multipart.ts index bb0850a5275..4ff87bccba5 100644 --- a/src/emulator/storage/multipart.ts +++ b/src/emulator/storage/multipart.ts @@ -46,11 +46,11 @@ function splitBufferByDelimiter(buffer: Buffer, delimiter: string, maxResults = } /** - * Parses a multipart request body buffer into a {@link MultipartRequestBody}. + * Splits a multipart request body buffer into its raw parts based on the boundary. * @param boundaryId the boundary id of the multipart request * @param body multipart request body as a Buffer */ -function parseMultipartRequestBody(boundaryId: string, body: Buffer): MultipartRequestBody { +function splitMultipartIntoParts(boundaryId: string, body: Buffer): Buffer[] { // strip additional surrounding single and double quotes, cloud sdks have additional quote here const cleanBoundaryId = boundaryId.replace(/^["'](.+(?=["']$))["']$/, "$1"); const boundaryString = `--${cleanBoundaryId}`; @@ -59,8 +59,23 @@ function parseMultipartRequestBody(boundaryId: string, body: Buffer): MultipartR return Buffer.from(buf.slice(2)); }); // A valid split request body should have two extra Buffers, one at the beginning and end. + if (bodyParts.length < 2) { + return []; + } + + return bodyParts.slice(1, bodyParts.length - 1); +} + +/** + * Parses a multipart request body buffer into a {@link MultipartRequestBody}. + * @param boundaryId the boundary id of the multipart request + * @param body multipart request body as a Buffer + */ +function parseMultipartRequestBody(boundaryId: string, body: Buffer): MultipartRequestBody { + const rawParts = splitMultipartIntoParts(boundaryId, body); const parsedParts: MultipartRequestBodyPart[] = []; - for (const bodyPart of bodyParts.slice(1, bodyParts.length - 1)) { + + for (const bodyPart of rawParts) { parsedParts.push(parseMultipartRequestBodyPart(bodyPart)); } return parsedParts; @@ -140,3 +155,113 @@ export function parseObjectUploadMultipartRequest( dataRaw: Buffer.from(parsedBody[1].dataRaw), }; } + +/** + * @param boundaryId the boundary id of the multipart request + * @param body a multipart request body part as a Buffer + */ +function parseMultipartFormDataBody(boundaryId: string, body: Buffer): MultipartFormDataPart[] { + const rawParts = splitMultipartIntoParts(boundaryId, body); + const parsedParts: MultipartFormDataPart[] = []; + + for (const bodyPart of rawParts) { + parsedParts.push(parseMultipartFormDataBodyPart(bodyPart)); + } + return parsedParts; +} + +/** + * Represents a single part of a multipart/form-data request. + * Can be a regular field or a file. + */ +export type MultipartField = { + type: "field"; + name: string; + value: string; +}; + +export type MultipartFile = { + type: "file"; + name: string; + filename: string; + contentType: string; + data: Buffer; +}; + +export type MultipartFormDataPart = MultipartField | MultipartFile; + +export type MultipartFormData = MultipartFormDataPart[]; + +function parseMultipartFormDataBodyPart(part: Buffer): MultipartFormDataPart { + const doubleCRLF = `${LINE_SEPARATOR}${LINE_SEPARATOR}`; + const sections = splitBufferByDelimiter(part, doubleCRLF, 2); + if (sections.length < 2) { + throw new Error("Failed to parse multipart part: Missing header-body separator."); + } + + const headerBuffer = sections[0]; + const bodyBuffer = sections[1]; + + const dataRaw = bodyBuffer.slice(0, bodyBuffer.byteLength - LINE_SEPARATOR.length); + + const headers = headerBuffer.toString().split(LINE_SEPARATOR); + let name = ""; + let filename: string | undefined; + let contentType: string | undefined; + + for (const h of headers) { + const lowerH = h.toLowerCase(); + if (lowerH.startsWith("content-disposition:")) { + name = extractParam(h, "name") ?? name; + filename = extractParam(h, "filename") ?? filename; + } else if (lowerH.startsWith("content-type:")) { + contentType = h.substring("content-type:".length).trim(); + } + } + + if (!name) { + throw new Error( + "Failed to parse multipart form-data. Missing 'name' in Content-Disposition header.", + ); + } + + if (filename) { + return { + type: "file", + name, + filename, + contentType: contentType || "application/octet-stream", + data: dataRaw, + }; + } else { + return { + type: "field", + name, + value: dataRaw.toString(), + }; + } +} + +function extractParam(header: string, param: string): string | undefined { + const match = header.match(new RegExp(`\\b${param}=["']?([^"';]+)["']?`, "i")); + return match ? match[1] : undefined; +} + +/** + * @param contentTypeHeader value of ContentType header passed in request. + * @param body string value of the body of the multipart request. + */ +export function parseFormDataMultipartRequest( + contentTypeHeader: string, + body: Buffer, +): MultipartFormData { + if (!contentTypeHeader.startsWith("multipart/form-data")) { + throw new Error(`Bad content type. ${contentTypeHeader}`); + } + const boundaryId = contentTypeHeader.split("boundary=")[1]?.split(";")[0]?.trim(); + if (!boundaryId) { + throw new Error(`Bad content type. ${contentTypeHeader}`); + } + + return parseMultipartFormDataBody(boundaryId, body); +}