Skip to content

Commit 6b5e5ea

Browse files
matt-aitkenmatt-aitken
committed
Support migrating presigned uploads
For large outputs - Old SDKs will continue to use the old object storage - New SDKs can use migrated object storage
1 parent dd0891e commit 6b5e5ea

File tree

8 files changed

+249
-13
lines changed

8 files changed

+249
-13
lines changed

‎.changeset/packet-v2-packets-api.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@trigger.dev/core": patch
3+
---
4+
5+
Large run outputs can use the new API which allows switching object storage providers.

‎apps/webapp/app/routes/api.v1.packets.$.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@ export async function action({ request, params }: ActionFunctionArgs) {
2929
authenticationResult.environment.project.externalRef,
3030
authenticationResult.environment.slug,
3131
filename,
32-
"PUT"
32+
"PUT",
33+
{ forceNoPrefix: true }
3334
);
3435

3536
if (!signed.success) {

‎apps/webapp/app/routes/api.v2.packets.$.ts‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
import type { ActionFunctionArgs } from "@remix-run/server-runtime";
2+
import { json } from "@remix-run/server-runtime";
3+
import { z } from "zod";
4+
import { authenticateApiRequest } from "~/services/apiAuth.server";
5+
import { generatePresignedUrl } from "~/v3/objectStore.server";
6+
7+
const ParamsSchema = z.object({
8+
"*": z.string(),
9+
});
10+
11+
/**
12+
* PUT-only presign for packet uploads (SDK offload). Uses OBJECT_STORE_DEFAULT_PROTOCOL for
13+
* unprefixed keys; returns canonical storagePath for IOPacket.data. GET presigns use v1.
14+
*/
15+
export async function action({ request, params }: ActionFunctionArgs) {
16+
if (request.method.toUpperCase() !== "PUT") {
17+
return { status: 405, body: "Method Not Allowed" };
18+
}
19+
20+
const authenticationResult = await authenticateApiRequest(request);
21+
22+
if (!authenticationResult) {
23+
return json({ error: "Invalid or Missing API key" }, { status: 401 });
24+
}
25+
26+
const parsedParams = ParamsSchema.parse(params);
27+
const filename = parsedParams["*"];
28+
29+
const signed = await generatePresignedUrl(
30+
authenticationResult.environment.project.externalRef,
31+
authenticationResult.environment.slug,
32+
filename,
33+
"PUT"
34+
);
35+
36+
if (!signed.success) {
37+
return json({ error: `Failed to generate presigned URL: ${signed.error}` }, { status: 500 });
38+
}
39+
40+
if (signed.storagePath === undefined) {
41+
return json({ error: "Failed to resolve storage path for packet upload" }, { status: 500 });
42+
}
43+
44+
return json({ presignedUrl: signed.url, storagePath: signed.storagePath });
45+
}

‎apps/webapp/app/v3/objectStore.server.ts‎

Lines changed: 58 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -196,11 +196,48 @@ export async function downloadPacketFromObjectStore(
196196
});
197197
}
198198

199+
export type GeneratePacketPresignOptions = {
200+
/**
201+
* When true (v1 packet PUT only), unprefixed keys use the legacy default object store only.
202+
* When false/omitted (v2 packet PUT), unprefixed keys also use OBJECT_STORE_DEFAULT_PROTOCOL.
203+
* Ignored for GET — reads never infer protocol from env for unprefixed keys.
204+
*/
205+
forceNoPrefix?: boolean;
206+
};
207+
208+
/**
209+
* Resolve object-store protocol for packet presigns.
210+
* GET: never apply OBJECT_STORE_DEFAULT_PROTOCOL to unprefixed keys.
211+
* PUT: optional forceNoPrefix for v1 legacy upload behavior.
212+
*/
213+
export function resolveStoreProtocolForPacketPresign(
214+
filename: string,
215+
method: "PUT" | "GET",
216+
forceNoPrefix?: boolean
217+
): { path: string; storeProtocol: string | undefined } {
218+
const { protocol: explicitProtocol, path } = parseStorageUri(filename);
219+
220+
if (method === "GET") {
221+
return { path, storeProtocol: explicitProtocol };
222+
}
223+
224+
if (explicitProtocol !== undefined) {
225+
return { path, storeProtocol: explicitProtocol };
226+
}
227+
228+
if (forceNoPrefix) {
229+
return { path, storeProtocol: undefined };
230+
}
231+
232+
return { path, storeProtocol: env.OBJECT_STORE_DEFAULT_PROTOCOL };
233+
}
234+
199235
export async function generatePresignedRequest(
200236
projectRef: string,
201237
envSlug: string,
202238
filename: string,
203-
method: "PUT" | "GET" = "PUT"
239+
method: "PUT" | "GET" = "PUT",
240+
options?: GeneratePacketPresignOptions
204241
): Promise<
205242
| {
206243
success: false;
@@ -209,23 +246,29 @@ export async function generatePresignedRequest(
209246
| {
210247
success: true;
211248
request: Request;
249+
/** Canonical pointer for IOPacket.data (PUT only). */
250+
storagePath?: string;
212251
}
213252
> {
214-
const { protocol, path } = parseStorageUri(filename);
253+
const { path, storeProtocol } = resolveStoreProtocolForPacketPresign(
254+
filename,
255+
method,
256+
options?.forceNoPrefix
257+
);
215258

216-
const config = getObjectStoreConfig(protocol);
259+
const config = getObjectStoreConfig(storeProtocol);
217260
if (!config?.baseUrl) {
218261
return {
219262
success: false,
220-
error: `Object store is not configured for protocol: ${protocol || "default"}`,
263+
error: `Object store is not configured for protocol: ${storeProtocol || "default"}`,
221264
};
222265
}
223266

224-
const client = getObjectStoreClient(protocol);
267+
const client = getObjectStoreClient(storeProtocol);
225268
if (!client) {
226269
return {
227270
success: false,
228-
error: `Object store is not configured for protocol: ${protocol || "default"}`,
271+
error: `Object store is not configured for protocol: ${storeProtocol || "default"}`,
229272
};
230273
}
231274

@@ -239,12 +282,15 @@ export async function generatePresignedRequest(
239282
projectRef,
240283
envSlug,
241284
filename,
242-
protocol: protocol || "default",
285+
protocol: storeProtocol || "default",
243286
});
244287

288+
const storagePath = method === "PUT" ? formatStorageUri(path, storeProtocol) : undefined;
289+
245290
return {
246291
success: true,
247292
request: new Request(url, { method }),
293+
storagePath,
248294
};
249295
} catch (error) {
250296
return {
@@ -260,7 +306,8 @@ export async function generatePresignedUrl(
260306
projectRef: string,
261307
envSlug: string,
262308
filename: string,
263-
method: "PUT" | "GET" = "PUT"
309+
method: "PUT" | "GET" = "PUT",
310+
options?: GeneratePacketPresignOptions
264311
): Promise<
265312
| {
266313
success: false;
@@ -269,9 +316,10 @@ export async function generatePresignedUrl(
269316
| {
270317
success: true;
271318
url: string;
319+
storagePath?: string;
272320
}
273321
> {
274-
const signed = await generatePresignedRequest(projectRef, envSlug, filename, method);
322+
const signed = await generatePresignedRequest(projectRef, envSlug, filename, method, options);
275323

276324
if (!signed.success) {
277325
return {
@@ -283,5 +331,6 @@ export async function generatePresignedUrl(
283331
return {
284332
success: true,
285333
url: signed.request.url,
334+
storagePath: signed.storagePath,
286335
};
287336
}

‎apps/webapp/test/objectStore.test.ts‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import {
1010
generatePresignedUrl,
1111
hasObjectStoreClient,
1212
parseStorageUri,
13+
resolveStoreProtocolForPacketPresign,
1314
uploadPacketToObjectStore,
1415
} from "~/v3/objectStore.server";
1516

@@ -103,6 +104,36 @@ describe("Object Storage", () => {
103104
});
104105
});
105106

107+
describe("resolveStoreProtocolForPacketPresign", () => {
108+
afterEach(() => {
109+
env.OBJECT_STORE_DEFAULT_PROTOCOL = originalEnvObj.OBJECT_STORE_DEFAULT_PROTOCOL;
110+
});
111+
112+
it("GET uses legacy default for unprefixed keys even when OBJECT_STORE_DEFAULT_PROTOCOL is set", () => {
113+
env.OBJECT_STORE_DEFAULT_PROTOCOL = "s3";
114+
expect(resolveStoreProtocolForPacketPresign("a/b.json", "GET").storeProtocol).toBeUndefined();
115+
});
116+
117+
it("PUT without forceNoPrefix uses OBJECT_STORE_DEFAULT_PROTOCOL for unprefixed keys", () => {
118+
env.OBJECT_STORE_DEFAULT_PROTOCOL = "s3";
119+
expect(resolveStoreProtocolForPacketPresign("a/b.json", "PUT", false).storeProtocol).toBe(
120+
"s3"
121+
);
122+
});
123+
124+
it("PUT with forceNoPrefix skips OBJECT_STORE_DEFAULT_PROTOCOL for unprefixed keys", () => {
125+
env.OBJECT_STORE_DEFAULT_PROTOCOL = "s3";
126+
expect(resolveStoreProtocolForPacketPresign("a/b.json", "PUT", true).storeProtocol).toBeUndefined();
127+
});
128+
129+
it("explicit protocol in key wins for PUT with forceNoPrefix", () => {
130+
env.OBJECT_STORE_DEFAULT_PROTOCOL = "r2";
131+
expect(resolveStoreProtocolForPacketPresign("s3://x/y.json", "PUT", true).storeProtocol).toBe(
132+
"s3"
133+
);
134+
});
135+
});
136+
106137
postgresAndMinioTest(
107138
"should upload and download data without protocol (legacy)",
108139
async ({ minioConfig, prisma }) => {
@@ -356,6 +387,7 @@ describe("Object Storage", () => {
356387
const putResult = await generatePresignedUrl(projectRef, envSlug, filename, "PUT");
357388
expect(putResult.success).toBe(true);
358389
if (!putResult.success) throw new Error(putResult.error);
390+
expect(putResult.storagePath).toBe(filename);
359391

360392
const putResponse = await fetch(putResult.url, {
361393
method: "PUT",
@@ -368,13 +400,106 @@ describe("Object Storage", () => {
368400
const getResult = await generatePresignedUrl(projectRef, envSlug, filename, "GET");
369401
expect(getResult.success).toBe(true);
370402
if (!getResult.success) throw new Error(getResult.error);
403+
expect(getResult.storagePath).toBeUndefined();
371404

372405
const getResponse = await fetch(getResult.url);
373406
expect(getResponse.ok).toBe(true);
374407
expect(await getResponse.text()).toBe(data);
375408
}
376409
);
377410

411+
postgresAndMinioTest(
412+
"generatePresignedUrl - PUT unprefixed with OBJECT_STORE_DEFAULT_PROTOCOL returns s3 storagePath",
413+
async ({ minioConfig }) => {
414+
env.OBJECT_STORE_BASE_URL = undefined;
415+
env.OBJECT_STORE_ACCESS_KEY_ID = undefined;
416+
env.OBJECT_STORE_SECRET_ACCESS_KEY = undefined;
417+
env.OBJECT_STORE_REGION = undefined;
418+
419+
process.env.OBJECT_STORE_S3_BASE_URL = minioConfig.baseUrl;
420+
process.env.OBJECT_STORE_S3_ACCESS_KEY_ID = minioConfig.accessKeyId;
421+
process.env.OBJECT_STORE_S3_SECRET_ACCESS_KEY = minioConfig.secretAccessKey;
422+
process.env.OBJECT_STORE_S3_REGION = minioConfig.region;
423+
process.env.OBJECT_STORE_S3_SERVICE = "s3";
424+
425+
env.OBJECT_STORE_DEFAULT_PROTOCOL = "s3";
426+
427+
const projectRef = "proj_presign_v2_style";
428+
const envSlug = "dev";
429+
const filename = "v2-style/payload.json";
430+
const data = JSON.stringify({ v2: true });
431+
432+
const putResult = await generatePresignedUrl(projectRef, envSlug, filename, "PUT");
433+
expect(putResult.success).toBe(true);
434+
if (!putResult.success) throw new Error(putResult.error);
435+
expect(putResult.storagePath).toBe(`s3://${filename}`);
436+
437+
const putResponse = await fetch(putResult.url, {
438+
method: "PUT",
439+
headers: { "Content-Type": "application/json" },
440+
body: data,
441+
});
442+
expect(putResponse.ok).toBe(true);
443+
444+
const getResult = await generatePresignedUrl(projectRef, envSlug, putResult.storagePath!, "GET");
445+
expect(getResult.success).toBe(true);
446+
if (!getResult.success) throw new Error(getResult.error);
447+
448+
const getResponse = await fetch(getResult.url);
449+
expect(getResponse.ok).toBe(true);
450+
expect(await getResponse.text()).toBe(data);
451+
452+
delete process.env.OBJECT_STORE_S3_BASE_URL;
453+
delete process.env.OBJECT_STORE_S3_ACCESS_KEY_ID;
454+
delete process.env.OBJECT_STORE_S3_SECRET_ACCESS_KEY;
455+
delete process.env.OBJECT_STORE_S3_REGION;
456+
delete process.env.OBJECT_STORE_S3_SERVICE;
457+
env.OBJECT_STORE_DEFAULT_PROTOCOL = undefined;
458+
}
459+
);
460+
461+
postgresAndMinioTest(
462+
"generatePresignedUrl - forceNoPrefix PUT fails without legacy default when only S3 named",
463+
async ({ minioConfig }) => {
464+
env.OBJECT_STORE_BASE_URL = undefined;
465+
env.OBJECT_STORE_ACCESS_KEY_ID = undefined;
466+
env.OBJECT_STORE_SECRET_ACCESS_KEY = undefined;
467+
env.OBJECT_STORE_REGION = undefined;
468+
469+
process.env.OBJECT_STORE_S3_BASE_URL = minioConfig.baseUrl;
470+
process.env.OBJECT_STORE_S3_ACCESS_KEY_ID = minioConfig.accessKeyId;
471+
process.env.OBJECT_STORE_S3_SECRET_ACCESS_KEY = minioConfig.secretAccessKey;
472+
process.env.OBJECT_STORE_S3_REGION = minioConfig.region;
473+
process.env.OBJECT_STORE_S3_SERVICE = "s3";
474+
475+
env.OBJECT_STORE_DEFAULT_PROTOCOL = "s3";
476+
477+
const putLegacy = await generatePresignedUrl(
478+
"proj_force_noprefix",
479+
"dev",
480+
"only-legacy/payload.json",
481+
"PUT",
482+
{ forceNoPrefix: true }
483+
);
484+
expect(putLegacy.success).toBe(false);
485+
486+
const getUnprefixed = await generatePresignedUrl(
487+
"proj_force_noprefix",
488+
"dev",
489+
"any.json",
490+
"GET"
491+
);
492+
expect(getUnprefixed.success).toBe(false);
493+
494+
delete process.env.OBJECT_STORE_S3_BASE_URL;
495+
delete process.env.OBJECT_STORE_S3_ACCESS_KEY_ID;
496+
delete process.env.OBJECT_STORE_S3_SECRET_ACCESS_KEY;
497+
delete process.env.OBJECT_STORE_S3_REGION;
498+
delete process.env.OBJECT_STORE_S3_SERVICE;
499+
env.OBJECT_STORE_DEFAULT_PROTOCOL = undefined;
500+
}
501+
);
502+
378503
postgresAndMinioTest(
379504
"generatePresignedUrl - PUT then GET round-trip (IAM credential chain / AWS SDK path)",
380505
async ({ minioConfig }) => {
@@ -398,6 +523,7 @@ describe("Object Storage", () => {
398523
const putResult = await generatePresignedUrl(projectRef, envSlug, filename, "PUT");
399524
expect(putResult.success).toBe(true);
400525
if (!putResult.success) throw new Error(putResult.error);
526+
expect(putResult.storagePath).toBe(filename);
401527

402528
const putResponse = await fetch(putResult.url, {
403529
method: "PUT",

‎packages/core/src/v3/apiClient/index.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -559,9 +559,10 @@ export class ApiClient {
559559
}
560560

561561
createUploadPayloadUrl(filename: string, requestOptions?: ZodFetchOptions) {
562+
const encoded = encodeURIComponent(filename);
562563
return zodfetch(
563564
CreateUploadPayloadUrlResponseBody,
564-
`${this.baseUrl}/api/v1/packets/${filename}`,
565+
`${this.baseUrl}/api/v2/packets/${encoded}`,
565566
{
566567
method: "PUT",
567568
headers: this.#getHeaders(false),
@@ -571,9 +572,10 @@ export class ApiClient {
571572
}
572573

573574
getPayloadUrl(filename: string, requestOptions?: ZodFetchOptions) {
575+
const encoded = encodeURIComponent(filename);
574576
return zodfetch(
575577
CreateUploadPayloadUrlResponseBody,
576-
`${this.baseUrl}/api/v1/packets/${filename}`,
578+
`${this.baseUrl}/api/v1/packets/${encoded}`,
577579
{
578580
method: "GET",
579581
headers: this.#getHeaders(false),

‎packages/core/src/v3/schemas/api.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -788,6 +788,8 @@ export const DeploymentEventFromString = z
788788

789789
export const CreateUploadPayloadUrlResponseBody = z.object({
790790
presignedUrl: z.string(),
791+
/** Present on `/api/v2/packets` PUT (upload handshake); omitted on v1 GET download presign. */
792+
storagePath: z.string().optional(),
791793
});
792794

793795
export const WorkersListResponseBody = z

0 commit comments

Comments
 (0)