Skip to content

Commit 8db3816

Browse files
committed
feat(storage-objects): Adds a helper method to create an Object from a local directory
1 parent 58ac02b commit 8db3816

5 files changed

Lines changed: 299 additions & 5 deletions

File tree

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242
"form-data-encoder": "1.7.2",
4343
"formdata-node": "^4.3.2",
4444
"node-fetch": "^2.6.7",
45+
"tar": "^7.5.2",
4546
"uuidv7": "^1.0.2",
4647
"zod": "^3.24.1"
4748
},

src/sdk.ts

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -488,6 +488,47 @@ export class StorageObjectOps {
488488
): Promise<StorageObject> {
489489
return StorageObject.uploadFromBuffer(this.client, buffer, name, contentType, options);
490490
}
491+
492+
/**
493+
* Upload a local directory as a gzipped tarball (Node.js only).
494+
* This method creates a tar archive of the directory contents, gzips it, and uploads it.
495+
*
496+
* @example
497+
* ```typescript
498+
* const runloop = new RunloopSDK();
499+
* // Upload a directory with default settings
500+
* const object = await runloop.storageObject.uploadFromDir(
501+
* './my-project',
502+
* 'my-project.tar.gz',
503+
* {
504+
* ttlMs: 3600000, // 1 hour
505+
* metadata: { project: 'demo' }
506+
* }
507+
* );
508+
* console.log(`Uploaded directory as ${object.id}`);
509+
* ```
510+
*
511+
* @param {string} dirPath - The path to the directory to upload.
512+
* @param {string} name - The name to use for the storage object.
513+
* @param {Core.RequestOptions & {
514+
* ttlMs?: number | null;
515+
* metadata?: Record<string, string>;
516+
* }} [options] - Request options inluding metadata.
517+
* @returns {Promise<StorageObject>} A {@link StorageObject} instance.
518+
*/
519+
async uploadFromDir(
520+
dirPath: string,
521+
name: string,
522+
options?: Core.RequestOptions & {
523+
/**
524+
* An optional Time-To-Live for the object, in milliseconds, after which it will be automatically deleted.
525+
*/
526+
ttlMs?: number | null;
527+
metadata?: Record<string, string>;
528+
},
529+
): Promise<StorageObject> {
530+
return StorageObject.uploadFromDir(this.client, dirPath, name, options);
531+
}
491532
}
492533

493534
// @deprecated Use {@link RunloopSDK} instead.

src/sdk/storage-object.ts

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import type {
88
} from '../resources/objects';
99
import * as fs from 'node:fs/promises';
1010
import * as path from 'node:path';
11+
import * as tar from 'tar';
1112

1213
// Extract the content type from the API types
1314
type ContentType = ObjectCreateParams['content_type'];
@@ -488,6 +489,79 @@ export class StorageObject {
488489
return storageObject;
489490
}
490491

492+
/**
493+
* @hidden
494+
*/
495+
static async uploadFromDir(
496+
client: Runloop,
497+
dirPath: string,
498+
name: string,
499+
options?: Core.RequestOptions & {
500+
ttlMs?: number | null;
501+
metadata?: Record<string, string>;
502+
},
503+
): Promise<StorageObject> {
504+
assertNodeEnvironment();
505+
506+
// Verify directory exists and is actually a directory
507+
try {
508+
const stats = await fs.stat(dirPath);
509+
if (!stats.isDirectory()) {
510+
throw new Error(`Path is not a directory: ${dirPath}`);
511+
}
512+
} catch (error) {
513+
throw new Error(
514+
`Failed to access directory ${dirPath}: ${error instanceof Error ? error.message : 'Unknown error'}`,
515+
);
516+
}
517+
518+
// Prepare the tarball stream
519+
let tarStream;
520+
try {
521+
tarStream = tar.create({ gzip: true, cwd: dirPath }, ['.']);
522+
} catch (error) {
523+
throw new Error(
524+
`Failed to create tarball from directory ${dirPath}: ${error instanceof Error ? error.message : 'Unknown error'}`,
525+
);
526+
}
527+
528+
// Create the object.
529+
const createParams: ObjectCreateParams = {
530+
name,
531+
content_type: 'tgz',
532+
metadata: options?.metadata || null,
533+
ttl_ms: options?.ttlMs !== undefined ? options.ttlMs : null,
534+
};
535+
536+
const objectData = await client.objects.create(createParams, options);
537+
const storageObject = new StorageObject(client, objectData.id, objectData.upload_url);
538+
539+
const uploadUrl = objectData.upload_url;
540+
if (!uploadUrl) {
541+
throw new Error('No upload URL available. Object may already be completed or deleted.');
542+
}
543+
544+
// Stream the tarball to the upload URL.
545+
try {
546+
const response = await fetch(uploadUrl, {
547+
method: 'PUT',
548+
body: tarStream,
549+
});
550+
551+
if (!response.ok) {
552+
throw new Error(`Upload failed: ${response.status} ${response.statusText}`);
553+
}
554+
} catch (error) {
555+
throw new Error(
556+
`Failed to upload tarball: ${error instanceof Error ? error.message : 'Unknown error'}`,
557+
);
558+
}
559+
560+
await storageObject.complete();
561+
562+
return storageObject;
563+
}
564+
491565
/**
492566
* Get the object ID.
493567
*/

tests/objects/storage-object.test.ts

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,12 @@ jest.mock('node:path', () => ({
1919
const ext = path.split('.').pop();
2020
return ext ? `.${ext}` : '';
2121
}),
22+
join: jest.fn((...paths) => paths.join('/')),
23+
}));
24+
25+
// Mock tar module
26+
jest.mock('tar', () => ({
27+
create: jest.fn(),
2228
}));
2329

2430
describe('StorageObject (New API)', () => {
@@ -810,6 +816,143 @@ describe('StorageObject (New API)', () => {
810816
});
811817
});
812818

819+
describe('uploadFromDir', () => {
820+
let mockTar: any;
821+
822+
beforeEach(() => {
823+
// Clear all mocks
824+
jest.clearAllMocks();
825+
// Reset global fetch mock
826+
((global as any).fetch as jest.Mock).mockClear();
827+
// Get tar mock
828+
mockTar = require('tar');
829+
});
830+
831+
it('should upload a directory as gzipped tarball', async () => {
832+
// Mock directory exists
833+
mockFs.stat.mockResolvedValue({ isDirectory: () => true });
834+
835+
// Mock tar stream
836+
const mockTarballBuffer = Buffer.from('compressed tarball content');
837+
mockTar.create.mockReturnValue({
838+
[Symbol.asyncIterator]: async function* () {
839+
yield mockTarballBuffer;
840+
},
841+
});
842+
843+
const mockObjectData = { id: 'dir-123', upload_url: 'https://upload.example.com/dir' };
844+
const mockObjectInfo = { ...mockObjectData, name: 'project.tar.gz', state: 'UPLOADING' };
845+
const mockCompletedData = { ...mockObjectInfo, state: 'READ_ONLY' };
846+
847+
mockClient.objects.create.mockResolvedValue(mockObjectData);
848+
mockClient.objects.retrieve.mockResolvedValue(mockObjectInfo);
849+
mockClient.objects.complete.mockResolvedValue(mockCompletedData);
850+
851+
((global as any).fetch as jest.Mock).mockResolvedValue({
852+
ok: true,
853+
status: 200,
854+
statusText: 'OK',
855+
});
856+
857+
const result = await StorageObject.uploadFromDir(mockClient, './my-project', 'project.tar.gz');
858+
859+
expect(mockClient.objects.create).toHaveBeenCalledWith(
860+
{ name: 'project.tar.gz', content_type: 'tgz', metadata: null, ttl_ms: null },
861+
undefined,
862+
);
863+
expect(mockTar.create).toHaveBeenCalled();
864+
expect(result).toBeInstanceOf(StorageObject);
865+
expect(result.id).toBe('dir-123');
866+
});
867+
868+
it('should upload directory with TTL and metadata', async () => {
869+
// Mock directory exists
870+
mockFs.stat.mockResolvedValue({ isDirectory: () => true });
871+
872+
// Mock tar stream
873+
const mockTarballBuffer = Buffer.from('compressed tarball');
874+
mockTar.create.mockReturnValue({
875+
[Symbol.asyncIterator]: async function* () {
876+
yield mockTarballBuffer;
877+
},
878+
});
879+
880+
const mockObjectData = { id: 'dir-456', upload_url: 'https://upload.example.com/dir' };
881+
const mockCompletedData = { ...mockObjectData, state: 'READ_ONLY' };
882+
883+
mockClient.objects.create.mockResolvedValue(mockObjectData);
884+
mockClient.objects.complete.mockResolvedValue(mockCompletedData);
885+
886+
((global as any).fetch as jest.Mock).mockResolvedValue({
887+
ok: true,
888+
status: 200,
889+
statusText: 'OK',
890+
});
891+
892+
const result = await StorageObject.uploadFromDir(mockClient, './my-project', 'project.tar.gz', {
893+
ttlMs: 3600000,
894+
metadata: { project: 'demo' },
895+
});
896+
897+
expect(mockClient.objects.create).toHaveBeenCalledWith(
898+
{ name: 'project.tar.gz', content_type: 'tgz', metadata: { project: 'demo' }, ttl_ms: 3600000 },
899+
{ ttlMs: 3600000, metadata: { project: 'demo' } },
900+
);
901+
expect(result.id).toBe('dir-456');
902+
});
903+
904+
it('should throw error if path is not a directory', async () => {
905+
mockFs.stat.mockResolvedValue({ isDirectory: () => false });
906+
907+
await expect(StorageObject.uploadFromDir(mockClient, './file.txt', 'archive.tar.gz')).rejects.toThrow(
908+
'Path is not a directory: ./file.txt',
909+
);
910+
});
911+
912+
it('should throw error if directory does not exist', async () => {
913+
mockFs.stat.mockRejectedValue(new Error('ENOENT: no such file or directory'));
914+
915+
await expect(
916+
StorageObject.uploadFromDir(mockClient, './nonexistent', 'archive.tar.gz'),
917+
).rejects.toThrow('Failed to access directory ./nonexistent');
918+
});
919+
920+
it('should throw error in browser environment', async () => {
921+
const originalProcess = global.process;
922+
delete (global as any).process;
923+
924+
await expect(StorageObject.uploadFromDir(mockClient, './project', 'project.tar.gz')).rejects.toThrow(
925+
'File upload methods are only available in Node.js environment',
926+
);
927+
928+
global.process = originalProcess;
929+
});
930+
931+
it('should handle upload failures gracefully', async () => {
932+
mockFs.stat.mockResolvedValue({ isDirectory: () => true });
933+
934+
const mockTarballBuffer = Buffer.from('tarball');
935+
mockTar.create.mockReturnValue({
936+
[Symbol.asyncIterator]: async function* () {
937+
yield mockTarballBuffer;
938+
},
939+
});
940+
941+
const mockObjectData = { id: 'dir-999', upload_url: 'https://upload.example.com/dir' };
942+
mockClient.objects.create.mockResolvedValue(mockObjectData);
943+
944+
((global as any).fetch as jest.Mock).mockResolvedValue({
945+
ok: false,
946+
status: 500,
947+
statusText: 'Internal Server Error',
948+
});
949+
950+
await expect(StorageObject.uploadFromDir(mockClient, './project', 'project.tar.gz')).rejects.toThrow(
951+
'Failed to upload tarball: Upload failed: 500 Internal Server Error',
952+
);
953+
});
954+
});
955+
813956
describe('error handling', () => {
814957
it('should handle create errors', async () => {
815958
const error = new Error('Create failed');

yarn.lock

Lines changed: 40 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -339,6 +339,13 @@
339339
resolved "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz"
340340
integrity sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==
341341

342+
"@isaacs/fs-minipass@^4.0.0":
343+
version "4.0.1"
344+
resolved "https://registry.yarnpkg.com/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz#2d59ae3ab4b38fb4270bfa23d30f8e2e86c7fe32"
345+
integrity sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w==
346+
dependencies:
347+
minipass "^7.0.4"
348+
342349
"@istanbuljs/load-nyc-config@^1.0.0":
343350
version "1.1.0"
344351
resolved "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz"
@@ -1323,6 +1330,11 @@ char-regex@^1.0.2:
13231330
resolved "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz"
13241331
integrity sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==
13251332

1333+
chownr@^3.0.0:
1334+
version "3.0.0"
1335+
resolved "https://registry.yarnpkg.com/chownr/-/chownr-3.0.0.tgz#9855e64ecd240a9cc4267ce8a4aa5d24a1da15e4"
1336+
integrity sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g==
1337+
13261338
ci-info@^3.2.0:
13271339
version "3.9.0"
13281340
resolved "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz"
@@ -2825,6 +2837,18 @@ minimist@^1.2.5, minimist@^1.2.6:
28252837
resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz"
28262838
integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==
28272839

2840+
minipass@^7.0.4, minipass@^7.1.2:
2841+
version "7.1.2"
2842+
resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707"
2843+
integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==
2844+
2845+
minizlib@^3.1.0:
2846+
version "3.1.0"
2847+
resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-3.1.0.tgz#6ad76c3a8f10227c9b51d1c9ac8e30b27f5a251c"
2848+
integrity sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==
2849+
dependencies:
2850+
minipass "^7.1.2"
2851+
28282852
ms@^2.0.0, ms@^2.1.3:
28292853
version "2.1.3"
28302854
resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz"
@@ -3302,6 +3326,17 @@ synckit@^0.11.7:
33023326
dependencies:
33033327
"@pkgr/core" "^0.2.9"
33043328

3329+
tar@^7.5.2:
3330+
version "7.5.2"
3331+
resolved "https://registry.yarnpkg.com/tar/-/tar-7.5.2.tgz#115c061495ec51ff3c6745ff8f6d0871c5b1dedc"
3332+
integrity sha512-7NyxrTE4Anh8km8iEy7o0QYPs+0JKBTj5ZaqHg6B39erLg0qYXN3BijtShwbsNSvQ+LN75+KV+C4QR/f6Gwnpg==
3333+
dependencies:
3334+
"@isaacs/fs-minipass" "^4.0.0"
3335+
chownr "^3.0.0"
3336+
minipass "^7.1.2"
3337+
minizlib "^3.1.0"
3338+
yallist "^5.0.0"
3339+
33053340
test-exclude@^6.0.0:
33063341
version "6.0.0"
33073342
resolved "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz"
@@ -3455,11 +3490,6 @@ typedoc-material-theme@^1.4.1:
34553490
dependencies:
34563491
"@material/material-color-utilities" "^0.3.0"
34573492

3458-
"typedoc-mintlify@file:../typedoc-mintlify":
3459-
version "0.1.0"
3460-
dependencies:
3461-
typedoc-plugin-markdown "^4.9.0"
3462-
34633493
typedoc-plugin-include-example@^3.0.2:
34643494
version "3.0.2"
34653495
resolved "https://registry.npmjs.org/typedoc-plugin-include-example/-/typedoc-plugin-include-example-3.0.2.tgz"
@@ -3627,6 +3657,11 @@ yallist@^3.0.2:
36273657
resolved "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz"
36283658
integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==
36293659

3660+
yallist@^5.0.0:
3661+
version "5.0.0"
3662+
resolved "https://registry.yarnpkg.com/yallist/-/yallist-5.0.0.tgz#00e2de443639ed0d78fd87de0d27469fbcffb533"
3663+
integrity sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw==
3664+
36303665
yaml@^2.8.1:
36313666
version "2.8.1"
36323667
resolved "https://registry.npmjs.org/yaml/-/yaml-2.8.1.tgz"

0 commit comments

Comments
 (0)