diff --git a/package.json b/package.json index a55f2cd82..77c9a2f28 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "form-data-encoder": "1.7.2", "formdata-node": "^4.3.2", "node-fetch": "^2.6.7", + "tar": "^7.5.2", "uuidv7": "^1.0.2", "zod": "^3.24.1" }, diff --git a/src/sdk.ts b/src/sdk.ts index 62c9d474c..9d84c981f 100644 --- a/src/sdk.ts +++ b/src/sdk.ts @@ -488,6 +488,38 @@ export class StorageObjectOps { ): Promise { return StorageObject.uploadFromBuffer(this.client, buffer, name, contentType, options); } + + /** + * Upload a local directory as a gzipped tarball (Node.js only). + * This method creates a tar archive of the directory contents, gzips it, and uploads it. + * + * @example + * ```typescript + * const runloop = new RunloopSDK(); + * + * const object = await runloop.storageObject.uploadFromDir( + * './my-project', + * { + * name: 'my-project.tar.gz', + * ttl_ms: 3600000, // 1 hour + * metadata: { project: 'demo' } + * } + * ); + * console.log(`Uploaded directory as ${object.id}`); + * ``` + * + * @param {string} dirPath - The path to the directory to upload. + * @param {Omit} A {@link StorageObject} instance. + */ + async uploadFromDir( + dirPath: string, + params: Omit, + options?: Core.RequestOptions, + ): Promise { + return StorageObject.uploadFromDir(this.client, dirPath, params, options); + } } // @deprecated Use {@link RunloopSDK} instead. diff --git a/src/sdk/storage-object.ts b/src/sdk/storage-object.ts index 1555cf19f..723d73661 100644 --- a/src/sdk/storage-object.ts +++ b/src/sdk/storage-object.ts @@ -8,6 +8,7 @@ import type { } from '../resources/objects'; import * as fs from 'node:fs/promises'; import * as path from 'node:path'; +import * as tar from 'tar'; // Extract the content type from the API types type ContentType = ObjectCreateParams['content_type']; @@ -488,6 +489,75 @@ export class StorageObject { return storageObject; } + /** + * @hidden + */ + static async uploadFromDir( + client: Runloop, + dirPath: string, + params: Omit, + options?: Core.RequestOptions, + ): Promise { + assertNodeEnvironment(); + + // Verify directory exists and is actually a directory + try { + const stats = await fs.stat(dirPath); + if (!stats.isDirectory()) { + throw new Error(`Path is not a directory: ${dirPath}`); + } + } catch (error) { + throw new Error( + `Failed to access directory ${dirPath}: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); + } + + // Create the tarball in-memory. + let buffer; + try { + const tarStream = tar.create({ gzip: true, cwd: dirPath }, ['.']); + const chunks = []; + for await (const chunk of tarStream) { + chunks.push(chunk); + } + buffer = Buffer.concat(chunks); + } catch (error) { + throw new Error( + `Failed to create tarball from directory ${dirPath}: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); + } + + // Create the object. + const createParams: ObjectCreateParams = { ...params, content_type: 'tgz' }; + const objectData = await client.objects.create(createParams, options); + const storageObject = new StorageObject(client, objectData.id, objectData.upload_url); + + const uploadUrl = objectData.upload_url; + if (!uploadUrl) { + throw new Error('No upload URL available. Object may already be completed or deleted.'); + } + + // Write the tarball to the upload URL. + try { + const response = await fetch(uploadUrl, { + method: 'PUT', + body: buffer, + }); + + if (!response.ok) { + throw new Error(`Upload failed: ${response.status} ${response.statusText}`); + } + } catch (error) { + throw new Error( + `Failed to upload tarball: ${error instanceof Error ? error.message : 'Unknown error'}`, + ); + } + + await storageObject.complete(); + + return storageObject; + } + /** * Get the object ID. */ diff --git a/tests/objects/storage-object.test.ts b/tests/objects/storage-object.test.ts index 4b8282d12..d3d2e5572 100644 --- a/tests/objects/storage-object.test.ts +++ b/tests/objects/storage-object.test.ts @@ -19,6 +19,12 @@ jest.mock('node:path', () => ({ const ext = path.split('.').pop(); return ext ? `.${ext}` : ''; }), + join: jest.fn((...paths) => paths.join('/')), +})); + +// Mock tar module +jest.mock('tar', () => ({ + create: jest.fn(), })); describe('StorageObject (New API)', () => { @@ -810,6 +816,146 @@ describe('StorageObject (New API)', () => { }); }); + describe('uploadFromDir', () => { + let mockTar: any; + + beforeEach(() => { + // Clear all mocks + jest.clearAllMocks(); + // Reset global fetch mock + ((global as any).fetch as jest.Mock).mockClear(); + // Get tar mock + mockTar = require('tar'); + }); + + it('should upload a directory as gzipped tarball', async () => { + // Mock directory exists + mockFs.stat.mockResolvedValue({ isDirectory: () => true }); + + // Mock tar stream + const mockTarballBuffer = Buffer.from('compressed tarball content'); + mockTar.create.mockReturnValue({ + [Symbol.asyncIterator]: async function* () { + yield mockTarballBuffer; + }, + }); + + const mockObjectData = { id: 'dir-123', upload_url: 'https://upload.example.com/dir' }; + const mockObjectInfo = { ...mockObjectData, name: 'project.tar.gz', state: 'UPLOADING' }; + const mockCompletedData = { ...mockObjectInfo, state: 'READ_ONLY' }; + + mockClient.objects.create.mockResolvedValue(mockObjectData); + mockClient.objects.retrieve.mockResolvedValue(mockObjectInfo); + mockClient.objects.complete.mockResolvedValue(mockCompletedData); + + ((global as any).fetch as jest.Mock).mockResolvedValue({ + ok: true, + status: 200, + statusText: 'OK', + }); + + const result = await StorageObject.uploadFromDir(mockClient, './my-project', { + name: 'project.tar.gz', + }); + + expect(mockClient.objects.create).toHaveBeenCalledWith( + { name: 'project.tar.gz', content_type: 'tgz' }, + undefined, + ); + expect(mockTar.create).toHaveBeenCalled(); + expect(result).toBeInstanceOf(StorageObject); + expect(result.id).toBe('dir-123'); + }); + + it('should upload directory with TTL and metadata', async () => { + // Mock directory exists + mockFs.stat.mockResolvedValue({ isDirectory: () => true }); + + // Mock tar stream + const mockTarballBuffer = Buffer.from('compressed tarball'); + mockTar.create.mockReturnValue({ + [Symbol.asyncIterator]: async function* () { + yield mockTarballBuffer; + }, + }); + + const mockObjectData = { id: 'dir-456', upload_url: 'https://upload.example.com/dir' }; + const mockCompletedData = { ...mockObjectData, state: 'READ_ONLY' }; + + mockClient.objects.create.mockResolvedValue(mockObjectData); + mockClient.objects.complete.mockResolvedValue(mockCompletedData); + + ((global as any).fetch as jest.Mock).mockResolvedValue({ + ok: true, + status: 200, + statusText: 'OK', + }); + + const result = await StorageObject.uploadFromDir(mockClient, './my-project', { + name: 'project.tar.gz', + ttl_ms: 3600000, + metadata: { project: 'demo' }, + }); + + expect(mockClient.objects.create).toHaveBeenCalledWith( + { name: 'project.tar.gz', content_type: 'tgz', metadata: { project: 'demo' }, ttl_ms: 3600000 }, + undefined, + ); + expect(result.id).toBe('dir-456'); + }); + + it('should throw error if path is not a directory', async () => { + mockFs.stat.mockResolvedValue({ isDirectory: () => false }); + + await expect( + StorageObject.uploadFromDir(mockClient, './file.txt', { name: 'archive.tar.gz' }), + ).rejects.toThrow('Path is not a directory: ./file.txt'); + }); + + it('should throw error if directory does not exist', async () => { + mockFs.stat.mockRejectedValue(new Error('ENOENT: no such file or directory')); + + await expect( + StorageObject.uploadFromDir(mockClient, './nonexistent', { name: 'archive.tar.gz' }), + ).rejects.toThrow('Failed to access directory ./nonexistent'); + }); + + it('should throw error in browser environment', async () => { + const originalProcess = global.process; + delete (global as any).process; + + await expect( + StorageObject.uploadFromDir(mockClient, './project', { name: 'project.tar.gz' }), + ).rejects.toThrow('File upload methods are only available in Node.js environment'); + + global.process = originalProcess; + }); + + it('should handle upload failures gracefully', async () => { + mockFs.stat.mockResolvedValue({ isDirectory: () => true }); + + const mockTarballBuffer = Buffer.from('tarball'); + mockTar.create.mockReturnValue({ + [Symbol.asyncIterator]: async function* () { + yield mockTarballBuffer; + }, + }); + + const mockObjectData = { id: 'dir-999', upload_url: 'https://upload.example.com/dir' }; + mockClient.objects.create.mockResolvedValue(mockObjectData); + + ((global as any).fetch as jest.Mock).mockResolvedValue({ + ok: false, + status: 500, + statusText: 'Internal Server Error', + }); + + await expect( + StorageObject.uploadFromDir(mockClient, './project', { name: 'project.tar.gz' }), + ).rejects.toThrow('Failed to upload tarball: Upload failed: 500 Internal Server Error'); + }); + }); + describe('error handling', () => { it('should handle create errors', async () => { const error = new Error('Create failed'); diff --git a/tests/smoketests/object-oriented/storage-object.test.ts b/tests/smoketests/object-oriented/storage-object.test.ts index 563951c1b..b28708ed9 100644 --- a/tests/smoketests/object-oriented/storage-object.test.ts +++ b/tests/smoketests/object-oriented/storage-object.test.ts @@ -1,3 +1,4 @@ +import { ReadEntry } from 'tar'; import { THIRTY_SECOND_TIMEOUT, uniqueName, makeClientSDK } from '../utils'; import { Devbox, StorageObject } from '@runloop/api-client/sdk'; @@ -195,13 +196,13 @@ describe('smoketest: object-oriented storage object', () => { }); test('upload from file', async () => { - const fs = require('fs'); + const fs = require('fs/promises'); const path = require('path'); const os = require('os'); // Create a temporary file const tmpFile = path.join(os.tmpdir(), `test-upload-${Date.now()}.txt`); - fs.writeFileSync(tmpFile, 'Hello from uploadFromFile!'); + await fs.writeFile(tmpFile, 'Hello from uploadFromFile!'); try { const uploaded = await sdk.storageObject.uploadFromFile(tmpFile, uniqueName('sdk-file-upload'), { @@ -218,13 +219,52 @@ describe('smoketest: object-oriented storage object', () => { await uploaded.delete(); } finally { // Clean up temp file - if (fs.existsSync(tmpFile)) { - fs.unlinkSync(tmpFile); - } + await fs.unlink(tmpFile).catch(() => {}); } }); }); + test('upload from dir', async () => { + const fs = require('fs/promises'); + const path = require('path'); + const os = require('os'); + const tar = require('tar'); + + // Create a temporary directory with a file in it. + const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'dir-to-tar-')); + const contentPath = path.join(tmpDir, 'content'); + await fs.writeFile(contentPath, 'Hello from uploadFromDir!'); + try { + const uploaded = await sdk.storageObject.uploadFromDir(tmpDir, { name: uniqueName('sdk-dir-upload') }); + expect(uploaded).toBeDefined(); + expect(uploaded.id).toBeTruthy(); + + // Verify content type. + const info = await uploaded.getInfo(); + expect(info.content_type).toBe('tgz'); + + // Untar the downloaded object and check for the original + // file. + const data = await uploaded.downloadAsBuffer(); + const contentChunks: Buffer[] = []; + const tarStream = tar.list({ + onReadEntry: async (entry: ReadEntry) => { + if (entry.path == './content') { + for await (const chunk of entry) { + contentChunks.push(chunk); + } + } + }, + }); + await tarStream.write(data); + const content = Buffer.concat(contentChunks); + expect(content.toString('utf-8')).toBe('Hello from uploadFromDir!'); + } finally { + await fs.unlink(contentPath).catch(() => {}); + await fs.unlink(tmpDir).catch(() => {}); + } + }); + describe('storage object list and retrieval', () => { test('list storage objects via SDK', async () => { const objects = await sdk.storageObject.list({ limit: 10 }); diff --git a/yarn.lock b/yarn.lock index efbd22165..a52640fa4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -339,6 +339,13 @@ resolved "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz" integrity sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA== +"@isaacs/fs-minipass@^4.0.0": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz#2d59ae3ab4b38fb4270bfa23d30f8e2e86c7fe32" + integrity sha512-wgm9Ehl2jpeqP3zw/7mo3kRHFp5MEDhqAdwy1fTGkHAwnkGOVsgpvQhL8B5n1qlb01jV3n/bI0ZfZp5lWA1k4w== + dependencies: + minipass "^7.0.4" + "@istanbuljs/load-nyc-config@^1.0.0": version "1.1.0" resolved "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz" @@ -1323,6 +1330,11 @@ char-regex@^1.0.2: resolved "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz" integrity sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw== +chownr@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-3.0.0.tgz#9855e64ecd240a9cc4267ce8a4aa5d24a1da15e4" + integrity sha512-+IxzY9BZOQd/XuYPRmrvEVjF/nqj5kgT4kEq7VofrDoM1MxoRjEWkrCC3EtLi59TVawxTAn+orJwFQcrqEN1+g== + ci-info@^3.2.0: version "3.9.0" resolved "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz" @@ -2825,6 +2837,18 @@ minimist@^1.2.5, minimist@^1.2.6: resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== +minipass@^7.0.4, minipass@^7.1.2: + version "7.1.2" + resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707" + integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw== + +minizlib@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/minizlib/-/minizlib-3.1.0.tgz#6ad76c3a8f10227c9b51d1c9ac8e30b27f5a251c" + integrity sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw== + dependencies: + minipass "^7.1.2" + ms@^2.0.0, ms@^2.1.3: version "2.1.3" resolved "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" @@ -3302,6 +3326,17 @@ synckit@^0.11.7: dependencies: "@pkgr/core" "^0.2.9" +tar@^7.5.2: + version "7.5.2" + resolved "https://registry.yarnpkg.com/tar/-/tar-7.5.2.tgz#115c061495ec51ff3c6745ff8f6d0871c5b1dedc" + integrity sha512-7NyxrTE4Anh8km8iEy7o0QYPs+0JKBTj5ZaqHg6B39erLg0qYXN3BijtShwbsNSvQ+LN75+KV+C4QR/f6Gwnpg== + dependencies: + "@isaacs/fs-minipass" "^4.0.0" + chownr "^3.0.0" + minipass "^7.1.2" + minizlib "^3.1.0" + yallist "^5.0.0" + test-exclude@^6.0.0: version "6.0.0" resolved "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz" @@ -3455,11 +3490,6 @@ typedoc-material-theme@^1.4.1: dependencies: "@material/material-color-utilities" "^0.3.0" -"typedoc-mintlify@file:../typedoc-mintlify": - version "0.1.0" - dependencies: - typedoc-plugin-markdown "^4.9.0" - typedoc-plugin-include-example@^3.0.2: version "3.0.2" resolved "https://registry.npmjs.org/typedoc-plugin-include-example/-/typedoc-plugin-include-example-3.0.2.tgz" @@ -3627,6 +3657,11 @@ yallist@^3.0.2: resolved "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== +yallist@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-5.0.0.tgz#00e2de443639ed0d78fd87de0d27469fbcffb533" + integrity sha512-YgvUTfwqyc7UXVMrB+SImsVYSmTS8X/tSrtdNZMImM+n7+QTriRXyXim0mBrTXNeqzVF0KWGgHPeiyViFFrNDw== + yaml@^2.8.1: version "2.8.1" resolved "https://registry.npmjs.org/yaml/-/yaml-2.8.1.tgz"