feat(storage-objects): Adds a helper method to create an Object from a local directory (#656)

Author: adam-rl

package.json

@@ -42,6 +42,7 @@
     "form-data-encoder": "1.7.2",
     "formdata-node": "^4.3.2",
     "node-fetch": "^2.6.7",
+    "tar": "^7.5.2",
     "uuidv7": "^1.0.2",
     "zod": "^3.24.1"
   },

src/sdk.ts

@@ -488,6 +488,38 @@ export class StorageObjectOps {
   ): Promise<StorageObject> {
     return StorageObject.uploadFromBuffer(this.client, buffer, name, contentType, options);
   }
+
+  /**
+   * Upload a local directory as a gzipped tarball (Node.js only).
+   * This method creates a tar archive of the directory contents, gzips it, and uploads it.
+   *
+   * @example
+   * ```typescript
+   * const runloop = new RunloopSDK();
+   *
+   * const object = await runloop.storageObject.uploadFromDir(
+   *   './my-project',
+   *   {
+   *     name: 'my-project.tar.gz',
+   *     ttl_ms: 3600000, // 1 hour
+   *     metadata: { project: 'demo' }
+   *   }
+   * );
+   * console.log(`Uploaded directory as ${object.id}`);
+   * ```
+   *
+   * @param {string} dirPath - The path to the directory to upload.
+   * @param {Omit<ObjectCreateParams, 'content_type'} params - Parameters for creating the object.
+   * @param {Core.RequestOptions} options - Request options.
+   * @returns {Promise<StorageObject>} A {@link StorageObject} instance.
+   */
+  async uploadFromDir(
+    dirPath: string,
+    params: Omit<ObjectCreateParams, 'content_type'>,
+    options?: Core.RequestOptions,
+  ): Promise<StorageObject> {
+    return StorageObject.uploadFromDir(this.client, dirPath, params, options);
+  }
 }
 
 // @deprecated Use {@link RunloopSDK} instead.

src/sdk/storage-object.ts

@@ -8,6 +8,7 @@ import type {
 } from '../resources/objects';
 import * as fs from 'node:fs/promises';
 import * as path from 'node:path';
+import * as tar from 'tar';
 
 // Extract the content type from the API types
 type ContentType = ObjectCreateParams['content_type'];
@@ -488,6 +489,75 @@ export class StorageObject {
     return storageObject;
   }
 
+  /**
+   * @hidden
+   */
+  static async uploadFromDir(
+    client: Runloop,
+    dirPath: string,
+    params: Omit<ObjectCreateParams, 'content_type'>,
+    options?: Core.RequestOptions,
+  ): Promise<StorageObject> {
+    assertNodeEnvironment();
+
+    // Verify directory exists and is actually a directory
+    try {
+      const stats = await fs.stat(dirPath);
+      if (!stats.isDirectory()) {
+        throw new Error(`Path is not a directory: ${dirPath}`);
+      }
+    } catch (error) {
+      throw new Error(
+        `Failed to access directory ${dirPath}: ${error instanceof Error ? error.message : 'Unknown error'}`,
+      );
+    }
+
+    // Create the tarball in-memory.
+    let buffer;
+    try {
+      const tarStream = tar.create({ gzip: true, cwd: dirPath }, ['.']);
+      const chunks = [];
+      for await (const chunk of tarStream) {
+        chunks.push(chunk);
+      }
+      buffer = Buffer.concat(chunks);
+    } catch (error) {
+      throw new Error(
+        `Failed to create tarball from directory ${dirPath}: ${error instanceof Error ? error.message : 'Unknown error'}`,
+      );
+    }
+
+    // Create the object.
+    const createParams: ObjectCreateParams = { ...params, content_type: 'tgz' };
+    const objectData = await client.objects.create(createParams, options);
+    const storageObject = new StorageObject(client, objectData.id, objectData.upload_url);
+
+    const uploadUrl = objectData.upload_url;
+    if (!uploadUrl) {
+      throw new Error('No upload URL available. Object may already be completed or deleted.');
+    }
+
+    // Write the tarball to the upload URL.
+    try {
+      const response = await fetch(uploadUrl, {
+        method: 'PUT',
+        body: buffer,
+      });
+
+      if (!response.ok) {
+        throw new Error(`Upload failed: ${response.status} ${response.statusText}`);
+      }
+    } catch (error) {
+      throw new Error(
+        `Failed to upload tarball: ${error instanceof Error ? error.message : 'Unknown error'}`,
+      );
+    }
+
+    await storageObject.complete();
+
+    return storageObject;
+  }
+
   /**
    * Get the object ID.
    */

tests/objects/storage-object.test.ts

@@ -19,6 +19,12 @@ jest.mock('node:path', () => ({
     const ext = path.split('.').pop();
     return ext ? `.${ext}` : '';
   }),
+  join: jest.fn((...paths) => paths.join('/')),
+}));
+
+// Mock tar module
+jest.mock('tar', () => ({
+  create: jest.fn(),
 }));
 
 describe('StorageObject (New API)', () => {
@@ -810,6 +816,146 @@ describe('StorageObject (New API)', () => {
     });
   });
 
+  describe('uploadFromDir', () => {
+    let mockTar: any;
+
+    beforeEach(() => {
+      // Clear all mocks
+      jest.clearAllMocks();
+      // Reset global fetch mock
+      ((global as any).fetch as jest.Mock).mockClear();
+      // Get tar mock
+      mockTar = require('tar');
+    });
+
+    it('should upload a directory as gzipped tarball', async () => {
+      // Mock directory exists
+      mockFs.stat.mockResolvedValue({ isDirectory: () => true });
+
+      // Mock tar stream
+      const mockTarballBuffer = Buffer.from('compressed tarball content');
+      mockTar.create.mockReturnValue({
+        [Symbol.asyncIterator]: async function* () {
+          yield mockTarballBuffer;
+        },
+      });
+
+      const mockObjectData = { id: 'dir-123', upload_url: 'https://upload.example.com/dir' };
+      const mockObjectInfo = { ...mockObjectData, name: 'project.tar.gz', state: 'UPLOADING' };
+      const mockCompletedData = { ...mockObjectInfo, state: 'READ_ONLY' };
+
+      mockClient.objects.create.mockResolvedValue(mockObjectData);
+      mockClient.objects.retrieve.mockResolvedValue(mockObjectInfo);
+      mockClient.objects.complete.mockResolvedValue(mockCompletedData);
+
+      ((global as any).fetch as jest.Mock).mockResolvedValue({
+        ok: true,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const result = await StorageObject.uploadFromDir(mockClient, './my-project', {
+        name: 'project.tar.gz',
+      });
+
+      expect(mockClient.objects.create).toHaveBeenCalledWith(
+        { name: 'project.tar.gz', content_type: 'tgz' },
+        undefined,
+      );
+      expect(mockTar.create).toHaveBeenCalled();
+      expect(result).toBeInstanceOf(StorageObject);
+      expect(result.id).toBe('dir-123');
+    });
+
+    it('should upload directory with TTL and metadata', async () => {
+      // Mock directory exists
+      mockFs.stat.mockResolvedValue({ isDirectory: () => true });
+
+      // Mock tar stream
+      const mockTarballBuffer = Buffer.from('compressed tarball');
+      mockTar.create.mockReturnValue({
+        [Symbol.asyncIterator]: async function* () {
+          yield mockTarballBuffer;
+        },
+      });
+
+      const mockObjectData = { id: 'dir-456', upload_url: 'https://upload.example.com/dir' };
+      const mockCompletedData = { ...mockObjectData, state: 'READ_ONLY' };
+
+      mockClient.objects.create.mockResolvedValue(mockObjectData);
+      mockClient.objects.complete.mockResolvedValue(mockCompletedData);
+
+      ((global as any).fetch as jest.Mock).mockResolvedValue({
+        ok: true,
+        status: 200,
+        statusText: 'OK',
+      });
+
+      const result = await StorageObject.uploadFromDir(mockClient, './my-project', {
+        name: 'project.tar.gz',
+        ttl_ms: 3600000,
+        metadata: { project: 'demo' },
+      });
+
+      expect(mockClient.objects.create).toHaveBeenCalledWith(
+        { name: 'project.tar.gz', content_type: 'tgz', metadata: { project: 'demo' }, ttl_ms: 3600000 },
+        undefined,
+      );
+      expect(result.id).toBe('dir-456');
+    });
+
+    it('should throw error if path is not a directory', async () => {
+      mockFs.stat.mockResolvedValue({ isDirectory: () => false });
+
+      await expect(
+        StorageObject.uploadFromDir(mockClient, './file.txt', { name: 'archive.tar.gz' }),
+      ).rejects.toThrow('Path is not a directory: ./file.txt');
+    });
+
+    it('should throw error if directory does not exist', async () => {
+      mockFs.stat.mockRejectedValue(new Error('ENOENT: no such file or directory'));
+
+      await expect(
+        StorageObject.uploadFromDir(mockClient, './nonexistent', { name: 'archive.tar.gz' }),
+      ).rejects.toThrow('Failed to access directory ./nonexistent');
+    });
+
+    it('should throw error in browser environment', async () => {
+      const originalProcess = global.process;
+      delete (global as any).process;
+
+      await expect(
+        StorageObject.uploadFromDir(mockClient, './project', { name: 'project.tar.gz' }),
+      ).rejects.toThrow('File upload methods are only available in Node.js environment');
+
+      global.process = originalProcess;
+    });
+
+    it('should handle upload failures gracefully', async () => {
+      mockFs.stat.mockResolvedValue({ isDirectory: () => true });
+
+      const mockTarballBuffer = Buffer.from('tarball');
+      mockTar.create.mockReturnValue({
+        [Symbol.asyncIterator]: async function* () {
+          yield mockTarballBuffer;
+        },
+      });
+
+      const mockObjectData = { id: 'dir-999', upload_url: 'https://upload.example.com/dir' };
+      mockClient.objects.create.mockResolvedValue(mockObjectData);
+
+      ((global as any).fetch as jest.Mock).mockResolvedValue({
+        ok: false,
+        status: 500,
+        statusText: 'Internal Server Error',
+      });
+
+      await expect(
+        StorageObject.uploadFromDir(mockClient, './project', { name: 'project.tar.gz' }),
+      ).rejects.toThrow('Failed to upload tarball: Upload failed: 500 Internal Server Error');
+    });
+  });
+
   describe('error handling', () => {
     it('should handle create errors', async () => {
       const error = new Error('Create failed');

tests/smoketests/object-oriented/storage-object.test.ts

@@ -1,3 +1,4 @@
+import { ReadEntry } from 'tar';
 import { THIRTY_SECOND_TIMEOUT, uniqueName, makeClientSDK } from '../utils';
 import { Devbox, StorageObject } from '@runloop/api-client/sdk';
 
@@ -195,13 +196,13 @@ describe('smoketest: object-oriented storage object', () => {
     });
 
     test('upload from file', async () => {
-      const fs = require('fs');
+      const fs = require('fs/promises');
       const path = require('path');
       const os = require('os');
 
       // Create a temporary file
       const tmpFile = path.join(os.tmpdir(), `test-upload-${Date.now()}.txt`);
-      fs.writeFileSync(tmpFile, 'Hello from uploadFromFile!');
+      await fs.writeFile(tmpFile, 'Hello from uploadFromFile!');
 
       try {
         const uploaded = await sdk.storageObject.uploadFromFile(tmpFile, uniqueName('sdk-file-upload'), {
@@ -218,13 +219,52 @@ describe('smoketest: object-oriented storage object', () => {
         await uploaded.delete();
       } finally {
         // Clean up temp file
-        if (fs.existsSync(tmpFile)) {
-          fs.unlinkSync(tmpFile);
-        }
+        await fs.unlink(tmpFile).catch(() => {});
       }
     });
   });
 
+  test('upload from dir', async () => {
+    const fs = require('fs/promises');
+    const path = require('path');
+    const os = require('os');
+    const tar = require('tar');
+
+    // Create a temporary directory with a file in it.
+    const tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'dir-to-tar-'));
+    const contentPath = path.join(tmpDir, 'content');
+    await fs.writeFile(contentPath, 'Hello from uploadFromDir!');
+    try {
+      const uploaded = await sdk.storageObject.uploadFromDir(tmpDir, { name: uniqueName('sdk-dir-upload') });
+      expect(uploaded).toBeDefined();
+      expect(uploaded.id).toBeTruthy();
+
+      // Verify content type.
+      const info = await uploaded.getInfo();
+      expect(info.content_type).toBe('tgz');
+
+      // Untar the downloaded object and check for the original
+      // file.
+      const data = await uploaded.downloadAsBuffer();
+      const contentChunks: Buffer[] = [];
+      const tarStream = tar.list({
+        onReadEntry: async (entry: ReadEntry) => {
+          if (entry.path == './content') {
+            for await (const chunk of entry) {
+              contentChunks.push(chunk);
+            }
+          }
+        },
+      });
+      await tarStream.write(data);
+      const content = Buffer.concat(contentChunks);
+      expect(content.toString('utf-8')).toBe('Hello from uploadFromDir!');
+    } finally {
+      await fs.unlink(contentPath).catch(() => {});
+      await fs.unlink(tmpDir).catch(() => {});
+    }
+  });
+
   describe('storage object list and retrieval', () => {
     test('list storage objects via SDK', async () => {
       const objects = await sdk.storageObject.list({ limit: 10 });