Merge pull request #107 from appwrite/concurrent-chunk-uploads-version-bump-1-9-x

chore: bump SDK version to 0.31.0

Author: TorstenDittmann

docs/examples/avatars/get-screenshot.md

@@ -20,7 +20,7 @@ const result = avatars.getScreenshot({
     userAgent: 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X) AppleWebKit/605.1.15', // optional
     fullpage: true, // optional
     locale: 'en-US', // optional
-    timezone: Timezone.AmericaNewYork, // optional
+    timezone: Timezone.AfricaAbidjan, // optional
     latitude: 37.7749, // optional
     longitude: -122.4194, // optional
     accuracy: 100, // optional

src/services/realtime.ts

@@ -239,11 +239,13 @@ export class Realtime {
             try {
                 const connectionId = ++this.connectionId;
                 const WebSocketCtor: any = WebSocket;
-                const socket = (this.socket = new WebSocketCtor(url, undefined, {
-                    headers: {
-                        Origin: `appwrite-${Platform.OS}://${this.client.config.platform}`
-                    }
-                }));
+                const socket = (this.socket = Platform.OS === 'web'
+                    ? new WebSocketCtor(url)
+                    : new WebSocketCtor(url, undefined, {
+                        headers: {
+                            Origin: `appwrite-${Platform.OS}://${this.client.config.platform}`
+                        }
+                    }));
 
                 socket.addEventListener('open', () => {
                     if (connectionId !== this.connectionId) {

src/services/storage.ts

@@ -195,42 +195,139 @@ export class Storage extends Service {
         } catch(e) {
         }
 
-        let timestamp = new Date().getTime();
-        while (offset < size) {
-            let end = Math.min(offset + Service.CHUNK_SIZE - 1, size - 1);
+        const totalChunks = Math.ceil(size / Service.CHUNK_SIZE);
 
-            apiHeaders['content-range'] = 'bytes ' + offset + '-' + end + '/' + size;
-            if (response && response.$id) {
-                apiHeaders['x-appwrite-id'] = response.$id;
-            }
+        // Upload first chunk alone to get the upload ID
+        if (offset === 0) {
+            const firstChunkEnd = Math.min(Service.CHUNK_SIZE, size);
+            const firstChunkHeaders = { ...apiHeaders, 'content-range': 'bytes 0-' + (firstChunkEnd - 1) + '/' + size };
 
-            let chunk = await FileSystem.readAsStringAsync(file.uri, {
+            let firstChunk = await FileSystem.readAsStringAsync(file.uri, {
                 encoding: FileSystem.EncodingType.Base64,
-                position: offset,
+                position: 0,
                 length: Service.CHUNK_SIZE
             });
-            var path = `data:${file.type};base64,${chunk}`;
+            var firstPath = `data:${file.type};base64,${firstChunk}`;
             if (RNPlatform.OS.toLowerCase() === 'android') {
-                path = FileSystem.cacheDirectory + '/tmp_chunk_' + timestamp;
-                await FileSystem.writeAsStringAsync(path, chunk, {encoding: FileSystem.EncodingType.Base64});
+                firstPath = FileSystem.cacheDirectory + '/tmp_chunk_' + new Date().getTime();
+                await FileSystem.writeAsStringAsync(firstPath, firstChunk, {encoding: FileSystem.EncodingType.Base64});
             }
 
-            payload['file'] = { uri: path, name: file.name, type: file.type };
+            payload['file'] = { uri: firstPath, name: file.name, type: file.type };
 
-            response = await this.client.call('post', uri, apiHeaders, payload);
+            response = await this.client.call('post', uri, firstChunkHeaders, payload);
+            offset = firstChunkEnd;
 
             if (onProgress) {
                 onProgress({
                     $id: response.$id,
                     progress: (offset / size) * 100,
                     sizeUploaded: offset,
-                    chunksTotal: response.chunksTotal,
-                    chunksUploaded: response.chunksUploaded
+                    chunksTotal: totalChunks,
+                    chunksUploaded: 1
                 });
             }
-            offset += Service.CHUNK_SIZE;
         }
-        return response;
+
+        if (offset >= size) {
+            return response;
+        }
+
+        const uploadId = response?.$id;
+        const chunks: { index: number; start: number; end: number }[] = [];
+        const startChunkIndex = Math.ceil(offset / Service.CHUNK_SIZE);
+        for (let i = startChunkIndex; i < totalChunks; i++) {
+            const start = i * Service.CHUNK_SIZE;
+            const end = Math.min(start + Service.CHUNK_SIZE, size);
+            chunks.push({ index: i, start, end });
+        }
+
+        // Upload remaining chunks with max concurrency of 8
+        const CONCURRENCY = 8;
+        let completedCount = startChunkIndex;
+        let uploadedBytes = offset;
+        let finalResponse = null;
+        let failed = false;
+
+        const isUploadComplete = (chunkResponse: any) => {
+            const chunksUploaded = chunkResponse?.chunksUploaded;
+            const chunksTotal = chunkResponse?.chunksTotal ?? totalChunks;
+            return typeof chunksUploaded === 'number' && typeof chunksTotal === 'number' && chunksUploaded >= chunksTotal;
+        };
+
+        const uploadChunk = async (chunk: typeof chunks[0]) => {
+            const chunkHeaders = { ...apiHeaders };
+            if (uploadId) {
+                chunkHeaders['x-appwrite-id'] = uploadId;
+            }
+            chunkHeaders['content-range'] = 'bytes ' + chunk.start + '-' + (chunk.end - 1) + '/' + size;
+
+            const chunkData = await FileSystem.readAsStringAsync(file.uri, {
+                encoding: FileSystem.EncodingType.Base64,
+                position: chunk.start,
+                length: chunk.end - chunk.start
+            });
+
+            let chunkPath = `data:${file.type};base64,${chunkData}`;
+            if (RNPlatform.OS.toLowerCase() === 'android') {
+                chunkPath = FileSystem.cacheDirectory + '/tmp_chunk_' + new Date().getTime() + '_' + chunk.index;
+                await FileSystem.writeAsStringAsync(chunkPath, chunkData, {encoding: FileSystem.EncodingType.Base64});
+            }
+
+            const chunkPayload = { ...payload };
+            chunkPayload['file'] = { uri: chunkPath, name: file.name, type: file.type };
+
+            const chunkResponse = await this.client.call('post', uri, chunkHeaders, chunkPayload);
+
+            if (failed) {
+                return chunkResponse;
+            }
+
+            completedCount++;
+            uploadedBytes += (chunk.end - chunk.start);
+
+            response = chunkResponse;
+            if (isUploadComplete(chunkResponse)) {
+                finalResponse = chunkResponse;
+            }
+
+            if (onProgress) {
+                onProgress({
+                    $id: uploadId,
+                    progress: (uploadedBytes / size) * 100,
+                    sizeUploaded: uploadedBytes,
+                    chunksTotal: totalChunks,
+                    chunksUploaded: completedCount
+                });
+            }
+
+            return chunkResponse;
+        };
+
+        // Process with limited concurrency using a worker pool
+        const queue = [...chunks];
+        const workers: Promise<void>[] = [];
+        const workerCount = Math.min(CONCURRENCY, queue.length);
+
+        for (let i = 0; i < workerCount; i++) {
+            workers.push(
+                (async () => {
+                    while (!failed && queue.length > 0) {
+                        const chunk = queue.shift()!;
+                        try {
+                            await uploadChunk(chunk);
+                        } catch (error) {
+                            failed = true;
+                            throw error;
+                        }
+                    }
+                })()
+            );
+        }
+
+        await Promise.all(workers);
+
+        return finalResponse ?? response;
     }
 
     /**