Skip to content

feat: support concurrent chunk uploads #116

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
TorstenDittmann wants to merge 13 commits into
base: main
Choose a base branch
from
Open

feat: support concurrent chunk uploads #116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
349ba8c
feat: support concurrent chunk uploads
TorstenDittmann May 21, 2026
6d88893
Commit from GitHub Actions (Format and push)
TorstenDittmann May 21, 2026
13c8f55
feat: support concurrent chunk uploads
TorstenDittmann May 21, 2026
3ef20d6
Commit from GitHub Actions (Format and push)
TorstenDittmann May 21, 2026
103bf09
feat: support concurrent chunk uploads
TorstenDittmann May 21, 2026
88545f5
Commit from GitHub Actions (Format and push)
TorstenDittmann May 21, 2026
3229d20
feat: support concurrent chunk uploads
TorstenDittmann May 21, 2026
770aa80
Commit from GitHub Actions (Format and push)
TorstenDittmann May 21, 2026
9878e2e
feat: support concurrent chunk uploads
TorstenDittmann May 21, 2026
08eea66
Commit from GitHub Actions (Format and push)
TorstenDittmann May 21, 2026
3b0b501
chore: trigger CI
TorstenDittmann May 21, 2026
9629bc8
chore: bump SDK version to 24.2.0
TorstenDittmann May 21, 2026
5a875bb
Commit from GitHub Actions (Format and push)
TorstenDittmann May 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ Add this to your package's `pubspec.yaml` file:

```yml
dependencies:
dart_appwrite: ^24.1.0
dart_appwrite: ^24.2.0
```

You can install packages from the command line:
Expand Down
116 changes: 96 additions & 20 deletions lib/src/client_browser.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ class ClientBrowser extends ClientBase with ClientMixin {
'x-sdk-name': 'Dart',
'x-sdk-platform': 'server',
'x-sdk-language': 'dart',
'x-sdk-version': '24.1.0',
'x-sdk-version': '24.2.0',
'X-Appwrite-Response-Format': '1.9.5',
};

Expand Down Expand Up @@ -196,6 +196,7 @@ class ClientBrowser extends ClientBase with ClientMixin {
}

var offset = 0;
String? uploadId;
if (idParamName.isNotEmpty) {
//make a request to check if a file already exists
try {
Expand All @@ -206,33 +207,108 @@ class ClientBrowser extends ClientBase with ClientMixin {
);
final int chunksUploaded = res.data['chunksUploaded'] as int;
offset = chunksUploaded * chunkSize;
uploadId = res.data['\$id'] ?? params[idParamName]?.toString();
} on AppwriteException catch (_) {}
}

while (offset < size) {
if (offset >= size) {
return res;
}

final totalChunks = (size / chunkSize).ceil();

Future<Response> uploadChunk(
int index, int start, int end, String? id) async {
List<int> chunk = [];
final end = min(offset + chunkSize, size);
chunk = file.bytes!.getRange(offset, end).toList();
params[paramName] = http.MultipartFile.fromBytes(paramName, chunk,
chunk = file.bytes!.getRange(start, end).toList();

final chunkParams = Map<String, dynamic>.from(params);
chunkParams[paramName] = http.MultipartFile.fromBytes(paramName, chunk,
filename: file.filename);
headers['content-range'] =
'bytes $offset-${min<int>((offset + chunkSize - 1), size - 1)}/$size';
res = await call(HttpMethod.post,
path: path, headers: headers, params: params);
offset += chunkSize;
if (offset < size) {
headers['x-appwrite-id'] = res.data['\$id'];
final chunkHeaders = Map<String, String>.from(headers);
if (id != null && id.isNotEmpty) {
chunkHeaders['x-appwrite-id'] = id;
}
final progress = UploadProgress(
$id: res.data['\$id'] ?? '',
progress: min(offset, size) / size * 100,
sizeUploaded: min(offset, size),
chunksTotal: res.data['chunksTotal'] ?? 0,
chunksUploaded: res.data['chunksUploaded'] ?? 0,
chunkHeaders['content-range'] = 'bytes $start-${end - 1}/$size';

return call(
HttpMethod.post,
path: path,
headers: chunkHeaders,
params: chunkParams,
);
onProgress?.call(progress);
}
return res;

final firstStart = offset;
final firstEnd = min(firstStart + chunkSize, size);
final firstIndex = firstStart ~/ chunkSize;
res = await uploadChunk(firstIndex, firstStart, firstEnd, uploadId);
uploadId = res.data['\$id'] ?? uploadId;

var completedChunks = firstIndex + 1;
var uploadedBytes = firstEnd;
var lastResponse = res;
Response? finalResponse;

bool isUploadComplete(Response response) {
final chunksUploaded = response.data['chunksUploaded'];
final chunksTotal = response.data['chunksTotal'] ?? totalChunks;
return chunksUploaded is num &&
chunksTotal is num &&
chunksUploaded >= chunksTotal;
}

final progress = UploadProgress(
$id: uploadId ?? '',
progress: min(uploadedBytes, size) / size * 100,
sizeUploaded: min(uploadedBytes, size),
chunksTotal: totalChunks,
chunksUploaded: completedChunks,
);
onProgress?.call(progress);

final chunks = <Map<String, int>>[];
for (var start = firstEnd; start < size; start += chunkSize) {
final end = min(start + chunkSize, size);
chunks.add({
'index': start ~/ chunkSize,
'start': start,
'end': end,
});
}

var nextChunk = 0;
Future<void> uploadNext() async {
while (nextChunk < chunks.length) {
final chunk = chunks[nextChunk++];
final chunkResponse = await uploadChunk(
chunk['index']!,
chunk['start']!,
chunk['end']!,
uploadId,
);
completedChunks++;
uploadedBytes += chunk['end']! - chunk['start']!;
lastResponse = chunkResponse;
if (isUploadComplete(chunkResponse)) {
finalResponse = chunkResponse;
}
Comment thread
greptile-apps[bot] marked this conversation as resolved.

final progress = UploadProgress(
$id: uploadId ?? '',
progress: min(uploadedBytes, size) / size * 100,
sizeUploaded: min(uploadedBytes, size),
chunksTotal: totalChunks,
chunksUploaded: completedChunks,
);
onProgress?.call(progress);
}
}

final concurrency = min(8, chunks.length);
await Future.wait(List.generate(concurrency, (_) => uploadNext()));
Copy link
Copy Markdown

@greptile-apps greptile-apps Bot May 21, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Uncancelled workers continue uploading after a failure

Future.wait with the default eagerError: true propagates the first error immediately, but does not cancel the remaining uploadNext futures — they keep running on the event loop. Concretely: if chunk N receives a 5xx from the server, the caller's await throws, but up to 7 other workers are still sending HTTP requests. onProgress keeps firing on the closed-over closures, and completedChunks/lastResponse keep mutating. If the caller catches the error and retries, the resume-offset query will see a higher chunksUploaded than anticipated (those background requests may have been accepted by the server), leading to duplicate-chunk or skipped-chunk scenarios. The sequential implementation avoided this entirely because a thrown exception exited the loop immediately. The same issue is present in client_io.dart at the equivalent line.


return finalResponse ?? lastResponse;
}

@override
Expand Down
142 changes: 114 additions & 28 deletions lib/src/client_io.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@ class ClientIO extends ClientBase with ClientMixin {
'x-sdk-name': 'Dart',
'x-sdk-platform': 'server',
'x-sdk-language': 'dart',
'x-sdk-version': '24.1.0',
'x-sdk-version': '24.2.0',
'user-agent':
'AppwriteDartSDK/24.1.0 (${Platform.operatingSystem}; ${Platform.operatingSystemVersion})',
'AppwriteDartSDK/24.2.0 (${Platform.operatingSystem}; ${Platform.operatingSystemVersion})',
'X-Appwrite-Response-Format': '1.9.5',
};

Expand Down Expand Up @@ -217,6 +217,7 @@ class ClientIO extends ClientBase with ClientMixin {
}

var offset = 0;
String? uploadId;
if (idParamName.isNotEmpty) {
//make a request to check if a file already exists
try {
Expand All @@ -227,45 +228,130 @@ class ClientIO extends ClientBase with ClientMixin {
);
final int chunksUploaded = res.data['chunksUploaded'] as int;
offset = chunksUploaded * chunkSize;
uploadId = res.data['\$id'] ?? params[idParamName]?.toString();
} on AppwriteException catch (_) {}
}

RandomAccessFile? raf;
// read chunk and upload each chunk
if (iofile != null) {
raf = await iofile.open(mode: FileMode.read);
if (offset >= size) {
return res;
}

while (offset < size) {
final totalChunks = (size / chunkSize).ceil();

Future<Response> uploadChunk(int index, int start, int end, String? id,
[RandomAccessFile? raf]) async {
List<int> chunk = [];
if (file.bytes != null) {
final end = min(offset + chunkSize, size);
chunk = file.bytes!.getRange(offset, end).toList();
chunk = file.bytes!.getRange(start, end).toList();
} else {
raf!.setPositionSync(offset);
chunk = raf.readSync(chunkSize);
if (raf != null) {
await raf.setPosition(start);
chunk = await raf.read(end - start);
} else {
final chunkFile = await iofile!.open(mode: FileMode.read);
try {
await chunkFile.setPosition(start);
chunk = await chunkFile.read(end - start);
} finally {
await chunkFile.close();
}
}
}
params[paramName] = http.MultipartFile.fromBytes(paramName, chunk,

final chunkParams = Map<String, dynamic>.from(params);
chunkParams[paramName] = http.MultipartFile.fromBytes(paramName, chunk,
filename: file.filename);
headers['content-range'] =
'bytes $offset-${min<int>((offset + chunkSize - 1), size - 1)}/$size';
res = await call(HttpMethod.post,
path: path, headers: headers, params: params);
offset += chunkSize;
if (offset < size) {
headers['x-appwrite-id'] = res.data['\$id'];
final chunkHeaders = Map<String, String>.from(headers);
if (id != null && id.isNotEmpty) {
chunkHeaders['x-appwrite-id'] = id;
}
final progress = UploadProgress(
$id: res.data['\$id'] ?? '',
progress: min(offset, size) / size * 100,
sizeUploaded: min(offset, size),
chunksTotal: res.data['chunksTotal'] ?? 0,
chunksUploaded: res.data['chunksUploaded'] ?? 0,
chunkHeaders['content-range'] = 'bytes $start-${end - 1}/$size';

return call(
HttpMethod.post,
path: path,
headers: chunkHeaders,
params: chunkParams,
);
onProgress?.call(progress);
}
raf?.close();
return res;

final firstStart = offset;
final firstEnd = min(firstStart + chunkSize, size);
final firstIndex = firstStart ~/ chunkSize;
res = await uploadChunk(firstIndex, firstStart, firstEnd, uploadId);
uploadId = res.data['\$id'] ?? uploadId;

var completedChunks = firstIndex + 1;
var uploadedBytes = firstEnd;
var lastResponse = res;
Response? finalResponse;

bool isUploadComplete(Response response) {
final chunksUploaded = response.data['chunksUploaded'];
final chunksTotal = response.data['chunksTotal'] ?? totalChunks;
return chunksUploaded is num &&
chunksTotal is num &&
chunksUploaded >= chunksTotal;
}

final progress = UploadProgress(
$id: uploadId ?? '',
progress: min(uploadedBytes, size) / size * 100,
sizeUploaded: min(uploadedBytes, size),
chunksTotal: totalChunks,
chunksUploaded: completedChunks,
);
onProgress?.call(progress);

final chunks = <Map<String, int>>[];
for (var start = firstEnd; start < size; start += chunkSize) {
final end = min(start + chunkSize, size);
chunks.add({
'index': start ~/ chunkSize,
'start': start,
'end': end,
});
}

var nextChunk = 0;
Future<void> uploadNext() async {
final raf =
file.bytes == null ? await iofile!.open(mode: FileMode.read) : null;
try {
while (nextChunk < chunks.length) {
final chunk = chunks[nextChunk++];
final chunkResponse = await uploadChunk(
chunk['index']!,
chunk['start']!,
chunk['end']!,
uploadId,
raf,
);
completedChunks++;
uploadedBytes += chunk['end']! - chunk['start']!;
lastResponse = chunkResponse;
if (isUploadComplete(chunkResponse)) {
finalResponse = chunkResponse;
}

final progress = UploadProgress(
$id: uploadId ?? '',
progress: min(uploadedBytes, size) / size * 100,
sizeUploaded: min(uploadedBytes, size),
chunksTotal: totalChunks,
chunksUploaded: completedChunks,
);
onProgress?.call(progress);
}
} finally {
await raf?.close();
}
}

final concurrency = min(8, chunks.length);
await Future.wait(List.generate(concurrency, (_) => uploadNext()));

return finalResponse ?? lastResponse;
}

@override
Expand Down
2 changes: 1 addition & 1 deletion pubspec.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
name: dart_appwrite
version: 24.1.0
version: 24.2.0
description: Appwrite is an open-source self-hosted backend server that abstracts and simplifies complex and repetitive development tasks behind a very simple REST API
homepage: https://appwrite.io
repository: https://github.com/appwrite/sdk-for-dart
Expand Down