Skip to content

Commit e232f4e

Browse files
committed
fix: hoist invocationId to ensure persistence across upload retries
Hoists the generation of `persistentInvocationId` to the beginning of the upload process in `Bucket.upload` and `File.save`. This ensures that retried multipart upload attempts reuse the same invocation ID in the `x-goog-api-client` header, rather than generating a new one for each attempt.
1 parent b2e12e7 commit e232f4e

10 files changed

Lines changed: 344 additions & 1259 deletions

File tree

handwritten/storage/src/bucket.ts

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import * as http from 'http';
2828
import * as path from 'path';
2929
import {promisify} from 'util';
3030
import AsyncRetry from 'async-retry';
31+
import {randomUUID} from 'crypto';
3132
import {convertObjKeysToSnakeCase, handleContextValidation} from './util.js';
3233

3334
import {Acl, AclMetadata} from './acl.js';
@@ -4442,6 +4443,7 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
44424443
optionsOrCallback?: UploadOptions | UploadCallback,
44434444
callback?: UploadCallback,
44444445
): Promise<UploadResponse> | void {
4446+
const persistentInvocationId = randomUUID();
44454447
const upload = (numberOfRetries: number | undefined) => {
44464448
const returnValue = AsyncRetry(
44474449
async (bail: (err: GaxiosError | Error) => void) => {
@@ -4452,7 +4454,10 @@ class Bucket extends ServiceObject<Bucket, BucketMetadata> {
44524454
) {
44534455
newFile.storage.retryOptions.autoRetry = false;
44544456
}
4455-
const writable = newFile.createWriteStream(options);
4457+
const writable = newFile.createWriteStream({
4458+
...options,
4459+
invocationId: persistentInvocationId,
4460+
});
44564461
if (options.onUploadProgress) {
44574462
writable.on('progress', options.onUploadProgress);
44584463
}

handwritten/storage/src/file.ts

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import * as resumableUpload from './resumable-upload.js';
2727
import {Writable, Readable, pipeline, Transform, PipelineSource} from 'stream';
2828
import * as zlib from 'zlib';
2929
import * as http from 'http';
30+
import {randomUUID} from 'crypto';
3031

3132
import {
3233
ExceptionMessages,
@@ -248,6 +249,7 @@ export interface CreateResumableUploadOptions
248249
* @see {@link CRC32C.from} for possible values.
249250
*/
250251
resumeCRC32C?: Parameters<(typeof CRC32C)['from']>[0];
252+
invocationId?: string;
251253
preconditionOpts?: PreconditionOptions;
252254
[GCCL_GCS_CMD_KEY]?: resumableUpload.UploadConfig[typeof GCCL_GCS_CMD_KEY];
253255
}
@@ -4181,13 +4183,17 @@ class File extends ServiceObject<File, FileMetadata> {
41814183
) {
41824184
maxRetries = 0;
41834185
}
4186+
const persistentInvocationId = randomUUID();
41844187
const returnValue = AsyncRetry(
41854188
async (bail: (err: Error) => void) => {
41864189
return new Promise<void>((resolve, reject) => {
41874190
if (maxRetries === 0) {
41884191
this.storage.retryOptions.autoRetry = false;
41894192
}
4190-
const writable = this.createWriteStream(options);
4193+
const writable = this.createWriteStream({
4194+
...options,
4195+
invocationId: persistentInvocationId,
4196+
});
41914197

41924198
if (options.onUploadProgress) {
41934199
writable.on('progress', options.onUploadProgress);
@@ -4449,6 +4455,7 @@ class File extends ServiceObject<File, FileMetadata> {
44494455
chunkSize: options?.chunkSize,
44504456
highWaterMark: options?.highWaterMark,
44514457
universeDomain: this.bucket.storage.universeDomain,
4458+
invocationId: options.invocationId,
44524459
[GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY],
44534460
};
44544461

@@ -4508,6 +4515,7 @@ class File extends ServiceObject<File, FileMetadata> {
45084515
uploadType: 'multipart',
45094516
},
45104517
url,
4518+
invocationId: options.invocationId,
45114519
[GCCL_GCS_CMD_KEY]: options[GCCL_GCS_CMD_KEY],
45124520
method: 'POST',
45134521
responseType: 'json',

handwritten/storage/src/nodejs-common/service.ts

Lines changed: 0 additions & 316 deletions
This file was deleted.

0 commit comments

Comments
 (0)