feat: allow user-provided x-goog-gcs-idempotency-token and synchronize with gccl-invocation-id header

Author: thiyaguk09

handwritten/storage/src/nodejs-common/service.ts

@@ -269,15 +269,23 @@ export class Service {
     if (this.providedUserAgent) {
       userAgent = `${this.providedUserAgent} ${userAgent}`;
     }
-    const idempotencyToken = crypto.randomUUID();
+    const headers = reqOpts.headers || {};
+    const userTokenKey = Object.keys(headers).find(
+      key => key.toLowerCase() === 'x-goog-gcs-idempotency-token'
+    );
+    const idempotencyToken = userTokenKey
+      ? (headers[userTokenKey] as string)
+      : crypto.randomUUID();
     reqOpts.headers = {
-      ...reqOpts.headers,
+      ...headers,
       'User-Agent': userAgent,
       'x-goog-api-client': `${getRuntimeTrackingString()} gccl/${
         pkg.version
       }-${getModuleFormat()} gccl-invocation-id/${idempotencyToken}`,
-      'x-goog-gcs-idempotency-token': idempotencyToken,
     };
+    if (!userTokenKey) {
+      reqOpts.headers['x-goog-gcs-idempotency-token'] = idempotencyToken;
+    }
 
     if (reqOpts[GCCL_GCS_CMD_KEY]) {
       reqOpts.headers['x-goog-api-client'] +=

handwritten/storage/src/resumable-upload.ts

@@ -338,7 +338,11 @@ export class Upload extends Writable {
   timeOfFirstRequest: number;
   isPartialUpload: boolean;
 
-  private currentInvocationId = {
+  private currentInvocationId: {
+    checkUploadStatus: string;
+    chunk: string;
+    uri: string;
+  } = {
     checkUploadStatus: crypto.randomUUID(),
     chunk: crypto.randomUUID(),
     uri: crypto.randomUUID(),
@@ -806,6 +810,13 @@ export class Upload extends Writable {
       delete metadata.contentType;
     }
 
+    const userTokenKey = Object.keys(this.customRequestOptions.headers || {}).find(
+      key => key.toLowerCase() === 'x-goog-gcs-idempotency-token'
+    );
+    if (userTokenKey) {
+      this.currentInvocationId.uri = this.customRequestOptions.headers![userTokenKey] as string;
+    }
+
     let googAPIClient = `${getRuntimeTrackingString()} gccl/${
       packageJson.version
     }-${getModuleFormat()} gccl-invocation-id/${this.currentInvocationId.uri}`;
@@ -829,11 +840,14 @@ export class Upload extends Writable {
       headers: {
         'User-Agent': getUserAgentString(),
         'x-goog-api-client': googAPIClient,
-        'x-goog-gcs-idempotency-token': this.currentInvocationId.uri,
         ...headers,
       },
     };
 
+    if (!userTokenKey) {
+      reqOpts.headers!['x-goog-gcs-idempotency-token'] = this.currentInvocationId.uri;
+    }
+
     if (metadata.contentLength) {
       reqOpts.headers!['X-Upload-Content-Length'] =
         metadata.contentLength.toString();
@@ -996,6 +1010,13 @@ export class Upload extends Writable {
       },
     });
 
+    const userTokenKey = Object.keys(this.customRequestOptions.headers || {}).find(
+      key => key.toLowerCase() === 'x-goog-gcs-idempotency-token'
+    );
+    if (userTokenKey) {
+      this.currentInvocationId.chunk = this.customRequestOptions.headers![userTokenKey] as string;
+    }
+
     let googAPIClient = `${getRuntimeTrackingString()} gccl/${
       packageJson.version
     }-${getModuleFormat()} gccl-invocation-id/${
@@ -1009,9 +1030,12 @@ export class Upload extends Writable {
     const headers: GaxiosOptions['headers'] = {
       'User-Agent': getUserAgentString(),
       'x-goog-api-client': googAPIClient,
-      'x-goog-gcs-idempotency-token': this.currentInvocationId.chunk,
     };
 
+    if (!userTokenKey) {
+      headers['x-goog-gcs-idempotency-token'] = this.currentInvocationId.chunk;
+    }
+
     // If using multiple chunk upload, set appropriate header
     if (multiChunkMode) {
       // We need to know how much data is available upstream to set the `Content-Range` header.
@@ -1211,6 +1235,13 @@ export class Upload extends Writable {
   async checkUploadStatus(
     config: CheckUploadStatusConfig = {},
   ): Promise<GaxiosResponse<FileMetadata | void>> {
+    const userTokenKey = Object.keys(this.customRequestOptions.headers || {}).find(
+      key => key.toLowerCase() === 'x-goog-gcs-idempotency-token'
+    );
+    if (userTokenKey) {
+      this.currentInvocationId.checkUploadStatus = this.customRequestOptions.headers![userTokenKey] as string;
+    }
+
     let googAPIClient = `${getRuntimeTrackingString()} gccl/${
       packageJson.version
     }-${getModuleFormat()} gccl-invocation-id/${
@@ -1229,10 +1260,13 @@ export class Upload extends Writable {
         'Content-Range': 'bytes */*',
         'User-Agent': getUserAgentString(),
         'x-goog-api-client': googAPIClient,
-        'x-goog-gcs-idempotency-token': this.currentInvocationId.checkUploadStatus,
       },
     };
 
+    if (!userTokenKey) {
+      opts.headers!['x-goog-gcs-idempotency-token'] = this.currentInvocationId.checkUploadStatus;
+    }
+
     try {
       const resp = await this.makeRequest(opts);
 

handwritten/storage/test/nodejs-common/service.ts

@@ -506,6 +506,36 @@ describe('Service', () => {
       service.request_(reqOpts, assert.ifError);
     });
 
+    it('should respect user-provided x-goog-gcs-idempotency-token case-insensitively and align it with gccl-invocation-id', done => {
+      const customToken = 'my-custom-token-123';
+      const customReqOpts = {
+        ...reqOpts,
+        headers: {
+          'X-Goog-Gcs-Idempotency-Token': customToken,
+        },
+      };
+
+      service.makeAuthenticatedRequest = (reqOpts: DecorateRequestOptions) => {
+        const pkg = service.packageJson;
+        const r = new RegExp(
+          `^gl-node/${process.versions.node} gccl/${
+            pkg.version
+          }-${getModuleFormat()} gccl-invocation-id/(?<gcclInvocationId>[^W]+)$`
+        );
+        const match = r.exec(reqOpts.headers!['x-goog-api-client']);
+        assert.ok(match);
+        const invocationId = match.groups!.gcclInvocationId;
+        assert.strictEqual(invocationId, customToken);
+
+        // Verify there is no duplicate x-goog-gcs-idempotency-token header
+        assert.strictEqual(reqOpts.headers!['x-goog-gcs-idempotency-token'], undefined);
+        assert.strictEqual(reqOpts.headers!['X-Goog-Gcs-Idempotency-Token'], customToken);
+        done();
+      };
+
+      service.request_(customReqOpts, assert.ifError);
+    });
+
     it('should add the `gccl-gcs-cmd` to the api-client header when provided', done => {
       const expected = 'example.expected/value';
       service.makeAuthenticatedRequest = (reqOpts: DecorateRequestOptions) => {

handwritten/storage/test/resumable-upload.ts

@@ -890,6 +890,31 @@ describe('resumable-upload', () => {
       up.createURI();
     });
 
+    it('should respect user-provided x-goog-gcs-idempotency-token case-insensitively and align it with gccl-invocation-id in createURI', async () => {
+      const customToken = 'my-custom-resumable-token';
+      up.customRequestOptions = {
+        headers: {
+          'X-Goog-Gcs-Idempotency-Token': customToken,
+        },
+      };
+
+      up.authClient.request = async (combinedReqOpts: GaxiosOptions) => {
+        assert(combinedReqOpts.headers);
+        const apiClientHeader = combinedReqOpts.headers['x-goog-api-client'];
+        const match = X_GOOG_API_HEADER_REGEX.exec(apiClientHeader as string);
+        assert.ok(match);
+        const invocationId = match.groups!.gcclInvocationId;
+        assert.strictEqual(invocationId, customToken);
+
+        // Verify there is no duplicate x-goog-gcs-idempotency-token header
+        assert.strictEqual(combinedReqOpts.headers['x-goog-gcs-idempotency-token'], undefined);
+        assert.strictEqual(combinedReqOpts.headers['X-Goog-Gcs-Idempotency-Token'], customToken);
+        return {headers: {location: '/foo'}};
+      };
+
+      await up.createURI();
+    });
+
     it('should reuse the same x-goog-gcs-idempotency-token on retry of createURI', async () => {
       let invocationCount = 0;
       let token1 = '';