refactor(storage): Migrate conformance and system tests to StorageTransport with enhanced retry logic

handwritten/storage/conformance-test/conformanceCommon.ts

@@ -15,22 +15,19 @@
  */
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 import * as jsonToNodeApiMapping from './test-data/retryInvocationMap.json';
-import * as libraryMethods from './libraryMethods';
-import {
-  Bucket,
-  File,
-  GaxiosOptions,
-  GaxiosOptionsPrepared,
-  HmacKey,
-  Notification,
-  Storage,
-} from '../src';
+import * as libraryMethods from './libraryMethods.js';
+import {Bucket, File, Gaxios, HmacKey, Notification, Storage} from '../src';
+import * as gaxios from 'gaxios';
 import * as crypto from 'crypto';
 import * as assert from 'assert';
 import {
   StorageRequestOptions,
   StorageTransport,
-} from '../src/storage-transport';
+  StorageTransportCallback,
+} from '../src/storage-transport.js';
+import {getDirName} from '../src/util.js';
+import path from 'path';
+import {GoogleAuth} from 'google-auth-library';
 interface RetryCase {
   instructions: String[];
 }
@@ -60,7 +57,7 @@ interface ConformanceTestResult {
 
 type LibraryMethodsModuleType = typeof import('./libraryMethods');
 const methodMap: Map<String, String[]> = new Map(
-  Object.entries({}), // TODO: replace with Object.entries(jsonToNodeApiMapping)
+  Object.entries(jsonToNodeApiMapping),
 );
 
 const DURATION_SECONDS = 600; // 10 mins.
@@ -70,6 +67,23 @@ const TESTBENCH_HOST =
 const CONF_TEST_PROJECT_ID = 'my-project-id';
 const TIMEOUT_FOR_INDIVIDUAL_TEST = 20000;
 const RETRY_MULTIPLIER_FOR_CONFORMANCE_TESTS = 0.01;
+const SERVICE_ACCOUNT = path.join(
+  getDirName(),
+  '../../../conformance-test/fixtures/signing-service-account.json',
+);
+
+const authClient = new GoogleAuth({
+  keyFilename: SERVICE_ACCOUNT,
+  scopes: ['https://www.googleapis.com/auth/devstorage.full_control'],
+}).fromJSON(require(SERVICE_ACCOUNT));
+
+authClient.getAccessToken = async () => ({token: 'unauthenticated-test-token'});
+authClient.request = async opts => {
+  if (typeof (opts as any).adapter === 'function') {
+    return (opts as any).adapter(opts);
+  }
+  return (gaxios as any).instance.request(opts);
+};
 
 export function executeScenario(testCase: RetryTestCase) {
   for (
@@ -89,16 +103,20 @@ export function executeScenario(testCase: RetryTestCase) {
         let bucket: Bucket;
         let file: File;
         let notification: Notification;
-        let creationResult: {id: string};
+        let creationResult: ConformanceTestCreationResult;
         let storage: Storage;
         let hmacKey: HmacKey;
         let storageTransport: StorageTransport;
 
         describe(`${storageMethodString}`, async () => {
           beforeEach(async () => {
-            storageTransport = new StorageTransport({
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            (gaxios as any)?.instance?.interceptors?.request?.clear();
+
+            const rawTransport = new StorageTransport({
               apiEndpoint: TESTBENCH_HOST,
-              authClient: undefined,
+              authClient: authClient,
+              keyFilename: SERVICE_ACCOUNT,
               baseUrl: TESTBENCH_HOST,
               packageJson: {name: 'test-package', version: '1.0.0'},
               retryOptions: {
@@ -117,173 +135,193 @@ export function executeScenario(testCase: RetryTestCase) {
               timeout: DURATION_SECONDS,
             });
 
+            creationResult = await createTestBenchRetryTest(
+              instructionSet.instructions,
+              jsonMethod?.name.toString(),
+              rawTransport,
+            );
+
             storage = new Storage({
               apiEndpoint: TESTBENCH_HOST,
               projectId: CONF_TEST_PROJECT_ID,
+              keyFilename: SERVICE_ACCOUNT,
+              authClient: authClient,
               retryOptions: {
                 retryDelayMultiplier: RETRY_MULTIPLIER_FOR_CONFORMANCE_TESTS,
               },
             });
 
-            creationResult = await createTestBenchRetryTest(
-              instructionSet.instructions,
-              jsonMethod?.name.toString(),
-              storageTransport,
+            bucket = await createBucketForTest(
+              storage,
+              testCase.preconditionProvided &&
+                !storageMethodString.includes('combine'),
+              storageMethodString,
             );
-            if (storageMethodString.includes('InstancePrecondition')) {
-              bucket = await createBucketForTest(
-                storage,
-                testCase.preconditionProvided,
-                storageMethodString,
-              );
-              file = await createFileForTest(
-                testCase.preconditionProvided,
-                storageMethodString,
-                bucket,
-              );
-            } else {
-              bucket = await createBucketForTest(
-                storage,
-                false,
-                storageMethodString,
-              );
-              file = await createFileForTest(
-                false,
-                storageMethodString,
-                bucket,
-              );
+            file = await createFileForTest(
+              testCase.preconditionProvided,
+              storageMethodString,
+              bucket,
+            );
+            if (
+              storageMethodString !== 'createNotification' &&
+              storageMethodString !== 'notificationCreate'
+            ) {
+              notification = bucket.notification(TESTS_PREFIX);
+              await notification.create();
             }
-            notification = bucket.notification(TESTS_PREFIX);
-            await notification.create();
 
-            [hmacKey] = await storage.createHmacKey(
-              `${TESTS_PREFIX}@email.com`,
-            );
+            if (
+              storageMethodString === 'deleteHMAC' ||
+              storageMethodString === 'getHMAC' ||
+              storageMethodString === 'getMetadataHMAC' ||
+              storageMethodString === 'setMetadataHMAC'
+            ) {
+              [hmacKey] = await storage.createHmacKey(
+                `${TESTS_PREFIX}@email.com`,
+              );
+            }
 
-            storage.interceptors.push({
-              resolved: (
-                requestConfig: GaxiosOptionsPrepared,
-              ): Promise<GaxiosOptionsPrepared> => {
-                const config = requestConfig as GaxiosOptions;
-                config.headers = config.headers || {};
-                Object.assign(config.headers, {
-                  'x-retry-test-id': creationResult.id,
-                });
-                return Promise.resolve(config as GaxiosOptionsPrepared);
-              },
-              rejected: error => {
-                return Promise.reject(error);
-              },
-            });
+            storageTransport = storage.storageTransport;
           });
 
           it(`${instructionNumber}`, async () => {
             const methodParameters: libraryMethods.ConformanceTestOptions = {
-              storage: storage,
-              bucket: bucket,
-              file: file,
-              storageTransport: storageTransport,
-              notification: notification,
-              hmacKey: hmacKey,
+              storage,
+              bucket,
+              file,
+              storageTransport,
+              notification,
+              hmacKey,
+              projectId: CONF_TEST_PROJECT_ID,
+              preconditionRequired: testCase.preconditionProvided,
+            };
+
+            const injectHeader = (reqOpts: any) => {
+              const url = reqOpts.url?.toString() || '';
+              if (url.includes('retry_test') || !creationResult?.id) {
+                return reqOpts;
+              }
+              reqOpts.headers = reqOpts.headers || {};
+              if (typeof reqOpts.headers.set === 'function') {
+                reqOpts.headers.set('x-retry-test-id', creationResult.id);
+              }
+              try {
+                reqOpts.headers['x-retry-test-id'] = creationResult.id;
+              } catch (e) {}
+              return reqOpts;
             };
-            if (testCase.preconditionProvided) {
-              methodParameters.preconditionRequired = true;
-            }
 
-            if (testCase.expectSuccess) {
-              assert.ifError(await storageMethodObject(methodParameters));
-            } else {
-              await assert.rejects(async () => {
+            const interceptor = {
+              resolved: injectHeader,
+              request: injectHeader,
+            };
+
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            (storage.storageTransport as any)?.gaxiosInstance?.interceptors?.request?.clear();
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            (gaxios as any)?.instance?.interceptors?.request?.clear();
+
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            (storage.storageTransport as any).gaxiosInstance.interceptors.request.add(interceptor);
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            (gaxios as any).instance.interceptors.request.add(interceptor);
+
+            try {
+              if (testCase.expectSuccess) {
                 await storageMethodObject(methodParameters);
-              }, undefined);
+                const testBenchResult = await getTestBenchRetryTest(
+                  creationResult.id,
+                  storageTransport,
+                );
+                assert.strictEqual(testBenchResult.completed, true);
+              } else {
+                await assert.rejects(async () => {
+                  await storageMethodObject(methodParameters);
+                }, undefined);
+              }
+            } finally {
+              // eslint-disable-next-line @typescript-eslint/no-explicit-any
+              (storage.storageTransport as any)?.gaxiosInstance?.interceptors?.request?.clear();
+              // eslint-disable-next-line @typescript-eslint/no-explicit-any
+              (gaxios as any)?.instance?.interceptors?.request?.clear();
             }
-
-            const testBenchResult = await getTestBenchRetryTest(
-              creationResult.id,
-              storageTransport,
-            );
-            assert.strictEqual(testBenchResult.completed, true);
           }).timeout(TIMEOUT_FOR_INDIVIDUAL_TEST);
         });
       });
     });
   }
 }
 
+
+
 async function createBucketForTest(
   storage: Storage,
-  preconditionShouldBeOnInstance: boolean,
-  storageMethodString: String,
+  withPrecondition: boolean,
+  method: String,
 ) {
-  const name = generateName(storageMethodString, 'bucket');
-  const bucket = storage.bucket(name);
-  await bucket.create();
+  const bucket = storage.bucket(generateName(method, 'bucket'));
+  const [metadata] = await bucket.create();
   await bucket.setRetentionPeriod(DURATION_SECONDS);
-
-  if (preconditionShouldBeOnInstance) {
+  if (withPrecondition) {
     return new Bucket(storage, bucket.name, {
       preconditionOpts: {
-        ifMetagenerationMatch: 2,
+        ifMetagenerationMatch: metadata.metageneration,
       },
     });
   }
   return bucket;
 }
 
 async function createFileForTest(
-  preconditionShouldBeOnInstance: boolean,
-  storageMethodString: String,
+  withPrecondition: boolean,
+  method: String,
   bucket: Bucket,
 ) {
-  const name = generateName(storageMethodString, 'file');
-  const file = bucket.file(name);
-  await file.save(name);
-  if (preconditionShouldBeOnInstance) {
+  const file = bucket.file(generateName(method, 'file'));
+  if (method === 'deleteBucket') {
+    return file;
+  }
+  await file.save('test-content');
+  if (withPrecondition) {
+    const [metadata] = await file.getMetadata();
     return new File(bucket, file.name, {
       preconditionOpts: {
-        ifMetagenerationMatch: file.metadata.metageneration,
-        ifGenerationMatch: file.metadata.generation,
+        ifMetagenerationMatch: metadata.metageneration,
+        ifGenerationMatch: metadata.generation,
       },
     });
   }
   return file;
 }
 
-function generateName(storageMethodString: String, bucketOrFile: string) {
-  return `${TESTS_PREFIX}${storageMethodString.toLowerCase()}${bucketOrFile}.${shortUUID()}`;
-}
-
 async function createTestBenchRetryTest(
   instructions: String[],
   methodName: string,
-  storageTransport: StorageTransport,
+  transport: StorageTransport,
 ): Promise<ConformanceTestCreationResult> {
-  const requestBody = {instructions: {[methodName]: instructions}};
-
-  const requestOptions: StorageRequestOptions = {
+  const response = await transport.makeRequest({
     method: 'POST',
     url: 'retry_test',
-    body: JSON.stringify(requestBody),
+    body: JSON.stringify({instructions: {[methodName]: instructions}}),
     headers: {'Content-Type': 'application/json'},
-  };
-
-  const response = await storageTransport.makeRequest(requestOptions);
-  return response as unknown as ConformanceTestCreationResult;
+  });
+  return response.data as ConformanceTestCreationResult;
 }
 
 async function getTestBenchRetryTest(
   testId: string,
-  storageTransport: StorageTransport,
+  transport: StorageTransport,
 ): Promise<ConformanceTestResult> {
-  const response = await storageTransport.makeRequest({
+  const response = await transport.makeRequest({
     url: `retry_test/${testId}`,
     method: 'GET',
-    retry: true,
-    headers: {
-      'x-retry-test-id': testId,
-    },
+    headers: {'x-retry-test-id': testId},
   });
-  return response as unknown as ConformanceTestResult;
+  return response.data as ConformanceTestResult;
+}
+
+function generateName(method: String, type: string) {
+  return `${TESTS_PREFIX}${method.toLowerCase()}${type}.${shortUUID()}`;
 }
 
 function shortUUID() {