fix(dataproc): add error handling for quota exhaustion (#4351)

* fix(dataproc): add robust error handling for quota exhaustion

* Added error handling to test cleanup.

* Added error handling to quickstart.test.js

Author: angelcaamal

dataproc/system-test/createCluster.test.js

@@ -35,18 +35,37 @@ const execSync = cmd =>
   });
 
 describe('create a dataproc cluster', () => {
-  it('should create a dataproc cluster', async () => {
-    const stdout = execSync(
-      `node createCluster.js "${projectId}" "${region}" "${clusterName}"`
-    );
-    assert.match(stdout, new RegExp(`${clusterName}`));
+  it('should create a dataproc cluster', async function () {
+    try {
+      const stdout = execSync(
+        `node createCluster.js "${projectId}" "${region}" "${clusterName}"`
+      );
+      assert.match(stdout, new RegExp(`${clusterName}`));
+    } catch (err) {
+      if (
+        err?.message?.includes('QUOTA') ||
+        err?.message?.includes('RESOURCE_EXHAUSTED') ||
+        err?.message?.includes('DISKS_TOTAL_GB')
+      ) {
+        console.warn(
+          `Quota limit reached in project ${projectId}. Skipping test.`
+        );
+        this.skip();
+      } else {
+        throw err;
+      }
+    }
   });
 
   after(async () => {
-    await clusterClient.deleteCluster({
-      projectId: projectId,
-      region: region,
-      clusterName: clusterName,
-    });
+    try {
+      await clusterClient.deleteCluster({
+        projectId: projectId,
+        region: region,
+        clusterName: clusterName,
+      });
+    } catch (err) {
+      // Ignore errors during cleanup
+    }
   });
 });

dataproc/system-test/instantiateInlineWorkflowTemplate.test.js

@@ -30,11 +30,26 @@ const {delay} = require('./util');
 
 describe('instantiate an inline workflow template', () => {
   it('should instantiate an inline workflow template', async function () {
-    this.retries(4);
-    await delay(this.test);
-    const stdout = execSync(
-      `node instantiateInlineWorkflowTemplate.js "${projectId}" "${region}"`
-    );
-    assert.match(stdout, /successfully/);
+    try {
+      this.retries(4);
+      await delay(this.test);
+      const stdout = execSync(
+        `node instantiateInlineWorkflowTemplate.js "${projectId}" "${region}"`
+      );
+      assert.match(stdout, /successfully/);
+    } catch (err) {
+      if (
+        err?.message?.includes('QUOTA') ||
+        err?.message?.includes('RESOURCE_EXHAUSTED') ||
+        err?.message?.includes('DISKS_TOTAL_GB')
+      ) {
+        console.warn(
+          `Quota limit reached in project ${projectId}. Skipping test.`
+        );
+        this.skip();
+      } else {
+        throw err;
+      }
+    }
   });
 });

dataproc/system-test/quickstart.test.js

@@ -55,14 +55,29 @@ describe('execute the quickstart', () => {
   });
 
   it('should execute the quickstart', async function () {
-    this.retries(4);
-    await delay(this.test);
-    const stdout = execSync(
-      `node quickstart.js "${projectId}" "${region}" "${clusterName}" "${jobFilePath}"`
-    );
-    assert.match(stdout, /Cluster created successfully/);
-    assert.match(stdout, /Job finished successfully/);
-    assert.match(stdout, /successfully deleted/);
+    try {
+      this.retries(4);
+      await delay(this.test);
+      const stdout = execSync(
+        `node quickstart.js "${projectId}" "${region}" "${clusterName}" "${jobFilePath}"`
+      );
+      assert.match(stdout, /Cluster created successfully/);
+      assert.match(stdout, /Job finished successfully/);
+      assert.match(stdout, /successfully deleted/);
+    } catch (err) {
+      if (
+        err?.message?.includes('QUOTA') ||
+        err?.message?.includes('RESOURCE_EXHAUSTED') ||
+        err?.message?.includes('DISKS_TOTAL_GB')
+      ) {
+        console.warn(
+          `Quota limit reached in project ${projectId}. Skipping test.`
+        );
+        this.skip();
+      } else {
+        throw err;
+      }
+    }
   });
 
   afterEach(async () => {

dataproc/system-test/submitJob.test.js

@@ -51,9 +51,24 @@ const execSync = cmd =>
   });
 
 describe('submit a Spark job to a Dataproc cluster', () => {
-  before(async () => {
-    const [operation] = await clusterClient.createCluster(cluster);
-    await operation.promise();
+  before(async function () {
+    try {
+      const [operation] = await clusterClient.createCluster(cluster);
+      await operation.promise();
+    } catch (err) {
+      if (
+        err?.message?.includes('QUOTA') ||
+        err?.message?.includes('RESOURCE_EXHAUSTED') ||
+        err?.message?.includes('DISKS_TOTAL_GB')
+      ) {
+        console.warn(
+          `Quota limit reached in project ${projectId}. Skipping test.`
+        );
+        this.skip();
+      } else {
+        throw err;
+      }
+    }
   });
 
   it('should submit a job to a dataproc cluster', async () => {
@@ -64,10 +79,14 @@ describe('submit a Spark job to a Dataproc cluster', () => {
   });
 
   after(async () => {
-    await clusterClient.deleteCluster({
-      projectId: projectId,
-      region: region,
-      clusterName: clusterName,
-    });
+    try {
+      await clusterClient.deleteCluster({
+        projectId: projectId,
+        region: region,
+        clusterName: clusterName,
+      });
+    } catch (err) {
+      // Ignore errors during cleanup
+    }
   });
 });