fix(e2e): fix flaky CodeBuild e2e tests - env var type, retries, timeouts

.github/actions/run-smoke-tests/action.yml

@@ -67,17 +67,7 @@ runs:
 
     - name: Run Smoke Tests
       shell: bash
-      run: |
-        set +e
-        yarn smoketest -- --forceExit --no-cache --maxWorkers=2
-        EXIT_CODE=$?
-        if [ $EXIT_CODE -ne 0 ]; then
-          echo "::warning::Smoke tests failed on first attempt (exit code $EXIT_CODE). Retrying in 30s..."
-          sleep 30
-          yarn smoketest -- --forceExit --no-cache --maxWorkers=2
-          EXIT_CODE=$?
-        fi
-        exit $EXIT_CODE
+      run: yarn smoketest -- --forceExit --no-cache --maxWorkers=2
       env:
         CLI_REGION: ${{ inputs.region }}
         CI: true

packages/amplify-e2e-core/src/categories/api.ts

@@ -477,6 +477,11 @@ export function addRestApi(cwd: string, settings: RestAPISettings) {
         .sendCarriageReturn() // Add another path
         .wait('Provide a path')
         .sendLine(settings.path)
+        .wait(/overlaps with.*Are you sure you want to continue|Choose a lambda source/, (data: string) => {
+          if (/overlaps with/.test(data)) {
+            chain.getProcess()?.write('y');
+          }
+        })
         .wait('Choose a lambda source');
 
       if (settings.existingLambda) {

packages/amplify-e2e-core/src/categories/custom.ts

@@ -2,6 +2,8 @@ import { nspawn as spawn, KEY_DOWN_ARROW, getCLIPath } from '..';
 import path from 'path';
 import { JSONUtilities } from '@aws-amplify/amplify-cli-core';
 
+const customResourceTimeoutMS = 1000 * 60 * 10; // 10 minutes
+
 export const addCDKCustomResource = async (cwd: string, settings: any): Promise<void> => {
   await spawn(getCLIPath(), ['add', 'custom'], { cwd, stripColors: true })
     .wait('How do you want to define this custom resource?')
@@ -36,7 +38,7 @@ export function buildCustomResources(cwd: string, usingLatestCodebase = false) {
   return new Promise((resolve, reject) => {
     const args = ['custom', 'build'];
 
-    spawn(getCLIPath(usingLatestCodebase), args, { cwd, stripColors: true })
+    spawn(getCLIPath(usingLatestCodebase), args, { cwd, stripColors: true, noOutputTimeout: customResourceTimeoutMS })
       .sendEof()
       .run((err: Error) => {
         if (!err) {

packages/amplify-e2e-core/src/categories/notifications.ts

@@ -1,5 +1,7 @@
 import { nspawn as spawn, getCLIPath } from '..';
 
+const notificationTimeoutMS = 1000 * 60 * 10; // 10 minutes
+
 /**
  * notifications settings
  */
@@ -11,7 +13,7 @@ type NotificationSettings = {
  * removes all the notification channel
  */
 export const removeAllNotificationChannel = async (cwd: string): Promise<void> =>
-  spawn(getCLIPath(), ['remove', 'notifications'], { cwd, stripColors: true })
+  spawn(getCLIPath(), ['remove', 'notifications'], { cwd, stripColors: true, noOutputTimeout: notificationTimeoutMS })
     .wait('Choose the notification channel to remove')
     .sendLine('All channels on Pinpoint resource')
     .wait(`All notifications have been disabled`)
@@ -22,7 +24,7 @@ export const removeAllNotificationChannel = async (cwd: string): Promise<void> =
  * removes the notification channel
  */
 export const removeNotificationChannel = async (cwd: string, channel: string): Promise<void> =>
-  spawn(getCLIPath(), ['remove', 'notifications'], { cwd, stripColors: true })
+  spawn(getCLIPath(), ['remove', 'notifications'], { cwd, stripColors: true, noOutputTimeout: notificationTimeoutMS })
     .wait('Choose the notification channel to remove')
     .sendLine(channel)
     .wait(`The channel has been successfully disabled`)
@@ -45,7 +47,11 @@ export const addNotificationChannel = async (
   hasAuth = false,
   testingWithLatestCodebase = false,
 ): Promise<void> => {
-  const chain = spawn(getCLIPath(testingWithLatestCodebase), ['add', 'notification'], { cwd, stripColors: true });
+  const chain = spawn(getCLIPath(testingWithLatestCodebase), ['add', 'notification'], {
+    cwd,
+    stripColors: true,
+    noOutputTimeout: notificationTimeoutMS,
+  });
 
   chain.wait('Choose the notification channel to enable').sendLine(channel);
 
@@ -92,7 +98,11 @@ export const updateNotificationChannel = async (
   enable = true,
   testingWithLatestCodebase = false,
 ): Promise<void> => {
-  const chain = spawn(getCLIPath(testingWithLatestCodebase), ['update', 'notification'], { cwd, stripColors: true });
+  const chain = spawn(getCLIPath(testingWithLatestCodebase), ['update', 'notification'], {
+    cwd,
+    stripColors: true,
+    noOutputTimeout: notificationTimeoutMS,
+  });
   chain.wait('Choose the notification channel to configure').sendLine(channel);
   chain.wait(`Do you want to ${enable ? 'enable' : 'disable'} the ${channel} channel`).sendYes();
 

packages/amplify-e2e-core/src/init/amplifyPush.ts

@@ -356,7 +356,7 @@ export const amplifyPushLayer = (cwd: string, settings: LayerPushSettings, testi
  * Function to test amplify push with iterativeRollback flag option
  */
 export const amplifyPushIterativeRollback = (cwd: string, testingWithLatestCodebase = false) =>
-  spawn(getCLIPath(testingWithLatestCodebase), ['push', '--iterative-rollback'], { cwd, stripColors: true })
+  spawn(getCLIPath(testingWithLatestCodebase), ['push', '--iterative-rollback'], { cwd, stripColors: true, noOutputTimeout: pushTimeoutMS })
     .wait('Are you sure you want to continue?')
     .sendYes()
     .runAsync();
@@ -365,7 +365,7 @@ export const amplifyPushIterativeRollback = (cwd: string, testingWithLatestCodeb
  * Function to test amplify push with missing environment variable
  */
 export const amplifyPushMissingEnvVar = (cwd: string, newEnvVarValue: string) =>
-  spawn(getCLIPath(), ['push'], { cwd, stripColors: true })
+  spawn(getCLIPath(), ['push'], { cwd, stripColors: true, noOutputTimeout: pushTimeoutMS })
     .wait('Enter a value for')
     .sendLine(newEnvVarValue)
     .wait('Are you sure you want to continue?')
@@ -376,7 +376,7 @@ export const amplifyPushMissingEnvVar = (cwd: string, newEnvVarValue: string) =>
  * Function to test amplify push with missing function secrets
  */
 export const amplifyPushMissingFuncSecret = (cwd: string, newSecretValue: string) =>
-  spawn(getCLIPath(), ['push'], { cwd, stripColors: true })
+  spawn(getCLIPath(), ['push'], { cwd, stripColors: true, noOutputTimeout: pushTimeoutMS })
     .wait('does not have a value in this environment. Specify one now:')
     .sendLine(newSecretValue)
     .wait('Are you sure you want to continue?')

packages/amplify-e2e-core/src/utils/nexpect.ts

@@ -98,6 +98,7 @@ export type ExecutionContext = {
    */
   run: (cb: (err: any, signal?: any) => void) => ExecutionContext;
   runAsync: (expectedErrorPredicate?: (err: Error) => boolean) => Promise<void>;
+  getProcess: () => Recorder | undefined;
 };
 
 /**
@@ -388,6 +389,9 @@ function chain(context: Context): ExecutionContext {
       context.queue.push(_delay);
       return chain(context);
     },
+    getProcess(): Recorder | undefined {
+      return context.process;
+    },
   };
   const run = (callback: (err: any, code?: number, signal?: string | number) => void): ExecutionContext => {
     let errState: any = null;
@@ -773,7 +777,7 @@ export function nspawn(command: string | string[], params: string[] = [], option
     // Undo ci-info detection, required for some tests
     // see https://github.com/watson/ci-info/blob/master/index.js#L57
     if (options.disableCIDetection === true) {
-      childEnv.CI = false;
+      childEnv.CI = 'false';
     }
   }
 

packages/amplify-e2e-tests/src/__tests__/smoke-tests/smoketest-amplify-app.test.ts

@@ -12,8 +12,6 @@ import {
 } from '../../amplify-app-helpers/amplify-app-validation';
 import execa from 'execa';
 
-jest.retryTimes(1);
-
 describe('amplify-app platform tests', () => {
   let projRoot: string;
 

packages/amplify-e2e-tests/src/__tests__/smoke-tests/smoketest-ios.test.ts

@@ -10,8 +10,6 @@ import {
 import * as fs from 'fs/promises';
 import * as path from 'path';
 
-jest.retryTimes(1);
-
 describe('Smoke Test - iOS', () => {
   if (process.platform == 'darwin') {
     let projRoot: string;

packages/amplify-e2e-tests/src/__tests__/smoke-tests/smoketest.test.ts

@@ -3,7 +3,7 @@ import * as path from 'path';
 import * as fs from 'fs-extra';
 import * as os from 'os';
 import { getCLIPath, initJSProjectWithProfile, nspawn as spawn } from '@aws-amplify/amplify-e2e-core';
-jest.retryTimes(1);
+jest.retryTimes(0);
 
 export type SmoketestArgs = {
   projectDirectory: string;

packages/amplify-e2e-tests/src/setup-tests.ts

@@ -16,7 +16,7 @@ removeYarnPaths();
 
 const JEST_TIMEOUT = 1000 * 60 * 60; // 1 hour
 jest.setTimeout(JEST_TIMEOUT);
-if (process.env.CIRCLECI) {
+if (process.env.CIRCLECI || process.env.CODEBUILD_BUILD_ID) {
   jest.retryTimes(1);
 }