Merge pull request #14189 from aws-amplify/gen2-migrations-execute

Author: abhi7cr

packages/amplify-migration-e2e/src/__tests__/migration_codegen_e2e.test.ts

@@ -1,9 +1,8 @@
 import path from 'node:path';
 import assert from 'node:assert';
-import { createNewProjectDir, generateRandomShortId, npmInstall } from '@aws-amplify/amplify-e2e-core';
+import { createNewProjectDir, generateRandomShortId, getSocialProviders, npmInstall } from '@aws-amplify/amplify-e2e-core';
 import { createGen2Renderer } from '@aws-amplify/amplify-gen2-codegen';
 import { copyFunctionFile, removeErrorThrowsFromFunctionFile } from '../function_utils';
-import { copyGen1Schema } from '../api_utils';
 import {
   cleanupProjects,
   setupAndPushDefaultGen1Project,
@@ -54,25 +53,40 @@ void describe('Gen 2 Codegen E2E tests', () => {
       await cleanupProjects(projRoot, projName);
     });
 
-    void it.only('should init a project & add auth, function, storage, api with defaults & perform full migration codegen flow', async () => {
+    void it('should init a project & add auth, function, storage, api with defaults & perform full migration codegen flow', async () => {
+      // Arrange
       await setupAndPushDefaultGen1Project(projRoot, projName);
+
+      // Act
       const { gen1UserPoolId, gen1ClientIds, gen1IdentityPoolId, gen1FunctionName, gen1BucketName, gen1GraphqlApiId, gen1Region, envName } =
         await assertDefaultGen1Setup(projRoot);
       runCodegenCommand(projRoot);
       copyFunctionFile(projRoot, 'function', gen1FunctionName);
-      copyGen1Schema(projRoot, projName);
       removeErrorThrowsFromFunctionFile(projRoot, 'function', extractFunctionResourceName(gen1FunctionName, envName));
       updateAmplifyBackendPackagesVersion(projRoot);
       npmInstall(projRoot);
       const gen2StackName = await runGen2SandboxCommand(projRoot, projName);
+
+      // Assert
       await assertAuthResource(projRoot, gen1UserPoolId, gen1ClientIds, gen1IdentityPoolId, gen1Region);
       await assertStorageResource(projRoot, gen1BucketName, gen1Region);
       await assertFunctionResource(projRoot, gen2StackName, gen1FunctionName, gen1Region);
       await assertDataResource(projRoot, gen2StackName, gen1GraphqlApiId, gen1Region);
     });
 
     void it('should init a project where all possible auth options are selected and perform full migration codegen flow ', async () => {
+      // Arrange
+      const socialProviders = getSocialProviders();
+      Object.entries(socialProviders).forEach(([socialProvider, value]) => {
+        // we expect APPLE_PRIVATE_KEY_2 in process.env but getSocialProviders returns as APPLE_PRIVATE_KEY
+        if (socialProvider === 'APPLE_PRIVATE_KEY') {
+          socialProvider = 'APPLE_PRIVATE_KEY_2';
+        }
+        process.env[socialProvider] = process.env[socialProvider] ?? value;
+      });
       await setupAndPushAuthWithMaxOptionsGen1Project(projRoot, projName);
+
+      // Act
       const { gen1UserPoolId, gen1ClientIds, gen1IdentityPoolId, gen1FunctionName, gen1Region } = await assertAuthWithMaxOptionsGen1Setup(
         projRoot,
       );
@@ -84,19 +98,26 @@ void describe('Gen 2 Codegen E2E tests', () => {
       await toggleSandboxSecrets(projRoot, projName, 'set');
       const gen2StackName = await runGen2SandboxCommand(projRoot, projName);
       await toggleSandboxSecrets(projRoot, projName, 'remove');
+
+      // Assert
       await assertAuthResource(projRoot, gen1UserPoolId, gen1ClientIds, gen1IdentityPoolId, gen1Region);
       await assertFunctionResource(projRoot, gen2StackName, gen1FunctionName, gen1Region);
     });
 
     void it('should init a project where default auth, all possible s3 bucket resource options are selected and perform full migration codegen flow ', async () => {
+      // Arrange
       await setupAndPushStorageWithMaxOptionsGen1Project(projRoot, projName);
-      const { gen1UserPoolId, gen1ClientIds, gen1BucketName, gen1IdentityPoolId, gen1Region, gen1FunctionName } =
+
+      // Act
+      const { gen1UserPoolId, gen1ClientIds, gen1BucketName, gen1IdentityPoolId, gen1Region, gen1FunctionName, envName } =
         await assertStorageWithMaxOptionsGen1Setup(projRoot);
       runCodegenCommand(projRoot);
       updateAmplifyBackendPackagesVersion(projRoot);
       npmInstall(projRoot);
       removeErrorThrowsFromFunctionFile(projRoot, 'storage', extractFunctionResourceName(gen1FunctionName, envName));
       await runGen2SandboxCommand(projRoot, projName);
+
+      // Assert
       await assertAuthResource(projRoot, gen1UserPoolId, gen1ClientIds, gen1IdentityPoolId, gen1Region);
       await assertStorageResource(projRoot, gen1BucketName, gen1Region);
     });

packages/amplify-migration-e2e/src/__tests__/migration_templategen_e2e.test.ts

@@ -1,56 +1,59 @@
 import path from 'node:path';
 import assert from 'node:assert';
-import { createNewProjectDir, npmInstall, deleteS3Bucket, generateRandomShortId } from '@aws-amplify/amplify-e2e-core';
+import { createNewProjectDir, npmInstall, generateRandomShortId } from '@aws-amplify/amplify-e2e-core';
 import { assertDefaultGen1Setup } from '../assertions';
-import { setupAndPushDefaultGen1Project, runCodegenCommand, runGen2SandboxCommand, cleanupProjects } from '..';
-import { copyFunctionFile } from '../function_utils';
-import { copyGen1Schema } from '../api_utils';
-import { createS3Bucket } from '../sdk_calls';
-import { runExecuteCommand, runRevertCommand } from '../templategen';
+import {
+  setupAndPushDefaultGen1Project,
+  runCodegenCommand,
+  runGen2SandboxCommand,
+  cleanupProjects,
+  extractFunctionResourceName,
+  updateAmplifyBackendPackagesVersion,
+} from '..';
+import { copyFunctionFile, removeErrorThrowsFromFunctionFile } from '../function_utils';
+import { assertExecuteCommand, RefactorCategory, runExecuteCommand, runGen2DeployPostExecute, runRevertCommand } from '../templategen';
+
+const CATEGORIES_TO_MOVE: RefactorCategory[] = ['auth', 'storage'];
 
 void describe('Templategen E2E tests', () => {
   void describe('Full Migration Templategen Flow', () => {
     let projRoot: string;
     let projName: string;
-    let bucketName: string;
 
     beforeEach(async () => {
       const baseDir = process.env.INIT_CWD ?? process.cwd();
       projRoot = await createNewProjectDir('templategen_e2e_flow_test', path.join(baseDir, '..', '..'));
       projName = `test${generateRandomShortId()}`;
-      bucketName = `testbucket${generateRandomShortId()}`;
     });
 
     afterEach(async () => {
       await cleanupProjects(projRoot, projName);
-      await deleteS3Bucket(bucketName);
     });
 
     void it('should init a project & add auth, function, storage, api with defaults & perform refactor', async () => {
+      // Arrange
       await setupAndPushDefaultGen1Project(projRoot, projName);
-      const { gen1StackName, gen1FunctionName } = await assertDefaultGen1Setup(projRoot);
-      assert(gen1StackName);
-      runCodegenCommand(projRoot);
-      copyFunctionFile(projRoot, 'function', gen1FunctionName);
-      copyGen1Schema(projRoot, projName);
-      npmInstall(projRoot);
-      const gen2StackName = await runGen2SandboxCommand(projRoot, projName);
-      assert(gen2StackName);
-      runExecuteCommand(projRoot, gen1StackName, gen2StackName);
-    });
 
-    void it('should init a project & add auth, function, storage, api with defaults, perform refactor and revert to the original state', async () => {
-      await setupAndPushDefaultGen1Project(projRoot, projName);
-      const { gen1StackName, gen1FunctionName, gen1Region } = await assertDefaultGen1Setup(projRoot);
-      await createS3Bucket(bucketName, gen1Region);
+      // Act
+      const { gen1StackName, gen1FunctionName, envName } = await assertDefaultGen1Setup(projRoot);
       assert(gen1StackName);
       runCodegenCommand(projRoot);
       copyFunctionFile(projRoot, 'function', gen1FunctionName);
-      copyGen1Schema(projRoot, projName);
+      removeErrorThrowsFromFunctionFile(projRoot, 'function', extractFunctionResourceName(gen1FunctionName, envName));
+      updateAmplifyBackendPackagesVersion(projRoot);
       npmInstall(projRoot);
+      // Below env is only needed for CI/CD deployments and is expected to be set by customers for their app
+      // To emulate the migration in sandbox, we set it explicitly.
+      process.env.AMPLIFY_GEN_1_ENV_NAME = envName;
       const gen2StackName = await runGen2SandboxCommand(projRoot, projName);
       assert(gen2StackName);
+
       runExecuteCommand(projRoot, gen1StackName, gen2StackName);
+      await runGen2DeployPostExecute(projRoot, projName, envName, CATEGORIES_TO_MOVE);
+
+      // Assert
+      await assertExecuteCommand(projRoot, CATEGORIES_TO_MOVE);
+
       runRevertCommand(projRoot, gen1StackName, gen2StackName);
     });
   });

packages/amplify-migration-e2e/src/assertions.ts

@@ -13,6 +13,8 @@ import { removeProperties } from '.';
 import { $TSAny } from '@aws-amplify/amplify-cli-core';
 import assert from 'node:assert';
 
+const DATA_SOURCE_PROPS_TO_REMOVE = ['dataSourceArn', 'serviceRoleArn', 'dynamodbConfig'];
+
 export async function assertUserPool(gen1Meta: $TSAny, gen1Region: string) {
   const { UserPoolId: gen1UserPoolId } = Object.keys(gen1Meta.auth).map((key) => gen1Meta.auth[key])[0].output;
   const cloudUserPool = await getUserPool(gen1UserPoolId, gen1Region);
@@ -129,17 +131,24 @@ export async function assertAuthWithMaxOptionsGen1Setup(projRoot: string) {
 
 export async function assertStorageWithMaxOptionsGen1Setup(projRoot: string) {
   const gen1Meta = getProjectMeta(projRoot);
+  const gen1StackName = gen1Meta.providers.awscloudformation.StackName;
   const gen1Region = gen1Meta.providers.awscloudformation.Region;
   const { gen1BucketName } = await assertStorage(gen1Meta, gen1Region);
   const { gen1UserPoolId } = await assertUserPool(gen1Meta, gen1Region);
   const { gen1FunctionName } = await assertFunction(gen1Meta, gen1Region);
   assert.match(gen1FunctionName, /S3Trigger/);
   const { gen1ClientIds } = await assertUserPoolClients(gen1Meta, gen1Region);
   const { gen1IdentityPoolId } = await assertIdentityPool(gen1Meta, gen1Region);
+  const envName = gen1StackName.split('-')[2];
 
-  return { gen1UserPoolId, gen1ClientIds, gen1BucketName, gen1IdentityPoolId, gen1Region, gen1FunctionName };
+  return { gen1UserPoolId, gen1ClientIds, gen1BucketName, gen1IdentityPoolId, gen1Region, gen1FunctionName, envName };
 }
 
+const extractUserPoolNamePrefix = (userPoolName: string) => {
+  const [userPoolNamePrefix] = userPoolName.split('-');
+  return userPoolNamePrefix;
+};
+
 async function assertUserPoolResource(projRoot: string, gen1UserPoolId: string, gen1Region: string) {
   const gen1Resource = await getResourceDetails('AWS::Cognito::UserPool', gen1UserPoolId, gen1Region);
   removeProperties(gen1Resource, ['ProviderURL', 'ProviderName', 'UserPoolId', 'Arn', 'LambdaConfig.PostConfirmation']);
@@ -156,6 +165,8 @@ async function assertUserPoolResource(projRoot: string, gen1UserPoolId: string,
   const gen2UserPoolId = gen2Meta.auth.user_pool_id;
   const gen2Region = gen2Meta.auth.aws_region;
   const gen2Resource = await getResourceDetails('AWS::Cognito::UserPool', gen2UserPoolId, gen2Region);
+  gen1Resource.UserPoolName = extractUserPoolNamePrefix(gen1Resource.UserPoolName);
+  gen2Resource.UserPoolName = extractUserPoolNamePrefix(gen2Resource.UserPoolName);
   if (gen1Resource.LambdaConfig.PostConfirmation) assert(gen2Resource.LambdaConfig.PostConfirmation);
   removeProperties(gen2Resource, ['ProviderURL', 'ProviderName', 'UserPoolId', 'Arn', 'LambdaConfig.PostConfirmation']);
   // TODO: remove below line after EmailMessage, EmailSubject, SmsMessage, SmsVerificationMessage, EmailVerificationMessage, EmailVerificationSubject, AccountRecoverySetting inconsistency is fixed
@@ -275,7 +286,7 @@ export async function assertFunctionResource(projRoot: string, gen2StackName: st
   const gen1Resource = await getResourceDetails('AWS::Lambda::Function', gen1FunctionName, gen1Region);
   removeProperties(gen1Resource, ['Arn', 'FunctionName', 'LoggingConfig.LogGroup', 'Role']);
   // TODO: remove below line after Tags inconsistency is fixed
-  removeProperties(gen1Resource, ['Tags']);
+  removeProperties(gen1Resource, ['Tags', 'Environment']);
 
   const gen2Meta = getProjectOutputs(projRoot);
   const gen2Region = gen2Meta.auth.aws_region;
@@ -285,15 +296,15 @@ export async function assertFunctionResource(projRoot: string, gen2StackName: st
   assert(gen2Resource.FunctionName);
   removeProperties(gen2Resource, ['Arn', 'FunctionName', 'LoggingConfig.LogGroup', 'Role']);
   // TODO: remove below line after Environment.Variables.AMPLIFY_SSM_ENV_CONFIG, Tags inconsistency is fixed
-  removeProperties(gen2Resource, ['Environment.Variables.AMPLIFY_SSM_ENV_CONFIG', 'Tags']);
+  removeProperties(gen2Resource, ['Environment.Variables.AMPLIFY_SSM_ENV_CONFIG', 'Tags', 'Environment']);
 
   expect(gen2Resource).toEqual(gen1Resource);
 }
 
 export async function assertDataResource(projRoot: string, gen2StackName: string, gen1GraphqlApiId: string, gen1Region: string) {
   const gen1Resource = await getAppSyncApi(gen1GraphqlApiId, gen1Region);
   const gen1DataSource = (await getAppSyncDataSource(gen1GraphqlApiId, 'TodoTable', gen1Region)) as Record<string, unknown>;
-  removeProperties(gen1DataSource, ['dataSourceArn', 'serviceRoleArn']);
+  removeProperties(gen1DataSource, DATA_SOURCE_PROPS_TO_REMOVE);
   removeProperties(gen1Resource.graphqlApi as Record<string, unknown>, ['name', 'apiId', 'arn', 'uris', 'tags', 'dns']);
   // TODO: remove below line after authenticationType inconsistency is fixed
   removeProperties(gen1Resource.graphqlApi as Record<string, unknown>, ['authenticationType']);
@@ -304,7 +315,7 @@ export async function assertDataResource(projRoot: string, gen2StackName: string
   const gen2GraphqlApiId = outputs?.find((output) => output.OutputKey === 'awsAppsyncApiId')?.OutputValue ?? '';
   const gen2Resource = await getAppSyncApi(gen2GraphqlApiId, gen2Region);
   const gen2DataSource = (await getAppSyncDataSource(gen2GraphqlApiId, 'TodoTable', gen1Region)) as Record<string, unknown>;
-  removeProperties(gen2DataSource, ['dataSourceArn', 'serviceRoleArn']);
+  removeProperties(gen2DataSource, DATA_SOURCE_PROPS_TO_REMOVE);
   removeProperties(gen2Resource.graphqlApi as Record<string, unknown>, [
     'name',
     'apiId',

packages/amplify-migration-e2e/src/gen1ResourceDetailsFetcher.ts

@@ -2,7 +2,6 @@ import path from 'path';
 import { RefactorCategory } from './templategen';
 import { getProjectMeta } from '@aws-amplify/amplify-e2e-core';
 import { assertIdentityPool, assertStorage, assertUserPool, assertUserPoolClients } from './assertions';
-import { getResourceDetails } from './sdk_calls';
 
 async function getGen1AuthResourceDetails(projRoot: string) {
   const gen1ProjRoot = path.join(projRoot, '.amplify', 'migration');
@@ -14,13 +13,7 @@ async function getGen1AuthResourceDetails(projRoot: string) {
   const gen1ClientIdWeb = gen1ClientIds[0];
   const gen1ResourceIds = [gen1UserPoolId, gen1IdentityPoolId, gen1ClientIdWeb];
 
-  const gen1ResourceDetails = await Promise.all([
-    getResourceDetails('AWS::Cognito::UserPool', gen1UserPoolId, gen1Region),
-    getResourceDetails('AWS::Cognito::IdentityPool', gen1IdentityPoolId, gen1Region),
-    getResourceDetails('AWS::Cognito::UserPoolClient', `${gen1UserPoolId}|${gen1ClientIdWeb}`, gen1Region),
-  ]);
-
-  return { gen1ResourceIds, gen1ResourceDetails };
+  return { gen1ResourceIds };
 }
 
 async function getGen1StorageResourceDetails(projRoot: string) {
@@ -29,14 +22,14 @@ async function getGen1StorageResourceDetails(projRoot: string) {
   const gen1Region = gen1Meta.providers.awscloudformation.Region;
   const { gen1BucketName } = await assertStorage(gen1Meta, gen1Region);
   const gen1ResourceIds = [gen1BucketName];
-  const gen1ResourceDetails = await getResourceDetails('AWS::S3::Bucket', gen1BucketName, gen1Region);
-  return { gen1ResourceIds, gen1ResourceDetails };
+  return { gen1ResourceIds };
 }
 
 export async function getGen1ResourceDetails(projRoot: string, category: RefactorCategory) {
   if (category === 'auth') {
     return await getGen1AuthResourceDetails(projRoot);
-  } else {
+  } else if (category === 'storage') {
     return await getGen1StorageResourceDetails(projRoot);
   }
+  throw new Error(`Invalid category for getting Gen 1 resource details ${category}`);
 }

packages/amplify-migration-e2e/src/gen2ResourceDetailsFetcher.ts

@@ -1,37 +1,28 @@
 import { getProjectOutputs } from './projectOutputs';
-import { getResourceDetails } from './sdk_calls';
 import { RefactorCategory } from './templategen';
 
 async function getGen2AuthResourceDetails(projRoot: string) {
   const gen2Meta = getProjectOutputs(projRoot);
-  const gen2Region = gen2Meta.auth.aws_region;
   const gen2UserPoolId = gen2Meta.auth.user_pool_id;
   const gen2ClientIdWeb = gen2Meta.auth.user_pool_client_id;
   const gen2IdentityPoolId = gen2Meta.auth.identity_pool_id;
   const gen2ResourceIds = [gen2UserPoolId, gen2IdentityPoolId, gen2ClientIdWeb];
 
-  const gen2ResourceDetails = await Promise.all([
-    getResourceDetails('AWS::Cognito::UserPool', gen2UserPoolId, gen2Region),
-    getResourceDetails('AWS::Cognito::IdentityPool', gen2IdentityPoolId, gen2Region),
-    getResourceDetails('AWS::Cognito::UserPoolClient', `${gen2UserPoolId}|${gen2ClientIdWeb}`, gen2Region),
-  ]);
-
-  return { gen2ResourceIds, gen2ResourceDetails };
+  return { gen2ResourceIds };
 }
 
 async function getGen2StorageResourceDetails(projRoot: string) {
   const gen2Meta = getProjectOutputs(projRoot);
-  const gen2Region = gen2Meta.auth.aws_region;
   const gen2BucketName = gen2Meta.storage.bucket_name;
   const gen2ResourceIds = [gen2BucketName];
-  const gen2ResourceDetails = await getResourceDetails('AWS::S3::Bucket', gen2BucketName, gen2Region);
-  return { gen2ResourceIds, gen2ResourceDetails };
+  return { gen2ResourceIds };
 }
 
 export async function getGen2ResourceDetails(projRoot: string, category: RefactorCategory) {
   if (category === 'auth') {
     return await getGen2AuthResourceDetails(projRoot);
-  } else {
+  } else if (category === 'storage') {
     return await getGen2StorageResourceDetails(projRoot);
   }
+  throw new Error(`Invalid category for getting Gen 2 resource details ${category}`);
 }