diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 3971b60b259..2d599c8739e 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -3,20 +3,20 @@ Please make sure to read the Pull Request Guidelines: https://github.com/aws-amplify/amplify-cli/blob/dev/CONTRIBUTING.md#pull-requests --> -#### Description of changes +### Description of changes -#### Issue #, if available +### Issue #, if available -#### Description of how you validated changes +### Description of how you validated changes -#### Checklist +### Checklist @@ -27,4 +27,6 @@ the requirements below. - [ ] New AWS SDK calls or CloudFormation actions have been added to relevant test and service IAM policies - [ ] [Pull request labels](https://github.com/aws-amplify/amplify-cli/blob/dev/CONTRIBUTING.md#labels) are added +--- + By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. diff --git a/.gitignore b/.gitignore index a91ddd3d46a..c01f3fd2f8c 100644 --- a/.gitignore +++ b/.gitignore @@ -115,4 +115,4 @@ amplify-migration-apps/**/_snapshot.*.actual* .pr-body.ai-generated.md .commit-message.ai-generated.txt .kiro-session.ai-generated.md -**/.amplify/refactor.operations +**/.gen2-migration/refactor.operations diff --git a/.kiro/skills/gen2-migration/SKILL.md b/.kiro/skills/gen2-migration/SKILL.md new file mode 100644 index 00000000000..c8d3ab508ed --- /dev/null +++ b/.kiro/skills/gen2-migration/SKILL.md @@ -0,0 +1,116 @@ +--- +name: gen2-migration +description: Development guidance for the Amplify Gen1-to-Gen2 migration tooling — architecture, commands, testing, and snapshot workflows +--- + +# Gen2 Migration Development + +This skill provides context for working on the `amplify gen2-migration` CLI feature, +which migrates Amplify Gen1 applications to Gen2. + +## Context + +### Documentation + +Before changing code, read the relevant docs files. They are the source of truth +for architecture and design decisions: + +| Doc | Covers | +| --------------------------------------------------------------------------------- | ---------------------------------------------------------------- | +| `docs/packages/amplify-cli/src/commands/gen2-migration.md` | Architecture, CLI interface, Plan lifecycle, subcommand design | +| `packages/amplify-gen2-migration-e2e-system/README.md` | E2E automation system, CLI options, migration workflow steps | +| `packages/amplify-cli/src/__tests__/commands/gen2-migration/_framework/README.md` | Test framework, mock clients, snapshot comparison, customization | +| https://docs.amplify.aws/gen1/react/tools/cli/ | Amplify Gen1 CLI documentation | +| https://docs.amplify.aws/react/build-a-backend/ | Amplify Gen2 backend documentation | + +### Code + +- `packages/amplify-cli/src/commands/gen2-migration/` — CLI commands and core logic +- `packages/amplify-cli/src/__tests__/commands/gen2-migration/` — Snapshot and unit tests +- `packages/amplify-gen2-migration-e2e-system/` — E2E testing automation + +### Apps + +Each subdirectory under `amplify-migration-apps/` is a test app representing a Gen1 project +with a specific combination of Amplify categories and configurations. See +`amplify-migration-apps/README.md` for the full structure and conventions. + +## Development Loop + +### Fix a Bug + +The snapshot inputs (`_snapshot.pre.*`) don't change — only the code and possibly the +expected outputs (`_snapshot.post.*`) change. + +1. Read the relevant Context above for the area you're touching. + +2. Analyze the bug by reading the affected app's snapshot files. Read the + `_snapshot.pre.generate/` and/or `_snapshot.pre.refactor/` files to understand the + input configuration, and the `_snapshot.post.*` files to identify what's wrong in the + current output. + +3. Reproduce the bug by running the appropriate E2E test: + + ```bash + cd amplify-migration-apps/ + npm run test:e2e + ``` + + After the E2E run, inspect the live Gen1 and Gen2 resources and CloudFormation stack + events using the AWS CLI to confirm the root cause. + +4. Present the root cause analysis to the user. + +5. Determine what the correct expected output should be after the fix. Present the proposed + change to the expected output to the user and get approval before writing code. + +6. Make the code change in `packages/amplify-cli/src/commands/gen2-migration/`. + +7. Determine if the fix requires a new or updated E2E validation test in the affected app's + `tests/` directory. These tests run against deployed stacks to verify the migrated app + works correctly (API queries, storage operations, auth flows). See existing app tests + for the pattern (e.g., `amplify-migration-apps/project-boards/tests/`). + +8. Ask the user for confirmation before running E2E tests — they take a long time and + require AWS credentials. If approved, run on the affected apps to regenerate snapshots: + + ```bash + cd amplify-migration-apps/ + UPDATE_SNAPSHOTS=1 npm run test:e2e + ``` + +9. Run `yarn build && yarn test` in `packages/amplify-cli/` to verify nothing else broke. + If tests fail at this point, only test code changes should be needed — the production + code was already validated by the E2E run. + +### Implement a New Feature + +1. Read the relevant Context above for the area you're extending. + +2. Read the existing test apps to find one that covers a similar configuration, or determine + that a new app is needed. + +3. Follow the "Adding an App" or "Modifying an App" instructions from + `amplify-migration-apps/README.md` to update or create the `_snapshot.pre.generate/` + inputs. Skip the E2E step — we'll run it after making code changes. + +4. Determine what the expected output should be. Present the proposed expected output to + the user and get approval before writing code. + +5. Make the code change in `packages/amplify-cli/src/commands/gen2-migration/`. + +6. Determine if the feature requires a new or updated E2E validation test in the affected + app's `tests/` directory. See existing app tests for the pattern + (e.g., `amplify-migration-apps/project-boards/tests/`). + +7. Ask the user for confirmation before running E2E tests — they take a long time and + require AWS credentials. If approved, run on the affected apps to regenerate snapshots: + + ```bash + cd amplify-migration-apps/ + UPDATE_SNAPSHOTS=1 npm run test:e2e + ``` + +8. Run `yarn build && yarn test` in `packages/amplify-cli/` to verify nothing else broke. + If tests fail at this point, only test code changes should be needed — the production + code was already validated by the E2E run. diff --git a/AGENTS.md b/AGENTS.md index 8adaa4d6046..6143080f671 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -104,6 +104,8 @@ When asked to create a PR, generate a body into `.pr-body.ai-generated.md` **at - Do a 30 second summary of the important design information. - Do not go overboard on technical details. A reviewer can read the code. - Keep it concise and scannable. +- Don't enumerate files changed. Instead, categorize changes into logical groups. Give each + group an h4 title on its own line, followed by a blank line, then the explanation. ## Delegating to Sub-Agents diff --git a/amplify-migration-apps/.gitignore b/amplify-migration-apps/.gitignore index bebfb062a13..0d76a7e0520 100644 --- a/amplify-migration-apps/.gitignore +++ b/amplify-migration-apps/.gitignore @@ -1,4 +1,8 @@ -# Ignore amplify dirs created by `amplify init` at each app root, -# but not the ones inside _snapshot.* folders which are tracked. +# Ignore dirs created by amplify CLI commands. */amplify/ +*/.gen2-migration/ +*amplifyconfiguration.json +*amplify_outputs.json + +# But not the ones inside _snapshot.* folders which are tracked and santized. !*/_snapshot.*/amplify/ diff --git a/amplify-migration-apps/README.md b/amplify-migration-apps/README.md index cec5359d8fd..9d3f48cd785 100644 --- a/amplify-migration-apps/README.md +++ b/amplify-migration-apps/README.md @@ -258,17 +258,17 @@ directories from a deployed Amplify app. It requires AWS credentials with access deployed app's CloudFormation stacks and Amplify resources. ```console -npx tsx snapshot.ts [deployed-app-path] +npx tsx snapshot.ts [deployed-app-path] [gen2-stack-name] ``` Where `` is one of: -| Step | Description | Requires `deployed-app-path`? | -| ---------------- | --------------------------------------------------------------------------- | ----------------------------- | -| `pre.generate` | Copies the Gen1 app's `amplify/`, `.gitignore`, and `package.json` | Yes | -| `post.generate` | Copies the Gen2 output (`amplify/`, `.gitignore`, `amplify.yml`) | Yes | -| `pre.refactor` | Downloads Gen1 and Gen2 CloudFormation templates from deployed stacks | No (reads from AWS directly) | -| `post.refactor` | Copies the refactor operations from `.amplify/refactor.operations` | Yes | +| Step | Description | Required args | +| ---------------- | --------------------------------------------------------------------------- | -------------------------------------- | +| `pre.generate` | Copies the Gen1 app's `amplify/`, `.gitignore`, and `package.json` | `deployed-app-path` | +| `post.generate` | Copies the Gen2 output (`amplify/`, `.gitignore`, `amplify.yml`) | `deployed-app-path` | +| `pre.refactor` | Downloads Gen1 and Gen2 CloudFormation templates from deployed stacks | `gen2-stack-name` | +| `post.refactor` | Copies the refactor operations from `.gen2-migration/refactor.operations` | `deployed-app-path` | Examples: @@ -280,7 +280,7 @@ npx tsx snapshot.ts pre.generate fitness-tracker /path/to/deployed/fitness-track npx tsx snapshot.ts post.generate fitness-tracker /path/to/deployed/fitness-tracker # Download CloudFormation templates for refactor input (requires AWS credentials) -npx tsx snapshot.ts pre.refactor fitness-tracker +npx tsx snapshot.ts pre.refactor fitness-tracker /path/to/deployed/fitness-tracker amplify-fitnesstracker-gen2main-branch-abc1234567 # Capture the expected refactor output npx tsx snapshot.ts post.refactor fitness-tracker /path/to/deployed/fitness-tracker @@ -288,49 +288,32 @@ npx tsx snapshot.ts post.refactor fitness-tracker /path/to/deployed/fitness-trac ## Adding an App -1. Create a new directory under `` that contains the entire Gen1 application, as is. -2. Add the following script directives to the `package.json` file: - - ```json - "scripts": { - "sanitize": "tsx ../sanitize.ts", - "typecheck": "cd _snapshot.post.generate/amplify && npx tsc --noEmit" - } - ``` - -3. Add the following to the `package.json` file: - - ```json - "installConfig": { - "hoistingLimits": "workspaces" - }, - ``` - - This ensures dependencies in the app don't interfere or conflict with the main repo dependencies. - -4. Use the [Snapshot Capture Tool](#snapshot-capture-tool) to capture all required snapshots. - Follow the app's migration guide, running the tool at each step: +1. Create a new directory under `` that contains the frontend code for your Gen1 application. +Make sure to follow the existing patterns and add tests as well. +2. Run `amplify init`. +3. Configure the backend using Gen1 CLI. +4. Run `amplify push`. +5. Use the [Snapshot Capture Tool](#snapshot-capture-tool) to capture the `pre.generate` snapshot. ```console - # Before running generate - npx tsx snapshot.ts pre.generate /path/to/deployed/ - - # After running generate - npx tsx snapshot.ts post.generate /path/to/deployed/ + npx tsx snapshot.ts pre.generate - # Before running refactor (requires AWS credentials) - npx tsx snapshot.ts pre.refactor +6. Run `UPDATE_SNAPSHOTS=1 npm run test:e2e` to execute the full migration flow and capture + the remaining snapshots (`post.generate`, `pre.refactor`, `post.refactor`). - # After running refactor - npx tsx snapshot.ts post.refactor /path/to/deployed/ - ``` +## Modifying an App -5. Run the sanitize script to replace sensitive values with placeholders: +1. `cd` into a specific app and run `npm run deploy`. +2. Locate the deployed app directory in output logs and `cd` into it. +3. Update the backend using Gen1 CLI. +4. Run `amplify push`. +5. Use the [Snapshot Capture Tool](#snapshot-capture-tool) to capture the `pre.generate` snapshot. ```console - cd amplify-migration-apps/ - npm run sanitize - ``` + npx tsx snapshot.ts pre.generate + +6. Run `UPDATE_SNAPSHOTS=1 npm run test:e2e` to execute the full migration flow and capture + the remaining snapshots (`post.generate`, `pre.refactor`, `post.refactor`). ## Snapshot Testing @@ -426,7 +409,6 @@ Always review the diff after updating to make sure the changes are intentional. > because it detects the diff before writing the updated files. Run the tests a second time > (without `--updateSnapshot`) to verify the snapshots are now correct. - ## Integration Testing (E2E) The [E2E system](../packages/amplify-gen2-migration-e2e-system/) automates the full migration @@ -441,19 +423,10 @@ cd packages/amplify-cli && yarn build export AMPLIFY_PATH=$(pwd)/.bin/amplify-dev # Run the full migration for a specific app -cd packages/amplify-gen2-migration-e2e-system -npx tsx src/cli.ts --app project-boards --profile default +cd amplify-migration-apps/ +npm run test:e2e ``` -This will: -1. Deploy the Gen1 app via `amplify push` -2. Run `test:gen1` to validate the Gen1 stack -3. Execute the full `gen2-migration` workflow (assess, lock, generate, deploy, refactor, redeploy) -4. Run `test:gen1` and `test:gen2` after each deployment - The system automatically runs npm scripts from the app's `package.json` at the right -points in the workflow — `post-generate` after generate, `post-refactor` after refactor, -and `post-push` after push. Scripts that resolve to `true` (no-op) are effectively skipped. - -See the [E2E system README](../packages/amplify-gen2-migration-e2e-system/README.md) for -CLI options and troubleshooting. +points in the workflow. See the [E2E system README](../packages/amplify-gen2-migration-e2e-system/README.md) for +more details. diff --git a/amplify-migration-apps/backend-only/package.json b/amplify-migration-apps/backend-only/package.json index d01a494e107..927be173178 100644 --- a/amplify-migration-apps/backend-only/package.json +++ b/amplify-migration-apps/backend-only/package.json @@ -13,6 +13,7 @@ "test:gen1": "true", "test:gen2": "true", "test:e2e": "cd ../../packages/amplify-gen2-migration-e2e-system && npx tsx src/cli.ts --app backend-only --profile ${AWS_PROFILE:-default}", + "deploy": "cd ../../packages/amplify-gen2-migration-e2e-system && npx tsx src/cli.ts --app backend-only --step deploy --profile ${AWS_PROFILE:-default}", "pre-push": "true", "post-generate": "npx tsx migration/post-generate.ts", "post-refactor": "npx tsx migration/post-refactor.ts", diff --git a/amplify-migration-apps/discussions/package.json b/amplify-migration-apps/discussions/package.json index 5e7cca8e013..40534d684c2 100644 --- a/amplify-migration-apps/discussions/package.json +++ b/amplify-migration-apps/discussions/package.json @@ -17,6 +17,7 @@ "test:gen1": "APP_CONFIG_PATH=${APP_CONFIG_PATH:-src/amplifyconfiguration.json} NODE_OPTIONS='--experimental-vm-modules' jest --verbose", "test:gen2": "APP_CONFIG_PATH=${APP_CONFIG_PATH:-amplify_outputs.json} NODE_OPTIONS='--experimental-vm-modules' jest --verbose", "test:e2e": "cd ../../packages/amplify-gen2-migration-e2e-system && npx tsx src/cli.ts --app discussions --profile ${AWS_PROFILE:-default}", + "deploy": "cd ../../packages/amplify-gen2-migration-e2e-system && npx tsx src/cli.ts --app discussions --step deploy --profile ${AWS_PROFILE:-default}", "pre-push": "true", "post-generate": "npx tsx migration/post-generate.ts", "post-refactor": "npx tsx migration/post-refactor.ts", diff --git a/amplify-migration-apps/fitness-tracker/package.json b/amplify-migration-apps/fitness-tracker/package.json index b0a410907dd..4623e97b4da 100644 --- a/amplify-migration-apps/fitness-tracker/package.json +++ b/amplify-migration-apps/fitness-tracker/package.json @@ -19,6 +19,7 @@ "test:gen1": "APP_CONFIG_PATH=${APP_CONFIG_PATH:-src/amplifyconfiguration.json} NODE_OPTIONS='--experimental-vm-modules' jest --verbose", "test:gen2": "APP_CONFIG_PATH=${APP_CONFIG_PATH:-amplify_outputs.json} NODE_OPTIONS='--experimental-vm-modules' jest --verbose", "test:e2e": "cd ../../packages/amplify-gen2-migration-e2e-system && npx tsx src/cli.ts --app fitness-tracker --profile ${AWS_PROFILE:-default}", + "deploy": "cd ../../packages/amplify-gen2-migration-e2e-system && npx tsx src/cli.ts --app fitness-tracker --step deploy --profile ${AWS_PROFILE:-default}", "pre-push": "true", "post-generate": "npx tsx migration/post-generate.ts", "post-refactor": "true", diff --git a/amplify-migration-apps/imported-resources/package.json b/amplify-migration-apps/imported-resources/package.json index 43ec22a002c..0898b352ab7 100644 --- a/amplify-migration-apps/imported-resources/package.json +++ b/amplify-migration-apps/imported-resources/package.json @@ -18,6 +18,7 @@ "test:gen1": "APP_CONFIG_PATH=${APP_CONFIG_PATH:-src/amplifyconfiguration.json} NODE_OPTIONS='--experimental-vm-modules' jest --verbose", "test:gen2": "APP_CONFIG_PATH=${APP_CONFIG_PATH:-amplify_outputs.json} NODE_OPTIONS='--experimental-vm-modules' jest --verbose", "test:e2e": "cd ../../packages/amplify-gen2-migration-e2e-system && npx tsx src/cli.ts --app imported-resources --profile ${AWS_PROFILE:-default}", + "deploy": "cd ../../packages/amplify-gen2-migration-e2e-system && npx tsx src/cli.ts --app imported-resources --step deploy --profile ${AWS_PROFILE:-default}", "pre-push": "true", "post-generate": "npx tsx migration/post-generate.ts", "post-refactor": "npx tsx migration/post-refactor.ts", diff --git a/amplify-migration-apps/media-vault/package.json b/amplify-migration-apps/media-vault/package.json index 81a79831463..7627aad24d5 100644 --- a/amplify-migration-apps/media-vault/package.json +++ b/amplify-migration-apps/media-vault/package.json @@ -17,6 +17,7 @@ "test:gen1": "APP_CONFIG_PATH=${APP_CONFIG_PATH:-src/amplifyconfiguration.json} NODE_OPTIONS='--experimental-vm-modules' jest --verbose", "test:gen2": "APP_CONFIG_PATH=${APP_CONFIG_PATH:-amplify_outputs.json} NODE_OPTIONS='--experimental-vm-modules' jest --verbose", "test:e2e": "cd ../../packages/amplify-gen2-migration-e2e-system && npx tsx src/cli.ts --app media-vault --profile ${AWS_PROFILE:-default}", + "deploy": "cd ../../packages/amplify-gen2-migration-e2e-system && npx tsx src/cli.ts --app media-vault --step deploy --profile ${AWS_PROFILE:-default}", "pre-push": "npx tsx migration/pre-push.ts", "post-generate": "npx tsx migration/post-generate.ts", "post-refactor": "true", diff --git a/amplify-migration-apps/mood-board/package.json b/amplify-migration-apps/mood-board/package.json index ec9db895445..b19ecc73e71 100644 --- a/amplify-migration-apps/mood-board/package.json +++ b/amplify-migration-apps/mood-board/package.json @@ -16,6 +16,7 @@ "test:gen1": "APP_CONFIG_PATH=${APP_CONFIG_PATH:-src/amplifyconfiguration.json} NODE_OPTIONS='--experimental-vm-modules' jest --verbose", "test:gen2": "APP_CONFIG_PATH=${APP_CONFIG_PATH:-amplify_outputs.json} NODE_OPTIONS='--experimental-vm-modules' jest --verbose", "test:e2e": "cd ../../packages/amplify-gen2-migration-e2e-system && npx tsx src/cli.ts --app mood-board --profile ${AWS_PROFILE:-default}", + "deploy": "cd ../../packages/amplify-gen2-migration-e2e-system && npx tsx src/cli.ts --app mood-board --step deploy --profile ${AWS_PROFILE:-default}", "pre-push": "true", "post-generate": "npx tsx migration/post-generate.ts", "post-refactor": "npx tsx migration/post-refactor.ts", diff --git a/amplify-migration-apps/product-catalog/package.json b/amplify-migration-apps/product-catalog/package.json index 27dd08fc2c7..cacecb0d7bb 100644 --- a/amplify-migration-apps/product-catalog/package.json +++ b/amplify-migration-apps/product-catalog/package.json @@ -17,6 +17,7 @@ "test:gen1": "APP_CONFIG_PATH=${APP_CONFIG_PATH:-src/amplifyconfiguration.json} NODE_OPTIONS='--experimental-vm-modules' jest --verbose", "test:gen2": "APP_CONFIG_PATH=${APP_CONFIG_PATH:-amplify_outputs.json} NODE_OPTIONS='--experimental-vm-modules' jest --verbose", "test:e2e": "cd ../../packages/amplify-gen2-migration-e2e-system && npx tsx src/cli.ts --app product-catalog --profile ${AWS_PROFILE:-default}", + "deploy": "cd ../../packages/amplify-gen2-migration-e2e-system && npx tsx src/cli.ts --app product-catalog --step deploy --profile ${AWS_PROFILE:-default}", "pre-push": "npx tsx migration/pre-push.ts", "post-generate": "npx tsx migration/post-generate.ts", "post-refactor": "npx tsx migration/post-refactor.ts", diff --git a/amplify-migration-apps/project-boards/package.json b/amplify-migration-apps/project-boards/package.json index 7e7a0a587e3..f407387d2ac 100644 --- a/amplify-migration-apps/project-boards/package.json +++ b/amplify-migration-apps/project-boards/package.json @@ -17,6 +17,7 @@ "test:gen1": "APP_CONFIG_PATH=${APP_CONFIG_PATH:-src/amplifyconfiguration.json} NODE_OPTIONS='--experimental-vm-modules' jest --verbose", "test:gen2": "APP_CONFIG_PATH=${APP_CONFIG_PATH:-amplify_outputs.json} NODE_OPTIONS='--experimental-vm-modules' jest --verbose", "test:e2e": "cd ../../packages/amplify-gen2-migration-e2e-system && npx tsx src/cli.ts --app project-boards --profile ${AWS_PROFILE:-default}", + "deploy": "cd ../../packages/amplify-gen2-migration-e2e-system && npx tsx src/cli.ts --app project-boards --step deploy --profile ${AWS_PROFILE:-default}", "pre-push": "true", "post-generate": "npx tsx migration/post-generate.ts", "post-refactor": "npx tsx migration/post-refactor.ts", diff --git a/amplify-migration-apps/sanitize.ts b/amplify-migration-apps/sanitize.ts index 12387253e39..332686fec2b 100644 --- a/amplify-migration-apps/sanitize.ts +++ b/amplify-migration-apps/sanitize.ts @@ -1,118 +1,19 @@ #!/usr/bin/env npx tsx +import * as path from 'path'; +import { sanitize } from '../packages/amplify-gen2-migration-e2e-system/src/core/sanitize'; + /** * Sanitizes sensitive values in Amplify migration app snapshot files for safe public commit. * * Usage: cd into an app directory under amplify-migration-apps/, then run: * npx tsx ../sanitize.ts - * - * Strategy: - * 1. Extract sensitive values from _snapshot.pre.generate/amplify/backend/amplify-meta.json - * 2. Replace all occurrences of those values across all _snapshot.* directories - * 3. Rename files whose names contain the Amplify App ID - * - * Targets: - * - AWS Account ID (from providers.awscloudformation AuthRoleArn) → replaced with 123456789012 - * - Amplify App ID (from providers.awscloudformation) → replaced with app name (dashes removed) - * - AppSync API Key (from api output, if present) → replaced with da2-fakeapikey00000000000000 */ - -import * as fs from 'fs'; -import * as path from 'path'; - -interface SensitiveValues { - accountId: string; - amplifyAppId: string; - apiKey: string | null; -} - -function extractAccountId(meta: any): string { - const authRoleArn = meta.providers.awscloudformation.AuthRoleArn; - const arnMatch = authRoleArn.match(/arn:aws:iam::(\d{12}):/); - if (!arnMatch) { - throw new Error('Could not extract AWS Account ID from AuthRoleArn'); - } - return arnMatch[1]; -} - -function extractAmplifyAppId(meta: any): string { - const appId = meta.providers.awscloudformation.AmplifyAppId; - if (!appId) { - throw new Error('Could not extract Amplify App ID from amplify-meta.json'); - } - return appId; -} - -function extractApiKey(meta: any): string | null { - if (!meta.api) return null; - const firstApiResource = Object.keys(meta.api)[0]; - return meta.api[firstApiResource]?.output?.GraphQLAPIKeyOutput ?? null; -} - -function extractSensitiveValues(meta: any): SensitiveValues { - return { - accountId: extractAccountId(meta), - amplifyAppId: extractAmplifyAppId(meta), - apiKey: extractApiKey(meta), - }; -} - -function getFilesRecursive(dir: string): string[] { - const files: string[] = []; - const entries = fs.readdirSync(dir, { withFileTypes: true }); - - for (const entry of entries) { - const fullPath = path.join(dir, entry.name); - if (fullPath.includes('node_modules')) { - continue; - } - if (entry.isDirectory()) { - files.push(...getFilesRecursive(fullPath)); - } else if (entry.isFile()) { - files.push(fullPath); - } - } - - return files; -} - -function sanitizeFileName(name: string, appId: string, appName: string): string { - return name.replaceAll(appId, appName); -} - -function getAllFiles(dir: string): string[] { - return getFilesRecursive(dir); -} - -function main(): void { - +async function main() { const appName = path.basename(process.cwd()); - const appNameNoDashes = appName.replaceAll('-', ''); - const appDir = path.join(__dirname, appName); - const metaPath = path.join(appDir, '_snapshot.pre.generate', 'amplify', 'backend', 'amplify-meta.json'); - const amplifyMeta: any = JSON.parse(fs.readFileSync(metaPath, 'utf-8')); - const values = extractSensitiveValues(amplifyMeta); - - - const snapshots = fs.readdirSync(appDir).filter(f => f.startsWith('_snapshot')); - const files = [...snapshots.flatMap(s => getAllFiles(path.join(appDir, s)))]; - - for (const file of files) { - let content = fs.readFileSync(file, 'utf-8'); - - content = content.replaceAll(values.accountId, '123456789012'); - content = content.replaceAll(values.amplifyAppId, appNameNoDashes); - - if (values.apiKey) { - content = content.replaceAll(values.apiKey, 'da2-fakeapikey00000000000000'); - } - - const sanitizedFileName = sanitizeFileName(file, values.amplifyAppId, appNameNoDashes); - - fs.writeFileSync(file, content, 'utf-8'); - fs.renameSync(file, sanitizedFileName); - } + sanitize(appName, appDir) } main(); + diff --git a/amplify-migration-apps/snapshot.ts b/amplify-migration-apps/snapshot.ts index 9729d7bd933..b048d91dd3e 100644 --- a/amplify-migration-apps/snapshot.ts +++ b/amplify-migration-apps/snapshot.ts @@ -2,245 +2,52 @@ import * as fs from 'fs-extra'; import * as path from 'path'; -import { AmplifyClient, paginateListApps, App } from '@aws-sdk/client-amplify'; -import { - CloudFormationClient, - DescribeStacksCommand, - GetTemplateCommand, - StackStatus, - paginateListStackResources, - paginateListStacks, -} from '@aws-sdk/client-cloudformation'; +import * as e2esnap from '../packages/amplify-gen2-migration-e2e-system/src/core/snapshot'; const STEPS = ['pre.generate', 'post.generate', 'pre.refactor', 'post.refactor'] as const; type Step = (typeof STEPS)[number]; -// --------------------------------------------------------------------------- -// Shared helpers -// --------------------------------------------------------------------------- - -function resetDir(dir: string): void { - if (fs.existsSync(dir)) { - fs.rmdirSync(dir, { recursive: true }); - } - fs.mkdirSync(dir, { recursive: true }); -} - -function writeFileSync(p: string, content: string): void { - console.log(p); - fs.writeFileSync(p, content, 'utf-8'); -} - -async function copySync(src: string, dest: string): Promise { - if (fs.existsSync(dest)) { - fs.rmSync(dest, { recursive: true }); - } - console.log(dest); - fs.copySync(src, dest, { recursive: true }); -} - -async function copyRequired(srcBasePath: string, destBasePath: string, toCopy: readonly string[]): Promise { - for (const required of toCopy) { - const inputPath = path.join(srcBasePath, required); - if (!fs.existsSync(inputPath)) { - throw new Error(`Required input not found: ${inputPath}`); - } - const destPath = path.join(destBasePath, required); - await copySync(inputPath, destPath); - } -} - -async function copyOptional(srcBasePath: string, destBasePath: string, toCopy: readonly string[]): Promise { - for (const optional of toCopy) { - const inputPath = path.join(srcBasePath, optional); - const destPath = path.join(destBasePath, optional); - if (fs.existsSync(inputPath)) { - await copySync(inputPath, destPath); - } - } -} - -// --------------------------------------------------------------------------- -// Amplify helpers -// --------------------------------------------------------------------------- - -const amplifyClient = new AmplifyClient({}); - -async function findAppByName(appName: string): Promise { - for await (const page of paginateListApps({ client: amplifyClient }, { maxResults: 25 })) { - const match = page.apps?.find((app) => app.name === appName); - if (match) { - return match; - } - } - throw new Error(`Amplify app "${appName}" not found`); -} - -// --------------------------------------------------------------------------- -// CloudFormation helpers (refactor.input only) -// --------------------------------------------------------------------------- - -const cfnClient = new CloudFormationClient({}); - -const ACTIVE_STATUSES = [StackStatus.CREATE_COMPLETE, StackStatus.UPDATE_COMPLETE, StackStatus.UPDATE_ROLLBACK_COMPLETE]; - -async function findStackByPattern(pattern: RegExp): Promise { - for await (const page of paginateListStacks({ client: cfnClient }, { StackStatusFilter: ACTIVE_STATUSES })) { - const match = page.StackSummaries?.find((s) => s.StackName && pattern.test(s.StackName)); - if (match?.StackName) return match.StackName; - } - throw new Error(`No stack found matching pattern "${pattern.source}"`); -} - -async function findGen2RootStack(appId: string, branchName: string): Promise { - const branchNoDashes = branchName.replace(/-/g, ''); - const rootPattern = new RegExp(`^amplify-${appId}-${branchNoDashes}-branch-[0-9a-f]{10}$`); - return findStackByPattern(rootPattern); -} - -async function findGen1RootStack(appName: string, envName: string): Promise { - const rootPattern = new RegExp(`^amplify-${appName}-${envName}-[0-9a-f]{5}$`); - return findStackByPattern(rootPattern); -} - -async function fetchTemplate(stackName: string): Promise { - const response = await cfnClient.send(new GetTemplateCommand({ StackName: stackName, TemplateStage: 'Original' })); - return response.TemplateBody!; -} - -async function fetchNestedStacks(stackName: string): Promise { - const ids: string[] = []; - for await (const page of paginateListStackResources({ client: cfnClient }, { StackName: stackName })) { - for (const r of page.StackResourceSummaries ?? []) { - if (r.ResourceType === 'AWS::CloudFormation::Stack' && r.PhysicalResourceId) { - ids.push(r.PhysicalResourceId); - } - } - } - return ids; -} - -function stackNameFromArn(arnOrName: string): string { - if (arnOrName.startsWith('arn:')) { - const parts = arnOrName.split('/'); - return parts[1] ?? arnOrName; - } - return arnOrName; -} - -async function downloadRecursive(stackNameOrArn: string, targetDir: string): Promise { - const stackName = stackNameFromArn(stackNameOrArn); - - const template = await fetchTemplate(stackName); - writeFileSync(path.join(targetDir, `${stackName}.template.json`), JSON.stringify(JSON.parse(template), null, 2)); - - const stackResponse = await cfnClient.send(new DescribeStacksCommand({ StackName: stackName })); - const stack = stackResponse.Stacks![0]; - - const outputs = stack.Outputs ?? []; - writeFileSync(path.join(targetDir, `${stackName}.outputs.json`), JSON.stringify(outputs, null, 2)); - - const parameters = stack.Parameters ?? []; - writeFileSync(path.join(targetDir, `${stackName}.parameters.json`), JSON.stringify(parameters, null, 2)); - - const description = stack.Description ?? ''; - writeFileSync(path.join(targetDir, `${stackName}.description.txt`), description); - - const nestedIds = await fetchNestedStacks(stackName); - for (const nestedId of nestedIds) { - await downloadRecursive(nestedId, targetDir); - } -} - -// --------------------------------------------------------------------------- -// Snapshot capture functions -// --------------------------------------------------------------------------- - -async function capturePreRefactor(appName: string, amplifyAppName?: string, gen2Branch?: string, gen1Env?: string): Promise { - const resolvedAppName = amplifyAppName ?? appName.replaceAll('-', ''); - const app = await findAppByName(resolvedAppName); - const gen2RootStack = await findGen2RootStack(app.appId!, gen2Branch ?? 'gen2-main'); - const gen1RootStack = await findGen1RootStack(app.name!, gen1Env ?? 'main'); - - const targetDir = path.resolve(path.join(__dirname, appName, '_snapshot.pre.refactor')); - resetDir(targetDir); - - await downloadRecursive(gen2RootStack, targetDir); - await downloadRecursive(gen1RootStack, targetDir); -} - -async function capturePostRefactor(appName: string, deployedAppPath: string): Promise { - const srcBasePath = path.join(deployedAppPath, '.amplify/refactor.operations'); - const dstBasePath = path.join(__dirname, appName, '_snapshot.post.refactor'); - resetDir(dstBasePath); - copySync(srcBasePath, dstBasePath); -} - -async function capturePreGenerate(appName: string, deployedAppPath: string): Promise { - const dstBasePath = path.join(__dirname, appName, '_snapshot.pre.generate'); - resetDir(dstBasePath); - - await copyRequired(deployedAppPath, dstBasePath, ['amplify', '.gitignore']); - await copyOptional(deployedAppPath, dstBasePath, ['package.json']); - - // For the snapshot we want to include all files - const gitIgnorePath = path.join(dstBasePath, '.gitignore'); - const gitIgnore = fs.readFileSync(gitIgnorePath, { encoding: 'utf-8' }); - const newGitIgnore = gitIgnore - .replaceAll('amplify/', '!amplify/') - .replaceAll('build/', '!build/') - .replaceAll('!amplify/.config/local-*', 'amplify/.config/local-*'); - fs.writeFileSync(gitIgnorePath, newGitIgnore); -} - -async function capturePostGenerate(appName: string, deployedAppPath: string): Promise { - const dstBasePath = path.join(__dirname, appName, '_snapshot.post.generate'); - resetDir(dstBasePath); - - await copyRequired(deployedAppPath, dstBasePath, ['amplify', '.gitignore', 'amplify.yml']); - await copyOptional(deployedAppPath, dstBasePath, ['package.json']); -} - -// --------------------------------------------------------------------------- -// CLI -// --------------------------------------------------------------------------- - function usage(): never { - console.error(`Usage: npx tsx snapshot.ts [deployed-app-path] [amplify-app-name] [gen2-branch] [gen1-env] + console.error(`Usage: npx tsx snapshot.ts [deployed-app-path] [gen2-stack-name] Steps: ${STEPS.join(', ')} - app-name: Directory name under amplify-migration-apps/ - deployed-app-path: Path to the deployed app (required for pre/post.generate and post.refactor) - amplify-app-name: Actual Amplify app name if different from app-name (default: app-name without dashes) - gen2-branch: Gen2 branch name (default: gen2-main) - gen1-env: Gen1 environment name (default: main)`); + app-dir: Directory name under amplify-migration-apps/ + deployed-app-path: Path to the deployed app (required for pre/post.generate and post.refactor; defaults to app-dir) + gen2-stack-name: Gen2 root stack name (required for pre.refactor)`); process.exit(1); } async function main(): Promise { - const [snapshot, appName, deployedAppPath, amplifyAppName, gen2Branch, gen1Env] = process.argv.slice(2); + const [snapshot, appDir, deployPath, gen2StackName] = process.argv.slice(2); - if (!snapshot || !STEPS.includes(snapshot as Step) || !appName) { + if (!snapshot || !STEPS.includes(snapshot as Step) || !appDir) { usage(); } + const sourceAppPath = path.resolve(path.join(__dirname, appDir)); + const deployedAppPath = deployPath ?? sourceAppPath; + switch (snapshot as Step) { case 'pre.generate': if (!deployedAppPath) usage(); - await capturePreGenerate(appName, deployedAppPath); + await e2esnap.capturePreGenerate(deployedAppPath, sourceAppPath) break; case 'post.generate': if (!deployedAppPath) usage(); - await capturePostGenerate(appName, deployedAppPath); + await e2esnap.capturePostGenerate(deployedAppPath, sourceAppPath); break; case 'pre.refactor': - await capturePreRefactor(appName, amplifyAppName, gen2Branch, gen1Env); + if (!gen2StackName) usage(); + const tpiPath = path.join(deployedAppPath, 'amplify', 'team-provider-info.json'); + const tpi = JSON.parse(fs.readFileSync(tpiPath, { encoding: 'utf-8'})); + const gen1StackName = (Object.values(tpi)[0] as any).awscloudformation.StackName; + await e2esnap.capturePreRefactor(gen1StackName, gen2StackName, sourceAppPath); break; case 'post.refactor': if (!deployedAppPath) usage(); - await capturePostRefactor(appName, deployedAppPath); + await e2esnap.capturePostRefactor(deployedAppPath, sourceAppPath); break; } } diff --git a/amplify-migration-apps/store-locator/package.json b/amplify-migration-apps/store-locator/package.json index 4e80af2accd..ae6c80fb4b5 100644 --- a/amplify-migration-apps/store-locator/package.json +++ b/amplify-migration-apps/store-locator/package.json @@ -16,6 +16,7 @@ "test:gen1": "APP_CONFIG_PATH=${APP_CONFIG_PATH:-src/amplifyconfiguration.json} NODE_OPTIONS='--experimental-vm-modules' jest --verbose", "test:gen2": "APP_CONFIG_PATH=${APP_CONFIG_PATH:-amplify_outputs.json} NODE_OPTIONS='--experimental-vm-modules' jest --verbose", "test:e2e": "cd ../../packages/amplify-gen2-migration-e2e-system && npx tsx src/cli.ts --app store-locator --profile ${AWS_PROFILE:-default}", + "deploy": "cd ../../packages/amplify-gen2-migration-e2e-system && npx tsx src/cli.ts --app store-locator --step deploy --profile ${AWS_PROFILE:-default}", "pre-push": "true", "post-generate": "npx tsx migration/post-generate.ts", "post-refactor": "true", diff --git a/packages/amplify-cli/src/commands/gen2-migration/generate/gitignore.generator.ts b/packages/amplify-cli/src/commands/gen2-migration/generate/gitignore.generator.ts index e2ee6fa5d63..9c62507dbc8 100644 --- a/packages/amplify-cli/src/commands/gen2-migration/generate/gitignore.generator.ts +++ b/packages/amplify-cli/src/commands/gen2-migration/generate/gitignore.generator.ts @@ -3,7 +3,16 @@ import fs from 'node:fs/promises'; import { Planner } from '../_infra/planner'; import { AmplifyMigrationOperation } from '../_infra/operation'; -const GEN2_GITIGNORE_ENTRIES = ['.amplify', 'amplify_outputs*', 'amplifyconfiguration*', 'aws-exports*', 'node_modules', 'build', 'dist']; +const GEN2_GITIGNORE_ENTRIES = [ + '.gen2-migration', + '.amplify', + 'amplify_outputs*', + 'amplifyconfiguration*', + 'aws-exports*', + 'node_modules', + 'build', + 'dist', +]; /** * Updates .gitignore: removes the Gen1 amplify block and adds Gen2 entries. diff --git a/packages/amplify-cli/src/commands/gen2-migration/refactor/cfn.ts b/packages/amplify-cli/src/commands/gen2-migration/refactor/cfn.ts index 613e5e8c855..3ccf6dd4e57 100644 --- a/packages/amplify-cli/src/commands/gen2-migration/refactor/cfn.ts +++ b/packages/amplify-cli/src/commands/gen2-migration/refactor/cfn.ts @@ -36,7 +36,7 @@ import { DiscoveredResource } from '../generate/_infra/gen1-app'; const MAX_WAIT_TIME_SECONDS = 900; const NO_UPDATES_MESSAGE = 'No updates are to be performed'; const CFN_IAM_CAPABILITY = 'CAPABILITY_NAMED_IAM'; -export const OUTPUT_DIRECTORY = '.amplify/refactor.operations'; +export const OUTPUT_DIRECTORY = '.gen2-migration/refactor.operations'; const EMPTY_HOLDING_TEMPLATE: CFNTemplate = { AWSTemplateFormatVersion: '2010-09-09', diff --git a/packages/amplify-gen2-migration-e2e-system/package.json b/packages/amplify-gen2-migration-e2e-system/package.json index a51f18216d0..e808f1f8c66 100644 --- a/packages/amplify-gen2-migration-e2e-system/package.json +++ b/packages/amplify-gen2-migration-e2e-system/package.json @@ -15,6 +15,8 @@ }, "dependencies": { "@aws-amplify/amplify-e2e-core": "workspace:^", + "@aws-sdk/client-amplify": "^3.919.0", + "@aws-sdk/client-cloudformation": "^3.919.0", "@paralleldrive/cuid2": "^3.0.6", "chalk": "^4.1.2", "execa": "^5.1.1", diff --git a/packages/amplify-gen2-migration-e2e-system/src/cli.ts b/packages/amplify-gen2-migration-e2e-system/src/cli.ts index 7d1c2b69688..858f73c6f5b 100644 --- a/packages/amplify-gen2-migration-e2e-system/src/cli.ts +++ b/packages/amplify-gen2-migration-e2e-system/src/cli.ts @@ -27,6 +27,12 @@ async function main(): Promise { description: 'AWS profile to use', string: true, }) + .option('step', { + type: 'string', + description: 'Stop migration workflow at this step', + choices: ['deploy', 'migrate'], + string: true, + }) .help() .alias('help', 'h') .version() @@ -43,70 +49,30 @@ async function main(): Promise { throw new Error('--profile must be specified'); } + const step = argv.step ?? 'migrate'; + const app = new App(argv.app, argv.profile, argv.verbose); try { - await migrate(app); - app.logger.info('Migration completed successfully'); + switch (step) { + case 'deploy': + await app.deploy(); + break; + case 'migrate': + await app.migrate(); + break; + default: + throw new Error(`Unrecognized step: ${step}`); + } + if (process.env.UPDATE_SNAPSHOTS === '1') { + app.updateSnapshots(); + } + app.logger.info(`Execution completed successfully (${app.targetAppPath})`); } catch (error) { - (error as Error).message = `Migration failed: ${chalk.red((error as Error).message)} (${app.targetAppPath})`; + (error as Error).message = `Execution failed: ${chalk.red((error as Error).message)} (${app.targetAppPath})`; throw error; } } -async function migrate(app: App): Promise { - app.logger.info(`Starting migration`); - - await app.gitInit(); - await app.init(); - await app.configure(); - await app.installDeps(); - await app.status(); - await app.prePush(); - await app.push(); - await app.postPush(); - await app.gitCommit('chore: post push'); - - await app.testGen1(); - - await app.assess(); - await app.lock(); - await app.gitCheckoutGen2(true); - await app.generate(); - await app.gitCommit('chore: generate'); - await app.installDeps(); - await app.gitCommit('chore: install dependencies'); - await app.postGenerate(); - await app.gitDiff(); - await app.gitCommit('chore: post generate'); - await app.preSandbox(); - const gen2StackName = await app.deployGen2Sandbox(); - await app.postSandbox(gen2StackName); - - await app.testGen1(); - await app.testGen2(); - - if (app.skipRefactor) { - app.logger.info('Skipping refactor (configured in migration/config.json)'); - return; - } - - await app.gitCheckoutGen1(); - await app.refactor(gen2StackName); - - await app.testGen1(); - await app.testGen2(); - - await app.gitCheckoutGen2(); - await app.postRefactor(); - await app.gitDiff(); - await app.gitCommit('chore: post refactor'); - - await app.deployGen2Sandbox(); - - await app.testGen1(); - await app.testGen2(); -} - function printBanner(): void { console.log( chalk.cyan(` diff --git a/packages/amplify-gen2-migration-e2e-system/src/core/app.ts b/packages/amplify-gen2-migration-e2e-system/src/core/app.ts index 17ff49e7097..5d0a67c6a93 100644 --- a/packages/amplify-gen2-migration-e2e-system/src/core/app.ts +++ b/packages/amplify-gen2-migration-e2e-system/src/core/app.ts @@ -5,8 +5,12 @@ import os from 'os'; import { getCLIPath, initJSProjectWithProfile } from '@aws-amplify/amplify-e2e-core'; import { Logger, LogLevel } from './logger'; import { Git } from './git'; +import * as snapshot from './snapshot'; +import { sanitize } from './sanitize'; +import { CloudFormationClient, paginateListStacks, StackStatus } from '@aws-sdk/client-cloudformation'; const MIGRATION_TARGET_DIR = path.join(os.tmpdir(), 'amplify-gen2-migration-e2e-system', 'output-apps'); +const MIGRATION_SNAPSHOT_DIR = path.join(os.tmpdir(), 'amplify-gen2-migration-e2e-system', 'snapshots'); const MIGRATION_APPS_DIR = path.join(__dirname, '..', '..', '..', '..', 'amplify-migration-apps'); interface MigrationConfig { @@ -42,10 +46,12 @@ interface RefactorConfig { export class App { private readonly deploymentName: string; private readonly gen2BranchName: string; + private readonly gen1BranchName = 'main'; private readonly sourceAppPath: string; private readonly envName: string; private readonly migrationConfig: MigrationConfig; + private readonly snapshotAppPath: string; /** * Whether the refactor step should be skipped entirely for this app. @@ -73,6 +79,12 @@ export class App { this.gen2BranchName = `gen2-${this.envName}`; this.amplifyPath = getCLIPath(true); + // temporary directory to store snapshot of each step + // callers can then call .updateSnapshot to copy over the snapshots + // into the original source path + this.snapshotAppPath = path.join(MIGRATION_SNAPSHOT_DIR, this.deploymentName); + fs.mkdirSync(this.snapshotAppPath, { recursive: true }); + // Copy source to temp directory this.targetAppPath = path.join(MIGRATION_TARGET_DIR, this.deploymentName); fs.mkdirSync(this.targetAppPath, { recursive: true }); @@ -80,6 +92,9 @@ export class App { filter: (src: string) => !src.includes('_snapshot') && !src.includes('node_modules'), }); + this.logger.info(`App directory: ${this.targetAppPath}`); + this.logger.info(`Snapshot directory: ${this.snapshotAppPath}`); + // Update package.json name for predictable Gen2 stack naming const packageJsonPath = path.join(this.targetAppPath, 'package.json'); const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8')) as { name: string }; @@ -89,7 +104,6 @@ export class App { this.migrationConfig = this.loadMigrationConfig(); this.git = new Git(this.targetAppPath, this.logger); - this.logger.info(`App ${appName} prepared at ${this.targetAppPath}`); this.logger.info(`Deployment name: ${this.deploymentName}, env: ${this.envName}`); } @@ -175,10 +189,84 @@ export class App { this.logger.info('amplify push completed'); } + /** + * Runs all steps to fully deploy the Gen1 app. + */ + public async deploy(): Promise { + await this.git.init(); + await this.init(); + await this.configure(); + await this.installDeps(); + await this.status(); + await this.prePush(); + await this.push(); + await this.postPush(); + await this.testGen1(); + + this.logger.info(`Capturing pre.generate snapshot`); + await snapshot.capturePreGenerate(this.targetAppPath, this.snapshotAppPath); + } + // ============================================================ // Gen2 Migration // ============================================================ + /** + * Runs the full migration workflow + */ + public async migrate(): Promise { + await this.deploy(); + await this.assess(); + await this.lock(); + await this.git.checkout(this.gen2BranchName, true); + await this.generate(); + + this.logger.info(`Capturing post.generate snapshot`); + await snapshot.capturePostGenerate(this.targetAppPath, this.snapshotAppPath); + + await this.git.commit('chore: generate'); + await this.installDeps(); + await this.git.commit('chore: install dependencies'); + await this.postGenerate(); + await this.git.diff(); + await this.git.commit('chore: post generate'); + await this.preSandbox(); + const gen2StackName = await this.deployGen2Sandbox(); + await this.postSandbox(gen2StackName); + + await this.testGen1(); + await this.testGen2(); + + if (this.skipRefactor) { + this.logger.info('Skipping refactor (configured in migration/config.json)'); + return; + } + + const gen1StackName = await this.findGen1RootStack(); + + this.logger.info(`Capturing pre.refactor snapshot`); + await snapshot.capturePreRefactor(gen1StackName, gen2StackName, this.snapshotAppPath); + + await this.git.checkout(this.gen1BranchName, false); + await this.refactor(gen2StackName); + + this.logger.info(`Capturing post.refactor snapshot`); + await snapshot.capturePostRefactor(this.targetAppPath, this.snapshotAppPath); + + await this.testGen1(); + await this.testGen2(); + + await this.git.checkout(this.gen2BranchName, false); + await this.postRefactor(); + await this.git.diff(); + await this.git.commit('chore: post refactor'); + + await this.deployGen2Sandbox(); + + await this.testGen1(); + await this.testGen2(); + } + /** * Run `amplify gen2-migration assess`. */ @@ -195,7 +283,7 @@ export class App { } /** - * Run `amplify gen2-migration generate` and install dependencies. + * Run `amplify gen2-migration generate`. */ public async generate(): Promise { await this.runMigrationStep('generate'); @@ -240,14 +328,14 @@ export class App { } // ============================================================ - // App Scripts + // App Tests // ============================================================ /** * Run the Jest tests against the Gen1 config. */ public async testGen1(): Promise { - await this.gitCheckoutGen1(); + await this.git.checkout(this.gen1BranchName, false); await this.runNpmScript('test:gen1'); } @@ -255,10 +343,14 @@ export class App { * Run the Jest tests against the Gen2 config. */ public async testGen2(): Promise { - await this.gitCheckoutGen2(); + await this.git.checkout(this.gen2BranchName, false); await this.runNpmScript('test:gen2'); } + // ============================================================ + // App Hooks + // ============================================================ + /** * Run the pre-push script. */ @@ -301,40 +393,20 @@ export class App { await this.runNpmScript('pre-sandbox'); } - // ============================================================ - // Git - // ============================================================ - - /** - * Initialize a git repo and create the initial commit. - */ - public async gitInit(): Promise { - await this.git.init(); - } - - /** - * Commit all changes. - */ - public async gitCommit(message: string): Promise { - await this.git.commit(message); - } - - public async gitDiff(): Promise { - await this.git.diff(); - } - /** - * Checkout the Gen1 (main) branch. + * Sanitizes and copies captured snapshots back to the source app directory. */ - public async gitCheckoutGen1(): Promise { - await this.git.checkout('main', false); - } - - /** - * Checkout the Gen2 branch (creates it if create is true). - */ - public async gitCheckoutGen2(create = false): Promise { - await this.git.checkout(this.gen2BranchName, create); + public updateSnapshots(): void { + this.logger.info(`Sanitizing snapshots`); + sanitize(this.deploymentName, this.snapshotAppPath); + for (const snapshot of fs.readdirSync(this.snapshotAppPath).filter((f) => f.includes('_snapshot'))) { + const sourceSnapshotPath = path.join(this.sourceAppPath, snapshot); + this.logger.info(`Updating snapshot: ${sourceSnapshotPath}`); + if (fs.existsSync(sourceSnapshotPath)) { + fs.removeSync(sourceSnapshotPath); + } + fs.copySync(path.join(this.snapshotAppPath, snapshot), sourceSnapshotPath); + } } // ============================================================ @@ -350,7 +422,6 @@ export class App { .filter((l) => l.trim() !== line) .join('\n'); fs.writeFileSync(gitignorePath, updated, 'utf-8'); - this.logger.info(`Removed '${line}' from .gitignore`); } private loadMigrationConfig(): MigrationConfig { @@ -410,6 +481,11 @@ export class App { } } + private async findGen1RootStack(): Promise { + const rootPattern = new RegExp(`^amplify-${this.deploymentName}-${this.envName}-[0-9a-f]{5}$`); + return findStackByPattern(rootPattern); + } + private async findGen2RootStack(stackPrefix: string): Promise { const result = await execa( 'aws', @@ -473,3 +549,15 @@ function generateRandomEnvName(): string { const length = Math.floor(Math.random() * 9) + 2; return Array.from({ length }, () => String.fromCharCode(97 + Math.floor(Math.random() * 26))).join(''); } + +async function findStackByPattern(pattern: RegExp): Promise { + const cfnClient = new CloudFormationClient({}); + for await (const page of paginateListStacks( + { client: cfnClient }, + { StackStatusFilter: [StackStatus.CREATE_COMPLETE, StackStatus.UPDATE_COMPLETE, StackStatus.UPDATE_ROLLBACK_COMPLETE] }, + )) { + const match = page.StackSummaries?.find((s) => s.StackName && pattern.test(s.StackName)); + if (match?.StackName) return match.StackName; + } + throw new Error(`No stack found matching pattern "${pattern.source}"`); +} diff --git a/packages/amplify-gen2-migration-e2e-system/src/core/sanitize.ts b/packages/amplify-gen2-migration-e2e-system/src/core/sanitize.ts new file mode 100644 index 00000000000..7e462404647 --- /dev/null +++ b/packages/amplify-gen2-migration-e2e-system/src/core/sanitize.ts @@ -0,0 +1,136 @@ +#!/usr/bin/env npx tsx +/* eslint-disable spellcheck/spell-checker */ +/* eslint-disable @typescript-eslint/no-unsafe-argument */ +/* eslint-disable @typescript-eslint/no-unsafe-return */ +/* eslint-disable @typescript-eslint/no-unsafe-call */ +/* eslint-disable @typescript-eslint/no-unsafe-member-access */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; + +interface SensitiveValues { + accountId: string; + amplifyAppId: string; + gen1ApiKey: string | null; + gen2ApiKey: string | null; +} + +function extractAccountId(meta: any): string { + const authRoleArn = meta.providers.awscloudformation.AuthRoleArn; + const arnMatch = authRoleArn.match(/arn:aws:iam::(\d{12}):/); + if (!arnMatch) { + throw new Error('Could not extract AWS Account ID from AuthRoleArn'); + } + return arnMatch[1]; +} + +function extractAmplifyAppId(meta: any): string { + const appId = meta.providers.awscloudformation.AmplifyAppId; + if (!appId) { + throw new Error('Could not extract Amplify App ID from amplify-meta.json'); + } + return appId; +} + +function extractGen1ApiKey(meta: any): string | null { + if (!meta.api) return null; + const firstApiResource = Object.keys(meta.api)[0]; + return meta.api[firstApiResource]?.output?.GraphQLAPIKeyOutput ?? null; +} + +function extractGen2ApiKey(appDir: string): string | null { + const preRefactor = path.join(appDir, '_snapshot.pre.refactor'); + for (const outputsFile of fs.readdirSync(preRefactor).filter((f) => f.endsWith('outputs.json'))) { + const outputs = JSON.parse(fs.readFileSync(path.join(preRefactor, outputsFile), { encoding: 'utf-8' })); + for (const output of outputs) { + if (output.OutputKey.includes('ApiKey')) { + return output.OutputValue; + } + } + } + return null; +} + +function extractSensitiveValues(meta: any, appDir: string): SensitiveValues { + return { + accountId: extractAccountId(meta), + amplifyAppId: extractAmplifyAppId(meta), + gen1ApiKey: extractGen1ApiKey(meta), + gen2ApiKey: extractGen2ApiKey(appDir), + }; +} + +function getFilesRecursive(dir: string): string[] { + const files: string[] = []; + const entries = fs.readdirSync(dir, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (fullPath.includes('node_modules')) { + continue; + } + if (entry.isDirectory()) { + files.push(...getFilesRecursive(fullPath)); + } else if (entry.isFile()) { + files.push(fullPath); + } + } + + return files; +} + +function sanitizeFileName(name: string, appId: string, appName: string): string { + // sandbox uses this + const username = os.userInfo().username; + return name.replaceAll(appId, appName).replaceAll(`-${username}-`, '-username-'); +} + +function getAllFiles(dir: string): string[] { + return getFilesRecursive(dir); +} + +/** + * Sanitizes sensitive values in Amplify migration app snapshot files for safe public commit. + * + * Strategy: + * 1. Extract sensitive values from _snapshot.pre.generate/amplify/backend/amplify-meta.json + * 2. Replace all occurrences of those values across all _snapshot.* directories + * 3. Rename files whose names contain the Amplify App ID + * + * Targets: + * - AWS Account ID (from providers.awscloudformation AuthRoleArn) → replaced with 123456789012 + * - Amplify App ID (from providers.awscloudformation) → replaced with app name (dashes removed) + * - AppSync API Key (from api output, if present) → replaced with da2-fakeapikey00000000000000 + */ +export function sanitize(appName: string, appDir: string): void { + const appNameNoDashes = appName.replaceAll('-', ''); + const metaPath = path.join(appDir, '_snapshot.pre.generate', 'amplify', 'backend', 'amplify-meta.json'); + const amplifyMeta: any = JSON.parse(fs.readFileSync(metaPath, 'utf-8')); + const values = extractSensitiveValues(amplifyMeta, appDir); + + const snapshots = fs.readdirSync(appDir).filter((f) => f.startsWith('_snapshot')); + const files = [...snapshots.flatMap((s) => getAllFiles(path.join(appDir, s)))]; + + for (const file of files) { + let content = fs.readFileSync(file, 'utf-8'); + + content = content.replaceAll(values.accountId, '123456789012'); + content = content.replaceAll(values.amplifyAppId, appNameNoDashes); + + if (values.gen1ApiKey) { + content = content.replaceAll(values.gen1ApiKey, 'da2-fakeapikey00000000000000'); + } + + if (values.gen2ApiKey) { + content = content.replaceAll(values.gen2ApiKey, 'da2-fakeapikey00000000000000'); + } + + const sanitizedFileName = sanitizeFileName(file, values.amplifyAppId, appNameNoDashes); + + fs.writeFileSync(file, content, 'utf-8'); + fs.renameSync(file, sanitizedFileName); + } +} diff --git a/packages/amplify-gen2-migration-e2e-system/src/core/snapshot.ts b/packages/amplify-gen2-migration-e2e-system/src/core/snapshot.ts new file mode 100644 index 00000000000..f84a3a9564e --- /dev/null +++ b/packages/amplify-gen2-migration-e2e-system/src/core/snapshot.ts @@ -0,0 +1,166 @@ +#!/usr/bin/env npx tsx + +import * as fs from 'fs-extra'; +import * as path from 'path'; +import { + CloudFormationClient, + DescribeStacksCommand, + GetTemplateCommand, + paginateListStackResources, +} from '@aws-sdk/client-cloudformation'; + +// --------------------------------------------------------------------------- +// Shared helpers +// --------------------------------------------------------------------------- + +function resetDir(dir: string): void { + if (fs.existsSync(dir)) { + fs.rmdirSync(dir, { recursive: true }); + } + fs.mkdirSync(dir, { recursive: true }); +} + +function writeFileSync(p: string, content: string): void { + console.log(p); + fs.writeFileSync(p, content, 'utf-8'); +} + +function copySync(src: string, dest: string): void { + if (fs.existsSync(dest)) { + fs.rmSync(dest, { recursive: true }); + } + console.log(dest); + fs.copySync(src, dest, { filter: (src) => !src.includes('node_modules') }); +} + +function copyRequired(srcBasePath: string, destBasePath: string, toCopy: readonly string[]): void { + for (const required of toCopy) { + const inputPath = path.join(srcBasePath, required); + if (!fs.existsSync(inputPath)) { + throw new Error(`Required input not found: ${inputPath}`); + } + const destPath = path.join(destBasePath, required); + copySync(inputPath, destPath); + } +} + +function copyOptional(srcBasePath: string, destBasePath: string, toCopy: readonly string[]): void { + for (const optional of toCopy) { + const inputPath = path.join(srcBasePath, optional); + const destPath = path.join(destBasePath, optional); + if (fs.existsSync(inputPath)) { + copySync(inputPath, destPath); + } + } +} + +// --------------------------------------------------------------------------- +// CloudFormation helpers (refactor.input only) +// --------------------------------------------------------------------------- + +const cfnClient = new CloudFormationClient({}); + +async function fetchTemplate(stackName: string): Promise { + const response = await cfnClient.send(new GetTemplateCommand({ StackName: stackName, TemplateStage: 'Original' })); + return response.TemplateBody!; +} + +async function fetchNestedStacks(stackName: string): Promise { + const ids: string[] = []; + for await (const page of paginateListStackResources({ client: cfnClient }, { StackName: stackName })) { + for (const r of page.StackResourceSummaries ?? []) { + if (r.ResourceType === 'AWS::CloudFormation::Stack' && r.PhysicalResourceId) { + ids.push(r.PhysicalResourceId); + } + } + } + return ids; +} + +function stackNameFromArn(arnOrName: string): string { + if (arnOrName.startsWith('arn:')) { + const parts = arnOrName.split('/'); + return parts[1] ?? arnOrName; + } + return arnOrName; +} + +async function downloadRecursive(stackNameOrArn: string, targetDir: string): Promise { + const stackName = stackNameFromArn(stackNameOrArn); + + const template = await fetchTemplate(stackName); + writeFileSync(path.join(targetDir, `${stackName}.template.json`), JSON.stringify(JSON.parse(template), null, 2)); + + const stackResponse = await cfnClient.send(new DescribeStacksCommand({ StackName: stackName })); + const stack = stackResponse.Stacks![0]; + + const outputs = stack.Outputs ?? []; + writeFileSync(path.join(targetDir, `${stackName}.outputs.json`), JSON.stringify(outputs, null, 2)); + + const parameters = stack.Parameters ?? []; + writeFileSync(path.join(targetDir, `${stackName}.parameters.json`), JSON.stringify(parameters, null, 2)); + + const description = stack.Description ?? ''; + writeFileSync(path.join(targetDir, `${stackName}.description.txt`), description); + + const nestedIds = await fetchNestedStacks(stackName); + for (const nestedId of nestedIds) { + await downloadRecursive(nestedId, targetDir); + } +} + +// --------------------------------------------------------------------------- +// Snapshot capture functions +// --------------------------------------------------------------------------- + +/** + * Downloads Gen1 and Gen2 CloudFormation templates into `_snapshot.pre.refactor/`. + */ +export async function capturePreRefactor(gen1RootStackName: string, gen2RootStackName: string, targetDir: string): Promise { + const destPath = path.join(targetDir, '_snapshot.pre.refactor'); + resetDir(destPath); + + await downloadRecursive(gen2RootStackName, destPath); + await downloadRecursive(gen1RootStackName, destPath); +} + +/** + * Copies refactor operations into `_snapshot.post.refactor/`. + */ +export async function capturePostRefactor(deployedAppPath: string, dstBasePath: string): Promise { + const srcBasePath = path.join(deployedAppPath, '.gen2-migration/refactor.operations'); + const destPath = path.join(dstBasePath, '_snapshot.post.refactor'); + resetDir(destPath); + copySync(srcBasePath, destPath); +} + +/** + * Copies the Gen1 app state into `_snapshot.pre.generate/`. + */ +export async function capturePreGenerate(deployedAppPath: string, dstBasePath: string): Promise { + const destPath = path.join(dstBasePath, '_snapshot.pre.generate'); + resetDir(destPath); + + copyRequired(deployedAppPath, destPath, ['amplify', '.gitignore']); + copyOptional(deployedAppPath, destPath, ['package.json']); + + // For the snapshot we want to include all files + const gitIgnorePath = path.join(destPath, '.gitignore'); + const gitIgnore = fs.readFileSync(gitIgnorePath, { encoding: 'utf-8' }); + const newGitIgnore = gitIgnore + .replaceAll('amplify/', '!amplify/') + .replaceAll('build/', '!build/') + .replaceAll('!amplify/.config/local-*', 'amplify/.config/local-*'); + fs.writeFileSync(gitIgnorePath, newGitIgnore); +} + +/** + * Copies the Gen2 output into `_snapshot.post.generate/`. + */ +export async function capturePostGenerate(deployedAppPath: string, dstBasePath: string): Promise { + const destPath = path.join(dstBasePath, '_snapshot.post.generate'); + resetDir(destPath); + + copyRequired(deployedAppPath, destPath, ['amplify', '.gitignore', 'amplify.yml']); + copyOptional(deployedAppPath, destPath, ['package.json']); +} diff --git a/yarn.lock b/yarn.lock index 5f46c0bfc27..8e3356b7abe 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1210,6 +1210,8 @@ __metadata: resolution: "@aws-amplify/amplify-gen2-migration-e2e-system@workspace:packages/amplify-gen2-migration-e2e-system" dependencies: "@aws-amplify/amplify-e2e-core": "workspace:^" + "@aws-sdk/client-amplify": ^3.919.0 + "@aws-sdk/client-cloudformation": ^3.919.0 "@cdklabs/cdk-atmosphere-client": latest "@paralleldrive/cuid2": ^3.0.6 "@types/fs-extra": ^11.0.4