Scale Test Infrastructure #75
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scale Test Infrastructure | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| action: | |
| description: 'Action to perform' | |
| required: true | |
| type: choice | |
| options: | |
| - plan | |
| - apply | |
| - destroy | |
| scale_multiplier: | |
| description: 'Resource multiplier (1=175, 10=1740, 50=8700 resources)' | |
| required: true | |
| type: choice | |
| default: '1' | |
| options: | |
| - '1' | |
| - '5' | |
| - '10' | |
| - '25' | |
| - '50' | |
| scenario: | |
| description: 'Test scenario (triggers specific risks in Overmind)' | |
| required: false | |
| type: choice | |
| default: 'none' | |
| options: | |
| - 'none' | |
| # AWS scenarios | |
| - 'lambda_timeout' | |
| - 'shared_sg_open' | |
| - 'vpc_peering_change' | |
| - 'central_sns_change' | |
| - 'combined_network' | |
| - 'combined_all' | |
| - 'combined_max' | |
| # GCP scenarios (requires cloud_provider=gcp or both) | |
| - 'shared_firewall_open' | |
| - 'central_pubsub_change' | |
| - 'gce_downgrade' | |
| - 'function_timeout' | |
| - 'combined_gcp_all' | |
| cloud_provider: | |
| description: 'Cloud provider to deploy' | |
| required: true | |
| type: choice | |
| default: 'aws' | |
| options: | |
| - 'aws' | |
| - 'gcp' | |
| - 'both' | |
| confirmation: | |
| description: 'For destroy: type DESTROY-SCALE-TEST to confirm' | |
| required: false | |
| type: string | |
| env: | |
| TF_VAR_scale_multiplier: ${{ inputs.scale_multiplier }} | |
| TF_VAR_scenario: ${{ inputs.scenario }} | |
| TF_VAR_cloud_provider: ${{ inputs.cloud_provider }} | |
| WORKING_DIR: scale-test | |
| jobs: | |
| validate: | |
| name: Validate Inputs | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Validate destroy confirmation | |
| if: ${{ inputs.action == 'destroy' }} | |
| run: | | |
| if [ "${{ inputs.confirmation }}" != "DESTROY-SCALE-TEST" ]; then | |
| echo "::error::Destroy requires confirmation. Type 'DESTROY-SCALE-TEST' in the confirmation field." | |
| exit 1 | |
| fi | |
| echo "Destroy confirmation validated" | |
| terraform: | |
| name: Terraform ${{ inputs.action }} (×${{ inputs.scale_multiplier }}, scenario=${{ inputs.scenario }}) | |
| needs: validate | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| id-token: write | |
| pull-requests: write | |
| concurrency: | |
| group: scale-test-tfstate | |
| cancel-in-progress: false | |
| defaults: | |
| run: | |
| working-directory: ${{ env.WORKING_DIR }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Terraform | |
| uses: hashicorp/setup-terraform@v3 | |
| with: | |
| terraform_wrapper: false | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| audience: sts.amazonaws.com | |
| aws-region: us-east-1 | |
| role-to-assume: ${{ vars.TERRAFORM_DEPLOY_ROLE }} | |
| # Always authenticate with GCP - Terraform validates all providers during init | |
| # even when cloud_provider=aws (no GCP resources created) | |
| - name: Configure GCP Credentials | |
| uses: google-github-actions/auth@v2 | |
| with: | |
| credentials_json: ${{ secrets.OVERMIND_SCALE_TEST }} | |
| - name: Setup GCP SDK | |
| uses: google-github-actions/setup-gcloud@v2 | |
| with: | |
| project_id: overmind-scale-test | |
| - name: Terraform Init | |
| id: init | |
| run: | | |
| terraform version | |
| terraform init -input=false | |
| - name: Terraform Plan | |
| id: plan | |
| if: ${{ inputs.action == 'plan' || inputs.action == 'apply' }} | |
| run: | | |
| set -o pipefail | |
| echo "Planning with scale_multiplier=${{ inputs.scale_multiplier }}, scenario=${{ inputs.scenario }}" | |
| terraform plan \ | |
| -compact-warnings \ | |
| -no-color \ | |
| -input=false \ | |
| -lock-timeout=5m \ | |
| -out=tfplan 2>&1 | tee terraform_plan.log | |
| # Generate JSON plan for Overmind | |
| terraform show -json tfplan > tfplan.json | |
| # Output summary | |
| echo "## Terraform Plan Summary" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Scale Multiplier:** ${{ inputs.scale_multiplier }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Scenario:** ${{ inputs.scenario }}" >> $GITHUB_STEP_SUMMARY | |
| echo "- **Action:** ${{ inputs.action }}" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "### Resource Changes" >> $GITHUB_STEP_SUMMARY | |
| terraform show -no-color tfplan | grep -E "^(Plan:|No changes)" >> $GITHUB_STEP_SUMMARY || true | |
| - name: Install Overmind CLI | |
| uses: overmindtech/actions/install-cli@main | |
| continue-on-error: true | |
| with: | |
| version: latest | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Submit Plan to Overmind | |
| uses: overmindtech/actions/submit-plan@main | |
| continue-on-error: true | |
| id: submit-plan | |
| if: ${{ inputs.action == 'plan' || inputs.action == 'apply' }} | |
| with: | |
| ovm-api-key: ${{ secrets.OVM_API_KEY }} | |
| plan-json: ${{ env.WORKING_DIR }}/tfplan.json | |
| tags: 'model=risks_v6' | |
| # Cost Analysis disabled for scale testing (plan too large) | |
| # - name: Cost Analysis | |
| # uses: overmindtech/cost-signals-action@v1 | |
| # continue-on-error: true | |
| # if: ${{ inputs.action == 'plan' || inputs.action == 'apply' }} | |
| # with: | |
| # overmind-api-key: ${{ secrets.OVM_API_KEY }} | |
| # infracost-api-key: ${{ secrets.INFRACOST_API_KEY }} | |
| # terraform-plan-json: ${{ env.WORKING_DIR }}/tfplan.json | |
| # ticket-link: ${{ steps.submit-plan.outputs.change-url }} | |
| - name: Start Overmind Change | |
| uses: overmindtech/actions/start-change@main | |
| continue-on-error: true | |
| if: ${{ inputs.action == 'apply' }} | |
| with: | |
| ovm-api-key: ${{ secrets.OVM_API_KEY }} | |
| - name: Terraform Apply | |
| id: apply | |
| if: ${{ inputs.action == 'apply' }} | |
| run: | | |
| echo "Applying scale test infrastructure (×${{ inputs.scale_multiplier }})" | |
| terraform apply \ | |
| -auto-approve \ | |
| -no-color \ | |
| -input=false \ | |
| -lock-timeout=5m \ | |
| tfplan | |
| echo "## Apply Complete" >> $GITHUB_STEP_SUMMARY | |
| echo "Scale test infrastructure deployed with multiplier ×${{ inputs.scale_multiplier }}" >> $GITHUB_STEP_SUMMARY | |
| - name: End Overmind Change | |
| uses: overmindtech/actions/end-change@main | |
| continue-on-error: true | |
| if: ${{ (inputs.action == 'apply') && (success() || failure() || cancelled()) }} | |
| with: | |
| ovm-api-key: ${{ secrets.OVM_API_KEY }} | |
| - name: Terraform Destroy | |
| id: destroy | |
| if: ${{ inputs.action == 'destroy' }} | |
| run: | | |
| echo "::warning::Destroying scale test infrastructure (×${{ inputs.scale_multiplier }})" | |
| terraform destroy \ | |
| -auto-approve \ | |
| -no-color \ | |
| -input=false \ | |
| -lock-timeout=10m | |
| echo "## Destroy Complete" >> $GITHUB_STEP_SUMMARY | |
| echo "Scale test infrastructure destroyed" >> $GITHUB_STEP_SUMMARY | |
| - name: Output Terraform Summary | |
| if: always() | |
| run: | | |
| echo "" | |
| echo "=== Scale Test Summary ===" | |
| echo "Action: ${{ inputs.action }}" | |
| echo "Multiplier: ${{ inputs.scale_multiplier }}" | |
| echo "Scenario: ${{ inputs.scenario }}" | |
| echo "Status: ${{ job.status }}" | |