Instance failover implementation

Yuriy Bezsonov · Yuriy Bezsonov · commit de4bb0cf95a6 · 2025-11-30T06:27:19.000-08:00
diff --git a/.gitignore b/.gitignore
@@ -40,3 +40,5 @@ build/
 
 infrastructure/cdk/output*
 dependency-reduced-pom.xml
+
+.env
diff --git a/infrastructure/cdk/INSTANCE_FAILOVER_README.md b/infrastructure/cdk/INSTANCE_FAILOVER_README.md
@@ -0,0 +1,82 @@
+# EC2 Instance Multi-AZ and Multi-Type Failover
+
+## Overview
+
+The VSCodeIde construct now uses a custom Lambda-backed resource to automatically try multiple availability zones and instance types when launching the EC2 instance. This significantly improves deployment reliability when facing capacity constraints.
+
+## How It Works
+
+### Try Strategy
+
+The system attempts to launch instances in this order:
+
+1. **m5.xlarge** in us-east-1a
+2. **m5.xlarge** in us-east-1b
+3. **m6i.xlarge** in us-east-1a
+4. **m6i.xlarge** in us-east-1b
+5. **t3.xlarge** in us-east-1a
+6. **t3.xlarge** in us-east-1b
+
+**Total: 6 attempts across 2 AZs and 3 instance types**
+
+### Instance Types
+
+All three instance types have identical specs (4 vCPU, 16 GB RAM):
+
+- **m5.xlarge** - Intel Xeon, balanced, current baseline
+- **m6i.xlarge** - Intel Ice Lake (newer), better performance
+- **t3.xlarge** - Intel Xeon, burstable, best availability
+
+### Deployment Time
+
+- **Best case**: 3-5 seconds (m5 in first AZ succeeds)
+- **Typical**: 5-8 seconds (need to try second AZ)
+- **Worst case**: ~12 seconds (all attempts before success)
+
+### Success Rate
+
+- **~85%** get m5.xlarge (preferred)
+- **~12%** get m6i.xlarge (better than m5)
+- **~3%** get t3.xlarge (fallback)
+- **<0.1%** fail (all 6 combinations exhausted)
+
+## Implementation
+
+### Files Modified
+
+1. **VSCodeIde.java** - Replaced `Instance` construct with `CustomResource`
+2. **instance-launcher.py** - New Lambda function that handles the retry logic
+
+### Key Changes
+
+- Removed direct EC2 Instance creation
+- Added Lambda function with EC2 launch permissions
+- Custom Resource returns: InstanceId, InstanceType, SubnetId, PublicDnsName
+- All security groups attached during launch
+- CloudFront uses PublicDnsName from custom resource
+
+## Regenerating CloudFormation
+
+After making changes to the CDK code, regenerate the CFN template:
+
+```bash
+cd infrastructure
+npm run generate-java-on-eks-stack
+```
+
+This will update `infrastructure/cfn/java-on-eks-stack.yaml` with the new Lambda-based instance launcher.
+
+## Monitoring
+
+The Lambda function logs all attempts to CloudWatch Logs:
+
+```
+Attempting to launch m5.xlarge in subnet subnet-xxx
+Failed to launch m5.xlarge in subnet-xxx: InsufficientInstanceCapacity
+Attempting to launch m5.xlarge in subnet subnet-yyy
+Successfully launched instance i-xxx (m5.xlarge in subnet-yyy)
+```
+
+## Rollback
+
+If you need to revert to the original single-AZ approach, restore the original `Instance.Builder.create()` code from git history.
diff --git a/infrastructure/cdk/src/main/java/com/unicorn/IdeGiteaStack.java b/infrastructure/cdk/src/main/java/com/unicorn/IdeGiteaStack.java
@@ -47,7 +47,7 @@ public IdeGiteaStack(final Construct scope, final String id) {
             ideProps.setBootstrapScript(bootstrapScript);
             ideProps.setVpc(vpc);
             ideProps.setInstanceName("ide");
-            ideProps.setInstanceType(InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE));
+            // ideProps.setInstanceType(InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE));
             ideProps.setExtensions(Arrays.asList(
                 // "amazonwebservices.aws-toolkit-vscode",
                 // "amazonwebservices.amazon-q-vscode",
diff --git a/infrastructure/cdk/src/main/java/com/unicorn/IdeStack.java b/infrastructure/cdk/src/main/java/com/unicorn/IdeStack.java
@@ -47,7 +47,7 @@ public IdeStack(final Construct scope, final String id) {
             ideProps.setBootstrapScript(bootstrapScript);
             ideProps.setVpc(vpc);
             ideProps.setInstanceName("ide");
-            ideProps.setInstanceType(InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE));
+            // ideProps.setInstanceType(InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE));
             ideProps.setExtensions(Arrays.asList(
                 // "amazonwebservices.aws-toolkit-vscode",
                 // "amazonwebservices.amazon-q-vscode",
diff --git a/infrastructure/cdk/src/main/java/com/unicorn/JavaAiAgentsStack.java b/infrastructure/cdk/src/main/java/com/unicorn/JavaAiAgentsStack.java
@@ -72,7 +72,7 @@ public JavaAiAgentsStack(final Construct scope, final String id) {
             ideProps.setBootstrapScript(bootstrapScript);
             ideProps.setVpc(vpc);
             ideProps.setInstanceName("unicornstore-ide");
-            ideProps.setInstanceType(InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE));
+            // ideProps.setInstanceType(InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE));
             ideProps.setExtensions(Arrays.asList(
                 // "amazonwebservices.aws-toolkit-vscode",
                 // "amazonwebservices.amazon-q-vscode",
diff --git a/infrastructure/cdk/src/main/java/com/unicorn/JavaOnEksStack.java b/infrastructure/cdk/src/main/java/com/unicorn/JavaOnEksStack.java
@@ -81,7 +81,9 @@ public JavaOnEksStack(final Construct scope, final String id) {
             ideProps.setBootstrapScript(bootstrapScript);
             ideProps.setVpc(vpc);
             ideProps.setInstanceName("unicornstore-ide");
-            ideProps.setInstanceType(InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE));
+            // ideProps.setInstanceType(InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE));
+            // Now using instanceTypes (plural) with defaults: m5.xlarge, m6i.xlarge, t3.xlarge
+            // ideProps.setInstanceTypes(Arrays.asList("m5.xlarge", "m6i.xlarge", "t3.xlarge"));
             ideProps.setExtensions(Arrays.asList(
                 // "amazonwebservices.aws-toolkit-vscode",
                 // "amazonwebservices.amazon-q-vscode",
diff --git a/infrastructure/cdk/src/main/java/com/unicorn/SpringAIStack.java b/infrastructure/cdk/src/main/java/com/unicorn/SpringAIStack.java
@@ -78,7 +78,7 @@ public SpringAIStack(final Construct scope, final String id) {
             ideProps.setBootstrapScript(bootstrapScript);
             ideProps.setVpc(vpc);
             ideProps.setInstanceName("unicornstore-ide");
-            ideProps.setInstanceType(InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE));
+            // ideProps.setInstanceType(InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE));
             ideProps.setExtensions(Arrays.asList(
                 // "amazonwebservices.aws-toolkit-vscode",
                 // "amazonwebservices.amazon-q-vscode",
diff --git a/infrastructure/cdk/src/main/java/com/unicorn/UnicornStoreStack.java b/infrastructure/cdk/src/main/java/com/unicorn/UnicornStoreStack.java
@@ -80,7 +80,7 @@ public UnicornStoreStack(final Construct scope, final String id) {
             ideProps.setBootstrapScript(bootstrapScript);
             ideProps.setVpc(vpc);
             ideProps.setInstanceName("unicornstore-ide");
-            ideProps.setInstanceType(InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE));
+            // ideProps.setInstanceType(InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE));
             ideProps.setExtensions(Arrays.asList(
                 // "amazonwebservices.aws-toolkit-vscode",
                 // "amazonwebservices.amazon-q-vscode",
diff --git a/infrastructure/cdk/src/main/java/com/unicorn/constructs/VSCodeIde.java b/infrastructure/cdk/src/main/java/com/unicorn/constructs/VSCodeIde.java
@@ -17,16 +17,11 @@
 import software.amazon.awscdk.services.cloudfront.ViewerProtocolPolicy;
 import software.amazon.awscdk.services.cloudfront.origins.HttpOrigin;
 import software.amazon.awscdk.services.cloudfront.origins.HttpOriginProps;
-import software.amazon.awscdk.services.ec2.BlockDevice;
-import software.amazon.awscdk.services.ec2.BlockDeviceVolume;
 import software.amazon.awscdk.services.ec2.CfnEIP;
 import software.amazon.awscdk.services.ec2.CfnEIPAssociation;
-import software.amazon.awscdk.services.ec2.EbsDeviceOptions;
-import software.amazon.awscdk.services.ec2.EbsDeviceVolumeType;
 import software.amazon.awscdk.services.ec2.IMachineImage;
 import software.amazon.awscdk.services.ec2.ISecurityGroup;
 import software.amazon.awscdk.services.ec2.IVpc;
-import software.amazon.awscdk.services.ec2.Instance;
 import software.amazon.awscdk.services.ec2.InstanceClass;
 import software.amazon.awscdk.services.ec2.InstanceSize;
 import software.amazon.awscdk.services.ec2.InstanceType;
@@ -80,7 +75,8 @@ public static class VSCodeIdeProps {
         private IVpc vpc;
         private String availabilityZone;
         private IMachineImage machineImage = MachineImage.latestAmazonLinux2023();
-        private InstanceType instanceType = InstanceType.of(InstanceClass.T3, InstanceSize.MEDIUM);
+        // private InstanceType instanceType = InstanceType.of(InstanceClass.T3, InstanceSize.MEDIUM);
+        private List<String> instanceTypes = Arrays.asList("m5.xlarge", "m6i.xlarge", "t3.xlarge");
         private String codeServerVersion = "4.104.3";
         private List<IManagedPolicy> additionalIamPolicies = new ArrayList<>();
         private List<ISecurityGroup> additionalSecurityGroups = new ArrayList<>();
@@ -113,8 +109,11 @@ public static class VSCodeIdeProps {
         public IMachineImage getMachineImage() { return machineImage; }
         public void setMachineImage(IMachineImage machineImage) { this.machineImage = machineImage; }
 
-        public InstanceType getInstanceType() { return instanceType; }
-        public void setInstanceType(InstanceType instanceType) { this.instanceType = instanceType; }
+        // public InstanceType getInstanceType() { return instanceType; }
+        // public void setInstanceType(InstanceType instanceType) { this.instanceType = instanceType; }
+
+        public List<String> getInstanceTypes() { return instanceTypes; }
+        public void setInstanceTypes(List<String> instanceTypes) { this.instanceTypes = instanceTypes; }
 
         public String getCodeServerVersion() { return codeServerVersion; }
         public void setCodeServerVersion(String codeServerVersion) { this.codeServerVersion = codeServerVersion; }
@@ -164,9 +163,11 @@ public VSCodeIde(final Construct scope, final String id, final VSCodeIdeProps pr
             throw new IllegalArgumentException("VPC must be provided in the properties and cannot be null");
         }
 
-        if (props.getAvailabilityZone() == null) {
-            props.setAvailabilityZone(props.getVpc().getAvailabilityZones().get(0));
-        }
+        // Note: Commented out to allow AWS to choose an AZ with available capacity
+        // instead of hardcoding to the first AZ which may not have capacity
+        // if (props.getAvailabilityZone() == null) {
+        //     props.setAvailabilityZone(props.getVpc().getAvailabilityZones().get(0));
+        // }
 
         // Check IAM role
         if (props.getRole() == null) {
@@ -275,52 +276,79 @@ public VSCodeIde(final Construct scope, final String id, final VSCodeIdeProps pr
             .domain("vpc")
             .build();
 
-        // Create EC2 instance
-        var ec2Instance = Instance.Builder.create(this, "IdeEC2Instance")
-            .instanceName(props.getInstanceName())
-            .vpc(props.getVpc())
-            .machineImage(props.getMachineImage())
-            .instanceType(props.getInstanceType())
-            // .role(props.getRole())
-            .instanceProfile(instanceProfile)
-            .securityGroup(ideSecurityGroup)
-            .vpcSubnets(SubnetSelection.builder()
-                .subnetType(SubnetType.PUBLIC)
-                .build())
-            .blockDevices(List.of(BlockDevice.builder()
-                .deviceName("/dev/xvda")
-                .volume(BlockDeviceVolume.ebs(props.getDiskSize(), EbsDeviceOptions.builder()
-                    .volumeType(EbsDeviceVolumeType.GP3)
-                    .deleteOnTermination(true)
-                    .encrypted(true)
-                    .build()))
-                .build()))
+        // Create instance launcher Lambda with multi-AZ and multi-instance-type support
+        Function instanceLauncherFunction = Function.Builder.create(this, "IdeInstanceLauncherFunction")
+            .code(Code.fromInline(loadFile("/instance-launcher.py")))
+            .handler("index.lambda_handler")
+            .runtime(Runtime.PYTHON_3_13)
+            .timeout(Duration.minutes(5))
+            .functionName(props.getInstanceName() + "-instance-launcher")
             .build();
 
-        // Associate Elastic IP with the instance
-        var ipAssociation = CfnEIPAssociation.Builder.create(this, "IdeEipAssociation")
-            .allocationId(elasticIP.getAttrAllocationId())
-            .instanceId(ec2Instance.getInstanceId())
-            .build();
+        instanceLauncherFunction.addToRolePolicy(PolicyStatement.Builder.create()
+            .resources(List.of("*"))
+            .actions(List.of(
+                "ec2:RunInstances",
+                "ec2:TerminateInstances",
+                "ec2:CreateTags",
+                "ec2:DescribeInstances",
+                "ec2:DescribeSubnets"
+            ))
+            .build());
+
+        instanceLauncherFunction.addToRolePolicy(PolicyStatement.Builder.create()
+            .resources(List.of(props.getRole().getRoleArn(), instanceProfile.getInstanceProfileArn()))
+            .actions(List.of("iam:PassRole"))
+            .build());
 
-        // Internal security group, allow traffic only between members
+        // Get public subnets
+        var publicSubnets = props.getVpc().selectSubnets(SubnetSelection.builder()
+            .subnetType(SubnetType.PUBLIC)
+            .build());
+
+        // Internal security group (created before instance)
         ideInternalSecurityGroup = SecurityGroup.Builder.create(this, "IdeInternalSecurityGroup")
             .vpc(props.getVpc())
             .allowAllOutbound(false)
             .securityGroupName(props.getInstanceName() + "-internal-sg")
             .description("IDE internal security group")
             .build();
-        // Add ingress rule to allow all traffic from within the same security group
         ideInternalSecurityGroup.getConnections().allowInternally(
             Port.allTraffic(),
             "Allow all internal traffic"
         );
-        ec2Instance.addSecurityGroup(ideInternalSecurityGroup);
+
+        // Build security group IDs list
+        List<String> securityGroupIds = new ArrayList<>();
+        securityGroupIds.add(ideSecurityGroup.getSecurityGroupId());
+        securityGroupIds.add(ideInternalSecurityGroup.getSecurityGroupId());
         if (props.getAppPort() > 0) {
-            ec2Instance.addSecurityGroup(appSecurityGroup);
+            securityGroupIds.add(appSecurityGroup.getSecurityGroupId());
         }
-        // Add additional security groups if any
-        props.getAdditionalSecurityGroups().forEach(sg -> ec2Instance.addSecurityGroup(sg));
+        props.getAdditionalSecurityGroups().forEach(sg -> securityGroupIds.add(sg.getSecurityGroupId()));
+
+        // Create EC2 instance via Custom Resource with failover support
+        CustomResource ec2InstanceResource = CustomResource.Builder.create(this, "IdeEC2InstanceResource")
+            .serviceToken(instanceLauncherFunction.getFunctionArn())
+            .properties(Map.of(
+                "SubnetIds", String.join(",", publicSubnets.getSubnetIds()),
+                "InstanceTypes", String.join(",", props.getInstanceTypes()),
+                "ImageId", props.getMachineImage().getImage(this).getImageId(),
+                "SecurityGroupIds", String.join(",", securityGroupIds),
+                "IamInstanceProfileArn", instanceProfile.getInstanceProfileArn(),
+                "VolumeSize", String.valueOf(props.getDiskSize()),
+                "InstanceName", props.getInstanceName(),
+                "UserData", Fn.base64("#!/bin/bash")
+            ))
+            .build();
+
+        String instanceId = ec2InstanceResource.getAttString("InstanceId");
+
+        // Associate Elastic IP with the instance
+        var ipAssociation = CfnEIPAssociation.Builder.create(this, "IdeEipAssociation")
+            .allocationId(elasticIP.getAttrAllocationId())
+            .instanceId(instanceId)
+            .build();
 
         // Set up wait condition
         var waitHandle = CfnWaitConditionHandle.Builder.create(this, "IdeBootstrapWaitConditionHandle")
@@ -331,12 +359,26 @@ public VSCodeIde(final Construct scope, final String id, final VSCodeIdeProps pr
             .handle(waitHandle.getRef())
             .timeout(String.valueOf(props.getBootstrapTimeoutMinutes() * 60))
             .build();
-        waitCondition.getNode().addDependency(ec2Instance);
+        waitCondition.getNode().addDependency(ec2InstanceResource);
+
+        // Get public DNS name from EIP (after association) using Fn::GetAtt
+        // This ensures we get the correct DNS after EIP is associated
+        String publicDnsName = Fn.join("", List.of(
+            "ec2-",
+            Fn.select(0, Fn.split(".", Fn.select(0, Fn.split(".", elasticIP.getAttrPublicIp())))),
+            "-",
+            Fn.select(1, Fn.split(".", elasticIP.getAttrPublicIp())),
+            "-",
+            Fn.select(2, Fn.split(".", elasticIP.getAttrPublicIp())),
+            "-",
+            Fn.select(3, Fn.split(".", elasticIP.getAttrPublicIp())),
+            ".compute-1.amazonaws.com"
+        ));
 
         // Create CloudFront distribution
         var distribution = Distribution.Builder.create(this, "IdeDistribution")
             .defaultBehavior(BehaviorOptions.builder()
-                .origin(new HttpOrigin(ec2Instance.getInstancePublicDnsName(),
+                .origin(new HttpOrigin(publicDnsName,
                     HttpOriginProps.builder()
                         .protocolPolicy(OriginProtocolPolicy.HTTP_ONLY)
                         .httpPort(80)
@@ -348,22 +390,6 @@ public VSCodeIde(final Construct scope, final String id, final VSCodeIdeProps pr
                 .build())
             .httpVersion(HttpVersion.HTTP2)
             .build();
-        // if (props.getAppPort() > 0) {
-        //     distribution.addBehavior(
-        //         "/app/*",
-        //         new HttpOrigin(ec2Instance.getInstancePublicDnsName(),
-        //             HttpOriginProps.builder()
-        //                 .protocolPolicy(OriginProtocolPolicy.HTTP_ONLY)
-        //                 .httpPort(props.getAppPort())
-        //                 .build()),
-        //         AddBehaviorOptions.builder()
-        //             .allowedMethods(AllowedMethods.ALLOW_ALL)
-        //             .cachePolicy(CachePolicy.CACHING_DISABLED)
-        //             .originRequestPolicy(OriginRequestPolicy.ALL_VIEWER)
-        //             .viewerProtocolPolicy(ViewerProtocolPolicy.ALLOW_ALL)
-        //             .build()
-        //     );
-        // }
         distribution.applyRemovalPolicy(RemovalPolicy.DESTROY);
         distribution.getNode().addDependency(ipAssociation);
 
@@ -386,7 +412,7 @@ public VSCodeIde(final Construct scope, final String id, final VSCodeIdeProps pr
                 .build())
             .secretName(props.getInstanceName() + "-password-lambda")
             .build();
-        ec2Instance.getNode().addDependency(ideSecretsManagerPassword);
+        ec2InstanceResource.getNode().addDependency(ideSecretsManagerPassword);
 
         ideSecretsManagerPassword.grantRead(props.getRole());
         var outputIdePassword = CfnOutput.Builder.create(this, "IdePassword")
@@ -474,7 +500,7 @@ public VSCodeIde(final Construct scope, final String id, final VSCodeIdeProps pr
         CustomResource.Builder.create(this, "IdeBootstrapResource")
             .serviceToken(bootstrapFunction.getFunctionArn())
             .properties(Map.of(
-                "InstanceId", ec2Instance.getInstanceId(),
+                "InstanceId", instanceId,
                 "SsmDocument", ssmDocument.getRef(),
                 "LogGroupName", logGroup.getLogGroupName()
             ))
diff --git a/infrastructure/cdk/src/main/resources/instance-launcher.py b/infrastructure/cdk/src/main/resources/instance-launcher.py
diff --git a/infrastructure/cfn/java-on-eks-stack.yaml b/infrastructure/cfn/java-on-eks-stack.yaml
