WinterYukky · remote-swe · Apr 25, 2026 · Apr 25, 2026 · Apr 25, 2026
diff --git a/src/hyper-pod/hyper-pod-vllm-nxd-inference-service.test.ts b/src/hyper-pod/hyper-pod-vllm-nxd-inference-service.test.ts
@@ -164,4 +164,51 @@ describe("HyperPodVllmNxdInferenceService", () => {
       }),
     });
   });
+
+  it("configures destroy-time tolerations on the addon's ALB and KEDA components", () => {
+    new HyperPodVllmNxdInferenceService(stack, "InferenceService", {
+      cluster,
+      compiledModel,
+    });
+    const template = Template.fromStack(stack);
+    // The addon's `configurationValues` is serialized into a tokenized
+    // `Fn::Join` in the CloudFormation template, so we inspect the
+    // concrete resource JSON and assert the substrings we need are
+    // present in one of the join parts.
+    const addons = template.findResources("AWS::EKS::Addon", {
+      Properties: { AddonName: "amazon-sagemaker-hyperpod-inference" },
+    });
+    expect(Object.keys(addons)).toHaveLength(1);
+    const [addon] = Object.values(addons);
+    const join = (addon as any).Properties.ConfigurationValues["Fn::Join"];
+    expect(join).toBeDefined();
+    const joinedLiteral = join[1]
+      .filter((part: unknown): part is string => typeof part === "string")
+      .join("");
+    // Both the ALB and KEDA sections get the same taint tolerations.
+    expect(joinedLiteral).toMatch(
+      /"alb":[\s\S]*"tolerations":\[[\s\S]*"node\.kubernetes\.io\/unreachable"[\s\S]*"node\.kubernetes\.io\/not-ready"/,
+    );
+    expect(joinedLiteral).toMatch(
+      /"keda":\{[\s\S]*"tolerations":\[[\s\S]*"node\.kubernetes\.io\/unreachable"[\s\S]*"node\.kubernetes\.io\/not-ready"/,
+    );
+  });
+
+  it("installs a KubernetesPatch that removes the InferenceEndpointConfig finalizer on destroy", () => {
+    new HyperPodVllmNxdInferenceService(stack, "InferenceService", {
+      cluster,
+      compiledModel,
+    });
+    const template = Template.fromStack(stack);
+    // The patch uses `MERGE` type with an empty `applyPatch` and a
+    // `restorePatch` that clears the finalizer list. We match on
+    // `ResourceName` + `PatchType` + the serialized `RestorePatchJson`.
+    template.hasResourceProperties("Custom::AWSCDK-EKS-KubernetesPatch", {
+      ResourceName: Match.stringLikeRegexp("^inferenceendpointconfig/"),
+      PatchType: "merge",
+      RestorePatchJson: Match.stringLikeRegexp(
+        '"finalizers"\\s*:\\s*\\[\\s*\\]',
+      ),
+    });
+  });
 });
diff --git a/src/hyper-pod/hyper-pod-vllm-nxd-inference-service.ts b/src/hyper-pod/hyper-pod-vllm-nxd-inference-service.ts
@@ -343,6 +343,43 @@ export class HyperPodVllmNxdInferenceService extends Construct {
       inferencePrerequisites.node.defaultChild as cdk.CfnResource,
     );
 
+    // On stack destroy the EKS system nodegroup is torn down before the
+    // HyperPod worker node is drained, which leaves the
+    // `hyperpod-inference-controller-manager` Deployment with no node to
+    // run on (the only remaining node carries the
+    // `node.kubernetes.io/unreachable:NoSchedule` taint and the
+    // controller-manager Deployment does not yet expose tolerations via
+    // the addon's configurationValues schema). With the controller down
+    // the `inference.sagemaker.aws.InferenceEndpointConfigFinalizer` on
+    // the `InferenceEndpointConfig` CR is never released, so the
+    // KubernetesManifest above hangs in `DELETE_IN_PROGRESS` for 45+
+    // minutes before CloudFormation gives up.
+    //
+    // Use `KubernetesPatch` with a destroy-time `restorePatch` that
+    // removes the finalizer list directly. The patch runs from the
+    // CDK-managed kubectl provider Lambda (not from a pod inside the
+    // cluster), so it is unaffected by whether the controller is
+    // scheduled. The dependency edge below ensures CloudFormation runs
+    // the patch *before* it deletes the manifest: on create/update the
+    // manifest is applied first (then the patch applies its no-op
+    // `applyPatch`), and on delete the order reverses so the finalizer
+    // is stripped before the CR is removed.
+    const removeFinalizer = new eks.KubernetesPatch(
+      this,
+      "RemoveInferenceEndpointFinalizerOnDestroy",
+      {
+        cluster: props.cluster.eksCluster,
+        resourceName: `inferenceendpointconfig/${this.endpointName}`,
+        resourceNamespace: namespace,
+        // No-op on create/update; the actual work is `restorePatch` on
+        // delete.
+        applyPatch: {},
+        restorePatch: { metadata: { finalizers: [] } },
+        patchType: eks.PatchType.MERGE,
+      },
+    );
+    removeFinalizer.node.addDependency(endpointManifest);
+
     // Grant S3 access for model loading
     props.compiledModel.bucket.grantRead(props.cluster.executionRole);
 
@@ -571,6 +608,34 @@ export class HyperPodVllmNxdInferenceService extends Construct {
     ) as cdk.CfnResource;
     const clusterArnForAddon = sagemakerCluster.ref;
 
+    // Tolerations that keep the addon-managed ALB controller and KEDA
+    // components scheduled while the cluster's system nodes are tainted
+    // (e.g. during stack deletion, when nodes are marked
+    // `node.kubernetes.io/unreachable` or `node.kubernetes.io/not-ready`
+    // while they drain). Without these tolerations the ALB controller pod
+    // is evicted before it can process the Ingress deletion event, which
+    // causes the ALB's SecurityGroup and TargetGroup to be orphaned
+    // outside of CloudFormation and leaves the VPC stuck in
+    // `DELETE_IN_PROGRESS` with `DependencyViolation`.
+    //
+    // Note: the HyperPod inference controller-manager Deployment does not
+    // currently expose tolerations through the addon's configurationValues
+    // schema, so the CR finalizer deadlock (handled by the KubernetesPatch
+    // restorePatch below) still needs a separate workaround until the
+    // upstream addon exposes that field.
+    const destroyTimeTolerations = [
+      {
+        key: "node.kubernetes.io/unreachable",
+        operator: "Exists",
+        effect: "NoSchedule",
+      },
+      {
+        key: "node.kubernetes.io/not-ready",
+        operator: "Exists",
+        effect: "NoSchedule",
+      },
+    ];
+
     const addon = new eks.Addon(this, "InferenceOperatorAddon", {
       addonName: "amazon-sagemaker-hyperpod-inference",
       cluster: cluster.eksCluster,
@@ -593,6 +658,10 @@ export class HyperPodVllmNxdInferenceService extends Construct {
             // Ingress the operator creates.
             roleArn: albControllerRole.roleArn,
           },
+          tolerations: destroyTimeTolerations,
+        },
+        keda: {
+          tolerations: destroyTimeTolerations,
         },
       },
     });

diff --git a/test/integ.hyper-pod-cluster.ts.snapshot/HyperPodClusterIntegTestStack.assets.json b/test/integ.hyper-pod-cluster.ts.snapshot/HyperPodClusterIntegTestStack.assets.json
@@ -183,16 +183,16 @@
         }
       }
     },
-    "c0a631df012f9e7697e7db21d0d1a8a5cdfd2c839fab189b3d9daca94d1748b9": {
+    "815ed9aecb45eaff7e12b1e8c9f24ef91bd0aff72a4fad755feea73faf737209": {
       "displayName": "HyperPodClusterIntegTestStack Template",
       "source": {
         "path": "HyperPodClusterIntegTestStack.template.json",
         "packaging": "file"
       },
       "destinations": {
-        "current_account-current_region-c085a4c8": {
+        "current_account-current_region-e1336041": {
           "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
-          "objectKey": "c0a631df012f9e7697e7db21d0d1a8a5cdfd2c839fab189b3d9daca94d1748b9.json",
+          "objectKey": "815ed9aecb45eaff7e12b1e8c9f24ef91bd0aff72a4fad755feea73faf737209.json",
           "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
         }
       }

diff --git a/test/integ.hyper-pod-cluster.ts.snapshot/HyperPodClusterIntegTestStack.template.json b/test/integ.hyper-pod-cluster.ts.snapshot/HyperPodClusterIntegTestStack.template.json
@@ -4647,7 +4647,7 @@
          "Arn"
         ]
        },
-       "\"}}}"
+       "\"},\"tolerations\":[{\"key\":\"node.kubernetes.io/unreachable\",\"operator\":\"Exists\",\"effect\":\"NoSchedule\"},{\"key\":\"node.kubernetes.io/not-ready\",\"operator\":\"Exists\",\"effect\":\"NoSchedule\"}]},\"keda\":{\"tolerations\":[{\"key\":\"node.kubernetes.io/unreachable\",\"operator\":\"Exists\",\"effect\":\"NoSchedule\"},{\"key\":\"node.kubernetes.io/not-ready\",\"operator\":\"Exists\",\"effect\":\"NoSchedule\"}]}}"
       ]
      ]
     }
@@ -4662,6 +4662,31 @@
     "InferenceServiceTlsCertificateBucket485D9F51"
    ]
   },
+  "InferenceServiceRemoveInferenceEndpointFinalizerOnDestroy854868F0": {
+   "Type": "Custom::AWSCDK-EKS-KubernetesPatch",
+   "Properties": {
+    "ServiceToken": {
+     "Fn::GetAtt": [
+      "HyperPodEksClusterKubectlProviderframeworkonEventFCB10CB8",
+      "Arn"
+     ]
+    },
+    "ResourceName": "inferenceendpointconfig/hyperpodclusterinteackinferenceservice5e8c33f7",
+    "ResourceNamespace": "default",
+    "ApplyPatchJson": "{}",
+    "RestorePatchJson": "{\"metadata\":{\"finalizers\":[]}}",
+    "ClusterName": {
+     "Ref": "HyperPodEksCluster2B4D66BB"
+    },
+    "PatchType": "merge"
+   },
+   "DependsOn": [
+    "HyperPodEksClusterKubectlReadyBarrierFC60BFEC",
+    "HyperPodEksClustermanifestInferenceEndpoint09B1D871"
+   ],
+   "UpdateReplacePolicy": "Delete",
+   "DeletionPolicy": "Delete"
+  },
   "AWSCDKCfnUtilsProviderCustomResourceProviderRoleFE0EE867": {
    "Type": "AWS::IAM::Role",
    "Properties": {

diff --git a/test/integ.hyper-pod-cluster.ts.snapshot/IntegTestAssertions.assets.json b/test/integ.hyper-pod-cluster.ts.snapshot/IntegTestAssertions.assets.json
@@ -14,16 +14,16 @@
         }
       }
     },
-    "c0c191e614abac0babc1680a1bd03875c5655c8c90310cfb5b3bf8b2de72ad42": {
+    "951b7da87cda6acd071a4be5eb908b63a6fd88c6baf1961251db660153bb1337": {
       "displayName": "IntegTestAssertions Template",
       "source": {
         "path": "IntegTestAssertions.template.json",
         "packaging": "file"
       },
       "destinations": {
-        "current_account-current_region-9a41aedb": {
+        "current_account-current_region-64086028": {
           "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
-          "objectKey": "c0c191e614abac0babc1680a1bd03875c5655c8c90310cfb5b3bf8b2de72ad42.json",
+          "objectKey": "951b7da87cda6acd071a4be5eb908b63a6fd88c6baf1961251db660153bb1337.json",
           "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
         }
       }

diff --git a/test/integ.hyper-pod-cluster.ts.snapshot/IntegTestAssertions.template.json b/test/integ.hyper-pod-cluster.ts.snapshot/IntegTestAssertions.template.json
@@ -30,7 +30,7 @@
      }
     },
     "flattenResponse": "false",
-    "salt": "1777027678879"
+    "salt": "1777088228577"
    },
    "UpdateReplacePolicy": "Delete",
    "DeletionPolicy": "Delete"
@@ -456,7 +456,7 @@
      }
     },
     "flattenResponse": "false",
-    "salt": "1777027678883"
+    "salt": "1777088228581"
    },
    "UpdateReplacePolicy": "Delete",
    "DeletionPolicy": "Delete"

diff --git a/test/integ.hyper-pod-cluster.ts.snapshot/manifest.json b/test/integ.hyper-pod-cluster.ts.snapshot/manifest.json
@@ -18,7 +18,7 @@
         "validateOnSynth": false,
         "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}",
         "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}",
-        "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/c0a631df012f9e7697e7db21d0d1a8a5cdfd2c839fab189b3d9daca94d1748b9.json",
+        "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/815ed9aecb45eaff7e12b1e8c9f24ef91bd0aff72a4fad755feea73faf737209.json",
         "requiresBootstrapStackVersion": 6,
         "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version",
         "additionalDependencies": [
@@ -1830,10 +1830,7 @@
         "/HyperPodClusterIntegTestStack/KubectlLayer/Resource": [
           {
             "type": "aws:cdk:logicalId",
-            "data": "KubectlLayer600207B5",
-            "trace": [
-              "!!DESTRUCTIVE_CHANGES: WILL_REPLACE"
-            ]
+            "data": "KubectlLayer600207B5"
           }
         ],
         "/HyperPodClusterIntegTestStack/HyperPod/EksCluster": [
@@ -2160,10 +2157,7 @@
         "/HyperPodClusterIntegTestStack/HyperPod/EksCluster/KubectlProvider/AwsCliLayer/Resource": [
           {
             "type": "aws:cdk:logicalId",
-            "data": "HyperPodEksClusterKubectlProviderAwsCliLayer956132D0",
-            "trace": [
-              "!!DESTRUCTIVE_CHANGES: WILL_REPLACE"
-            ]
+            "data": "HyperPodEksClusterKubectlProviderAwsCliLayer956132D0"
           }
         ],
         "/HyperPodClusterIntegTestStack/HyperPod/EksCluster/KubectlProvider/Provider/framework-onEvent": [
@@ -2746,10 +2740,7 @@
         "/HyperPodClusterIntegTestStack/HyperPod/LifecycleScriptsDeploy/AwsCliLayer/Resource": [
           {
             "type": "aws:cdk:logicalId",
-            "data": "HyperPodLifecycleScriptsDeployAwsCliLayerDB7979F7",
-            "trace": [
-              "!!DESTRUCTIVE_CHANGES: WILL_REPLACE"
-            ]
+            "data": "HyperPodLifecycleScriptsDeployAwsCliLayerDB7979F7"
           }
         ],
         "/HyperPodClusterIntegTestStack/HyperPod/LifecycleScriptsDeploy/CustomResourceHandler": [
@@ -3730,6 +3721,18 @@
             "data": "InferenceServiceInferenceOperatorAddonFEF09769"
           }
         ],
+        "/HyperPodClusterIntegTestStack/InferenceService/RemoveInferenceEndpointFinalizerOnDestroy/Resource": [
+          {
+            "type": "aws:cdk:analytics:construct",
+            "data": "*"
+          }
+        ],
+        "/HyperPodClusterIntegTestStack/InferenceService/RemoveInferenceEndpointFinalizerOnDestroy/Resource/Default": [
+          {
+            "type": "aws:cdk:logicalId",
+            "data": "InferenceServiceRemoveInferenceEndpointFinalizerOnDestroy854868F0"
+          }
+        ],
         "/HyperPodClusterIntegTestStack/AWSCDKCfnUtilsProviderCustomResourceProvider": [
           {
             "type": "aws:cdk:is-custom-resource-handler-customResourceProvider",
@@ -3916,7 +3919,7 @@
         "validateOnSynth": false,
         "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-deploy-role-${AWS::AccountId}-${AWS::Region}",
         "cloudFormationExecutionRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-cfn-exec-role-${AWS::AccountId}-${AWS::Region}",
-        "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/c0c191e614abac0babc1680a1bd03875c5655c8c90310cfb5b3bf8b2de72ad42.json",
+        "stackTemplateAssetObjectUrl": "s3://cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}/951b7da87cda6acd071a4be5eb908b63a6fd88c6baf1961251db660153bb1337.json",
         "requiresBootstrapStackVersion": 6,
         "bootstrapStackVersionSsmParameter": "/cdk-bootstrap/hnb659fds/version",
         "additionalDependencies": [

diff --git a/test/integ.hyper-pod-cluster.ts.snapshot/tree.json b/test/integ.hyper-pod-cluster.ts.snapshot/tree.json