WIP: refactoring - fix jvm

Author: Yuriy Bezsonov

infrastructure/cdk/src/main/java/com/unicorn/UnicornStoreStack.java

@@ -33,9 +33,8 @@ public class UnicornStoreStack extends Stack {
         sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/app.sh"
         sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/eks.sh"
 
-        sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/monitoring.sh"
-        sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/thread-dump-lambda/build-and-deploy.sh"
-        sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/monitoring-jvm.sh"
+        # sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/monitoring.sh"
+        # sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/monitoring-jvm.sh"
         """;
 
     private final String buildspec = """

infrastructure/cfn/unicornstore-stack.yaml

@@ -492,7 +492,7 @@ Resources:
   UnicornStoreIdeIdeLogGroup7A5EFA50:
     Type: AWS::Logs::LogGroup
     Properties:
-      LogGroupName: unicornstore-ide-bootstrap-log-20250725-203717
+      LogGroupName: unicornstore-ide-bootstrap-log-20250727-102705
       RetentionInDays: 7
     UpdateReplacePolicy: Retain
     DeletionPolicy: Retain
@@ -907,9 +907,9 @@ Resources:
         description: Bootstrap IDE
         parameters:
           BootstrapScript:
-            type: String
-            description: (Optional) Custom bootstrap script to run.
             default: ""
+            description: (Optional) Custom bootstrap script to run.
+            type: String
         mainSteps:
           - inputs:
               runCommand:
@@ -1174,30 +1174,14 @@ Resources:
                         Fn::GetAtt:
                           - UnicornStoreIdeIdeRole81A2CEAA
                           - Arn
-                      splashUrl: ""
-                      instanceIamRoleName:
-                        Ref: UnicornStoreIdeIdeRole81A2CEAA
-                      codeServerVersion: 4.101.2
-                      customBootstrapScript: |
-                        date
-
-                        echo '=== Clone Git repository ===
-                        '
-                        sudo -H -u ec2-user bash -c "git clone https://github.com/aws-samples/java-on-aws ~/java-on-aws/"
-                        sudo -H -u ec2-user bash -c "cd ~/java-on-aws && git checkout bedrock-tda"
-
-                        echo '=== Setup IDE ===
-                        '
-                        sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/ide.sh"
-
-                        echo '=== Additional Setup ===
-                        '
-                        sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/app.sh"
-                        sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/eks.sh"
-
-                        sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/monitoring.sh"
-                        sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/thread-dump-lambda/build-and-deploy.sh"
-                        sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/monitoring-jvm.sh"
+                      readmeUrl: ""
+                      waitConditionHandleUrl:
+                        Ref: UnicornStoreIdeIdeBootstrapWaitConditionHandle3D9B67F1
+                      extensions: vscjava.vscode-java-pack,ms-kubernetes-tools.vscode-kubernetes-tools,ms-azuretools.vscode-docker
+                      domain: ""
+                      installGitea: echo bootstrapGitea was not provided
+                      terminalOnStartup: "true"
+                      environmentContentsZip: ""
                       passwordName:
                         Fn::Join:
                           - "-"
@@ -1237,14 +1221,29 @@ Resources:
                                         - Fn::Split:
                                             - ":"
                                             - Ref: UnicornStoreIdeIdePasswordSecret514252E2
-                      environmentContentsZip: ""
-                      terminalOnStartup: "true"
-                      installGitea: echo bootstrapGitea was not provided
-                      domain: ""
-                      extensions: vscjava.vscode-java-pack,ms-kubernetes-tools.vscode-kubernetes-tools,ms-azuretools.vscode-docker
-                      waitConditionHandleUrl:
-                        Ref: UnicornStoreIdeIdeBootstrapWaitConditionHandle3D9B67F1
-                      readmeUrl: ""
+                      customBootstrapScript: |
+                        date
+
+                        echo '=== Clone Git repository ===
+                        '
+                        sudo -H -u ec2-user bash -c "git clone https://github.com/aws-samples/java-on-aws ~/java-on-aws/"
+                        sudo -H -u ec2-user bash -c "cd ~/java-on-aws && git checkout bedrock-tda"
+
+                        echo '=== Setup IDE ===
+                        '
+                        sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/ide.sh"
+
+                        echo '=== Additional Setup ===
+                        '
+                        sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/app.sh"
+                        sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/eks.sh"
+
+                        # sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/monitoring.sh"
+                        # sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/monitoring-jvm.sh"
+                      codeServerVersion: 4.101.2
+                      instanceIamRoleName:
+                        Ref: UnicornStoreIdeIdeRole81A2CEAA
+                      splashUrl: ""
             name: IdeBootstrapFunction
             action: aws:runShellScript
       DocumentFormat: YAML
@@ -1384,10 +1383,10 @@ Resources:
           - Arn
       InstanceId:
         Ref: UnicornStoreIdeIdeEC2Instance5F393FC5
-      SsmDocument:
-        Ref: UnicornStoreIdeIdeBootstrapDocumentE330810B
       LogGroupName:
         Ref: UnicornStoreIdeIdeLogGroup7A5EFA50
+      SsmDocument:
+        Ref: UnicornStoreIdeIdeBootstrapDocumentE330810B
     UpdateReplacePolicy: Delete
     DeletionPolicy: Delete
   UnicornStoreEksClusterEKSClusterRole0A87A100:
@@ -1748,7 +1747,7 @@ Resources:
             - Ref: AWS::AccountId
             - "-"
             - Ref: AWS::Region
-            - "-20250725203718"
+            - "-20250727102705"
       PublicAccessBlockConfiguration:
         BlockPublicAcls: true
         BlockPublicPolicy: true
@@ -2381,12 +2380,12 @@ Resources:
       Environment:
         Variables:
           S3_THREAD_DUMPS_PREFIX: thread-dumps/
-          EKS_CLUSTER_NAME: unicorn-store
+          APP_LABEL: unicorn-store-spring
+          KUBERNETES_AUTH_TYPE: aws
+          K8S_NAMESPACE: unicorn-store-spring
           S3_BUCKET_NAME:
             Ref: InfrastructureCoreWorkshopBucketB6510E67
-          K8S_NAMESPACE: unicorn-store-spring
-          KUBERNETES_AUTH_TYPE: aws
-          APP_LABEL: unicorn-store-spring
+          EKS_CLUSTER_NAME: unicorn-store
       FunctionName: unicornstore-thread-dump-lambda
       Handler: index.lambda_handler
       MemorySize: 512
@@ -2776,6 +2775,9 @@ Resources:
         Fn::GetAtt:
           - UnicornStoreDatabaseSetupDatabaseSetupFunction83636DD9
           - Arn
+      SqlStatements: |-
+        CREATE TABLE IF NOT EXISTS unicorns(id TEXT DEFAULT gen_random_uuid() PRIMARY KEY, name TEXT, age TEXT, size TEXT, type TEXT);
+        CREATE EXTENSION IF NOT EXISTS vector;
       SecretName:
         Fn::Join:
           - "-"
@@ -2806,9 +2808,6 @@ Resources:
                         - Fn::Split:
                             - ":"
                             - Ref: InfrastructureCorepostgresB0D0C6BB
-      SqlStatements: |-
-        CREATE TABLE IF NOT EXISTS unicorns(id TEXT DEFAULT gen_random_uuid() PRIMARY KEY, name TEXT, age TEXT, size TEXT, type TEXT);
-        CREATE EXTENSION IF NOT EXISTS vector;
     DependsOn:
       - InfrastructureCoreUnicornStoreDatabaseF56884BA
       - InfrastructureCoreUnicornStoreDatabaseSubnets55C412B8
@@ -2876,6 +2875,9 @@ Resources:
               - - "{{resolve:secretsmanager:"
                 - Ref: InfrastructureCorepostgresB0D0C6BB
                 - :SecretString:password::}}
+          JAVA_TOOL_OPTIONS: -XX:+TieredCompilation -XX:TieredStopAtLevel=1
+          AWS_SERVERLESS_JAVA_CONTAINER_INIT_GRACE_TIME: "500"
+          SPRING_DATASOURCE_HIKARI_maximumPoolSize: "1"
           SPRING_DATASOURCE_URL:
             Fn::Join:
               - ""
@@ -2884,9 +2886,6 @@ Resources:
                     - InfrastructureCoreUnicornStoreDatabaseF56884BA
                     - Endpoint.Address
                 - :5432/unicorns
-          SPRING_DATASOURCE_HIKARI_maximumPoolSize: "1"
-          AWS_SERVERLESS_JAVA_CONTAINER_INIT_GRACE_TIME: "500"
-          JAVA_TOOL_OPTIONS: -XX:+TieredCompilation -XX:TieredStopAtLevel=1
       FunctionName: unicorn-store-spring
       Handler: com.unicorn.store.StreamLambdaHandler::handleRequest
       MemorySize: 2048
@@ -3573,12 +3572,12 @@ Resources:
       Description: Build complete
       EventPattern:
         detail:
-          project-name:
-            - Ref: UnicornStoreCodeBuildCodeBuildProjectAF799FF7
           build-status:
             - SUCCEEDED
             - FAILED
             - STOPPED
+          project-name:
+            - Ref: UnicornStoreCodeBuildCodeBuildProjectAF799FF7
         detail-type:
           - CodeBuild Build State Change
         source:
@@ -3612,11 +3611,11 @@ Resources:
           - Arn
       ProjectName:
         Ref: UnicornStoreCodeBuildCodeBuildProjectAF799FF7
-      ContentHash: fd70d4491df3a6a44292446f6b364c8a
       CodeBuildIamRoleArn:
         Fn::GetAtt:
           - UnicornStoreCodeBuildCodeBuildRoleB9CE0DE7
           - Arn
+      ContentHash: fd70d4491df3a6a44292446f6b364c8a
     DependsOn:
       - UnicornStoreCodeBuildBuildCompleteRuleAllowEventRuleunicornstorestackUnicornStoreCodeBuildReportBuildFunction6189FCDE5C03542B
       - UnicornStoreCodeBuildBuildCompleteRule7E287561

infrastructure/scripts/setup/monitoring-jvm.sh

@@ -279,3 +279,32 @@ fi
 ALERT_RESPONSE=$(curl -s -X POST -H "Content-Type: application/json" \
   -u "$GRAFANA_USER:$GRAFANA_PASSWORD" \
   -d "$ALERT_PAYLOAD" \
+  "$GRAFANA_URL/api/v1/provisioning/alert-rules")
+
+log "✅ Alert rule created"
+
+# Ensure notification policy routes to our contact point
+log "🔧 Configuring notification policy..."
+POLICY_PAYLOAD="{
+  \"receiver\": \"$CONTACT_POINT_NAME\",
+  \"group_by\": [\"alertname\"],
+  \"group_wait\": \"30s\",
+  \"group_interval\": \"5m\",
+  \"repeat_interval\": \"1h\"
+}"
+
+POLICY_RESPONSE=$(curl -s -X PUT -H "Content-Type: application/json" \
+  -u "$GRAFANA_USER:$GRAFANA_PASSWORD" \
+  -d "$POLICY_PAYLOAD" \
+  "$GRAFANA_URL/api/v1/provisioning/policies")
+
+if echo "$POLICY_RESPONSE" | grep -q "policies updated"; then
+  log "✅ Notification policy configured"
+else
+  log "❌ Notification policy configuration failed:"
+  echo "$POLICY_RESPONSE"
+fi
+log "✅ JVM monitoring setup complete"
+log "🌍 Grafana: $GRAFANA_URL"
+log "📊 Dashboard shows jvm_threads_live_threads from both EKS and ECS"
+log "🚨 Alert triggers Lambda thread dump when threads > $THREAD_THRESHOLD, stops when threads < $THREAD_THRESHOLD"

infrastructure/scripts/setup/monitoring.sh

@@ -61,6 +61,24 @@ helm upgrade --install prometheus prometheus-community/prometheus \
   --namespace "$NAMESPACE" \
   --values "$VALUES_FILE"
 
+# Wait for Prometheus to be ready
+log "⏳ Waiting for Prometheus to be ready..."
+kubectl wait --for=condition=available --timeout=300s deployment/prometheus-server -n "$NAMESPACE"
+
+# Verify Prometheus is responding
+kubectl port-forward -n "$NAMESPACE" svc/prometheus-server 9090:80 &
+PF_PID=$!
+sleep 5
+if curl -s http://localhost:9090/-/healthy > /dev/null 2>&1; then
+    log "✅ Prometheus is healthy"
+else
+    log "❌ Prometheus health check failed"
+    kubectl logs -n "$NAMESPACE" deployment/prometheus-server -c prometheus-server --tail=10
+    kill $PF_PID 2>/dev/null
+    exit 1
+fi
+kill $PF_PID 2>/dev/null
+
 # Grafana values
 cat > "$GRAFANA_VALUES_FILE" <<EOF
 admin: