diff --git a/apps/unicorn-store-spring/pom.xml b/apps/unicorn-store-spring/pom.xml index 7b6614da..1196d6b0 100644 --- a/apps/unicorn-store-spring/pom.xml +++ b/apps/unicorn-store-spring/pom.xml @@ -58,6 +58,12 @@ org.springframework.boot spring-boot-starter-actuator + + + io.micrometer + micrometer-registry-prometheus + 1.14.7 + software.amazon.awssdk diff --git a/apps/unicorn-store-spring/src/main/java/com/unicorn/store/config/MonitoringConfig.java b/apps/unicorn-store-spring/src/main/java/com/unicorn/store/config/MonitoringConfig.java new file mode 100644 index 00000000..64578b92 --- /dev/null +++ b/apps/unicorn-store-spring/src/main/java/com/unicorn/store/config/MonitoringConfig.java @@ -0,0 +1,168 @@ +package com.unicorn.store.config; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.config.MeterFilter; +import org.springframework.boot.actuate.autoconfigure.metrics.MeterRegistryCustomizer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.io.File; +import java.io.IOException; +import java.net.InetAddress; +import java.net.URI; +import java.net.UnknownHostException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.util.Optional; + +@Configuration +public class MonitoringConfig { + + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + private static final File NAMESPACE_FILE = new File("/var/run/secrets/kubernetes.io/serviceaccount/namespace"); + + @Bean + public MeterRegistryCustomizer meterRegistryCustomizer() { + return registry -> { + String clusterType = System.getenv("ECS_CONTAINER_METADATA_URI_V4") != null ? "ecs" : "eks"; + String cluster = clusterType.equals("ecs") ? extractClusterNameFromMetadata().orElse("unknown") : Optional.ofNullable(System.getenv("CLUSTER")).orElse("unknown"); + String containerName = "unicorn-store-spring"; + String taskOrPodId = extractTaskOrPodId().orElse("unknown"); + String namespace = clusterType.equals("eks") ? readNamespaceFile().orElse("default") : ""; + + // Get the container/pod IP address + String ipAddress = getContainerOrPodIp().orElse("unknown"); + + registry.config().commonTags( + "cluster", cluster, + "cluster_type", clusterType, + "container_name", containerName, + "task_pod_id", taskOrPodId, + "instance", ipAddress, // Keep this for backward compatibility + "container_ip", ipAddress // Add this new tag that won't be overwritten + ); + + if (!namespace.isEmpty()) { + registry.config().commonTags("namespace", namespace); + } else { + registry.config().commonTags("namespace", ""); + } + + registry.config().meterFilter( + MeterFilter.deny(id -> + id.getName().equals("jvm.gc.pause") && + !id.getTags().stream().allMatch(tag -> + tag.getKey().equals("action") || + tag.getKey().equals("cause") || + tag.getKey().equals("gc") + ) + ) + ); + }; + } + + private Optional extractTaskOrPodId() { + String metadataUri = System.getenv("ECS_CONTAINER_METADATA_URI_V4"); + if (metadataUri != null) { + try { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(metadataUri + "/task")) + .build(); + + HttpResponse response = HttpClient.newHttpClient() + .send(request, HttpResponse.BodyHandlers.ofString()); + + JsonNode root = OBJECT_MAPPER.readTree(response.body()); + String taskArn = root.path("TaskARN").asText(); + String[] parts = taskArn.split("/"); + return parts.length > 1 ? Optional.of(parts[parts.length - 1]) : Optional.empty(); + + } catch (IOException | InterruptedException e) { + return Optional.empty(); + } + } + + // EKS fallback: read pod name from Downward API + return readFile("/etc/podinfo/name"); + } + + private Optional extractClusterNameFromMetadata() { + String metadataUri = System.getenv("ECS_CONTAINER_METADATA_URI_V4"); + if (metadataUri != null) { + try { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(metadataUri + "/task")) + .build(); + + HttpResponse response = HttpClient.newHttpClient() + .send(request, HttpResponse.BodyHandlers.ofString()); + + JsonNode root = OBJECT_MAPPER.readTree(response.body()); + String clusterArn = root.path("Cluster").asText(); + String[] parts = clusterArn.split("/"); + return parts.length > 1 ? Optional.of(parts[parts.length - 1]) : Optional.empty(); + + } catch (IOException | InterruptedException e) { + return Optional.empty(); + } + } + return Optional.empty(); + } + + private Optional readNamespaceFile() { + return readFile(NAMESPACE_FILE.getAbsolutePath()); + } + + private Optional readFile(String path) { + try { + return Optional.of(Files.readString(new File(path).toPath()).trim()); + } catch (IOException e) { + return Optional.empty(); + } + } + + // New method to get the container or pod IP address + private Optional getContainerOrPodIp() { + // For ECS + String metadataUri = System.getenv("ECS_CONTAINER_METADATA_URI_V4"); + if (metadataUri != null) { + try { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(metadataUri)) + .build(); + + HttpResponse response = HttpClient.newHttpClient() + .send(request, HttpResponse.BodyHandlers.ofString()); + + JsonNode root = OBJECT_MAPPER.readTree(response.body()); + + if (root.has("Networks") && root.path("Networks").isArray() && !root.path("Networks").isEmpty()) { + JsonNode network = root.path("Networks").get(0); + if (network.has("IPv4Addresses") && network.path("IPv4Addresses").isArray() && + !network.path("IPv4Addresses").isEmpty()) { + return Optional.of(network.path("IPv4Addresses").get(0).asText()); + } + } + } catch (IOException | InterruptedException e) { + // Fall through to next method + } + } + + // For Kubernetes/EKS + String podIp = System.getenv("KUBERNETES_POD_IP"); + if (podIp != null && !podIp.isEmpty()) { + return Optional.of(podIp); + } + + // Try to get local IP as fallback + try { + return Optional.of(InetAddress.getLocalHost().getHostAddress()); + } catch (UnknownHostException e) { + return Optional.empty(); + } + } +} \ No newline at end of file diff --git a/apps/unicorn-store-spring/src/main/java/com/unicorn/store/controller/ThreadManagementController.java b/apps/unicorn-store-spring/src/main/java/com/unicorn/store/controller/ThreadManagementController.java new file mode 100644 index 00000000..af2f4d04 --- /dev/null +++ b/apps/unicorn-store-spring/src/main/java/com/unicorn/store/controller/ThreadManagementController.java @@ -0,0 +1,42 @@ +// src/main/java/com/unicorn/store/controller/ThreadManagementController.java +package com.unicorn.store.controller; + +import com.unicorn.store.service.ThreadGeneratorService; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +@RestController +@RequestMapping("/api/threads") +public class ThreadManagementController { + + private final ThreadGeneratorService threadGeneratorService; + + public ThreadManagementController(ThreadGeneratorService threadGeneratorService) { + this.threadGeneratorService = threadGeneratorService; + } + + @PostMapping("/start") + public ResponseEntity startThreads(@RequestParam(defaultValue = "500") int count) { + try { + threadGeneratorService.startThreads(count); + return ResponseEntity.ok("Successfully started " + count + " threads"); + } catch (IllegalStateException e) { + return ResponseEntity.badRequest().body(e.getMessage()); + } + } + + @PostMapping("/stop") + public ResponseEntity stopThreads() { + try { + threadGeneratorService.stopThreads(); + return ResponseEntity.ok("Successfully stopped all threads"); + } catch (IllegalStateException e) { + return ResponseEntity.badRequest().body(e.getMessage()); + } + } + + @GetMapping("/count") + public ResponseEntity getThreadCount() { + return ResponseEntity.ok(threadGeneratorService.getActiveThreadCount()); + } +} diff --git a/apps/unicorn-store-spring/src/main/java/com/unicorn/store/monitoring/ThreadMonitoringMBean.java b/apps/unicorn-store-spring/src/main/java/com/unicorn/store/monitoring/ThreadMonitoringMBean.java new file mode 100644 index 00000000..2c8b87d5 --- /dev/null +++ b/apps/unicorn-store-spring/src/main/java/com/unicorn/store/monitoring/ThreadMonitoringMBean.java @@ -0,0 +1,23 @@ +// src/main/java/com/unicorn/store/monitoring/ThreadMonitoringMBean.java +package com.unicorn.store.monitoring; + +import com.unicorn.store.service.ThreadGeneratorService; +import org.springframework.jmx.export.annotation.ManagedAttribute; +import org.springframework.jmx.export.annotation.ManagedResource; +import org.springframework.stereotype.Component; + +@Component +@ManagedResource(objectName = "com.unicorn.store:type=ThreadMonitoring,name=ThreadStats") +public class ThreadMonitoringMBean { + + private final ThreadGeneratorService threadGeneratorService; + + public ThreadMonitoringMBean(ThreadGeneratorService threadGeneratorService) { + this.threadGeneratorService = threadGeneratorService; + } + + @ManagedAttribute(description = "Number of active custom threads") + public int getActiveThreadCount() { + return threadGeneratorService.getActiveThreadCount(); + } +} diff --git a/apps/unicorn-store-spring/src/main/java/com/unicorn/store/service/ThreadGeneratorService.java b/apps/unicorn-store-spring/src/main/java/com/unicorn/store/service/ThreadGeneratorService.java new file mode 100644 index 00000000..7081b360 --- /dev/null +++ b/apps/unicorn-store-spring/src/main/java/com/unicorn/store/service/ThreadGeneratorService.java @@ -0,0 +1,86 @@ +// src/main/java/com/unicorn/store/service/ThreadGeneratorService.java +package com.unicorn.store.service; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +@Service +public class ThreadGeneratorService { + private static final Logger logger = LoggerFactory.getLogger(ThreadGeneratorService.class); + private final List activeThreads = new ArrayList<>(); + private final AtomicBoolean running = new AtomicBoolean(false); + + public synchronized void startThreads(int threadCount) { + if (running.get()) { + throw new IllegalStateException("Threads are already running"); + } + + running.set(true); + logger.info("Starting {} threads", threadCount); + + for (int i = 0; i < threadCount; i++) { + Thread thread = new Thread(new DummyWorkload(running)); + thread.setName("DummyThread-" + i); + activeThreads.add(thread); + thread.start(); + } + + logger.info("Started {} threads", threadCount); + } + + public synchronized void stopThreads() { + if (!running.get()) { + throw new IllegalStateException("No threads are running"); + } + + logger.info("Stopping {} threads", activeThreads.size()); + running.set(false); + + // Wait for all threads to complete + activeThreads.forEach(thread -> { + try { + thread.join(5000); // Wait up to 5 seconds for each thread + } catch (InterruptedException e) { + logger.warn("Interrupted while waiting for thread {} to stop", thread.getName()); + } + }); + + activeThreads.clear(); + logger.info("All threads stopped"); + } + + public int getActiveThreadCount() { + return activeThreads.size(); + } + + private static class DummyWorkload implements Runnable { + private final AtomicBoolean running; + + public DummyWorkload(AtomicBoolean running) { + this.running = running; + } + + @Override + public void run() { + while (running.get()) { + // Simulate some work + try { + // Calculate some dummy values to keep CPU busy + double result = 0; + for (int i = 0; i < 1000; i++) { + result += Math.sqrt(i) * Math.random(); + } + Thread.sleep(100); // Sleep to prevent excessive CPU usage + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + } + } +} diff --git a/apps/unicorn-store-spring/src/main/resources/application.properties b/apps/unicorn-store-spring/src/main/resources/application.properties index 2651de3d..cfa6c1a2 100644 --- a/apps/unicorn-store-spring/src/main/resources/application.properties +++ b/apps/unicorn-store-spring/src/main/resources/application.properties @@ -1,20 +1,30 @@ spring.sql.init.mode=always -# spring.datasource.url=jdbc:postgresql://localhost:5432/unicorns spring.datasource.username=postgres -# spring.datasource.password=postgres spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect server.error.include-message=always spring.jpa.open-in-view=false -spring.datasource.hikari.maximumPoolSize=1 -spring.datasource.hikari.data-source-properties.preparedStatementCacheQueries=0 -management.endpoint.health.probes.enabled=true -# Disable Hibernate usage of JDBC metadata spring.jpa.properties.hibernate.temp.use_jdbc_metadata_defaults=false - -# Database initialization will typically be performed outside of Spring lifecycle spring.jpa.hibernate.ddl-auto=none + +spring.datasource.hikari.initialization-fail-timeout=0 +spring.datasource.hikari.maximumPoolSize=1 spring.datasource.hikari.allow-pool-suspension=true +spring.datasource.hikari.data-source-properties.preparedStatementCacheQueries=0 + +# Virtual Threads +spring.threads.virtual.enabled=true + +# Actuator config +spring.jmx-enabled=true +management.endpoints.web.exposure.include=threaddump,prometheus,health,info +management.endpoint.health.probes.enabled=true +management.endpoint.prometheus.enabled=true +management.endpoint.health.group.liveness.include=livenessState +management.endpoint.health.group.readiness.include=readinessState + +management.metrics.enable.all=true +management.metrics.tags.application=otel-jmx-unicorn-store -# Enable virtual threads -spring.threads.virtual.enabled=true \ No newline at end of file +server.port=8080 +server.address=0.0.0.0 \ No newline at end of file diff --git a/infrastructure/cdk/pom.xml b/infrastructure/cdk/pom.xml index a7d93d10..cd6135cf 100644 --- a/infrastructure/cdk/pom.xml +++ b/infrastructure/cdk/pom.xml @@ -75,5 +75,10 @@ ${junit.version} test + + software.constructs + constructs + 10.3.0 + diff --git a/infrastructure/cdk/src/main/java/com/unicorn/IdeStack.java b/infrastructure/cdk/src/main/java/com/unicorn/IdeStack.java index c9b903ab..a71e0769 100644 --- a/infrastructure/cdk/src/main/java/com/unicorn/IdeStack.java +++ b/infrastructure/cdk/src/main/java/com/unicorn/IdeStack.java @@ -22,8 +22,8 @@ public class IdeStack extends Stack { private final String bootstrapScript = """ date - echo '=== Clone Git repository ===' - sudo -H -u ec2-user bash -c "git clone https://github.com/aws-samples/java-on-aws ~/java-on-aws/" + echo '=== Clone Git repository ==='bash + sudo -H -u ec2-user -c "git clone https://github.com/smoell/java-on-aws ~/java-on-aws/" # sudo -H -u ec2-user bash -c "cd ~/java-on-aws && git checkout refactoring" echo '=== Setup IDE ===' diff --git a/infrastructure/cdk/src/main/java/com/unicorn/UnicornStoreStack.java b/infrastructure/cdk/src/main/java/com/unicorn/UnicornStoreStack.java index 613cb798..eab289c1 100644 --- a/infrastructure/cdk/src/main/java/com/unicorn/UnicornStoreStack.java +++ b/infrastructure/cdk/src/main/java/com/unicorn/UnicornStoreStack.java @@ -8,37 +8,33 @@ import com.unicorn.constructs.CodeBuildResource; import com.unicorn.constructs.CodeBuildResource.CodeBuildResourceProps; import com.unicorn.constructs.EksCluster; -import com.unicorn.core.InfrastructureCore; -import com.unicorn.core.InfrastructureContainers; -import com.unicorn.core.DatabaseSetup; -import com.unicorn.core.UnicornStoreSpringLambda; +import com.unicorn.core.*; -import software.amazon.awscdk.Stack; -import software.amazon.awscdk.StackProps; +import software.amazon.awscdk.*; import software.amazon.awscdk.services.ec2.InstanceClass; import software.amazon.awscdk.services.ec2.InstanceSize; import software.amazon.awscdk.services.ec2.InstanceType; import software.amazon.awscdk.services.iam.ManagedPolicy; import software.constructs.Construct; -import software.amazon.awscdk.DefaultStackSynthesizer; -import software.amazon.awscdk.DefaultStackSynthesizerProps; - public class UnicornStoreStack extends Stack { private final String bootstrapScript = """ date - echo '=== Clone Git repository ===' - sudo -H -u ec2-user bash -c "git clone https://github.com/aws-samples/java-on-aws ~/java-on-aws/" + echo '=== Clone Git repository ===\n' + sudo -H -u ec2-user bash -c "git clone https://github.com/smoell/java-on-aws ~/java-on-aws/" # sudo -H -u ec2-user bash -c "cd ~/java-on-aws && git checkout refactoring" - echo '=== Setup IDE ===' + echo '=== Setup IDE ===\n' sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/ide.sh" - echo '=== Additional Setup ===' + echo '=== Additional Setup ===\n' + + echo '=== Checking directory ===\n' sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/app.sh" sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/eks.sh" + sudo -H -i -u ec2-user bash -c "~/java-on-aws/infrastructure/scripts/setup/monitoring.sh" """; private final String buildspec = """ @@ -54,6 +50,7 @@ public class UnicornStoreStack extends Stack { commands: - | # Resolution for when creating the first service in the account + aws organizations enable-all-features aws iam create-service-linked-role --aws-service-name ecs.amazonaws.com 2>/dev/null || true aws iam create-service-linked-role --aws-service-name apprunner.amazonaws.com 2>/dev/null || true aws iam create-service-linked-role --aws-service-name elasticloadbalancing.amazonaws.com 2>/dev/null || true @@ -70,9 +67,14 @@ public UnicornStoreStack(final Construct scope, final String id) { // Create VPC var vpc = new WorkshopVpc(this, "UnicornStoreVpc", "unicornstore-vpc").getVpc(); - // Create Workshop IDE + String bootstrapScriptWithRegion = bootstrapScript + String.format( + "sudo -H -i -u ec2-user bash -c \"cd ~/java-on-aws/infrastructure/lambda/ && ./build-and-deploy.sh --region %s\"", + this.getRegion() + ); + + // Create VSCode IDE var ideProps = new VSCodeIdeProps(); - ideProps.setBootstrapScript(bootstrapScript); + ideProps.setBootstrapScript(bootstrapScriptWithRegion); ideProps.setVpc(vpc); ideProps.setInstanceName("unicornstore-ide"); ideProps.setInstanceType(InstanceType.of(InstanceClass.M5, InstanceSize.XLARGE)); @@ -87,6 +89,30 @@ public UnicornStoreStack(final Construct scope, final String id) { var ideRole = ideProps.getRole(); var ideInternalSecurityGroup = ide.getIdeInternalSecurityGroup(); + // Create S3 bucket for thread dumps + AnalysisBucketConstruct analysisBucket = new AnalysisBucketConstruct( + this, + "ThreadDumpBucket", + AnalysisBucketProps.builder() + .bucketPrefix("unicorn-analysis") + .versioningEnabled(true) + .retentionDays(90) + .noncurrentVersionRetentionDays(30) + .removalPolicy(RemovalPolicy.DESTROY) + .build() + ); + + // Create EKS cluster for the workshop + var eksCluster = new EksCluster(this, "UnicornStoreEksCluster", "unicorn-store", "1.33", + vpc, ideInternalSecurityGroup); + eksCluster.createAccessEntry(ideRole.getRoleArn(), "unicorn-store", "unicornstore-ide-user"); + + // Create Lambda function to create thread dump + InfrastructureLambdaBedrock lambdaBedrock = new InfrastructureLambdaBedrock(this, "InfrastructureLambdaBedrock", this.getRegion(), analysisBucket.getBucket(), eksCluster, vpc); + + // Create Prometheus und Grafana infrastructure + MonitoringConstruct monitoring = new MonitoringConstruct(this, "Monitoring", vpc, eksCluster.getCluster(), lambdaBedrock.getThreadDumpFunction()); // Create Grafana dashboards + // Create Core infrastructure var infrastructureCore = new InfrastructureCore(this, "InfrastructureCore", vpc); // var accountId = Stack.of(this).getAccount(); @@ -94,15 +120,12 @@ public UnicornStoreStack(final Construct scope, final String id) { // Create additional infrastructure for Containers modules of Java on AWS Immersion Day new InfrastructureContainers(this, "InfrastructureContainers", infrastructureCore); - // Create EKS cluster for the workshop - var eksCluster = new EksCluster(this, "UnicornStoreEksCluster", "unicorn-store", "1.32", - vpc, ideInternalSecurityGroup); - eksCluster.createAccessEntry(ideRole.getRoleArn(), "unicorn-store", "unicornstore-ide-user"); - // Execute Database setup var databaseSetup = new DatabaseSetup(this, "UnicornStoreDatabaseSetup", infrastructureCore); databaseSetup.getNode().addDependency(infrastructureCore.getDatabase()); + String bucketName = analysisBucket.getBucket().getBucketName(); + // Create UnicornStoreSpringLambda new UnicornStoreSpringLambda(this, "UnicornStoreSpringLambda", infrastructureCore); diff --git a/infrastructure/cdk/src/main/java/com/unicorn/constructs/EksCluster.java b/infrastructure/cdk/src/main/java/com/unicorn/constructs/EksCluster.java index d7455dd0..b83127e4 100644 --- a/infrastructure/cdk/src/main/java/com/unicorn/constructs/EksCluster.java +++ b/infrastructure/cdk/src/main/java/com/unicorn/constructs/EksCluster.java @@ -1,6 +1,6 @@ package com.unicorn.constructs; -import software.amazon.awscdk.services.ec2.IVpc; +import software.amazon.awscdk.services.ec2.*; import software.amazon.awscdk.services.eks.CfnCluster; import software.amazon.awscdk.services.eks.CfnCluster.LoggingProperty; import software.amazon.awscdk.services.eks.CfnCluster.ClusterLoggingProperty; @@ -17,12 +17,6 @@ import software.amazon.awscdk.services.eks.CfnAccessEntry.AccessScopeProperty; import software.amazon.awscdk.services.eks.CfnAccessEntry.AccessPolicyProperty; import software.amazon.awscdk.services.eks.CfnPodIdentityAssociation; -// import software.amazon.awscdk.services.eks.OpenIdConnectProvider; -// import software.amazon.awscdk.ResolutionTypeHint; -import software.amazon.awscdk.services.ec2.SubnetType; -import software.amazon.awscdk.services.ec2.SubnetSelection; -import software.amazon.awscdk.services.ec2.ISubnet; -import software.amazon.awscdk.services.ec2.ISecurityGroup; import software.amazon.awscdk.services.iam.ManagedPolicy; import software.amazon.awscdk.services.iam.Role; import software.amazon.awscdk.services.iam.ServicePrincipal; @@ -190,4 +184,18 @@ public void createPodIdentity(final String principalArn, final String clusterNam .build(); podIdentityAssociation.getNode().addDependency(cluster); } + + // Get the default cluster security group + public String getClusterSecurityGroupId() { + return cluster.getAttrClusterSecurityGroupId(); + } + + // Get the default cluster security group as ISecurityGroup + public ISecurityGroup getClusterSecurityGroup() { + return SecurityGroup.fromSecurityGroupId( + this, + "APIClusterSecurityGroup", + cluster.getAttrClusterSecurityGroupId() + ); + } } diff --git a/infrastructure/cdk/src/main/java/com/unicorn/constructs/VSCodeIde.java b/infrastructure/cdk/src/main/java/com/unicorn/constructs/VSCodeIde.java index c26bb064..a58ecf49 100644 --- a/infrastructure/cdk/src/main/java/com/unicorn/constructs/VSCodeIde.java +++ b/infrastructure/cdk/src/main/java/com/unicorn/constructs/VSCodeIde.java @@ -1,11 +1,6 @@ package com.unicorn.constructs; -import software.amazon.awscdk.CfnWaitCondition; -import software.amazon.awscdk.CfnWaitConditionHandle; -import software.amazon.awscdk.CustomResource; -import software.amazon.awscdk.Duration; -import software.amazon.awscdk.Fn; -import software.amazon.awscdk.RemovalPolicy; +import software.amazon.awscdk.*; // import software.amazon.awscdk.services.cloudfront.AddBehaviorOptions; import software.amazon.awscdk.services.cloudfront.AllowedMethods; import software.amazon.awscdk.services.cloudfront.BehaviorOptions; @@ -36,13 +31,7 @@ import software.amazon.awscdk.services.ec2.SecurityGroup; import software.amazon.awscdk.services.ec2.SubnetSelection; import software.amazon.awscdk.services.ec2.SubnetType; -import software.amazon.awscdk.services.iam.IManagedPolicy; -import software.amazon.awscdk.services.iam.InstanceProfile; -import software.amazon.awscdk.services.iam.ManagedPolicy; -import software.amazon.awscdk.services.iam.PolicyDocument; -import software.amazon.awscdk.services.iam.PolicyStatement; -import software.amazon.awscdk.services.iam.Role; -import software.amazon.awscdk.services.iam.ServicePrincipal; +import software.amazon.awscdk.services.iam.*; import software.amazon.awscdk.services.lambda.Code; import software.amazon.awscdk.services.lambda.Function; import software.amazon.awscdk.services.lambda.Runtime; @@ -51,7 +40,6 @@ import software.amazon.awscdk.services.secretsmanager.Secret; import software.amazon.awscdk.services.secretsmanager.SecretStringGenerator; import software.amazon.awscdk.services.ssm.CfnDocument; -import software.amazon.awscdk.CfnOutput; import software.constructs.Construct; import org.json.JSONObject; @@ -181,13 +169,23 @@ public VSCodeIde(final Construct scope, final String id, final VSCodeIdeProps pr props.getRole().addManagedPolicy(ManagedPolicy.fromAwsManagedPolicyName("AmazonSSMManagedInstanceCore")); var filePath = props.getAdditionalIamPolicyPath(); - if (Files.exists(Path.of(getClass().getResource(filePath).getPath()))) { - var policyDocumentJson = loadFile(filePath); - var policyDocument = PolicyDocument.fromJson(new JSONObject(policyDocumentJson).toMap()); - var policy = ManagedPolicy.Builder.create(this, "WorkshopIdeUserPolicy") - .document(policyDocument) - .build(); - props.getRole().addManagedPolicy(policy); + try { + var policyPath = Path.of(getClass().getResource(filePath).toURI()); + if (Files.exists(policyPath)) { + var jsonPolicy = loadFile(filePath); + // AccountId dynamisch einsetzen + String accountId = Stack.of(this).getAccount(); + String replaced = jsonPolicy.replace("{{.AccountId}}", accountId); + var policyDoc = new JSONObject(replaced).toMap(); + + CfnManagedPolicy.Builder.create(this, "WorkshopIdeUserPolicy") + .policyDocument(policyDoc) + .managedPolicyName("WorkshopIdeUserPolicy") + .roles(List.of(props.getRole().getRoleName())) + .build(); + } + } catch (Exception e) { + throw new RuntimeException("Failed to read or parse IAM policy file: " + filePath, e); } DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss"); diff --git a/infrastructure/cdk/src/main/java/com/unicorn/constructs/WorkshopVpc.java b/infrastructure/cdk/src/main/java/com/unicorn/constructs/WorkshopVpc.java index b3ba3fa5..3291f73d 100644 --- a/infrastructure/cdk/src/main/java/com/unicorn/constructs/WorkshopVpc.java +++ b/infrastructure/cdk/src/main/java/com/unicorn/constructs/WorkshopVpc.java @@ -23,6 +23,8 @@ public WorkshopVpc(final Construct scope, final String id, final String vpcName) vpc = Vpc.Builder.create(this, "Vpc") .vpcName(vpcName) .ipAddresses(IpAddresses.cidr("10.0.0.0/16")) + .enableDnsSupport(true) + .enableDnsHostnames(true) .maxAzs(2) // Use 2 Availability Zones .subnetConfiguration(Arrays.asList( SubnetConfiguration.builder() diff --git a/infrastructure/cdk/src/main/java/com/unicorn/core/AnalysisBucketConstruct.java b/infrastructure/cdk/src/main/java/com/unicorn/core/AnalysisBucketConstruct.java new file mode 100644 index 00000000..936da433 --- /dev/null +++ b/infrastructure/cdk/src/main/java/com/unicorn/core/AnalysisBucketConstruct.java @@ -0,0 +1,73 @@ +package com.unicorn.core; + +import software.amazon.awscdk.Duration; +import software.amazon.awscdk.RemovalPolicy; +import software.amazon.awscdk.services.iam.AnyPrincipal; +import software.amazon.awscdk.services.iam.Effect; +import software.amazon.awscdk.services.iam.PolicyStatement; +import software.amazon.awscdk.services.s3.*; +import software.constructs.Construct; + +import java.time.Instant; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +public class AnalysisBucketConstruct extends Construct { + private final Bucket bucket; + + public AnalysisBucketConstruct(final Construct scope, final String id, AnalysisBucketProps props) { + super(scope, id); + + // Safe timestamp format for S3 bucket name + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss") + .withZone(ZoneOffset.UTC); + String timestamp = formatter.format(Instant.now()); + + // Generate a lowercase, valid bucket name + String rawName = props.getBucketPrefix() + "-" + timestamp; + String bucketName = rawName.toLowerCase().replaceAll("[^a-z0-9.-]", ""); + bucketName = bucketName.replaceAll("[-.]+$", ""); // Remove trailing dashes/dots + + // Create the S3 bucket + this.bucket = Bucket.Builder.create(this, "AnalysisBucket") + .bucketName(bucketName) + .encryption(BucketEncryption.S3_MANAGED) + .versioned(props.isVersioningEnabled()) + .publicReadAccess(false) + .blockPublicAccess(BlockPublicAccess.BLOCK_ALL) + .removalPolicy(props.getRemovalPolicy()) + .lifecycleRules(List.of( + LifecycleRule.builder() + .enabled(true) + .expiration(Duration.days(props.getRetentionDays())) + .build(), + LifecycleRule.builder() + .enabled(true) + .noncurrentVersionExpiration(Duration.days(props.getNoncurrentVersionRetentionDays())) + .build() + )) + .build(); + + // Enforce SSL access with a bucket policy (CDK Nag: AwsSolutions-S10) + bucket.addToResourcePolicy(PolicyStatement.Builder.create() + .sid("EnforceSSLOnly") + .effect(Effect.DENY) + .principals(List.of(new AnyPrincipal())) + .actions(List.of("s3:*")) + .resources(List.of( + bucket.getBucketArn(), + bucket.getBucketArn() + "/*" + )) + .conditions(Map.of( + "Bool", Map.of("aws:SecureTransport", "false") + )) + .build()); + } + + public Bucket getBucket() { + return bucket; + } +} \ No newline at end of file diff --git a/infrastructure/cdk/src/main/java/com/unicorn/core/AnalysisBucketProps.java b/infrastructure/cdk/src/main/java/com/unicorn/core/AnalysisBucketProps.java new file mode 100644 index 00000000..00648703 --- /dev/null +++ b/infrastructure/cdk/src/main/java/com/unicorn/core/AnalysisBucketProps.java @@ -0,0 +1,82 @@ +package com.unicorn.core; + +// AnalysisBucketProps.java + +import software.amazon.awscdk.RemovalPolicy; + +public class AnalysisBucketProps { + private final String bucketPrefix; + private final boolean versioningEnabled; + private final int retentionDays; + private final int noncurrentVersionRetentionDays; + private final RemovalPolicy removalPolicy; + + private AnalysisBucketProps(Builder builder) { + this.bucketPrefix = builder.bucketPrefix; + this.versioningEnabled = builder.versioningEnabled; + this.retentionDays = builder.retentionDays; + this.noncurrentVersionRetentionDays = builder.noncurrentVersionRetentionDays; + this.removalPolicy = builder.removalPolicy; + } + + public String getBucketPrefix() { + return bucketPrefix; + } + + public boolean isVersioningEnabled() { + return versioningEnabled; + } + + public int getRetentionDays() { + return retentionDays; + } + + public int getNoncurrentVersionRetentionDays() { + return noncurrentVersionRetentionDays; + } + + public RemovalPolicy getRemovalPolicy() { + return removalPolicy; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private String bucketPrefix = "analysis"; + private boolean versioningEnabled = true; + private int retentionDays = 90; + private int noncurrentVersionRetentionDays = 30; + private RemovalPolicy removalPolicy = RemovalPolicy.RETAIN; + + public Builder bucketPrefix(String bucketPrefix) { + this.bucketPrefix = bucketPrefix; + return this; + } + + public Builder versioningEnabled(boolean versioningEnabled) { + this.versioningEnabled = versioningEnabled; + return this; + } + + public Builder retentionDays(int retentionDays) { + this.retentionDays = retentionDays; + return this; + } + + public Builder noncurrentVersionRetentionDays(int noncurrentVersionRetentionDays) { + this.noncurrentVersionRetentionDays = noncurrentVersionRetentionDays; + return this; + } + + public Builder removalPolicy(RemovalPolicy removalPolicy) { + this.removalPolicy = removalPolicy; + return this; + } + + public AnalysisBucketProps build() { + return new AnalysisBucketProps(this); + } + } +} diff --git a/infrastructure/cdk/src/main/java/com/unicorn/core/GrafanaDashboardConstruct.java b/infrastructure/cdk/src/main/java/com/unicorn/core/GrafanaDashboardConstruct.java new file mode 100644 index 00000000..721c9f1d --- /dev/null +++ b/infrastructure/cdk/src/main/java/com/unicorn/core/GrafanaDashboardConstruct.java @@ -0,0 +1,169 @@ +package com.unicorn.core; + +import software.amazon.awscdk.CfnOutput; +import software.amazon.awscdk.Duration; +import software.amazon.awscdk.CustomResource; +import software.amazon.awscdk.services.grafana.CfnWorkspace; +import software.amazon.awscdk.services.iam.*; +import software.amazon.awscdk.services.lambda.*; +import software.constructs.Construct; + +import java.util.List; +import java.util.Map; + +/** + * Construct to create and provision Grafana dashboards + */ +public class GrafanaDashboardConstruct extends Construct { + + public GrafanaDashboardConstruct(Construct scope, String id, + CfnWorkspace grafanaWorkspace, + software.amazon.awscdk.services.aps.CfnWorkspace ampWorkspace) { + super(scope, id); + + // Create a role for the Lambda function + Role lambdaRole = Role.Builder.create(this, "GrafanaDashboardLambdaRole") + .assumedBy(new ServicePrincipal("lambda.amazonaws.com")) + .managedPolicies(List.of( + ManagedPolicy.fromAwsManagedPolicyName("service-role/AWSLambdaBasicExecutionRole") + )) + .build(); + + // Permissions to Grafana workspace + lambdaRole.addToPolicy(PolicyStatement.Builder.create() + .effect(Effect.ALLOW) + .actions(List.of( + "grafana:CreateWorkspaceApiKey", + "grafana:DescribeWorkspace", + "grafana:UpdateDashboard", + "grafana:CreateDashboard" + )) + .resources(List.of(grafanaWorkspace.getAttrId())) + .build()); + + // Inline Python code for the Lambda + String lambdaCode = createLambdaCode(); + + // Create Lambda function + Function dashboardProvisionerFunction = Function.Builder.create(this, "GrafanaDashboardProvisioner") + .runtime(software.amazon.awscdk.services.lambda.Runtime.PYTHON_3_9) + .handler("index.handler") + .code(Code.fromInline(lambdaCode)) + .role(lambdaRole) + .timeout(Duration.seconds(300)) + .environment(Map.of( + "GRAFANA_WORKSPACE_ID", grafanaWorkspace.getAttrId(), + "AMP_WORKSPACE_ID", ampWorkspace.getAttrWorkspaceId(), + "DASHBOARD_JSON", createDashboardJson() + )) + .build(); + + // Custom resource to trigger the Lambda + CustomResource.Builder.create(this, "DashboardProvisionerResource") + .serviceToken(dashboardProvisionerFunction.getFunctionArn()) + .build(); + + // Output URL + CfnOutput.Builder.create(this, "GrafanaDashboardOutput") + .description("Grafana Dashboard URL") + .value("https://" + grafanaWorkspace.getAttrEndpoint() + "/d/unicornstore/unicorn-store-dashboard") + .exportName("GrafanaDashboardUrl") + .build(); + } + + private String createDashboardJson() { + return "{" + + "\\\"title\\\": \\\"Unicorn Store Dashboard\\\"," + + "\\\"uid\\\": \\\"unicornstore\\\"," + + "\\\"panels\\\": []" + + "}"; + // Replace this with your actual dashboard JSON string, escaped properly + } + + private String createLambdaCode() { + return """ + import boto3 + import json + import os + import time + import urllib3 + import logging + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + + http = urllib3.PoolManager() + + def handler(event, context): + logger.info('Event: %s', json.dumps(event)) + + grafana_workspace_id = os.environ['GRAFANA_WORKSPACE_ID'] + amp_workspace_id = os.environ['AMP_WORKSPACE_ID'] + dashboard_json = os.environ['DASHBOARD_JSON'] + + grafana_client = boto3.client('grafana') + workspace = grafana_client.describe_workspace(workspaceId=grafana_workspace_id) + grafana_endpoint = workspace['workspace']['endpoint'] + + api_key_response = grafana_client.create_workspace_api_key( + workspaceId=grafana_workspace_id, + keyName='dashboard-provisioner-' + str(int(time.time())), + keyRole='ADMIN', + secondsToLive=900 + ) + api_key = api_key_response['key'] + + datasource_payload = { + 'name': 'AMP', + 'type': 'prometheus', + 'access': 'proxy', + 'url': f'https://aps-workspaces.{os.environ["AWS_REGION"]}.amazonaws.com/workspaces/{amp_workspace_id}/', + 'jsonData': { + 'httpMethod': 'GET', + 'sigV4Auth': True, + 'sigV4AuthType': 'default', + 'sigV4Region': os.environ['AWS_REGION'] + }, + 'isDefault': True + } + + http.request( + 'POST', + f'https://{grafana_endpoint}/api/datasources', + body=json.dumps(datasource_payload), + headers={ + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {api_key}' + } + ) + + response = http.request( + 'GET', + f'https://{grafana_endpoint}/api/datasources/name/AMP', + headers={ + 'Authorization': f'Bearer {api_key}' + } + ) + datasource_data = json.loads(response.data.decode('utf-8')) + prometheus_uid = datasource_data['uid'] + + dashboard_data = json.loads(dashboard_json.replace('${prometheusUid}', prometheus_uid)) + + dashboard_payload = { + 'dashboard': dashboard_data, + 'overwrite': True + } + + http.request( + 'POST', + f'https://{grafana_endpoint}/api/dashboards/db', + body=json.dumps(dashboard_payload), + headers={ + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {api_key}' + } + ) + return {'statusCode': 200, 'body': 'Dashboard created'} + """; + } +} \ No newline at end of file diff --git a/infrastructure/cdk/src/main/java/com/unicorn/core/InfrastructureContainers.java b/infrastructure/cdk/src/main/java/com/unicorn/core/InfrastructureContainers.java index dbdeb2b2..b9876d59 100644 --- a/infrastructure/cdk/src/main/java/com/unicorn/core/InfrastructureContainers.java +++ b/infrastructure/cdk/src/main/java/com/unicorn/core/InfrastructureContainers.java @@ -21,7 +21,7 @@ public class InfrastructureContainers extends Construct { private final InfrastructureCore infrastructureCore; public InfrastructureContainers(final Construct scope, final String id, - final InfrastructureCore infrastructureCore) { + final InfrastructureCore infrastructureCore) { super(scope, id); // Get previously created infrastructure construct @@ -36,77 +36,68 @@ public InfrastructureContainers(final Construct scope, final String id, private Repository createUnicornStoreSpringEcr() { return Repository.Builder.create(this, "UnicornStoreSpringEcr") - .repositoryName("unicorn-store-spring") - .imageScanOnPush(false) - .removalPolicy(RemovalPolicy.DESTROY) - .emptyOnDelete(true) // This will force delete all images when repository is deleted - .build(); + .repositoryName("unicorn-store-spring") + .imageScanOnPush(false) + .removalPolicy(RemovalPolicy.DESTROY) + .emptyOnDelete(true) // This will force delete all images when repository is deleted + .build(); } private void createVpcConnector() { VpcConnector.Builder.create(this, "UnicornStoreVpcConnector") - .vpc(infrastructureCore.getVpc()) - .vpcSubnets(SubnetSelection.builder() - .subnetType(SubnetType.PRIVATE_WITH_EGRESS) - .build()) - .vpcConnectorName("unicornstore-vpc-connector") - .build(); + .vpc(infrastructureCore.getVpc()) + .vpcSubnets(SubnetSelection.builder() + .subnetType(SubnetType.PRIVATE_WITH_EGRESS) + .build()) + .vpcConnectorName("unicornstore-vpc-connector") + .build(); } private void createRolesAppRunner() { var unicornStoreApprunnerRole = Role.Builder.create(this, "UnicornStoreApprunnerRole") - .roleName("unicornstore-apprunner-role") - .assumedBy(new ServicePrincipal("tasks.apprunner.amazonaws.com")).build(); + .roleName("unicornstore-apprunner-role") + .assumedBy(new ServicePrincipal("tasks.apprunner.amazonaws.com")).build(); unicornStoreApprunnerRole.addToPolicy(PolicyStatement.Builder.create() - .actions(List.of("xray:PutTraceSegments")) - .resources(List.of("*")) - .build()); + .actions(List.of("xray:PutTraceSegments")) + .resources(List.of("*")) + .build()); infrastructureCore.getEventBridge().grantPutEventsTo(unicornStoreApprunnerRole); infrastructureCore.getDatabaseSecret().grantRead(unicornStoreApprunnerRole); infrastructureCore.getSecretPassword().grantRead(unicornStoreApprunnerRole); infrastructureCore.getParamDBConnectionString().grantRead(unicornStoreApprunnerRole); var appRunnerECRAccessRole = Role.Builder.create(this, "UnicornStoreApprunnerEcrAccessRole") - .roleName("unicornstore-apprunner-ecr-access-role") - .assumedBy(new ServicePrincipal("build.apprunner.amazonaws.com")).build(); + .roleName("unicornstore-apprunner-ecr-access-role") + .assumedBy(new ServicePrincipal("build.apprunner.amazonaws.com")).build(); appRunnerECRAccessRole.addManagedPolicy(ManagedPolicy.fromManagedPolicyArn(this, - "UnicornStoreApprunnerEcrAccessRole-" + "AWSAppRunnerServicePolicyForECRAccess", - "arn:aws:iam::aws:policy/service-role/AWSAppRunnerServicePolicyForECRAccess")); - - // // Create the App Runner service-linked role - // Long tsLong = System.currentTimeMillis()/1000; - // String timestamp = tsLong.toString(); - // CfnServiceLinkedRole appRunnerServiceLinkedRole = CfnServiceLinkedRole.Builder.create(this, "AppRunnerServiceLinkedRole") - // .awsServiceName("apprunner.amazonaws.com") - // .description("Service-linked role for AWS App Runner service") - // .customSuffix(timestamp) - // .build(); + "UnicornStoreApprunnerEcrAccessRole-" + "AWSAppRunnerServicePolicyForECRAccess", + "arn:aws:iam::aws:policy/service-role/AWSAppRunnerServicePolicyForECRAccess")); } private void createRolesEcs() { var AWSOpenTelemetryPolicy = PolicyStatement.Builder.create() - .effect(Effect.ALLOW) - .actions(List.of("logs:PutLogEvents", "logs:CreateLogGroup", "logs:CreateLogStream", - "logs:DescribeLogStreams", "logs:DescribeLogGroups", - "logs:PutRetentionPolicy", "xray:PutTraceSegments", - "xray:PutTelemetryRecords", "xray:GetSamplingRules", - "xray:GetSamplingTargets", "xray:GetSamplingStatisticSummaries", - "cloudwatch:PutMetricData", "ssm:GetParameters")) - .resources(List.of("*")).build(); + .effect(Effect.ALLOW) + .actions(List.of("logs:PutLogEvents", "logs:CreateLogGroup", "logs:CreateLogStream", + "logs:DescribeLogStreams", "logs:DescribeLogGroups", + "logs:PutRetentionPolicy", "xray:PutTraceSegments", + "xray:PutTelemetryRecords", "xray:GetSamplingRules", + "xray:GetSamplingTargets", "xray:GetSamplingStatisticSummaries", + "cloudwatch:PutMetricData", "ssm:GetParameters")) + .resources(List.of("*")).build(); var unicornStoreEscTaskRole = Role.Builder.create(this, "UnicornStoreEcsTaskRole") - .roleName("unicornstore-ecs-task-role") - .assumedBy(new ServicePrincipal("ecs-tasks.amazonaws.com")).build(); + .roleName("unicornstore-ecs-task-role") + .assumedBy(new ServicePrincipal("ecs-tasks.amazonaws.com")).build(); unicornStoreEscTaskRole.addToPolicy(PolicyStatement.Builder.create() - .actions(List.of("xray:PutTraceSegments")) - .resources(List.of("*")) - .build()); + .actions(List.of("xray:PutTraceSegments")) + .resources(List.of("*")) + .build()); unicornStoreEscTaskRole.addManagedPolicy(ManagedPolicy.fromManagedPolicyArn(this, - "UnicornStoreEcsTaskRole-" + "CloudWatchLogsFullAccess", - "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess")); + "UnicornStoreEcsTaskRole-" + "CloudWatchLogsFullAccess", + "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess")); unicornStoreEscTaskRole.addManagedPolicy(ManagedPolicy.fromManagedPolicyArn(this, - "UnicornStoreEcsTaskRole-" + "AmazonSSMReadOnlyAccess", - "arn:aws:iam::aws:policy/AmazonSSMReadOnlyAccess")); + "UnicornStoreEcsTaskRole-" + "AmazonSSMReadOnlyAccess", + "arn:aws:iam::aws:policy/AmazonSSMReadOnlyAccess")); unicornStoreEscTaskRole.addToPolicy(AWSOpenTelemetryPolicy); infrastructureCore.getEventBridge().grantPutEventsTo(unicornStoreEscTaskRole); @@ -115,21 +106,21 @@ private void createRolesEcs() { infrastructureCore.getParamDBConnectionString().grantRead(unicornStoreEscTaskRole); Role unicornStoreEscTaskExecutionRole = Role.Builder.create(this, "UnicornStoreEcsTaskExecutionRole") - .roleName("unicornstore-ecs-task-execution-role") - .assumedBy(new ServicePrincipal("ecs-tasks.amazonaws.com")).build(); + .roleName("unicornstore-ecs-task-execution-role") + .assumedBy(new ServicePrincipal("ecs-tasks.amazonaws.com")).build(); unicornStoreEscTaskExecutionRole.addToPolicy(PolicyStatement.Builder.create() - .actions(List.of("logs:CreateLogGroup")) - .resources(List.of("*")) - .build()); + .actions(List.of("logs:CreateLogGroup")) + .resources(List.of("*")) + .build()); unicornStoreEscTaskExecutionRole.addManagedPolicy(ManagedPolicy.fromManagedPolicyArn(this, - "UnicornStoreEcsTaskExecutionRole-" + "AmazonECSTaskExecutionRolePolicy", - "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy")); + "UnicornStoreEcsTaskExecutionRole-" + "AmazonECSTaskExecutionRolePolicy", + "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy")); unicornStoreEscTaskExecutionRole.addManagedPolicy(ManagedPolicy.fromManagedPolicyArn(this, - "UnicornStoreEcsTaskExecutionRole-" + "CloudWatchLogsFullAccess", - "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess")); + "UnicornStoreEcsTaskExecutionRole-" + "CloudWatchLogsFullAccess", + "arn:aws:iam::aws:policy/CloudWatchLogsFullAccess")); unicornStoreEscTaskExecutionRole.addManagedPolicy(ManagedPolicy.fromManagedPolicyArn(this, - "UnicornStoreEcsTaskExecutionRole-" + "AmazonSSMReadOnlyAccess", - "arn:aws:iam::aws:policy/AmazonSSMReadOnlyAccess")); + "UnicornStoreEcsTaskExecutionRole-" + "AmazonSSMReadOnlyAccess", + "arn:aws:iam::aws:policy/AmazonSSMReadOnlyAccess")); unicornStoreEscTaskExecutionRole.addToPolicy(AWSOpenTelemetryPolicy); infrastructureCore.getEventBridge().grantPutEventsTo(unicornStoreEscTaskExecutionRole); @@ -140,55 +131,68 @@ private void createRolesEcs() { private void createRolesEks() { ServicePrincipal eksPods = new ServicePrincipal("pods.eks.amazonaws.com"); - // EKS Pod Identity role var unicornStoreEksPodRole = Role.Builder.create(this, "UnicornStoreEksPodRole") - .roleName("unicornstore-eks-pod-role") - .assumedBy(eksPods.withSessionTags()) - .build(); + .roleName("unicornstore-eks-pod-role") + .assumedBy(eksPods.withSessionTags()) + .build(); unicornStoreEksPodRole.addToPolicy(PolicyStatement.Builder.create() - .actions(List.of("xray:PutTraceSegments")) - .resources(List.of("*")) - .build()); + .actions(List.of("xray:PutTraceSegments")) + .resources(List.of("*")) + .build()); unicornStoreEksPodRole.addManagedPolicy(ManagedPolicy.fromManagedPolicyArn(this, - "UnicornStoreEksPodRole-" + "CloudWatchAgentServerPolicy", - "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy")); + "UnicornStoreEksPodRole-" + "CloudWatchAgentServerPolicy", + "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy")); + + unicornStoreEksPodRole.addToPolicy(PolicyStatement.Builder.create() + .effect(Effect.ALLOW) + .actions(List.of( + "ecs:ListTasks", + "ecs:DescribeTasks", + "ecs:ListServices", + "ecs:DescribeServices", + "ecs:ListClusters", + "ecs:DescribeClusters", + "ecs:ListContainerInstances", + "ecs:DescribeContainerInstances" + )) + .resources(List.of("*")) + .build()); infrastructureCore.getEventBridge().grantPutEventsTo(unicornStoreEksPodRole); infrastructureCore.getDatabaseSecret().grantRead(unicornStoreEksPodRole); infrastructureCore.getParamDBConnectionString().grantRead(unicornStoreEksPodRole); var dbSecretPolicy = ManagedPolicy.Builder.create(this, "UnicornStoreDbSecretsManagerPolicy") - .managedPolicyName("unicornstore-db-secret-policy") - .statements(List.of( - PolicyStatement.Builder.create() - .effect(Effect.ALLOW) - .actions(List.of("secretsmanager:ListSecrets")) - .resources(List.of("*")) - .build(), - PolicyStatement.Builder.create() - .effect(Effect.ALLOW) - .actions(List.of( - "secretsmanager:GetResourcePolicy", - "secretsmanager:DescribeSecret", - "secretsmanager:GetSecretValue", - "secretsmanager:ListSecretVersionIds" - )) - .resources(List.of(infrastructureCore.getDatabaseSecret().getSecretFullArn())) - .build() - )) - .build(); - - // External Secrets Operator roles + .managedPolicyName("unicornstore-db-secret-policy") + .statements(List.of( + PolicyStatement.Builder.create() + .effect(Effect.ALLOW) + .actions(List.of("secretsmanager:ListSecrets")) + .resources(List.of("*")) + .build(), + PolicyStatement.Builder.create() + .effect(Effect.ALLOW) + .actions(List.of( + "secretsmanager:GetResourcePolicy", + "secretsmanager:DescribeSecret", + "secretsmanager:GetSecretValue", + "secretsmanager:ListSecretVersionIds" + )) + .resources(List.of(infrastructureCore.getDatabaseSecret().getSecretFullArn())) + .build() + )) + .build(); + Role unicornStoreEksEsoRole = Role.Builder.create(this, "UnicornStoreEksEsoRole") - .roleName("unicornstore-eks-eso-role") - .assumedBy(eksPods.withSessionTags()) - .build(); + .roleName("unicornstore-eks-eso-role") + .assumedBy(eksPods.withSessionTags()) + .build(); ArnPrincipal unicornStoreEksEsoRolePrincipal = new ArnPrincipal(unicornStoreEksEsoRole.getRoleArn()); Role unicornStoreEksEsoSmRole = Role.Builder.create(this, "UnicornStoreEksEsoSmRole") - .roleName("unicornstore-eks-eso-sm-role") - .assumedBy(unicornStoreEksEsoRolePrincipal.withSessionTags()) - .build(); + .roleName("unicornstore-eks-eso-sm-role") + .assumedBy(unicornStoreEksEsoRolePrincipal.withSessionTags()) + .build(); unicornStoreEksEsoSmRole.addManagedPolicy(dbSecretPolicy); } -} +} \ No newline at end of file diff --git a/infrastructure/cdk/src/main/java/com/unicorn/core/InfrastructureLambdaBedrock.java b/infrastructure/cdk/src/main/java/com/unicorn/core/InfrastructureLambdaBedrock.java new file mode 100644 index 00000000..f700f5bc --- /dev/null +++ b/infrastructure/cdk/src/main/java/com/unicorn/core/InfrastructureLambdaBedrock.java @@ -0,0 +1,242 @@ +package com.unicorn.core; + +import com.unicorn.constructs.EksCluster; +import software.amazon.awscdk.*; +import software.amazon.awscdk.services.ec2.*; +import software.amazon.awscdk.services.iam.*; +import software.amazon.awscdk.services.lambda.Code; +import software.amazon.awscdk.services.lambda.Function; +import software.amazon.awscdk.services.lambda.Runtime; +import software.amazon.awscdk.services.logs.LogGroup; +import software.amazon.awscdk.services.logs.RetentionDays; +import software.amazon.awscdk.services.s3.Bucket; + +import software.constructs.Construct; +import software.amazon.awscdk.services.lambda.FunctionUrl; +import software.amazon.awscdk.services.lambda.FunctionUrlAuthType; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class InfrastructureLambdaBedrock extends Construct { + + private final Function threadDumpFunction; + + public InfrastructureLambdaBedrock(Construct scope, String id, String region, Bucket s3Bucket, EksCluster eksCluster, IVpc vpc) { + super(scope, id); + + // Create a security group for the Lambda function + SecurityGroup lambdaSg = SecurityGroup.Builder.create(this, "LambdaSecurityGroup") + .vpc(vpc) + .securityGroupName("unicornstore-thread-dump-lambda-sg") + .description("Security group for Thread Dump Lambda function") + .allowAllOutbound(true) + .build(); + + ISecurityGroup clusterSG = eksCluster.getClusterSecurityGroup(); + + // Allow Lambda to communicate with EKS API server + // The Kubernetes API typically runs on port 443 (HTTPS) + clusterSG.addIngressRule( + Peer.securityGroupId(lambdaSg.getSecurityGroupId()), + Port.tcp(443), + "Allow Lambda access to Kubernetes API" + ); + + // Allow Lambda to reach EKS cluster + lambdaSg.addEgressRule( + Peer.securityGroupId(clusterSG.getSecurityGroupId()), + Port.tcp(443), + "Allow Lambda to reach Kubernetes API" + ); + + // Allow EKS cluster to respond back to Lambda + clusterSG.addIngressRule( + Peer.securityGroupId(lambdaSg.getSecurityGroupId()), + Port.tcp(443), + "Allow Lambda access to Kubernetes API" + ); + + // IAM Role for Bedrock + Role bedrockRole = Role.Builder.create(this, "BedrockAccessRole") + .assumedBy(new ServicePrincipal("bedrock.amazonaws.com")) + .description("Role for Bedrock Claude 3.7 access") + .build(); + + bedrockRole.addToPolicy(PolicyStatement.Builder.create() + .effect(Effect.ALLOW) + .actions(List.of("bedrock:InvokeModel", "bedrock:ListFoundationModels")) + .resources(List.of("arn:aws:bedrock:*:*:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0")) + .build()); + + // IAM Role for Lambda + Role lambdaRole = Role.Builder.create(this, "LambdaBedrockRole") + .assumedBy(new ServicePrincipal("lambda.amazonaws.com")) + .description("Role for Lambda to access Bedrock and EKS") + .managedPolicies(List.of( + ManagedPolicy.fromAwsManagedPolicyName("service-role/AWSLambdaBasicExecutionRole"), + // Add VPC access policy + ManagedPolicy.fromAwsManagedPolicyName("service-role/AWSLambdaVPCAccessExecutionRole") + )) + .build(); + + // Add permissions for Bedrock, S3, and SNS + // Add permissions for logs + lambdaRole.addToPolicy(PolicyStatement.Builder.create() + .effect(Effect.ALLOW) + .actions(List.of( + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + )) + .resources(List.of("arn:aws:logs:*:*:*")) + .build()); + + // Add broader permissions for Bedrock + lambdaRole.addToPolicy(PolicyStatement.Builder.create() + .effect(Effect.ALLOW) + .actions(List.of( + "bedrock:InvokeModel", + "bedrock:InvokeModelWithResponseStream", + "bedrock:ListFoundationModels" + )) + .resources(List.of("*")) // Grant access to all Bedrock resources + .build()); + + // Add permissions for AWS Secrets Manager access + lambdaRole.addToPolicy(PolicyStatement.Builder.create() + .effect(Effect.ALLOW) + .actions(List.of( + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret" + )) + .resources(List.of( + // Allow access to the specific secret for webhook credentials + String.format("arn:aws:secretsmanager:%s:*:secret:grafana-webhook-credentials*", region) + )) + .build()); + + // Add separate policy for S3 and SNS + lambdaRole.addToPolicy(PolicyStatement.Builder.create() + .effect(Effect.ALLOW) + .actions(List.of( + "s3:PutObject", "s3:GetObject", "s3:ListBucket", + "sns:Publish" + )) + .resources(List.of( + String.format("arn:aws:s3:::%s/*", s3Bucket.getBucketName()), + String.format("arn:aws:s3:::%s", s3Bucket.getBucketName()), + "arn:aws:sns:*:*:*" + )) + .build()); + + // Add permissions for EKS API access + lambdaRole.addToPolicy(PolicyStatement.Builder.create() + .effect(Effect.ALLOW) + .actions(List.of( + "eks:DescribeCluster", + "eks:AccessKubernetesApi", + "eks:ListClusters", + "sts:GetCallerIdentity" + )) + .resources(List.of("*")) + .build()); + + // Lambda function definition with inline dummy code + this.threadDumpFunction = Function.Builder.create(this, "unicornstore-thread-dump-lambda-eks") + .functionName("unicornstore-thread-dump-lambda") + .runtime(Runtime.PYTHON_3_13) + .code(Code.fromInline( + "import json\n" + + "import logging\n" + + "\n" + + "logger = logging.getLogger()\n" + + "logger.setLevel(logging.INFO)\n" + + "\n" + + "def lambda_handler(event, context):\n" + + " logger.info('Dummy Lambda function invoked')\n" + + " logger.info(f'Event: {json.dumps(event)}')\n" + + " \n" + + " return {\n" + + " 'statusCode': 200,\n" + + " 'headers': {\n" + + " 'Content-Type': 'application/json',\n" + + " 'Access-Control-Allow-Origin': '*'\n" + + " },\n" + + " 'body': json.dumps({\n" + + " 'message': 'Hello from dummy Lambda function!',\n" + + " 'timestamp': context.aws_request_id,\n" + + " 'function_name': context.function_name\n" + + " })\n" + + " }\n" + )) + .handler("index.lambda_handler") + .role(lambdaRole) + .timeout(Duration.minutes(5)) + .memorySize(512) + // Add VPC configuration - use private subnets with NAT + .vpc(vpc) + .vpcSubnets(SubnetSelection.builder() + .subnetType(SubnetType.PRIVATE_WITH_EGRESS) + .build()) + .securityGroups(List.of(lambdaSg)) + .environment(Map.of( + "APP_LABEL", "unicorn-store-spring", + "EKS_CLUSTER_NAME", Objects.requireNonNull(eksCluster.getCluster().getName()), + "K8S_NAMESPACE", "unicorn-store-spring", + "S3_BUCKET_NAME", s3Bucket.getBucketName(), + "KUBERNETES_AUTH_TYPE", "aws" // Use AWS IAM authentication for EKS + )) + .build(); + + s3Bucket.grantWrite(this.threadDumpFunction); + s3Bucket.grantRead(this.threadDumpFunction); + + // Create Log Group with retention + LogGroup.Builder.create(this, "ThreadDumpLogGroup") + .logGroupName("/aws/lambda/unicornstore-thread-dump-lambda") + .retention(RetentionDays.ONE_WEEK) + .removalPolicy(RemovalPolicy.DESTROY) + .build(); + + // Create Function URL for Grafana webhook integration + FunctionUrl functionUrl = FunctionUrl.Builder.create(this, "ThreadDumpFunctionUrl") + .function(this.threadDumpFunction) + .authType(FunctionUrlAuthType.NONE) + .build(); + + // Add resource-based policy to allow public access to Function URL + this.threadDumpFunction.addPermission("AllowPublicInvocation", + software.amazon.awscdk.services.lambda.Permission.builder() + .principal(new AnyPrincipal()) + .action("lambda:InvokeFunctionUrl") + .functionUrlAuthType(FunctionUrlAuthType.NONE) + .build()); + + // Use EKS Access Entries API instead of aws-auth ConfigMap + if (eksCluster != null) { + // Create an access entry for the Lambda role + eksCluster.createAccessEntry(lambdaRole.getRoleArn(), eksCluster.getCluster().getName(), "lambda-eks-acces-role"); + } + + // Output the Function URL for reference + CfnOutput.Builder.create(this, "ThreadDumpFunctionUrlOutput") + .exportName("ThreadDumpFunctionUrl") + .description("URL for invoking the Thread Dump Lambda function") + .value(functionUrl.getUrl()) + .build(); + + // Output the Lambda security group ID for reference + CfnOutput.Builder.create(this, "LambdaSecurityGroupOutput") + .exportName("LambdaSecurityGroupId") + .description("Security Group ID for the Lambda function") + .value(lambdaSg.getSecurityGroupId()) + .build(); + + } + + public Function getThreadDumpFunction() { + return threadDumpFunction; + } +} \ No newline at end of file diff --git a/infrastructure/cdk/src/main/java/com/unicorn/core/MonitoringConstruct.java b/infrastructure/cdk/src/main/java/com/unicorn/core/MonitoringConstruct.java new file mode 100644 index 00000000..fe72c8ff --- /dev/null +++ b/infrastructure/cdk/src/main/java/com/unicorn/core/MonitoringConstruct.java @@ -0,0 +1,64 @@ +package com.unicorn.core; + +import software.amazon.awscdk.CfnOutput; +import software.amazon.awscdk.services.ec2.IVpc; +import software.amazon.awscdk.services.iam.*; +import software.amazon.awscdk.services.lambda.*; +import software.amazon.awscdk.services.sns.Topic; +import software.amazon.awscdk.services.sns.TopicPolicy; +import software.amazon.awscdk.services.sns.subscriptions.LambdaSubscription; +import software.amazon.awscdk.services.eks.CfnCluster; +import software.constructs.Construct; + +import java.util.List; +import java.util.Map; + +public class MonitoringConstruct extends Construct { + + private final Topic alarmTopic; + private String prometheusInternalUrl; + + public MonitoringConstruct(Construct scope, String id, IVpc vpc, CfnCluster eksCluster, Function alertHandlerLambda) { + super(scope, id); + + alarmTopic = Topic.Builder.create(this, "AlarmTopic") + .topicName("UnicornStoreAlarms") + .displayName("Unicorn Store Alarms") + .build(); + + TopicPolicy.Builder.create(this, "AlarmTopicPolicy") + .topics(List.of(alarmTopic)) + .build() + .getDocument() + .addStatements(PolicyStatement.Builder.create() + .effect(Effect.DENY) + .actions(List.of("sns:Publish")) + .principals(List.of(new AnyPrincipal())) + .resources(List.of(alarmTopic.getTopicArn())) + .conditions(Map.of("Bool", Map.of("aws:SecureTransport", "false"))) + .build()); + + alarmTopic.addSubscription(new LambdaSubscription(alertHandlerLambda)); + + prometheusInternalUrl = "http://prometheus-server.monitoring.svc.cluster.local"; + + CfnOutput.Builder.create(this, "PrometheusInternalUrl") + .description("Prometheus internal service URL (for ECS use)") + .value(prometheusInternalUrl) + .exportName("PrometheusInternalUrl") + .build(); + + CfnOutput.Builder.create(this, "AlarmTopicARN") + .value(alarmTopic.getTopicArn()) + .exportName("AlarmTopicArn") + .build(); + } + + public Topic getAlarmTopic() { + return alarmTopic; + } + + public String getPrometheusInternalUrl() { + return prometheusInternalUrl; + } +} diff --git a/infrastructure/cdk/src/main/resources/iam-policy.json b/infrastructure/cdk/src/main/resources/iam-policy.json index 344b62db..57fea8e3 100644 --- a/infrastructure/cdk/src/main/resources/iam-policy.json +++ b/infrastructure/cdk/src/main/resources/iam-policy.json @@ -9,7 +9,7 @@ "Resource": "*", "Condition": { "Null": { - "aws-marketplace:ProductId": "false" + "aws-marketplace:ProductId": false }, "ForAllValues:StringEquals": { "aws-marketplace:ProductId": [ @@ -98,7 +98,9 @@ "logs:*", "lambda:*", "dynamodb:*", + "route53:*", "s3:*", + "servicediscovery:*", "tag:*", "application-signals:*" ], diff --git a/infrastructure/cfn/unicornstore-stack.assets.yaml b/infrastructure/cfn/unicornstore-stack.assets.yaml new file mode 100644 index 00000000..21452ab0 --- /dev/null +++ b/infrastructure/cfn/unicornstore-stack.assets.yaml @@ -0,0 +1,34 @@ +{ + "version": "41.0.0", + "files": { + "ca11027cde47a37b71c83bdbc52be2b39152c2dbd444a9cc0fe1197531e58d8e": { + "displayName": "InfrastructureLambdaBedrock/unicornstore-thread-dump-lambda-eks/Code", + "source": { + "path": "asset.ca11027cde47a37b71c83bdbc52be2b39152c2dbd444a9cc0fe1197531e58d8e.zip", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "ca11027cde47a37b71c83bdbc52be2b39152c2dbd444a9cc0fe1197531e58d8e.zip", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "14ee2c363e6e41697c04b74e233835d01563e61ac7568255cd4eec4e5cb8f783": { + "displayName": "unicornstore-stack Template", + "source": { + "path": "unicornstore-stack.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "14ee2c363e6e41697c04b74e233835d01563e61ac7568255cd4eec4e5cb8f783.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": {} +} diff --git a/infrastructure/cfn/unicornstore-stack.yaml b/infrastructure/cfn/unicornstore-stack.yaml index 5444b4ad..f6a2d4e8 100644 --- a/infrastructure/cfn/unicornstore-stack.yaml +++ b/infrastructure/cfn/unicornstore-stack.yaml @@ -1023,53 +1023,719 @@ Resources: sudo -u ec2-user bash -c 'touch ~/.local/share/code-server/User/settings.json' tee /home/ec2-user/.local/share/code-server/User/settings.json < { - var responseBody = JSON.stringify({ - Status: responseStatus, - Reason: "See the details in CloudWatch Log Stream: " + context.logGroupName + " " + context.logStreamName, - PhysicalResourceId: physicalResourceId || context.logStreamName, - StackId: event.StackId, - RequestId: event.RequestId, - LogicalResourceId: event.LogicalResourceId, - NoEcho: noEcho || false, - Data: responseData - }); - - console.log("Response body:\\n", responseBody); - - var https = require("https"); - var url = require("url"); - - var parsedUrl = url.parse(event.ResponseURL); - var options = { - hostname: parsedUrl.hostname, - port: 443, - path: parsedUrl.path, - method: "PUT", - headers: { - "content-type": "", - "content-length": responseBody.length + }, + { + "Fn::Select": [ + 1, + { + "Fn::Split": [ + "-", + { + "Fn::Select": [ + 6, + { + "Fn::Split": [ + ":", + { + "Ref": "UnicornStoreIdeIdePasswordSecret514252E2" + } + ] + } + ] + } + ] } - }; - - var request = https.request(options, function(response) { - console.log("Status code: " + response.statusCode); - console.log("Status message: " + response.statusMessage); - resolve(); - }); - - request.on("error", function(error) { - console.log("respond(..) failed executing https.request(..): " + error); - resolve(); - }); - - request.write(responseBody); - request.end(); - }); - }; - const { CodeBuildClient, BatchGetBuildsCommand } = require("@aws-sdk/client-codebuild"); - - exports.handler = async function (event, context) { - console.log(JSON.stringify(event, null, 4)); - - const projectName = event['detail']['project-name']; - - const codebuild = new CodeBuildClient(); - - const buildId = event['detail']['build-id']; - const { builds } = await codebuild.send(new BatchGetBuildsCommand({ - ids: [ buildId ] - })); - - console.log(JSON.stringify(builds, null, 4)); - - const build = builds[0]; - // Fetch the CFN resource and response parameters from the build environment. - const environment = {}; - build.environment.environmentVariables.forEach(e => environment[e.name] = e.value); - - const response = { - ResponseURL: environment.CFN_RESPONSE_URL, - StackId: environment.CFN_STACK_ID, - LogicalResourceId: environment.CFN_LOGICAL_RESOURCE_ID, - RequestId: environment.CFN_REQUEST_ID - }; - - if (event['detail']['build-status'] === 'SUCCEEDED') { - await respond(response, context, 'SUCCESS', {}, 'build'); - } else { - await respond(response, context, 'FAILED', { Error: 'Build failed' }); + ] + }, + { + "Fn::Select": [ + 2, + { + "Fn::Split": [ + "-", + { + "Fn::Select": [ + 6, + { + "Fn::Split": [ + ":", + { + "Ref": "UnicornStoreIdeIdePasswordSecret514252E2" + } + ] + } + ] + } + ] + } + ] + }, + { + "Fn::Select": [ + 3, + { + "Fn::Split": [ + "-", + { + "Fn::Select": [ + 6, + { + "Fn::Split": [ + ":", + { + "Ref": "UnicornStoreIdeIdePasswordSecret514252E2" + } + ] + } + ] + } + ] + } + ] + } + ] + ] + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "aws:cdk:path": "unicornstore-stack/UnicornStoreIde/IdePasswordExporter/Default" + } + }, + "UnicornStoreIdeIdeBootstrapDocumentE330810B": { + "Type": "AWS::SSM::Document", + "Properties": { + "Content": { + "schemaVersion": "2.2", + "description": "Bootstrap IDE", + "parameters": { + "BootstrapScript": { + "default": "", + "description": "(Optional) Custom bootstrap script to run.", + "type": "String" } }; FunctionName: unicornstore-codebuild-report-lambda diff --git a/infrastructure/lambda/build-and-deploy.sh b/infrastructure/lambda/build-and-deploy.sh new file mode 100755 index 00000000..03fa9372 --- /dev/null +++ b/infrastructure/lambda/build-and-deploy.sh @@ -0,0 +1,236 @@ +#!/bin/bash +# build-and-deploy.sh - Build and deploy Lambda function using virtualenv + +# Exit on any error +set -e + +# Configuration +FUNCTION_NAME="unicornstore-thread-dump-lambda" +PACKAGE_NAME="lambda_function" +PYTHON_VERSION="3.13" +TEMP_DIR="build_temp" +DIST_DIR="dist" +OUTPUT_ZIP="$DIST_DIR/$PACKAGE_NAME.zip" +REGION="${AWS_DEFAULT_REGION:-us-east-1}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Print colored output +print_status() { + echo -e "${BLUE}=== $1 ===${NC}" +} + +print_success() { + echo -e "${GREEN}โœ… $1${NC}" +} + +print_warning() { + echo -e "${YELLOW}โš ๏ธ $1${NC}" +} + +print_error() { + echo -e "${RED}โŒ $1${NC}" +} + +# Check if AWS CLI is configured +check_aws_config() { + if ! aws sts get-caller-identity &>/dev/null; then + print_error "AWS CLI is not configured or credentials are invalid" + echo "Please run 'aws configure' or set AWS environment variables" + exit 1 + fi + + local account_id=$(aws sts get-caller-identity --query Account --output text) + local current_region=$(aws configure get region || echo "not set") + print_success "AWS configured - Account: $account_id, Region: $current_region" +} + +# Check if Python 3.13 is available +check_python() { + if ! command -v python3 &> /dev/null; then + print_error "Python 3 is not installed" + exit 1 + fi + + local python_version=$(python3 --version) + print_success "Python available: $python_version" +} + +# Build the Lambda package +build_package() { + print_status "Building Lambda Deployment Package" + echo "Function name: $FUNCTION_NAME" + echo "Python version: $PYTHON_VERSION" + echo "Target region: $REGION" + + # Create clean directories + print_status "Creating clean build directories" + rm -rf "$TEMP_DIR" "$DIST_DIR" + mkdir -p "$TEMP_DIR/package" "$DIST_DIR" + + # Create and activate virtual environment + print_status "Creating virtual environment" + python3 -m venv "$TEMP_DIR/venv" + source "$TEMP_DIR/venv/bin/activate" + + # Upgrade pip + print_status "Upgrading pip" + pip install --upgrade pip --quiet + + # Install dependencies if requirements.txt exists + if [ -f "requirements.txt" ]; then + print_status "Installing dependencies" + pip install -r requirements.txt --target "$TEMP_DIR/package" --quiet + print_success "Dependencies installed" + else + print_warning "No requirements.txt found, skipping dependency installation" + fi + + # Copy function code + print_status "Copying function code" + if [ -d "src" ]; then + cp src/*.py "$TEMP_DIR/package/" 2>/dev/null || true + print_success "Source files copied" + else + print_warning "No src directory found" + fi + + # Create deployment package + print_status "Creating deployment package" + cd "$TEMP_DIR/package" + zip -r "../../$OUTPUT_ZIP" . > /dev/null + cd - > /dev/null + + # Calculate package size + local package_size=$(du -h "$OUTPUT_ZIP" | cut -f1) + print_success "Package created: $OUTPUT_ZIP ($package_size)" + + # Clean up virtual environment + deactivate + rm -rf "$TEMP_DIR" +} + +# Deploy to AWS Lambda +deploy_function() { + print_status "Deploying to AWS Lambda" + + # Check if function exists + if aws lambda get-function --function-name "$FUNCTION_NAME" --region "$REGION" &>/dev/null; then + print_status "Updating existing function code" + aws lambda update-function-code \ + --function-name "$FUNCTION_NAME" \ + --zip-file "fileb://$OUTPUT_ZIP" \ + --region "$REGION" \ + --output table + print_success "Function code updated successfully" + else + print_error "Function '$FUNCTION_NAME' does not exist" + echo "Please deploy the CDK stack first to create the function" + exit 1 + fi + + # Wait for function to be updated + print_status "Waiting for function update to complete" + aws lambda wait function-updated \ + --function-name "$FUNCTION_NAME" \ + --region "$REGION" + print_success "Function update completed" + + # Get function info + print_status "Function Information" + aws lambda get-function \ + --function-name "$FUNCTION_NAME" \ + --region "$REGION" \ + --query '{FunctionName:Configuration.FunctionName,Runtime:Configuration.Runtime,Handler:Configuration.Handler,CodeSize:Configuration.CodeSize,LastModified:Configuration.LastModified}' \ + --output table +} + +# Test the deployed function +test_function() { + print_status "Testing deployed function" + + local test_payload='{"test": true, "message": "Hello from build script"}' + + aws lambda invoke \ + --function-name "$FUNCTION_NAME" \ + --region "$REGION" \ + --payload "$test_payload" \ + --cli-binary-format raw-in-base64-out \ + response.json + + if [ -f "response.json" ]; then + print_success "Function test completed" + echo "Response:" + cat response.json | python3 -m json.tool + rm -f response.json + fi +} + +# Main execution +main() { + print_status "Lambda Build and Deploy Script" + + # Parse command line arguments + BUILD_ONLY=false + SKIP_TEST=false + + while [[ $# -gt 0 ]]; do + case $1 in + --build-only) + BUILD_ONLY=true + shift + ;; + --skip-test) + SKIP_TEST=true + shift + ;; + --region) + REGION="$2" + shift 2 + ;; + --help) + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --build-only Only build the package, don't deploy" + echo " --skip-test Skip function testing after deployment" + echo " --region REGION Set AWS region (default: $REGION)" + echo " --help Show this help message" + exit 0 + ;; + *) + print_error "Unknown option: $1" + echo "Use --help for usage information" + exit 1 + ;; + esac + done + + # Run checks + check_python + + if [ "$BUILD_ONLY" = false ]; then + check_aws_config + fi + + # Build package + build_package + + if [ "$BUILD_ONLY" = false ]; then + # Deploy and test + deploy_function + + if [ "$SKIP_TEST" = false ]; then + test_function + fi + fi + + print_success "Script completed successfully!" +} + +# Run main function +main "$@" diff --git a/infrastructure/lambda/build.sh b/infrastructure/lambda/build.sh new file mode 100755 index 00000000..d7343923 --- /dev/null +++ b/infrastructure/lambda/build.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# build.sh + +# Exit on any error +set -e + +# Configuration +FUNCTION_NAME="container-thread-dump" +PACKAGE_NAME="lambda_function" +PYTHON_VERSION="3.13" +TEMP_DIR="build_temp" +DIST_DIR="dist" +OUTPUT_ZIP="$DIST_DIR/$PACKAGE_NAME.zip" + +# Print header +echo "=== Building Lambda Deployment Package ===" +echo "Function name: $FUNCTION_NAME" +echo "Python version: $PYTHON_VERSION" + +# Create clean directories +echo "Creating clean build directories..." +rm -rf "$TEMP_DIR" "$DIST_DIR" +mkdir -p "$TEMP_DIR/package" "$DIST_DIR" + +# Create and activate virtual environment +echo "Creating virtual environment..." +python3 -m venv "$TEMP_DIR/venv" +source "$TEMP_DIR/venv/bin/activate" + +# Upgrade pip +echo "Upgrading pip..." +pip install --upgrade pip + +# Install dependencies +echo "Installing dependencies..." +pip install -r requirements.txt --target "$TEMP_DIR/package" + +# Copy function code +echo "Copying function code..." +cp src/index.py "$TEMP_DIR/package/" +cp src/eks_client.py "$TEMP_DIR/package/" + +# Create deployment package +echo "Creating deployment package..." +cd "$TEMP_DIR/package" +zip -r "../../$OUTPUT_ZIP" . > /dev/null +cd - > /dev/null + +# โœ… Required for CDK bundling: copy to /asset-output +if [ -d "/asset-output" ]; then + echo "Copying zip to /asset-output for CDK..." + cp "$OUTPUT_ZIP" /asset-output/ +else + echo "Warning: /asset-output not found โ€” not running inside CDK bundling" +fi + +# Calculate package size +PACKAGE_SIZE=$(du -h "$OUTPUT_ZIP" | cut -f1) + +# Clean up +echo "Cleaning up build files..." +deactivate +rm -rf "$TEMP_DIR" + +# Print summary +echo "" +echo "=== Build Summary ===" +echo "Package created: $OUTPUT_ZIP" +echo "Package size: $PACKAGE_SIZE" +echo "Build complete!" + +# Optional: Deploy to AWS if specified +if [ "$1" == "--deploy" ]; then + echo "" + echo "=== Deploying to AWS ===" + aws lambda update-function-code \ + --function-name "$FUNCTION_NAME" \ + --zip-file "fileb://$OUTPUT_ZIP" + echo "Deployment complete!" +fi diff --git a/infrastructure/lambda/cleanup.sh b/infrastructure/lambda/cleanup.sh new file mode 100755 index 00000000..56a49f44 --- /dev/null +++ b/infrastructure/lambda/cleanup.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Create clean package directory +rm -rf package/* +rm -f lambda_function.zip + +echo "Clean up directories" \ No newline at end of file diff --git a/infrastructure/lambda/requirements.txt b/infrastructure/lambda/requirements.txt new file mode 100644 index 00000000..c8e9ac9e --- /dev/null +++ b/infrastructure/lambda/requirements.txt @@ -0,0 +1,4 @@ +boto3==1.37.34 +kubernetes==32.0.1 +awscli==1.38.34 +requests==2.31.0 \ No newline at end of file diff --git a/infrastructure/lambda/src/eks_client.py b/infrastructure/lambda/src/eks_client.py new file mode 100644 index 00000000..44facd86 --- /dev/null +++ b/infrastructure/lambda/src/eks_client.py @@ -0,0 +1,437 @@ +import inspect +import json +import logging +import threading +import time +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional, Type, TypeVar +import re + +import boto3 +from awscli.customizations.eks.get_token import ( + TOKEN_EXPIRATION_MINS, + STSClientFactory, + TokenGenerator, +) +from kubernetes import client as k8s_client +from kubernetes.client.rest import ApiException +from kubernetes.stream import stream + +logger = logging.getLogger(__name__) + +# Type variable for the Kubernetes API client classes +T = TypeVar("T") + +class Token: + __token: str = None + __expiration: datetime = None + + def __init__(self) -> None: + self._lock = threading.RLock() + + def is_expired(self) -> bool: + with self._lock: + return datetime.now(timezone.utc) >= self.__expiration + + @property + def token(self) -> str: + with self._lock: + return self.__token + + @token.setter + def token(self, val: str) -> None: + with self._lock: + self.__token = val + + @property + def expiration(self) -> datetime: + with self._lock: + return self.__expiration + + @expiration.setter + def expiration(self, val: datetime) -> None: + with self._lock: + self.__expiration = val + + def __repr__(self): + return f"Token(token={self.token}, expiration={self.expiration})" + +class EKSClient: + TOKEN_EXPIRATION_BUFFER_SECONDS = 60 + + def __init__(self, cluster_name) -> None: + """ + Initialize the client. Using the provided cluster name we will use the EKS API to look up the + endpoint for the cluster, so that we can then generate a client. + + Args: + cluster_name: the name of the cluster that we will work with + """ + self._cluster_name = cluster_name + self._eks_client = boto3.client("eks") + self._cluster_endpoint = self._get_cluster_endpoint() + self._client_cache: Dict[T, Any] = {} + + # Let's start by creating the STS client + sess = boto3.Session() + self._sts_client = STSClientFactory(sess._session).get_sts_client() + self._token_generator = TokenGenerator(self._sts_client) + self._token = Token() + + logger.info(f"Initialized new {__name__} for {self._cluster_name}") + + # Start the token refresh thread + self._refresh_token() + self._start_token_refresh() + + # Initialize the Kubernetes configuration + try: + self._k8s_configuration = k8s_client.Configuration() + self._k8s_configuration.host = self.endpoint + self._k8s_configuration.verify_ssl = False + self._k8s_configuration.assert_hostname = True + + self._update_kube_authz() + self._start_kube_authz_refresh() + + k8s_client.Configuration.set_default(self._k8s_configuration) + self.core_v1_api = self.make_kube_client(k8s_client.CoreV1Api) + except Exception as e: + logger.error(f"Failed to initialize Kubernetes configuration: {str(e)}") + raise + + def _get_cluster_endpoint(self) -> str: + """ + Get the cluster endpoint URL + + Returns: + str: The cluster endpoint URL + """ + try: + resp = self._eks_client.describe_cluster(name=self._cluster_name) + endpoint = resp["cluster"]["endpoint"] + logger.info(f"Discovered endpoint {endpoint} for cluster {self._cluster_name}") + return endpoint + except self._eks_client.exceptions.ResourceNotFoundException as e: + logger.error(f"Cluster {self._cluster_name} not found: {str(e)}") + raise + except self._eks_client.exceptions.ClientError as e: + logger.error(f"Failed to describe cluster {self._cluster_name}: {str(e)}") + raise + + @property + def endpoint(self) -> str: + return self._cluster_endpoint + + def _refresh_token(self) -> None: + """Refresh both token and expiration timestamp""" + logger.info(f"Refreshing token for cluster {self._cluster_name}") + tok = self._token_generator.get_token(self._cluster_name) + exp = datetime.now(timezone.utc) + timedelta(minutes=TOKEN_EXPIRATION_MINS) + self._token.token = tok + self._token.expiration = exp + logger.info(f"Token refreshed: {self._token}") + + def _start_token_refresh(self) -> None: + """Start a background thread to periodically refresh the token""" + def refresh_loop(): + while True: + try: + logger.info(f"Token refresh loop for cluster {self._cluster_name}") + seconds_until_expire = ( + self._token.expiration - datetime.now(timezone.utc) + ).total_seconds() + if seconds_until_expire < self.TOKEN_EXPIRATION_BUFFER_SECONDS: + logger.info(f"Token will expire in {seconds_until_expire} seconds, refreshing immediately") + self._refresh_token() + time.sleep(10) + except Exception as e: + logger.error(f"Error in token refresh loop: {str(e)}") + time.sleep(5) + + thread = threading.Thread(target=refresh_loop, daemon=True) + thread.start() + logger.info("Token refresh thread started") + + def _update_kube_authz(self) -> None: + """Update the Kubernetes client authorization header""" + tok = self._token.token + self._k8s_configuration.api_key["authorization"] = f"Bearer {tok}" + + def _start_kube_authz_refresh(self) -> None: + """Start a background thread to periodically refresh the Kubernetes authorization""" + def refresh_loop(): + while True: + try: + logger.info(f"Kube authz refresh loop for cluster {self._cluster_name}") + seconds_until_expire = ( + self._token.expiration - datetime.now(timezone.utc) + ).total_seconds() + if seconds_until_expire < self.TOKEN_EXPIRATION_BUFFER_SECONDS: + logger.info(f"Token will expire in {seconds_until_expire} seconds, refreshing immediately") + self._update_kube_authz() + time.sleep(10) + except Exception as e: + logger.error(f"Error in kube authz refresh loop: {str(e)}") + time.sleep(5) + + thread = threading.Thread(target=refresh_loop, daemon=True) + thread.start() + logger.info("Kube authz refresh thread started") + + def make_kube_client(self, api_cls: Type[T]) -> T: + """ + Create a Kubernetes client of the specified type + + Args: + api_cls: The Kubernetes API client class to create + + Returns: + An instance of the specified client class + """ + if api_cls in self._client_cache: + return self._client_cache[api_cls] + + if (not inspect.isclass(api_cls) or + not hasattr(api_cls, "__module__") or + not api_cls.__module__.startswith("kubernetes.client")): + raise ValueError(f"Invalid Kubernetes API client class: {api_cls}") + + try: + instance = api_cls() + self._client_cache[api_cls] = instance + return instance + except Exception as e: + logger.error(f"Failed to create client of type {api_cls.__name__}: {str(e)}") + raise + + def find_pod_by_pattern(self, namespace: str, name_pattern: str) -> str: + """ + Find a pod in the namespace matching the given pattern + + Args: + namespace: The namespace to search in + name_pattern: Pattern to match against pod names + + Returns: + str: Name of the first matching pod + + Raises: + Exception: If no matching pod is found or on API errors + """ + try: + pods = self.core_v1_api.list_namespaced_pod(namespace=namespace) + + for pod in pods.items: + if re.search(name_pattern, pod.metadata.name): + logger.info(f"Found matching pod: {pod.metadata.name}") + return pod.metadata.name + + raise ValueError(f"No pod found matching pattern '{name_pattern}' in namespace '{namespace}'") + + except ApiException as e: + logger.error(f"Kubernetes API error while finding pod: {str(e)}") + raise Exception(f"Kubernetes API error: {e}") + + def create_heap_dump(self, namespace: str, pod_name: str, container_name: Optional[str] = None, + output_path: str = "/tmp/heapdump.hprof") -> str: + """ + Create a heap dump from a Java application running in a pod + + Args: + namespace: Kubernetes namespace where the pod is running + pod_name: Name of the pod or pattern to match + container_name: Optional name of the container (if pod has multiple containers) + output_path: Path where the heap dump should be saved in the container + + Returns: + str: The path to the created heap dump file + + Raises: + Exception: On various error conditions + """ + try: + # If pod_name contains wildcards, find matching pod + if any(char in pod_name for char in ['*', '?', '[']): + matched_pod = self.find_pod_by_pattern(namespace, pod_name) + logger.info(f"Using matched pod: {matched_pod}") + pod_name = matched_pod + + # Get container name if not specified + if not container_name: + pod = self.core_v1_api.read_namespaced_pod( + name=pod_name, + namespace=namespace + ) + if len(pod.spec.containers) > 0: + container_name = pod.spec.containers[0].name + logger.info(f"Using container: {container_name}") + else: + raise ValueError("No containers found in pod") + + # Command to create heap dump + exec_command = [ + '/bin/sh', + '-c', + (f'if command -v jcmd >/dev/null 2>&1; then ' + f'PID=$(jcmd | grep -v jcmd | cut -d" " -f1); ' + f'jcmd $PID GC.heap_dump {output_path}; ' + f'elif command -v jmap >/dev/null 2>&1; then ' + f'PID=$(ps -ef | grep java | grep -v grep | awk \'{{print $2}}\'); ' + f'jmap -dump:format=b,file={output_path} $PID; ' + f'else echo "Neither jcmd nor jmap found"; exit 1; ' + f'fi; ' + f'echo "Heap dump created at {output_path}"; ' + f'ls -l {output_path}') + ] + + logger.info(f"Executing heap dump command in pod {pod_name}, container {container_name}") + + resp = stream( + self.core_v1_api.connect_get_namespaced_pod_exec, + pod_name, + namespace, + container=container_name, + command=exec_command, + stderr=True, + stdin=False, + stdout=True, + tty=False + ) + + if "Heap dump created at" in resp: + logger.info("Successfully created heap dump") + return output_path + else: + logger.error(f"Failed to create heap dump. Output: {resp}") + raise Exception("Failed to create heap dump") + + except ApiException as e: + logger.error(f"Kubernetes API error in create_heap_dump: {str(e)}") + raise Exception(f"Kubernetes API error: {e}") + except ValueError as e: + logger.error(f"Value error in create_heap_dump: {str(e)}") + raise Exception(f"Value error: {e}") + except Exception as e: + logger.error(f"Unexpected error in create_heap_dump: {str(e)}") + raise Exception(f"Unexpected error while creating heap dump: {e}") + + def copy_file_from_pod(self, namespace: str, pod_name: str, src_path: str, + dest_path: str, container_name: Optional[str] = None) -> None: + """ + Copy a file from a pod to the local filesystem + + Args: + namespace: Kubernetes namespace where the pod is running + pod_name: Name of the pod + src_path: Source path of the file in the pod + dest_path: Destination path on the local filesystem + container_name: Optional name of the container (if pod has multiple containers) + + Raises: + Exception: On various error conditions + """ + try: + exec_command = ['cat', src_path] + resp = stream( + self.core_v1_api.connect_get_namespaced_pod_exec, + pod_name, + namespace, + container=container_name, + command=exec_command, + stderr=True, + stdin=False, + stdout=True, + tty=False + ) + + with open(dest_path, 'wb') as f: + f.write(resp.encode('utf-8')) + + logger.info(f"Successfully copied file from {src_path} to {dest_path}") + + except ApiException as e: + logger.error(f"Kubernetes API error in copy_file_from_pod: {str(e)}") + raise Exception(f"Kubernetes API error: {e}") + except Exception as e: + logger.error(f"Unexpected error in copy_file_from_pod: {str(e)}") + raise Exception(f"Unexpected error while copying file from pod: {e}") + + def get_thread_dump(self, namespace: str, pod_name: str, container_name: str = None) -> str: + """ + Get a thread dump from a Java application running in a pod + + Args: + namespace: Kubernetes namespace where the pod is running + pod_name: Name of the pod or pattern to match + container_name: Optional name of the container (if pod has multiple containers) + + Returns: + str: The thread dump output + + Raises: + Exception: On various error conditions + """ + try: + # If pod_name contains wildcards, find matching pod + if any(char in pod_name for char in ['*', '?', '[']): + matched_pod = self.find_pod_by_pattern(namespace, pod_name) + logger.info(f"Using matched pod: {matched_pod}") + pod_name = matched_pod + + # Get container name if not specified + if not container_name: + pod = self.core_v1_api.read_namespaced_pod( + name=pod_name, + namespace=namespace + ) + if len(pod.spec.containers) > 0: + container_name = pod.spec.containers[0].name + logger.info(f"Using container: {container_name}") + else: + raise ValueError("No containers found in pod") + + # Command to get thread dump + exec_command = [ + '/bin/sh', + '-c', + ('if command -v jcmd >/dev/null 2>&1; then ' + 'PID=$(jcmd | grep -v jcmd | cut -d" " -f1); ' + 'jcmd $PID Thread.print; ' + 'elif command -v jstack >/dev/null 2>&1; then ' + 'PID=$(ps -ef | grep java | grep -v grep | awk \'{print $2}\'); ' + 'jstack $PID; ' + 'else echo "Neither jcmd nor jstack found"; ' + 'fi') + ] + + logger.info(f"Executing thread dump command in pod {pod_name}, container {container_name}") + + resp = stream( + self.core_v1_api.connect_get_namespaced_pod_exec, + pod_name, + namespace, + container=container_name, + command=exec_command, + stderr=True, + stdin=False, + stdout=True, + tty=False + ) + + if resp: + logger.info("Successfully obtained thread dump") + return resp + else: + logger.warning("No output received from thread dump command") + return "No output received from thread dump command" + + except ApiException as e: + logger.error(f"Kubernetes API error in get_thread_dump: {str(e)}") + raise Exception(f"Kubernetes API error: {e}") + except ValueError as e: + logger.error(f"Value error in get_thread_dump: {str(e)}") + raise Exception(f"Value error: {e}") + except Exception as e: + logger.error(f"Unexpected error in get_thread_dump: {str(e)}") + raise Exception(f"Unexpected error while getting thread dump: {e}") diff --git a/infrastructure/lambda/src/index.py b/infrastructure/lambda/src/index.py new file mode 100644 index 00000000..2f92b7f3 --- /dev/null +++ b/infrastructure/lambda/src/index.py @@ -0,0 +1,433 @@ +import json +import logging +import os +import boto3 +import requests +import time +import random +import re +import base64 +from datetime import datetime +from typing import Dict, Any +from kubernetes import client, config +from botocore.exceptions import ClientError +from eks_client import EKSClient # Your own implementation + +logger = logging.getLogger() +logger.setLevel(logging.INFO) + +def is_invalid(value: str) -> bool: + return value is None or value.strip() in ["", "[no value]"] + +def verify_basic_auth(event: Dict[str, Any]) -> bool: + """ + Verify Basic Authentication credentials from the request headers. + Returns True if authentication is successful, False otherwise. + """ + try: + # Get headers from the event + headers = event.get('headers', {}) + if not headers: + logger.warning("No headers found in the request") + return False + + # Look for authorization header (case-insensitive) + auth_header = None + for key in headers: + if key.lower() == 'authorization': + auth_header = headers[key] + break + + if not auth_header or not auth_header.startswith('Basic '): + logger.warning("No Basic Authorization header found") + return False + + # Extract and decode credentials + encoded_credentials = auth_header.split(' ')[1] + decoded_credentials = base64.b64decode(encoded_credentials).decode('utf-8') + username, password = decoded_credentials.split(':') + + # Get expected credentials from AWS Secrets Manager + secret_name = "grafana-webhook-credentials" + region_name = os.environ.get('AWS_REGION', 'us-east-1') + + session = boto3.session.Session() + client = session.client( + service_name='secretsmanager', + region_name=region_name + ) + + response = client.get_secret_value(SecretId=secret_name) + secret = json.loads(response['SecretString']) + + # Validate credentials + if username != secret['username'] or password != secret['password']: + logger.warning("Invalid credentials provided") + return False + + logger.info("Authentication successful") + return True + + except ClientError as e: + logger.error(f"Error retrieving credentials from Secrets Manager: {str(e)}") + return False + except Exception as e: + logger.error(f"Authentication error: {str(e)}") + return False + +def extract_pod_info_from_valuestring(value_string: str) -> Dict[str, str]: + """Extract pod information from Grafana alert valueString""" + try: + logger.info(f"Extracting pod info from valueString: {value_string}") + + # First, extract the labels section + labels_match = re.search(r'labels=\{([^}]+)\}', value_string) + if not labels_match: + logger.warning("Could not find labels section in valueString") + return {} + + labels_str = labels_match.group(1) + logger.info(f"Extracted labels section: {labels_str}") + + # Now extract individual labels from the labels section + labels = {} + for label_pair in labels_str.split(', '): + if '=' in label_pair: + key, value = label_pair.split('=', 1) + labels[key.strip()] = value.strip() + logger.info(f"Parsed label: {key.strip()}={value.strip()}") + + logger.info(f"All parsed labels: {labels}") + + # Map the labels to our expected keys + result = { + 'cluster': labels.get('cluster'), + 'cluster_type': labels.get('cluster_type'), + 'container_name': labels.get('container_name'), + 'task_pod_id': labels.get('task_pod_id') or labels.get('pod'), # Use pod if task_pod_id not found + 'namespace': labels.get('namespace'), + 'container_ip': labels.get('container_ip') or labels.get('exported_instance') # Add container_ip extraction + } + + logger.info(f"Final extracted values: {result}") + return result + except Exception as e: + logger.error(f"Error extracting pod info from valueString: {str(e)}") + return {} + +def analyze_thread_dump(thread_dump: str) -> str: + bedrock = boto3.client("bedrock-runtime", region_name="us-west-2") + + logger.info(f"Using Bedrock in region: {bedrock.meta.region_name}") + prompt = f"""Please analyze the following Java thread dump. Your task is to identify performance issues and provide actionable insights. Structure the output into the following four sections: + +1. **Summary of Thread States**: Count and categorize all thread states (e.g., RUNNABLE, WAITING). +2. **Key Issues Identified**: Describe any threads that appear stuck, blocked, or problematic (e.g., deadlocks, high CPU). +3. **Optimization Recommendations**: Suggest practical improvements based on your findings (e.g., code, configuration, GC tuning). +4. **Detailed Analysis**: Provide a technical breakdown of the most interesting or problematic threads. + +Thread Dump: +{thread_dump} +""" + + payload = json.dumps({ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 8192, + "messages": [{"role": "user", "content": prompt}], + "temperature": 0.7 + }) + + max_attempts = 5 + base_delay = 10 + + # Cap backoff to avoid over-waiting + max_delay = 30 + + for attempt in range(1, max_attempts + 1): + logger.info(f"Invoking Bedrock with payload ... ") + try: + response = bedrock.invoke_model( + modelId="us.anthropic.claude-3-7-sonnet-20250219-v1:0", + body=payload + ) + body = json.loads(response.get("body").read()) + return body.get("content")[0].get("text") + except ClientError as e: + if e.response["Error"]["Code"] == "ThrottlingException" and attempt < max_attempts: + + logger.error(f"ClientError during analysis: {str(e)}") + + delay = min(random.uniform(0, base_delay * (2 ** attempt)), max_delay) + logger.warning(f"Bedrock throttled (attempt {attempt}) โ€” retrying in {delay:.2f}s") + time.sleep(delay) + else: + logger.error(f"ClientError during analysis: {str(e)}") + return f"ClientError during analysis: {str(e)}" + except Exception as e: + logger.error(f"Unexpected error during analysis: {str(e)}") + return f"Unexpected error during analysis: {str(e)}" + + return "Failed to analyze thread dump after multiple retries due to throttling." + + +class ECSClient: + def __init__(self, cluster_name: str): + self.cluster_name = cluster_name + self.ecs = boto3.client('ecs') + + def get_container_ip(self, task_id: str) -> str: + task = self.ecs.describe_tasks(cluster=self.cluster_name, tasks=[task_id])['tasks'][0] + eni_details = task['attachments'][0]['details'] + private_ip = next((item['value'] for item in eni_details if item['name'] == 'privateIPv4Address'), None) + if not private_ip: + raise ValueError("Failed to get private IP of container") + logger.info(f"Found private IP: {private_ip}") + return private_ip + + +def process_alert(cluster_type, cluster_name, task_pod_id, container_name, namespace, s3_bucket, container_ip=None): + """Extract the alert processing logic into a separate function for reuse""" + # Get the thread dump based on cluster type + if cluster_type == 'ecs': + if container_ip: + # Use the container_ip provided in the alert + logger.info(f"Using container IP from alert: {container_ip}") + else: + # Fallback to getting IP from ECS API + ecs_client = ECSClient(cluster_name) + container_ip = ecs_client.get_container_ip(task_pod_id) + + response = requests.get(f"http://{container_ip}:8080/actuator/threaddump", timeout=10) + response.raise_for_status() + thread_dump = response.text + + elif cluster_type == 'eks': + eks_client = EKSClient(cluster_name) + thread_dump = eks_client.get_thread_dump( + namespace=namespace, + pod_name=task_pod_id, + container_name=container_name + ) + + else: + raise ValueError(f"Unsupported cluster type: {cluster_type}") + + # Analyze and store + analysis = analyze_thread_dump(thread_dump) + s3 = boto3.client('s3') + timestamp = datetime.utcnow().strftime('%Y-%m-%d-%H-%M-%S') + dump_key = f"thread-dumps/{task_pod_id}/{timestamp}.txt" + analysis_key = f"thread-dumps/{task_pod_id}/{timestamp}_analysis.md" + + s3.put_object(Bucket=s3_bucket, Key=dump_key, Body=thread_dump.encode('utf-8')) + s3.put_object(Bucket=s3_bucket, Key=analysis_key, Body=analysis.encode('utf-8')) + + result = { + 'message': 'Thread dump handled from alert', + 'taskPodId': task_pod_id, + 'cluster': cluster_name, + 'threadDumpUrl': f"s3://{s3_bucket}/{dump_key}", + 'analysisUrl': f"s3://{s3_bucket}/{analysis_key}" + } + + logger.info(json.dumps(result)) + return result + +def lambda_handler(event: Dict[str, Any], context: Any) -> Dict[str, Any]: + try: + logger.info(f"Received event: {json.dumps(event)}") + s3_bucket = os.environ['S3_BUCKET_NAME'] + + # Check if this is a direct webhook call from Grafana + if 'body' in event: + logger.info("Processing direct webhook from Grafana") + + # Verify authentication for webhook calls + if not verify_basic_auth(event): + logger.warning("Authentication failed") + return { + 'statusCode': 401, + 'headers': {'WWW-Authenticate': 'Basic'}, + 'body': json.dumps({'error': 'Authentication failed'}) + } + + try: + # Parse the webhook payload + if isinstance(event['body'], str): + body = json.loads(event['body']) + else: + body = event['body'] + + # Rest of your webhook handling code remains the same + alerts = body.get('alerts', []) + if not alerts: + return { + 'statusCode': 200, + 'body': json.dumps({'message': 'No alerts in webhook payload'}) + } + + results = [] + for alert in alerts: + # Your existing alert processing code... + # No changes needed here + if alert.get('status') != 'firing': + logger.info("Skipping resolved alert") + continue + + # Extract labels from Grafana alert + labels = alert.get('labels', {}) + cluster_type = labels.get('cluster_type') + cluster_name = labels.get('cluster') + task_pod_id = labels.get('task_pod_id') + container_name = labels.get('container_name') + namespace = labels.get('namespace', 'default') + container_ip = labels.get('instance') # Get container_ip from the instance label + + # Check if we need to extract from valueString + if any(is_invalid(val) for val in [cluster_type, cluster_name, task_pod_id, container_name]): + logger.info("Some labels are missing or invalid, trying to extract from valueString") + + # Try to extract from valueString + if 'valueString' in alert: + extracted_info = extract_pod_info_from_valuestring(alert['valueString']) + logger.info(f"Raw extracted info: {extracted_info}") + + # Update missing values + if is_invalid(cluster_type) and extracted_info.get('cluster_type'): + cluster_type = extracted_info['cluster_type'] + + if is_invalid(cluster_name) and extracted_info.get('cluster'): + cluster_name = extracted_info['cluster'] + + if is_invalid(task_pod_id) and extracted_info.get('task_pod_id'): + task_pod_id = extracted_info['task_pod_id'] + + if is_invalid(container_name) and extracted_info.get('container_name'): + container_name = extracted_info['container_name'] + + if is_invalid(container_ip) and extracted_info.get('container_ip'): + container_ip = extracted_info['container_ip'] + logger.info(f"Using container_ip from valueString: {container_ip}") + + # IMPORTANT: Always override namespace with extracted value if available + if extracted_info.get('namespace'): + namespace = extracted_info['namespace'] + logger.info(f"Overriding namespace with extracted value: {namespace}") + + logger.info(f"Final extracted values: cluster_type={cluster_type}, cluster={cluster_name}, task_pod_id={task_pod_id}, container_name={container_name}, namespace={namespace}") + + # Validate inputs after extraction attempt + if any(is_invalid(val) for val in [cluster_type, cluster_name, task_pod_id, container_name]): + logger.warning(f"Still missing or invalid alert labels after extraction: cluster_type={cluster_type}, cluster={cluster_name}, task_pod_id={task_pod_id}, container_name={container_name}") + continue + + # Process the alert (reusing your existing logic) + result = process_alert(cluster_type, cluster_name, task_pod_id, container_name, namespace, s3_bucket, container_ip) + results.append(result) + + if results: + return { + 'statusCode': 200, + 'body': json.dumps({'message': f'Processed {len(results)} alerts', 'results': results}) + } + else: + return { + 'statusCode': 200, + 'body': json.dumps({'message': 'No valid alerts to process'}) + } + + except Exception as e: + logger.error(f"Error processing webhook: {str(e)}") + return { + 'statusCode': 500, + 'body': json.dumps({'error': f"Error processing webhook: {str(e)}"}) + } + + # Original SNS handling logic + elif 'Records' in event and event['Records'][0].get('Sns'): + record = event['Records'][0] + sns_message = record['Sns']['Message'] + message_json = json.loads(sns_message) + + # Process only alerts with status 'firing' + for alert in message_json.get('alerts', []): + if alert.get('status') != 'firing': + logger.info("Skipping resolved alert") + continue + + labels = alert.get('labels', {}) + cluster_type = labels.get('cluster_type') + cluster_name = labels.get('cluster') + task_pod_id = labels.get('task_pod_id') + container_name = labels.get('container_name') + namespace = labels.get('namespace', 'default') + + # Check if we need to extract from valueString (adding the same logic to SNS handling) + if any(is_invalid(val) for val in [cluster_type, cluster_name, task_pod_id, container_name]): + logger.info("Some labels are missing or invalid in SNS message, trying to extract from valueString") + + # Try to extract from valueString + if 'valueString' in alert: + extracted_info = extract_pod_info_from_valuestring(alert['valueString']) + + # Update missing values + if is_invalid(cluster_type) and 'cluster_type' in extracted_info: + cluster_type = extracted_info['cluster_type'] + + if is_invalid(cluster_name) and 'cluster' in extracted_info: + cluster_name = extracted_info['cluster'] + + if is_invalid(task_pod_id) and 'task_pod_id' in extracted_info: + task_pod_id = extracted_info['task_pod_id'] + + if is_invalid(container_name) and 'container_name' in extracted_info: + container_name = extracted_info['container_name'] + + if is_invalid(namespace) and 'namespace' in extracted_info: + namespace = extracted_info['namespace'] + + logger.info(f"Extracted values from valueString in SNS: cluster_type={cluster_type}, cluster={cluster_name}, task_pod_id={task_pod_id}, container_name={container_name}, namespace={namespace}") + + # Validate inputs after extraction attempt + if any(is_invalid(val) for val in [cluster_type, cluster_name, task_pod_id, container_name]): + raise ValueError(f"Missing or invalid alert labels: cluster_type={cluster_type}, cluster={cluster_name}, task_pod_id={task_pod_id}, container_name={container_name}") + + # Process the alert + result = process_alert(cluster_type, cluster_name, task_pod_id, container_name, namespace, s3_bucket) + + return { + 'statusCode': 200, + 'body': json.dumps(result) + } + + # No firing alerts found + logger.info("No firing alerts to process") + return {'statusCode': 204, 'body': json.dumps({'message': 'No active alerts'})} + + # Direct invocation (non-webhook, non-SNS) + else: + # Check if this is a direct invocation with parameters + cluster_type = event.get('cluster_type') + cluster_name = event.get('cluster') + task_pod_id = event.get('task_pod_id') + container_name = event.get('container_name') + namespace = event.get('namespace', 'default') + + # Validate inputs + if any(is_invalid(val) for val in [cluster_type, cluster_name, task_pod_id, container_name]): + logger.warning("Event doesn't match expected formats and is missing required parameters") + return { + 'statusCode': 400, + 'body': json.dumps({'error': 'Invalid event format or missing required parameters'}) + } + + # Process direct invocation + result = process_alert(cluster_type, cluster_name, task_pod_id, container_name, namespace, s3_bucket) + return { + 'statusCode': 200, + 'body': json.dumps(result) + } + + except Exception as e: + logger.error(f"Unexpected error: {str(e)}") + return {'statusCode': 500, 'body': json.dumps({'error': str(e)})} \ No newline at end of file diff --git a/infrastructure/scripts/cleanup/monitoring.sh b/infrastructure/scripts/cleanup/monitoring.sh new file mode 100755 index 00000000..ce67b6a0 --- /dev/null +++ b/infrastructure/scripts/cleanup/monitoring.sh @@ -0,0 +1,300 @@ +#!/bin/bash + +set -euo pipefail + +log() { + echo "[$(date +'%H:%M:%S')] $*" +} + +# --- Configuration --- +NAMESPACE="monitoring" +GRAFANA_SECRET_NAME="grafana-admin" +SECRET_NAME="grafana-webhook-credentials" +PARAM_NAME="/unicornstore/prometheus/internal-dns" + +# Temporary files to clean up +VALUES_FILE="prometheus-values.yaml" +EXTRA_SCRAPE_FILE="extra-scrape-configs.yaml" +DATASOURCE_FILE="grafana-datasource.yaml" +DASHBOARD_JSON_FILE="jvm-dashboard.json" +DASHBOARD_PROVISIONING_FILE="dashboard-provisioning.yaml" +ALERT_RULE_FILE="grafana-alert-rules.yaml" +GRAFANA_VALUES_FILE="grafana-values.yaml" +LAMBDA_ALERT_RULE_FILE="lambda-alert-rule.json" +NOTIFICATION_POLICY_CONFIGMAP_FILE="notification-policy.yaml" + +cleanup_temp_files() { + log "๐Ÿงน Cleaning up temporary files..." + rm -f "$VALUES_FILE" "$EXTRA_SCRAPE_FILE" "$DATASOURCE_FILE" "$NOTIFICATION_POLICY_CONFIGMAP_FILE" \ + "$DASHBOARD_JSON_FILE" "$DASHBOARD_PROVISIONING_FILE" \ + "$ALERT_RULE_FILE" "$GRAFANA_VALUES_FILE" "$LAMBDA_ALERT_RULE_FILE" \ + "grafana-credentials.txt" +} +trap cleanup_temp_files EXIT + +log "๐Ÿšจ Starting monitoring stack cleanup..." + +# --- Get Grafana credentials before cleanup --- +GRAFANA_USER="admin" +GRAFANA_PASSWORD="" + +if kubectl get secret "$GRAFANA_SECRET_NAME" -n "$NAMESPACE" >/dev/null 2>&1; then + GRAFANA_PASSWORD=$(kubectl get secret "$GRAFANA_SECRET_NAME" -n "$NAMESPACE" -o jsonpath="{.data.password}" | base64 --decode) + log "๐Ÿ“‹ Retrieved Grafana password from existing secret" +fi + +# Get Grafana LoadBalancer hostname before cleanup +GRAFANA_LB=$(kubectl get svc grafana -n "$NAMESPACE" -o jsonpath="{.status.loadBalancer.ingress[0].hostname}" 2>/dev/null || true) +if [[ -n "$GRAFANA_LB" && "$GRAFANA_LB" != "" ]]; then + GRAFANA_URL="http://$GRAFANA_LB" + log "๐Ÿ“‹ Found Grafana URL: $GRAFANA_URL" +fi + +# Get Prometheus LoadBalancer hostname before cleanup +PROM_LB_HOSTNAME=$(kubectl get svc prometheus-server -n "$NAMESPACE" -o jsonpath="{.status.loadBalancer.ingress[0].hostname}" 2>/dev/null || true) +if [[ -n "$PROM_LB_HOSTNAME" && "$PROM_LB_HOSTNAME" != "" ]]; then + log "๐Ÿ“‹ Found Prometheus hostname: $PROM_LB_HOSTNAME" +fi + +# --- Clean up Grafana alert rules (if Grafana is accessible) --- +if [[ -n "$GRAFANA_LB" && -n "$GRAFANA_PASSWORD" ]]; then + log "๐Ÿ”ง Cleaning up Grafana alert rules..." + + # Wait briefly for Grafana to be accessible + for i in {1..5}; do + if curl -s -o /dev/null -w "%{http_code}" -u "$GRAFANA_USER:$GRAFANA_PASSWORD" "$GRAFANA_URL/api/health" | grep -q "200"; then + log "โœ… Grafana is accessible for cleanup" + break + fi + log "โณ Waiting for Grafana access... ($i/5)" + sleep 2 + done + + # Delete alert rules + ALERT_RULES=$(curl -s -u "$GRAFANA_USER:$GRAFANA_PASSWORD" "$GRAFANA_URL/api/v1/provisioning/alert-rules" 2>/dev/null || echo "[]") + if [[ "$ALERT_RULES" != "[]" ]]; then + echo "$ALERT_RULES" | jq -r '.[].uid' | while read -r rule_uid; do + if [[ -n "$rule_uid" && "$rule_uid" != "null" ]]; then + log "๐Ÿ—‘๏ธ Deleting alert rule: $rule_uid" + curl -s -X DELETE -u "$GRAFANA_USER:$GRAFANA_PASSWORD" \ + "$GRAFANA_URL/api/v1/provisioning/alert-rules/$rule_uid" >/dev/null || true + fi + done + fi + + # Delete contact points + CONTACT_POINTS=$(curl -s -u "$GRAFANA_USER:$GRAFANA_PASSWORD" "$GRAFANA_URL/api/v1/provisioning/contact-points" 2>/dev/null || echo "[]") + if [[ "$CONTACT_POINTS" != "[]" ]]; then + echo "$CONTACT_POINTS" | jq -r '.[] | select(.name=="lambda-webhook") | .uid' | while read -r cp_uid; do + if [[ -n "$cp_uid" && "$cp_uid" != "null" ]]; then + log "๐Ÿ—‘๏ธ Deleting contact point: $cp_uid" + curl -s -X DELETE -u "$GRAFANA_USER:$GRAFANA_PASSWORD" \ + "$GRAFANA_URL/api/v1/provisioning/contact-points/$cp_uid" >/dev/null || true + fi + done + fi + + # Delete folders + FOLDERS=$(curl -s -u "$GRAFANA_USER:$GRAFANA_PASSWORD" "$GRAFANA_URL/api/folders" 2>/dev/null || echo "[]") + if [[ "$FOLDERS" != "[]" ]]; then + echo "$FOLDERS" | jq -r '.[] | select(.title=="Unicorn Store Dashboards") | .uid' | while read -r folder_uid; do + if [[ -n "$folder_uid" && "$folder_uid" != "null" ]]; then + log "๐Ÿ—‘๏ธ Deleting folder: $folder_uid" + curl -s -X DELETE -u "$GRAFANA_USER:$GRAFANA_PASSWORD" \ + "$GRAFANA_URL/api/folders/$folder_uid" >/dev/null || true + fi + done + fi +fi + +# --- Clean up Prometheus LoadBalancer Security Group rules --- +if [[ -n "$PROM_LB_HOSTNAME" ]]; then + log "๐Ÿ” Cleaning up Prometheus LoadBalancer Security Group rules..." + + VPC_ID=$(aws ec2 describe-vpcs --filters "Name=tag:Name,Values=unicornstore-vpc" --query "Vpcs[0].VpcId" --output text 2>/dev/null || true) + if [[ -n "$VPC_ID" && "$VPC_ID" != "None" ]]; then + VPC_CIDR=$(aws ec2 describe-vpcs --vpc-ids "$VPC_ID" --query "Vpcs[0].CidrBlock" --output text 2>/dev/null || true) + + LB_ARN=$(aws elbv2 describe-load-balancers --output json 2>/dev/null | jq -r \ + --arg dns "$PROM_LB_HOSTNAME" ' + .LoadBalancers[] | select(.DNSName == $dns) | .LoadBalancerArn' || true) + + if [[ -n "$LB_ARN" ]]; then + ILB_SG_ID=$(aws elbv2 describe-load-balancers \ + --load-balancer-arns "$LB_ARN" \ + --query "LoadBalancers[0].SecurityGroups[0]" \ + --output text 2>/dev/null || true) + + if [[ -n "$ILB_SG_ID" && "$ILB_SG_ID" != "None" ]]; then + log "๐Ÿ—‘๏ธ Removing security group rule from $ILB_SG_ID" + aws ec2 revoke-security-group-ingress \ + --group-id "$ILB_SG_ID" \ + --protocol tcp \ + --port 9090 \ + --cidr "$VPC_CIDR" \ + --output text 2>/dev/null || log "โ„น๏ธ Security group rule may not exist" + fi + fi + fi +fi + +# --- Uninstall Helm releases --- +log "๐Ÿ—‘๏ธ Uninstalling Helm releases..." + +if helm list -n "$NAMESPACE" | grep -q "grafana"; then + log "๐Ÿ—‘๏ธ Uninstalling Grafana..." + helm uninstall grafana --namespace "$NAMESPACE" || log "โš ๏ธ Failed to uninstall Grafana" +fi + +if helm list -n "$NAMESPACE" | grep -q "prometheus"; then + log "๐Ÿ—‘๏ธ Uninstalling Prometheus..." + helm uninstall prometheus --namespace "$NAMESPACE" || log "โš ๏ธ Failed to uninstall Prometheus" +fi + +# --- Clean up Kubernetes resources --- +log "๐Ÿ—‘๏ธ Cleaning up Kubernetes resources..." + +# Delete ConfigMaps +kubectl delete configmap unicornstore-datasource -n "$NAMESPACE" 2>/dev/null || log "โ„น๏ธ ConfigMap unicornstore-datasource not found" +kubectl delete configmap unicornstore-dashboard -n "$NAMESPACE" 2>/dev/null || log "โ„น๏ธ ConfigMap unicornstore-dashboard not found" +kubectl delete configmap prometheus-extra-scrape -n "$NAMESPACE" 2>/dev/null || log "โ„น๏ธ ConfigMap prometheus-extra-scrape not found" +kubectl delete configmap unicornstore-notification-policy -n "$NAMESPACE" 2>/dev/null || log "โ„น๏ธ ConfigMap unicornstore-notification-policy not found" + +# Delete Secrets +kubectl delete secret "$GRAFANA_SECRET_NAME" -n "$NAMESPACE" 2>/dev/null || log "โ„น๏ธ Secret $GRAFANA_SECRET_NAME not found" + +# Delete PVCs (Persistent Volume Claims) +log "๐Ÿ—‘๏ธ Cleaning up Persistent Volume Claims..." +kubectl get pvc -n "$NAMESPACE" -o name 2>/dev/null | while read -r pvc; do + if [[ -n "$pvc" ]]; then + log "๐Ÿ—‘๏ธ Deleting $pvc" + kubectl delete "$pvc" -n "$NAMESPACE" || log "โš ๏ธ Failed to delete $pvc" + fi +done + +# Wait for PVCs to be deleted +log "โณ Waiting for PVCs to be fully deleted..." +for i in {1..30}; do + PVC_COUNT=$(kubectl get pvc -n "$NAMESPACE" --no-headers 2>/dev/null | wc -l || echo "0") + if [[ "$PVC_COUNT" -eq 0 ]]; then + log "โœ… All PVCs deleted" + break + fi + log "โณ Waiting for $PVC_COUNT PVCs to be deleted... ($i/30)" + sleep 5 +done + +# --- Delete namespace --- +log "๐Ÿ—‘๏ธ Deleting namespace $NAMESPACE..." +kubectl delete namespace "$NAMESPACE" --timeout=60s 2>/dev/null || log "โš ๏ธ Failed to delete namespace or namespace not found" + +# Wait for namespace deletion +log "โณ Waiting for namespace deletion..." +for i in {1..30}; do + if ! kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then + log "โœ… Namespace $NAMESPACE deleted" + break + fi + log "โณ Waiting for namespace deletion... ($i/30)" + sleep 5 +done + +# --- Clean up AWS resources --- +log "๐Ÿ—‘๏ธ Cleaning up AWS resources..." + +# Delete SSM Parameter +if aws ssm get-parameter --name "$PARAM_NAME" >/dev/null 2>&1; then + log "๐Ÿ—‘๏ธ Deleting SSM parameter $PARAM_NAME" + aws ssm delete-parameter --name "$PARAM_NAME" || log "โš ๏ธ Failed to delete SSM parameter" +else + log "โ„น๏ธ SSM parameter $PARAM_NAME not found" +fi + +# Delete Secrets Manager secret +if aws secretsmanager describe-secret --secret-id "$SECRET_NAME" >/dev/null 2>&1; then + log "๐Ÿ—‘๏ธ Deleting Secrets Manager secret $SECRET_NAME" + aws secretsmanager delete-secret --secret-id "$SECRET_NAME" --force-delete-without-recovery || log "โš ๏ธ Failed to delete secret" +else + log "โ„น๏ธ Secrets Manager secret $SECRET_NAME not found" +fi + +# --- Clean up Lambda Function URL (optional) --- +log "๐Ÿ”ง Checking Lambda Function URL..." +LAMBDA_FUNCTION_NAME="unicornstore-thread-dump-lambda" + +if aws lambda get-function --function-name "$LAMBDA_FUNCTION_NAME" >/dev/null 2>&1; then + # Check if Function URL exists + if aws lambda get-function-url-config --function-name "$LAMBDA_FUNCTION_NAME" >/dev/null 2>&1; then + log "๐Ÿ—‘๏ธ Removing Lambda Function URL..." + aws lambda delete-function-url-config --function-name "$LAMBDA_FUNCTION_NAME" || log "โš ๏ธ Failed to delete Function URL" + + # Remove the permission + aws lambda remove-permission \ + --function-name "$LAMBDA_FUNCTION_NAME" \ + --statement-id AllowPublicAccess 2>/dev/null || log "โ„น๏ธ Permission may not exist" + else + log "โ„น๏ธ Lambda Function URL not found" + fi +else + log "โ„น๏ธ Lambda function $LAMBDA_FUNCTION_NAME not found" +fi + +# --- Remove Helm repositories (optional) --- +log "๐Ÿ—‘๏ธ Cleaning up Helm repositories..." +helm repo remove prometheus-community 2>/dev/null || log "โ„น๏ธ prometheus-community repo not found" +helm repo remove grafana 2>/dev/null || log "โ„น๏ธ grafana repo not found" + +# --- Final validation --- +log "๐Ÿ” Validating cleanup..." + +# Check if namespace still exists +if kubectl get namespace "$NAMESPACE" >/dev/null 2>&1; then + log "โš ๏ธ Warning: Namespace $NAMESPACE still exists" +else + log "โœ… Namespace $NAMESPACE successfully deleted" +fi + +# Check if Helm releases still exist +REMAINING_RELEASES=$(helm list -A | grep -E "(prometheus|grafana)" || true) +if [[ -n "$REMAINING_RELEASES" ]]; then + log "โš ๏ธ Warning: Some Helm releases may still exist:" + echo "$REMAINING_RELEASES" +else + log "โœ… All monitoring Helm releases cleaned up" +fi + +# Check AWS resources +if aws secretsmanager describe-secret --secret-id "$SECRET_NAME" >/dev/null 2>&1; then + log "โš ๏ธ Warning: Secrets Manager secret $SECRET_NAME still exists" +else + log "โœ… Secrets Manager secret cleaned up" +fi + +if aws ssm get-parameter --name "$PARAM_NAME" >/dev/null 2>&1; then + log "โš ๏ธ Warning: SSM parameter $PARAM_NAME still exists" +else + log "โœ… SSM parameter cleaned up" +fi + +log "โœ… Monitoring stack cleanup completed!" +log "โ„น๏ธ Note: Some AWS resources (like Load Balancers) may take additional time to fully terminate" +log "โ„น๏ธ Note: Persistent Volumes may need manual cleanup if they were not automatically deleted" + +# --- Optional: List remaining resources for manual cleanup --- +log "๐Ÿ“‹ Checking for any remaining resources that may need manual cleanup..." + +# Check for remaining PVs +REMAINING_PVS=$(kubectl get pv | grep "$NAMESPACE" || true) +if [[ -n "$REMAINING_PVS" ]]; then + log "โš ๏ธ Warning: Found Persistent Volumes that may need manual cleanup:" + echo "$REMAINING_PVS" +fi + +# Check for remaining Load Balancers +REMAINING_LBS=$(aws elbv2 describe-load-balancers --output table | grep -E "(prometheus|grafana)" || true) +if [[ -n "$REMAINING_LBS" ]]; then + log "โš ๏ธ Warning: Found Load Balancers that may need manual cleanup:" + echo "$REMAINING_LBS" +fi + +log "๐ŸŽ‰ Cleanup script execution completed!" diff --git a/infrastructure/scripts/deploy/config/prometheus-values.yaml b/infrastructure/scripts/deploy/config/prometheus-values.yaml new file mode 100644 index 00000000..267ace90 --- /dev/null +++ b/infrastructure/scripts/deploy/config/prometheus-values.yaml @@ -0,0 +1,37 @@ +server: + service: + type: LoadBalancer + annotations: + service.beta.kubernetes.io/aws-load-balancer-internal: "true" + + persistentVolume: + enabled: true + storageClass: gp3 + + files: + prometheus.yml: + scrape_configs: + - job_name: 'ecs-adot-cloudmap' + scrape_interval: 15s + dns_sd_configs: + - names: ['adot.unicornstore.local'] + type: A + port: 9464 + + - job_name: 'k8s-unicorn-pods' + scrape_interval: 15s + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + separator: ":" + regex: (.+);(.+) + target_label: __address__ + replacement: $1:$2 diff --git a/infrastructure/scripts/deploy/eks.sh b/infrastructure/scripts/deploy/eks.sh index e2cbd8c3..30a6bd51 100755 --- a/infrastructure/scripts/deploy/eks.sh +++ b/infrastructure/scripts/deploy/eks.sh @@ -144,4 +144,50 @@ curl --location --request POST $SVC_URL'/unicorns' --header 'Content-Type: appli "size": "Very big" }' | jq +cat < ~/environment/unicorn-store-spring/k8s/storage-class.yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: gp3 + annotations: + storageclass.kubernetes.io/is-default-class: "true" +provisioner: ebs.csi.eks.amazonaws.com +volumeBindingMode: WaitForFirstConsumer +parameters: + type: gp3 + encrypted: "true" +EOF +kubectl apply -f ~/environment/unicorn-store-spring/k8s/storage-class.yaml + +cat < ~/environment/unicorn-store-spring/k8s/rbac.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: otel-collector + namespace: unicorn-store-spring +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: otel-collector + namespace: unicorn-store-spring +rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: otel-collector + namespace: unicorn-store-spring +subjects: + - kind: ServiceAccount + name: otel-collector + namespace: unicorn-store-spring +roleRef: + kind: Role + name: otel-collector + apiGroup: rbac.authorization.k8s.io +EOF +kubectl apply -f ~/environment/unicorn-store-spring/k8s/rbac.yaml + echo "App deployment to EKS cluster is complete." diff --git a/infrastructure/scripts/setup/README.md b/infrastructure/scripts/setup/README.md new file mode 100644 index 00000000..62b3aad1 --- /dev/null +++ b/infrastructure/scripts/setup/README.md @@ -0,0 +1,119 @@ +# ๐Ÿ“ˆ Unicorn Store Monitoring Stack + +This setup provisions a complete observability solution for the **Unicorn Store** application running on Amazon EKS: + +- โœ… **Prometheus** (scraping JVM metrics via OpenTelemetry) +- โœ… **Grafana** (with dashboards, alert rules, contact point, and notification policy) +- โœ… **AWS Lambda integration** (webhook triggered alerts for JVM thread dump generation) + +--- + +## ๐Ÿ”ง Components + +| Component | Description | +|---------------|-----------------------------------------------------------------------------| +| **Prometheus**| Scrapes metrics from OTEL Collector, exposed via internal LoadBalancer | +| **Grafana** | Public LoadBalancer, preconfigured with datasources, dashboards, and alerts | +| **OTEL Collector** | Exposes metrics from Spring Boot actuator `/actuator/prometheus` | +| **AWS Lambda**| Triggered via webhook when JVM thread threshold is exceeded | +| **ConfigMaps**| Provisioned via Helm sidecars for dashboard, datasource, alerts, etc. | +| **Security Group**| Ingress on port 9090 for Prometheus ILB, limited to the VPC CIDR | + +--- + +## ๐Ÿš€ Automated Setup Workflow + +Executed by `monitoring.sh`: + +1. ๐Ÿ” Generates a random Grafana admin password and stores it as a Kubernetes Secret +2. ๐Ÿ“ฆ Installs **Prometheus** and **Grafana** via Helm with dynamic `values.yaml` +3. ๐Ÿ“Š Downloads a JVM dashboard from Grafana.com (ID: `22108`) +4. ๐Ÿ“ก Waits for the Grafana LoadBalancer to become available +5. ๐Ÿ” Searches the dashboard for panels using `jvm_threads_live_threads` +6. โš ๏ธ Creates an alert rule `High JVM Threads` with a configurable threshold +7. ๐Ÿ›ฐ๏ธ Creates a **Contact Point** named `lambda-webhook` (if not already present) +8. ๐Ÿ“ฌ Creates a **Notification Policy** targeting the Lambda webhook +9. ๐Ÿ” Updates the **Security Group** of the Prometheus ILB to allow VPC ingress on port 9090 +10. ๐Ÿงช Sends a test alert to the Lambda function to verify integration + +--- + +## ๐Ÿ“ Output + +After successful execution, you'll see: + +- ๐ŸŒ **Grafana URL** (public ELB) +- ๐Ÿ‘ค **Username**: `admin` +- ๐Ÿ”‘ **Password**: Randomly generated (saved in `grafana-credentials.txt`) +- โš ๏ธ Confirmation of alert rule and Lambda webhook setup +- โœ… Manual test payload sent to the Lambda endpoint + +--- + +## ๐Ÿ“ File Structure + +```bash +scripts/ +โ”œโ”€โ”€ monitoring.sh # Main setup script (self-contained) +โ”œโ”€โ”€ grafana-credentials.txt # Generated credentials for Grafana access +โ”œโ”€โ”€ prometheus-values.yaml # Generated Prometheus Helm values +โ”œโ”€โ”€ grafana-values.yaml # Generated Grafana Helm values +โ”œโ”€โ”€ grafana-datasource.yaml # Prometheus datasource config (ConfigMap) +โ”œโ”€โ”€ jvm-dashboard.json # Downloaded JVM dashboard +โ”œโ”€โ”€ dashboard-provisioning.yaml # Dashboard provisioning provider +โ”œโ”€โ”€ grafana-alert-rules.yaml # Alert rule group for live thread alert +โ”œโ”€โ”€ lambda-alert-rule.json # JSON for API-based alert rule provisioning +``` + +--- + +## ๐Ÿ›ก๏ธ Prerequisites + +- Kubernetes cluster (Amazon EKS) +- IAM permissions for: + - Lambda: `GetFunctionUrlConfig`, `CreateFunctionUrlConfig`, `AddPermission` + - EC2: `DescribeLoadBalancers`, `AuthorizeSecurityGroupIngress` +- Installed tools: `kubectl`, `helm`, `aws`, `jq`, `curl`, `openssl`, `dig` + +--- + +## โœ… Monitoring Coverage + +- JVM Live Threads via `/actuator/prometheus` +- Alerts on high thread count (> 200 threads by default) +- Automatic Lambda invocation on threshold breach + +--- + +## ๐Ÿ“ฌ Example Lambda Payload + +```json +{ + "alerts": [ + { + "status": "firing", + "labels": { + "alertname": "High JVM Threads", + "severity": "critical", + "cluster_type": "eks", + "cluster": "unicorn-store", + "task_pod_id": "example-pod", + "container_name": "unicorn-store-spring", + "namespace": "unicorn-store-spring" + }, + "annotations": { + "summary": "Test Alert", + "description": "This is a test alert from Grafana setup script" + } + } + ] +} +``` + +--- + +## ๐Ÿ“Ž Notes + +- The script is **idempotent**: it safely replaces existing config +- All resources (dashboards, alerts, contacts, policies) are **provisioned via API or ConfigMaps** +- This setup is ideal for production observability of Spring Boot JVM apps on EKS \ No newline at end of file diff --git a/infrastructure/scripts/setup/eks.sh b/infrastructure/scripts/setup/eks.sh index f5dc01ae..3900aa04 100755 --- a/infrastructure/scripts/setup/eks.sh +++ b/infrastructure/scripts/setup/eks.sh @@ -172,4 +172,52 @@ aws eks associate-access-policy --cluster-name unicorn-store \ mkdir -p ~/environment/unicorn-store-spring/k8s +echo "Setting up StorageClass gp3" +cat </dev/null || true +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts || true +helm repo add grafana https://grafana.github.io/helm-charts || true +helm repo update + +# --- Grafana secret --- +kubectl delete secret "$GRAFANA_SECRET_NAME" -n "$NAMESPACE" 2>/dev/null || true +kubectl create secret generic "$GRAFANA_SECRET_NAME" \ + --from-literal=username="$GRAFANA_USER" \ + --from-literal=password="$GRAFANA_PASSWORD" \ + -n "$NAMESPACE" + +# --- Prometheus extra scrape configs --- +cat > "$EXTRA_SCRAPE_FILE" < "$VALUES_FILE" < "$GRAFANA_VALUES_FILE" </dev/null || true) + if [[ -n "$GRAFANA_LB" && "$GRAFANA_LB" != "" ]]; then + if dig +short "$GRAFANA_LB" | grep -qE "^[0-9.]+$"; then + log "โœ… Grafana LB: http://$GRAFANA_LB" + break + fi + fi + log "โณ Waiting for Grafana LB DNS... ($i/30)" + sleep 10 +done + +# --- Datasource ConfigMap --- +cat > "$DATASOURCE_FILE" < "$DASHBOARD_PROVISIONING_FILE" </dev/null || echo "") +if [[ -z "$LAMBDA_URL" ]]; then + log "Creating Lambda Function URL..." + LAMBDA_URL=$(aws lambda create-function-url-config --function-name unicornstore-thread-dump-lambda --auth-type NONE --query 'FunctionUrl' --output text) + aws lambda add-permission \ + --function-name unicornstore-thread-dump-lambda \ + --statement-id AllowPublicAccess \ + --action lambda:InvokeFunctionUrl \ + --principal "*" \ + --function-url-auth-type NONE || log "โš ๏ธ Permission may already exist" +fi +log "โœ… Lambda Function URL: $LAMBDA_URL" + +GRAFANA_URL="http://$GRAFANA_LB" + +# 1. Search for a dashboard containing "JVM" in the title + +set -x + +log "โณ Waiting for Grafana to become healthy..." +for i in {1..20}; do + STATUS=$(curl -s -u "$GRAFANA_USER:$GRAFANA_PASSWORD" "$GRAFANA_URL/api/health" | jq -r .database || true) + if [[ "$STATUS" == "ok" ]]; then + log "โœ… Grafana is healthy" + break + fi + log "โณ ($i/20) Grafana not ready yet..." + sleep 5 +done + +if [[ "$STATUS" != "ok" ]]; then + log "โŒ Grafana did not become healthy in time" + exit 1 +fi + +log "๐Ÿ” Searching for dashboard with 'JVM' in title..." +DASHBOARD_SEARCH=$(curl --connect-timeout 5 --max-time 10 -s -u "$GRAFANA_USER:$GRAFANA_PASSWORD" \ + "$GRAFANA_URL/api/search?query=JVM") + +if [[ -z "$DASHBOARD_SEARCH" ]]; then + log "โŒ Empty response from dashboard search API" + exit 1 +fi + +if ! echo "$DASHBOARD_SEARCH" | jq . >/dev/null 2>&1; then + log "โŒ Invalid JSON returned from dashboard search" + echo "$DASHBOARD_SEARCH" + exit 1 +fi + +DASHBOARD_UID=$(echo "$DASHBOARD_SEARCH" | jq -r ' + .[] | + select(.title | test("(?i)jvm")) | + .uid' | head -n1 +) + +if [[ -z "$DASHBOARD_UID" ]]; then + log "โŒ No dashboard found with title matching 'JVM'" + exit 1 +fi +log "โœ… Found dashboard UID: $DASHBOARD_UID" + +# 2. Fetch the full dashboard JSON +DASHBOARD_JSON=$(curl -s -u "$GRAFANA_USER:$GRAFANA_PASSWORD" \ + "$GRAFANA_URL/api/dashboards/uid/$DASHBOARD_UID") + +# 3. Extract a panel that contains your JVM metric expression +PANEL_ID=$(echo "$DASHBOARD_JSON" | jq -r ' + .dashboard.panels[] | + select(.targets[]?.expr | test("jvm_threads_live_threads")) | + .id' | head -n1 +) + +if [[ -z "$PANEL_ID" ]]; then + log "โŒ No panel found with expression 'jvm_threads_live_threads'" + exit 1 +fi +log "โœ… Found panel ID: $PANEL_ID" + +# 4. Build the alert rule JSON using the dashboardUID and panelID + +log "๐Ÿ”ง Resolving contact point and folder..." + +# Get Contact Point UID +# Check and create contact point if necessary +NOTIF_UID=$(curl -s -u "$GRAFANA_USER:$GRAFANA_PASSWORD" \ + "$GRAFANA_URL/api/v1/provisioning/contact-points" | \ + jq -r '.[] | select(.name=="lambda-webhook") | .uid') + +if [[ -z "$NOTIF_UID" ]]; then + log "๐Ÿ”ง Contact point not found, creating..." + + CONTACT_POINT_JSON=$(jq -n \ + --arg name "lambda-webhook" \ + --arg url "$LAMBDA_URL" \ + --arg user "$WEBHOOK_USER" \ + --arg pass "$WEBHOOK_PASSWORD" \ + '{ + name: $name, + type: "webhook", + settings: { + url: $url, + httpMethod: "POST", + username: $user, + password: $pass, + authorization_scheme: "basic" + }, + disableResolveMessage: false, + isDefault: false + }') + + + curl -s -X POST -u "$GRAFANA_USER:$GRAFANA_PASSWORD" \ + -H "Content-Type: application/json" \ + -d "$CONTACT_POINT_JSON" \ + "$GRAFANA_URL/api/v1/provisioning/contact-points" + + sleep 2 + + NOTIF_UID=$(curl -s -u "$GRAFANA_USER:$GRAFANA_PASSWORD" \ + "$GRAFANA_URL/api/v1/provisioning/contact-points" | \ + jq -r '.[] | select(.name=="lambda-webhook") | .uid') +fi + +if [[ -z "$NOTIF_UID" ]]; then + log "โŒ Failed to create contact point 'lambda-webhook'" + exit 1 +else + log "โœ… Contact point UID: $NOTIF_UID" +fi + +# Get or create folder +FOLDER_TITLE="Unicorn Store Dashboards" +FOLDER_UID=$(curl -s -u "$GRAFANA_USER:$GRAFANA_PASSWORD" \ + "$GRAFANA_URL/api/folders" | jq -r --arg title "$FOLDER_TITLE" '.[] | select(.title == $title) | .uid') + +if [[ -z "$FOLDER_UID" ]]; then + log "๐Ÿ“ Folder not found. Creating '$FOLDER_TITLE'..." + FOLDER_UID=$(curl -s -u "$GRAFANA_USER:$GRAFANA_PASSWORD" \ + -H "Content-Type: application/json" \ + -d "{\"title\": \"$FOLDER_TITLE\"}" \ + "$GRAFANA_URL/api/folders" | jq -r '.uid') + + if [[ -z "$FOLDER_UID" ]]; then + log "โŒ Failed to create folder '$FOLDER_TITLE'" + exit 1 + fi + log "๐Ÿ“ Folder '$FOLDER_TITLE' created with UID: $FOLDER_UID" +else + log "๐Ÿ“ Found folder UID: $FOLDER_UID" +fi + +# Build alert rule JSON +log "๐Ÿ› ๏ธ Generating alert rule JSON..." + +ALERT_RULE_JSON=$(jq -n \ + --arg url "$LAMBDA_URL" \ + --arg uid "$DASHBOARD_UID" \ + --argjson pid "$PANEL_ID" \ + --arg notifUid "$NOTIF_UID" \ + --arg folderUid "$FOLDER_UID" ' +{ + dashboardUID: $uid, + panelId: $pid, + folderUID: $folderUid, + ruleGroup: "lambda-alerts", + title: "High JVM Threads - Lambda", + condition: "B", + data: [ + { + refId: "A", + relativeTimeRange: { from: 600, to: 0 }, + datasourceUid: "promds", + model: { + expr: "sum(jvm_threads_live_threads) by (task_pod_id, cluster_type, cluster, container_name, namespace, container_ip)", + instant: true, + intervalMs: 1000, + maxDataPoints: 43200, + refId: "A" + } + }, + { + refId: "B", + relativeTimeRange: { from: 0, to: 0 }, + datasourceUid: "-100", + model: { + conditions: [ + { + evaluator: { params: [200], type: "gt" }, + operator: { type: "and" }, + query: { params: ["A"] }, + reducer: { params: [], type: "last" }, + type: "query" + } + ], + refId: "B", + type: "classic_conditions" + } + } + ], + noDataState: "NoData", + execErrState: "Error", + for: "1m", + annotations: { + summary: "High JVM Threads", + description: "High number of JVM threads detected. Triggering Lambda thread dump.", + webhookUrl: $url + }, + labels: { + severity: "critical", + alertname: "High JVM Threads", + cluster: "{{ $labels.cluster }}", + cluster_type: "{{ $labels.cluster_type }}", + container_name: "{{ $labels.container_name }}", + namespace: "{{ $labels.namespace }}", + task_pod_id: "{{ $labels.task_pod_id }}", + container_ip: "{{ $labels.container_ip }}" + }, + notifications: [ + { uid: $notifUid } + ] +} +') + +# 5. Post the rule via Grafana HTTP API +log "๐Ÿ“ค Creating alert rule for dashboard '$DASHBOARD_UID', panel $PANEL_ID..." +RESPONSE=$(curl -s -X POST -H "Content-Type: application/json" \ + -u "$GRAFANA_USER:$GRAFANA_PASSWORD" \ + -d "$ALERT_RULE_JSON" \ + "$GRAFANA_URL/api/v1/provisioning/alert-rules") + +if echo "$RESPONSE" | jq -e '.uid' > /dev/null; then + log "โœ… Lambda alert rule created: RULE_UID $(echo "$RESPONSE" | jq -r '.uid')" +else + log "โŒ Failed to create Lambda alert rule" + echo "$RESPONSE" + exit 1 +fi + +set +x + +# --- Final output --- +echo -e "\nโœ… Monitoring Stack + Lambda Alert Setup Complete!" +echo "๐ŸŒ Grafana URL: http://$GRAFANA_LB" +echo "๐Ÿ‘ค Username: $GRAFANA_USER" +echo "๐Ÿ”‘ Password: $GRAFANA_PASSWORD" +echo -e "Grafana URL: http://$GRAFANA_LB\nUsername: $GRAFANA_USER\nPassword: $GRAFANA_PASSWORD" > grafana-credentials.txt +log "๐Ÿ’พ Credentials saved to grafana-credentials.txt" + +# --- Get Prometheus LoadBalancer Hostname --- +PROM_LB_HOSTNAME=$(kubectl get svc prometheus-server -n "$NAMESPACE" -o jsonpath="{.status.loadBalancer.ingress[0].hostname}" 2>/dev/null || true) + +if [[ -z "$PROM_LB_HOSTNAME" ]]; then + log "โŒ Prometheus Load Balancer Hostname not found" + exit 1 +fi + +# Store Prometheus hostname in SSM Parameter Store +PARAM_NAME="/unicornstore/prometheus/internal-dns" +log "๐Ÿ“ Storing Prometheus hostname in SSM Parameter Store..." + +# First, put or update the parameter without tags +aws ssm put-parameter \ + --name "$PARAM_NAME" \ + --value "$PROM_LB_HOSTNAME" \ + --type "String" \ + --overwrite \ + --description "Prometheus internal load balancer hostname" \ + --region "$AWS_REGION" \ + --output text + +if [ $? -eq 0 ]; then + # Then, add tags separately + aws ssm add-tags-to-resource \ + --resource-type "Parameter" \ + --resource-id "$PARAM_NAME" \ + --tags "Key=Project,Value=UnicornStore" "Key=Environment,Value=Development" \ + --region "$AWS_REGION" + + log "โœ… Successfully stored Prometheus hostname in SSM Parameter Store at $PARAM_NAME" +else + log "โš ๏ธ Warning: Failed to store Prometheus hostname in SSM Parameter Store" +fi + +set -x + +# --- Contact Point and Notification Policy for Lambda --- + +for i in {1..5}; do + NOTIF_UID=$(curl -s -u "$GRAFANA_USER:$GRAFANA_PASSWORD" \ + "$GRAFANA_URL/api/v1/provisioning/contact-points" \ + | jq -r '.[] | select(.name=="lambda-webhook") | .uid') + + if [[ -n "$NOTIF_UID" ]]; then + log "โœ… Contact Point UID resolved: $NOTIF_UID" + break + fi + + log "โณ Waiting for Contact Point to be available... ($i/5)" + sleep 2 +done + +if [[ -z "$NOTIF_UID" ]]; then + log "โŒ Failed to resolve Contact Point UID after creation" + exit 1 +fi + +# --- Create and apply notification policy --- +log "๐Ÿ”” Setting up notification policy for lambda-webhook..." + +# Wait for Grafana to be ready +log "โณ Waiting for Grafana API to be available..." +for i in {1..30}; do + if curl -s -o /dev/null -w "%{http_code}" -u "$GRAFANA_USER:$GRAFANA_PASSWORD" "http://$GRAFANA_LB/api/health" | grep -q "200"; then + log "โœ… Grafana API is available" + break + fi + log "โณ Waiting for Grafana API... ($i/30)" + sleep 5 +done + +# Create notification policy via API +POLICY_JSON=$(cat < "$NOTIFICATION_POLICY_CONFIGMAP_FILE" <