ci: fix Windows, EKS, batch naming, and test reliability (#1839)

Author: TravisStark

.github/workflows/CI.yml

@@ -0,0 +1 @@
+# NTP fix trigger

cdk_infra/lib/config/cluster-config/clusters.yml

@@ -4,127 +4,127 @@ clusters:
   - name: collector-ci-arm64-1-29
     version: "1.29"
     launch_type: ec2
-    instance_type: m6g.large
+    instance_type: m6g.xlarge
   - name: collector-ci-amd64-1-29
     version: "1.29"
     launch_type: ec2
-    instance_type: "m5.large"
+    instance_type: "m5.xlarge"
   - name: collector-ci-fargate-1-29
     version: "1.29"
     launch_type: fargate
   - name: collector-ci-arm64-1-30
     version: "1.30"
     launch_type: ec2
-    instance_type: m6g.large
+    instance_type: m6g.xlarge
   - name: collector-ci-amd64-1-30
     version: "1.30"
     launch_type: ec2
-    instance_type: "m5.large"
+    instance_type: "m5.xlarge"
   - name: collector-ci-fargate-1-30
     version: "1.30"
     launch_type: fargate
   - name: collector-ci-arm64-1-31
     version: "1.31"
     launch_type: ec2
-    instance_type: m6g.large
+    instance_type: m6g.xlarge
   - name: collector-ci-amd64-1-31
     version: "1.31"
     launch_type: ec2
-    instance_type: "m5.large"
+    instance_type: "m5.xlarge"
   - name: collector-ci-fargate-1-31
     version: "1.31"
     launch_type: fargate
   - name: collector-ci-arm64-1-32
     version: "1.32"
     launch_type: ec2
-    instance_type: m6g.large
+    instance_type: m6g.xlarge
   - name: collector-ci-amd64-1-32
     version: "1.32"
     launch_type: ec2
-    instance_type: "m5.large"
+    instance_type: "m5.xlarge"
   - name: collector-ci-fargate-1-32
     version: "1.32"
     launch_type: fargate
   - name: collector-ci-arm64-1-33
     version: "1.33"
     launch_type: ec2
-    instance_type: m6g.large
+    instance_type: m6g.xlarge
   - name: collector-ci-amd64-1-33
     version: "1.33"
     launch_type: ec2
-    instance_type: "m5.large"
+    instance_type: "m5.xlarge"
   - name: collector-ci-fargate-1-33
     version: "1.33"
     launch_type: fargate
   - name: collector-ci-arm64-1-34
     version: "1.34"
     launch_type: ec2
-    instance_type: m6g.large
+    instance_type: m6g.xlarge
   - name: collector-ci-amd64-1-34
     version: "1.34"
     launch_type: ec2
-    instance_type: "m5.large"
+    instance_type: "m5.xlarge"
   - name: collector-ci-fargate-1-34
     version: "1.34"
     launch_type: fargate
 # operator test clusters
   - name: operator-ci-amd64-1-29
     version: "1.29"
     launch_type: ec2
-    instance_type: m5.large
+    instance_type: m5.xlarge
     cert_manager: true
   - name: operator-ci-arm64-1-29
     version: "1.29"
     launch_type: ec2
-    instance_type: m6g.large
+    instance_type: m6g.xlarge
     cert_manager: true
   - name: operator-ci-amd64-1-30
     version: "1.30"
     launch_type: ec2
-    instance_type: m5.large
+    instance_type: m5.xlarge
     cert_manager: true
   - name: operator-ci-arm64-1-30
     version: "1.30"
     launch_type: ec2
-    instance_type: m6g.large
+    instance_type: m6g.xlarge
     cert_manager: true
   - name: operator-ci-amd64-1-31
     version: "1.31"
     launch_type: ec2
-    instance_type: m5.large
+    instance_type: m5.xlarge
     cert_manager: true
   - name: operator-ci-arm64-1-31
     version: "1.31"
     launch_type: ec2
-    instance_type: m6g.large
+    instance_type: m6g.xlarge
     cert_manager: true
   - name: operator-ci-amd64-1-32
     version: "1.32"
     launch_type: ec2
-    instance_type: m5.large
+    instance_type: m5.xlarge
     cert_manager: true
   - name: operator-ci-arm64-1-32
     version: "1.32"
     launch_type: ec2
-    instance_type: m6g.large
+    instance_type: m6g.xlarge
     cert_manager: true
   - name: operator-ci-amd64-1-33
     version: "1.33"
     launch_type: ec2
-    instance_type: m5.large
+    instance_type: m5.xlarge
     cert_manager: true
   - name: operator-ci-arm64-1-33
     version: "1.33"
     launch_type: ec2
-    instance_type: m6g.large
+    instance_type: m6g.xlarge
     cert_manager: true
   - name: operator-ci-amd64-1-34
     version: "1.34"
     launch_type: ec2
-    instance_type: m5.large
+    instance_type: m5.xlarge
     cert_manager: true
   - name: operator-ci-arm64-1-34
     version: "1.34"
     launch_type: ec2
-    instance_type: m6g.large
+    instance_type: m6g.xlarge
     cert_manager: true

cdk_infra/lib/stacks/eks/ec2-cluster-stack.ts

@@ -69,6 +69,8 @@ export class EC2Stack extends Stack {
       instanceTypes: props.instanceTypes,
       cluster: this.cluster,
       minSize: 2,
+      maxSize: 4,
+      desiredSize: 3,
       subnets: { subnetType: ec2.SubnetType.PUBLIC },
       launchTemplateSpec: {
         id: lt.launchTemplateId as string,

terraform/Makefile

@@ -22,7 +22,7 @@ postBatchClean:
 
 .PHONY: checkCacheHits
 checkCacheHits:
-	cat test-case-batch | xargs -L1 -P1 ./checkCacheHit.sh
+	cat test-case-batch | xargs -L1 -P10 ./checkCacheHit.sh
 
 .PHONY: terraformCleanup
 terraformCleanup: test-case-batch

terraform/basic_components/main.tf

@@ -23,7 +23,7 @@ module "common" {
 locals {
   otconfig_path = fileexists("${var.testcase}/otconfig.tpl") ? "${var.testcase}/otconfig.tpl" : module.common.default_otconfig_path
 
-  subnet_ids_list = data.aws_subnets.aoc_public_subnet_ids.ids
+  subnet_ids_list = data.aws_subnets.aoc_instance_subnet_ids.ids
 
   subnet_ids_random_index = random_id.subnetSelector.dec % length(local.subnet_ids_list)
 
@@ -61,7 +61,7 @@ data "aws_subnets" "aoc_private_subnet_ids" {
   }
 }
 
-# return public subnets
+# return public subnets (one per AZ — used for ALBs/ECS which require exactly one subnet per AZ)
 data "aws_subnets" "aoc_public_subnet_ids" {
   filter {
     name   = "vpc-id"
@@ -77,6 +77,22 @@ data "aws_subnets" "aoc_public_subnet_ids" {
   }
 }
 
+# return /23 public subnets for EC2 instance placement (more IPs, avoids ALB conflicts)
+data "aws_subnets" "aoc_instance_subnet_ids" {
+  filter {
+    name   = "vpc-id"
+    values = [data.aws_vpc.aoc_vpc.id]
+  }
+  filter {
+    name = "tag:Name"
+    values = [
+      "${module.common.aoc_vpc_name}-public-${var.region}a-2",
+      "${module.common.aoc_vpc_name}-public-${var.region}b-2",
+      "${module.common.aoc_vpc_name}-public-${var.region}c-2",
+    ]
+  }
+}
+
 resource "random_id" "subnetSelector" {
   byte_length = 2
 }

terraform/basic_components/outputs.tf

@@ -38,7 +38,8 @@ output "otconfig_content" {
 }
 
 output "mocked_server_cert_content" {
-  value = local.mocked_server_cert_rendered_template
+  value     = local.mocked_server_cert_rendered_template
+  sensitive = true
 }
 
 output "sample_app_image_repo" {

terraform/common.tf

@@ -74,4 +74,13 @@ variable "aoc_vpc_name" {
 
 variable "aoc_vpc_security_group" {
   default = "aoc-vpc-security-group"
+}
+
+variable "runner_sg_id" {
+  default = ""
+}
+
+variable "runner_ip" {
+  description = "CIDR of the CI runner (e.g. 1.2.3.4/32) for WinRM/SSH access"
+  default     = ""
 }

terraform/data_aoc_msk/outputs.tf

@@ -1,4 +1,4 @@
 output "cluster_data" {
   description = "Cluster data for the MSK Cluster as return by aws_msk_cluster terraform data moduel + the topic name that the testcase should use"
-  value       = merge(local.cluster_data, { topic = local.topic })
+  value       = merge(local.cluster_data, { topic = local.topic, kafka_version = var.cluster_version })
 }

terraform/ec2/amis.tf

@@ -49,30 +49,17 @@ variable "ami_family" {
       instance_type            = "c5a.large"
       otconfig_destination     = "C:\\ot-default.yml"
       download_command_pattern = "powershell -command \"Invoke-WebRequest -Uri %s -OutFile C:\\aws-otel-collector.msi\""
-      install_command          = "msiexec /i C:\\aws-otel-collector.msi"
-      start_command            = "$url = [System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String('CONFIGURATION_URI_PLACEHOLDER'))\n. 'C:\\Program Files\\Amazon\\AwsOtelCollector\\aws-otel-collector-ctl.ps1' -ConfigLocation $url -FeatureGates 'FEATUREGATE_PLACEHOLDER' -Action start"
+      install_command          = "msiexec /i C:\\aws-otel-collector.msi /qn /norestart"
+      start_command            = "powershell -Command \"$url = [System.Text.Encoding]::ASCII.GetString([System.Convert]::FromBase64String('CONFIGURATION_URI_PLACEHOLDER')); & 'C:\\Program Files\\Amazon\\AwsOtelCollector\\aws-otel-collector-ctl.ps1' -ConfigLocation $url -FeatureGates 'FEATUREGATE_PLACEHOLDER' -Action start; Start-Sleep -Seconds 15\""
       status_command           = "powershell \"& 'C:\\Program Files\\Amazon\\AwsOtelCollector\\aws-otel-collector-ctl.ps1' -Action status\""
       ssm_validate             = "powershell \"& 'C:\\Program Files\\Amazon\\AwsOtelCollector\\aws-otel-collector-ctl.ps1' -Action status\" | findstr running"
       connection_type          = "winrm"
       user_data                = <<EOF
 <powershell>
-winrm quickconfig -q
-winrm set winrm/config/winrs '@{MaxShellsPerUser="100"}'
-winrm set winrm/config/winrs '@{MaxConcurrentUsers="30"}'
-winrm set winrm/config/winrs '@{MaxProcessesPerShell="100"}'
-winrm set winrm/config/winrs '@{MaxMemoryPerShellMB="1024"}'
-winrm set winrm/config '@{MaxTimeoutms="1800000"}'
-winrm set winrm/config/service '@{AllowUnencrypted="true"}'
-winrm set winrm/config/service/auth '@{Basic="true"}'
-netsh advfirewall firewall add rule name="WinRM 5985" protocol=TCP dir=in localport=5985 action=allow
-netsh advfirewall firewall add rule name="WinRM 5986" protocol=TCP dir=in localport=5986 action=allow
-net stop winrm
-sc.exe config winrm start=auto
-net start winrm
-Set-NetFirewallProfile -Profile Public -Enabled False
+Set-NetFirewallProfile -Profile Domain,Public,Private -Enabled False
 </powershell>
 EOF
-      wait_cloud_init          = " "
+      wait_cloud_init          = "powershell -Command \"w32tm /resync /force; $i=0; while($i -lt 30) { $o = w32tm /stripchart /computer:169.254.169.123 /samples:1 /dataonly 2>&1; if($o -match '[+-]\\d+\\.\\d+s' -and [math]::Abs([double]($Matches[0] -replace 's','')) -lt 5) { break }; Start-Sleep 2; $i++ }; Write-Host 'Clock synced'\""
     }
   }
 }