firelens-stability: add cases

matthewfala · bendu · commit 37fea75ca53a · 2024-04-24T13:41:05.000-07:00
Signed-off-by: Matthew Fala &lt;falamatt@amazon.com&gt;
diff --git a/apps/firelens-stability/collections/ecs-firelens-one-flow-tests/rainbow-stability/case-config.json b/apps/firelens-stability/collections/ecs-firelens-one-flow-tests/rainbow-stability/case-config.json
@@ -1,6 +1,6 @@
 {
     "config": {
-        "taskCount": 5
+        "taskCount": 1
     },
     "definitions": {
         "kbps_throughput_tcp": 10,
diff --git a/apps/firelens-stability/collections/ecs-firelens-one-flow-tests/rainbow-stability/cases/cloudwatch-go-content-input-tcp.json b/apps/firelens-stability/collections/ecs-firelens-one-flow-tests/rainbow-stability/cases/cloudwatch-go-content-input-tcp.json
@@ -0,0 +1,7 @@
+{
+    "definitions": {
+        "rainbow_cw_c_on": false,
+        "rainbow_cw_go_on": true,
+        "datajet_output_stdout": false
+    }
+}
diff --git a/apps/firelens-stability/collections/ecs-firelens-one-flow-tests/rainbow-stability/cases/cloudwatch-go-content-random.json b/apps/firelens-stability/collections/ecs-firelens-one-flow-tests/rainbow-stability/cases/cloudwatch-go-content-random.json
@@ -0,0 +1,7 @@
+{
+    "definitions": {
+        "rainbow_cw_c_on": false,
+        "rainbow_cw_go_on": true,
+        "datajet_generator_basic_contentType": "random"
+    }
+}
diff --git a/apps/firelens-stability/collections/ecs-firelens-one-flow-tests/rainbow-stability/cases/cloudwatch-go-firelens-vcpu-256.json b/apps/firelens-stability/collections/ecs-firelens-one-flow-tests/rainbow-stability/cases/cloudwatch-go-firelens-vcpu-256.json
@@ -0,0 +1,7 @@
+{
+    "definitions": {
+        "rainbow_cw_c_on": false,
+        "rainbow_cw_go_on": true,
+        "firelens_fluent_bit_container_vcpu": 256
+    }
+}
diff --git a/apps/firelens-stability/collections/ecs-firelens-one-flow-tests/rainbow-stability/cases/cloudwatch-go-length-1mb.json b/apps/firelens-stability/collections/ecs-firelens-one-flow-tests/rainbow-stability/cases/cloudwatch-go-length-1mb.json
@@ -0,0 +1,7 @@
+{
+    "definitions": {
+        "rainbow_cw_c_on": false,
+        "rainbow_cw_go_on": true,
+        "datajet_generator_basic_contentType": "random"
+    }
+}
diff --git a/apps/firelens-stability/collections/ecs-firelens-one-flow-tests/rainbow-stability/cases/cloudwatch-go-multiplex-5out.json b/apps/firelens-stability/collections/ecs-firelens-one-flow-tests/rainbow-stability/cases/cloudwatch-go-multiplex-5out.json
@@ -0,0 +1,7 @@
+{
+    "definitions": {
+        "rainbow_cw_c_on": false,
+        "rainbow_cw_go_on": true,
+        "rainbow_cw_go_multiple": 5
+    }
+}
diff --git a/apps/firelens-stability/collections/ecs-firelens-one-flow-tests/rainbow-stability/cases/cloudwatch-go-throughput-1mbps.json b/apps/firelens-stability/collections/ecs-firelens-one-flow-tests/rainbow-stability/cases/cloudwatch-go-throughput-1mbps.json
@@ -0,0 +1,7 @@
+{
+    "definitions": {
+        "rainbow_cw_c_on": false,
+        "rainbow_cw_go_on": true,
+        "kbps_throughput_tcp": 1000
+    }
+}
diff --git a/apps/firelens-stability/collections/ecs-firelens-one-flow-tests/rainbow-stability/cases/cloudwatch-go.json b/apps/firelens-stability/collections/ecs-firelens-one-flow-tests/rainbow-stability/cases/cloudwatch-go.json
@@ -1,7 +1,6 @@
 {
     "definitions": {
         "rainbow_cw_c_on": false,
-        "rainbow_cw_go_on": true,
-        "kbps_throughput_tcp": 1000
+        "rainbow_cw_go_on": true
     }
 }
diff --git a/apps/firelens-stability/templates/golden-path-mountebank-fargate-v01-11-2023/firelens-datajet.json b/apps/firelens-stability/templates/golden-path-mountebank-fargate-v01-11-2023/firelens-datajet.json
@@ -4,7 +4,8 @@
     "repeat": 100,
     "waitBefore": 5,
     "waitAfter": 10,
-    "waitBetween": 0
+    "waitBetween": 0,
+    "isAsync": true
   },
   "children": [
     {
diff --git a/apps/firelens-stability/templates/rainbow-fargate/default-config.json b/apps/firelens-stability/templates/rainbow-fargate/default-config.json
@@ -12,6 +12,10 @@
         "include_onepod": false,
         "kbps_throughput_tcp": 10,
         "debugS3Bucket": "{{managed.s3OutputBucket}}",
-        "debugS3KeyPrefix": "{{managed.s3OutputExecutionPath}}/{{managed.caseNameUnique}}"
+        "debugS3KeyPrefix": "{{managed.s3OutputExecutionPath}}/{{managed.caseNameUnique}}",
+        "datajet_generator_basic_contentLength": 1000,
+        "datajet_output_stdout": true,
+        "rainbow_cw_go_multiple": 1,
+        "firelens_fluent_bit_container_vcpu": 1024
     }
 }
diff --git a/apps/firelens-stability/templates/rainbow-fargate/firelens-datajet.json b/apps/firelens-stability/templates/rainbow-fargate/firelens-datajet.json
@@ -3,13 +3,18 @@
   "generator": {
     "name": "basic",
     "config": {
-      "contentLength": 1000,
+      "contentLength": {{definitions.datajet_generator_basic_contentLength}},
       "batchSize": 1,
       "logKey": "log",
       "contentType": "{{definitions.datajet_generator_basic_contentType}}",
       "contentRandomValueSet": "{{definitions.datajet_generator_basic_contentRandomValueSet}}"
     }
   },
+  {{#if definitions.datajet_output_stdout}}
+  "datajet": {
+    "name": "stdout"
+  },
+  {{else}}
   "datajet": {
     "name": "tcp",
     "config": {
@@ -18,7 +23,8 @@
       "addNewline": true
     }
   },
+  {{/if}}
   "stage": {
     "batchRate": {{definitions.kbps_throughput_tcp}}
   }
-}
+}
diff --git a/apps/firelens-stability/templates/rainbow-fargate/fluent-bit.conf b/apps/firelens-stability/templates/rainbow-fargate/fluent-bit.conf
@@ -17,6 +17,11 @@
     command   kill -11 `pgrep fluent-bit`
 {{/if}}
 
+{{#if definitions.datajet_output_stdout}}
+[INPUT]
+    Name        forward
+    unix_path   /var/run/fluent.sock
+{{else}}
 [INPUT]
     Name          tcp
     Listen        0.0.0.0
@@ -42,26 +47,29 @@
     # Mem_Buf_Limit <= Memory_limit / 2 / (number of INPUT sections + 1)
     # 64MB = 512MB / 2 / (4 INPUT sections)
     Mem_Buf_Limit 64MB
+{{/if}}
 
 {{#if definitions.rainbow_cw_c_on}}
 [OUTPUT]
     # Send any Application Logs to A CloudWatch Logs Group
     Name                cloudwatch_logs
-    Match               ApplicationLogs
+    Match               **
     region              {{config.region}}
     log_group_name      {{definitions.cw_log_group_name}}
-    log_stream_prefix   {{definitions.cw_log_stream_name_prefix}}_cw-c_${ECS_TASK_ID}
+    log_stream_prefix   {{definitions.cw_log_stream_name_prefix}}_multiple{{_idx}}_cw-c_${ECS_TASK_ID}__
     log_key             log
     auto_create_group   {{definitions.cw_auto_create_group}}
     retry_limit         3
 {{/if}}
 
 {{#if definitions.rainbow_cw_go_on}}
+{{#times definitions.rainbow_cw_go_multiple}}
 [OUTPUT]
     Name                cloudwatch
-    Match               ApplicationLogs
+    Match               **
     region              {{config.region}}
     log_group_name      {{definitions.cw_log_group_name}}
-    log_stream_prefix   {{definitions.cw_log_stream_name_prefix}}_cw-go_${ECS_TASK_ID}
+    log_stream_prefix   {{definitions.cw_log_stream_name_prefix}}_multiple{{_idx}}_cw-go_${ECS_TASK_ID}__
     auto_create_group   {{definitions.cw_auto_create_group}}
+{{/times}}
 {{/if}}
diff --git a/apps/firelens-stability/templates/rainbow-fargate/task-definition.json b/apps/firelens-stability/templates/rainbow-fargate/task-definition.json
@@ -87,9 +87,9 @@
         "image": "{{definitions.imageAwsForFluentBit}}",
         "command": [
           "/bin/sh", "-c",
-          "/init/fluent_bit_init_process; chmod +x /init/invoke_fluent_bit.sh; exec env ECS_TASK_ID=$(curl -s \"${ECS_CONTAINER_METADATA_URI_V4}/task\" | awk -F '\"TaskARN\":\"' '{print $2}' | cut -d \\\" -f 1 | cut -d \\\\ -f 3) /init/invoke_fluent_bit.sh"
+          "/init/fluent_bit_init_process; sed -i.bak '/@INCLUDE \\/fluent-bit\\/etc\\/fluent-bit.conf/d' /init/fluent-bit-init.conf; chmod +x /init/invoke_fluent_bit.sh; exec env ECS_TASK_ID=$(curl -s \"${ECS_CONTAINER_METADATA_URI_V4}/task\" | awk -F '\"TaskARN\":\"' '{print $2}' | cut -d \\\" -f 1 | cut -d \\\\ -f 3) /init/invoke_fluent_bit.sh"
         ],
-        "cpu": 1024,
+        "cpu": {{definitions.firelens_fluent_bit_container_vcpu}},
         "firelensConfiguration": {
           "type": "fluentbit"
         },
@@ -151,6 +151,19 @@
             "value": "{{managed.s3ResourcesPath}}/firelens-datajet.json"
           }
         ],
+        {{#if definitions.datajet_output_stdout}}
+        "logConfiguration": {
+          "logDriver":"awsfirelens",
+          "options": {
+            "Name": "cloudwatch_logs",
+            "region": "{{config.region}}",
+            "log_group_name": "garbage-group-trashed-via-container-task-definition-command-override",
+            "auto_create_group": "true",
+            "log_stream_name": "garbage-stream",
+            "retry_limit": "2"
+          }
+        },
+        {{else}}
         "logConfiguration": {
             "logDriver": "awslogs",
             "options": {
@@ -160,14 +173,15 @@
                 "awslogs-stream-prefix": "{{managed.caseNameUnique}}-datajet"
             }
         },
+        {{/if}}
         "linuxParameters": {
           "initProcessEnabled": true
         },
         "dependsOn": [{
             "containerName": "fluent-bit",
             "condition": "START"
         }]
-      },
+      }{{#unless definitions.datajet_output_stdout}},
       {
         "essential": false,
         "image": "{{definitions.imageHttpd}}",
@@ -189,6 +203,7 @@
         }],
         "memoryReservation": 100
       }
+      {{/unless}}
     ],
     "memory": "4096",
     "cpu": "1024"

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"config": {`
`3`		`- "taskCount": 5`
	`3`	`+ "taskCount": 1`
`4`	`4`	`},`
`5`	`5`	`"definitions": {`
`6`	`6`	`"kbps_throughput_tcp": 10,`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"definitions": {`
`3`	`3`	`"rainbow_cw_c_on": false,`
`4`		`- "rainbow_cw_go_on": true,`
`5`		`- "kbps_throughput_tcp": 1000`
	`4`	`+ "rainbow_cw_go_on": true`
`6`	`5`	`}`
`7`	`6`	`}`
Original file line number	Diff line number	Diff line change
`@@ -12,6 +12,10 @@`
`12`	`12`	`"include_onepod": false,`
`13`	`13`	`"kbps_throughput_tcp": 10,`
`14`	`14`	`"debugS3Bucket": "{{managed.s3OutputBucket}}",`
`15`		`- "debugS3KeyPrefix": "{{managed.s3OutputExecutionPath}}/{{managed.caseNameUnique}}"`
	`15`	`+ "debugS3KeyPrefix": "{{managed.s3OutputExecutionPath}}/{{managed.caseNameUnique}}",`
	`16`	`+ "datajet_generator_basic_contentLength": 1000,`
	`17`	`+ "datajet_output_stdout": true,`
	`18`	`+ "rainbow_cw_go_multiple": 1,`
	`19`	`+ "firelens_fluent_bit_container_vcpu": 1024`
`16`	`20`	`}`
`17`	`21`	`}`