fix: optimize spark configs for EMR on EKS (#571)

Author: vgkowski

core/src/emr-eks-platform/resources/k8s/emr-eks-config/critical.json

@@ -10,10 +10,10 @@
                 "spark.dynamicAllocation.minExecutors": "8",
                 "spark.dynamicAllocation.maxExecutors": "40",
                 "spark.kubernetes.allocation.batch.size": "8",
-                "spark.kubernetes.executor.request.cores": "7.5",
-                "spark.executor.memory": "26g",
-                "spark.kubernetes.driver.request.cores": "1.7",
-                "spark.driver.memory": "6g",
+                "spark.kubernetes.executor.request.cores": "7.7",
+                "spark.executor.memory": "20800",
+                "spark.kubernetes.driver.request.cores": "2.5",
+                "spark.driver.memory": "6800m",
                 "spark.dynamicAllocation.executorAllocationRatio": "1",
                 "spark.dynamicAllocation.shuffleTracking.enabled": "true",
                 "spark.dynamicAllocation.shuffleTracking.timeout": "300s",

core/src/emr-eks-platform/resources/k8s/emr-eks-config/job-template-critical.json

@@ -19,10 +19,10 @@
                         "spark.dynamicAllocation.minExecutors": "8",
                         "spark.dynamicAllocation.maxExecutors": "40",
                         "spark.kubernetes.allocation.batch.size": "8",
-                        "spark.kubernetes.executor.request.cores": "7.5",
-                        "spark.executor.memory": "26G",
-                        "spark.kubernetes.driver.request.cores": "1.7",
-                        "spark.driver.memory": "6G",
+                        "spark.kubernetes.executor.request.cores": "7.7",
+                        "spark.executor.memory": "20800m",
+                        "spark.kubernetes.driver.request.cores": "2.5",
+                        "spark.driver.memory": "6800m",
                         "spark.dynamicAllocation.executorAllocationRatio": "1",
                         "spark.dynamicAllocation.shuffleTracking.enabled": "true",
                         "spark.dynamicAllocation.shuffleTracking.timeout": "300s",

core/src/emr-eks-platform/resources/k8s/emr-eks-config/notebook-pod-template-ready.json

@@ -10,10 +10,10 @@
                 "spark.dynamicAllocation.minExecutors": "0",
                 "spark.dynamicAllocation.maxExecutors": "20",
                 "spark.kubernetes.allocation.batch.size": "2",
-                "spark.kubernetes.executor.request.cores": "3.3",
-                "spark.executor.memory": "12700m",
-                "spark.kubernetes.driver.request.cores": "1.7",
-                "spark.driver.memory": "2500m",
+                "spark.kubernetes.executor.request.cores": "3.7",
+                "spark.executor.memory": "10200m",
+                "spark.kubernetes.driver.request.cores": "1.6",
+                "spark.driver.memory": "4700m",
                 "spark.dynamicAllocation.executorAllocationRatio": "1",
                 "spark.dynamicAllocation.shuffleTracking.enabled": "true",
                 "spark.dynamicAllocation.shuffleTracking.timeout": "300s",

core/src/emr-eks-platform/resources/k8s/emr-eks-config/shared.json

@@ -10,10 +10,10 @@
                 "spark.dynamicAllocation.minExecutors": "8",
                 "spark.dynamicAllocation.maxExecutors": "40",
                 "spark.kubernetes.allocation.batch.size": "8",
-                "spark.kubernetes.executor.request.cores": "3.3",
-                "spark.executor.memory": "12700m",
+                "spark.kubernetes.executor.request.cores": "3.8",
+                "spark.executor.memory": "10400m",
                 "spark.kubernetes.driver.request.cores": "1.7",
-                "spark.driver.memory": "6g",
+                "spark.driver.memory": "4900m",
                 "spark.dynamicAllocation.executorAllocationRatio": "1",
                 "spark.dynamicAllocation.shuffleTracking.enabled": "true",
                 "spark.dynamicAllocation.shuffleTracking.timeout": "300s",