HDDS-14635. kubernetes check fails after restarting datanodes (#9772)

Author: adoroszlai

hadoop-ozone/dist/src/main/k8s/definitions/ozone/config.yaml

@@ -38,7 +38,10 @@ data:
   OZONE-SITE.XML_ozone.scm.client.address: "scm-0.scm"
   OZONE-SITE.XML_ozone.scm.datanode.id.dir: "/data/metadata"
   OZONE-SITE.XML_ozone.scm.names: "scm-0.scm"
-  LOG4J.PROPERTIES_log4j.rootLogger: "INFO, stdout"
-  LOG4J.PROPERTIES_log4j.appender.stdout: "org.apache.log4j.ConsoleAppender"
-  LOG4J.PROPERTIES_log4j.appender.stdout.layout: "org.apache.log4j.PatternLayout"
-  LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern: "%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n"
+  OZONE-SITE.XML_ozone.client.max.retries: "15"
+  OZONE-SITE.XML_ozone.client.retry.interval: "1"
+  LOG4J.PROPERTIES_hadoop.root.logger: "INFO,console"
+  LOG4J.PROPERTIES_log4j.rootLogger: "${hadoop.root.logger}"
+  LOG4J.PROPERTIES_log4j.appender.console: "org.apache.log4j.ConsoleAppender"
+  LOG4J.PROPERTIES_log4j.appender.console.layout: "org.apache.log4j.PatternLayout"
+  LOG4J.PROPERTIES_log4j.appender.console.layout.ConversionPattern: "%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n"

hadoop-ozone/dist/src/main/k8s/examples/ozone/test.sh

@@ -35,6 +35,7 @@ execute_robot_test ${SCM} -v PREFIX:pre smoketest/freon/validate.robot
 kubectl delete pod datanode-0 datanode-1 datanode-2
 
 wait_for_startup
+wait_for_pipeline
 
 execute_robot_test ${SCM} -v PREFIX:pre smoketest/freon/validate.robot
 execute_robot_test ${SCM} -v PREFIX:post smoketest/freon/generate.robot

hadoop-ozone/dist/src/main/k8s/examples/testlib.sh

@@ -50,6 +50,23 @@ wait_for_startup(){
    fi
 }
 
+wait_for_pipeline() {
+  retry assert_pipeline_exists
+}
+
+execute_command_in_container() {
+   CONTAINER="$1"
+   shift 1 #Remove first argument which was the container name
+
+   kubectl exec "${CONTAINER}" -- "$@"
+}
+
+assert_pipeline_exists() {
+   local count
+   count=$(execute_command_in_container scm-0 ozone admin pipeline list --state OPEN --filter-by-factor THREE --json | jq -r 'length')
+   [[ $count -gt 0 ]]
+}
+
 all_pods_are_running() {
    local -i running=$(kubectl get pod --field-selector status.phase=Running | grep -v 'STATUS' | wc -l)
    local -i all=$(kubectl get pod | grep -v 'STATUS' | wc -l)