Merge pull request #474 from dciabrin/chainsaw-backup-timeout

Make chainsaw wait timeout configurable

Author: openshift-merge-bot[bot]

test/chainsaw/config.yaml

@@ -7,7 +7,7 @@ spec:
     assert: 180s
     delete: 180s
     cleanup: 180s
-    exec: 120s
+    exec: 180s
   execution:
     failFast: true
     parallel: 1

test/chainsaw/tests/backup-capture/chainsaw-test.yaml

@@ -3,6 +3,9 @@ kind: Test
 metadata:
   name: backup-capture
 spec:
+  bindings:
+  - name: ocWaitTimeout
+    value: 160s
   steps:
   - name: create a backup CR without Galera cluster
     description: backup CR should not create objects without Galera CR detected
@@ -51,14 +54,18 @@ spec:
     try:
     - script: &backup
         env:
+        - &wait_env
+          name: WAIT_TIMEOUT
+          value: ($ocWaitTimeout)
         - name: REPLICAS
           value: ($replicas)
         content: |
           set -e
           oc -n $NAMESPACE create job --from=cronjob/backup-openstack backup-${REPLICAS}node
-          oc -n $NAMESPACE wait --for=condition=complete job/backup-${REPLICAS}node
+          oc -n $NAMESPACE wait --for=condition=complete --timeout=${WAIT_TIMEOUT} job/backup-${REPLICAS}node
     - script: &backup_check
         env:
+        - *wait_env
         - name: REPLICAS
           value: ($replicas)
         content: |
@@ -123,9 +130,11 @@ spec:
           oc -n $NAMESPACE create job --from=cronjob/backup-openstack backup-1node
           oc -n $NAMESPACE create job --from=cronjob/backup-openstack2nd backup-1node-2nd
     - script:
+        env:
+        - *wait_env
         content: |
           set -e
-          oc -n $NAMESPACE wait --for=condition=complete job/backup-1node job/backup-1node-2nd
+          oc -n $NAMESPACE wait --timeout=${WAIT_TIMEOUT} --for=condition=complete job/backup-1node job/backup-1node-2nd
           ../../common/backup-assert.sh backup-1node
           ../../common/backup-assert.sh backup-1node-2nd
 
@@ -137,9 +146,11 @@ spec:
           set -o pipefail
           oc -n $NAMESPACE create job --from=cronjob/backup-openstack custom-timestamp --dry-run=client -o json | jq '.spec.template.spec.containers[0].env += [{ "name": "BACKUP_TIMESTAMP", value:"chainsaw_unit_test" }]' | oc -n $NAMESPACE apply -f -
     - script:
+        env:
+        - *wait_env
         content: |
           set -e
           set -o pipefail
-          oc -n $NAMESPACE wait --for=condition=complete job/custom-timestamp
+          oc -n $NAMESPACE wait --timeout=${WAIT_TIMEOUT} --for=condition=complete job/custom-timestamp
           ../../common/backup-assert.sh custom-timestamp
           oc -n $NAMESPACE get pod -o name -l job-name=custom-timestamp | xargs oc -n $NAMESPACE logs | grep 'Starting backup .* - chainsaw_unit_test'

test/chainsaw/tests/backup-configs/chainsaw-test.yaml

@@ -3,6 +3,9 @@ kind: Test
 metadata:
   name: backup-configs
 spec:
+  bindings:
+  - name: ocWaitTimeout
+    value: 160s
   steps:
   - name: setup a 1-node cluster and a backup CR with dedicated transfer PVC
     description: prechecks that backup CR is correctly set up
@@ -40,11 +43,13 @@ spec:
     try:
     - script:
         env:
+        - name: WAIT_TIMEOUT
+          value: ($ocWaitTimeout)
         - name: REPLICAS
           value: ($replicas)
         content: |
           oc -n $NAMESPACE create job --from=cronjob/backup-openstack backup-${REPLICAS}node
-          oc -n $NAMESPACE wait --for=condition=complete job/backup-${REPLICAS}node
+          oc -n $NAMESPACE wait --for=condition=complete --timeout=${WAIT_TIMEOUT} job/backup-${REPLICAS}node
     - script:
         env:
         - name: REPLICAS

test/chainsaw/tests/restore/chainsaw-test.yaml

@@ -3,6 +3,9 @@ kind: Test
 metadata:
   name: restore
 spec:
+  bindings:
+  - name: ocWaitTimeout
+    value: 160s
   steps:
   - name: setup two different 1-node Galera clusters (TLS and non-TLS)
     bindings:
@@ -47,11 +50,14 @@ spec:
     description: checks that the cluster is correctly backed up
     try:
     - script:
+        env:
+        - name: WAIT_TIMEOUT
+          value: ($ocWaitTimeout)
         content: |
           oc -n $NAMESPACE create job --from=cronjob/backup-openstackbackup backupjob1
           oc -n $NAMESPACE create job --from=cronjob/backup-cluster2backup backupjob2
-          oc -n $NAMESPACE wait --for=condition=complete job/backupjob1
-          oc -n $NAMESPACE wait --for=condition=complete job/backupjob2
+          oc -n $NAMESPACE wait --for=condition=complete --timeout=${WAIT_TIMEOUT} job/backupjob1
+          oc -n $NAMESPACE wait --for=condition=complete --timeout=${WAIT_TIMEOUT} job/backupjob2
 
   - name: remove data in the running Galera clusters
     try:

test/chainsaw/tests/service/chainsaw-test.yaml

@@ -4,6 +4,9 @@ metadata:
   name: failover
 spec:
   description: Service failover test case
+  bindings:
+  - name: ocWaitTimeout
+    value: 160s
   steps:
   - name: Setup
     description: Deploy a cluster and a background pod with long-running DB connection
@@ -32,14 +35,16 @@ spec:
           value: ($stdout)
     - script:
         env:
+        - name: WAIT_TIMEOUT
+          value: ($ocWaitTimeout)
         - name: PASSWORD
           value: ($db_password)
         - name: IMAGE
           value: ($pod_image)
         skipLogOutput: true
         content: |
           oc -n $NAMESPACE run failover-check --image=$IMAGE --env="DB_ROOT_PASSWORD=$PASSWORD" --command -- dumb-init bash -c "$(cat ../../scripts/connect.sh)"
-          oc -n $NAMESPACE wait --for=jsonpath='{.status.phase}'=Running pod failover-check
+          oc -n $NAMESPACE wait --for=jsonpath='{.status.phase}'=Running --timeout=${WAIT_TIMEOUT} pod failover-check
 
     cleanup:
     - delete:
@@ -58,11 +63,13 @@ spec:
         - name: endpoint
           value: ($stdout)
     - script:
-        env:
+        env: &wait_env
+        - name: WAIT_TIMEOUT
+          value: ($ocWaitTimeout)
         - name: ENDPOINT
           value: ($endpoint)
         content: |
-          oc wait -n $NAMESPACE --for=jsonpath='{.status.readyReplicas}'=3 statefulset openstack-galera
+          oc wait -n $NAMESPACE --for=jsonpath='{.status.readyReplicas}'=3 --timeout=${WAIT_TIMEOUT} statefulset openstack-galera
           oc delete -n $NAMESPACE --wait pod $ENDPOINT
     - script:
         content: *endpoint_cmd
@@ -85,11 +92,9 @@ spec:
     try:
     - script: *get_endpoint
     - script:
-        env:
-        - name: ENDPOINT
-          value: ($endpoint)
+        env: *wait_env
         content: |
-          oc wait -n $NAMESPACE --for=jsonpath='{.status.readyReplicas}'=3 statefulset openstack-galera
+          oc wait -n $NAMESPACE --for=jsonpath='{.status.readyReplicas}'=3 --timeout=${WAIT_TIMEOUT} statefulset openstack-galera
           current=$ENDPOINT
           oc rsh -n $NAMESPACE $ENDPOINT killall -s STOP /usr/libexec/mysqld
           while [ "$current" = "$ENDPOINT" ]; do
@@ -109,11 +114,9 @@ spec:
     try:
     - script: *get_endpoint
     - script:
-        env:
-        - name: ENDPOINT
-          value: ($endpoint)
+        env: *wait_env
         content: |
-          oc wait -n $NAMESPACE --for=jsonpath='{.status.readyReplicas}'=3 statefulset openstack-galera
+          oc wait -n $NAMESPACE --for=jsonpath='{.status.readyReplicas}'=3 --timeout=${WAIT_TIMEOUT} statefulset openstack-galera
           echo -ne openstack-galera-{0,1,2}\\n | grep -v $ENDPOINT | head -1 | xargs oc delete --wait -n $NAMESPACE --wait pod
     - script:
         content: *endpoint_cmd