Merge pull request #477 from dciabrin/chainsaw-err-reporting

Improve chainsaw error reporting

Author: openshift-merge-bot[bot]

test/chainsaw/common/dump-resources.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Dump various resources still running in the environment, for debugging purpose
+# This is useful because all resources created by chainsaw are automatically deleted
+# before the we capture logs or the environment state in CI
+oc -n $NAMESPACE get jobs
+oc -n $NAMESPACE get pods
+oc -n $NAMESPACE get pvc
+oc -n $NAMESPACE get pv
+
+# Optional objects to be described
+for i in $*; do
+    echo inspecting $i
+    oc -n $NAMESPACE describe $i
+    oc -n $NAMESPACE logs $i
+done
+true

test/chainsaw/tests/backup-capture/chainsaw-test.yaml

@@ -27,7 +27,9 @@ spec:
     cleanup:
     - script:
         content: |
-          oc -n $NAMESPACE delete pvc mysql-backup-openstack || true
+          if ! oc -n $NAMESPACE get galerabackup/openstack; then
+            oc -n $NAMESPACE delete pvc mysql-backup-openstack || true
+          fi
 
   - name: setup a backup for a 1-node Galera cluster
     description: checks that backup CR is correctly set up
@@ -70,6 +72,16 @@ spec:
           value: ($replicas)
         content: |
           ../../common/backup-assert.sh backup-${REPLICAS}node
+    cleanup:
+    - script: &backup_cleanup
+        env:
+        - name: REPLICAS
+          value: ($replicas)
+        content: |
+          if oc -n $NAMESPACE get pods -l batch.kubernetes.io/job-name=backup-${REPLICAS}node --no-headers | grep -v -q Completed; then
+            echo ERROR during backup job
+            ../../common/dump-resources.sh `oc -n $NAMESPACE get pods -l batch.kubernetes.io/job-name=backup-${REPLICAS}node -o name`
+          fi
 
   - name: Scale Galera cluster to 3-node
     bindings:
@@ -90,6 +102,8 @@ spec:
     try:
     - script: *backup
     - script: *backup_check
+    cleanup:
+    - script: *backup_cleanup
 
   - name: Scale Galera cluster to 1-node
     bindings:
@@ -117,16 +131,17 @@ spec:
     cleanup:
     - script:
         content: |
-          oc -n $NAMESPACE delete pvc mysql-backup-openstack2nd || true
-
+          if ! oc -n $NAMESPACE get galerabackup/openstack2nd; then
+            oc -n $NAMESPACE delete pvc mysql-backup-openstack2nd || true
+          fi
 
   - name: backup the two running clusters
     description: checks that the two clusters can be backed up concurrently
     try:
     - script:
         content: |
           set -e
-          oc -n $NAMESPACE delete --ignore-not-found=true job/backup-1node
+          oc -n $NAMESPACE delete --ignore-not-found=true job/backup-1node --wait
           oc -n $NAMESPACE create job --from=cronjob/backup-openstack backup-1node
           oc -n $NAMESPACE create job --from=cronjob/backup-openstack2nd backup-1node-2nd
     - script:
@@ -137,6 +152,13 @@ spec:
           oc -n $NAMESPACE wait --timeout=${WAIT_TIMEOUT} --for=condition=complete job/backup-1node job/backup-1node-2nd
           ../../common/backup-assert.sh backup-1node
           ../../common/backup-assert.sh backup-1node-2nd
+    cleanup:
+    - script:
+        content: |
+          if [ "$(oc -n $NAMESPACE get pods | grep '^backup-1node' | grep -c Completed)" -ne 2 ]; then
+            echo ERROR during backup job
+            ../../common/dump-resources.sh `oc -n $NAMESPACE get pods | sed -ne 's%^\(backup-1node[^ ]*\).*%pod/\1%p' | xargs echo`
+          fi
 
   - name: Trigger a backup with a custom timestamp marker
     description: make sure that a custom timestamp can be passed to the backup script
@@ -154,3 +176,10 @@ spec:
           oc -n $NAMESPACE wait --timeout=${WAIT_TIMEOUT} --for=condition=complete job/custom-timestamp
           ../../common/backup-assert.sh custom-timestamp
           oc -n $NAMESPACE get pod -o name -l job-name=custom-timestamp | xargs oc -n $NAMESPACE logs | grep 'Starting backup .* - chainsaw_unit_test'
+    cleanup:
+    - script:
+        content: |
+          if oc -n $NAMESPACE get pods -l batch.kubernetes.io/job-name=custom-timestamp --no-headers | grep -v -q Completed; then
+            echo ERROR during backup job
+            ../../common/dump-resources.sh `oc -n $NAMESPACE get pods -l batch.kubernetes.io/job-name=custom-timestamp -o name`
+          fi

test/chainsaw/tests/backup-configs/chainsaw-test.yaml

@@ -33,7 +33,9 @@ spec:
     cleanup:
     - script:
         content: |
-          oc -n $NAMESPACE delete pvc mysql-backup-openstack mysql-transfer-openstack || true
+          if ! oc -n $NAMESPACE get galerabackup/openstack; then
+            oc -n $NAMESPACE delete pvc mysql-backup-openstack mysql-transfer-openstack || true
+          fi
 
   - name: backup the 1-node Galera cluster
     description: checks that the cluster is correctly backed up
@@ -56,3 +58,10 @@ spec:
           value: ($replicas)
         content: |
           ../../common/backup-assert.sh backup-${REPLICAS}node
+    cleanup:
+    - script:
+        content: |
+          if oc -n $NAMESPACE get pods -l batch.kubernetes.io/job-name=backup-1node --no-headers | grep -v -q Completed; then
+            echo ERROR during backup job
+            ../../common/dump-resources.sh `oc -n $NAMESPACE get pods -l batch.kubernetes.io/job-name=backup-1node -o name`
+          fi

test/chainsaw/tests/restore/chainsaw-test.yaml

@@ -44,7 +44,12 @@ spec:
     cleanup:
     - script:
         content: |
-          oc -n $NAMESPACE delete pvc mysql-backup-cluster2backup mysql-backup-openstackbackup || true
+          if ! oc -n $NAMESPACE get galerabackup/openstackbackup; then
+            oc -n $NAMESPACE delete pvc mysql-backup-openstackbackup || true
+          fi
+          if ! oc -n $NAMESPACE get galerabackup/cluster2backup; then
+            oc -n $NAMESPACE delete pvc mysql-backup-cluster2backup || true
+          fi
 
   - name: trigger a backup of Galera cluster
     description: checks that the cluster is correctly backed up
@@ -58,6 +63,13 @@ spec:
           oc -n $NAMESPACE create job --from=cronjob/backup-cluster2backup backupjob2
           oc -n $NAMESPACE wait --for=condition=complete --timeout=${WAIT_TIMEOUT} job/backupjob1
           oc -n $NAMESPACE wait --for=condition=complete --timeout=${WAIT_TIMEOUT} job/backupjob2
+    cleanup:
+    - script:
+        content: |
+          if [ "$(oc -n $NAMESPACE get pods | grep -e '^openstackbackup' -e 'cluster2backup' | grep -c Completed)" -ne 2 ]; then
+            echo ERROR during backup job
+            ../../common/dump-resources.sh `oc -n $NAMESPACE get pods | sed -ne 's%^\(\(openstackbackup|cluster2backup\)[^ ]*\).*%pod/\1%p' | xargs echo`
+          fi
 
   - name: remove data in the running Galera clusters
     try: