Add kuttl test to catch any missing nodeSelectors

dnsmasq that is created by the kuttl tests and
ovn-controller-*-config is the only exception (sets a specific nodeName)

Author: olliewalsh

config/samples/nodeselectors/global/kustomization.yaml

@@ -0,0 +1,14 @@
+resources:
+- ../../base/openstackcontrolplane
+
+patches:
+- target:
+    kind: OpenStackControlPlane
+    name: .*
+  patch: |-
+    - op: replace
+      path: /metadata/name
+      value: openstack
+- target:
+    kind: OpenStackControlPlane
+  path: patch.yaml

config/samples/nodeselectors/global/patch.yaml

@@ -0,0 +1,7 @@
+apiVersion: core.openstack.org/v1beta1
+kind: OpenStackControlPlane
+metadata:
+  name: openstack
+spec:
+  nodeSelector:
+    node-role.kubernetes.io/worker: ""

tests/kuttl/tests/ctlplane-nodeselectors/01-assert-deploy-openstack.yaml

@@ -0,0 +1 @@
+../../common/assert-sample-deployment.yaml

tests/kuttl/tests/ctlplane-nodeselectors/01-deploy-openstack.yaml

@@ -0,0 +1,5 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |
+      oc kustomize ../../../../config/samples/nodeselectors/global | oc apply -n $NAMESPACE -f -

tests/kuttl/tests/ctlplane-nodeselectors/02-assert-nodeselector.yaml

@@ -0,0 +1,23 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - script: |
+      echo "Checking all pods have expected nodeselector"
+      EXPECTED_NODE_SELECTOR="node-role.kubernetes.io/worker:"
+      BAD_OR_MISSING_NODE_SELECTOR=$(oc get pods -n $NAMESPACE -l service!=dnsmasq -o=go-template --template='{{ range .items }}{{ .metadata.name}}: {{ .spec.nodeSelector }}{{"\n"}}{{ end }}' | grep -v 'ovn-controller-.*-config' | sed -e '\!map\['"$EXPECTED_NODE_SELECTOR"'\]$!d')
+      BAD_OR_MISSING_NODE_SELECTOR_COUNT=$(echo -n "$BAD_OR_MISSING_NODE_SELECTOR" | wc -l)
+      if [ $BAD_OR_MISSING_NODE_SELECTOR_COUNT -ne 0 ]; then
+        echo "Found $BAD_OR_MISSING_NODE_SELECTOR_COUNT pods with bad or missing nodeselector:"
+        echo "$BAD_OR_MISSING_NODE_SELECTOR"
+        exit 1
+      fi
+  - script: |
+      echo "Checking all cronjobs have expected nodeselector"
+      EXPECTED_NODE_SELECTOR="node-role.kubernetes.io/worker:"
+      BAD_OR_MISSING_NODE_SELECTOR=$(oc get cronjobs -n $NAMESPACE -o=go-template --template='{{ range .items }}{{ .metadata.name}}: {{ .spec.jobTemplate.spec.template.spec.nodeSelector }}{{"\n"}}{{ end }}' | grep -v 'ovn-controller-.*-config' | sed -e '\!map\['"$EXPECTED_NODE_SELECTOR"'\]$!d')
+      BAD_OR_MISSING_NODE_SELECTOR_COUNT=$(echo -n "$BAD_OR_MISSING_NODE_SELECTOR" | wc -l)
+      if [ $BAD_OR_MISSING_NODE_SELECTOR_COUNT -ne 0 ]; then
+        echo "Found $BAD_OR_MISSING_NODE_SELECTOR_COUNT cronjobs with bad or missing nodeselector:"
+        echo "$BAD_OR_MISSING_NODE_SELECTOR"
+        exit 1
+      fi

tests/kuttl/tests/ctlplane-nodeselectors/03-assert-deploy-openstack.yaml

@@ -0,0 +1 @@
+../../common/assert-sample-deployment.yaml

tests/kuttl/tests/ctlplane-nodeselectors/03-update-nodeselector.yaml

@@ -0,0 +1,16 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 60
+commands:
+  - script: |
+      oc patch dnsmasq -n $NAMESPACE dnsmasq --type='json' -p='[{
+        "op": "replace",
+        "path": "/spec/nodeSelector",
+        "value": {"kubernetes.io/os":"linux"}
+      }]'
+  - script: |
+      oc patch openstackcontrolplane -n $NAMESPACE openstack --type='json' -p='[{
+        "op": "replace",
+        "path": "/spec/nodeSelector",
+        "value": {"kubernetes.io/os":"linux"}
+      }]'

tests/kuttl/tests/ctlplane-nodeselectors/04-assert-nodeselector.yaml

@@ -0,0 +1,23 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - script: |
+      echo "Checking all running pods have new nodeselector"
+      EXPECTED_NODE_SELECTOR="kubernetes.io/os:linux"
+      BAD_OR_MISSING_NODE_SELECTOR=$(oc get pods -n $NAMESPACE -l service!=dnsmasq --field-selector=status.phase=Running -o=go-template --template='{{ range .items }}{{ .metadata.name}}: {{ .spec.nodeSelector }}{{"\n"}}{{ end }}' | grep -v 'ovn-controller-.*-config' | sed -e '\!map\['"$EXPECTED_NODE_SELECTOR"'\]$!d')
+      BAD_OR_MISSING_NODE_SELECTOR_COUNT=$(echo -n "$BAD_OR_MISSING_NODE_SELECTOR" | wc -l)
+      if [ $BAD_OR_MISSING_NODE_SELECTOR_COUNT -ne 0 ]; then
+        echo "Found $BAD_OR_MISSING_NODE_SELECTOR_COUNT pods with bad or missing nodeselector:"
+        echo "$BAD_OR_MISSING_NODE_SELECTOR"
+        exit 1
+      fi
+  - script: |
+      echo "Checking all cronjobs have expected nodeselector"
+      EXPECTED_NODE_SELECTOR="kubernetes.io/os:linux"
+      BAD_OR_MISSING_NODE_SELECTOR=$(oc get cronjobs -n $NAMESPACE -o=go-template --template='{{ range .items }}{{ .metadata.name}}: {{ .spec.jobTemplate.spec.template.spec.nodeSelector }}{{"\n"}}{{ end }}' | grep -v 'ovn-controller-.*-config' | sed -e '\!map\['"$EXPECTED_NODE_SELECTOR"'\]$!d')
+      BAD_OR_MISSING_NODE_SELECTOR_COUNT=$(echo -n "$BAD_OR_MISSING_NODE_SELECTOR" | wc -l)
+      if [ $BAD_OR_MISSING_NODE_SELECTOR_COUNT -ne 0 ]; then
+        echo "Found $BAD_OR_MISSING_NODE_SELECTOR_COUNT cronjobs with bad or missing nodeselector:"
+        echo "$BAD_OR_MISSING_NODE_SELECTOR"
+        exit 1
+      fi

tests/kuttl/tests/ctlplane-nodeselectors/05-cleanup.yaml

@@ -0,0 +1,13 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+- apiVersion: core.openstack.org/v1beta1
+  kind: OpenStackControlPlane
+  name: openstack
+commands:
+- script: |
+    oc delete --ignore-not-found=true -n $NAMESPACE pvc \
+    srv-swift-storage-0
+    oc delete secret --ignore-not-found=true combined-ca-bundle -n $NAMESPACE
+    oc delete secret -l service-cert -n $NAMESPACE
+    oc delete secret -l ca-cert -n $NAMESPACE

tests/kuttl/tests/ctlplane-nodeselectors/05-errors-cleanup.yaml

@@ -0,0 +1 @@
+../../common/errors_cleanup_openstack.yaml