Parallel KUTTL test support

Author: abays

test/kuttl/common/create_custom_cert.sh

@@ -8,7 +8,7 @@ function wait_for_state() {
     local object="$1"
     local state="$2"
     local timeout="$3"
-    local namespace="${4:-openstack-kuttl-tests}"
+    local namespace="${4:-${NAMESPACE:-openstack-kuttl-tests}}"
 
     echo "Waiting for '${object}' in namespace '${namespace}' to become '${state}'..."
     oc wait --for=${state} --timeout=${timeout} ${object} -n="${namespace}"
@@ -17,7 +17,7 @@ function wait_for_state() {
 
 # Create a self-signed issuer (prerequisite for CA certificate)
 function create_selfsigned_issuer() {
-    local namespace="${1:-openstack-kuttl-tests}"
+    local namespace="${1:-${NAMESPACE:-openstack-kuttl-tests}}"
 
     echo "Creating self-signed issuer in namespace ${namespace}..."
     oc apply -f - << EOF
@@ -43,7 +43,7 @@ EOF
 # Usage: create_custom_issuer <issuer-name> <namespace>
 function create_custom_issuer() {
     local issuer_name="${1:-rootca-custom}"
-    local namespace="${2:-openstack-kuttl-tests}"
+    local namespace="${2:-${NAMESPACE:-openstack-kuttl-tests}}"
 
     echo "Creating custom root CA and issuer: ${issuer_name} in namespace ${namespace}..."
 
@@ -107,7 +107,7 @@ function create_wildcard_certificate() {
     local cert_name="${1}"
     local domain="${2}"
     local issuer_name="${3:-rootca-custom}"
-    local namespace="${4:-openstack-kuttl-tests}"
+    local namespace="${4:-${NAMESPACE:-openstack-kuttl-tests}}"
 
     if [[ -z "${cert_name}" ]] || [[ -z "${domain}" ]]; then
         echo "Error: cert_name and domain are required"
@@ -157,7 +157,7 @@ EOF
 function create_service_route_certificate() {
     local service_name="${1}"
     local ingress_domain="${2}"
-    local namespace="${3:-openstack-kuttl-tests}"
+    local namespace="${3:-${NAMESPACE:-openstack-kuttl-tests}}"
     local issuer_name="rootca-ingress-custom"
     local cert_name="${service_name}-custom-route-cert"
     local route_secret_name="${service_name}-custom-route"
@@ -218,7 +218,7 @@ EOF
 # This creates ingress and internal issuers for comprehensive testing
 # Usage: setup_custom_certificate_infrastructure <namespace>
 function setup_custom_certificate_infrastructure() {
-    local namespace="${1:-openstack-kuttl-tests}"
+    local namespace="${1:-${NAMESPACE:-openstack-kuttl-tests}}"
 
     echo "Setting up custom certificate infrastructure in namespace ${namespace}..."
 
@@ -238,7 +238,7 @@ function setup_custom_certificate_infrastructure() {
 # Usage: create_barbican_placement_routes <ingress-domain> <namespace>
 function create_barbican_placement_routes() {
     local ingress_domain="${1}"
-    local namespace="${2:-openstack-kuttl-tests}"
+    local namespace="${2:-${NAMESPACE:-openstack-kuttl-tests}}"
 
     if [[ -z "${ingress_domain}" ]]; then
         echo "Error: ingress_domain is required"
@@ -265,7 +265,7 @@ function create_barbican_placement_routes() {
 # Cleanup custom certificates and issuers
 # Usage: cleanup_custom_certificates <namespace>
 function cleanup_custom_certificates() {
-    local namespace="${1:-openstack-kuttl-tests}"
+    local namespace="${1:-${NAMESPACE:-openstack-kuttl-tests}}"
 
     echo "Cleaning up custom certificates and issuers in namespace ${namespace}..."
 

test/kuttl/common/osp_check_fingerprints.sh

@@ -1,15 +1,17 @@
 #!/bin/bash
 set -x
 
+NS_SUFFIX="${NAMESPACE:+.${NAMESPACE}}"
+
 # Check if all services from before are present in after and have valid fingerprints
 while IFS= read -r before; do
     eval $(echo "$before" | awk '{print "service_name="$1" fp_before="$2}')
-    fp_after=$(grep -F "$service_name" /tmp/endpoint_fingerprints_after | awk '{ print $2}')
+    fp_after=$(grep -F "$service_name" /tmp/endpoint_fingerprints_after${NS_SUFFIX} | awk '{ print $2}')
 
     echo -n "Endpoint $service_name - "
 
     if [ -z "$fp_after" ]; then
-        echo "not found in endpoint_fingerprints_after"
+        echo "not found in endpoint_fingerprints_after${NS_SUFFIX}"
         exit 1
     fi
 
@@ -19,6 +21,6 @@ while IFS= read -r before; do
     fi
 
     echo "OK cert rotated - before: $fp_before - after: $fp_after"
-done < /tmp/endpoint_fingerprints_before
+done < /tmp/endpoint_fingerprints_before${NS_SUFFIX}
 
 exit 0

test/kuttl/common/prepare_placement_certs.sh

@@ -39,9 +39,11 @@ echo "Fetching certificates from placement-custom-route secret and creating Conf
 
 # Get the directory where the script is located
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-# Calculate the path to the kustomize directory
-KUSTOMIZE_DIR="${SCRIPT_DIR}/../../../config/samples/tls/custom_route_cert"
-CONFIGMAP_FILE="${KUSTOMIZE_DIR}/placement-cert-data.yaml"
+# Write to a temp file to avoid parallel workers clobbering a shared repo path.
+# The kustomize overlay at config/samples/tls/custom_route_cert references this
+# file, but since the parallel runner assigns each test directory to a single
+# worker, only one invocation of this test can run at a time.
+CONFIGMAP_FILE="/tmp/placement-cert-data-${NAMESPACE}.yaml"
 
 echo "Creating ConfigMap file at: ${CONFIGMAP_FILE}"
 
@@ -60,8 +62,10 @@ $(oc get secret placement-custom-route -n ${NAMESPACE} -o jsonpath='{.data.tls\.
 $(oc get secret placement-custom-route -n ${NAMESPACE} -o jsonpath='{.data.ca\.crt}' | base64 -d | sed 's/^/    /')
 EOF
 
-# Also apply it to the cluster for verification
+# Copy to the kustomize directory so oc kustomize can find it
+KUSTOMIZE_DIR="${SCRIPT_DIR}/../../../config/samples/tls/custom_route_cert"
+cp "${CONFIGMAP_FILE}" "${KUSTOMIZE_DIR}/placement-cert-data.yaml"
+
 oc apply -f "${CONFIGMAP_FILE}"
 
 echo "ConfigMap placement-cert-data created at ${CONFIGMAP_FILE} and applied to namespace ${NAMESPACE}"
-echo "This file will be used by kustomize as a resource"

test/kuttl/tests/ctlplane-tls-cert-rotation/02-get-endpoints-certs.yaml

@@ -3,4 +3,4 @@ kind: TestStep
 commands:
   - script: |
       echo "Get fingerprints of all service certs"
-      oc exec -i openstackclient -n $NAMESPACE -- bash -s < ../../common/osp_endpoint_fingerprints.sh > /tmp/endpoint_fingerprints_before
+      oc exec -i openstackclient -n $NAMESPACE -- bash -s < ../../common/osp_endpoint_fingerprints.sh > /tmp/endpoint_fingerprints_before.${NAMESPACE}

test/kuttl/tests/ctlplane-tls-cert-rotation/04-assert-service-cert-rotation.yaml

@@ -9,14 +9,14 @@ commands:
   - script: |
       echo "Checking endpoint cert rotation..."
       for i in $(seq 1 30); do
-        oc exec -i openstackclient -n $NAMESPACE -- bash -s < ../../common/osp_endpoint_fingerprints.sh > /tmp/endpoint_fingerprints_after
-        if bash ../../common/osp_check_fingerprints.sh 2>/dev/null; then
+        oc exec -i openstackclient -n $NAMESPACE -- bash -s < ../../common/osp_endpoint_fingerprints.sh > /tmp/endpoint_fingerprints_after.${NAMESPACE}
+        if NAMESPACE=$NAMESPACE bash ../../common/osp_check_fingerprints.sh 2>/dev/null; then
           echo "All endpoint certs rotated successfully."
           exit 0
         fi
         echo "Attempt $i/30: Not all certs rotated yet, waiting 20s..."
         sleep 20
       done
       echo "Collecting final fingerprints for failure diagnostics..."
-      oc exec -i openstackclient -n $NAMESPACE -- bash -s < ../../common/osp_endpoint_fingerprints.sh > /tmp/endpoint_fingerprints_after
-      bash -s < ../../common/osp_check_fingerprints.sh
+      oc exec -i openstackclient -n $NAMESPACE -- bash -s < ../../common/osp_endpoint_fingerprints.sh > /tmp/endpoint_fingerprints_after.${NAMESPACE}
+      NAMESPACE=$NAMESPACE bash ../../common/osp_check_fingerprints.sh

test/kuttl/tests/ctlplane-tls-custom-issuers/04-rotate-service-certs.yaml

@@ -3,5 +3,5 @@ kind: TestStep
 commands:
   - script: |
       echo "Deleting secrets..."
-      oc get secret -l service-cert -n $NAMESPACE -o name > /tmp/deleted-secrets.txt
+      oc get secret -l service-cert -n $NAMESPACE -o name > /tmp/deleted-secrets.${NAMESPACE}.txt
       oc delete secret -l service-cert -n $NAMESPACE

test/kuttl/tests/ctlplane-tls-custom-issuers/10-rotate-service-certs.yaml

@@ -3,5 +3,5 @@ kind: TestStep
 commands:
   - script: |
       echo "Deleting secrets..."
-      oc get secret -l service-cert -n $NAMESPACE -o name > /tmp/deleted-secrets.txt
+      oc get secret -l service-cert -n $NAMESPACE -o name > /tmp/deleted-secrets.${NAMESPACE}.txt
       oc delete secret -l service-cert -n $NAMESPACE

test/kuttl/tests/dataplane-deploy-global-service-test/00-assert.yaml

@@ -3,7 +3,7 @@ kind: TestAssert
 timeout: 600
 collectors:
 - type: command
-  command: oc get OpenStackDataPlaneDeployment -n openstack-kuttl-tests edpm-compute-global -o yaml
+  command: oc get OpenStackDataPlaneDeployment -n $NAMESPACE edpm-compute-global -o yaml
   name: edpm-compute-global-deployment
 - type: command
   command: oc logs -n openstack-operators -l openstack.org/operator-name=openstack
@@ -13,7 +13,6 @@ apiVersion: dataplane.openstack.org/v1beta1
 kind: OpenStackDataPlaneNodeSet
 metadata:
   name: edpm-compute-global
-  namespace: openstack-kuttl-tests
 spec:
   preProvisioned: true
   services:

test/kuttl/tests/dataplane-deploy-global-service-test/01-assert.yaml

@@ -3,7 +3,7 @@ kind: TestAssert
 timeout: 600
 collectors:
 - type: command
-  command: oc get OpenStackDataPlaneDeployment -n openstack-kuttl-tests edpm-compute-global -o yaml
+  command: oc get OpenStackDataPlaneDeployment -n $NAMESPACE edpm-compute-global -o yaml
   name: edpm-compute-global-deployment
 - type: command
   command: oc logs -n openstack-operators -l openstack.org/operator-name=openstack
@@ -13,7 +13,6 @@ apiVersion: dataplane.openstack.org/v1beta1
 kind: OpenStackDataPlaneNodeSet
 metadata:
   name: edpm-compute-global
-  namespace: openstack-kuttl-tests
 spec:
   services:
   - download-cache
@@ -80,7 +79,6 @@ metadata:
     openstackdataplanenodeset: edpm-compute-global
     openstackdataplaneservice: custom-global-service
   name: custom-global-service-edpm-compute-global
-  namespace: openstack-kuttl-tests
   ownerReferences:
   - apiVersion: dataplane.openstack.org/v1beta1
     blockOwnerDeletion: true
@@ -188,7 +186,6 @@ metadata:
     openstackdataplanenodeset: edpm-compute-global
     openstackdataplaneservice: download-cache
   name: download-cache-edpm-compute-global-edpm-compute-global
-  namespace: openstack-kuttl-tests
   ownerReferences:
   - apiVersion: dataplane.openstack.org/v1beta1
     blockOwnerDeletion: true
@@ -286,7 +283,6 @@ metadata:
     openstackdataplanenodeset: edpm-compute-global
     openstackdataplaneservice: bootstrap
   name: bootstrap-edpm-compute-global-edpm-compute-global
-  namespace: openstack-kuttl-tests
   ownerReferences:
   - apiVersion: dataplane.openstack.org/v1beta1
     blockOwnerDeletion: true
@@ -390,7 +386,6 @@ metadata:
     openstackdataplanenodeset: edpm-compute-global
     openstackdataplaneservice: configure-network
   name: configure-network-edpm-compute-global-edpm-compute-global
-  namespace: openstack-kuttl-tests
   ownerReferences:
   - apiVersion: dataplane.openstack.org/v1beta1
     blockOwnerDeletion: true
@@ -488,7 +483,6 @@ metadata:
     openstackdataplanenodeset: edpm-compute-global
     openstackdataplaneservice: validate-network
   name: validate-network-edpm-compute-global-edpm-compute-global
-  namespace: openstack-kuttl-tests
   ownerReferences:
   - apiVersion: dataplane.openstack.org/v1beta1
     blockOwnerDeletion: true
@@ -586,7 +580,6 @@ metadata:
     openstackdataplanenodeset: edpm-compute-global
     openstackdataplaneservice: configure-os
   name: configure-os-edpm-compute-global-edpm-compute-global
-  namespace: openstack-kuttl-tests
   ownerReferences:
   - apiVersion: dataplane.openstack.org/v1beta1
     blockOwnerDeletion: true
@@ -684,7 +677,6 @@ metadata:
     openstackdataplanenodeset: edpm-compute-global
     openstackdataplaneservice: install-certs
   name: install-certs-edpm-compute-global-edpm-compute-global
-  namespace: openstack-kuttl-tests
   ownerReferences:
   - apiVersion: dataplane.openstack.org/v1beta1
     blockOwnerDeletion: true
@@ -831,7 +823,6 @@ metadata:
     openstackdataplanenodeset: edpm-compute-global
     openstackdataplaneservice: ovn
   name: ovn-edpm-compute-global-edpm-compute-global
-  namespace: openstack-kuttl-tests
   ownerReferences:
   - apiVersion: dataplane.openstack.org/v1beta1
     blockOwnerDeletion: true
@@ -940,7 +931,6 @@ metadata:
     openstackdataplanenodeset: edpm-compute-global
     openstackdataplaneservice: neutron-metadata
   name: neutron-metadata-edpm-compute-global-edpm-compute-global
-  namespace: openstack-kuttl-tests
   ownerReferences:
   - apiVersion: dataplane.openstack.org/v1beta1
     blockOwnerDeletion: true
@@ -1079,7 +1069,6 @@ metadata:
     openstackdataplanenodeset: edpm-compute-global
     openstackdataplaneservice: neutron-ovn
   name: neutron-ovn-edpm-compute-global-edpm-compute-global
-  namespace: openstack-kuttl-tests
   ownerReferences:
   - apiVersion: dataplane.openstack.org/v1beta1
     blockOwnerDeletion: true
@@ -1218,7 +1207,6 @@ metadata:
     openstackdataplanenodeset: edpm-compute-global
     openstackdataplaneservice: neutron-sriov
   name: neutron-sriov-edpm-compute-global-edpm-compute-global
-  namespace: openstack-kuttl-tests
   ownerReferences:
   - apiVersion: dataplane.openstack.org/v1beta1
     blockOwnerDeletion: true
@@ -1327,7 +1315,6 @@ metadata:
     openstackdataplanenodeset: edpm-compute-global
     openstackdataplaneservice: neutron-dhcp
   name: neutron-dhcp-edpm-compute-global-edpm-compute-global
-  namespace: openstack-kuttl-tests
   ownerReferences:
   - apiVersion: dataplane.openstack.org/v1beta1
     blockOwnerDeletion: true
@@ -1436,7 +1423,6 @@ metadata:
     openstackdataplanenodeset: edpm-compute-global
     openstackdataplaneservice: libvirt
   name: libvirt-edpm-compute-global-edpm-compute-global
-  namespace: openstack-kuttl-tests
   ownerReferences:
   - apiVersion: dataplane.openstack.org/v1beta1
     blockOwnerDeletion: true
@@ -1545,7 +1531,6 @@ metadata:
     openstackdataplanenodeset: edpm-compute-global
     openstackdataplaneservice: nova
   name: nova-edpm-compute-global-edpm-compute-global
-  namespace: openstack-kuttl-tests
   ownerReferences:
   - apiVersion: dataplane.openstack.org/v1beta1
     blockOwnerDeletion: true

test/kuttl/tests/dataplane-deploy-global-service-test/02-assert.yaml

@@ -10,7 +10,6 @@ apiVersion: dataplane.openstack.org/v1beta1
 kind: OpenStackDataPlaneNodeSet
 metadata:
   name: edpm-compute-beta-nodeset
-  namespace: openstack-kuttl-tests
 spec:
   services:
   - download-cache
@@ -67,7 +66,6 @@ apiVersion: dataplane.openstack.org/v1beta1
 kind: OpenStackDataPlaneDeployment
 metadata:
   name: edpm-multinodeset
-  namespace: openstack-kuttl-tests
 spec:
   nodeSets:
     - edpm-compute-global
@@ -81,7 +79,6 @@ metadata:
     openstackdataplanenodeset: edpm-compute-beta-nodeset
     openstackdataplaneservice: download-cache
   name: download-cache-edpm-multinodeset-edpm-compute-beta-nodeset
-  namespace: openstack-kuttl-tests
   ownerReferences:
   - apiVersion: dataplane.openstack.org/v1beta1
     blockOwnerDeletion: true
@@ -178,7 +175,6 @@ metadata:
     openstackdataplanenodeset: edpm-compute-beta-nodeset
     openstackdataplaneservice: bootstrap
   name: bootstrap-edpm-multinodeset-edpm-compute-beta-nodeset
-  namespace: openstack-kuttl-tests
   ownerReferences:
   - apiVersion: dataplane.openstack.org/v1beta1
     blockOwnerDeletion: true