[WIP][ovn] switch backup/restore procedure

With backup/restore uuids for the records are being
changed and that's result into geneve tunnels to be
disabled with the compute nodes and thus centralized
SNAT traffic is impacted when ovn_controller are
started post adoption on the networker nodes. So
switch to use full db files for restoration after
converting the DBs to standalone. With this also
cleaned the helper pod and other resources.

Resolves: #OSPRH-29914
Signed-off-by: Yatin Karel <ykarel@redhat.com>

Author: karelyatin

tests/roles/control_plane_rollback/tasks/main.yaml

@@ -157,7 +157,6 @@
 
     oc delete --ignore-not-found=true --wait=false pod mariadb-copy-data
     oc delete --ignore-not-found=true --wait=false pvc mariadb-data
-    oc delete --ignore-not-found=true --wait=false pod ovn-copy-data
     oc delete --ignore-not-found=true secret osp-secret
 
 - name: verify that the source control plane is responding

tests/roles/ovn_adoption/defaults/main.yaml

@@ -20,12 +20,12 @@ ovs_db_patch: |
       template:
         ovnDBCluster:
           ovndbcluster-nb:
-            replicas: 3
+            replicas: 1
             dbType: NB
             storageRequest: 10G
             networkAttachment: internalapi
           ovndbcluster-sb:
-            replicas: 3
+            replicas: 1
             dbType: SB
             storageRequest: 10G
             networkAttachment: internalapi

tests/roles/ovn_adoption/handlers/main.yaml

@@ -1,8 +1 @@
-- name: delete adoption helper pod and pvc
-  ansible.builtin.shell: |
-    {{ shell_header }}
-    {{ oc_header }}
-    oc delete --ignore-not-found=true pod ovn-copy-data
-    oc delete --ignore-not-found=true certificate ovn-data-cert
-    oc delete --ignore-not-found=true secret ovn-data-cert
-    {% if storage_reclaim_policy.lower() == "delete" %}oc delete pvc --ignore-not-found=true ovn-data{% endif %}
+---

tests/roles/ovn_adoption/tasks/cluster_to_standalone.yaml

@@ -0,0 +1,75 @@
+---
+# Cluster-to-standalone conversion for OVN database adoption
+
+- name: convert OVN databases to standalone and fetch to ansible controller
+  no_log: "{{ use_no_log }}"
+  ansible.builtin.shell: |
+    {{ shell_header }}
+    {{ ovn_copy_shell_vars }}
+
+    # Create working directory on controller
+    $CONTROLLER1_SSH sudo mkdir -p /tmp/ovn_standalone_conversion
+
+    # Convert cluster databases to standalone on the controller
+    $CONTROLLER1_SSH sudo ovsdb-tool cluster-to-standalone /tmp/ovn_standalone_conversion/ovnnb_db.db /var/lib/openvswitch/ovn/ovnnb_db.db
+    $CONTROLLER1_SSH sudo ovsdb-tool cluster-to-standalone /tmp/ovn_standalone_conversion/ovnsb_db.db /var/lib/openvswitch/ovn/ovnsb_db.db
+
+    # Set permissions for copying
+    $CONTROLLER1_SSH sudo chmod 644 /tmp/ovn_standalone_conversion/ovn*.db
+
+    # Create local temp directory
+    mkdir -p /tmp/ovn_adoption_dbs
+
+    # Use scp like other parts of the codebase
+    CONTROLLER1_SCP=$(echo "$CONTROLLER1_SSH" | sed 's/^ssh/scp/')
+
+    echo "Copying NB database to ansible controller..."
+    ${CONTROLLER1_SCP}:/tmp/ovn_standalone_conversion/ovnnb_db.db /tmp/ovn_adoption_dbs/
+
+    echo "Copying SB database to ansible controller..."
+    ${CONTROLLER1_SCP}:/tmp/ovn_standalone_conversion/ovnsb_db.db /tmp/ovn_adoption_dbs/
+
+    # Cleanup on controller
+    $CONTROLLER1_SSH sudo rm -rf /tmp/ovn_standalone_conversion
+
+    echo "Successfully converted and copied databases to ansible controller"
+
+- name: copy standalone databases to OVN database pods
+  ansible.builtin.shell: |
+    {{ shell_header }}
+    {{ oc_header }}
+
+    echo "Copying standalone databases directly to OVN database pods..."
+
+    # Copy NB database to the NB pod
+    oc cp /tmp/ovn_adoption_dbs/ovnnb_db.db ovsdbserver-nb-0:/etc/ovn/ovnnb_db.db
+
+    # Copy SB database to the SB pod
+    oc cp /tmp/ovn_adoption_dbs/ovnsb_db.db ovsdbserver-sb-0:/etc/ovn/ovnsb_db.db
+
+    echo "Database files copied successfully to pods"
+
+- name: restart OVN database pods to load standalone databases
+  ansible.builtin.shell: |
+    {{ shell_header }}
+    {{ oc_header }}
+
+    echo "Restarting OVN database pods to load standalone databases..."
+
+    # Restart NB database pod
+    oc delete pod -l service=ovsdbserver-nb
+    oc wait --for=condition=Ready pod -l service=ovsdbserver-nb --timeout=180s
+
+    # Restart SB database pod
+    oc delete pod -l service=ovsdbserver-sb
+    oc wait --for=condition=Ready pod -l service=ovsdbserver-sb --timeout=180s
+
+    echo "OVN database pods restarted successfully"
+
+- name: cleanup temporary files
+  ansible.builtin.shell: |
+    # Remove temporary files from ansible controller
+    rm -f /tmp/ovn_adoption_dbs/ovn*.db
+    rmdir /tmp/ovn_adoption_dbs 2>/dev/null || true
+
+    echo "Temporary files cleaned up"

tests/roles/ovn_adoption/tasks/main.yaml

@@ -61,94 +61,6 @@
       CONTROLLER3_SSH="{{ controller3_ssh }}"
   when: not ospdo_src| bool
 
-- name: start an adoption helper pod
-  vars:
-    internalapi_default_nad: >-
-      {% if not (bgp | bool) -%}
-      '[{"name":"internalapi","namespace":"openstack"}]'
-      {% else -%}
-      '[{"name":"internalapi","namespace":"openstack","default-route":["{{ internalapi_prefix | default('172.17.0') }}.1"]}]'
-      {% endif -%}
-  ansible.builtin.shell: |-
-    {{ shell_header }}
-    {{ oc_header }}
-    {{ ovn_copy_shell_vars }}
-
-    oc apply -f - <<EOF
-    ---
-    apiVersion: cert-manager.io/v1
-    kind: Certificate
-    metadata:
-      name: ovn-data-cert
-    spec:
-      commonName: ovn-data-cert
-      secretName: ovn-data-cert
-      issuerRef:
-        name: rootca-internal
-    ---
-    apiVersion: v1
-    kind: PersistentVolumeClaim
-    metadata:
-      name: ovn-data
-      labels:
-        app: adoption
-    spec:
-      storageClassName: $STORAGE_CLASS
-      accessModes:
-        - ReadWriteOnce
-      resources:
-        requests:
-          storage: 10Gi
-    ---
-    apiVersion: v1
-    kind: Pod
-    metadata:
-      name: ovn-copy-data
-      annotations:
-        openshift.io/scc: anyuid
-        k8s.v1.cni.cncf.io/networks: {{ copy_pods_custom_networks | default(internalapi_default_nad) }}
-      labels:
-        app: adoption
-    spec:
-      {{ copy_pods_custom_spec|default('') }}
-      containers:
-      - image: $OVSDB_IMAGE
-        command: [ "sh", "-c", "sleep infinity"]
-        name: adoption
-        volumeMounts:
-        - mountPath: /backup
-          name: ovn-data
-        - mountPath: /etc/pki/tls/misc
-          name: ovn-data-cert
-          readOnly: true
-      securityContext:
-        allowPrivilegeEscalation: false
-        capabilities:
-          drop: ALL
-        runAsNonRoot: true
-        seccompProfile:
-          type: RuntimeDefault
-      volumes:
-      - name: ovn-data
-        persistentVolumeClaim:
-          claimName: ovn-data
-      - name: ovn-data-cert
-        secret:
-          secretName: ovn-data-cert
-    EOF
-  changed_when: true
-  notify: delete adoption helper pod and pvc
-
-- name: wait for the pod to come up
-  ansible.builtin.shell: |
-    {{ shell_header }}
-    {{ oc_header }}
-    oc wait --for condition=Ready pod/ovn-copy-data --timeout=30s
-  register: ovn_data_pod_result
-  until: ovn_data_pod_result is success
-  retries: 2
-  delay: "{{ ovn_retry_delay }}"
-
 - name: stop northd service
   no_log: "{{ use_no_log }}"
   ansible.builtin.shell: |
@@ -203,73 +115,8 @@
   ansible.builtin.pause:
     seconds: 60
 
-- name: dump OVN databases using tcp connection
-  no_log: "{{ use_no_log }}"
-  ansible.builtin.shell: |
-    {{ shell_header }}
-    {{ oc_header }}
-    {{ ovn_copy_shell_vars }}
-
-    oc exec ovn-copy-data -- bash -c "ovsdb-client backup tcp:$SOURCE_OVSDB_IP:6641 > /backup/ovs-nb.db"
-    oc exec ovn-copy-data -- bash -c "ovsdb-client backup tcp:$SOURCE_OVSDB_IP:6642 > /backup/ovs-sb.db"
-  when: enable_tlse|bool is false
-
-- name: dump OVN databases using ssl connection
-  no_log: "{{ use_no_log }}"
-  ansible.builtin.shell: |
-    {{ shell_header }}
-    {{ oc_header }}
-    {{ ovn_copy_shell_vars }}
-
-    oc exec ovn-copy-data -- bash -c "ovsdb-client backup --ca-cert=/etc/pki/tls/misc/ca.crt --private-key=/etc/pki/tls/misc/tls.key --certificate=/etc/pki/tls/misc/tls.crt ssl:$SOURCE_OVSDB_IP:6641 > /backup/ovs-nb.db"
-    oc exec ovn-copy-data -- bash -c "ovsdb-client backup --ca-cert=/etc/pki/tls/misc/ca.crt --private-key=/etc/pki/tls/misc/tls.key --certificate=/etc/pki/tls/misc/tls.crt ssl:$SOURCE_OVSDB_IP:6642 > /backup/ovs-sb.db"
-  when: enable_tlse|bool
-
-- name: upgrade OVN databases to the latest schema from podified ovsdb-servers
-  no_log: "{{ use_no_log }}"
-  ansible.builtin.shell: |
-    {{ shell_header }}
-    {{ oc_header }}
-    {{ ovn_copy_shell_vars }}
-
-    oc exec ovn-copy-data -- bash -c "ovsdb-client get-schema tcp:$PODIFIED_OVSDB_NB_IP:6641 > /backup/ovs-nb.ovsschema && ovsdb-tool convert /backup/ovs-nb.db /backup/ovs-nb.ovsschema"
-    oc exec ovn-copy-data -- bash -c "ovsdb-client get-schema tcp:$PODIFIED_OVSDB_SB_IP:6642 > /backup/ovs-sb.ovsschema && ovsdb-tool convert /backup/ovs-sb.db /backup/ovs-sb.ovsschema"
-  when: enable_tlse|bool is false
-
-- name: upgrade OVN databases to the latest schema from podified ovsdb-servers (tls)
-  no_log: "{{ use_no_log }}"
-  ansible.builtin.shell: |
-    {{ shell_header }}
-    {{ oc_header }}
-    {{ ovn_copy_shell_vars }}
-
-    oc exec ovn-copy-data -- bash -c "ovsdb-client get-schema --ca-cert=/etc/pki/tls/misc/ca.crt --private-key=/etc/pki/tls/misc/tls.key \
-    --certificate=/etc/pki/tls/misc/tls.crt ssl:$PODIFIED_OVSDB_NB_IP:6641 > /backup/ovs-nb.ovsschema && ovsdb-tool convert /backup/ovs-nb.db /backup/ovs-nb.ovsschema"
-    oc exec ovn-copy-data -- bash -c "ovsdb-client get-schema --ca-cert=/etc/pki/tls/misc/ca.crt --private-key=/etc/pki/tls/misc/tls.key \
-    --certificate=/etc/pki/tls/misc/tls.crt ssl:$PODIFIED_OVSDB_SB_IP:6642 > /backup/ovs-sb.ovsschema && ovsdb-tool convert /backup/ovs-sb.db /backup/ovs-sb.ovsschema"
-  when: enable_tlse|bool
-
-- name: restore OVN database backup to podified ovsdb-servers - {{ item }}
-  vars:
-    ovn_db_ip: "{{ 'PODIFIED_OVSDB_NB_IP' if item == 'nb' else 'PODIFIED_OVSDB_SB_IP' }}"
-    ovn_db_port: "{{ '6641' if item == 'nb' else '6642' }}"
-    ovn_db_file: "ovs-{{ item }}.db"
-    tls_opts: "{{ '--ca-cert=/etc/pki/tls/misc/ca.crt --private-key=/etc/pki/tls/misc/tls.key --certificate=/etc/pki/tls/misc/tls.crt' if enable_tlse | bool else '' }}"
-    proto: "{{ 'ssl' if enable_tlse | bool else 'tcp' }}"
-  no_log: "{{ use_no_log }}"
-  ansible.builtin.shell: |
-    {{ shell_header }}
-    {{ oc_header }}
-    {{ ovn_copy_shell_vars }}
-
-    oc exec ovn-copy-data -- bash -c "ovsdb-client -v restore {{ tls_opts }} {{ proto }}:${{ ovn_db_ip }}:{{ ovn_db_port }} < /backup/{{ ovn_db_file }}"
-  register: podified_ovn_db_restore
-  until: podified_ovn_db_restore is success
-  retries: 5
-  delay: "{{ ovn_retry_delay }}"
-  loop:
-    - nb
-    - sb
+- name: migrate OVN databases using cluster-to-standalone conversion
+  ansible.builtin.include_tasks: cluster_to_standalone.yaml
 
 - name: deploy podified OVN northd service to keep databases in sync
   ansible.builtin.shell: |

tests/roles/ovn_adoption/tasks/ovn_ospdo_src_vars.yaml

@@ -33,15 +33,3 @@
       CONTROLLER1_SSH="{{ controller1_ssh }}"
       CONTROLLER2_SSH="echo ospdo src env, skipping on controller-1 "
       CONTROLLER3_SSH="echo ospdo src env ,skipping on controller-2 "
-
-- name: create ovn-data-cert secret
-  ansible.builtin.shell: |
-    oc apply -f - <<EOF
-    ---
-    apiVersion: v1
-    kind: Secret
-    metadata:
-      name: ovn-data-cert
-      namespace: {{ director_namespace }}
-    type: Opaque
-    EOF

tests/roles/pcp_cleanup/tasks/main.yaml

@@ -18,7 +18,6 @@
     # Delete Adoption helper pods
     oc delete --ignore-not-found=true --wait=false pod mariadb-copy-data
     oc delete --ignore-not-found=true --wait=false pvc mariadb-data
-    oc delete --ignore-not-found=true --wait=false pod ovn-copy-data
 
     # Delete secrets
     for secret in $(oc get secrets -o name); do