Update PVC migration script (#1345)

Author: BraisVQ

CHANGELOG.md

@@ -7,8 +7,10 @@
 
 ### Changed
 - Nexus storage change ([#1341](https://github.com/opendevstack/ods-core/issues/1341))
+- Update PVC migration script, adding threads to rsync execution ([#1345](https://github.com/opendevstack/ods-core/pull/1345))
 - Update Aqua cli to 760 ([#1344](https://github.com/opendevstack/ods-core/pull/1344))
 
+
 ### Fixed
 
 

Makefile

@@ -196,9 +196,9 @@ backup-sonarqube:
 
 
 # PVC MIGRATION
-## Migrate data from one PVC to another.
+## Migrate data from one PVC to another. Options: SOURCE_PVC, TARGET_PVC, THREADS (default: 5), CPU_REQUEST (default: 1), MEMORY (default: 2)
 migrate-pvc-data:
-	./scripts/migrate_pvc_data.sh --source-pvc $(SOURCE_PVC) --target-pvc $(TARGET_PVC) --namespace $(ODS_NAMESPACE)
+	./scripts/migrate_pvc_data.sh --source-pvc $(SOURCE_PVC) --target-pvc $(TARGET_PVC) --namespace $(ODS_NAMESPACE) --threads $(THREADS) --cpu $(CPU_REQUEST) --memory $(MEMORY)
 .PHONY: migrate-pvc-data
 
 

docs/modules/administration/pages/pvc-migration.adoc

@@ -16,13 +16,16 @@ To migrate data between PVCs, follow these steps:
 
 [source,sh]
 ----
-make migrate-pvc-data SOURCE_PVC=<source-pvc-name> TARGET_PVC=<target-pvc-name>
+make migrate-pvc-data SOURCE_PVC=<source-pvc-name> TARGET_PVC=<target-pvc-name> [THREADS=<num-threads>] [CPU_REQUEST=<num-cpu>] [MEMORY=<memory-in-gb>]
 ----
 
-Replace `<source-pvc-name>`, and `<target-pvc-name>` with the appropriate values for your environment.
+Replace `<source-pvc-name>`, `<target-pvc-name>`, `<num-threads>`, `<num-cpu>`, and `<memory-in-gb>` with the appropriate values for your environment.
+- `THREADS` (optional): Number of parallel threads to use for migration (default: 5).
+- `CPU_REQUEST` (optional): Number of CPU cores to request for the migration pod (default: 1).
+- `MEMORY` (optional): Memory request and limit in Gigabytes for the migration pod (default: 2).
 
 == Details
 
-- The migration process creates a temporary pod that uses `rsync` to copy data from the source PVC to the target PVC.
-- Logs of the migration are stored in `/tmp/target/rsync.log` within the target PVC.
-- After the migration, verify the data integrity and delete the temporary pod if necessary.
+- The migration process creates a temporary pod that uses parallelized `rsync` to copy data from the source PVC to the target PVC.
+- You can control the number of parallel threads and CPU/memory resources via the `THREADS`, `CPU_REQUEST`, and `MEMORY` options.
+- After the migration, verify the data integrity and delete the temporary pod.

scripts/migrate_pvc_data.sh

@@ -1,8 +1,5 @@
 #!/bin/bash
 
-# Exit on error is removed to allow the script to continue on failure
-# set -e
-
 # Function to display usage
 usage() {
   printf "Migrate data from one PVC to another within the OpenShift cluster.\n\n"
@@ -13,6 +10,9 @@ usage() {
   printf "\t-s|--source-pvc\t\tName of the source PVC\n"
   printf "\t-t|--target-pvc\t\tName of the target PVC\n"
   printf "\t-n|--namespace\t\tNamespace where the PVCs are located\n"
+  printf "\t-p|--threads\t\tNumber of parallel threads (default: 10)\n"
+  printf "\t-c|--cpu\t\tNumber of CPU cores to request in cores (default: 2)\n"
+  printf "\t-m|--memory\tMemory request and limit in Gigabytes (default: 5)\n"
 }
 
 # Parse arguments
@@ -26,11 +26,25 @@ while [[ "$#" -gt 0 ]]; do
     -t=*|--target-pvc=*) TARGET_PVC="${1#*=}" ;;
     -n|--namespace) NAMESPACE="$2"; shift ;;
     -n=*|--namespace=*) NAMESPACE="${1#*=}" ;;
+    -p|--threads) THREADS="$2"; shift ;;
+    -p=*|--threads=*) THREADS="${1#*=}" ;;
+    -c|--cpu) CPU_REQUEST="$2"; shift ;;
+    -c=*|--cpu=*) CPU_REQUEST="${1#*=}" ;;
+    -m|--memory) MEMORY="$2"; shift ;;
+    -m=*|--memory=*) MEMORY="${1#*=}" ;;
     *) echo "Unknown parameter passed: $1"; usage; exit 1 ;;
   esac
   shift
 done
 
+# Set default threads, cpu, and memory if not provided
+THREADS="${THREADS:-10}"
+CPU_REQUEST="${CPU_REQUEST:-2}"
+MEMORY="${MEMORY:-5}"
+
+# Calculate CPU limit (always 2 more than request)
+CPU_LIMIT=$((CPU_REQUEST + 2))
+
 # Validate arguments
 if [[ -z "$SOURCE_PVC" || -z "$TARGET_PVC" || -z "$NAMESPACE" ]]; then
   echo "Error: Missing required arguments."
@@ -50,14 +64,31 @@ spec:
   containers:
   - name: rsync-container
     image: image-registry.openshift-image-registry.svc:5000/openshift/tools
-    command: ["/bin/sh", "-c", "time rsync -avh --omit-dir-times --stats --human-readable --info=progress2 --partial --ignore-errors /tmp/source/ /tmp/target/; while true; do sleep 3600; done"]
+    command:
+      - /bin/sh
+      - -c
+      - |
+        start_time=\$(date +%s)
+        find /tmp/source -type f -print0 | xargs -0 -n1 -P${THREADS} -I{} sh -c '
+          target_dir="/tmp/target/\$(dirname "{}" | sed "s|/tmp/source||")"
+          mkdir -p "\$target_dir"
+          rsync -avh --omit-dir-times --stats --human-readable --info=progress2 --partial --ignore-errors "{}" "\$target_dir/"
+        '
+        end_time=\$(date +%s)
+        duration=\$((end_time - start_time))
+        duration_min=\$((duration / 60))
+        duration_hr=\$((duration / 3600))
+        echo "Data migration completed successfully."
+        echo "Time taken: \${duration} seconds (\${duration_min} minutes, \${duration_hr} hours)."
+        echo "You can now safely remove the migration pod."
+        while true; do sleep 3600; done
     resources:
       requests:
-        memory: "2Gi"
-        cpu: "250m"
+        memory: "${MEMORY}Gi"
+        cpu: "${CPU_REQUEST}"
       limits:
-        memory: "2Gi"
-        cpu: '2'
+        memory: "${MEMORY}Gi"
+        cpu: '${CPU_LIMIT}'
     volumeMounts:
     - name: source-pvc
       mountPath: /tmp/source