RBD cluster 구분에 대응하는 HA heartbeat 커맨드 수정

Author: jschoiRR

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java

@@ -17,6 +17,8 @@
 package com.cloud.hypervisor.kvm.storage;
 
 import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
 import java.util.List;
 import java.util.Map;
 
@@ -362,6 +364,7 @@ public String createHeartBeatCommand(HAStoragePool primaryStoragePool, String ho
                     cmd.add("-c");
                 }
         } else if (primaryStoragePool.getPool().getType() == StoragePoolType.RBD) {
+            createRBDSecretKeyFileIfNoExist(primaryStoragePool.getPoolUUID(), "/var/lib/libvirt/images/", primaryStoragePool.getPoolAuthSecret());
             cmd.add("-i", primaryStoragePool.getPoolSourceHost());
             cmd.add("-p", primaryStoragePool.getPoolMountSourcePath());
             cmd.add("-n", primaryStoragePool.getPoolAuthUserName());
@@ -613,6 +616,7 @@ public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activit
             cmd.add("-t", String.valueOf(String.valueOf(System.currentTimeMillis() / 1000)));
             cmd.add("-d", String.valueOf(duration));
         } else if (pool.getPool().getType() == StoragePoolType.RBD) {
+            cmd.add("-i", pool.getPoolSourceHost());
             cmd.add("-p", pool.getPoolMountSourcePath());
             cmd.add("-n", pool.getPoolAuthUserName());
             cmd.add("-s", pool.getPoolAuthSecret());
@@ -704,4 +708,21 @@ public Boolean vmActivityCheck(HAStoragePool pool, HostTO host, Duration activit
             return true;
         }
     }
+
+        public void createRBDSecretKeyFileIfNoExist(String uuid, String localPath, String skey) {
+        File file = new File(localPath + File.separator + uuid);
+        try {
+            // 파일이 존재하지 않을 때만 생성
+            if (!file.exists()) {
+                boolean isCreated = file.createNewFile();
+                if (isCreated) {
+                    // 파일 생성 후 내용 작성
+                    FileWriter writer = new FileWriter(file);
+                    writer.write(skey);
+                    writer.close();
+                }
+            }
+        } catch (IOException e) {}
+    }
+
 }

scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh

@@ -39,6 +39,7 @@ interval=0
 rflag=0
 cflag=0
 UUIDList=
+skeyPath="/var/lib/libvirt/images/"
 
 while getopts 'p:n:s:h:i:t:u:r:c' OPTION
 do
@@ -81,38 +82,38 @@ if [ -z "$PoolName" ]; then
 fi
 
 # rados object touch action for vol list
-res=$(rbd -p $PoolName ls --id $PoolAuthUserName | grep MOLD-AC)
+res=$(rbd -p $PoolName ls --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret | grep MOLD-AC)
 if [ $? -gt 0 ]; then
-  rbd -p $PoolName create --size 1 --id $PoolAuthUserName MOLD-AC
+  rbd -p $PoolName create --size 1 --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret MOLD-AC
 fi
 
 timestamp=$(date +%s)
 
 if [ -n "$UUIDList" ]; then
     for uuid in $(echo $UUIDList | sed 's/,/ /g'); do
-      objId=$(rbd -p $PoolName info $uuid --id $PoolAuthUserName | grep 'id:')
+      objId=$(rbd -p $PoolName info $uuid --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret | grep 'id:')
       objId=${objId#*id: }
       res=$(timeout 3s bash -c "rados -p $PoolName touch rbd_object_map.$objId")
       if [ $? -eq 0 ]; then
         # 정상적인 touch 상태면 image meta에 key: uuid / value : timestamp 입력
-        rbd -p $PoolName --id $PoolAuthUserName image-meta set MOLD-AC $uuid $HostIP:$timestamp
+        rbd -p $PoolName --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret image-meta set MOLD-AC $uuid $HostIP:$timestamp
       else
         # 정상적으로 touch 상태가 아니면 image meta에 key : uuid 삭제
-        rbd -p $PoolName --id $PoolAuthUserName image-meta rm MOLD-AC $uuid
+        rbd -p $PoolName --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret image-meta rm MOLD-AC $uuid
       fi
     done
 fi
 
 #write the heart beat log
 write_hbLog() {
   Timestamp=$(date +%s)
-  obj=$(rbd -p $PoolName ls --id $PoolAuthUserName | grep MOLD-HB)
+  obj=$(rbd -p $PoolName ls --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret | grep MOLD-HB)
 
   if [ $? -gt 0 ]; then
-     rbd -p $PoolName create --size 1 --id $PoolAuthUserName MOLD-HB
+     rbd -p $PoolName create --size 1 --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret MOLD-HB
   fi
 
-  obj=$(rbd -p $PoolName --id $PoolAuthUserName image-meta set MOLD-HB $HostIP $Timestamp)
+  obj=$(rbd -p $PoolName --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret image-meta set MOLD-HB $HostIP $Timestamp)
   if [ $? -gt 0 ]; then
    	printf "Failed to create rbd file and set image-meta"
     return 2
@@ -123,7 +124,7 @@ write_hbLog() {
 #check the heart beat log
 check_hbLog() {
   now=$(date +%s)
-  getHbTime=$(rbd -p $PoolName --id $PoolAuthUserName image-meta get MOLD-HB $HostIP)
+  getHbTime=$(rbd -p $PoolName --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret image-meta get MOLD-HB $HostIP)
   if [ $? -gt 0 ] || [ -z "$getHbTime" ]; then
     return 1
   fi

scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh

@@ -31,10 +31,12 @@ PoolName=
 PoolAuthUserName=
 PoolAuthSecret=
 HostIP=
+SourceHostIP=
 UUIDList=
 interval=0
+skeyPath="/var/lib/libvirt/images/"
 
-while getopts 'p:n:s:h:u:t:' OPTION
+while getopts 'p:n:s:h:i:u:t:' OPTION
 do
   case $OPTION in
   p)
@@ -49,6 +51,9 @@ do
   h)
      HostIP="$OPTARG"
      ;;
+  i)
+     SourceHostIP="$OPTARG"
+     ;;
   u)
      UUIDList="$OPTARG"
      ;;
@@ -72,7 +77,7 @@ fi
 # First check: heartbeat filei
 
 now=$(date +%s)
-getHbTime=$(rbd -p $PoolName --id $PoolAuthUserName image-meta get MOLD-HB $HostIP)
+getHbTime=$(rbd -p $PoolName --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret image-meta get MOLD-HB $HostIP)
 
 if [ $? -eq 0 ]; then
    diff=$(expr $now - $getHbTime)
@@ -89,12 +94,12 @@ fi
 
 # Second check: disk activity check
 for uuid in $(echo $UUIDList | sed 's/,/ /g'); do
-   acTime=$(rbd -p $PoolName --id $PoolAuthUserName image-meta get MOLD-AC $uuid > /dev/null 2>&1)
+   acTime=$(rbd -p $PoolName --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret image-meta get MOLD-AC $uuid > /dev/null 2>&1)
    if [ $? -gt 0 ]; then
       echo "### [HOST STATE : DEAD] Unable to confirm normal activity of volume image list => Considered host down in [PoolType : RBD] ### "
       exit 0
    else
-      acTime=$(rbd -p $PoolName --id $PoolAuthUserName image-meta get MOLD-AC $uuid)
+      acTime=$(rbd -p $PoolName --id $PoolAuthUserName -m $SourceHostIP -K $skeyPath$PoolAuthSecret image-meta get MOLD-AC $uuid)
       if [ -z "$acTime" ]; then
          echo "### [HOST STATE : DEAD] Unable to confirm normal activity of volume image list => Considered host down in [PoolType : RBD] ### "
          exit 0