Merge branch 'fv-bm-expon-63318@@3' into '5.1.0'

gitlab · gitlab · commit a5f1f203eb1c · 2024-04-12T08:39:16.000Z
&lt;feature&gt;[expon]: support expon block volume for baremetal2

See merge request zstackio/zstack-utility!4652
diff --git a/bm-instance-agent/bm_instance_agent/__init__.py b/bm-instance-agent/bm_instance_agent/__init__.py
@@ -1 +1 @@
-__version__ = "2.0.5"
+__version__ = "2.0.6"
diff --git a/bm-instance-agent/bm_instance_agent/systems/linux/driver.py b/bm-instance-agent/bm_instance_agent/systems/linux/driver.py
@@ -40,7 +40,7 @@ def _check_initiator_config(instance_uuid):
     stdout, stderr = processutils.trycmd(cmd, shell=True)
     if stderr:
         LOG.info("get iscsid status failed, try to restart iscsid service")
-        cmd = 'systemctl restart iscsid'
+        cmd = 'systemctl restart iscsid || service iscsid restart'
         processutils.execute(cmd, shell=True)
 
     conf_path = '/etc/iscsi/initiatorname.iscsi'
@@ -82,6 +82,36 @@ def _check_multi_path_config():
     processutils.execute(cmd, shell=True)
 
 
+def rescan_sids_target_name(target_name):
+    cmd = "iscsiadm -m session | grep {} | awk '{{print $2}}' | tr -d '[]'".format(target_name)
+    LOG.info(cmd)
+    stdout, stderr = processutils.trycmd(cmd, shell=True)
+    if stderr:
+        LOG.info("iscsiadm -m session grep %s failed, because %s" % (target_name, stderr))
+        return
+    session_ids = stdout.strip().split('\n')
+    for session_id in session_ids:
+        if not session_id:
+            continue
+        rescan_cmd = 'iscsiadm -m session -r %s --rescan' % session_id
+        LOG.info(rescan_cmd)
+        stdout, stderr = processutils.trycmd(rescan_cmd, shell=True)
+        if stderr:
+            LOG.info("iscsiadm -m session -r %s --rescan fail, because %s" % (session_id, stderr))
+
+
+def rescan_for_detach(volume_obj):
+    volume_iqn = volume_obj.iscsi_path.replace('iscsi://', '').split("/")[1].strip()
+    if volume_iqn:
+        LOG.info("skip rescan for detach")
+        return
+
+    stdout, stderr = processutils.trycmd("timeout 30 iscsiadm -m session -R", shell=True)
+    if stderr:
+        LOG.info("timeout 30 iscsiadm -m session -R failed, because %s" % stderr)
+        return
+
+
 class LinuxDriver(base.SystemDriverBase):
     driver_name = 'linux'
 
@@ -120,7 +150,7 @@ def discovery_target(self, instance_obj):
         stdout, stderr = processutils.trycmd(cmd, shell=True)
         if not stderr:
             LOG.info("iscsi target:%s has logged" % target_name)
-            return target_name
+            return
 
         discovery_cmd = 'iscsiadm -m discovery -t sendtargets -p {address}:{port}'.format(
                                     address=instance_obj.gateway_ip,
@@ -140,10 +170,9 @@ def discovery_target(self, instance_obj):
                 processutils.execute(target_login_cmd, shell=True)
             else:
                 LOG.info("discovered targets not contains %s, skip login" % target_name)
-            return target_name
         except processutils.ProcessExecutionError:
             LOG.info("no iscsi target found, skip login")
-            return None
+        rescan_sids_target_name(target_name)
 
     def discovery_volume_target(self, instance_obj, volume_obj, volume_access_path_gateway_ips):
         if not volume_obj.iscsi_path:
@@ -153,12 +182,14 @@ def discovery_volume_target(self, instance_obj, volume_obj, volume_access_path_g
 
     def discovery_target_through_access_path_gateway_ips(self, target_name, volume_access_path_gateway_ips):
         for gateway_ip in volume_access_path_gateway_ips:
+            if not target_name:
+                discovery_cmd = "timeout 5 iscsiadm -m discovery -t sendtargets -p {address}:{port} | awk '{{print $2}}'".format(address=gateway_ip, port=3260)
+                stdout, stderr = processutils.trycmd(discovery_cmd, shell=True)
+                if stderr:
+                    raise Exception("discovered targets fail, %s" % stderr)
+                target_name = stdout.strip()
             self.login_target(target_name, gateway_ip)
-
-        cmd = 'iscsiadm -m session --rescan'
-        stdout, stderr = processutils.trycmd(cmd, shell=True)
-        if stderr:
-            LOG.info("iscsiadm -m session --rescan fail, because %s" % (stderr))
+        rescan_sids_target_name(target_name)
 
     def login_target(self, target_name, address_ip, port=3260):
         LOG.info("start login_target:%s by ip %s" % (target_name, address_ip))
@@ -196,25 +227,11 @@ def attach_volume(self, instance_obj, volume_obj, volume_access_path_gateway_ips
         First check the /etc/iscsi/initiatorname.iscsi whether corrent, if
         not, corrent the configuration, then rescan the iscsi session.
         """
-        target_name = self.discovery_target(instance_obj)
+        self.discovery_target(instance_obj)
         self.discovery_volume_target(instance_obj, volume_obj, volume_access_path_gateway_ips)
         _check_initiator_config(instance_obj.uuid)
         _check_multi_path_config()
 
-        if not target_name:
-            raise Exception("instance[%s] target name is not exist, can not rescan session" % instance_obj)
-        cmd = "iscsiadm -m session | grep {} | awk '{{print $2}}' | tr -d '[]'".format(target_name)
-        LOG.info(cmd)
-        stdout, stderr = processutils.trycmd(cmd, shell=True)
-        if stderr:
-            LOG.info("iscsiadm -m session grep %s failed, because %s" % (target_name, stderr))
-            return
-        rescan_cmd = 'iscsiadm -m session -r %s --rescan' % stdout.strip()
-        LOG.info(rescan_cmd)
-        stdout, stderr = processutils.trycmd(rescan_cmd, shell=True)
-        if stderr:
-            LOG.info("iscsiadm -m session --rescan fail, because %s" % stderr)
-
     def detach_volume(self, instance_obj, volume_obj, volume_access_path_gateway_ips):
         """ Detach a given iSCSI lun
 
@@ -238,11 +255,14 @@ def detach_volume(self, instance_obj, volume_obj, volume_access_path_gateway_ips
         """
         for volume_access_path_gateway_ip in volume_access_path_gateway_ips:
             self.detach_volume_for_target_ip(instance_obj, volume_obj, volume_access_path_gateway_ip)
+        rescan_for_detach(volume_obj)
 
     def detach_volume_for_target_ip(self, instance_obj, volume_obj, target_ip):
         # Get the session id
         sid = None
         volume_iqn = volume_obj.iscsi_path.replace('iscsi://', '').split("/")[1]
+        if not volume_iqn:
+            return
         if instance_obj.custom_iqn:
             iqn = instance_obj.custom_iqn
         elif volume_iqn:
diff --git a/bm-instance-agent/tools/install.sh b/bm-instance-agent/tools/install.sh
@@ -32,8 +32,9 @@ popd
 sed -i '/SELINUX=/s/enforcing/permissive/g' /etc/selinux/config >/dev/null 2>&1 || true
 setenforce 0 >/dev/null 2>&1 || true
 
-mkdir -p /usr/lib/systemd/system/
-cat << EOF > /usr/lib/systemd/system/zstack-baremetal-instance-agent.service
+if which systemctl &> /dev/null; then
+    mkdir -p /usr/lib/systemd/system/
+    cat << EOF > /usr/lib/systemd/system/zstack-baremetal-instance-agent.service
 [Unit]
 Description=ZStack baremetal instance agent
 After=network.target
@@ -54,9 +55,90 @@ ExecStart=/var/lib/zstack/baremetalv2/bm-instance-agent/bm-instance-agent.pex
 WantedBy=multi-user.target
 EOF
 
-systemctl daemon-reload \
-    && systemctl enable zstack-baremetal-instance-agent \
-    && systemctl restart zstack-baremetal-instance-agent
+    systemctl daemon-reload \
+        && systemctl enable zstack-baremetal-instance-agent \
+        && systemctl restart zstack-baremetal-instance-agent
+else
+    cat << 'EOF' > /etc/init.d/zstack-baremetal-instance-agent
+#!/bin/bash
+# chkconfig: 2345 77 22
+# description: ZStack Baremetal Agent Service
+
+# return value
+RETVAL=0
+server_dir="/var/lib/zstack/baremetalv2/bm-instance-agent"
+server_name="zstack-baremetal-instance-agent"
+
+
+start() {
+    echo -n "Starting ${server_name}:"
+        /bin/sh -c "/sbin/iptables -t filter -C INPUT -p tcp --dport=7090 -j ACCEPT 2>/dev/null || /sbin/iptables -t filter -I INPUT -p tcp --dport=7090 -j ACCEPT || true"
+        /bin/sh -c "/sbin/iptables -t filter -C INPUT -p tcp --dport=4200 -j ACCEPT 2>/dev/null || /sbin/iptables -t filter -I INPUT -p tcp --dport=4200 -j ACCEPT || true"
+        PIDS=`ps -ef |grep .pex/unzipped_pexes |grep -v grep | awk '{print $2}'`
+    if [ "$PIDS" != "" ]; then
+        echo "${server_name} is runing!"
+    else
+        nohup /var/lib/zstack/baremetalv2/bm-instance-agent/bm-instance-agent.pex >> /nohup.log 2>&1 &
+        echo ".... SUCCESS"
+    fi
+}
+
+
+stop(){
+    echo -n "Stopping ${server_name}:"
+    PIDS=`ps -ef |grep .pex/unzipped_pexes |grep -v grep | awk '{print $2}'`
+    if [ "$PIDS" == "" ];then
+        echo "No pids exist, is ${server_name} running?"
+    else
+        for pid in $PIDS; do
+            kill -9 "$pid"
+            sleep 5
+        done
+        echo ".... SUCCESS"
+    fi
+}
+
+status(){
+    PIDS=`ps -ef |grep .pex/unzipped_pexes |grep -v grep | awk '{print $2}'`
+    if [ "$PIDS" != "" ];then
+        echo "${server_name} is running"
+    else
+        echo "${server_name} is stopped"
+    fi
+}
+
+case "$1" in
+        start)
+                start
+                ;;
+        stop)
+                stop
+                ;;
+        status)
+                status
+                ;;
+        restart)
+                stop
+                status
+                start
+                sleep 5
+                status
+                RETVAL=$?
+                ;;
+        *)
+                echo $"Usage: $0 {start|stop|status|restart}"
+                exit 1
+esac
+
+exit $RETVAL
+EOF
+
+    cd /etc/init.d/
+    chkconfig --add zstack-baremetal-instance-agent
+    chkconfig --level 2345 zstack-baremetal-instance-agent on
+    chmod +x zstack-baremetal-instance-agent
+    bash zstack-baremetal-instance-agent restart
+fi
 
 chmod +x /var/lib/zstack/baremetalv2/bm-instance-agent/zwatch-vm-agent-`uname -m`
 /var/lib/zstack/baremetalv2/bm-instance-agent/zwatch-vm-agent-`uname -m` -i
diff --git a/kvmagent/kvmagent/plugins/baremetal_v2_gateway_agent.py b/kvmagent/kvmagent/plugins/baremetal_v2_gateway_agent.py
@@ -951,7 +951,8 @@ def _create_import_data_config(self, volume_driver, cmd):
             chassis_port=cmd.chassisInfo.port,
             api_id=cmd.threadContext.api,
             task_name=cmd.threadContext["task-name"],
-            provision_mac=instance_obj.provision_mac
+            provision_mac=instance_obj.provision_mac,
+            instance_uuid=instance_obj.uuid
         )
 
         with open(ks_config_path, 'w') as f:
diff --git a/kvmagent/kvmagent/plugins/bmv2_gateway_agent/object.py b/kvmagent/kvmagent/plugins/bmv2_gateway_agent/object.py
@@ -155,7 +155,8 @@ class VolumeObj(Base):
         'token': 'token',
         'tpTimeout': 'tpTimeout',
         'monIp': 'monIp',
-        'iscsiPath': 'iscsiPath'
+        'iscsiPath': 'iscsiPath',
+        'targetIqn': 'targetIqn'
     }
 
     @classmethod
diff --git a/kvmagent/kvmagent/plugins/bmv2_gateway_agent/templates/import-data.ks.j2 b/kvmagent/kvmagent/plugins/bmv2_gateway_agent/templates/import-data.ks.j2
@@ -65,9 +65,17 @@ def get_dest_dev(dest_wwn):
     raise Exception("Failed to find dest disk[%s]" % dest_wwn)
 
 def get_src_dev():
+    cmd = "mkdir -p /etc/iscsi && touch /etc/iscsi/initiatorname.iscsi && echo 'InitiatorName=iqn.2015-01.io.zstack:initiator.instance.{{instance_uuid}}' > /etc/iscsi/initiatorname.iscsi"
+    shell_cmd(cmd)
+    cmd = "systemctl restart iscsid"
+    shell_cmd(cmd)
+    time.sleep(1)
     cmd = "iscsiadm -m discovery -t sendtargets -p {{gateway_ip}}:3260"
     shell_cmd(cmd)
     time.sleep(1)
+    cmd = "iscsiadm --mode node --targetname {{iqn_name}} -p {{gateway_ip}}:3260 --login -o new"
+    shell_cmd(cmd)
+    time.sleep(1)
     cmd = "iscsiadm --mode node --targetname {{iqn_name}} -p {{gateway_ip}}:3260 --login"
     shell_cmd(cmd)
     time.sleep(3)
diff --git a/kvmagent/kvmagent/plugins/bmv2_gateway_agent/volume/__init__.py b/kvmagent/kvmagent/plugins/bmv2_gateway_agent/volume/__init__.py
@@ -2,23 +2,35 @@
 from kvmagent.plugins.bmv2_gateway_agent.volume import ceph
 from kvmagent.plugins.bmv2_gateway_agent.volume import sharedblock
 from kvmagent.plugins.bmv2_gateway_agent.volume import thirdparty_ceph
+from kvmagent.plugins.bmv2_gateway_agent.volume import expon
 
 mapping = {
     'ceph': ceph.CephVolume,
     'sharedblock': sharedblock.SharedBlockVolume,
-    'thirdpartyCeph': thirdparty_ceph.ThirdPartyCephVolume
+    'thirdpartyCeph': thirdparty_ceph.ThirdPartyCephVolume,
+    'expon': expon.ExponVolume
 }
 
 
 def get_driver(instance_obj, volume_obj):
     ps_type = volume_obj.primary_storage_type.lower()
     if is_third_party_ceph(volume_obj):
         ps_type = 'thirdpartyCeph'
+    if is_third_party_addon(ps_type):
+        ps_type = get_type_from_third_party_addon(volume_obj)
     if ps_type not in mapping:
         raise exception.PrimaryStorageTypeNotSupport(
             primary_storage_type=ps_type)
     return mapping.get(ps_type)(instance_obj, volume_obj)
 
 
 def is_third_party_ceph(token_object):
-    return hasattr(token_object, "token") and token_object.token
+    return hasattr(token_object, "token") and token_object.token
+
+
+def is_third_party_addon(ps_type):
+    return ps_type == "addon"
+
+
+def get_type_from_third_party_addon(volume_obj):
+    return volume_obj.path.split(":")[0]
diff --git a/kvmagent/kvmagent/plugins/bmv2_gateway_agent/volume/expon.py b/kvmagent/kvmagent/plugins/bmv2_gateway_agent/volume/expon.py
@@ -0,0 +1,74 @@
+from kvmagent.plugins.bmv2_gateway_agent.volume import base
+
+from zstacklib.utils import shell, linux
+
+
+class ExponVolume(base.BaseVolume):
+
+    def __init__(self, *args, **kwargs):
+        super(ExponVolume, self).__init__(*args, **kwargs)
+
+    def check_exist(self):
+        pass
+
+    @property
+    def nbd_backend(self):
+        return self.real_path
+
+    @property
+    def iscsi_backend(self):
+        return self.nbd_dev
+
+    @property
+    def iscsi_target(self):
+        return self.volume_obj.targetIqn
+
+    @property
+    def real_path(self):
+        return self.volume_obj.path.replace('expon://', '')
+
+    def attach(self):
+        pass
+
+    def detach(self):
+        pass
+
+    def detach_volume(self):
+        pass
+
+# monIp: 172.27.16.181,172.27.16.99,...
+    def prepare_instance_resource(self):
+        instance_gateway_ip = self.instance_obj.gateway_ip
+        mon_ip = self.volume_obj.monIp
+        dev_name = linux.find_route_interface_by_destination_ip(mon_ip)
+        shell.run("iptables -t nat -A PREROUTING -s %s -d %s -p tcp --dport 3260 -j DNAT --to-destination %s:3260" %
+                  (self.instance_obj.provision_ip, instance_gateway_ip, mon_ip))
+        shell.run("iptables -t nat -A POSTROUTING -s %s -o %s -j SNAT --to-source %s" % (
+            self.instance_obj.provision_ip, dev_name, mon_ip))
+
+    def destroy_instance_resource(self):
+        instance_gateway_ip = self.instance_obj.gateway_ip
+        mon_ip = self.volume_obj.monIp
+        dev_name = linux.find_route_interface_by_destination_ip(mon_ip)
+        shell.run("iptables -t nat -D PREROUTING -s %s -d %s -p tcp --dport 3260 -j DNAT --to-destination %s:3260" %
+                  (self.instance_obj.provision_ip, instance_gateway_ip, mon_ip))
+        shell.run("iptables -t nat -D POSTROUTING -s %s -o %s -j SNAT --to-source %s" % (
+            self.instance_obj.provision_ip, dev_name, mon_ip))
+
+    def pre_take_volume_snapshot(self):
+        pass
+
+    def post_take_volume_snapshot(self, src_vol):
+        pass
+
+    def resume(self):
+        pass
+
+    def rollback_volume_snapshot(self, src_vol):
+        pass
+
+    def get_lun_id(self):
+        pass
+
+    def roll_back_attach_volume(self):
+        self.detach_volume()

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-__version__ = "2.0.5"`
	`1`	`+__version__ = "2.0.6"`
Original file line number	Diff line number	Diff line change
`@@ -951,7 +951,8 @@ def _create_import_data_config(self, volume_driver, cmd):`
`951`	`951`	`chassis_port=cmd.chassisInfo.port,`
`952`	`952`	`api_id=cmd.threadContext.api,`
`953`	`953`	`task_name=cmd.threadContext["task-name"],`
`954`		`- provision_mac=instance_obj.provision_mac`
	`954`	`+ provision_mac=instance_obj.provision_mac,`
	`955`	`+ instance_uuid=instance_obj.uuid`
`955`	`956`	`)`
`956`	`957`
`957`	`958`	`with open(ks_config_path, 'w') as f:`
Original file line number	Diff line number	Diff line change
`@@ -155,7 +155,8 @@ class VolumeObj(Base):`
`155`	`155`	`'token': 'token',`
`156`	`156`	`'tpTimeout': 'tpTimeout',`
`157`	`157`	`'monIp': 'monIp',`
`158`		`- 'iscsiPath': 'iscsiPath'`
	`158`	`+ 'iscsiPath': 'iscsiPath',`
	`159`	`+ 'targetIqn': 'targetIqn'`
`159`	`160`	`}`
`160`	`161`
`161`	`162`	`@classmethod`