Fix issue with files containing non-utf8 characters

Author: fabi200123

coriolis/osmorphing/base.py

@@ -335,7 +335,7 @@ def _read_config_file_sudo(self, chroot_path, check_exists=False):
             if check_exists:
                 raise IOError("could not find %s" % chroot_path)
             return {}
-        content = self._read_file_sudo(chroot_path).decode()
+        content = self._read_file_sudo(chroot_path)
         config = utils.parse_ini_config(content)
         return config
 
@@ -484,7 +484,7 @@ def _test_path_chroot(self, path):
         if path.startswith('/') is False:
             path = "/%s" % path
         exists = self._exec_cmd_chroot(
-            '[ -f "%s" ] && echo 1 || echo 0' % path).decode().rstrip('\n')
+            '[ -f "%s" ] && echo 1 || echo 0' % path).rstrip('\n')
         return exists == "1"
 
     def _read_file_sudo(self, chroot_path):
@@ -497,7 +497,7 @@ def _read_file_sudo(self, chroot_path):
     def _read_grub_config(self, config):
         if self._test_path_chroot(config) is False:
             raise IOError("could not find %s" % config)
-        contents = self._read_file_sudo(config).decode()
+        contents = self._read_file_sudo(config)
         ret = {}
         for line in contents.split('\n'):
             if line.startswith("#"):
@@ -512,7 +512,7 @@ def _get_grub_config_obj(self, grub_conf=None):
         grub_conf = grub_conf or "/etc/default/grub"
         if self._test_path_chroot(grub_conf) is False:
             raise IOError("could not find %s" % grub_conf)
-        tmp_file = self._exec_cmd_chroot("mktemp").decode().rstrip('\n')
+        tmp_file = self._exec_cmd_chroot("mktemp").rstrip('\n')
         self._exec_cmd_chroot(
             "/bin/cp -fp %s %s" % (grub_conf, tmp_file))
         config_file = self._read_grub_config(tmp_file)
@@ -561,7 +561,7 @@ def replace_in_cfg(opt, val):
                 replace_in_cfg(option, value)
         else:
             append_to_cfg(option, value)
-        cfg = self._read_file_sudo(config_obj["location"]).decode()
+        cfg = self._read_file_sudo(config_obj["location"])
         LOG.warning("TEMP CONFIG IS: %r" % cfg)
 
     def _set_grub2_cmdline(self, config_obj, options, clobber=False):

coriolis/osmorphing/debian.py

@@ -42,7 +42,7 @@ def disable_predictable_nic_names(self):
         grub_cfg = "etc/default/grub"
         if self._test_path_chroot(grub_cfg) is False:
             return
-        contents = self._read_file_sudo(grub_cfg).decode()
+        contents = self._read_file_sudo(grub_cfg)
         cfg = utils.Grub2ConfigEditor(contents)
         cfg.append_to_option(
             "GRUB_CMDLINE_LINUX_DEFAULT",
@@ -127,8 +127,7 @@ def set_net_config(self, nics_info, dhcp):
     def get_installed_packages(self):
         cmd = "dpkg-query -f '${binary:Package}\\n' -W"
         try:
-            self.installed_packages = self._exec_cmd_chroot(
-                cmd).decode('utf-8').splitlines()
+            self.installed_packages = self._exec_cmd_chroot(cmd).splitlines()
         except exception.CoriolisException:
             self.installed_packages = []
 

coriolis/osmorphing/osmount/base.py

@@ -122,7 +122,7 @@ def get_connection(self):
 
 class BaseLinuxOSMountTools(BaseSSHOSMountTools):
     def _get_pvs(self):
-        out = self._exec_cmd("sudo pvdisplay -c").decode().splitlines()
+        out = self._exec_cmd("sudo pvdisplay -c").splitlines()
         LOG.debug("Output of 'pvdisplay -c' command: %s", out)
         pvs = {}
         for line in out:
@@ -150,7 +150,7 @@ def _get_vgs(self):
         """
         vgs_cmd = (
             "sudo vgs -o vg_name,pv_name,vg_uuid, --noheadings --separator :")
-        out = self._exec_cmd(vgs_cmd).decode().splitlines()
+        out = self._exec_cmd(vgs_cmd).splitlines()
         LOG.debug("Output of %s command: %s", vgs_cmd, out)
         vgs_uuid_map = {}
         for line in out:
@@ -197,8 +197,7 @@ def _check_vgs(self):
 
     def _get_vgnames(self):
         vg_names = []
-        vgscan_out_lines = self._exec_cmd(
-            "sudo vgscan").decode().splitlines()
+        vgscan_out_lines = self._exec_cmd("sudo vgscan").splitlines()
         LOG.debug("Output of vgscan commnad: %s", vgscan_out_lines)
         for vgscan_out_line in vgscan_out_lines:
             m = re.match(
@@ -212,7 +211,7 @@ def _get_vgnames(self):
     def _get_lv_paths(self):
         """ Returns list with paths of available LVM volumes. """
         lvm_paths = []
-        out = self._exec_cmd("sudo lvdisplay -c").decode().strip()
+        out = self._exec_cmd("sudo lvdisplay -c").strip()
         if out:
             LOG.debug("Decoded `lvdisplay` output data: %s", out)
             out_lines = out.splitlines()
@@ -353,7 +352,7 @@ def _check_mount_fstab_partitions(
     def _get_symlink_target(self, symlink):
         target = symlink
         try:
-            target = self._exec_cmd('readlink -en %s' % symlink).decode()
+            target = self._exec_cmd('readlink -en %s' % symlink)
             LOG.debug("readlink %s returned: %s" % (symlink, target))
         except Exception:
             LOG.warn('Target not found for symlink: %s. Original link path '
@@ -375,8 +374,7 @@ def _get_device_file_paths(self, symlink_list):
         return dev_file_paths
 
     def _get_mounted_devices(self):
-        mounts = self._exec_cmd(
-            "cat /proc/mounts").decode().splitlines()
+        mounts = self._exec_cmd("cat /proc/mounts").splitlines()
         ret = []
         mounted_device_numbers = []
         dev_nmb_cmd = "mountpoint -x %s"
@@ -390,10 +388,9 @@ def _get_mounted_devices(self):
                     continue
                 ret.append(dev_name)
                 mounted_device_numbers.append(
-                    self._exec_cmd(dev_nmb_cmd % dev_name).decode().rstrip())
+                    self._exec_cmd(dev_nmb_cmd % dev_name).rstrip())
 
-        block_devs = self._exec_cmd(
-            "ls -al /dev | grep ^b").decode().splitlines()
+        block_devs = self._exec_cmd("ls -al /dev | grep ^b").splitlines()
         for dev_line in block_devs:
             dev = dev_line.split()
             major_minor = "%s:%s" % (
@@ -409,8 +406,7 @@ def _get_mounted_devices(self):
         return ret
 
     def _get_mount_destinations(self):
-        mounts = self._exec_cmd(
-            "cat /proc/mounts").decode().splitlines()
+        mounts = self._exec_cmd("cat /proc/mounts").splitlines()
         ret = set()
         for line in mounts:
             colls = line.split()
@@ -424,8 +420,7 @@ def _get_volume_block_devices(self):
         #   where 'ln -s /dev/dm-N /dev/<VG-name>/<LV-name>'
         # Querying for the kernel device name (KNAME) should ensure we get the
         # device names we desire both for physical and logical volumes.
-        volume_devs = self._exec_cmd(
-            "lsblk -lnao KNAME").decode().splitlines()
+        volume_devs = self._exec_cmd("lsblk -lnao KNAME").splitlines()
         LOG.debug("All block devices: %s", str(volume_devs))
 
         volume_devs = ["/dev/%s" % d for d in volume_devs if
@@ -446,7 +441,7 @@ def _find_dev_with_contents(self, devices, all_files=None,
         dev_name = None
         for dev_path in devices:
             dirs = None
-            tmp_dir = self._exec_cmd('mktemp -d').decode().splitlines()[0]
+            tmp_dir = self._exec_cmd('mktemp -d').splitlines()[0]
             try:
                 self._exec_cmd('sudo mount %s %s' % (dev_path, tmp_dir))
                 # NOTE: it's possible that the device was mounted successfully
@@ -508,7 +503,7 @@ def _find_and_mount_root(self, devices):
                 "being migrated.")
 
         try:
-            tmp_dir = self._exec_cmd('mktemp -d').decode().splitlines()[0]
+            tmp_dir = self._exec_cmd('mktemp -d').splitlines()[0]
             self._exec_cmd('sudo mount %s %s' % (os_root_device, tmp_dir))
             os_root_dir = tmp_dir
         except Exception as ex:
@@ -560,7 +555,7 @@ def mount_os(self):
         for volume_dev in volume_devs:
             self._exec_cmd("sudo partx -v -a %s || true" % volume_dev)
             dev_paths += self._exec_cmd(
-                "sudo ls -1 %s*" % volume_dev).decode().splitlines()
+                "sudo ls -1 %s*" % volume_dev).splitlines()
         LOG.debug("All simple devices to scan: %s", dev_paths)
 
         lvm_dev_paths = []
@@ -573,7 +568,7 @@ def mount_os(self):
             self._exec_cmd("sudo vgchange -ay -S vg_uuid=%s" % vg_uuid)
             self._exec_cmd("sudo vgchange --refresh")
             dev_paths_for_group = self._exec_cmd(
-                f"sudo ls -1 /dev/{vg_props['name']}/*").decode().splitlines()
+                f"sudo ls -1 /dev/{vg_props['name']}/*").splitlines()
             lvm_dev_paths.extend(dev_paths_for_group)
         LOG.debug("All LVM vols to scan: %s", lvm_dev_paths)
 
@@ -592,8 +587,8 @@ def mount_os(self):
                     dev_path, dev_target)
                 continue
             fs_type = self._exec_cmd(
-                "sudo blkid -o value -s TYPE %s || true" %
-                dev_path).decode().splitlines()
+                "sudo blkid -o value -s TYPE %s || true" % dev_path
+            ).splitlines()
             LOG.debug('Device %s filesystem types: %s', dev_path, fs_type)
             if fs_type and fs_type[0] in valid_filesystems:
                 if fs_type[0] == "xfs":

coriolis/osmorphing/redhat.py

@@ -172,8 +172,7 @@ def set_net_config(self, nics_info, dhcp):
     def get_installed_packages(self):
         cmd = 'rpm -qa --qf "%{NAME}\\n"'
         try:
-            self.installed_packages = self._exec_cmd_chroot(
-                cmd).decode('utf-8').splitlines()
+            self.installed_packages = self._exec_cmd_chroot(cmd).splitlines()
         except exception.CoriolisException:
             LOG.warning("Failed to get installed packages")
             LOG.trace(utils.get_exception_details())
@@ -231,7 +230,7 @@ def _find_yum_repos(self, repos_to_enable=[]):
         for file in repofiles:
             path = os.path.join(reposdir_path, file)
             try:
-                content = self._read_file_sudo(path).decode()
+                content = self._read_file_sudo(path)
             except Exception as e:
                 LOG.warning(
                     "Could not read yum repository file %s: %s", path, e)

coriolis/osmorphing/suse.py

@@ -68,8 +68,7 @@ def set_net_config(self, nics_info, dhcp):
     def get_installed_packages(self):
         cmd = 'rpm -qa --qf "%{NAME}\\n"'
         try:
-            self.installed_packages = self._exec_cmd_chroot(
-                cmd).decode('utf-8').splitlines()
+            self.installed_packages = self._exec_cmd_chroot(cmd).splitlines()
         except exception.CoriolisException:
             LOG.warning("Failed to get installed packages")
             LOG.trace(utils.get_exception_details())
@@ -134,7 +133,7 @@ def post_packages_install(self, package_names):
 
     def _enable_sles_module(self, module):
         available_modules = self._exec_cmd_chroot(
-            "SUSEConnect --list-extensions").decode()
+            "SUSEConnect --list-extensions")
         module_match = re.search("%s.*" % module, available_modules)
         try:
             module_path = module_match.group(0)
@@ -170,8 +169,7 @@ def _add_cloud_tools_repo(self):
     def _get_repos(self):
         repos = {}
         repos_list = self._exec_cmd_chroot(
-            "zypper repos -u | awk -F '|' '/^\s[0-9]+/ {print $2 $7}'"
-        ).decode()
+            "zypper repos -u | awk -F '|' '/^\s[0-9]+/ {print $2 $7}'")
         for repo in repos_list.splitlines():
             alias, uri = repo.strip().split()
             repos[alias] = uri

coriolis/providers/backup_writers.py

@@ -912,7 +912,7 @@ def _setup_certificates(self, ssh):
     def _read_remote_file_sudo(self, remote_path):
         contents = utils.exec_ssh_cmd(
             self._ssh, "sudo cat %s" % remote_path, get_pty=True)
-        return contents.decode()
+        return contents
 
     def _init_writer(self, ssh, cert_paths):
         cmdline = ("%(cmd)s run -ca-cert %(ca_cert)s -key "