copy_file_range optimization tidy up

Author: Fallen-Breath

README.md

@@ -105,11 +105,19 @@ mcd_root/
 
 是否在备份时临时关闭自动保存
 
-### copy_on_write
+### enable_copy_file_range
 
 默认值: `false`
 
-使用某些文件系统的写时拷贝(一种增量备份)
+使用 `os.copy_file_range` 进行文件的复制
+
+在某些文件系统中，它会使用基于写时复制（copy-on-write）的 reflink 技术，从而极大地提升复制速度
+
+需求：
+
+- Linux 平台
+- Python >= 3.8
+- 选项 `backup_format` 为 `plain`
 
 ### ignored_files
 

README_en.md

@@ -105,11 +105,19 @@ Default: `true`
 
 If turn off auto save when making backup or not
 
-### copy_on_write
+### enable_copy_file_range
 
 Default: `false`
 
-Useing copy_on_write in some File system(incremental backup)
+Use `os.copy_file_range` for file copying
+
+In some file system, it will use the copy-on-write based reflink technique to greatly accelerate the copy speed
+
+Requirements:
+
+- Linux
+- Python >= 3.8
+- Option `backup_format` set to `plain`
 
 ### ignored_files
 

quick_backup_multi/__init__.py

@@ -82,27 +82,31 @@ def get_backup_file_name(backup_format: BackupFormat):
 	return backup_format.get_file_name('backup')
 
 
-copy_file_range_supported=hasattr(os, "copy_file_range")
-COW_COPY_BUFFER_SIZE = 2**30  # 1GB / need int, may overflow, so cannot copy files larger than 2GB in a single pass
+COPY_FILE_RANGE_SUPPORTED = hasattr(os, 'copy_file_range')
+COPY_FILE_RANGE_BUFFER_SIZE = 2 ** 30  # 1GiB
 
-#copy using "Copy On Write"
-def _cpcow(src_path: str, dst_path: str):
-	if not copy_file_range_supported or not config.copy_on_write:
+
+def copy_file_fast(src_path: str, dst_path: str) -> str:
+	"""
+	A ``shutil.copy2`` alternative that uses ``os.copy_file_range`` whenever possible
+
+	``os.copy_file_range`` may support copy-on-write, which is much faster than regular copy
+	"""
+	if not COPY_FILE_RANGE_SUPPORTED or not config.enable_copy_file_range:
 		return shutil.copy2(src_path, dst_path)
 
-	if os.path.isdir(dst_path):
+	if os.path.isdir(dst_path):  # ref: shutil.copy2
 		dst_path = os.path.join(dst_path, os.path.basename(src_path))
 
 	try:
-		with open(src_path,'rb') as fsrc, open(dst_path,'wb+') as fdst:
-			while os.copy_file_range(fsrc.fileno(), fdst.fileno(), COW_COPY_BUFFER_SIZE):
+		with open(src_path, 'rb') as f_src, open(dst_path, 'wb+') as f_dst:
+			while os.copy_file_range(f_src.fileno(), f_dst.fileno(), COPY_FILE_RANGE_BUFFER_SIZE):
 				pass
-
 	except Exception as e:
-		server_inst.logger.warning(str(e) + str(src_path) + "->" + str(dst_path) + ",Retry with other functions")
+		server_inst.logger.warning('copy_file_range {} -> {} failed ({}), retrying with shutil.copy'.format(src_path, dst_path, e))
 		shutil.copy(src_path, dst_path)
 
-	shutil.copystat(src_path, dst_path)  # copy2 
+	shutil.copystat(src_path, dst_path)  # ref: shutil.copy2
 	return dst_path
 
 
@@ -126,12 +130,12 @@ def copy_worlds(src: str, dst: str, intent: CopyWorldIntent, *, backup_format: O
 
 			server_inst.logger.info('copying {} -> {}'.format(src_path, dst_path))
 			if os.path.isdir(src_path):
-				shutil.copytree(src_path, dst_path, ignore=lambda path, files: set(filter(config.is_file_ignored, files)), copy_function=_cpcow)
+				shutil.copytree(src_path, dst_path, ignore=lambda path, files: set(filter(config.is_file_ignored, files)), copy_function=copy_file_fast)
 			elif os.path.isfile(src_path):
 				dst_dir = os.path.dirname(dst_path)
 				if not os.path.isdir(dst_dir):
 					os.makedirs(dst_dir)
-				_cpcow(src_path, dst_path)
+				copy_file_fast(src_path, dst_path)
 			else:
 				server_inst.logger.warning('{} does not exist while copying ({} -> {})'.format(src_path, src_path, dst_path))
 	elif backup_format in [BackupFormat.tar, BackupFormat.tar_gz, BackupFormat.tar_xz]:

quick_backup_multi/config.py

@@ -1,4 +1,4 @@
-from typing import List, Dict, Optional
+from typing import List, Dict
 
 from mcdreforged.api.utils.serializer import Serializable
 
@@ -10,7 +10,7 @@ class SlotInfo(Serializable):
 class Configuration(Serializable):
 	size_display: bool = True
 	turn_off_auto_save: bool = True
-	copy_on_write: bool = False
+	enable_copy_file_range: bool = False
 	ignored_files: List[str] = [
 		'session.lock'
 	]