optimize CreateBackupAction cache_previous_chunks_for_fixed_auto prettychunk querying

Author: Fallen-Breath

prime_backup/action/create_backup_action.py

@@ -227,6 +227,7 @@ def __cache_previous_chunks_for_fixed_auto(self, session: DbSession, scan_result
 		):
 			return
 
+		wanted_file_blobs: List[Tuple[Path, int]] = []  # list of (file path, blob id)
 		for file_entry in scan_result.all_files:
 			if not file_entry.is_file() or file_entry.path in self.__pre_calc_result.reused_files:
 				continue
@@ -243,11 +244,15 @@ def __cache_previous_chunks_for_fixed_auto(self, session: DbSession, scan_result
 			):
 				continue
 
+			blob_id = previous_file.blob_id
+			assert blob_id is not None
+			wanted_file_blobs.append((file_entry.path, blob_id))
+
+		if len(wanted_file_blobs) > 0:
 			with self.__time_costs.measure_time_cost(CreateBackupTimeCostKey.kind_db):
-				previous_file_chunks[file_entry.path] = [
-					PrettyChunk(offset_chunk.offset, offset_chunk.chunk.raw_size, offset_chunk.chunk.hash)
-					for offset_chunk in session.get_blob_chunks(previous_file.blob_id)
-				]
+				chunks_by_blob_id = session.batch_get_blob_pretty_chunks([blob_id for _, blob_id in wanted_file_blobs])
+			for file_path, blob_id in wanted_file_blobs:
+				previous_file_chunks[file_path] = chunks_by_blob_id[blob_id]
 
 	@classmethod
 	def _pre_calculate_hash_worker(

prime_backup/db/session.py

@@ -21,6 +21,7 @@
 
 if TYPE_CHECKING:
 	from sqlalchemy.sql.type_api import TypeEngine
+	from prime_backup.types.chunker import PrettyChunk
 
 
 _T = TypeVar('_T')
@@ -1097,6 +1098,43 @@ def get_blob_chunks(self, blob_id: int, *, limit: Optional[int] = None) -> List[
 		result: Sequence[Row[Tuple[int, schema.Chunk]]] = self.session.execute(stmt).all()
 		return [OffsetChunk(offset, chunk) for offset, chunk in result]
 
+	def batch_get_blob_pretty_chunks(self, blob_ids: List[int]) -> Dict[int, List['PrettyChunk']]:
+		"""
+		The result contains all given blob ids. Chunk lists are sorted.
+		"""
+		from prime_backup.types.chunker import PrettyChunk
+
+		blob_ids = collection_utils.deduplicated_list(blob_ids)
+		result: Dict[int, List['PrettyChunk']] = {blob_id: [] for blob_id in blob_ids}
+		if len(blob_ids) == 0:
+			return result
+
+		absolute_offset = (schema.BlobChunkGroupBinding.chunk_group_offset + schema.ChunkGroupChunkBinding.chunk_offset).label('absolute_offset')
+		for view in collection_utils.slicing_iterate(blob_ids, self.__safe_var_limit):
+			stmt = (
+				select(
+					schema.BlobChunkGroupBinding.blob_id,
+					absolute_offset,
+					schema.Chunk.raw_size,
+					schema.Chunk.hash,
+				).
+				select_from(schema.BlobChunkGroupBinding).
+				join(
+					schema.ChunkGroupChunkBinding,
+					schema.BlobChunkGroupBinding.chunk_group_id == schema.ChunkGroupChunkBinding.chunk_group_id
+				).
+				join(
+					schema.Chunk,
+					schema.ChunkGroupChunkBinding.chunk_id == schema.Chunk.id
+				).
+				where(schema.BlobChunkGroupBinding.blob_id.in_(view)).
+				order_by(schema.BlobChunkGroupBinding.blob_id, absolute_offset)
+			)
+			rows: Sequence[Row[Tuple[int, int, int, str]]] = self.session.execute(stmt).all()
+			for blob_id, offset, raw_size, chunk_hash in rows:
+				result[blob_id].append(PrettyChunk(offset, raw_size, chunk_hash))
+		return result
+
 	@dataclasses.dataclass(frozen=True)
 	class ListBlobChunkGroupBindingsItem:
 		binding: schema.BlobChunkGroupBinding