optimize DbSession.get_blob_chunks to return read-only row objects

avoiding creating lots of orm objects

Author: Fallen-Breath

prime_backup/action/helpers/chunk_grouper.py

@@ -1,12 +1,13 @@
 import dataclasses
 import logging
 import time
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Union
 
 from prime_backup.action.helpers.create_backup_utils import CreateBackupTimeCostKey
 from prime_backup.constants import chunk_constants
 from prime_backup.db import schema
 from prime_backup.db.session import DbSession
+from prime_backup.db.values import ChunkRow
 from prime_backup.utils import chunk_utils
 from prime_backup.utils.time_cost_stats import TimeCostStats
 
@@ -22,7 +23,7 @@ class ChunkLike:
 		stored_size: int
 
 		@classmethod
-		def of(cls, chunk: schema.Chunk) -> 'ChunkGrouper.ChunkLike':
+		def of(cls, chunk: Union[schema.Chunk, ChunkRow]) -> 'ChunkGrouper.ChunkLike':
 			return cls(
 				id=chunk.id,
 				hash=chunk.hash,

prime_backup/action/migrate_hash_method_action.py

@@ -13,7 +13,7 @@
 from prime_backup.db import schema
 from prime_backup.db.access import DbAccess
 from prime_backup.db.session import DbSession
-from prime_backup.db.values import BlobStorageMethod
+from prime_backup.db.values import BlobStorageMethod, ChunkRow
 from prime_backup.exceptions import PrimeBackupError
 from prime_backup.types.chunk_info import ChunkInfo
 from prime_backup.types.hash_method import HashMethod
@@ -183,7 +183,7 @@ def __regroup_chunked_blobs(self, session: DbSession):
 		# Step 1 - collect blob -> ordered chunks before destroying the binding chain
 		chunked_blobs = session.list_blobs_by_storage_method(BlobStorageMethod.chunked)
 		total = len(chunked_blobs)
-		blob_chunks_map: Dict[int, Dict[int, schema.Chunk]] = {}
+		blob_chunks_map: Dict[int, Dict[int, ChunkRow]] = {}
 		for blob in chunked_blobs:
 			offset_chunks = session.get_blob_chunks(blob.id)  # sorted by absolute_offset
 			blob_chunks_map[blob.id] = {oc.offset: oc.chunk for oc in offset_chunks}

prime_backup/db/session.py

@@ -13,7 +13,7 @@
 
 from prime_backup.db import schema, db_constants
 from prime_backup.db.db_features import DbFeatures
-from prime_backup.db.values import FileRole, BackupTagDict, OffsetChunk, OffsetChunkGroup, BlobStorageMethod, ChunkGroupChunkBindingIdentifier, BlobChunkGroupBindingIdentifier, FileIdentifier
+from prime_backup.db.values import FileRole, BackupTagDict, ChunkRow, OffsetChunk, OffsetChunkRow, OffsetChunkGroup, BlobStorageMethod, ChunkGroupChunkBindingIdentifier, BlobChunkGroupBindingIdentifier, FileIdentifier
 from prime_backup.exceptions import BackupNotFound, BackupFileNotFound, BlobHashNotFound, PrimeBackupError, FilesetNotFound, FilesetFileNotFound, BlobIdNotFound, ChunkHashNotFound, ChunkIdNotFound, ChunkGroupChunkBindingNotFound, BlobChunkGroupBindingNotFound, ChunkGroupIdNotFound, ChunkGroupHashNotFound, PackIdNotFound
 from prime_backup.types.backup_filter import BackupFilter, BackupTagFilter, BackupSortOrder
 from prime_backup.types.pack_info import PackEntryInfo
@@ -1097,15 +1097,21 @@ def get_blob_chunk_count(self, blob_id: int) -> int:
 			where(schema.BlobChunkGroupBinding.blob_id == blob_id)
 		).scalar_one())
 
-	def get_blob_chunks(self, blob_id: int, *, limit: Optional[int] = None) -> List[OffsetChunk]:
+	def get_blob_chunks(self, blob_id: int, *, limit: Optional[int] = None) -> List[OffsetChunkRow]:
 		"""
 		result is sorted
 		"""
 		absolute_offset = (schema.BlobChunkGroupBinding.chunk_group_offset + schema.ChunkGroupChunkBinding.chunk_offset).label('absolute_offset')
 		stmt = (
 			select(
 				absolute_offset,
-				schema.Chunk
+				schema.Chunk.id,
+				schema.Chunk.hash,
+				schema.Chunk.compress,
+				schema.Chunk.raw_size,
+				schema.Chunk.stored_size,
+				schema.Chunk.pack_id,
+				schema.Chunk.pack_offset,
 			).
 			select_from(schema.BlobChunkGroupBinding).
 			join(
@@ -1121,8 +1127,22 @@ def get_blob_chunks(self, blob_id: int, *, limit: Optional[int] = None) -> List[
 		)
 		if limit is not None:
 			stmt = stmt.limit(limit)
-		result: Sequence[Row[Tuple[int, schema.Chunk]]] = self.session.execute(stmt).all()
-		return [OffsetChunk(offset, chunk) for offset, chunk in result]
+		result: Sequence[Row[Tuple[int, int, str, str, int, int, int, int]]] = self.session.execute(stmt).all()
+		return [
+			OffsetChunkRow(
+				offset=offset,
+				chunk=ChunkRow(
+					id=chunk_id,
+					hash=chunk_hash,
+					compress=compress,
+					raw_size=raw_size,
+					stored_size=stored_size,
+					pack_id=pack_id,
+					pack_offset=pack_offset,
+				),
+			)
+			for offset, chunk_id, chunk_hash, compress, raw_size, stored_size, pack_id, pack_offset in result
+		]
 
 	def batch_get_blob_pretty_chunks(self, blob_ids: List[int]) -> Dict[int, List['PrettyChunk']]:
 		"""

prime_backup/db/values.py

@@ -14,6 +14,26 @@ class BlobStorageMethod(enum.IntEnum):
 	chunked = 2  # split into chunks and stored as pack entries
 
 
+@dataclasses.dataclass(frozen=True)
+class ChunkRow:
+	id: int
+	hash: str
+	compress: str
+	raw_size: int
+	stored_size: int
+	pack_id: int
+	pack_offset: int
+
+
+@dataclasses.dataclass(frozen=True)
+class OffsetChunkRow:
+	offset: int
+	chunk: ChunkRow
+
+	def __lt__(self, other: 'OffsetChunkRow'):
+		return self.offset < other.offset
+
+
 @dataclasses.dataclass(frozen=True)
 class OffsetChunk:
 	offset: int

prime_backup/types/chunk_info.py

@@ -1,12 +1,12 @@
 import dataclasses
 from pathlib import Path
-from typing import Iterable, TYPE_CHECKING
+from typing import Iterable, TYPE_CHECKING, Union
 
 from typing_extensions import Self
 
 from prime_backup.compressors import CompressMethod
 from prime_backup.db import schema
-from prime_backup.db.values import OffsetChunk
+from prime_backup.db.values import ChunkRow, OffsetChunk, OffsetChunkRow
 from prime_backup.types.pack_info import PackEntryLocation
 from prime_backup.utils import misc_utils
 
@@ -24,7 +24,7 @@ class ChunkInfo:
 	pack_entry: PackEntryLocation
 
 	@classmethod
-	def of(cls, chunk: schema.Chunk) -> 'ChunkInfo':
+	def of(cls, chunk: Union[schema.Chunk, ChunkRow]) -> 'ChunkInfo':
 		return ChunkInfo(
 			id=chunk.id,
 			hash=chunk.hash,
@@ -50,7 +50,7 @@ def size(self) -> int:
 		return self.chunk.raw_size
 
 	@classmethod
-	def of(cls, offset_chunk: OffsetChunk) -> 'OffsetChunkInfo':
+	def of(cls, offset_chunk: Union[OffsetChunk, OffsetChunkRow]) -> 'OffsetChunkInfo':
 		return cls(offset_chunk.offset, ChunkInfo.of(offset_chunk.chunk))
 
 	def __lt__(self, other: 'OffsetChunkInfo') -> bool: