up docs

Author: Mantisus

docs/guides/storage_clients.mdx

@@ -208,6 +208,15 @@ class dataset_records {
     + data
 }
 
+class dataset_metadata_buffer {
+    <<table>>
+    + id (PK)
+    + accessed_at
+    + modified_at
+    + dataset_id (FK)
+    + delta_item_count
+}
+
 %% ========================
 %% Key-Value Store Tables
 %% ========================
@@ -231,15 +240,25 @@ class key_value_store_records {
     + size
 }
 
+class key_value_store_metadata_buffer {
+    <<table>>
+    + id (PK)
+    + accessed_at
+    + modified_at
+    + key_value_store_id (FK)
+}
+
 %% ========================
 %% Client to Table arrows
 %% ========================
 
 SqlDatasetClient --> datasets
 SqlDatasetClient --> dataset_records
+SqlDatasetClient --> dataset_metadata_buffer
 
 SqlKeyValueStoreClient --> key_value_stores
 SqlKeyValueStoreClient --> key_value_store_records
+SqlKeyValueStoreClient --> key_value_store_metadata_buffer
 ```
 ```mermaid
 ---
@@ -294,13 +313,27 @@ class request_queue_state {
     + forefront_sequence_counter
 }
 
+class request_queue_metadata_buffer {
+    <<table>>
+    + id (PK)
+    + accessed_at
+    + modified_at
+    + request_queue_id (FK)
+    + client_id
+    + delta_handled_count
+    + delta_pending_count
+    + delta_total_count
+    + need_recalc
+}
+
 %% ========================
 %% Client to Table arrows
 %% ========================
 
 SqlRequestQueueClient --> request_queues
 SqlRequestQueueClient --> request_queue_records
 SqlRequestQueueClient --> request_queue_state
+SqlRequestQueueClient --> request_queue_metadata_buffer
 ```
 
 Configuration options for the <ApiLink to="class/SqlStorageClient">`SqlStorageClient`</ApiLink> can be set through environment variables or the <ApiLink to="class/Configuration">`Configuration`</ApiLink> class:

src/crawlee/storage_clients/_sql/_client_mixin.py

@@ -272,10 +272,8 @@ async def _purge(self, metadata_kwargs: MetadataUpdateParams) -> None:
         await self._process_buffers()
 
         stmt_records = delete(self._ITEM_TABLE).where(self._ITEM_TABLE.storage_id == self._id)
-        stmt_buffers = delete(self._BUFFER_TABLE).where(self._BUFFER_TABLE.storage_id == self._id)
         async with self.get_session(with_simple_commit=True) as session:
             await session.execute(stmt_records)
-            await session.execute(stmt_buffers)
             await self._update_metadata(session, **metadata_kwargs)
 
     async def _drop(self) -> None:

src/crawlee/storage_clients/_sql/_storage_client.py

@@ -273,10 +273,9 @@ def _get_or_create_engine(self, configuration: Configuration) -> AsyncEngine:
         self._engine = create_async_engine(
             connection_string,
             future=True,
-            pool_size=5,
-            max_overflow=10,
-            pool_timeout=30,
-            pool_recycle=600,
+            pool_size=10,
+            max_overflow=50,
+            pool_timeout=60,
             pool_pre_ping=True,
             echo=False,
             connect_args={'timeout': 30},