add support mariadb dialect

Author: Mantisus

src/crawlee/storage_clients/_sql/_client_mixin.py

@@ -228,7 +228,7 @@ def _build_insert_stmt_with_ignore(
         if dialect == 'sqlite':
             return lite_insert(table_model).values(insert_values).on_conflict_do_nothing()
 
-        if dialect == 'mysql':
+        if dialect in {'mysql', 'mariadb'}:
             return mysql_insert(table_model).values(insert_values).prefix_with('IGNORE')
 
         raise NotImplementedError(f'Insert with ignore not supported for dialect: {dialect}')
@@ -264,7 +264,7 @@ def _build_upsert_stmt(
             set_ = {col: getattr(lite_stmt.excluded, col) for col in update_columns}
             return lite_stmt.on_conflict_do_update(index_elements=conflict_cols, set_=set_)
 
-        if dialect == 'mysql':
+        if dialect in {'mysql', 'mariadb'}:
             mysql_stmt = mysql_insert(table_model).values(insert_values)
             set_ = {col: getattr(mysql_stmt.inserted, col) for col in update_columns}
             return mysql_stmt.on_duplicate_key_update(**set_)
@@ -416,7 +416,7 @@ async def _try_acquire_buffer_lock(self, session: AsyncSession) -> bool:
         lock_until = now + self._BLOCK_BUFFER_TIME
         dialect = self._storage_client.get_dialect_name()
 
-        if dialect in ('postgresql', 'mysql'):
+        if dialect in {'postgresql', 'mysql', 'mariadb'}:
             select_stmt = (
                 select(self._METADATA_TABLE)
                 .where(

src/crawlee/storage_clients/_sql/_request_queue_client.py

@@ -433,7 +433,7 @@ async def fetch_next_request(self) -> Request | None:
 
         async with self.get_session(with_simple_commit=True) as session:
             # We use the `skip_locked` database mechanism to prevent the 'interception' of requests by another client
-            if dialect in ('postgresql', 'mysql'):
+            if dialect in {'postgresql', 'mysql', 'mariadb'}:
                 stmt = stmt.with_for_update(skip_locked=True)
                 result = await session.execute(stmt)
                 requests_db = result.scalars().all()

src/crawlee/storage_clients/_sql/_storage_client.py

@@ -116,10 +116,10 @@ async def initialize(self, configuration: Configuration) -> None:
             async with engine.begin() as conn:
                 self._dialect_name = engine.dialect.name
 
-                if self._dialect_name not in ('sqlite', 'postgresql', 'mysql'):
+                if self._dialect_name not in {'sqlite', 'postgresql', 'mysql', 'mariadb'}:
                     raise ValueError(
-                        f'Unsupported database dialect: {self._dialect_name}. Supported: sqlite, postgresql, mysql. '
-                        'Consider using a different database.',
+                        f'Unsupported database dialect: {self._dialect_name}. Supported: sqlite, postgresql, mysql, '
+                        'mariadb. Consider using a different database.',
                     )
 
                 # Create tables if they don't exist.
@@ -260,14 +260,16 @@ def _get_or_create_engine(self, configuration: Configuration) -> AsyncEngine:
             ('sqlite' not in connection_string)
             and ('postgresql' not in connection_string)
             and ('mysql' not in connection_string)
+            and ('mariadb' not in connection_string)
         ):
             raise ValueError(
-                'Unsupported database. Supported: sqlite, postgresql, mysql. Consider using a different database.'
+                'Unsupported database. Supported: sqlite, postgresql, mysql, mariadb. Consider using a different '
+                'database.'
             )
 
         connect_args: dict[str, Any]
         kwargs: dict[str, Any] = {}
-        if 'mysql' in connection_string:
+        if 'mysql' in connection_string or 'mariadb' in connection_string:
             connect_args: dict[str, Any] = {'connect_timeout': 30}
             kwargs['isolation_level'] = 'READ COMMITTED'
         else: