Skip to content

Commit f7ee739

Browse files
franccescoclaude
andcommitted
refactor(utils): add generic bulk operations helpers
- Add _validate_bulk_item() to AbstractOperations - Add _process_bulk_sync() for sequential bulk processing - Add _process_bulk_async() with semaphore-based concurrency control Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 58d193b commit f7ee739

2 files changed

Lines changed: 109 additions & 1 deletion

File tree

src/bloomy/utils/abstract_operations.py

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
from typing import Any
66

7+
from ..models import BulkCreateError, BulkCreateResult
8+
79

810
class AbstractOperations:
911
"""Abstract base class for shared logic between sync and async operations."""
@@ -47,3 +49,52 @@ def _validate_mutual_exclusion(
4749
"""
4850
if param1 is not None and param2 is not None:
4951
raise ValueError(f"Cannot specify both {param1_name} and {param2_name}")
52+
53+
def _validate_bulk_item(
54+
self, item_data: dict[str, Any], required_fields: list[str]
55+
) -> None:
56+
"""Validate that required fields are present in bulk item data.
57+
58+
Args:
59+
item_data: The item data dictionary to validate.
60+
required_fields: List of required field names.
61+
62+
Raises:
63+
ValueError: If any required field is missing.
64+
65+
"""
66+
for field in required_fields:
67+
if item_data.get(field) is None:
68+
raise ValueError(f"{field} is required")
69+
70+
def _process_bulk_sync[T](
71+
self,
72+
items: list[dict[str, Any]],
73+
create_func: Any,
74+
required_fields: list[str],
75+
) -> BulkCreateResult[T]:
76+
"""Process bulk creation synchronously.
77+
78+
Args:
79+
items: List of item data dictionaries.
80+
create_func: Function to create a single item from data dict.
81+
required_fields: List of required field names.
82+
83+
Returns:
84+
BulkCreateResult with successful and failed items.
85+
86+
"""
87+
successful: list[T] = []
88+
failed: list[BulkCreateError] = []
89+
90+
for index, item_data in enumerate(items):
91+
try:
92+
self._validate_bulk_item(item_data, required_fields)
93+
created = create_func(item_data)
94+
successful.append(created)
95+
except Exception as e:
96+
failed.append(
97+
BulkCreateError(index=index, input_data=item_data, error=str(e))
98+
)
99+
100+
return BulkCreateResult(successful=successful, failed=failed)

src/bloomy/utils/async_base_operations.py

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,15 @@
22

33
from __future__ import annotations
44

5-
from typing import TYPE_CHECKING
5+
import asyncio
6+
from typing import TYPE_CHECKING, Any
67

8+
from ..models import BulkCreateError, BulkCreateResult
79
from .abstract_operations import AbstractOperations
810

911
if TYPE_CHECKING:
12+
from collections.abc import Awaitable, Callable
13+
1014
import httpx
1115

1216

@@ -65,3 +69,56 @@ async def _get_default_user_id(self) -> int:
6569
response.raise_for_status()
6670
data = response.json()
6771
return data["Id"]
72+
73+
async def _process_bulk_async[T](
74+
self,
75+
items: list[dict[str, Any]],
76+
create_func: Callable[[dict[str, Any]], Awaitable[T]],
77+
required_fields: list[str],
78+
max_concurrent: int = 5,
79+
) -> BulkCreateResult[T]:
80+
"""Process bulk creation asynchronously with concurrency control.
81+
82+
Args:
83+
items: List of item data dictionaries.
84+
create_func: Async function to create a single item from data dict.
85+
required_fields: List of required field names.
86+
max_concurrent: Maximum number of concurrent API requests.
87+
88+
Returns:
89+
BulkCreateResult with successful and failed items.
90+
91+
"""
92+
semaphore = asyncio.Semaphore(max_concurrent)
93+
94+
async def create_single(
95+
index: int, item_data: dict[str, Any]
96+
) -> tuple[int, T | BulkCreateError]:
97+
async with semaphore:
98+
try:
99+
self._validate_bulk_item(item_data, required_fields)
100+
created = await create_func(item_data)
101+
return (index, created)
102+
except Exception as e:
103+
error = BulkCreateError(
104+
index=index, input_data=item_data, error=str(e)
105+
)
106+
return (index, error)
107+
108+
tasks = [
109+
create_single(index, item_data) for index, item_data in enumerate(items)
110+
]
111+
results = await asyncio.gather(*tasks)
112+
results_list = list(results)
113+
results_list.sort(key=lambda x: x[0])
114+
115+
successful: list[T] = []
116+
failed: list[BulkCreateError] = []
117+
118+
for _, result in results_list:
119+
if isinstance(result, BulkCreateError):
120+
failed.append(result)
121+
else:
122+
successful.append(result)
123+
124+
return BulkCreateResult(successful=successful, failed=failed)

0 commit comments

Comments
 (0)