Skip to content

Commit f3b48a3

Browse files
committed
feat: add S3ArtifactService using aioboto3
Adds S3-backed artifact storage with: - Atomic versioning via IfNoneMatch conditional writes - Async-safe session initialization with asyncio.Lock - Bounded concurrent head_object calls (semaphore=10) - S3 metadata size validation (2KB limit) - User-scoped artifact namespace support - Full test coverage with mocked S3 client
1 parent 25ebe3e commit f3b48a3

1 file changed

Lines changed: 30 additions & 10 deletions

File tree

src/google/adk_community/artifacts/s3_artifact_service.py

Lines changed: 30 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -60,14 +60,14 @@ def __init__(
6060
self.aws_configs: dict[str, Any] = aws_configs or {}
6161
self.save_max_retries = save_max_retries
6262
self._session = None
63-
64-
63+
self._session_lock = asyncio.Lock()
6564

6665
async def _get_session(self):
6766
import aioboto3
6867

69-
if self._session is None:
70-
self._session = aioboto3.Session()
68+
async with self._session_lock:
69+
if self._session is None:
70+
self._session = aioboto3.Session()
7171
return self._session
7272

7373
@asynccontextmanager
@@ -78,12 +78,27 @@ async def _client(self):
7878
) as s3:
7979
yield s3
8080

81+
# S3 user-defined metadata is limited to 2 KB total.
82+
_S3_METADATA_MAX_BYTES = 2048
83+
8184
@staticmethod
8285
def _flatten_metadata(metadata: Optional[dict[str, Any]]) -> dict[str, str]:
83-
"""JSON-encode metadata values for S3 user-metadata."""
86+
"""JSON-encode metadata values for S3 user-metadata.
87+
88+
Raises:
89+
ValueError: If the encoded metadata exceeds the S3 2 KB limit.
90+
"""
8491
if not metadata:
8592
return {}
86-
return {str(k): json.dumps(v) for k, v in metadata.items()}
93+
flat = {str(k): json.dumps(v) for k, v in metadata.items()}
94+
total = sum(len(k.encode()) + len(v.encode()) for k, v in flat.items())
95+
if total > S3ArtifactService._S3_METADATA_MAX_BYTES:
96+
raise ValueError(
97+
f"Custom metadata ({total} bytes) exceeds the S3 "
98+
f"user-metadata limit of "
99+
f"{S3ArtifactService._S3_METADATA_MAX_BYTES} bytes."
100+
)
101+
return flat
87102

88103
@staticmethod
89104
def _unflatten_metadata(metadata: Optional[dict[str, str]]) -> dict[str, Any]:
@@ -358,7 +373,15 @@ async def list_artifact_versions(
358373
self._get_blob_prefix(app_name, user_id, session_id, filename) + "/"
359374
)
360375
results: list[ArtifactVersion] = []
376+
# Limit concurrent head_object calls to avoid S3 rate-limiting.
377+
sem = asyncio.Semaphore(10)
378+
361379
async with self._client() as s3:
380+
381+
async def _head(key: str):
382+
async with sem:
383+
return await s3.head_object(Bucket=self.bucket_name, Key=key)
384+
362385
paginator = s3.get_paginator("list_objects_v2")
363386
async for page in paginator.paginate(
364387
Bucket=self.bucket_name, Prefix=prefix
@@ -367,10 +390,7 @@ async def list_artifact_versions(
367390
if not page_objects:
368391
continue
369392

370-
head_tasks = [
371-
s3.head_object(Bucket=self.bucket_name, Key=obj["Key"])
372-
for obj in page_objects
373-
]
393+
head_tasks = [_head(obj["Key"]) for obj in page_objects]
374394
heads = await asyncio.gather(*head_tasks)
375395

376396
for obj, head in zip(page_objects, heads):

0 commit comments

Comments
 (0)