|
| 1 | +from core.classes import Storage, FileInfo, FileList |
| 2 | +from core.logger import logger |
| 3 | +from botocore.config import Config |
| 4 | +from botocore.exceptions import ClientError |
| 5 | +from typing import Literal, Union |
| 6 | +from tqdm import tqdm |
| 7 | +import boto3 |
| 8 | +import humanize |
| 9 | +import io |
| 10 | +import asyncio |
| 11 | +import secrets |
| 12 | + |
| 13 | + |
| 14 | +class S3Storage(Storage): |
| 15 | + def __init__( |
| 16 | + self, |
| 17 | + endpoint: str, |
| 18 | + access_key_id: str, |
| 19 | + secret_access_key: str, |
| 20 | + signature_version: str, |
| 21 | + bucket: str, |
| 22 | + addressing_style: Literal["auto", "path", "virtual"] = "auto", |
| 23 | + session_token: Union[None, str] = None, |
| 24 | + ): |
| 25 | + self.client = boto3.client( |
| 26 | + "s3", |
| 27 | + aws_access_key_id=access_key_id, |
| 28 | + aws_secret_access_key=secret_access_key, |
| 29 | + aws_session_token=session_token, |
| 30 | + endpoint_url=endpoint, |
| 31 | + config=Config(s3={"addressing_style": addressing_style}, signature_version=signature_version), |
| 32 | + ) |
| 33 | + self.bucket = bucket |
| 34 | + |
| 35 | + async def init(self) -> None: |
| 36 | + pass |
| 37 | + |
| 38 | + async def writeFile( |
| 39 | + self, file: FileInfo, content: io.BytesIO, delay: int, retry: int |
| 40 | + ) -> bool: |
| 41 | + file_path = f"{file.hash[:2]}/{file.hash}" |
| 42 | + try: |
| 43 | + response = self.client.head_object(Bucket=self.bucket, Key=file_path) |
| 44 | + if response["ContentLength"] == len(content.getvalue()): |
| 45 | + return True |
| 46 | + except Exception: |
| 47 | + pass |
| 48 | + |
| 49 | + for _ in range(retry): |
| 50 | + try: |
| 51 | + self.client.put_object(Bucket=self.bucket, Key=file_path, Body=content.getbuffer()) |
| 52 | + response = self.client.head_object(Bucket=self.bucket, Key=file_path) |
| 53 | + uploaded_size = response["ContentLength"] |
| 54 | + if uploaded_size == file.size: |
| 55 | + return True |
| 56 | + else: |
| 57 | + logger.terror( |
| 58 | + "storage.error.s3.write_file.size_mismatch", |
| 59 | + file=file.hash, |
| 60 | + file_size=humanize.naturalsize(file.size, binary=True), |
| 61 | + actual_file_size=humanize.naturalsize(uploaded_size, binary=True), |
| 62 | + ) |
| 63 | + return False |
| 64 | + |
| 65 | + except ClientError as e: |
| 66 | + logger.terror( |
| 67 | + "storage.error.s3.write_file.retry", |
| 68 | + file=file.hash, |
| 69 | + e=e, |
| 70 | + retry=delay, |
| 71 | + ) |
| 72 | + |
| 73 | + await asyncio.sleep(delay) |
| 74 | + |
| 75 | + logger.terror("storage.error.s3.write_file.failed", file=file.hash) |
| 76 | + return False |
| 77 | + |
| 78 | + async def check(self) -> None: |
| 79 | + file_path = secrets.token_hex(8) |
| 80 | + try: |
| 81 | + self.client.put_object(Bucket=self.bucket, Key=file_path, Body=b"") |
| 82 | + self.client.head_object(Bucket=self.bucket, Key=file_path) |
| 83 | + self.client.delete_object(Bucket=self.bucket, Key=file_path) |
| 84 | + logger.tsuccess("storage.success.s3.check") |
| 85 | + except Exception as e: |
| 86 | + logger.terror("storage.error.s3.check", e=e) |
| 87 | + |
| 88 | + async def getMissingFiles(self, files: FileList, pbar: tqdm) -> FileList: |
| 89 | + async def checkFile(file: FileInfo, pbar: tqdm) -> bool: |
| 90 | + pbar.update(1) |
| 91 | + file_path = f"{file.hash[:2]}/{file.hash}" |
| 92 | + try: |
| 93 | + response = self.client.head_object(Bucket=self.bucket, Key=file_path) |
| 94 | + s3_file_size = response["ContentLength"] |
| 95 | + return s3_file_size != file.size |
| 96 | + except Exception: |
| 97 | + pass |
| 98 | + |
| 99 | + results = await asyncio.gather(*[checkFile(file, pbar) for file in files.files]) |
| 100 | + missing_files = [ |
| 101 | + file for file, is_missing in zip(files.files, results) if is_missing |
| 102 | + ] |
| 103 | + return FileList(files=missing_files) |
0 commit comments