feat(s3): query minio metrics for free space (#293)

Author: phwissmann

backend/api/.openapi-generator/FILES

@@ -28,5 +28,7 @@ src/openapi_server/models/http_validation_error.py
 src/openapi_server/models/internal_error.py
 src/openapi_server/models/presigned_url_body.py
 src/openapi_server/models/presigned_url_resp.py
+src/openapi_server/models/upload_request_body.py
+src/openapi_server/models/upload_request_resp.py
 src/openapi_server/models/validation_error.py
 src/openapi_server/models/validation_error_loc_inner.py

backend/api/.vscode/launch.json

@@ -9,18 +9,19 @@
       "type": "debugpy",
       "request": "launch",
       "env": {
-        "PREFECT_API_URL": "http://localhost:4200/api",
+        "PREFECT_API_URL": "https://scopem-openem2.ethz.ch/archiver/prefect/api",
         "PREFECT_LOGGING_LEVEL": "DEBUG",
         "MINIO_ENDPOINT": "scopem-openemdata.ethz.ch:9090",
         "SECRETS_DIR": "${workspaceFolder}/../../.secrets",
         "UVICORN_ROOT_PATH": "",
         "UVICORN_RELOAD": "true",
         "UVICORN_LOG_LEVEL": "debug",
         "UVICORN_PORT": "8888",
-        "IDP_CLIENT_SECRET":"ASFD",
-        "IDP_PASSWORD":"ASFD",
-        "JOB_ENDPOINT_PASSWORD":"pw",
-        "JOB_ENDPOINT_USERNAME":"username"
+        "IDP_CLIENT_SECRET": "ASFD",
+        "IDP_PASSWORD": "ASFD",
+        "JOB_ENDPOINT_PASSWORD": "pw",
+        "JOB_ENDPOINT_USERNAME": "username",
+        "SCICAT_INGESTOR_GROUP": "ingestor"
       },
       "cwd": "${workspaceFolder}/src",
       "module": "openapi_server",

backend/api/openapi.yaml

@@ -138,6 +138,38 @@ paths:
       summary: Create Archiver Job
       tags:
         - archiving
+  /s3/requestDatasetUpload:
+    post:
+      operationId: request_dataset_upload
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/UploadRequestBody"
+        required: true
+      responses:
+        "201":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/UploadRequestResp"
+          description: Upload requested
+        "422":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/HTTPValidationError"
+          description: Validation Error
+        "500":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/InternalError"
+          description: Internal Server Error
+      summary: Request a new upload
+      tags:
+        - s3upload
+
   /s3/presignedUrls:
     post:
       operationId: get_presigned_urls
@@ -562,6 +594,32 @@ components:
         - upload_id
         - urls
       title: PresignedUrlResp
+    UploadRequestBody:
+      example:
+        DatasetPID:
+          - dataset/pid
+        TotalGigabytes: 1000
+      properties:
+        DatasetPID:
+          title: DatasetPID
+          type: string
+        TotalGigabytes:
+          title: TotalGigabytes
+          type: integer
+      required:
+        - DatasetPID
+        - TotalGigabytes
+      title: UploadRequestBody
+    UploadRequestResp:
+      example:
+        Ok: true
+      properties:
+        Ok:
+          title: Ok
+          type: boolean
+      required:
+        - Ok 
+      title: UploadRequestResp
     ValidationError:
       example:
         msg: msg

backend/api/pyproject.toml

@@ -69,5 +69,6 @@ lint = [
     "mypy>=1.14.1",
 ]
 dev = [
+    "pytest-asyncio>=1.3.0",
     "requests-mock>=1.12.1",
 ]

backend/api/src/openapi_server/__main__.py

@@ -6,6 +6,7 @@
 """
 
 from importlib.metadata import version
+from logging import getLogger
 import pathlib
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
@@ -18,7 +19,6 @@
 from starlette.middleware.base import BaseHTTPMiddleware
 
 from .settings import GetSettings
-from logging import getLogger
 
 __version__ = version("archiver-service-api")
 
@@ -78,6 +78,10 @@ def create_app() -> FastAPI:
 
     log_config = uvicorn.config.LOGGING_CONFIG
     log_config["loggers"]["uvicorn"]["level"] = settings.UVICORN_LOG_LEVEL.upper()
+    log_config["formatters"]["default"]["fmt"] = (
+        "%(asctime)s  |  " + log_config["formatters"]["default"]["fmt"]
+    )
+    log_config["formatters"]["access"]["fmt"] = "%(asctime)s  |  " + log_config["formatters"]["access"]["fmt"]
     uvi_config = uvicorn.Config(
         app,
         host="0.0.0.0",

backend/api/src/openapi_server/apis/s3upload_api.py

@@ -33,6 +33,8 @@
 from openapi_server.models.internal_error import InternalError
 from openapi_server.models.presigned_url_body import PresignedUrlBody
 from openapi_server.models.presigned_url_resp import PresignedUrlResp
+from openapi_server.models.upload_request_body import UploadRequestBody
+from openapi_server.models.upload_request_resp import UploadRequestResp
 from openapi_server.security_api import get_token_BearerAuth
 
 router = APIRouter()
@@ -128,3 +130,25 @@ async def get_presigned_urls(
     if not BaseS3uploadApi.subclasses:
         raise HTTPException(status_code=500, detail="Not implemented")
     return await BaseS3uploadApi.subclasses[0]().get_presigned_urls(presigned_url_body)
+
+
+@router.post(
+    "/s3/requestDatasetUpload",
+    responses={
+        201: {"model": UploadRequestResp, "description": "Upload requested"},
+        422: {"model": HTTPValidationError, "description": "Validation Error"},
+        500: {"model": InternalError, "description": "Internal Server Error"},
+    },
+    tags=["s3upload"],
+    summary="Request a new upload",
+    response_model_by_alias=True,
+)
+async def request_dataset_upload(
+    upload_request_body: UploadRequestBody = Body(None, description=""),
+    token_BearerAuth: TokenModel = Security(
+        get_token_BearerAuth
+    ),
+) -> UploadRequestResp:
+    if not BaseS3uploadApi.subclasses:
+        raise HTTPException(status_code=500, detail="Not implemented")
+    return await BaseS3uploadApi.subclasses[0]().request_dataset_upload(upload_request_body)

backend/api/src/openapi_server/apis/s3upload_api_base.py

@@ -12,6 +12,8 @@
 from openapi_server.models.internal_error import InternalError
 from openapi_server.models.presigned_url_body import PresignedUrlBody
 from openapi_server.models.presigned_url_resp import PresignedUrlResp
+from openapi_server.models.upload_request_body import UploadRequestBody
+from openapi_server.models.upload_request_resp import UploadRequestResp
 from openapi_server.security_api import get_token_BearerAuth
 
 class BaseS3uploadApi:
@@ -46,3 +48,10 @@ async def get_presigned_urls(
         presigned_url_body: PresignedUrlBody,
     ) -> PresignedUrlResp:
         ...
+
+
+    async def request_dataset_upload(
+        self,
+        upload_request_body: UploadRequestBody,
+    ) -> UploadRequestResp:
+        ...