Fix timeout when waiting for flow (#299)

Author: phwissmann

backend/api/.openapi-generator/FILES

@@ -9,6 +9,8 @@ src/openapi_server/apis/service_token_api.py
 src/openapi_server/apis/service_token_api_base.py
 src/openapi_server/impl/__init__.py
 src/openapi_server/models/__init__.py
+src/openapi_server/models/abort_dataset_upload_body.py
+src/openapi_server/models/abort_dataset_upload_resp.py
 src/openapi_server/models/abort_upload_body.py
 src/openapi_server/models/abort_upload_resp.py
 src/openapi_server/models/complete_part.py

backend/api/.vscode/launch.json

@@ -10,13 +10,13 @@
       "request": "launch",
       "env": {
         "PREFECT_API_URL": "https://scopem-openem2.ethz.ch/archiver/prefect/api",
-        "PREFECT_LOGGING_LEVEL": "DEBUG",
+        "PREFECT_LOGGING_LEVEL": "INFO",
         "MINIO_ENDPOINT": "scopem-openemdata.ethz.ch:9090",
         "SECRETS_DIR": "${workspaceFolder}/../../.secrets",
         "UVICORN_ROOT_PATH": "",
         "UVICORN_RELOAD": "true",
         "UVICORN_LOG_LEVEL": "debug",
-        "UVICORN_PORT": "8888",
+        "UVICORN_PORT": "8800",
         "IDP_CLIENT_SECRET": "ASFD",
         "IDP_PASSWORD": "ASFD",
         "JOB_ENDPOINT_PASSWORD": "pw",

backend/api/Pipfile

@@ -0,0 +1,11 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+
+[dev-packages]
+
+[requires]
+python_version = "3.13"

backend/api/openapi.yaml

@@ -294,6 +294,37 @@ paths:
       summary: Finalize Dataset Upload
       tags:
         - s3upload
+  /s3/abortDatasetUpload:
+    post:
+      operationId: AbortDatasetUpload
+      requestBody:
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/AbortDatasetUploadBody"
+        required: true
+      responses:
+        "201":
+          description: Finalize dataset upload requested
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/AbortDatasetUploadResp"
+        "422":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/HTTPValidationError"
+          description: Validation Error
+        "500":
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/InternalError"
+          description: Internal Server Error
+      summary: Abort Dataset Upload
+      tags:
+        - s3upload
 components:
   securitySchemes:
     BasicAuth:
@@ -349,6 +380,9 @@ components:
         upload_id: upload_id
         object_name: object_name
       properties:
+        dataset_id:
+          title: dataset_id
+          type: string
         upload_id:
           title: upload_id
           type: string
@@ -358,6 +392,7 @@ components:
       required:
         - object_name
         - upload_id
+        - dataset_id
       title: AbortUploadBody
     AbortUploadResp:
       example:
@@ -412,6 +447,9 @@ components:
         object_name: object_name
         checksum_sha256: checksum_sha256
       properties:
+        dataset_id:
+          title: dataset_id
+          type: string
         object_name:
           title: object_name
           type: string
@@ -427,6 +465,7 @@ components:
           title: checksum_sha256
           type: string
       required:
+        - dataset_id
         - checksum_sha256
         - object_name
         - parts
@@ -474,16 +513,41 @@ components:
         - uuid
         - name
       title: CreateJobResp
+    AbortDatasetUploadBody:
+      example:
+        dataset_id: c2c6bc13-2c14-43d7-91a9-46cd556b258d
+      properties:
+        dataset_id:
+          title: dataset_id
+          type: string
+      required:
+        - dataset_id
+      title: AbortDatasetUploadBody
+    AbortDatasetUploadResp:
+      example:
+        dataset_id: dataset_id
+        message: "Upload finalized"
+      properties:
+        dataset_id:
+          title: dataset_id
+          type: string
+        message:
+          title: message
+          type: string
+      required:
+        - dataset_id
+        - message
+      title: AbortDatasetUploadResp
     FinalizeDatasetUploadBody:
       example:
-        dataset_pid: c2c6bc13-2c14-43d7-91a9-46cd556b258d
+        dataset_id: c2c6bc13-2c14-43d7-91a9-46cd556b258d
         create_archiving_job: true
         owner_group: Group
         owner_user: User
         contact_email: user@mail.com
       properties:
-        dataset_pid:
-          title: dataset_pid
+        dataset_id:
+          title: dataset_id
           type: string
         create_archiving_job:
           title: create_archiving_job
@@ -499,7 +563,7 @@ components:
           type: string
           format: email
       required:
-        - dataset_pid
+        - dataset_id
         - create_archiving_job
         - owner_group
         - owner_user
@@ -564,6 +628,9 @@ components:
         parts: 0
         object_name: object_name
       properties:
+        dataset_id:
+          title: DatasetPID
+          type: string
         object_name:
           title: object_name
           type: string
@@ -574,6 +641,7 @@ components:
       required:
         - object_name
         - parts
+        - dataset_id
       title: PresignedUrlBody
     PresignedUrlResp:
       example:
@@ -596,29 +664,29 @@ components:
       title: PresignedUrlResp
     UploadRequestBody:
       example:
-        DatasetPID:
+        dataset_id:
           - dataset/pid
-        TotalGigabytes: 1000
+        total_gigabytes: 1000
       properties:
-        DatasetPID:
+        dataset_id:
           title: DatasetPID
           type: string
-        TotalGigabytes:
+        total_gigabytes:
           title: TotalGigabytes
           type: integer
       required:
-        - DatasetPID
-        - TotalGigabytes
+        - dataset_id
+        - total_gigabytes
       title: UploadRequestBody
     UploadRequestResp:
       example:
-        Ok: true
+        ok: true
       properties:
-        Ok:
-          title: Ok
+        ok:
+          title: ok
           type: boolean
       required:
-        - Ok 
+        - ok 
       title: UploadRequestResp
     ValidationError:
       example:

backend/api/pyproject.toml

@@ -14,6 +14,7 @@ dependencies = [
     "gunicorn>=23.0.0",
     "httpx>=0.28.1",
     "importlib-metadata>=8.5.0",
+    "minio>=7.2.18",
     "prefect==3.6.1",
     "pyjwt>=2.10.1",
     "pytest>=8.3.5",

backend/api/src/openapi_server/apis/s3upload_api.py

@@ -23,6 +23,8 @@
 )
 
 from openapi_server.models.extra_models import TokenModel  # noqa: F401
+from openapi_server.models.abort_dataset_upload_body import AbortDatasetUploadBody
+from openapi_server.models.abort_dataset_upload_resp import AbortDatasetUploadResp
 from openapi_server.models.abort_upload_body import AbortUploadBody
 from openapi_server.models.abort_upload_resp import AbortUploadResp
 from openapi_server.models.complete_upload_body import CompleteUploadBody
@@ -44,6 +46,28 @@
     importlib.import_module(name)
 
 
+@router.post(
+    "/s3/abortDatasetUpload",
+    responses={
+        201: {"model": AbortDatasetUploadResp, "description": "Finalize dataset upload requested"},
+        422: {"model": HTTPValidationError, "description": "Validation Error"},
+        500: {"model": InternalError, "description": "Internal Server Error"},
+    },
+    tags=["s3upload"],
+    summary="Abort Dataset Upload",
+    response_model_by_alias=True,
+)
+async def abort_dataset_upload(
+    abort_dataset_upload_body: AbortDatasetUploadBody = Body(None, description=""),
+    token_BearerAuth: TokenModel = Security(
+        get_token_BearerAuth
+    ),
+) -> AbortDatasetUploadResp:
+    if not BaseS3uploadApi.subclasses:
+        raise HTTPException(status_code=500, detail="Not implemented")
+    return await BaseS3uploadApi.subclasses[0]().abort_dataset_upload(abort_dataset_upload_body)
+
+
 @router.post(
     "/s3/abortMultipartUpload",
     responses={

backend/api/src/openapi_server/apis/s3upload_api_base.py

@@ -2,6 +2,8 @@
 
 from typing import ClassVar, Dict, List, Tuple  # noqa: F401
 
+from openapi_server.models.abort_dataset_upload_body import AbortDatasetUploadBody
+from openapi_server.models.abort_dataset_upload_resp import AbortDatasetUploadResp
 from openapi_server.models.abort_upload_body import AbortUploadBody
 from openapi_server.models.abort_upload_resp import AbortUploadResp
 from openapi_server.models.complete_upload_body import CompleteUploadBody
@@ -22,6 +24,13 @@ class BaseS3uploadApi:
     def __init_subclass__(cls, **kwargs):
         super().__init_subclass__(**kwargs)
         BaseS3uploadApi.subclasses = BaseS3uploadApi.subclasses + (cls,)
+    async def abort_dataset_upload(
+        self,
+        abort_dataset_upload_body: AbortDatasetUploadBody,
+    ) -> AbortDatasetUploadResp:
+        ...
+
+
     async def abort_multipart_upload(
         self,
         abort_upload_body: AbortUploadBody,