Merge pull request #567 from MerginMaps/api-update-batch-endpoint

Update batch API endpoint, openapi documentation and add batch test

Author: MarcelGeo

LICENSES/CLA-signed-list.md

@@ -21,3 +21,4 @@ C/ My company has custom contribution contract with Lutra Consulting Ltd. or I a
 * luxusko, 25th August 2023
 * jozef-budac, 30th January 2024
 * fernandinand, 13th March 2025
+* xkello, 26th January 2026

server/mergin/sync/config.py

@@ -78,3 +78,5 @@ class Configuration(object):
     EXCLUDED_CLONE_FILENAMES = config(
         "EXCLUDED_CLONE_FILENAMES", default="qgis_cfg.xml", cast=Csv()
     )
+    # max batch size for fetch projects in batch endpoint
+    MAX_BATCH_SIZE = config("MAX_BATCH_SIZE", default=100, cast=int)

server/mergin/sync/errors.py

@@ -95,3 +95,8 @@ def to_dict(self) -> Dict:
 class BigChunkError(ResponseError):
     code = "BigChunkError"
     detail = f"Chunk size exceeds maximum allowed size {MAX_CHUNK_SIZE} MB"
+
+
+class BatchLimitError(ResponseError):
+    code = "BatchLimitExceeded"
+    detail = f"Batch size exceeds maximum allowed size {Configuration.MAX_BATCH_SIZE}"

server/mergin/sync/permissions.py

@@ -248,6 +248,29 @@ def require_project_by_uuid(
     return project
 
 
+def check_project_permissions(
+    project: Project, permission: ProjectPermissions
+) -> int | None:
+    """Check project permissions and return appropriate HTTP error code if check fails.
+    :param project: project
+    :type project: Project
+    :param permission: permission to check
+    :type permission: ProjectPermissions
+    :return: HTTP error code if permission check fails, None otherwise
+    :rtype: int | None
+    """
+
+    if not permission.check(project, current_user):
+        # logged in - NO, have acccess - NONE, public project - NO
+        if current_user.is_anonymous:
+            # we don't want to tell anonymous user if a private project exists
+            return 404
+        # logged in - YES, have access - NO, public project - NO
+        return 403
+
+    return None
+
+
 def get_upload(transaction_id):
     upload = Upload.query.get_or_404(transaction_id)
     # upload to 'removed' projects is forbidden

server/mergin/sync/project_handler.py

@@ -28,3 +28,13 @@ def get_email_receivers(self, project: Project) -> List[User]:
             )
             .all()
         )
+
+    @staticmethod
+    def get_projects_by_uuids(uuids: List[str]) -> [Project]:
+        """Gets non-deleted projects"""
+        return (
+            Project.query.filter(Project.id.in_(uuids))
+            .filter(Project.storage_params.isnot(None))
+            .filter(Project.removed_at.is_(None))
+            .all()
+        )

server/mergin/sync/public_api_v2.yaml

@@ -367,6 +367,53 @@ paths:
                 $ref: "#/components/schemas/ProjectLocked"
 
       x-openapi-router-controller: mergin.sync.public_api_v2_controller
+
+  /projects/batch:
+    post:
+      tags:
+        - project
+      summary: Get multiple projects by UUIDs
+      operationId: list_batch_projects
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              type: object
+              required: [ids]
+              properties:
+                ids:
+                  type: array
+                  description: List of project UUIDs to fetch
+                  items:
+                    $ref: "#/components/schemas/ProjectId"
+      responses:
+        "200":
+          description: Projects returned as a list of simple project objects and/or error objects.
+          content:
+            application/json:
+              schema:
+                type: object
+                required: [projects]
+                properties:
+                  projects:
+                    type: array
+                    items:
+                      oneOf:
+                        - $ref: "#/components/schemas/Project"
+                        - $ref: "#/components/schemas/BatchItemError"
+        "400": 
+          description: Batch limit exceeded or one or more UUIDs were invalid
+          content:
+            application/problem+json:
+              schema:
+                $ref: "#/components/schemas/CustomError"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "404":
+          $ref: "#/components/responses/NotFound"
+      x-openapi-router-controller: mergin.sync.public_api_v2_controller
+
   /workspaces/{workspace_id}/projects:
     get:
       tags:
@@ -457,9 +504,7 @@ components:
       description: UUID of the project
       required: true
       schema:
-        type: string
-        format: uuid
-        pattern: \b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b
+        $ref: "#/components/schemas/ProjectId"
     WorkspaceId:
       name: workspace_id
       in: path
@@ -468,6 +513,10 @@ components:
       schema:
         type: integer
   schemas:
+    ProjectId:
+      type: string
+      format: uuid
+      pattern: \b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b
     # Errors
     CustomError:
       type: object
@@ -547,6 +596,17 @@ components:
       example:
         code: UploadError
         detail: "Project version could not be created (UploadError)"
+    BatchItemError:
+      type: object
+      properties:
+        id:
+          $ref: "#/components/schemas/ProjectId"
+        error:
+          type: integer
+          example: 404
+      required:
+        - id
+        - error
     # Data
     ProjectRole:
       type: string

server/mergin/sync/public_api_v2_controller.py

@@ -17,12 +17,13 @@
 
 from mergin.sync.tasks import remove_transaction_chunks
 
-from .schemas_v2 import ProjectSchema as ProjectSchemaV2
+from .schemas_v2 import BatchErrorSchema, ProjectSchema as ProjectSchemaV2
 from ..app import db
 from ..auth import auth_required
 from ..auth.models import User
 from .errors import (
     AnotherUploadRunning,
+    BatchLimitError,
     BigChunkError,
     DataSyncError,
     ProjectLocked,
@@ -41,14 +42,25 @@
     project_version_created,
     push_finished,
 )
-from .permissions import ProjectPermissions, require_project_by_uuid, projects_query
+from .permissions import (
+    ProjectPermissions,
+    check_project_permissions,
+    require_project_by_uuid,
+    projects_query,
+)
 from .public_api_controller import catch_sync_failure
 from .schemas import (
     ProjectMemberSchema,
     UploadChunkSchema,
 )
 from .storages.disk import move_to_tmp, save_to_file
-from .utils import get_device_id, get_ip, get_user_agent, get_chunk_location
+from .utils import (
+    get_device_id,
+    get_ip,
+    get_user_agent,
+    get_chunk_location,
+    is_valid_uuid,
+)
 from .workspace import WorkspaceRole
 from ..utils import parse_order_params, get_schema_fields_map
 
@@ -457,3 +469,40 @@ def list_workspace_projects(workspace_id, page, per_page, order_params=None, q=N
 
     data = ProjectSchemaV2(many=True).dump(result)
     return jsonify(projects=data, count=total, page=page, per_page=per_page), 200
+
+
+def list_batch_projects(body):
+    """List projects by given list of UUIDs. Limit to 100 projects per request.
+
+    :param ids: List of project UUIDs
+    :type ids: List[str]
+    :rtype: Dict[str: List[Project]]
+    """
+    ids = list(dict.fromkeys(body.get("ids", [])))
+    # remove duplicates while preserving the order
+    max_batch = current_app.config.get("MAX_BATCH_SIZE", 100)
+    if len(ids) > max_batch:
+        return BatchLimitError().response(400)
+
+    projects = current_app.project_handler.get_projects_by_uuids(ids)
+    by_id = {str(project.id): project for project in projects}
+
+    filtered_projects = []
+    for uuid in ids:
+        project = by_id.get(uuid)
+
+        if not project:
+            filtered_projects.append(
+                BatchErrorSchema().dump({"id": uuid, "error": 404})
+            )
+            continue
+
+        err = check_project_permissions(project, ProjectPermissions.Read)
+        if err is not None:
+            filtered_projects.append(
+                BatchErrorSchema().dump({"id": uuid, "error": err})
+            )
+        else:
+            filtered_projects.append(ProjectSchemaV2().dump(project))
+
+    return jsonify(projects=filtered_projects), 200

server/mergin/sync/schemas_v2.py

@@ -46,3 +46,8 @@ class Meta:
             "workspace",
             "role",
         )
+
+
+class BatchErrorSchema(ma.Schema):
+    id = fields.UUID(required=True)
+    error = fields.Integer(required=True)

server/mergin/tests/test_permissions.py

@@ -2,15 +2,29 @@
 #
 # SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
 
+import pytest
+from unittest.mock import patch
 import datetime
 from flask_login import AnonymousUserMixin
 
-from ..sync.permissions import require_project, ProjectPermissions
-from ..sync.models import ProjectRole
+from mergin.tests import DEFAULT_USER
+
+from ..sync.permissions import (
+    require_project,
+    check_project_permissions,
+    ProjectPermissions,
+)
+from ..sync.models import Project, ProjectRole
 from ..auth.models import User
 from ..app import db
 from ..config import Configuration
-from .utils import add_user, create_project, create_workspace
+from .utils import (
+    add_user,
+    create_project,
+    create_workspace,
+    login,
+    logout,
+)
 
 
 def test_project_permissions(client):
@@ -116,3 +130,47 @@ def test_project_permissions(client):
     assert ProjectPermissions.All.check(project, user)
     assert ProjectPermissions.Edit.check(project, user)
     assert ProjectPermissions.get_user_project_role(project, user) == ProjectRole.OWNER
+
+
+def test_check_project_permissions(client):
+    """Test check_project_permissions with various permission scenarios."""
+    admin = User.query.filter_by(username=DEFAULT_USER[0]).first()
+    test_workspace = create_workspace()
+
+    private_proj = create_project("batch_private", test_workspace, admin)
+    public_proj = create_project("batch_public", test_workspace, admin)
+
+    p = Project.query.get(public_proj.id)
+    p.public = True
+    db.session.commit()
+
+    priv_proj = Project.query.get(private_proj.id)
+    pub_proj = Project.query.get(public_proj.id)
+
+    # First user with access to both projects
+    login(client, DEFAULT_USER[0], DEFAULT_USER[1])
+
+    with client:
+        client.get("/")
+        assert check_project_permissions(priv_proj, ProjectPermissions.Read) is None
+        assert check_project_permissions(pub_proj, ProjectPermissions.Read) is None
+
+    # Second user with no access to private project (ensure global perms disabled)
+    with patch.object(Configuration, "GLOBAL_READ", False), patch.object(
+        Configuration, "GLOBAL_WRITE", False
+    ), patch.object(Configuration, "GLOBAL_ADMIN", False):
+        user2 = add_user("user_batch", "password")
+        login(client, user2.username, "password")
+
+        with client:
+            client.get("/")
+            assert check_project_permissions(pub_proj, ProjectPermissions.Read) is None
+            assert check_project_permissions(priv_proj, ProjectPermissions.Read) == 403
+
+    # Logged-out (anonymous) user
+    logout(client)
+
+    with client:
+        client.get("/")
+        assert check_project_permissions(priv_proj, ProjectPermissions.Read) == 404
+        assert check_project_permissions(pub_proj, ProjectPermissions.Read) is None

server/mergin/tests/test_project_handler.py

@@ -1,3 +1,6 @@
+from datetime import datetime
+
+from . import DEFAULT_USER
 from ..sync.models import Project, ProjectRole
 from .utils import add_user, create_project, create_workspace
 from ..sync.project_handler import ProjectHandler
@@ -51,3 +54,26 @@ def test_email_receivers(client):
     db.session.commit()
     receivers = project_handler.get_email_receivers(project)
     assert len(receivers) == 0
+
+
+def test_get_projects_by_uuids(client):
+    """Test getting projects with their UUIDs"""
+    project_handler = ProjectHandler()
+    test_workspace = create_workspace()
+    user = User.query.filter_by(username=DEFAULT_USER[0]).first()
+    p_found = create_project("p_found", test_workspace, user)
+    p_removed = create_project("p_removed", test_workspace, user)
+    p_removed.removed_at = datetime.now()
+    db.session.commit()
+    p_other = create_project("p_other", test_workspace, user)
+    ids = [
+        str(p_found.id),
+        str(p_removed.id),
+    ]
+
+    projects = project_handler.get_projects_by_uuids(ids)
+    returned_ids = [str(p.id) for p in projects]
+    assert str(p_found.id) in returned_ids
+    assert str(p_removed.id) not in returned_ids
+    assert str(p_other.id) not in returned_ids
+    assert len(projects) == 1