Merge pull request #228 from MerginMaps/develop

Release 2024.3.0

Author: MarcelGeo

server/mergin/auth/models.py

@@ -10,7 +10,7 @@
 from sqlalchemy import or_
 
 from .. import db
-from ..sync.utils import get_user_agent, get_ip
+from ..sync.utils import get_user_agent, get_ip, get_device_id
 
 
 class User(db.Model):
@@ -219,19 +219,22 @@ class LoginHistory(db.Model):
     user_agent = db.Column(db.String, index=True)
     ip_address = db.Column(db.String, index=True)
     ip_geolocation_country = db.Column(db.String, index=True)
+    device_id = db.Column(db.String, index=True, nullable=True)
 
-    def __init__(self, user_id: int, ua: str, ip: str):
+    def __init__(self, user_id: int, ua: str, ip: str, device_id: Optional[str] = None):
         self.user_id = user_id
         self.user_agent = ua
         self.ip_address = ip
+        self.device_id = device_id
 
     @staticmethod
     def add_record(user_id: int, req: request) -> None:
         ua = get_user_agent(req)
         ip = get_ip(req)
+        device_id = get_device_id(req)
         # ignore login attempts coming from urllib - related to db sync tool
         if "DB-sync" in ua:
             return
-        lh = LoginHistory(user_id, ua, ip)
+        lh = LoginHistory(user_id, ua, ip, device_id)
         db.session.add(lh)
         db.session.commit()

server/mergin/sync/models.py

@@ -480,16 +480,20 @@ class ProjectVersion(db.Model):
         "Project",
         uselist=False,
     )
+    device_id = db.Column(db.String, index=True, nullable=True)
     __table_args__ = (db.UniqueConstraint("project_id", "name"),)
 
-    def __init__(self, project, name, author, changes, files, ip, user_agent=None):
+    def __init__(
+        self, project, name, author, changes, files, ip, user_agent=None, device_id=None
+    ):
         self.project_id = project.id
         self.name = name
         self.author = author
         self.changes = changes
         self.files = files
         self.user_agent = user_agent
         self.ip_address = ip
+        self.device_id = device_id
         self.project_size = sum(f["size"] for f in self.files) if self.files else 0
         # clean up changes metadata from chunks upload info
         for change in self.changes["updated"] + self.changes["added"]:

server/mergin/sync/public_api_controller.py

@@ -70,6 +70,7 @@
     get_path_from_files,
     get_project_path,
     clean_upload,
+    get_device_id,
 )
 from .errors import StorageLimitHit
 from ..utils import format_time_delta
@@ -155,6 +156,7 @@ def add_project(namespace):  # noqa: E501
 
     if request.is_json:
         ua = get_user_agent(request)
+        device_id = get_device_id(request)
         workspace = current_app.ws_handler.get_by_name(namespace)
         if not workspace:
             # return special message if former 'user workspace' was used
@@ -215,11 +217,12 @@ def add_project(namespace):  # noqa: E501
                 p.files,
                 get_ip(request),
                 user_agent,
+                device_id,
             )
         else:
             changes = {"added": [], "updated": [], "removed": []}
             version = ProjectVersion(
-                p, "v0", current_user.username, changes, [], ip, ua
+                p, "v0", current_user.username, changes, [], ip, ua, device_id
             )
             p.latest_version = "v0"
             try:
@@ -857,6 +860,7 @@ def project_push(namespace, project_name):
         flag_modified(project, "files")
         project.disk_usage = sum(file["size"] for file in project.files)
         user_agent = get_user_agent(request)
+        device_id = get_device_id(request)
         pv = ProjectVersion(
             project,
             next_version,
@@ -865,6 +869,7 @@ def project_push(namespace, project_name):
             project.files,
             get_ip(request),
             user_agent,
+            device_id,
         )
         project.latest_version = next_version
         db.session.add(pv)
@@ -1027,6 +1032,7 @@ def push_finish(transaction_id):
         flag_modified(project, "files")
         project.disk_usage = sum(file["size"] for file in project.files)
         user_agent = get_user_agent(request)
+        device_id = get_device_id(request)
         pv = ProjectVersion(
             project,
             next_version,
@@ -1035,6 +1041,7 @@ def push_finish(transaction_id):
             project.files,
             get_ip(request),
             user_agent,
+            device_id,
         )
         project.latest_version = next_version
         db.session.add(pv)
@@ -1156,10 +1163,19 @@ def clone_project(namespace, project_name):  # noqa: E501
 
     version = "v1" if p.files else "v0"
     changes = {"added": p.files, "updated": [], "removed": []}
+    # TODO: add user_agent and device_id handling to class
     user_agent = get_user_agent(request)
+    device_id = get_device_id(request)
     p.latest_version = version
     version = ProjectVersion(
-        p, version, current_user.username, changes, p.files, get_ip(request), user_agent
+        p,
+        version,
+        current_user.username,
+        changes,
+        p.files,
+        get_ip(request),
+        user_agent,
+        device_id,
     )
     db.session.add(p)
     db.session.add(pa)

server/mergin/sync/schemas.py

@@ -252,7 +252,7 @@ def get_diff_summary(self, obj):
 
     class Meta:
         model = ProjectVersion
-        exclude = ["id", "ip_address", "ip_geolocation_country", "project"]
+        exclude = ["id", "ip_address", "ip_geolocation_country", "project", "device_id"]
         load_instance = True
 
 
@@ -262,7 +262,7 @@ class FullVersionSchema(ma.SQLAlchemyAutoSchema):
 
     class Meta:
         model = ProjectVersion
-        exclude = ["id"]
+        exclude = ["id", "device_id"]
         load_instance = True
 
 

server/mergin/sync/utils.py

@@ -14,6 +14,8 @@
 from shapely import wkb
 from shapely.errors import WKBReadingError
 from gevent import sleep
+from flask import Request
+from typing import Optional
 
 
 def generate_checksum(file, chunk_size=4096):
@@ -377,3 +379,8 @@ def clean_upload(transaction_id):
     db.session.commit()
     move_to_tmp(upload_dir, transaction_id)
     return NoContent, 200
+
+
+def get_device_id(request: Request) -> Optional[str]:
+    """Get device uuid from http header X-Device-Id"""
+    return request.headers.get("X-Device-Id")

server/mergin/tests/__init__.py

@@ -4,6 +4,7 @@
 
 import os
 import tempfile
+import uuid
 
 # constants
 test_workspace_name = "mergin"
@@ -12,6 +13,10 @@
 test_project_dir = os.path.join(
     os.path.dirname(os.path.realpath(__file__)), "test_projects", test_project
 )
-json_headers = {"Content-Type": "application/json", "Accept": "application/json"}
+json_headers = {
+    "Content-Type": "application/json",
+    "Accept": "application/json",
+    "X-Device-Id": f"{uuid.uuid4()}",
+}
 DEFAULT_USER = ("mergin", "ilovemergin")  # username, password - is a super user
 TMP_DIR = tempfile.gettempdir()

server/mergin/tests/test_auth.py

@@ -35,6 +35,11 @@ def client(app):
     ({"login": "mergin", "password": "ilovemergin"}, json_headers, 200),
     ({"login": "mergin  ", "password": "ilovemergin"}, json_headers, 200),
     ({"login": "mergin@mergin.com", "password": "ilovemergin"}, json_headers, 200),
+    (
+        {"login": "mergin", "password": "ilovemergin"},
+        {**json_headers, "X-Device-Id": None},
+        200,
+    ),
     ({"login": "mergin", "password": "ilovemergi"}, json_headers, 401),
     ({"login": "mergin"}, json_headers, 401),
     ({"login": "mergin", "password": "ilovemergin"}, {}, 415),
@@ -57,6 +62,7 @@ def test_login(client, data, headers, expected):
             .first()
         )
         assert login_history
+        assert login_history.device_id == str(headers.get("X-Device-Id"))
 
 
 def test_logout(client):

server/mergin/tests/test_project_controller.py

@@ -400,7 +400,6 @@ def test_add_project(client, app, data, expected):
     )
     upload_chunks(upload_dir, upload.changes)
     resp = client.post("/v1/project/push/finish/{}".format(upload.id))
-    print(resp.json)
     assert resp.status_code == 200
 
     # add TEMPLATES user and make him creator of test_project (to become template)
@@ -434,6 +433,7 @@ def test_add_project(client, app, data, expected):
         )
         pv = project.get_latest_version()
         assert pv.user_agent is not None
+        assert pv.device_id == json_headers["X-Device-Id"]
         # check if there is no diffs in cloned files
         assert not any("diff" in file for file in project.files)
         assert not any("diff" in file for file in pv.files)
@@ -459,6 +459,7 @@ def test_versioning(client):
     pv = project.get_latest_version()
     assert pv.name == "v0"
     assert pv.project_size == 0
+    assert pv.device_id == json_headers["X-Device-Id"]
 
 
 def test_delete_project(client):
@@ -1367,7 +1368,7 @@ def test_push_finish(client):
     upload, upload_dir = create_transaction("mergin", changes)
     url = "/v1/project/push/finish/{}".format(upload.id)
 
-    resp = client.post(url)
+    resp = client.post(url, headers=json_headers)
     assert resp.status_code == 422
     assert "corrupted_files" in resp.json["detail"].keys()
     assert not os.path.exists(os.path.join(upload_dir, "files", "test.txt"))
@@ -1384,11 +1385,12 @@ def test_push_finish(client):
                 with open(os.path.join(upload_dir, "chunks", chunk), "wb") as out_file:
                     out_file.write(in_file.read(CHUNK_SIZE))
 
-    resp2 = client.post(url)
+    resp2 = client.post(url, headers={**json_headers, "User-Agent": "Werkzeug"})
     assert resp2.status_code == 200
     assert not os.path.exists(upload_dir)
     version = upload.project.get_latest_version()
     assert version.user_agent
+    assert version.device_id == json_headers["X-Device-Id"]
     # chunks is only temporal information, it should not be in db
     assert "chunks" not in version.changes["added"][0].keys()
     assert "chunks" not in version.changes["updated"][0].keys()
@@ -1699,6 +1701,7 @@ def test_clone_project(client, data, username, expected):
         assert not any("diff" in file for file in pv.files)
         assert pv.changes.get("removed") == []
         assert pv.changes.get("updated") == []
+        assert pv.device_id == json_headers["X-Device-Id"]
         assert "diff" not in pv.changes.get("added")
         # cleanup
         shutil.rmtree(project.storage.project_dir)

server/migrations/community/0e3fc92aeaaa_add_device_id.py

@@ -0,0 +1,38 @@
+"""Add device id
+
+Revision ID: 0e3fc92aeaaa
+Revises: a5d4defded55
+Create Date: 2024-05-03 11:24:46.040646
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "0e3fc92aeaaa"
+down_revision = "a5d4defded55"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("login_history", sa.Column("device_id", sa.String(), nullable=True))
+    op.create_index(
+        op.f("ix_login_history_device_id"), "login_history", ["device_id"], unique=False
+    )
+    op.add_column("project_version", sa.Column("device_id", sa.String(), nullable=True))
+    op.create_index(
+        op.f("ix_project_version_device_id"),
+        "project_version",
+        ["device_id"],
+        unique=False,
+    )
+
+
+def downgrade():
+    op.drop_index(op.f("ix_project_version_device_id"), table_name="project_version")
+    op.drop_column("project_version", "device_id")
+    op.drop_index(op.f("ix_login_history_device_id"), table_name="login_history")
+    op.drop_column("login_history", "device_id")