Merge remote-tracking branch 'origin/develop' into feat/aasql-query-endpoints

Author: zrgt

.github/workflows/release.yml

@@ -4,6 +4,9 @@ on:
   release:
     types: [published]
 
+env:
+  TARGET_PLATFORM: "linux/amd64,linux/arm/v7,linux/arm64"
+
 jobs:
   sdk-publish:
     # This job publishes the package to PyPI
@@ -59,7 +62,54 @@ jobs:
         with:
           password: ${{ secrets.PYPI_ORG_TOKEN }}
 
-  server-publish:
+  server-repository-publish:
+    # This job publishes the server docker image to DockerHub
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./server
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Extract Docker image metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: eclipsebasyx/basyx-python-repository
+          # This fetches the latest git tag and adds an additional "latest" to it, so e.g. `2.1.0,latest`
+          tags: |
+            type=semver,pattern={{version}}
+            type=raw,value=latest
+          labels: |
+            org.opencontainers.image.title=BaSyx Python Repository Service
+            org.opencontainers.image.description=Eclipse BaSyx Python SDK - Repository HTTP Server
+            org.opencontainers.image.source=https://github.com/eclipse-basyx/basyx-python-sdk/tree/main/server
+            org.opencontainers.image.licenses=MIT
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Build and Push Repository Docker Image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./server/docker/repository/Dockerfile
+          platforms: ${{ env.TARGET_PLATFORM }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  server-discovery-publish:
     # This job publishes the server docker image to DockerHub
     runs-on: ubuntu-latest
     defaults:
@@ -73,14 +123,14 @@ jobs:
         id: meta
         uses: docker/metadata-action@v5
         with:
-          images: eclipsebasyx/basyx-python-server
+          images: eclipsebasyx/basyx-python-discovery
           # This fetches the latest git tag and adds an additional "latest" to it, so e.g. `2.1.0,latest`
           tags: |
             type=semver,pattern={{version}}
             type=raw,value=latest
           labels: |
-            org.opencontainers.image.title=BaSyx Python Server
-            org.opencontainers.image.description=Eclipse BaSyx Python SDK - HTTP Server
+            org.opencontainers.image.title=BaSyx Python Discovery Service
+            org.opencontainers.image.description=Eclipse BaSyx Python SDK - Discovery HTTP Server
             org.opencontainers.image.source=https://github.com/eclipse-basyx/basyx-python-sdk/tree/main/server
             org.opencontainers.image.licenses=MIT
 
@@ -90,11 +140,65 @@ jobs:
           username: ${{ secrets.DOCKER_HUB_USER }}
           password: ${{ secrets.DOCKER_HUB_TOKEN }}
 
-      - name: Build and Push Docker Image
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Build and Push Repository Docker Image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./server/docker/discovery/Dockerfile
+          platforms: ${{ env.TARGET_PLATFORM }}
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  server-registry-publish:
+    # This job publishes the server docker image to DockerHub
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./server
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Extract Docker image metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: eclipsebasyx/basyx-python-registry
+          # This fetches the latest git tag and adds an additional "latest" to it, so e.g. `2.1.0,latest`
+          tags: |
+            type=semver,pattern={{version}}
+            type=raw,value=latest
+          labels: |
+            org.opencontainers.image.title=BaSyx Python Registry Service
+            org.opencontainers.image.description=Eclipse BaSyx Python SDK - Registry HTTP Server
+            org.opencontainers.image.source=https://github.com/eclipse-basyx/basyx-python-sdk/tree/main/server
+            org.opencontainers.image.licenses=MIT
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKER_HUB_USER }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v4
+
+      - name: Build and Push Repository Docker Image
         uses: docker/build-push-action@v6
         with:
           context: .
-          file: ./server/Dockerfile  # Todo: Update paths
+          file: ./server/docker/registry/Dockerfile
+          platforms: ${{ env.TARGET_PLATFORM }}
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

sdk/basyx/aas/backend/couchdb.py

@@ -160,22 +160,17 @@ def get_identifiable_by_couchdb_id(self, couchdb_id: str) -> model.Identifiable:
                 raise KeyError("No Identifiable with couchdb-id {} found in CouchDB database".format(couchdb_id)) from e
             raise
 
-        # Add CouchDB metadata (for later commits) to object
         obj = data['data']
         if not isinstance(obj, model.Identifiable):
             raise CouchDBResponseError("The CouchDB document with id {} does not contain an identifiable AAS object."
                                        .format(couchdb_id))
         set_couchdb_revision("{}/{}/{}".format(self.url, self.database_name, urllib.parse.quote(couchdb_id, safe='')),
                              data["_rev"])
 
-        # If we still have a local replication of that object (since it is referenced from anywhere else), update that
-        # replication and return it.
         with self._object_cache_lock:
             if obj.id in self._object_cache:
-                old_obj = self._object_cache[obj.id]
-                old_obj.update_from(obj)
-                return old_obj
-        self._object_cache[obj.id] = obj
+                return self._object_cache[obj.id]
+            self._object_cache[obj.id] = obj
         return obj
 
     def get_item(self, identifier: model.Identifier) -> model.Identifiable:
@@ -186,6 +181,9 @@ def get_item(self, identifier: model.Identifier) -> model.Identifiable:
         :raises CouchDBError: If error occur during the request to the CouchDB server
                               (see ``_do_request()`` for details)
         """
+        with self._object_cache_lock:
+            if identifier in self._object_cache:
+                return self._object_cache[identifier]
         try:
             return self.get_identifiable_by_couchdb_id(self._transform_id(identifier, False))
         except KeyError as e:
@@ -220,6 +218,37 @@ def add(self, x: model.Identifiable) -> None:
         with self._object_cache_lock:
             self._object_cache[x.id] = x
 
+    def commit(self, x: model.Identifiable) -> None:
+        """
+        Write the current in-memory state of a stored object back to the CouchDB.
+
+        :param x: The object to persist
+        :raises KeyError: If the object is not present in the store or no revision is known
+        :raises CouchDBConflictError: If the object was modified in the database since it was last fetched
+        :raises CouchDBError: If error occur during the request to the CouchDB server
+                              (see ``_do_request()`` for details)
+        """
+        doc_url = "{}/{}/{}".format(self.url, self.database_name, self._transform_id(x.id))
+        rev = get_couchdb_revision(doc_url)
+        if rev is None:
+            raise KeyError("No revision found for object with id {} — not fetched from this store".format(x.id))
+        data = json.dumps({'data': x}, cls=json_serialization.AASToJsonEncoder)
+        try:
+            response = self._do_request(
+                "{}?rev={}".format(doc_url, rev),
+                'PUT',
+                {'Content-type': 'application/json'},
+                data.encode('utf-8'))
+            set_couchdb_revision(doc_url, response["rev"])
+        except CouchDBServerError as e:
+            if e.code == 404:
+                raise KeyError("No AAS object with id {} exists in CouchDB database".format(x.id)) from e
+            elif e.code == 409:
+                raise CouchDBConflictError(
+                    "Object with id {} has been modified in the database since it was last fetched."
+                    .format(x.id)) from e
+            raise
+
     def discard(self, x: model.Identifiable, safe_delete=False) -> None:
         """
         Delete an :class:`~basyx.aas.model.base.Identifiable` AAS object from the CouchDB database

sdk/basyx/aas/backend/local_file.py

@@ -16,6 +16,7 @@
 import json
 import os
 import hashlib
+import tempfile
 import threading
 import warnings
 import weakref
@@ -31,6 +32,13 @@ class LocalFileIdentifiableStore(model.AbstractObjectStore[model.Identifier, mod
     """
     An ObjectStore implementation for :class:`~basyx.aas.model.base.Identifiable` BaSyx Python SDK objects backed
     by a local file based local backend
+
+    .. warning::
+        This backend is intended for development and testing only. It provides no
+        concurrency control across processes: concurrent writes to the same object
+        (e.g. under a multi-worker WSGI server) will silently overwrite each other,
+        with the last writer winning and no error raised. Use a dedicated database
+        backend for any production deployment.
     """
     def __init__(self, directory_path: str):
         """
@@ -68,21 +76,16 @@ def get_identifiable_by_hash(self, hash_: str) -> model.Identifiable:
 
         :raises KeyError: If the respective file could not be found
         """
-        # Try to get the correct file
         try:
             with open("{}/{}.json".format(self.directory_path, hash_), "r") as file:
                 data = json.load(file, cls=json_deserialization.AASFromJsonDecoder)
                 obj = data["data"]
         except FileNotFoundError as e:
             raise KeyError("No Identifiable with hash {} found in local file database".format(hash_)) from e
-        # If we still have a local replication of that object (since it is referenced from anywhere else), update that
-        # replication and return it.
         with self._object_cache_lock:
             if obj.id in self._object_cache:
-                old_obj = self._object_cache[obj.id]
-                old_obj.update_from(obj)
-                return old_obj
-        self._object_cache[obj.id] = obj
+                return self._object_cache[obj.id]
+            self._object_cache[obj.id] = obj
         return obj
 
     def get_item(self, identifier: model.Identifier) -> model.Identifiable:
@@ -91,11 +94,33 @@ def get_item(self, identifier: model.Identifier) -> model.Identifiable:
 
         :raises KeyError: If the respective file could not be found
         """
+        with self._object_cache_lock:
+            if identifier in self._object_cache:
+                return self._object_cache[identifier]
         try:
             return self.get_identifiable_by_hash(self._transform_id(identifier))
         except KeyError as e:
             raise KeyError("No Identifiable with id {} found in local file database".format(identifier)) from e
 
+    def _write_atomic(self, x: model.Identifiable) -> None:
+        """
+        Serialize x to a temp file in the store directory, then atomically replace the final file.
+
+        Using os.replace() (rename(2) on POSIX) ensures readers always see a complete file — never
+        a partially-written one from a crash or concurrent access mid-write.
+        """
+        final_path = "{}/{}.json".format(self.directory_path, self._transform_id(x.id))
+        tmp_fd, tmp_path = tempfile.mkstemp(dir=self.directory_path, suffix=".tmp")
+        try:
+            with os.fdopen(tmp_fd, "w") as tmp_file:
+                json.dump({"data": x}, tmp_file, cls=json_serialization.AASToJsonEncoder, indent=4)
+            os.replace(tmp_path, final_path)
+        # Catch all `Exception`s, as well as `KeyboardInterrupt` and `SystemExit` too, so the temp
+        # file is never left behind even if the process is being torn down:
+        except BaseException:
+            os.unlink(tmp_path)
+            raise
+
     def add(self, x: model.Identifiable) -> None:
         """
         Add an object to the store
@@ -105,10 +130,20 @@ def add(self, x: model.Identifiable) -> None:
         logger.debug("Adding object %s to Local File Store ...", repr(x))
         if os.path.exists("{}/{}.json".format(self.directory_path, self._transform_id(x.id))):
             raise KeyError("Identifiable with id {} already exists in local file database".format(x.id))
-        with open("{}/{}.json".format(self.directory_path, self._transform_id(x.id)), "w") as file:
-            json.dump({"data": x}, file, cls=json_serialization.AASToJsonEncoder, indent=4)
-            with self._object_cache_lock:
-                self._object_cache[x.id] = x
+        self._write_atomic(x)
+        with self._object_cache_lock:
+            self._object_cache[x.id] = x
+
+    def commit(self, x: model.Identifiable) -> None:
+        """
+        Write the current in-memory state of a stored object back to its file.
+
+        :param x: The object to persist
+        :raises KeyError: If the object is not present in the store
+        """
+        if not os.path.exists("{}/{}.json".format(self.directory_path, self._transform_id(x.id))):
+            raise KeyError("No AAS object with id {} exists in local file database".format(x.id))
+        self._write_atomic(x)
 
     def discard(self, x: model.Identifiable) -> None:
         """

sdk/basyx/aas/model/provider.py

@@ -59,6 +59,18 @@ class AbstractObjectStore(AbstractObjectProvider[_KEY, _VALUE], MutableSet[_VALU
     def __init__(self):
         pass
 
+    def commit(self, x: _VALUE) -> None:
+        """
+        Persist an in-memory mutation of a stored object back to the underlying storage.
+
+        Persistent backends (e.g. file-based or database-backed stores) must override this to
+        write the updated object back to storage. In-memory stores should override this with an
+        explicit no-op to make the intent clear.
+
+        :param x: The object whose current in-memory state should be persisted
+        """
+        raise NotImplementedError()
+
     def update(self, other: Iterable[_VALUE]) -> None:
         for x in other:
             self.add(x)
@@ -146,6 +158,9 @@ def add(self, x: _IDENTIFIABLE) -> None:
                            .format(x.id))
         self._backend[x.id] = x
 
+    def commit(self, x: _IDENTIFIABLE) -> None:
+        pass
+
     def discard(self, x: _IDENTIFIABLE) -> None:
         if self._backend.get(x.id) is x:
             del self._backend[x.id]
@@ -223,6 +238,9 @@ def add(self, x: _IDENTIFIABLE) -> None:
         else:
             raise KeyError(f"Identifiable object with same id {x.id} is already stored in this store")
 
+    def commit(self, x: _IDENTIFIABLE) -> None:
+        pass
+
     def discard(self, x: _IDENTIFIABLE) -> None:
         self._backend.discard(x)