Fix Discovery persistence and lookup handling (#543)

Previously, the Discovery service tried to persist its in-memory data structure
directly. This could corrupt the storage file discovery_store.json and prevented reliable lookup after restart.

This change persists only the AAS-to-asset-ID mapping in a JSON-safe format,
rebuilds the reverse lookup index on load, creates the storage file when
persistent mode is enabled, and fixes lookup responses to pass paging metadata
correctly.

---------

Co-authored-by: s-heppner <iat@s-heppner.com>

Author: Ornella33

.gitignore

@@ -36,3 +36,4 @@ server/example_configurations/repository_standalone/input/
 server/example_configurations/repository_standalone/storage/
 server/example_configurations/registry_standalone/input/
 server/example_configurations/registry_standalone/storage/
+server/example_configurations/discovery_standalone/storage/

server/app/interfaces/discovery.py

@@ -5,6 +5,7 @@
 """
 
 import json
+import os
 from typing import Dict, List, Set, Type
 
 import werkzeug.exceptions
@@ -65,28 +66,58 @@ def _delete_aas_id_from_specific_asset_ids(self, asset_id: model.SpecificAssetId
     @classmethod
     def from_file(cls, filename: str) -> "DiscoveryStore":
         """
-        Load the state of the `DiscoveryStore` from a local file.
-        Safely handles files that are missing expected keys.
+        Load a persisted discovery store from JSON.
 
+        The file stores the AAS-to-asset-id mapping as the source of truth.
+        While loading, the reverse asset-id-to-AAS index is rebuilt in memory so
+        lookup by asset ID works without persisting duplicate state.
         """
         with open(filename, "r") as file:
             data = json.load(file, cls=jsonization.ServerAASFromJsonDecoder)
-            discovery_store = DiscoveryStore()
-            discovery_store.aas_id_to_asset_ids = data.get("aas_id_to_asset_ids", {})
-            discovery_store.asset_id_to_aas_ids = data.get("asset_id_to_aas_ids", {})
-            return discovery_store
+
+        discovery_store = DiscoveryStore()
+
+        for aas_id, asset_ids in data.get("aas_id_to_asset_ids", {}).items():
+            parsed_asset_ids = set()
+
+            for asset_id in asset_ids:
+                if isinstance(asset_id, model.SpecificAssetId):
+                    parsed_asset_id = asset_id
+                else:
+                    parsed_asset_id = model.SpecificAssetId(
+                        name=asset_id["name"],
+                        value=asset_id["value"],
+                    )
+
+                parsed_asset_ids.add(parsed_asset_id)
+                discovery_store._add_aas_id_to_specific_asset_id(parsed_asset_id, aas_id)
+
+            discovery_store.aas_id_to_asset_ids[aas_id] = parsed_asset_ids
+
+        return discovery_store
 
     def to_file(self, filename: str) -> None:
         """
-        Write the current state of the `DiscoveryStore` to a local JSON file for persistence.
+        Persist the discovery store as JSON.
+
+        Only the AAS-to-asset-id mapping is written because the reverse lookup
+        index can be rebuilt when the store is loaded. The data is written to a
+        temporary file first and then atomically moved into place to avoid
+        corrupting the existing store if serialization fails.
         """
-        with open(filename, "w") as file:
-            data = {
-                "aas_id_to_asset_ids": self.aas_id_to_asset_ids,
-                "asset_id_to_aas_ids": self.asset_id_to_aas_ids,
+        data = {
+            "aas_id_to_asset_ids": {
+                aas_id: list(asset_ids)
+                for aas_id, asset_ids in self.aas_id_to_asset_ids.items()
             }
+        }
+
+        temp_filename = f"{filename}.tmp"
+        with open(temp_filename, "w") as file:
             json.dump(data, file, cls=jsonization.ServerAASToJsonEncoder, indent=4)
 
+        os.replace(temp_filename, filename)
+
 
 class DiscoveryAPI(BaseWSGIApp):
     def __init__(self, persistent_store: DiscoveryStore, base_path: str = "/api/v3.1"):
@@ -160,8 +191,8 @@ def get_all_aas_ids_by_asset_link(
             aas_keys = self.persistent_store.search_aas_ids_by_asset_link(asset_link)
             matching_aas_keys.update(aas_keys)
 
-        paginated_slice, cursor = self._get_slice(request, list(matching_aas_keys))
-        return response_t(list(paginated_slice), cursor=cursor)
+        paginated_slice, paging_metadata = self._get_slice(request, list(matching_aas_keys))
+        return response_t(list(paginated_slice), paging_metadata=paging_metadata)
 
     def search_all_aas_ids_by_asset_link(
         self, request: Request, url_args: dict, response_t: Type[APIResponse], **_kwargs
@@ -171,8 +202,8 @@ def search_all_aas_ids_by_asset_link(
         for asset_link in asset_links:
             aas_keys = self.persistent_store.search_aas_ids_by_asset_link(asset_link)
             matching_aas_keys.update(aas_keys)
-        paginated_slice, cursor = self._get_slice(request, list(matching_aas_keys))
-        return response_t(list(paginated_slice), cursor=cursor)
+        paginated_slice, paging_metadata = self._get_slice(request, list(matching_aas_keys))
+        return response_t(list(paginated_slice), paging_metadata=paging_metadata)
 
     def get_all_specific_asset_ids_by_aas_id(
         self, request: Request, url_args: dict, response_t: Type[APIResponse], **_kwargs

server/app/services/run_discovery.py

@@ -8,13 +8,22 @@
 
 wsgi_optparams = {}
 
+
 if base_path is not None:
     wsgi_optparams["base_path"] = base_path
 
 
 # Load DiscoveryStore from disk, if `storage_path` is set
 if storage_path:
-    discovery_store: DiscoveryStore = DiscoveryStore.from_file(storage_path)
+    # If the storage file exists, we load it
+    if os.path.exists(storage_path) and os.path.getsize(storage_path) > 0:
+        discovery_store = DiscoveryStore.from_file(storage_path)
+
+    # If the file doesn't exist at the given path, we initiate a new file
+    else:
+        os.makedirs(os.path.dirname(storage_path), exist_ok=True)
+        discovery_store = DiscoveryStore()
+        discovery_store.to_file(storage_path)
 else:
     discovery_store = DiscoveryStore()
 

server/example_configurations/discovery_standalone/README.md

@@ -1,7 +1,7 @@
 # Eclipse BaSyx Python SDK - Discovery Service
 
 This is a Python-based implementation of the **BaSyx Asset Administration Shell (AAS) Discovery Service**.
-It provides basic discovery functionality for AAS IDs and their corresponding assets, as specified in the official [Discovery Service Specification v3.1.0_SSP-001](https://app.swaggerhub.com/apis/Plattform_i40/DiscoveryServiceSpecification/V3.1.0_SSP-001).
+It provides basic discovery functionality for AAS IDs and their corresponding assets, as specified in the official [Discovery Service Specification v3.1.1_SSP-001](https://app.swaggerhub.com/apis/Plattform_i40/DiscoveryServiceSpecification/V3.1.1_SSP-001).
 
 ## Overview
 
@@ -11,38 +11,36 @@ The Discovery Service stores and retrieves relations between AAS identifiers and
 
 | Function                                 | Description                                              | Example URL                                                           |
 |------------------------------------------|----------------------------------------------------------|-----------------------------------------------------------------------|
-| **search_all_aas_ids_by_asset_link**     | Find AAS identifiers by providing asset link values      | `POST http://localhost:8084/api/v3.0/lookup/shellsByAssetLink`        |
-| **get_all_specific_asset_ids_by_aas_id** | Return specific asset ids associated with an AAS ID      | `GET http://localhost:8084/api/v3.0/lookup/shells/{aasIdentifier}`    |
-| **post_all_asset_links_by_id**           | Register specific asset ids linked to an AAS             | `POST http://localhost:8084/api/v3.0/lookup/shells/{aasIdentifier}`   |
-| **delete_all_asset_links_by_id**         | Delete all asset links associated with a specific AAS ID | `DELETE http://localhost:8084/api/v3.0/lookup/shells/{aasIdentifier}` |
-| 
+| **get_description**                      | Return the supported Discovery Service profiles          | `GET http://localhost:8084/api/v3.1/description`                      |
+| **get_all_aas_ids_by_asset_link**        | Find AAS identifiers by asset link query parameter       | `GET http://localhost:8084/api/v3.1/lookup/shells?assetIds={assetIds}` |
+| **search_all_aas_ids_by_asset_link**     | Find AAS identifiers by providing asset link values      | `POST http://localhost:8084/api/v3.1/lookup/shellsByAssetLink`        |
+| **get_all_specific_asset_ids_by_aas_id** | Return specific asset ids associated with an AAS ID      | `GET http://localhost:8084/api/v3.1/lookup/shells/{aasIdentifier}`    |
+| **post_all_asset_links_by_id**           | Register specific asset ids linked to an AAS             | `POST http://localhost:8084/api/v3.1/lookup/shells/{aasIdentifier}`   |
+| **delete_all_asset_links_by_id**         | Delete all asset links associated with a specific AAS ID | `DELETE http://localhost:8084/api/v3.1/lookup/shells/{aasIdentifier}` |
+
 
 ## Configuration
-Add discovery_store as directory
-The service can be configured to use either:
+This example Docker compose configuration starts a discovery server. 
+
+The container image can also be built and run via:
+```
+$ docker compose up
+```
 
-- **In-memory storage** (default): Temporary data storage that resets on service restart.
-- **MongoDB storage**: Persistent backend storage using MongoDB.
+## Persistence
 
-### Configuration via Environment Variables
+The discovery service can run in persistent or non-persistent mode.
 
-| Variable         | Description                                | Default                     |
-|------------------|--------------------------------------------|-----------------------------|
-| `STORAGE_TYPE`   | `inmemory` or `mongodb`                    | `inmemory`                  |
-| `MONGODB_URI`    | MongoDB connection URI                     | `mongodb://localhost:27017` |
-| `MONGODB_DBNAME` | Name of the MongoDB database               | `basyx_registry`            |
+### Persistent Mode
 
-## Deployment via Docker
+Persistent mode configuration is provided in the `compose.yaml`.
 
-A `Dockerfile` and `docker-compose.yml` are provided for simple deployment.
-The container image can be built and run via:
-```bash
-docker compose up --build
-```
-## Test
+Only the AAS-to-asset-ID mapping is persisted. The reverse lookup index is rebuilt in memory when the service starts.
+
+### Non-Persistent Mode
 
-Examples of asset links and specific asset IDs for testing purposes are provided as JSON files in the [storage](./storage) folder.
+If `storage_path` is not set, the discovery service runs in memory only.
 
-## Acknowledgments
+## Notes
+- Stop the service before manually editing `discovery_store.json`.
 
-This Dockerfile is inspired by the [tiangolo/uwsgi-nginx-docker](https://github.com/tiangolo/uwsgi-nginx-docker) repository.

server/example_configurations/discovery_standalone/compose.yml

@@ -6,7 +6,7 @@ services:
       dockerfile: server/docker/discovery/Dockerfile
     ports:
     - "8084:80"
-    #environment:
-      #- storage_path=/discovery_store.json
-    #volumes:
-     # - ./discovery_store.json:/discovery_store.json
+    environment:
+      storage_path: /storage/discovery_store.json
+    volumes:
+      - ./storage:/storage