Skip to content

Commit 7ea897e

Browse files
FrankSchnickeFrankSchnicke
authored
Merge pull request #10 from acplt/feature/file_backend
Add a local file backend
2 parents 80cf2a2 + dab334a commit 7ea897e

2 files changed

Lines changed: 365 additions & 0 deletions

File tree

‎basyx/aas/backend/local_file.py‎

Lines changed: 233 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,233 @@
1+
# Copyright (c) 2022 the Eclipse BaSyx Authors
2+
#
3+
# This program and the accompanying materials are made available under the terms of the MIT License, available in
4+
# the LICENSE file of this project.
5+
#
6+
# SPDX-License-Identifier: MIT
7+
"""
8+
This module adds the functionality of storing and retrieving :class:`~aas.model.base.Identifiable` objects in local
9+
files.
10+
11+
The :class:`~.LocalFileBackend` takes care of updating and committing objects from and to the files, while the
12+
:class:`~LocalFileObjectStore` handles adding, deleting and otherwise managing the AAS objects in a specific Directory.
13+
"""
14+
import copy
15+
from typing import List, Iterator, Iterable, Union
16+
import logging
17+
import json
18+
import os
19+
import hashlib
20+
import threading
21+
import weakref
22+
23+
from . import backends
24+
from ..adapter.json import json_serialization, json_deserialization
25+
from basyx.aas import model
26+
27+
28+
logger = logging.getLogger(__name__)
29+
30+
31+
class LocalFileBackend(backends.Backend):
32+
"""
33+
This Backend stores each Identifiable object as a single JSON document as a local file in a directory.
34+
Each document's id is build from the object's identifier using a SHA256 sum of its identifiable; the document's
35+
contents comprise a single property "data", containing the JSON serialization of the BaSyx Python SDK object. The
36+
:ref:`adapter.json <adapter.json.__init__>` package is used for serialization and deserialization of objects.
37+
"""
38+
39+
@classmethod
40+
def update_object(cls,
41+
updated_object: model.Referable,
42+
store_object: model.Referable,
43+
relative_path: List[str]) -> None:
44+
45+
if not isinstance(store_object, model.Identifiable):
46+
raise FileBackendSourceError("The given store_object is not Identifiable, therefore cannot be found "
47+
"in the FileBackend")
48+
file_name: str = store_object.source.replace("file://localhost/", "")
49+
with open(file_name, "r") as file:
50+
data = json.load(file, cls=json_deserialization.AASFromJsonDecoder)
51+
updated_store_object = data["data"]
52+
store_object.update_from(updated_store_object)
53+
54+
@classmethod
55+
def commit_object(cls,
56+
committed_object: model.Referable,
57+
store_object: model.Referable,
58+
relative_path: List[str]) -> None:
59+
if not isinstance(store_object, model.Identifiable):
60+
raise FileBackendSourceError("The given store_object is not Identifiable, therefore cannot be found "
61+
"in the FileBackend")
62+
file_name: str = store_object.source.replace("file://localhost/", "")
63+
with open(file_name, "w") as file:
64+
json.dump({'data': store_object}, file, cls=json_serialization.AASToJsonEncoder, indent=4)
65+
66+
67+
backends.register_backend("file", LocalFileBackend)
68+
69+
70+
class LocalFileObjectStore(model.AbstractObjectStore):
71+
"""
72+
An ObjectStore implementation for :class:`~aas.model.base.Identifiable` BaSyx Python SDK objects backed by a local
73+
file based local backend
74+
"""
75+
def __init__(self, directory_path: str):
76+
"""
77+
Initializer of class LocalFileObjectStore
78+
79+
:param directory_path: Path to the local file backend (the path where you want to store your AAS JSON files)
80+
"""
81+
super().__init__()
82+
self.directory_path: str = directory_path.rstrip("/")
83+
84+
# A dictionary of weak references to local replications of stored objects. Objects are kept in this cache as
85+
# long as there is any other reference in the Python application to them. We use this to make sure that only one
86+
# local replication of each object is kept in the application and retrieving an object from the store always
87+
# returns the **same** (not only equal) object. Still, objects are forgotten, when they are not referenced
88+
# anywhere else to save memory.
89+
self._object_cache: weakref.WeakValueDictionary[model.Identifier, model.Identifiable] \
90+
= weakref.WeakValueDictionary()
91+
self._object_cache_lock = threading.Lock()
92+
93+
def check_directory(self, create=False):
94+
"""
95+
Check if the directory exists and created it if not (and requested to do so)
96+
97+
:param create: If True and the database does not exist, try to create it
98+
"""
99+
if not os.path.exists(self.directory_path):
100+
if not create:
101+
raise FileNotFoundError("The given directory ({}) does not exist".format(self.directory_path))
102+
# Create directory
103+
os.mkdir(self.directory_path)
104+
logger.info("Creating directory {}".format(self.directory_path))
105+
106+
def get_identifiable(self, identifier: Union[str, model.Identifier]) -> model.Identifiable:
107+
"""
108+
Retrieve an AAS object from the local file by its :class:`~aas.model.base.Identifier`
109+
110+
If the :class:`~.aas.model.base.Identifier` is a string, it is assumed that the string is a correct
111+
local-file-ID-string (as it is outputted by LocalFileObjectStore._transform_id() )
112+
113+
:raises KeyError: If the respective file could not be found
114+
"""
115+
input_identifier = copy.copy(identifier)
116+
if isinstance(identifier, model.Identifier):
117+
identifier = self._transform_id(identifier)
118+
119+
# Try to get the correct file
120+
try:
121+
with open("{}/{}.json".format(self.directory_path, identifier), "r") as file:
122+
data = json.load(file, cls=json_deserialization.AASFromJsonDecoder)
123+
obj = data["data"]
124+
self.generate_source(obj)
125+
except FileNotFoundError as e:
126+
raise KeyError("No Identifiable with id {} found in local file database".format(input_identifier)) from e
127+
# If we still have a local replication of that object (since it is referenced from anywhere else), update that
128+
# replication and return it.
129+
with self._object_cache_lock:
130+
if obj.identification in self._object_cache:
131+
old_obj = self._object_cache[obj.identification]
132+
# If the source does not match the correct source for this CouchDB backend, the object seems to belong
133+
# to another backend now, so we return a fresh copy
134+
if old_obj.source == obj.source:
135+
old_obj.update_from(obj)
136+
return old_obj
137+
self._object_cache[obj.identification] = obj
138+
return obj
139+
140+
def add(self, x: model.Identifiable) -> None:
141+
"""
142+
Add an object to the store
143+
144+
:raises KeyError: If an object with the same id exists already in the object store
145+
"""
146+
logger.debug("Adding object %s to Local File Store ...", repr(x))
147+
if os.path.exists("{}/{}.json".format(self.directory_path, self._transform_id(x.identification))):
148+
raise KeyError("Identifiable with id {} already exists in local file database".format(x.identification))
149+
with open("{}/{}.json".format(self.directory_path, self._transform_id(x.identification)), "w") as file:
150+
json.dump({"data": x}, file, cls=json_serialization.AASToJsonEncoder, indent=4)
151+
with self._object_cache_lock:
152+
self._object_cache[x.identification] = x
153+
self.generate_source(x) # Set the source of the object
154+
155+
def discard(self, x: model.Identifiable) -> None:
156+
"""
157+
Delete an :class:`~aas.model.base.Identifiable` AAS object from the local file store
158+
159+
:param x: The object to be deleted
160+
:raises KeyError: If the object does not exist in the database
161+
"""
162+
logger.debug("Deleting object %s from Local File Store database ...", repr(x))
163+
try:
164+
os.remove("{}/{}.json".format(self.directory_path, self._transform_id(x.identification)))
165+
except FileNotFoundError as e:
166+
raise KeyError("No AAS object with id {} exists in local file database".format(x.identification)) from e
167+
with self._object_cache_lock:
168+
del self._object_cache[x.identification]
169+
x.source = ""
170+
171+
def __contains__(self, x: object) -> bool:
172+
"""
173+
Check if an object with the given :class:`~aas.model.base.Identifier` or the same
174+
:class:`~aas.model.base.Identifier` as the given object is contained in the local file database
175+
176+
:param x: AAS object :class:`~aas.model.base.Identifier` or :class:`~aas.model.base.Identifiable` AAS object
177+
:return: `True` if such an object exists in the database, `False` otherwise
178+
"""
179+
if isinstance(x, model.Identifier):
180+
identifier = x
181+
elif isinstance(x, model.Identifiable):
182+
identifier = x.identification
183+
else:
184+
return False
185+
logger.debug("Checking existence of object with id %s in database ...", repr(x))
186+
return os.path.exists("{}/{}.json".format(self.directory_path, self._transform_id(identifier)))
187+
188+
def __len__(self) -> int:
189+
"""
190+
Retrieve the number of objects in the local file database
191+
192+
:return: The number of objects (determined from the number of documents)
193+
"""
194+
logger.debug("Fetching number of documents from database ...")
195+
return len(os.listdir(self.directory_path))
196+
197+
def __iter__(self) -> Iterator[model.Identifiable]:
198+
"""
199+
Iterate all :class:`~aas.model.base.Identifiable` objects in the CouchDB database.
200+
201+
This method returns an iterator, containing only a list of all identifiers in the database and retrieving
202+
the identifiable objects on the fly.
203+
"""
204+
logger.debug("Iterating over objects in database ...")
205+
for name in os.listdir(self.directory_path):
206+
yield self.get_identifiable(name.rstrip(".json"))
207+
208+
@staticmethod
209+
def _transform_id(identifier: model.Identifier) -> str:
210+
"""
211+
Helper method to represent an ASS Identifier as a string to be used as Local file document id
212+
"""
213+
return hashlib.sha256("{}-{}".format(identifier.id_type.name, identifier.id).encode("utf-8")).hexdigest()
214+
215+
def generate_source(self, identifiable: model.Identifiable) -> str:
216+
"""
217+
Generates the source string for an :class:`~aas.model.base.Identifiable` object that is backed by the File
218+
219+
:param identifiable: Identifiable object
220+
"""
221+
source: str = "file://localhost/{}/{}.json".format(
222+
self.directory_path,
223+
self._transform_id(identifiable.identification)
224+
)
225+
identifiable.source = source
226+
return source
227+
228+
229+
class FileBackendSourceError(Exception):
230+
"""
231+
Raised, if the given object's source is not resolvable as a local file
232+
"""
233+
pass

‎test/backend/test_local_file.py‎

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
# Copyright (c) 2022 the Eclipse BaSyx Authors
2+
#
3+
# This program and the accompanying materials are made available under the terms of the MIT License, available in
4+
# the LICENSE file of this project.
5+
#
6+
# SPDX-License-Identifier: MIT
7+
import os.path
8+
import shutil
9+
import unittest
10+
import unittest.mock
11+
12+
from basyx.aas.backend import local_file
13+
from basyx.aas.examples.data.example_aas import *
14+
15+
16+
store_path: str = os.path.dirname(__file__) + "/local_file_test_folder"
17+
source_core: str = "file://localhost/{}/".format(store_path)
18+
19+
20+
class LocalFileBackendTest(unittest.TestCase):
21+
def setUp(self) -> None:
22+
self.object_store = local_file.LocalFileObjectStore(store_path)
23+
self.object_store.check_directory(create=True)
24+
25+
def tearDown(self) -> None:
26+
try:
27+
self.object_store.clear()
28+
finally:
29+
shutil.rmtree(store_path)
30+
31+
def test_object_store_add(self):
32+
test_object = create_example_submodel()
33+
self.object_store.add(test_object)
34+
self.assertEqual(
35+
test_object.source,
36+
source_core+"bfe69a634a188d106286585170ba06dfbbd26dd000c641cab5b0f374e94c9611.json"
37+
)
38+
39+
def test_retrieval(self):
40+
test_object = create_example_submodel()
41+
self.object_store.add(test_object)
42+
43+
# When retrieving the object, we should get the *same* instance as we added
44+
test_object_retrieved = self.object_store.get_identifiable(
45+
model.Identifier(id_='https://acplt.org/Test_Submodel', id_type=model.IdentifierType.IRI))
46+
self.assertIs(test_object, test_object_retrieved)
47+
48+
# When retrieving it again, we should still get the same object
49+
del test_object
50+
test_object_retrieved_again = self.object_store.get_identifiable(
51+
model.Identifier(id_='https://acplt.org/Test_Submodel', id_type=model.IdentifierType.IRI))
52+
self.assertIs(test_object_retrieved, test_object_retrieved_again)
53+
54+
# However, a changed source should invalidate the cached object, so we should get a new copy
55+
test_object_retrieved.source = "couchdb://example.com/example/IRI-https%3A%2F%2Facplt.org%2FTest_Submodel"
56+
test_object_retrieved_third = self.object_store.get_identifiable(
57+
model.Identifier(id_='https://acplt.org/Test_Submodel', id_type=model.IdentifierType.IRI))
58+
self.assertIsNot(test_object_retrieved, test_object_retrieved_third)
59+
60+
def test_example_submodel_storing(self) -> None:
61+
example_submodel = create_example_submodel()
62+
63+
# Add exmaple submodel
64+
self.object_store.add(example_submodel)
65+
self.assertEqual(1, len(self.object_store))
66+
self.assertIn(example_submodel, self.object_store)
67+
68+
# Restore example submodel and check data
69+
submodel_restored = self.object_store.get_identifiable(
70+
model.Identifier(id_='https://acplt.org/Test_Submodel', id_type=model.IdentifierType.IRI))
71+
assert (isinstance(submodel_restored, model.Submodel))
72+
checker = AASDataChecker(raise_immediately=True)
73+
check_example_submodel(checker, submodel_restored)
74+
75+
# Delete example submodel
76+
self.object_store.discard(submodel_restored)
77+
self.assertNotIn(example_submodel, self.object_store)
78+
79+
def test_iterating(self) -> None:
80+
example_data = create_full_example()
81+
82+
# Add all objects
83+
for item in example_data:
84+
self.object_store.add(item)
85+
86+
self.assertEqual(6, len(self.object_store))
87+
88+
# Iterate objects, add them to a DictObjectStore and check them
89+
retrieved_data_store: model.provider.DictObjectStore[model.Identifiable] = model.provider.DictObjectStore()
90+
for item in self.object_store:
91+
retrieved_data_store.add(item)
92+
checker = AASDataChecker(raise_immediately=True)
93+
check_full_example(checker, retrieved_data_store)
94+
95+
def test_key_errors(self) -> None:
96+
# Double adding an object should raise a KeyError
97+
example_submodel = create_example_submodel()
98+
self.object_store.add(example_submodel)
99+
with self.assertRaises(KeyError) as cm:
100+
self.object_store.add(example_submodel)
101+
self.assertEqual("'Identifiable with id Identifier(IRI=https://acplt.org/Test_Submodel) already exists in "
102+
"local file database'", str(cm.exception))
103+
104+
# Querying a deleted object should raise a KeyError
105+
retrieved_submodel = self.object_store.get_identifiable(
106+
model.Identifier('https://acplt.org/Test_Submodel', model.IdentifierType.IRI))
107+
self.object_store.discard(example_submodel)
108+
with self.assertRaises(KeyError) as cm:
109+
self.object_store.get_identifiable(model.Identifier('https://acplt.org/Test_Submodel',
110+
model.IdentifierType.IRI))
111+
self.assertEqual("'No Identifiable with id Identifier(IRI=https://acplt.org/Test_Submodel) "
112+
"found in local file database'",
113+
str(cm.exception))
114+
115+
# Double deleting should also raise a KeyError
116+
with self.assertRaises(KeyError) as cm:
117+
self.object_store.discard(retrieved_submodel)
118+
self.assertEqual("'No AAS object with id Identifier(IRI=https://acplt.org/Test_Submodel) exists in "
119+
"local file database'", str(cm.exception))
120+
121+
def test_editing(self):
122+
test_object = create_example_submodel()
123+
self.object_store.add(test_object)
124+
125+
# Test if commit uploads changes
126+
test_object.id_short = "SomeNewIdShort"
127+
test_object.commit()
128+
129+
# Test if update restores changes
130+
test_object.id_short = "AnotherIdShort"
131+
test_object.update()
132+
self.assertEqual("SomeNewIdShort", test_object.id_short)

0 commit comments

Comments
 (0)