Added support for datetime column in datebase created by hydrus (#599)

* changes in helpers
* function to modify object
* datetime support
* changes in tests
* changes in sample doc
* mapping for xsd:string
* added docstring and annotations
* added missing type annotation
* iso datetime format
* added exception for invalid date format
* fix get_modified_object

Author: farazkhanfk7

hydrus/data/crud.py

@@ -49,6 +49,7 @@
 
 # from sqlalchemy.orm.session import Session
 from sqlalchemy.orm.scoping import scoped_session
+from hydra_python_core.doc_writer import HydraDoc
 from typing import Dict, Optional, Any, List
 
 from hydrus.data.resource_based_classes import (
@@ -97,12 +98,14 @@ def get(
 
 
 def insert(
+    doc_: HydraDoc,
     object_: Dict[str, Any],
     session: scoped_session,
     id_: Optional[str] = None,
     collection: bool = False,
 ) -> str:
     """Insert an object to database [POST] and returns the inserted object.
+    :param doc_ : Hydra Doc object
     :param object_: object to be inserted
     :param session: sqlalchemy scoped session
     :param id_: id of the object to be inserted (optional param)
@@ -124,15 +127,19 @@ def insert(
     object_template = copy.deepcopy(object_)
     if id_ is not None:
         object_template["id"] = id_
-    inserted_object_id = insert_object(object_template, session, collection)
+    inserted_object_id = insert_object(doc_, object_template, session, collection)
     return inserted_object_id
 
 
 def insert_multiple(
-    objects_: List[Dict[str, Any]], session: scoped_session, id_: Optional[str] = ""
+    doc: HydraDoc,
+    objects_: List[Dict[str, Any]],
+    session: scoped_session,
+    id_: Optional[str] = ""
 ) -> List[str]:
     """
     Adds a list of object with given ids to the database
+    :param doc : Hydra Doc object
     :param objects_: List of dict's to be added to the database
     :param session: scoped session from getSession in utils
     :param id_: optional parameter containing the ids of objects that have to be inserted
@@ -167,7 +174,7 @@ def insert_multiple(
             pass
         except TypeError:
             pass
-        inserted_object_id = insert(object_, session, id_of_object_)
+        inserted_object_id = insert(doc, object_, session, id_of_object_)
         instance_id_list.append(inserted_object_id)
 
     return instance_id_list
@@ -209,6 +216,7 @@ def delete_multiple(id_: List[int], type_: str, session: scoped_session) -> None
 
 
 def update(
+    doc_: HydraDoc,
     id_: str,
     type_: str,
     object_: Dict[str, str],
@@ -218,6 +226,7 @@ def update(
     collection: bool = False,
 ) -> str:
     """Update an object properties based on the given object [PUT].
+    :param doc_ : Hydra Doc object
     :param id_: if of object to be updated
     :param type_: type of object to be updated
     :param object_: object that has to be inserted
@@ -228,7 +237,7 @@ def update(
     :return: id of updated object
     """
     query_info = {"@type": type_, "id_": id_}
-    updated_object_id = update_object(object_, query_info, session, collection)
+    updated_object_id = update_object(doc_, object_, query_info, session, collection)
     return updated_object_id
 
 

hydrus/data/db_models.py

@@ -101,7 +101,11 @@ def get_attr_dict(self):
                     foreign_table_name, title
                 )
             else:
-                datatype_keys = {'integer': Integer, 'float': Float, 'decimal': Float}
+                datatype_keys = {'integer': Integer,
+                                 'float': Float,
+                                 'decimal': Float,
+                                 'string': String,
+                                 'dateTime': DateTime}
                 if 'range' in supported_property:
                     datatype = supported_property['range'].split('#')[1]
                     if datatype in datatype_keys:

hydrus/data/exceptions.py

@@ -195,3 +195,17 @@ def get_HTTP(self) -> HydraError:
         """Return the HTTP response for the Exception."""
         description = f"The property {self.type_} is not given."
         return HydraError(code=400, title="Property not given", desc=description)
+
+
+class InvalidDateTimeFormat(Exception):
+    """Error when a datetime field input is invalid"""
+
+    def __init__(self, field_: str) -> None:
+        """Constructor."""
+        self.field_ = field_
+
+    def get_HTTP(self) -> HydraError:
+        """Return the HTTP response for the Exception."""
+        description = (f"The format of {self.field_} is invalid."
+                       f" Datetime input should be in ISO format.")
+        return HydraError(code=400, title="Invalid Datetime format", desc=description)

hydrus/data/helpers/item_collection_helpers.py

@@ -15,6 +15,7 @@
     InvalidSearchParameter,
     OffsetOutOfRange,
     PropertyNotGiven,
+    InvalidDateTimeFormat
 )
 
 from hydrus.data.helpers import (
@@ -34,6 +35,7 @@
     get_hydrus_server_url,
     get_page_size,
     get_pagination,
+    get_doc
 )
 
 
@@ -99,6 +101,7 @@ def item_collection_put_response(path: str) -> Response:
     :rtype: Response
     """
     object_ = json.loads(request.data.decode("utf-8"))
+    doc_object = get_doc()
     collections, parsed_classes = get_collections_and_parsed_classes()
     is_collection = False
     if path in parsed_classes:
@@ -118,7 +121,7 @@ def item_collection_put_response(path: str) -> Response:
         try:
             # Insert object and return location in Header
             object_id = crud.insert(
-                object_=object_, session=get_session(), collection=is_collection
+                object_=object_, session=get_session(), doc_=doc_object, collection=is_collection
             )
             resource_url = (
                 f"{get_hydrus_server_url()}{get_api_name()}/{path}/{object_id}"
@@ -133,7 +136,8 @@ def item_collection_put_response(path: str) -> Response:
             return set_response_headers(
                 jsonify(status_response), headers=headers_, status_code=status.code
             )
-        except (ClassNotFound, InstanceExists, PropertyNotFound, PropertyNotGiven) as e:
+        except (ClassNotFound, InstanceExists, PropertyNotFound,
+                PropertyNotGiven, InvalidDateTimeFormat) as e:
             error = e.get_HTTP()
             return error_response(error)
     else:

hydrus/data/helpers/itemhelpers.py

@@ -9,6 +9,7 @@
     ClassNotFound,
     InstanceNotFound,
     InstanceExists,
+    InvalidDateTimeFormat,
     PropertyNotFound,
     MemberInstanceNotFound,
 )
@@ -24,7 +25,7 @@
     parse_collection_members,
     get_collections_and_parsed_classes,
 )
-from hydrus.utils import get_session, get_api_name, get_hydrus_server_url
+from hydrus.utils import get_session, get_api_name, get_hydrus_server_url, get_doc
 from hydrus.extensions.socketio_factory import socketio
 
 
@@ -52,6 +53,7 @@ def items_get_check_support(id_, class_type, class_path, path, is_collection=Fal
 def items_post_check_support(id_, object_, class_path, path, is_collection):
     """Check if class_type supports POST operation"""
     collections, parsed_classes = get_collections_and_parsed_classes()
+    doc = get_doc()
     if path in parsed_classes:
         class_path = path
         obj_type = getType(path, "PUT")
@@ -68,6 +70,7 @@ def items_post_check_support(id_, object_, class_path, path, is_collection):
             # Update the right ID if the object is valid and matches
             # type of Item
             object_id = crud.update(
+                doc,
                 object_=object_,
                 id_=id_,
                 type_=object_["@type"],
@@ -99,7 +102,8 @@ def items_post_check_support(id_, object_, class_path, path, is_collection):
             status_response["iri"] = resource_url
             return set_response_headers(jsonify(status_response), headers=headers_)
 
-        except (ClassNotFound, InstanceNotFound, InstanceExists, PropertyNotFound) as e:
+        except (ClassNotFound, InstanceNotFound, InstanceExists,
+                PropertyNotFound, InvalidDateTimeFormat) as e:
             error = e.get_HTTP()
             return error_response(error)
     else:
@@ -110,6 +114,7 @@ def items_post_check_support(id_, object_, class_path, path, is_collection):
 def items_put_check_support(id_, class_path, path, is_collection):
     """Check if class_type supports PUT operation"""
     object_ = json.loads(request.data.decode("utf-8"))
+    doc = get_doc()
     collections, parsed_classes = get_collections_and_parsed_classes()
     if path in parsed_classes:
         class_path = path
@@ -126,6 +131,7 @@ def items_put_check_support(id_, class_path, path, is_collection):
         try:
             # Add the object with given ID
             object_id = crud.insert(
+                doc,
                 object_=object_,
                 id_=id_,
                 session=get_session(),
@@ -144,7 +150,8 @@ def items_put_check_support(id_, class_path, path, is_collection):
             return set_response_headers(
                 jsonify(status_response), headers=headers_, status_code=status.code
             )
-        except (ClassNotFound, InstanceExists, PropertyNotFound) as e:
+        except (ClassNotFound, InstanceExists,
+                PropertyNotFound, InvalidDateTimeFormat) as e:
             error = e.get_HTTP()
             return error_response(error)
     else:

hydrus/data/helpers/items_helpers.py

@@ -8,6 +8,7 @@
     InstanceExists,
     PropertyNotFound,
     InstanceNotFound,
+    InvalidDateTimeFormat
 )
 from hydrus.data.helpers import (
     checkClassOp,
@@ -26,6 +27,7 @@
     get_api_name,
     get_hydrus_server_url,
     set_response_headers,
+    get_doc
 )
 from hydrus.extensions.socketio_factory import socketio
 
@@ -42,6 +44,7 @@ def items_put_response(path: str, int_list="") -> Response:
     :rtype: Response
     """
     object_ = json.loads(request.data.decode("utf-8"))
+    doc = get_doc()
     object_ = object_["data"]
     _, parsed_classes = get_collections_and_parsed_classes()
     if path in parsed_classes:
@@ -65,7 +68,7 @@ def items_put_response(path: str, int_list="") -> Response:
                 try:
                     # Insert object and return location in Header
                     object_id = crud.insert_multiple(
-                        objects_=object_, session=get_session(), id_=int_list
+                        doc, objects_=object_, session=get_session(), id_=int_list
                     )
                     headers_ = [
                         {
@@ -100,7 +103,8 @@ def items_put_response(path: str, int_list="") -> Response:
                             headers=headers_,
                             status_code=status.code,
                         )
-                except (ClassNotFound, InstanceExists, PropertyNotFound) as e:
+                except (ClassNotFound, InstanceExists,
+                        PropertyNotFound, InvalidDateTimeFormat) as e:
                     error = e.get_HTTP()
                     return error_response(error)
 

hydrus/data/resource_based_classes.py

@@ -4,6 +4,7 @@
 """
 import copy
 import uuid
+from dateutil import parser
 from hydrus.data.db_models import Resource
 from hydrus.data.exceptions import (
     ClassNotFound,
@@ -13,11 +14,13 @@
     PropertyNotFound,
     PropertyNotGiven,
     MemberInstanceNotFound,
+    InvalidDateTimeFormat
 )
 from sqlalchemy import exists
 from sqlalchemy.exc import InvalidRequestError, IntegrityError
 from sqlalchemy.orm.scoping import scoped_session
 from sqlalchemy.orm.exc import NoResultFound
+from hydra_python_core.doc_writer import HydraDoc
 from typing import Dict, Any
 
 
@@ -34,6 +37,39 @@ def get_type(object_: Dict[str, Any]) -> str:
     return type_
 
 
+def get_modified_object(object_: Dict[str, Any], doc: HydraDoc, path: str) -> Dict[str, Any]:
+    """
+    Return the object after modifying properties.
+    :param object_: Dict containing object properties
+    :param doc: HydraDoc object
+    :param path: Path of the Class or Collection
+    :return: object_ containing properties
+    """
+    class_ = doc.parsed_classes[path]
+    properties = class_["class"].supportedProperty
+    datetimefields = []
+    for prop in properties:
+        kwargs = getattr(prop, 'kwargs', None)
+        range = kwargs.get('range')
+        title = getattr(prop, 'title')
+        if range is not None:
+            if "dateTime" in range:
+                datetimefield = title
+                datetimefields.append(datetimefield)
+    if len(datetimefields) != 0:
+        for field in datetimefields:
+            try:
+                datetime_value = object_.get(field)
+                dt_object = parser.isoparse(datetime_value)
+                object_[field] = dt_object
+            except ValueError:
+                raise InvalidDateTimeFormat(field)
+            except TypeError:
+                datetime_value = object_.get(field)
+                object_[field] = datetime_value
+    return object_
+
+
 def get_database_class(type_: str):
     """
     Get the sqlalchemy class object from given classname
@@ -47,10 +83,11 @@ def get_database_class(type_: str):
 
 
 def insert_object(
-    object_: Dict[str, Any], session: scoped_session, collection: bool = False
+    doc_: HydraDoc, object_: Dict[str, Any], session: scoped_session, collection: bool = False
 ) -> str:
     """
     Insert the object in the database
+    :param doc_: HydraDoc object
     :param object_: Dict containing object properties
     :param session: sqlalchemy session
     :return: The ID of the inserted object
@@ -94,16 +131,20 @@ def insert_object(
             fk_column = fk.info["column_name"]
             try:
                 fk_object = object_[fk_column]
+                fk_object_path = fk_object.get('@type')
+                fk_object = get_modified_object(fk_object, doc_, fk_object_path)
             except KeyError as e:
                 wrong_property = e.args[0]
                 raise PropertyNotGiven(type_=wrong_property)
             # insert the foreign key object
-            fk_object_id = insert_object(fk_object, session)
+            fk_object_id = insert_object(doc_, fk_object, session)
             # put the id of the foreign instance in this table's column
             object_[fk_column] = fk_object_id
         try:
             # remove the @type from object before using the object to make a
             # instance of it using sqlalchemy class
+            object_path = object_.get('@type')
+            object_ = get_modified_object(object_, doc_, object_path)
             object_.pop("@type")
             inserted_object = database_class(**object_)
         except TypeError as e:
@@ -198,13 +239,15 @@ def delete_object(
 
 
 def update_object(
+    doc_: HydraDoc,
     object_: Dict[str, Any],
     query_info: Dict[str, str],
     session: scoped_session,
     collection: bool = False,
 ) -> str:
     """
     Update the object from the database
+    :param doc_: HydraDoc object
     :param object_: Dict containing updated object properties
     :param query_info: Dict containing the id and @type of object that has to retrieved
     :param session: sqlalchemy session
@@ -218,11 +261,11 @@ def update_object(
     # Try inserting new object
     try:
         object_["id"] = id_
-        d = insert_object(object_, session, collection)
+        d = insert_object(doc_, object_, session, collection)
     except Exception as e:
         # Put old object back
         old_object["id"] = id_
-        d = insert_object(old_object, session, collection)
+        d = insert_object(doc_, old_object, session, collection)
         raise e
     return id_