Version 0.5 - add datastore creation and tests. Small update of API: update_from_yaml instead of update_yaml

Author: mcarans

.idea/workspace.xml

@@ -45,7 +45,7 @@ The first task is to create an API key file. By default this is assumed to be ca
 
 To include the HDX Python library in your project, pip install the line below or add the following to your `requirements.txt` file:
 
-    git+git://github.com/ocha-dap/hdx-python-api.git@v0.42#egg=hdx-python-api
+    git+git://github.com/ocha-dap/hdx-python-api.git@v0.5#egg=hdx-python-api
 
 If you get errors, it is probably the dependencies of the cryptography package that are missing eg. for Ubuntu: python-dev, libffi-dev and libssl-dev. See [cryptography dependencies](https://cryptography.io/en/latest/installation/#building-cryptography-on-linux)
 
@@ -68,7 +68,7 @@ Let's start with a simple example that also ensures that the library is working
         source test/bin/activate
 4. Install the HDX Python library:
 
-        pip install git+git://github.com/ocha-dap/hdx-python-api.git@v0.42#egg=hdx-python-api
+        pip install git+git://github.com/ocha-dap/hdx-python-api.git@v0.5#egg=hdx-python-api
 5. If you get errors, it is probably the [dependencies of the cryptography package](#installing-the-library)
 6. Launch python:
 
@@ -246,9 +246,9 @@ You can also do so by the standard dictionary `update` method, which takes a d
 
 Larger amounts of static metadata are best added from files. YAML is very human readable and recommended, while JSON is also accepted eg.
 
-    dataset.update_yaml([path])
+    dataset.update_from_yaml([path])
 
-    dataset.update_json([path])
+    dataset.update_from_json([path])
 
 The default path if unspecified is `config/hdx_TYPE_static.yml` for YAML and `config/hdx_TYPE_static.json` for JSON where TYPE is an HDX object's type like dataset or resource eg. `config/hdx_galleryitem_static.json`. The YAML file takes the following form:
 
@@ -328,7 +328,7 @@ Next create a file called `run.py` and copy into it the code below.
     '''
     import logging
     from hdx.facades.scraperwiki import facade
-    from my_code import generate_dataset
+    from .my_code import generate_dataset
 
     logger = logging.getLogger(__name__)
 

README.md

@@ -211,7 +211,7 @@ def get_gallery(self) -> List[GalleryItem]:
         """
         return self.gallery
 
-    def update_yaml(self, path: Optional[str] = join('config', 'hdx_dataset_static.yml')) -> None:
+    def update_from_yaml(self, path: Optional[str] = join('config', 'hdx_dataset_static.yml')) -> None:
         """Update dataset metadata with static metadata from YAML file
 
         Args:
@@ -220,11 +220,11 @@ def update_yaml(self, path: Optional[str] = join('config', 'hdx_dataset_static.y
         Returns:
             None
         """
-        super(Dataset, self).update_yaml(path)
+        super(Dataset, self).update_from_yaml(path)
         self.separate_resources()
         self.separate_gallery()
 
-    def update_json(self, path: Optional[str] = join('config', 'hdx_dataset_static.json')) -> None:
+    def update_from_json(self, path: Optional[str] = join('config', 'hdx_dataset_static.json')) -> None:
         """Update dataset metadata with static metadata from JSON file
 
         Args:
@@ -233,7 +233,7 @@ def update_json(self, path: Optional[str] = join('config', 'hdx_dataset_static.j
         Returns:
             None
         """
-        super(Dataset, self).update_json(path)
+        super(Dataset, self).update_from_json(path)
         self.separate_resources()
         self.separate_gallery()
 
@@ -453,4 +453,6 @@ def search_in_hdx(configuration: Configuration, query: str) -> List['Dataset']:
                     dataset.data = datasetdict
                     dataset._dataset_create_resources_gallery()
                     datasets.append(dataset)
+        else:
+            logger.debug(result)
         return datasets

hdx/data/dataset.py

@@ -3,7 +3,6 @@
 """GalleryItem class containing all logic for creating, checking, and updating gallery items."""
 import logging
 from os.path import join
-
 from typing import Optional, List
 
 from hdx.configuration import Configuration
@@ -39,7 +38,7 @@ def actions() -> dict:
             'list': 'related_list'
         }
 
-    def update_yaml(self, path: str = join('config', 'hdx_galleryitem_static.yml')) -> None:
+    def update_from_yaml(self, path: str = join('config', 'hdx_galleryitem_static.yml')) -> None:
         """Update gallery item metadata with static metadata from YAML file
 
         Args:
@@ -48,9 +47,9 @@ def update_yaml(self, path: str = join('config', 'hdx_galleryitem_static.yml'))
         Returns:
             None
         """
-        super(GalleryItem, self).update_yaml(path)
+        super(GalleryItem, self).update_from_yaml(path)
 
-    def update_json(self, path: str = join('config', 'hdx_galleryitem_static.json')) -> None:
+    def update_from_json(self, path: str = join('config', 'hdx_galleryitem_static.json')) -> None:
         """Update gallery item metadata with static metadata from JSON file
 
         Args:
@@ -59,7 +58,7 @@ def update_json(self, path: str = join('config', 'hdx_galleryitem_static.json'))
         Returns:
             None
         """
-        super(GalleryItem, self).update_json(path)
+        super(GalleryItem, self).update_from_json(path)
 
     @staticmethod
     def read_from_hdx(configuration: Configuration, identifier: str) -> Optional['GalleryItem']:

hdx/data/galleryitem.py

@@ -60,7 +60,7 @@ def get_old_data_dict(self) -> None:
         """
         return self.old_data
 
-    def update_yaml(self, path: str) -> None:
+    def update_from_yaml(self, path: str) -> None:
         """Update metadata with static metadata from YAML file
 
         Args:
@@ -71,7 +71,7 @@ def update_yaml(self, path: str) -> None:
         """
         self.data = load_yaml_into_existing_dict(self.data, path)
 
-    def update_json(self, path: str):
+    def update_from_json(self, path: str):
         """Update metadata with static metadata from JSON file
 
         Args:
@@ -83,17 +83,19 @@ def update_json(self, path: str):
         self.data = load_json_into_existing_dict(self.data, path)
 
     def _read_from_hdx(self, object_type: str, value: str, fieldname: Optional[str] = 'id',
-                       action: Optional[str] = None) -> Tuple[bool, dict]:
+                       action: Optional[str] = None,
+                       other_fields: dict = {}) -> Union[Tuple[bool, dict], Tuple[bool, str]]:
         """Makes a read call to HDX passing in given parameter.
 
         Args:
             object_type (str): Description of HDX object type (for messages)
             value (str): Value of HDX field
             fieldname (Optional[str]): HDX field name. Defaults to id.
             action (Optional[str]): Replacement CKAN action url to use. Defaults to None.
+            other_fields (dict): Other fields to pass to CKAN. Defaults to empty dict.
 
         Returns:
-            (bool, dict): (True/False, HDX object metadata/Error)
+            (bool, dict/str): (True/False, HDX object metadata/Error)
         """
         if not value:
             raise HDXError("Empty %s value!" % object_type)
@@ -102,14 +104,16 @@ def _read_from_hdx(self, object_type: str, value: str, fieldname: Optional[str]
                 action = self.actions()['search']
             else:
                 action = self.actions()['show']
+        data = {fieldname: value}
+        data.update(other_fields)
         try:
-            result = self.hdxpostsite.call_action(action, {fieldname: value},
+            result = self.hdxpostsite.call_action(action, data,
                                                   requests_kwargs={'auth': self.configuration._get_credentials()})
             return True, result
-        except NotFound as e:
+        except NotFound:
             return False, "%s=%s: not found!" % (fieldname, value)
         except Exception as e:
-            raise HDXError('HTTP Get failed when trying to read: %s=%s' % (fieldname, value)) from e
+            raise HDXError('HTTP Post failed when trying to read: %s=%s' % (fieldname, value)) from e
 
     def _load_from_hdx(self, object_type: str, id_field: str) -> bool:
         """Helper method to load the HDX object given by identifier from HDX
@@ -126,6 +130,7 @@ def _load_from_hdx(self, object_type: str, id_field: str) -> bool:
             self.old_data = self.data
             self.data = result
             return True
+        logger.debug(result)
         return False
 
     @staticmethod
@@ -224,21 +229,20 @@ def _update_in_hdx(self, object_type: str, id_field_name: str) -> None:
         self._check_load_existing_object(object_type, id_field_name)
         self._merge_hdx_update(object_type, id_field_name)
 
-    def _write_to_hdx(self, action: str, data: dict, id_field_name: str) -> Union[Tuple[bool, dict], Tuple[bool, str]]:
+    def _write_to_hdx(self, action: str, data: dict, id_field_name: str) -> dict:
         """Creates or updates an HDX object in HDX and return HDX object metadata dict
 
         Args:
-            action (str): Action to perform: 'create' or 'update'
+            action (str): Action to perform eg. 'create', 'update'
             data (dict): Data to write to HDX
             id_field_name (str): Name of field containing HDX object identifier
 
         Returns:
-            (bool, dict): (True/False, HDX object metadata/Error)
+            dict: HDX object metadata
         """
         try:
-            result = self.hdxpostsite.call_action(self.actions()[action], data,
-                                                  requests_kwargs={'auth': self.configuration._get_credentials()})
-            return True, result
+            return self.hdxpostsite.call_action(self.actions()[action], data,
+                                                requests_kwargs={'auth': self.configuration._get_credentials()})
         except Exception as e:
             raise HDXError('HTTP Post failed when trying to %s %s' % (action, self.data[id_field_name])) from e
 
@@ -253,13 +257,9 @@ def _save_to_hdx(self, action: str, id_field_name: str) -> None:
         Returns:
             None
         """
-        success, result = self._write_to_hdx(action, self.data, id_field_name)
-
-        if success:
-            self.old_data = self.data
-            self.data = result
-        else:
-            raise HDXError('Failed to %s %s\n%s' % (action, self.data[id_field_name], result))
+        result = self._write_to_hdx(action, self.data, id_field_name)
+        self.old_data = self.data
+        self.data = result
 
     @abc.abstractmethod
     def create_in_hdx(self) -> None: