Merge pull request #68 from mikofski/fields

implement a Field class similar to fields in Django, Marshmallow and DRF

Author: mikofski

carousel/contrib/readers.py

@@ -7,8 +7,8 @@
 import numpy as np
 import h5py
 from carousel.core.data_readers import DataReader
-from carousel.core import Q_  # UREG
-from django.db.models import AutoField
+from carousel.core.data_sources import DataParameter
+from carousel.core import Q_
 import logging
 
 LOGGER = logging.getLogger(__name__)
@@ -17,11 +17,18 @@
 
 def copy_model_instance(obj):
     """
-    https://djangosnippets.org/snippets/1040/
+    Copy Django model instance as a dictionary excluding automatically created
+    fields like an auto-generated sequence as a primary key or an auto-created
+    many-to-one reverse relation.
+
+    :param obj: Django model object
+    :return: copy of model instance as dictionary
     """
-    return {f.name: getattr(obj, f.name) for f in obj._meta.get_fields()
-            if not isinstance(f, AutoField) and
-            f not in obj._meta.parents.values()}
+    meta = getattr(obj, '_meta')  # make pycharm happy
+    # dictionary of model values excluding auto created and related fields
+    return {f.name: getattr(obj, f.name)
+            for f in meta.get_fields(include_parents=False)
+            if not f.auto_created}
 
 
 # TODO: make parameters consistent for all readers
@@ -58,8 +65,10 @@ def load_data(self, *args, **kwargs):
         """
         # get positional argument names from parameters and apply them to args
         # update data with additional kwargs
-        argpos = {v['argpos']: k for k, v in self.parameters.iteritems()
-                  if 'argpos' in v}
+        argpos = {
+            v['extras']['argpos']: k for k, v in self.parameters.iteritems()
+            if 'argpos' in v['extras']
+        }
         data = dict(
             {argpos[n]: a for n, a in enumerate(args)}, **kwargs
         )
@@ -91,30 +100,32 @@ def __init__(self, parameters=None, meta=None):
             raise AttributeError('model not specified in Meta class')
         #: Django model
         self.model = meta.model
-        all_field_names = [f.name for f in self.model._meta.get_fields()]
+        model_meta = getattr(self.model, '_meta')  # make pycharm happy
+        # model fields excluding AutoFields and related fields like one-to-many
+        all_model_fields = [
+            f for f in model_meta.get_fields(include_parents=False)
+            if not f.auto_created
+        ]
+        all_field_names = [f.name for f in all_model_fields]  # field names
+        # use all fields if no parameters given
         if parameters is None:
-            parameters = dict.fromkeys(
+            parameters = DataParameter.fromkeys(
                 all_field_names, {}
             )
-        fields = getattr(meta, 'fields', all_field_names)
+        fields = getattr(meta, 'fields', all_field_names)  # specified fields
         LOGGER.debug('fields:\n%r', fields)
-        exclude = getattr(meta, 'exclude', [])
-        model_meta_parents_values = self.model._meta.parents.values()
-        for f in self.model._meta.fields:
-            # pop and skip any AutoFields or parents
-            if isinstance(f, AutoField) or f in model_meta_parents_values:
-                parameters.pop(f.name, None)
-                continue
+        exclude = getattr(meta, 'exclude', [])  # specifically excluded fields
+        for f in all_model_fields:
             # skip any fields not specified in data source
             if f.name not in fields or f.name in exclude:
                 LOGGER.debug('skipping %s', f.name)
                 continue
             # add field to parameters or update parameters with field type
             param_dict = {'ftype': f.get_internal_type()}
             if f.name in parameters:
-                parameters[f.name].update(param_dict)
+                parameters[f.name]['extras'].update(param_dict)
             else:
-                parameters[f.name] = param_dict
+                parameters[f.name] = DataParameter(**param_dict)
         super(DjangoModelReader, self).__init__(parameters)
 
     def load_data(self, model_instance, *args, **kwargs):
@@ -138,8 +149,9 @@ def load_data(self, h5file, *args, **kwargs):
             h5data = dict.fromkeys(self.parameters)
             for param, attrs in self.parameters.iteritems():
                 LOGGER.debug('parameter:\n%r', param)
-                node = attrs['node']  # full name of node
-                member = attrs.get('member')  # composite datatype member
+                node = attrs['extras']['node']  # full name of node
+                # composite datatype member
+                member = attrs['extras'].get('member')
                 if member is not None:
                     # if node is a table then get column/field/description
                     h5data[param] = np.asarray(h5f[node][member])  # copy member

carousel/contrib/tests/test_data_readers.py

@@ -6,7 +6,7 @@
 from carousel.contrib.readers import (
     ArgumentReader, DjangoModelReader, HDF5Reader
 )
-from carousel.core.data_sources import DataSource
+from carousel.core.data_sources import DataSource, DataParameter
 from datetime import datetime
 from carousel.core import UREG
 from django.db import models
@@ -91,7 +91,8 @@ class MyApp(AppConfig):
 
 class MyModel(models.Model):
     """
-    Django model for testing :class:`~carousel.contrib.readers.DjangoModelReader`.
+    Django model for testing
+    :class:`~carousel.contrib.readers.DjangoModelReader`.
     """
     air_temp = models.FloatField()
     latitude = models.FloatField()
@@ -108,8 +109,8 @@ class Meta:
 
 def test_arg_reader():
     """
-    Test :class:`~carousel.contrib.readers.ArgumentReader` is instantiated and can
-    load argument data units and values correctly.
+    Test :class:`~carousel.contrib.readers.ArgumentReader` is instantiated and
+    can load argument data units and values correctly.
 
     :return: arg reader and data
     :raises: AssertionError
@@ -122,11 +123,11 @@ def test_arg_reader():
     air_temp = TAIR
     location = {'latitude': LAT, 'longitude': LON, 'timezone': TZ}
     parameters = {
-        'pvmodule': {'argpos': 0},
-        'air_temp': {'units': 'celsius', 'argpos': 1},
-        'latitude': {'units': 'degrees'},
-        'longitude': {'units': 'degrees'},
-        'timezone': {'units': 'hours'}
+        'pvmodule': {'extras': {'argpos': 0}},
+        'air_temp': {'units': 'celsius', 'extras': {'argpos': 1}},
+        'latitude': {'units': 'degrees', 'extras': {}},
+        'longitude': {'units': 'degrees', 'extras': {}},
+        'timezone': {'units': 'hours', 'extras': {}}
     }
     arg_reader = ArgumentReader(parameters)
     assert isinstance(arg_reader, DataReader)  # instance of ArgumentReader
@@ -156,10 +157,10 @@ def test_arg_data_src():
     class ArgSrcTest(DataSource):
         data_reader = ArgumentReader
         data_cache_enabled = False
-        air_temp = {'units': 'celsius', 'argpos': 0}
-        latitude = {'units': 'degrees', 'isconstant': True}
-        longitude = {'units': 'degrees', 'isconstant': True}
-        timezone = {'units': 'hours'}
+        air_temp = DataParameter(**{'units': 'celsius', 'argpos': 0})
+        latitude = DataParameter(**{'units': 'degrees', 'isconstant': True})
+        longitude = DataParameter(**{'units': 'degrees', 'isconstant': True})
+        timezone = DataParameter(**{'units': 'hours'})
 
         def __prepare_data__(self):
             pass
@@ -180,13 +181,13 @@ def __prepare_data__(self):
 
 def test_django_reader():
     """
-    Test :class:`~carousel.contrib.readers.DjangoModelReader` is instantiated and
-    can load argument data units and values correctly.
+    Test :class:`~carousel.contrib.readers.DjangoModelReader` is instantiated
+    and can load argument data units and values correctly.
 
     :return: django reader and data
     :raises: AssertionError
     """
-    params = {'air_temp': {'units': 'celsius'}}
+    params = {'air_temp': {'units': 'celsius', 'extras': {}}}
     meta = type('Meta', (), {'model': MyModel})
     django_reader = DjangoModelReader(params, meta)
     assert isinstance(django_reader, (DataReader, ArgumentReader))
@@ -216,9 +217,9 @@ class DjangoSrcTest1(DataSource):
         data_reader = DjangoModelReader
         data_cache_enabled = False
         # parameters
-        air_temp = {'units': 'celsius'}
-        latitude = {'units': 'degrees'}
-        longitude = {'units': 'degrees'}
+        air_temp = DataParameter(**{'units': 'celsius'})
+        latitude = DataParameter(**{'units': 'degrees'})
+        longitude = DataParameter(**{'units': 'degrees'})
 
         class Meta:
             model = MyModel
@@ -244,9 +245,9 @@ class DjangoSrcTest2(DataSource):
         data_reader = DjangoModelReader
         data_cache_enabled = False
         # parameters
-        air_temp = {'units': 'celsius'}
-        latitude = {'units': 'degrees'}
-        longitude = {'units': 'degrees'}
+        air_temp = DataParameter(**{'units': 'celsius'})
+        latitude = DataParameter(**{'units': 'degrees'})
+        longitude = DataParameter(**{'units': 'degrees'})
 
         class Meta:
             model = MyModel
@@ -298,9 +299,9 @@ def test_hdf5_reader():
     setup_hdf5_test_data()
     # test 1: load data from hdf5 dataset array by node
     params = {
-        'GHI': {'units': 'W/m**2', 'node': '/data/GHI'},
-        'DNI': {'units': 'W/m**2', 'node': '/data/DNI'},
-        'Tdry': {'units': 'degC', 'node': '/data/Tdry'}
+        'GHI': {'units': 'W/m**2', 'extras': {'node': '/data/GHI'}},
+        'DNI': {'units': 'W/m**2', 'extras': {'node': '/data/DNI'}},
+        'Tdry': {'units': 'degC', 'extras': {'node': '/data/Tdry'}}
     }
     reader1 = HDF5Reader(params)
     assert isinstance(reader1, DataReader)
@@ -312,12 +313,12 @@ def test_hdf5_reader():
     assert np.allclose(data1['Tdry'], H5TABLE['DryBulbTemperature'])
     assert data1['Tdry'].units == UREG.degC
     # test 2: load data from hdf5 dataset table by node and member name
-    params['GHI']['node'] = 'data'
-    params['GHI']['member'] = 'GlobalHorizontalRadiation'
-    params['DNI']['node'] = 'data'
-    params['DNI']['member'] = 'DirectNormalRadiation'
-    params['Tdry']['node'] = 'data'
-    params['Tdry']['member'] = 'DryBulbTemperature'
+    params['GHI']['extras']['node'] = 'data'
+    params['GHI']['extras']['member'] = 'GlobalHorizontalRadiation'
+    params['DNI']['extras']['node'] = 'data'
+    params['DNI']['extras']['member'] = 'DirectNormalRadiation'
+    params['Tdry']['extras']['node'] = 'data'
+    params['Tdry']['extras']['member'] = 'DryBulbTemperature'
     reader2 = HDF5Reader(params)
     assert isinstance(reader1, DataReader)
     data2 = reader2.load_data(H5TEST2)
@@ -329,3 +330,11 @@ def test_hdf5_reader():
     assert data1['Tdry'].units == UREG.degC
     teardown_hdf5_test_data()
     return reader1, data1, reader2, data2
+
+
+if __name__ == '__main__':
+    ar, d1 = test_arg_reader()
+    a = test_arg_data_src()
+    dr, d2 = test_django_reader()
+    test_django_data_src()
+    h5r1, h5d1, h5r2, h5d2 = test_hdf5_reader()

carousel/core/__init__.py

@@ -77,24 +77,28 @@ class Registry(dict):
     calling the :func:`super` built-in function.
 
     By default there are no meta attributes, only the register method.
-    To set meta attributes, in a subclass, add them in the constructor::
+    To set meta attributes, in a subclass, set the ``meta_names`` class
+    attribute in the subclass::
 
-        def __init__(self):
-            self.meta1 = {}
-            self.meta2 = {}
-            ...
+        class MyRegistry(Registry):
+            meta_names = ['meta1', 'meta2', ...]
+
+    The ``Registry`` superclass will check that the meta names are not already
+    attributes and then set instance attributes as empty dictionaries in the
+    subclass. To document them, use the class docstring or document them in the
+    documentation API.
     """
+    meta_names = []
+
     def __init__(self):
-        if hasattr(self, 'meta_names'):
-            self.meta_names = _listify(self.meta_names)
-            if [m for m in self.meta_names if m.startswith('_')]:
-                raise AttributeError('No underscores in meta names.')
-            for m in self.meta_names:
-                # check for m in cls and bases
-                if m in dir(Registry):
-                    msg = ('Class %s already has %s member.' %
-                           (self.__class__.__name__, m))
-                    raise AttributeError(msg)
+        self.meta_names = _listify(self.meta_names)  # convert to list
+        for m in self.meta_names:
+            # check for m in cls and bases
+            if m in dir(Registry):
+                msg = ('Class %s already has %s member.' %
+                       (self.__class__.__name__, m))
+                raise AttributeError(msg)
+            setattr(self, m, {})  # create instance attribute and set to dict()
         super(Registry, self).__init__()
 
     def register(self, newitems, *args, **kwargs):
@@ -105,13 +109,10 @@ def register(self, newitems, *args, **kwargs):
             items, keys are not allowed to override existing keys in the
             registry.
         :type newitems: mapping
-        :param args: Key-value pairs of meta-data. The key is the meta-name,
-            and the value is a map of the corresponding meta-data for new
-            item-keys. Each set of meta-keys must be a subset of new item-keys.
-        :type args: tuple or list
+        :param args: Positional arguments with meta data corresponding to order
+            of meta names class attributes
         :param kwargs: Maps of corresponding meta for new keys. Each set of
             meta keys must be a subset of the new item keys.
-        :type kwargs: mapping
         :raises:
             :exc:`~carousel.core.exceptions.DuplicateRegItemError`,
             :exc:`~carousel.core.exceptions.MismatchRegMetaKeysError`
@@ -121,19 +122,13 @@ def register(self, newitems, *args, **kwargs):
             raise DuplicateRegItemError(self.viewkeys() & newkeys)
         self.update(newitems)  # register new item
         # update meta fields
-        if any(isinstance(_, dict) for _ in args):
-            # don't allow kwargs to passed as args!
-            raise TypeError('*args should be all named tuples.')
-        # combine the meta args and kwargs together
-        kwargs.update(args)  # doesn't work for combo of dicts and tuples
+        kwargs.update(zip(self.meta_names, args))
         for k, v in kwargs.iteritems():
             meta = getattr(self, k)  # get the meta attribute
             if v:
                 if not v.viewkeys() <= newkeys:
                     raise MismatchRegMetaKeysError(newkeys - v.viewkeys())
                 meta.update(v)  # register meta
-        # TODO: default "tag" meta field for all registries?
-        # TODO: append "meta" to all meta fields, so they're easier to find?
 
     def unregister(self, items):
         """
@@ -258,7 +253,7 @@ def set_param_file_or_parameters(mcs, attr):
             attr['param_file'] = os.path.join(cls_path, cls_file)
         else:
             attr['parameters'] = dict.fromkeys(
-                k for k in attr if not k.startswith('_')
+                k for k, v in attr.iteritems() if isinstance(v, Parameter)
             )
             for k in attr['parameters']:
                 attr['parameters'][k] = attr.pop(k)
@@ -277,3 +272,24 @@ def get_parents(bases, parent):
         :rtype: list
         """
         return [b for b in bases if isinstance(b, parent)]
+
+
+class Parameter(dict):
+    _attrs = []
+
+    def __init__(self, *args, **kwargs):
+        items = dict(zip(self._attrs, args))
+        extras = {}
+        for key, val in kwargs.iteritems():
+            if key in self._attrs:
+                items[key] = val
+            else:
+                extras[key] = val
+                LOGGER.warning('This key: "%s" is not an attribute.', key)
+        super(Parameter, self).__init__(items, extras=extras)
+
+    def __repr__(self):
+        fmt = ('<%s(' % self.__class__.__name__)
+        fmt += ', '.join('%s=%r' % (k, v) for k, v in self.iteritems())
+        fmt += ')>'
+        return fmt