Cleanup chunksize and chunkslice arguments

Author: tritemio

pybromo/diffusion.py

@@ -486,8 +486,7 @@ def concentration(self, pM=False):
                 All the previously stored data in that file will be lost.
         """[1:]
 
-    def _open_store(self, store, prefix='', path='./', chunksize=2**19,
-                    chunkslice='bytes', mode='w'):
+    def _open_store(self, store, prefix='', path='./', mode='w'):
         """Open and setup the on-disk storage file (pytables HDF5 file).
 
         Low level method used to implement different stores.
@@ -498,13 +497,10 @@ def _open_store(self, store, prefix='', path='./', chunksize=2**19,
         Returns:
             Store object.
         """
-        nparams = self.numeric_params
-        self.chunksize = chunksize
-        nparams.update(chunksize=(chunksize, 'Chunksize for arrays'))
         store_fname = '%s_%s.hdf5' % (prefix, self.compact_name())
         attr_params = dict(particles=self.particles.to_json(), box=self.box)
-        kwargs = dict(path=path, nparams=nparams, attr_params=attr_params,
-                      mode=mode)
+        kwargs = dict(path=path, nparams=self.numeric_params,
+                      attr_params=attr_params, mode=mode)
         store_obj = store(store_fname, **kwargs)
         return store_obj
 
@@ -519,8 +515,6 @@ def open_store_traj(self, path='./', chunksize=2**19, chunkslice='bytes',
         self.store = self._open_store(TrajectoryStore,
                                       prefix=ParticlesSimulation._PREFIX_TRAJ,
                                       path=path,
-                                      chunksize=chunksize,
-                                      chunkslice=chunkslice,
                                       mode=mode)
 
         self.psf_pytables = self.psf.to_hdf5(self.store.h5file, '/psf')
@@ -530,13 +524,12 @@ def open_store_traj(self, path='./', chunksize=2**19, chunkslice='bytes',
         self.traj_group = self.store.h5file.root.trajectories
         self.traj_group._v_attrs['psf_name'] = self.psf.fname
 
-        kwargs = dict(chunksize=self.chunksize, chunkslice=chunkslice)
+        kwargs = dict(chunksize=chunksize, chunkslice=chunkslice)
         self.emission_tot = self.store.add_emission_tot(**kwargs)
         self.emission = self.store.add_emission(**kwargs)
         self.position = self.store.add_position(radial=radial, **kwargs)
 
-    def open_store_timestamp(self, path=None, chunksize=2**19,
-                             chunkslice='bytes', mode='w'):
+    def open_store_timestamp(self, path=None, mode='w'):
         """Open and setup the on-disk storage file (pytables HDF5 file).
 
         Arguments:
@@ -553,8 +546,6 @@ def open_store_timestamp(self, path=None, chunksize=2**19,
         self.ts_store = self._open_store(TimestampStore,
                                          prefix=ParticlesSimulation._PREFIX_TS,
                                          path=path,
-                                         chunksize=chunksize,
-                                         chunkslice=chunkslice,
                                          mode=mode)
         self.ts_group = self.ts_store.h5file.root.timestamps
 
@@ -964,7 +955,7 @@ def simulate_timestamps_mix(self, max_rates, populations, bg_rate,
             timeslice (float or None): timestamps are simulated until
                 `timeslice` seconds. If None, simulate until `self.t_max`.
         """
-        self.open_store_timestamp(chunksize=chunksize, path=path)
+        self.open_store_timestamp(path=path)
         rs = self._get_group_randomstate(rs, seed, self.ts_group)
         if t_chunksize is None:
             t_chunksize = self.emission.chunkshape[1]
@@ -1084,7 +1075,7 @@ def simulate_timestamps_mix_da(self, max_rates_d, max_rates_a,
             timeslice (float or None): timestamps are simulated until
                 `timeslice` seconds. If None, simulate until `self.t_max`.
         """
-        self.open_store_timestamp(chunksize=chunksize, path=path)
+        self.open_store_timestamp(path=path)
         rs = self._get_group_randomstate(rs, seed, self.ts_group)
         if t_chunksize is None:
             t_chunksize = self.emission.chunkshape[1]
@@ -1206,7 +1197,7 @@ def simulate_timestamps_mix_da_online(self, max_rates_d, max_rates_a,
             timeslice (float or None): timestamps are simulated until
                 `timeslice` seconds. If None, simulate until `self.t_max`.
         """
-        self.open_store_timestamp(chunksize=chunksize, path=path)
+        self.open_store_timestamp(path=path)
         rs = self._get_group_randomstate(rs, seed, self.ts_group)
         if t_chunksize is None:
             t_chunksize = 2**19

pybromo/storage.py

@@ -40,13 +40,13 @@ def calc_chunkshape(chunksize, shape, kind='bytes'):
         if chunksize is None:
             return None
 
+        divisor = 1
         if kind == 'bytes':
-            divisor = 1
             for dimsize in shape[:-1]:
                 divisor *= dimsize
 
         if len(shape) == 1:
-            chunkshape = (chunksize,)
+            chunkshape = (chunksize / divisor,)
         elif len(shape) == 2:
             chunkshape = (shape[0], chunksize / divisor)
         elif len(shape) == 3:
@@ -89,16 +89,18 @@ def open(self):
     def set_sim_params(self, nparams, attr_params):
         """Store parameters in `params` in `h5file.root.parameters`.
 
-        `nparams` (dict)
-            A dict as returned by `get_params()` in `ParticlesSimulation()`
-            The format is:
-            keys:
-                used as parameter name
-            values: (2-elements tuple)
-                first element is the parameter value
-                second element is a string used as "title" (description)
-        `attr_params` (dict)
-            A dict whole items are stored as attributes in '/parameters'
+        Argument:
+            nparams (dict): a dict of parameters from
+                `ParticlesSimulation().nparams`. The format is:
+
+                keys:
+                    used as parameter name
+                values: (2-elements tuple)
+                    first element is the parameter value
+                    second element is a string used as "title" (description)
+
+            attr_params (dict): dict items will be stored as attributes in
+                '/parameters'
         """
         for name, value in nparams.items():
             val = value[0] if value[0] is not None else 'none'
@@ -112,24 +114,25 @@ def numeric_params(self):
         """Return a dict containing all (key, values) stored in '/parameters'
         """
         nparams = dict()
-        for p in self.h5file.root.parameters:
-            nparams[p.name] = p.read()
+        for par in self.h5file.root.parameters:
+            nparams[par.name] = par.read()
         return nparams
 
     @property
     def numeric_params_meta(self):
         """Return a dict with all parameters and metadata in '/parameters'.
 
-        This returns the same dict format as returned by get_params() method
-        in ParticlesSimulation().
+        This returns the same dict format as `ParticlesSimulation().nparams`.
         """
         nparams = dict()
-        for p in self.h5file.root.parameters:
-            nparams[p.name] = (p.read(), p.title)
+        for par in self.h5file.root.parameters:
+            nparams[par.name] = (par.read(), par.title)
         return nparams
 
 
 class TrajectoryStore(BaseStore):
+    """An on-disk HDF5 store for trajectories.
+    """
     def __init__(self, datafile, path='./', nparams=dict(), attr_params=dict(),
                  mode='r'):
         """Return a new HDF5 file to store simulation results.
@@ -152,9 +155,9 @@ def __init__(self, datafile, path='./', nparams=dict(), attr_params=dict(),
             self.h5file.create_group('/', 'psf', 'PSFs used in the simulation')
 
     def add_trajectory(self, name, overwrite=False, shape=(0,), title='',
-                       chunksize=2**19, comp_filter=default_compression,
-                       atom=tables.Float64Atom(), params=dict(),
-                       chunkslice='bytes'):
+                       chunksize=2**19, chunkslice='bytes',
+                       comp_filter=default_compression,
+                       atom=tables.Float64Atom(), params=dict()):
         """Add an trajectory array in '/trajectories'.
         """
         group = self.h5file.root.trajectories
@@ -186,25 +189,28 @@ def add_trajectory(self, name, overwrite=False, shape=(0,), title='',
         store_array.set_attr('creation_time', current_time())
         return store_array
 
-    def add_emission_tot(self, chunksize=2**19, comp_filter=default_compression,
-                         overwrite=False, params=dict(),
-                         chunkslice='bytes'):
+    def add_emission_tot(self, chunksize=2**19, chunkslice='bytes',
+                         comp_filter=default_compression,
+                         overwrite=False, params=dict()):
         """Add the `emission_tot` array in '/trajectories'.
         """
-        kwargs = dict(overwrite=overwrite, chunksize=chunksize, params=params,
+        kwargs = dict(overwrite=overwrite, params=params,
+                      chunksize=chunksize, chunkslice=chunkslice,
                       comp_filter=comp_filter, atom=tables.Float32Atom(),
                       title='Summed emission trace of all the particles')
         return self.add_trajectory('emission_tot', **kwargs)
 
-    def add_emission(self, chunksize=2**19, comp_filter=default_compression,
-                     overwrite=False, params=dict(), chunkslice='bytes'):
+    def add_emission(self, chunksize=2**19, chunkslice='bytes',
+                     comp_filter=default_compression,
+                     overwrite=False, params=dict()):
         """Add the `emission` array in '/trajectories'.
         """
         nparams = self.numeric_params
         num_particles = nparams['np']
 
         return self.add_trajectory('emission', shape=(num_particles, 0),
                                    overwrite=overwrite, chunksize=chunksize,
+                                   chunkslice=chunkslice,
                                    comp_filter=comp_filter,
                                    atom=tables.Float32Atom(),
                                    title='Emission trace of each particle',
@@ -224,13 +230,16 @@ def add_position(self, radial=False, chunksize=2**19, chunkslice='bytes',
         title = '%s position trace of each particle' % prefix
         return self.add_trajectory(name, shape=(num_particles, ncoords, 0),
                                    overwrite=overwrite, chunksize=chunksize,
+                                   chunkslice=chunkslice,
                                    comp_filter=comp_filter,
                                    atom=tables.Float32Atom(),
                                    title=title,
                                    params=params)
 
 
 class TimestampStore(BaseStore):
+    """An on-disk HDF5 store for timestamps.
+    """
     def __init__(self, datafile, path='./', nparams=dict(), attr_params=dict(),
                  mode='r'):
         """Return a new HDF5 file to store simulation results.
@@ -248,7 +257,6 @@ def __init__(self, datafile, path='./', nparams=dict(), attr_params=dict(),
                          attr_params=attr_params, mode=mode)
         if mode != 'r':
             if 'timestamps' not in self.h5file.root:
-                # Create the groups
                 self.h5file.create_group('/', 'timestamps',
                                          'Simulated timestamps')