Merge pull request #7 from bbayles/new-release

Preparing for a new release (with Python 3 support)

Author: bbayles

.gitignore

@@ -0,0 +1,111 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json

README.md

@@ -1,15 +1,55 @@
-Manipulate DJB's Constant Database files. These are 2 level disk-based hash tables that efficiently handle thousands of keys, while remaining space-efficient.
+## Introduction
 
-http://cr.yp.to/cdb.html
+`python-pure-cdb` (`pure-cdb` on [PyPI](https://pypi.org/project/pure-cdb/)) is a Python library for reading and writing djb's "constant database" files. These disk-based associative arrays that allow efficient storage and retrieval of thousands of keys. For more information on `cdb`, see [djb's page](https://cr.yp.to/cdb.html) and [Wikipedia](https://en.wikipedia.org/wiki/Cdb_(software)). 
 
-Note the Reader class reads the entire CDB into memory.
+### Installation and usage
 
-------
+Install `pure-cdb` with `pip`:
+
+```
+pip install pure-cdb
+```
+
+Then import `cdblib` to access the `Reader` and `Writer` classes.
+
+---
+
+`Reader` provides an interface to a `cdb` file's contents - use the `.iterkeys()` method to iterate over keys, and the `.get()` method to retrieve a key's value.
+
+```python
+import cdblib
+
+with open('info.cdb', 'rb') as f:
+    data = f.read()
+
+reader = cdblib.Reader(data)
+
+reader.get(b'hello')
+```
+
+All keys are `bytes` objects (`str` in Python 2). By default all values are also retrieved as `bytes` objects, but the `.getstring()` and `.getint()` methods can be used to retrieve decoded strings and integers, respectively.
+
+---
+
+`Writer` allows for creating new `cdb` files. Use the `.put()` method to insert a key / value pair, and the `.finalize()` method to create the CDB structure.
+
+```python
+with open('/home/bo/Desktop/new.cdb', 'wb') as f:
+    writer = cdblib.Writer(f)
+    writer.put(b'key', b'value')
+    writer.finalize()
+```
+
+As with the `Reader` class, keys and values are `bytes` objects. The `.putstrings()` and `.putint()` methods can be used to insert encoded text data and integers, respectively.
+
+---
+
+### Remarks on usage
 
 Constant databases have the desirable property of requiring low overhead to open. This makes them ideal for use in environments where it's not always possible to have data hanging around in RAM awaiting use, for example in a CGI script or Google App Engine.
 
 On App Engine, reading a 700kb database into memory from the filesystem costs around 90ms, after which the cost of individual lookups is negligible (one simple benchmark achieved 94k lookups/sec running on App Engine, and that was using `djb_hash()`). This makes CDBs ideal for storing many small keys that are infrequently accessed, for example in per-language corpora containing 10k+ internationalized strings (my original use case).
 
 The format might also be useful as a composite file storage alternative to the `zipfile` module. Since the Reader interface only requires a file-like object, it is possible to wrap a string stored in a Datastore entity in a `cStringIO` object, allowing convenient access to it as a CDB.
 
-When storing many thousands of keys in a Datastore entity, CDBs might enable you to bypass the "indexed fields" limit of the `Expando` class without resorting to `pickle`, as well as reduce deserialization overhead when only a few keys of such an entity are normally accessed and rarely updated.
+When storing many thousands of keys in a Datastore entity, CDBs might enable you to bypass the "indexed fields" limit of the `Expando` class without resorting to `pickle`, as well as reduce deserialization overhead when only a few keys of such an entity are normally accessed and rarely updated.

appengine/cdblib.py

@@ -0,0 +1 @@
+../cdblib/cdblib.py

appengine/cdplib.py

@@ -1 +1 @@
-../testdata
+../tests/testdata

appengine/testdata

@@ -0,0 +1,7 @@
+from __future__ import unicode_literals
+
+from .djb_hash import djb_hash
+from .cdblib import Reader, Reader64, Writer, Writer64
+
+
+__all__ = ['djb_hash', 'Reader', 'Reader64', 'Writer', 'Writer64']

cdblib/__init__.py

@@ -28,7 +28,7 @@ static /*const*/ PyMethodDef methods[] = {
 
 
 PyMODINIT_FUNC
-init_cdblib(void)
+init_djb_hash(void)
 {
-    Py_InitModule("_cdblib", methods);
+    Py_InitModule("_djb_hash", methods);
 }

_cdblib.c cdblib/_djb_hash.c_cdblib.c renamed to cdblib/_djb_hash.c

@@ -5,22 +5,16 @@
     http://cr.yp.to/cdb.html
 
 '''
+from __future__ import unicode_literals
 
-from _struct import Struct
+from struct import Struct
 from itertools import chain
 
+import six
+from six.moves import range
 
-def py_djb_hash(s):
-    '''Return the value of DJB's hash function for the given 8-bit string.'''
-    h = 5381
-    for c in s:
-        h = (((h << 5) + h) ^ ord(c)) & 0xffffffff
-    return h
+from .djb_hash import djb_hash
 
-try:
-    from _cdblib import djb_hash
-except ImportError:
-    djb_hash = py_djb_hash
 
 read_2_le4 = Struct('<LL').unpack
 read_2_le8 = Struct('<QQ').unpack
@@ -44,7 +38,7 @@ def __init__(self, data, hashfn=djb_hash):
         self.data = data
         self.hashfn = hashfn
         self.index = [self.read_pair(data[i:i+self.pair_size])
-                      for i in xrange(0, 256*self.pair_size, self.pair_size)]
+                      for i in range(0, 256*self.pair_size, self.pair_size)]
         self.table_start = min(p[0] for p in self.index)
         # Assume load load factor is 0.5 like official CDB.
         self.length = sum(p[1] >> 1 for p in self.index)
@@ -111,8 +105,8 @@ def gets(self, key):
             end = start + (nslots * self.pair_size)
             slot_off = start + (((h >> 8) % nslots) * self.pair_size)
 
-            for pos in chain(xrange(slot_off, end, self.pair_size),
-                             xrange(start, slot_off, self.pair_size)):
+            for pos in chain(range(slot_off, end, self.pair_size),
+                             range(start, slot_off, self.pair_size)):
                 rec_h, rec_pos = self.read_pair(
                     self.data[pos:pos+self.pair_size]
                 )
@@ -132,7 +126,7 @@ def gets(self, key):
     def get(self, key, default=None):
         '''Get the first value for key, returning default if missing.'''
         # Avoid exception catch when handling default case; much faster.
-        return chain(self.gets(key), (default,)).next()
+        return next(chain(self.gets(key), (default,)))
 
     def getint(self, key, default=None, base=0):
         '''Get the first value for key converted it to an int, returning
@@ -182,12 +176,12 @@ def __init__(self, fp, hashfn=djb_hash):
         self.fp = fp
         self.hashfn = hashfn
 
-        fp.write('\x00' * (256 * self.pair_size))
-        self._unordered = [[] for i in xrange(256)]
+        fp.write(b'\x00' * (256 * self.pair_size))
+        self._unordered = [[] for i in range(256)]
 
-    def put(self, key, value=''):
+    def put(self, key, value=b''):
         '''Write a string key/value pair to the output file.'''
-        assert type(key) is str and type(value) is str
+        assert type(key) is six.binary_type and type(value) is six.binary_type
 
         pos = self.fp.tell()
         self.fp.write(self.write_pair(len(key), len(value)))
@@ -206,22 +200,22 @@ def puts(self, key, values):
     def putint(self, key, value):
         '''Write an integer as a base-10 string associated with the given key
         to the output file.'''
-        self.put(key, str(value))
+        self.put(key, str(value).encode('ascii'))
 
     def putints(self, key, values):
         '''Write zero or more integers for the same key to the output file.
         Equivalent to calling putint() in a loop.'''
-        self.puts(key, (str(value) for value in values))
+        self.puts(key, (str(value).encode('ascii') for value in values))
 
     def putstring(self, key, value, encoding='utf-8'):
         '''Write a unicode string associated with the given key to the output
         file after encoding it as UTF-8 or the given encoding.'''
-        self.put(key, unicode.encode(value, encoding))
+        self.put(key, six.text_type.encode(value, encoding))
 
     def putstrings(self, key, values, encoding='utf-8'):
         '''Write zero or more unicode strings to the output file. Equivalent to
         calling putstring() in a loop.'''
-        self.puts(key, (unicode.encode(value, encoding) for value in values))
+        self.puts(key, (six.text_type.encode(value, encoding) for value in values))
 
     def finalize(self):
         '''Write the final hash tables to the output file, and write out its
@@ -232,7 +226,7 @@ def finalize(self):
             ordered = [(0, 0)] * length
             for pair in tbl:
                 where = (pair[0] >> 8) % length
-                for i in chain(xrange(where, length), xrange(0, where)):
+                for i in chain(range(where, length), range(0, where)):
                     if not ordered[i][0]:
                         ordered[i] = pair
                         break