storage: add Redis storage backend

This change adds Redis as an alternative storage backend.

Implementation notes:
- The database name (which is a number) should be provided in the
connection URI.
- The keys are prefixed with the 'collection' string, so that it is
possible to share the same database with other applications.
- All values are stored as JSON documents ({"value": value}) in order to
try to keep the original data type, since Redis stores all values as bytes.
Note however that it is still possible that the retrieved object differs
from the stored one. (Dictionaries with non-string keys are an example.)
- All values are stored together with a TTL. The TTL is None by default,
which means that the record never expires. The TTL value can be set
during initialisation.

This commit changes the storage tests to use 'fakeredis' and 'mongomock'
instead of trying to launch a real MongoDB server. Expiration tests
have been added, too.

Author: bajnokk

setup.py

@@ -12,6 +12,7 @@
     description='OpenID Connect Provider (OP) library in Python.',
     install_requires=[
         'oic >= 1.2.1',
-        'pymongo'
+        'pymongo',
+        'redis'
     ]
 )

src/pyop/storage.py

@@ -1,11 +1,59 @@
 # -*- coding: utf-8 -*-
 
+from abc import ABC, abstractmethod
 import copy
+import json
 import pymongo
+from redis.client import Redis
 from time import time
 
 
-class MongoWrapper(object):
+class StorageBase(ABC):
+    _ttl = None
+
+    @abstractmethod
+    def __setitem__(self, key, value):
+        pass
+
+    @abstractmethod
+    def __getitem__(self, key):
+        pass
+
+    @abstractmethod
+    def __delitem__(self, key):
+        pass
+
+    @abstractmethod
+    def __contains__(self, key):
+        pass
+
+    @abstractmethod
+    def items(self):
+        pass
+
+    def pop(self, key, default=None):
+        try:
+            data = self[key]
+        except KeyError:
+            return default
+        del self[key]
+        return data
+
+    @classmethod
+    def from_uri(cls, db_uri, collection, db_name=None, ttl=None):
+        if db_uri.startswith("mongodb"):
+            return MongoWrapper(db_uri, db_name, collection, ttl)
+        if db_uri.startswith("redis") or db_uri.startswith("unix"):
+            return RedisWrapper(db_uri, collection, ttl)
+
+        return ValueError(f"Invalid DB URI: {db_uri}")
+
+    @property
+    def ttl(self):
+        return self._ttl
+
+
+class MongoWrapper(StorageBase):
     def __init__(self, db_uri, db_name, collection):
         self._db_uri = db_uri
         self._coll_name = collection
@@ -38,13 +86,52 @@ def items(self):
         for doc in self._coll.find():
             yield (doc['lookup_key'], doc['data'])
 
-    def pop(self, key, default=None):
-        try:
-            data = self[key]
-        except KeyError:
-            return default
-        del self[key]
-        return data
+
+class RedisWrapper(StorageBase):
+    """
+    Simple wrapper for a dict-like storage in Redis.
+    Supports JSON-serializable data types.
+    """
+
+    def __init__(self, db_uri, collection, ttl=None):
+        self._db = Redis.from_url(db_uri, decode_responses=True)
+        self._collection = collection
+        if ttl is None or (isinstance(ttl, int) and ttl >= 0):
+            self._ttl = ttl
+        else:
+            raise ValueError("TTL must be a non-negative integer or None")
+
+    def _make_key(self, key):
+        if not isinstance(key, str):
+            raise TypeError(f"Keys must be strings, {type(key).__name__} given")
+
+        return ":".join([self._collection, key])
+
+    def __setitem__(self, key, value):
+        # Replacing the value of a key resets the ttl counter
+        encoded = json.dumps({ "value": value })
+        self._db.set(self._make_key(key), encoded, ex=self.ttl)
+
+    def __getitem__(self, key):
+        encoded = self._db.get(self._make_key(key))
+        if encoded is None:
+            raise KeyError(key)
+        return json.loads(encoded).get("value")
+
+    def __delitem__(self, key):
+        # Deleting a non-existent key is allowed
+        self._db.delete(self._make_key(key))
+
+    def __contains__(self, key):
+        return (self._db.get(self._make_key(key)) is not None)
+
+    def items(self):
+        for key in self._db.keys(self._collection + "*"):
+            visible_key = key[len(self._collection) + 1 :]
+            try:
+                yield (visible_key, self[visible_key])
+            except KeyError:
+                pass
 
 
 class MongoDB(object):

tests/pyop/conftest.py

@@ -2,15 +2,37 @@
 
 import pytest
 
-from pyop.storage import MongoWrapper
+from abc import ABC, abstractmethod
+from contextlib import contextmanager
+from redis.client import Redis
+import datetime
+import fakeredis
+import mongomock
+import pymongo
+import time
+
+from pyop.storage import StorageBase
 
 __author__ = 'lundberg'
 
 
-class TestMongoStorage(object):
-    @pytest.fixture()
-    def db(self, mongodb_instance):
-        return MongoWrapper(mongodb_instance.get_uri(), 'pyop', 'test')
+uri_list = ["mongodb://localhost:1234/pyop", "redis://localhost/0"]
+
+@pytest.fixture(autouse=True)
+def mock_redis(monkeypatch):
+    def mockreturn(*args, **kwargs):
+        return fakeredis.FakeStrictRedis(decode_responses=True)
+    monkeypatch.setattr(Redis, "from_url", mockreturn)
+
+@pytest.fixture(autouse=True)
+def mock_mongo():
+    pymongo.MongoClient = mongomock.MongoClient
+
+
+class TestStorage(object):
+    @pytest.fixture(params=uri_list)
+    def db(self, request):
+        return StorageBase.from_uri(request.param, db_name="pyop", collection='test')
 
     def test_write(self, db):
         db['foo'] = 'bar'
@@ -41,3 +63,108 @@ def test_items(self, db):
         for key, item in db.items():
             assert key
             assert item
+
+    @pytest.mark.parametrize(
+        "args,kwargs",
+        [
+            (["redis://localhost"], {"collection": "test"}),
+            (["redis://localhost", "test"], {}),
+            (["unix://localhost/0"], {"collection": "test", "ttl": 3}),
+            (["mongodb://localhost/pyop"], {"collection": "test", "ttl": 3}),
+            (["mongodb://localhost"], {"db_name": "pyop", "collection": "test"}),
+            (["mongodb://localhost", "test", "pyop"], {}),
+            (["mongodb://localhost/pyop", "test"], {}),
+            (["mongodb://localhost/pyop"], {"db_name": "other", "collection": "test"}),
+            (["redis://localhost/0"], {"db_name": "pyop", "collection": "test"}),
+        ],
+    )
+    def test_from_uri(self, args, kwargs):
+        store = StorageBase.from_uri(*args, **kwargs)
+        store["test"] = "value"
+        assert store["test"] == "value"
+
+    @pytest.mark.parametrize(
+        "error,args,kwargs",
+        [
+            (
+                TypeError,
+                ["redis://localhost", "ouch"],
+                {"db_name": 3, "collection": "test", "ttl": None},
+            ),
+            (
+                TypeError,
+                ["mongodb://localhost", "ouch"],
+                {"db_name": 3, "collection": "test", "ttl": None},
+            ),
+            (
+                TypeError,
+                ["mongodb://localhost", "ouch"],
+                {"db_name": "pyop", "collection": "test", "ttl": None},
+            ),
+            (
+                TypeError,
+                ["mongodb://localhost", "pyop"],
+                {"collection": "test", "ttl": None},
+            ),
+            (
+                TypeError,
+                ["mongodb://localhost"],
+                {"db_name": "pyop", "collection": "test", "ttl": None, "extra": True},
+            ),
+            (TypeError, ["redis://localhost/0"], {}),
+            (TypeError, ["redis://localhost/0"], {"db_name": "pyop"}),
+            (ValueError, ["mongodb://localhost"], {"collection": "test", "ttl": None}),
+        ],
+    )
+    def test_from_uri_invalid_parameters(self, error, args, kwargs):
+        with pytest.raises(error):
+            StorageBase.from_uri(*args, **kwargs)
+
+
+class StorageTTLTest(ABC):
+    def prepare_db(self, uri, ttl):
+        self.db = StorageBase.from_uri(
+            uri,
+            collection="test",
+            ttl=ttl,
+        )
+        self.db["foo"] = {"bar": "baz"}
+
+    @abstractmethod
+    def set_time(self, offset, monkey):
+        pass
+
+    @contextmanager
+    def adjust_time(self, offset):
+        mp = pytest.MonkeyPatch()
+        try:
+            yield self.set_time(offset, mp)
+        finally:
+            mp.undo()
+
+    def execute_ttl_test(self, uri, ttl):
+        self.prepare_db(uri, ttl)
+        assert self.db["foo"]
+        with self.adjust_time(offset=int(ttl / 2)):
+            assert self.db["foo"]
+        with self.adjust_time(offset=int(ttl * 2)):
+            with pytest.raises(KeyError):
+                self.db["foo"]
+
+    @pytest.mark.parametrize("uri", uri_list)
+    @pytest.mark.parametrize("ttl", ["invalid", -1, 2.3, {}])
+    def test_invalid_ttl(self, uri, ttl):
+        with pytest.raises(ValueError):
+            self.prepare_db(uri, ttl)
+
+
+class TestRedisTTL(StorageTTLTest):
+    def set_time(self, offset, monkeypatch):
+        now = time.time()
+        def new_time():
+            return now + offset
+
+        monkeypatch.setattr(time, "time", new_time)
+
+    def test_ttl(self):
+        self.execute_ttl_test("redis://localhost/0", 3600)

tests/pyop/test_storage.py

@@ -1,3 +1,6 @@
-pytest
+pytest >= 6.2
+pip >= 19.0
 responses
 pycryptodomex
+fakeredis
+mongomock