Created new module helpers.cache for easy caching

**Caching:**

 - The main `privex.helpers.cache` module exposes a `cached` attribute (`from privex.helpers import cached`) which
   acts as a singleton adapter wrapper, so the global cache adapter can easily be switched out without any risk
   of other parts of your code using the "old" adapter.
 - Created `CacheAdapter` which is an abstract base class designed to set the standard for the cache API
 - Created `MemoryCache` which is a simple caching layer with expiration support that simply stores cache items in memory
   using a static `dict` attribute.
 - Created `RedisCache` which as it sounds, is a cache layer that uses Redis for it's backend. It uses the global Redis from `privex.helpers.plugin` by default,
   however you can pass a `redis.Redis` class instance to it's constructor to use a custom instance instead.

*Unit testing:**

 - Created unit tests for `MemoryCache`
 - Created unit tests for `RedisCache` - baseed on the unit tests for `MemoryCache` to avoid code duplication

**Other Updates:**

 - Added two new functions to `privex.helpers.plugin`
    - `configure_redis` for updating the global redis settings and automatically replacing the Redis instance
    - `reset_redis` to close the current Redis connection, then delete and re-instantiate the Redis connector class.
 - Added two new exceptions `CacheNotFound` and `NotConfigured`
 - Possibly some other small changes

Author: Someguy123

privex/helpers/__init__.py

@@ -66,6 +66,7 @@ def _setup_logging(level=logging.WARNING):
 
 VERSION = '1.3.0.post3'
 
+
 class ImproperlyConfigured(Exception):
     """Placeholder in-case this fails to import from django.core.exceptions"""
     pass
@@ -97,6 +98,7 @@ class AppRegistryNotReady(Exception):
 from privex.helpers.net import *
 from privex.helpers.exceptions import *
 from privex.helpers.plugin import *
+from privex.helpers.cache import CacheNotFound, CacheAdapter, CacheWrapper, MemoryCache, RedisCache, cached
 
 try:
     from privex.helpers.asyncx import *

privex/helpers/cache/CacheAdapter.py

@@ -0,0 +1,144 @@
+from abc import ABC, abstractmethod
+from typing import Any, Optional, Union
+
+from privex.helpers.exceptions import CacheNotFound
+from privex.helpers.settings import DEFAULT_CACHE_TIMEOUT
+
+
+class CacheAdapter(ABC):
+    """
+    **CacheAdapter** is an abstract base class which scaffolds methods for implementing a Cache, allowing for
+    consistent methods and method signatures across all child classes which implement it.
+    
+    This class cannot be instantiated by itself, only child classes which extend :class:`.CacheAdapter` and implement
+    all methods marked with ``@abstractmethod`` can be instantiated.
+    
+    For an example implementation of CacheAdapter, see :class:`privex.helpers.cache.MemoryCache`
+    
+    """
+    
+    def __init__(self, *args, **kwargs):
+        pass
+    
+    @abstractmethod
+    def get(self, key: str, default: Any = None, fail: bool = False) -> Any:
+        """
+        Return the value of cache key ``key``. If the key wasn't found, or it was expired, then ``default`` will be
+        returned.
+        
+        Optionally, you may choose to pass ``fail=True``, which will cause this method to raise :class:`.CacheNotFound`
+        instead of returning ``default`` when a key is non-existent / expired.
+        
+        :param str key: The cache key (as a string) to get the value for, e.g. ``example:test``
+        :param Any default: If the cache key ``key`` isn't found / is expired, return this value (Default: ``None``)
+        :param bool fail: If set to ``True``, will raise :class:`.CacheNotFound` instead of returning ``default``
+                          when a key is non-existent / expired.
+        
+        :raises CacheNotFound: Raised when ``fail=True`` and ``key`` was not found in cache / expired.
+        
+        :return Any value: The value of the cache key ``key``, or ``default`` if it wasn't found.
+        """
+        raise NotImplemented(f'{self.__class__.__name__} must implement .get()')
+
+    @abstractmethod
+    def set(self, key: str, value: Any, timeout: Optional[int] = DEFAULT_CACHE_TIMEOUT):
+        """
+        Set the cache key ``key`` to the value ``value``, and automatically expire the key after ``timeout`` seconds
+        from now.
+        
+        If ``timeout`` is ``None``, then the key will never expire (unless the cache implementation loses it's
+        persistence, e.g. memory caches with no disk writes).
+        
+        :param str key: The cache key (as a string) to set the value for, e.g. ``example:test``
+        :param Any value: The value to store in the cache key ``key``
+        :param int timeout: The amount of seconds to keep the data in cache. Pass ``None`` to disable expiration.
+        """
+        raise NotImplemented(f'{self.__class__.__name__} must implement .set()')
+
+    
+    @abstractmethod
+    def remove(self, key: str) -> bool:
+        """
+        Remove a key from the cache.
+        
+        If the cache key existed before removal, ``True`` will be returned. If it didn't exist (and thus couldn't
+        remove), then ``False`` will be returned.
+        
+        :param str key: The cache key to remove
+        :return bool removed: ``True`` if ``key`` existed and was removed
+        :return bool removed: ``False`` if ``key`` didn't exist, and no action was taken.
+        """
+        raise NotImplemented(f'{self.__class__.__name__} must implement .remove()')
+
+    @abstractmethod
+    def update_timeout(self, key: str, timeout: int = DEFAULT_CACHE_TIMEOUT) -> Any:
+        """
+        Update the timeout for a given ``key`` to ``datetime.utcnow() + timedelta(seconds=timeout)``
+        
+        This method should accept keys which are already expired, allowing expired cache keys to have their timeout
+        extended **after** expiry.
+        
+        **Example**::
+        
+            >>> c = CacheAdapter()
+            >>> c.set('example', 'test', timeout=60)
+            >>> sleep(70)
+            >>> c.update_timeout('example', timeout=60)   # Reset the timeout for ``'example'`` to ``now + 60 seconds``
+            >>> c.get('example')
+            'test'
+        
+        :param str key: The cache key to update the timeout for
+        :param int timeout: Reset the timeout to this many seconds from ``datetime.utcnow()``
+        :raises CacheNotFound: Raised when ``key`` was not found in cache (thus cannot extend timeout)
+        :return Any value: The value of the cache key
+        """
+        raise NotImplemented(f'{self.__class__.__name__} must implement .extend_timeout()')
+
+    def get_or_set(self, key: str, value: Union[Any, callable], timeout: int = DEFAULT_CACHE_TIMEOUT) -> Any:
+        """
+        Attempt to return the value of ``key`` in the cache. If ``key`` doesn't exist or is expired, then it will be
+        set to ``value``, and ``value`` will be returned.
+
+        The ``value`` parameter can be any standard type such as ``str`` or ``dict`` - or it can be a callable
+        function / method which returns the value to set and return.
+
+        **Basic Usage**::
+
+            >>> c = CacheAdapter()
+            >>> c.get('testing')
+            None
+            >>> c.get_or_set('testing', 'hello world')
+            'hello world'
+            >>> c.get('testing')
+            'hello world'
+
+        **Set and get the value from a function if ``key`` didn't exist / was expired**::
+
+            >>> def my_func(key): return "hello {} world".format(key)
+            >>> c = CacheAdapter()
+            >>> c.get_or_set('example', my_func)
+            'hello example world'
+            >>> c.get('example')
+            'hello example world'
+
+        :param str key: The cache key (as a string) to get/set the value for, e.g. ``example:test``
+        :param Any value: The value to store in the cache key ``key``. Can be a standard type, or a callable function.
+        :param int timeout: The amount of seconds to keep the data in cache. Pass ``None`` to disable expiration.
+        :return Any value: The value of the cache key ``key``, or ``value`` if it wasn't found.
+        """
+        key, timeout = str(key), int(timeout)
+        try:
+            k = self.get(key, fail=True)
+        except CacheNotFound:
+            k = value(key) if callable(value) else value
+            self.set(key=key, value=k, timeout=timeout)
+        return k
+
+    def __getitem__(self, item):
+        try:
+            return self.get(key=item, fail=True)
+        except CacheNotFound:
+            raise KeyError(f'Key "{item}" not found in cache.')
+
+    def __setitem__(self, key, value):
+        return self.set(key=key, value=value)

privex/helpers/cache/MemoryCache.py

@@ -0,0 +1,77 @@
+import logging
+from datetime import datetime, timedelta
+from typing import Any, Optional, Union
+
+from privex.helpers.exceptions import CacheNotFound
+from privex.helpers.settings import DEFAULT_CACHE_TIMEOUT
+from privex.helpers.cache.CacheAdapter import CacheAdapter
+
+log = logging.getLogger(__name__)
+
+
+class MemoryCache(CacheAdapter):
+    """
+    A very basic cache adapter which implements :class:`.CacheAdapter` - stores the cache in memory using
+    the static attribute :py:attr:`.__CACHE`
+    
+    As the cache is simply stored in memory, any python object can be cached without needing any form of serialization.
+    
+    Fully supports cache expiration.
+    
+    **Basic Usage**::
+        
+        >>> from time import sleep
+        >>> c = MemoryCache()
+        >>> c.set('test:example', 'hello world', timeout=60)
+        >>> c.get('test:example')
+        'hello world'
+        >>> sleep(60)
+        >>> c.get('test:example', 'NOT FOUND')
+        'NOT FOUND'
+    
+    """
+    __CACHE = {}
+    
+    def get(self, key: str, default: Any = None, fail: bool = False) -> Any:
+        key = str(key)
+        c = self.__CACHE
+        if key in c:
+            vc = c[key]
+            if str(vc['timeout']) != 'never' and vc['timeout'] < datetime.utcnow():
+                log.debug('Cache key "%s" has expired. Removing from cache.')
+                del c[key]
+                if fail:
+                    raise CacheNotFound(f'Cache key "{key}" was expired.')
+                return default
+            return vc['value']
+        if fail:
+            raise CacheNotFound(f'Cache key "{key}" was not found.')
+        return default
+
+    def set(self, key: str, value: Any, timeout: Optional[int] = DEFAULT_CACHE_TIMEOUT):
+        key, timeout = str(key), int(timeout)
+        c = self.__CACHE
+        c[key] = dict(value=value, timeout=datetime.utcnow() + timedelta(seconds=timeout))
+        return c[key]
+
+    def get_or_set(self, key: str, value: Union[Any, callable], timeout: int = DEFAULT_CACHE_TIMEOUT) -> Any:
+        key, timeout = str(key), int(timeout)
+        try:
+            k = self.get(key, fail=True)
+        except CacheNotFound:
+            k = value(key) if callable(value) else value
+            self.set(key=key, value=k, timeout=timeout)
+        return k
+    
+    def remove(self, key: str) -> bool:
+        key = str(key)
+        if key in self.__CACHE:
+            del self.__CACHE[key]
+            return True
+        return False
+
+    def update_timeout(self, key: str, timeout: int = DEFAULT_CACHE_TIMEOUT) -> Any:
+        key, timeout = str(key), int(timeout)
+        v = self.get(key=key, fail=True)
+        self.__CACHE[key]['timeout'] = datetime.utcnow() + timedelta(seconds=timeout)
+        return v

privex/helpers/cache/RedisCache.py

@@ -0,0 +1,120 @@
+import pickle
+from typing import Any, Union, Optional
+
+from privex.helpers.common import empty
+
+from privex.helpers import plugin
+from privex.helpers.cache.CacheAdapter import CacheAdapter
+from privex.helpers.exceptions import CacheNotFound
+from privex.helpers.settings import DEFAULT_CACHE_TIMEOUT
+
+
+if plugin.HAS_REDIS:
+    from privex.helpers.plugin import get_redis
+    import redis
+    
+    class RedisCache(CacheAdapter):
+        """
+        An Redis backed implementation of :class:`.CacheAdapter`. Uses the global Redis instance from
+        :py:mod:`privex.helpers.plugin` by default, however custom Redis instances can be passed in via
+        the constructor argument ``redis_instance``.
+        
+        To allow for a wide variety of Python objects to be safely stored and retrieved from Redis, this class
+        uses the :py:mod:`pickle` module for serialising + un-serialising values to/from Redis.
+        
+        **Basic Usage**::
+        
+            >>> from privex.helpers import RedisCache
+            >>> rc = RedisCache()
+            >>> rc.set('hello', 'world')
+            >>> rc['hello']
+            'world'
+        
+        **Disabling Pickling**::
+        
+        In some cases, you may need interoperable caching with other languages. The :py:mod:`pickle` serialisation
+        technique is extremely specific to Python and is largely unsupported outside of Python. Thus if you need
+        to share Redis cache data with applications in other languages, then you must disable pickling.
+        
+        **WARNING:** If you disable pickling, then you must perform your own serialisation + de-serialization on
+        complex objects such as ``dict``, ``list``, ``Decimal``, or arbitrary classes/functions after getting
+        or setting cache keys.
+        
+        **Disabling Pickle per instance**::
+        
+        Pass ``use_pickle=False`` to the constructor, or access the attribute directly to disable pickling for a
+        single instance of RedisCache (not globally)
+        
+            >>> rc = RedisCache(use_pickle=False)  # Opt 1. Disable pickle in constructor
+            >>> rc.use_pickle = False              # Opt 2. Disable pickle on an existing instance
+        
+        **Disabling Pickle by default on any new instances**::
+        
+        Change the static attribute :py:attr:`.pickle_default` to ``False`` to disable the use of pickle by default
+        across any new instances of RedisCache.
+        
+            >>> RedisCache.pickle_default = False
+        
+        """
+        
+        pickle_default: bool = True
+        """
+        Change this to ``False`` to disable the use of :py:mod:`pickle` by default for any new instances
+        of this class.
+        """
+        
+        use_pickle: bool
+        """If ``True``, will use :py:mod:`pickle` for serializing objects before inserting into Redis, and
+        un-serialising objects retrieved from Redis. This attribute is set in :py:meth:`.__init__`.
+        
+        Change this to ``False`` to disable the use of :py:mod:`pickle` - instead values will be passed to / returned
+        from Redis as-is, with no serialisation (this may require you to manually serialize complex types such
+        as ``dict`` and ``Decimal`` before insertion, and un-serialise after retrieval).
+        """
+        
+        def __init__(self, use_pickle: bool = None, redis_instance: redis.Redis = None, *args, **kwargs):
+            """
+            RedisCache by default uses the global Redis instance from :py:mod:`privex.helpers.plugin`.
+            
+            It's recommended to use :py:func:`privex.helpers.plugin.configure_redis` if you need to change any
+            Redis settings, as this will adjust the global settings and re-instantiate the global instance if required.
+            
+            Alternatively, you may pass an instance of :class:`redis.Redis` as ``redis_instance``, then that will
+            be used instead of the global instance from :py:func:`.get_redis`
+            
+            :param bool use_pickle: (Default: ``True``) Use the built-in ``pickle`` to serialise values before
+                                    storing in Redis, and un-serialise when loading from Redis
+            
+            :param redis.Redis redis_instance: If this isn't ``None`` / ``False``, then this Redis instance will be
+                                               used instead of the global one from :py:func:`.get_redis`
+            
+            """
+            super().__init__(*args, **kwargs)
+            self.redis = get_redis() if not redis_instance else redis_instance
+            self.use_pickle = self.pickle_default if use_pickle is None else use_pickle
+        
+        def get(self, key: str, default: Any = None, fail: bool = False) -> Any:
+            key = str(key)
+            res = self.redis.get(key)
+            if empty(res):
+                if fail: raise CacheNotFound(f'Cache key "{key}" was not found.')
+                return default
+            return pickle.loads(res) if self.use_pickle else res
+
+        def set(self, key: str, value: Any, timeout: Optional[int] = DEFAULT_CACHE_TIMEOUT):
+            v = pickle.dumps(value) if self.use_pickle else value
+            return self.redis.set(str(key), v, ex=timeout)
+
+        def remove(self, key: str) -> bool:
+            try:
+                self.get(key, fail=True)
+                self.redis.delete(key)
+                return True
+            except CacheNotFound:
+                return False
+
+        def update_timeout(self, key: str, timeout: int = DEFAULT_CACHE_TIMEOUT) -> Any:
+            key, timeout = str(key), int(timeout)
+            v = self.get(key=key, fail=True)
+            self.set(key=key, value=v, timeout=timeout)
+            return v