Add support of redis clusters (#236)

* Add support of redis clusters (#1)

Add Redis cluster support as session management backend

Author: Millefeuille42

beaker/cache.py

@@ -22,6 +22,7 @@
 import beaker.ext.google as google
 import beaker.ext.mongodb as mongodb
 import beaker.ext.redisnm as redisnm
+import beaker.ext.redisclusternm as redisclusternm
 from functools import wraps
 
 # Initialize the cache region dict
@@ -126,7 +127,8 @@ def _init(self):
     'ext:sqla': sqla.SqlaNamespaceManager,
     'ext:google': google.GoogleNamespaceManager,
     'ext:mongodb': mongodb.MongoNamespaceManager,
-    'ext:redis': redisnm.RedisNamespaceManager
+    'ext:redis': redisnm.RedisNamespaceManager,
+    'ext:rediscluster': redisclusternm.RedisClusterNamespaceManager,
 })
 
 

beaker/docs/modules/rediscluster.rst

@@ -0,0 +1,10 @@
+:mod:`beaker.ext.redisclusternm` -- Redis cluster NameSpace Manager and Synchronizer
+==============================================================================
+
+.. automodule:: beaker.ext.redisclusternm
+
+Module Contents
+---------------
+
+.. autoclass:: RedisClusterNamespaceManager
+.. autoclass:: RedisClusterSynchronizer

beaker/ext/redisclusternm.py

@@ -0,0 +1,87 @@
+import os
+import threading
+import time
+import pickle
+
+from beaker.container import NamespaceManager
+
+try:
+    import redis
+except ImportError:
+    redis = None
+
+from beaker.ext.redisnm import RedisNamespaceManager, RedisSynchronizer
+from beaker._compat import string_type
+
+
+class RedisClusterNamespaceManager(RedisNamespaceManager):
+    """Provides the :class:`.NamespaceManager` API over Redis cluster.
+
+    Provided ``urls`` can be both multiple redis connection strings separated by a comma or
+    an already existing RedisCluster instance.
+
+    Unlike a StrictRedis connection string, a RedisCluster one does not support
+    database indicators, it is zero by default.
+
+    Example: `redis://node-1:7001,redis://node-2:7002`
+
+    Additional options can be passed in kwargs (e.g. `username="redis", password="secure_password"`).
+
+    The data will be stored into redis keys, with their name
+    starting with ``beaker_cache:``.
+    """
+
+    def __init__(self, namespace, urls, timeout=None, **kwargs):
+        super(RedisNamespaceManager, self).__init__(namespace)
+        self.lock_dir = None  # Redis uses redis itself for locking.
+        self.timeout = timeout
+        self.nodes = []
+        self.options = kwargs
+
+        if redis is None:
+            raise RuntimeError('redis is not available')
+
+        if isinstance(urls, string_type):
+            for url in urls.split(','):
+                url_options = redis.connection.parse_url(url)
+                if 'db' in url_options:
+                    raise redis.exceptions.RedisClusterException(
+                        "A ``db`` querystring option can only be 0 in cluster mode"
+                    )
+                self.nodes.append(redis.cluster.ClusterNode(
+                    host=url_options.get('host'),
+                    port=url_options.get('port')
+                ))
+            self.client = RedisClusterNamespaceManager.clients.get(
+                urls, redis.cluster.RedisCluster, startup_nodes=self.nodes, **kwargs
+            )
+        else:
+            self.client = urls
+
+    def get_creation_lock(self, key):
+        return RedisClusterSynchronizer(self._format_key(key), self.client, self.nodes, **self.options)
+
+
+class RedisClusterSynchronizer(RedisSynchronizer):
+    """Synchronizer based on redis cluster.
+
+    Provided ``urls`` can be both multiple redis connection strings separated by a comma or
+    an already existing RedisCluster instance.
+
+    Unlike a StrictRedis connection string, a RedisCluster one does not support
+    database indicators, it is zero by default.
+
+    Example: ``redis://node-1:7001,redis://node-2:7002,
+
+    This Synchronizer only supports 1 reader or 1 writer at time, not concurrent readers.
+    """
+
+    def __init__(self, identifier, urls, nodes=None, **kwargs):
+        super(RedisSynchronizer, self).__init__()
+        self.identifier = 'beaker_lock:%s' % identifier
+        if isinstance(urls, string_type):
+            self.client = RedisClusterNamespaceManager.clients.get(
+                urls, redis.cluster.RedisCluster, startup_nodes=nodes, **kwargs
+            )
+        else:
+            self.client = urls

tests/test_managers/test_ext_rediscluster.py

@@ -0,0 +1,16 @@
+from beaker.cache import Cache
+from . import base
+
+
+class TestRedis(base.CacheManagerBaseTests):
+    CACHE_ARGS = {
+        'type': 'ext:rediscluster',
+        'url': 'redis://localhost:6379'
+    }
+
+    def test_client_reuse(self):
+        cache1 = Cache('test1', **self.CACHE_ARGS)
+        cli1 = cache1.namespace.client
+        cache2 = Cache('test2', **self.CACHE_ARGS)
+        cli2 = cache2.namespace.client
+        self.assertTrue(cli1 is cli2)