From 977c4a5b5014632f6b316d7193bf2ee087f5d21f Mon Sep 17 00:00:00 2001 From: Millefeuille Date: Wed, 20 Mar 2024 21:17:23 +0100 Subject: [PATCH] Add support of redis clusters (#236) * Add support of redis clusters (#1) Add Redis cluster support as session management backend --- beaker/cache.py | 4 +- beaker/docs/modules/rediscluster.rst | 10 +++ beaker/ext/redisclusternm.py | 87 ++++++++++++++++++++ tests/test_managers/test_ext_rediscluster.py | 16 ++++ 4 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 beaker/docs/modules/rediscluster.rst create mode 100644 beaker/ext/redisclusternm.py create mode 100644 tests/test_managers/test_ext_rediscluster.py diff --git a/beaker/cache.py b/beaker/cache.py index 5a1ad6a4..61b8e4e9 100644 --- a/beaker/cache.py +++ b/beaker/cache.py @@ -22,6 +22,7 @@ import beaker.ext.google as google import beaker.ext.mongodb as mongodb import beaker.ext.redisnm as redisnm +import beaker.ext.redisclusternm as redisclusternm from functools import wraps # Initialize the cache region dict @@ -126,7 +127,8 @@ def _init(self): 'ext:sqla': sqla.SqlaNamespaceManager, 'ext:google': google.GoogleNamespaceManager, 'ext:mongodb': mongodb.MongoNamespaceManager, - 'ext:redis': redisnm.RedisNamespaceManager + 'ext:redis': redisnm.RedisNamespaceManager, + 'ext:rediscluster': redisclusternm.RedisClusterNamespaceManager, }) diff --git a/beaker/docs/modules/rediscluster.rst b/beaker/docs/modules/rediscluster.rst new file mode 100644 index 00000000..5546656f --- /dev/null +++ b/beaker/docs/modules/rediscluster.rst @@ -0,0 +1,10 @@ +:mod:`beaker.ext.redisclusternm` -- Redis cluster NameSpace Manager and Synchronizer +============================================================================== + +.. automodule:: beaker.ext.redisclusternm + +Module Contents +--------------- + +.. autoclass:: RedisClusterNamespaceManager +.. autoclass:: RedisClusterSynchronizer diff --git a/beaker/ext/redisclusternm.py b/beaker/ext/redisclusternm.py new file mode 100644 index 00000000..0024ae89 --- /dev/null +++ b/beaker/ext/redisclusternm.py @@ -0,0 +1,87 @@ +import os +import threading +import time +import pickle + +from beaker.container import NamespaceManager + +try: + import redis +except ImportError: + redis = None + +from beaker.ext.redisnm import RedisNamespaceManager, RedisSynchronizer +from beaker._compat import string_type + + +class RedisClusterNamespaceManager(RedisNamespaceManager): + """Provides the :class:`.NamespaceManager` API over Redis cluster. + + Provided ``urls`` can be both multiple redis connection strings separated by a comma or + an already existing RedisCluster instance. + + Unlike a StrictRedis connection string, a RedisCluster one does not support + database indicators, it is zero by default. + + Example: `redis://node-1:7001,redis://node-2:7002` + + Additional options can be passed in kwargs (e.g. `username="redis", password="secure_password"`). + + The data will be stored into redis keys, with their name + starting with ``beaker_cache:``. + """ + + def __init__(self, namespace, urls, timeout=None, **kwargs): + super(RedisNamespaceManager, self).__init__(namespace) + self.lock_dir = None # Redis uses redis itself for locking. + self.timeout = timeout + self.nodes = [] + self.options = kwargs + + if redis is None: + raise RuntimeError('redis is not available') + + if isinstance(urls, string_type): + for url in urls.split(','): + url_options = redis.connection.parse_url(url) + if 'db' in url_options: + raise redis.exceptions.RedisClusterException( + "A ``db`` querystring option can only be 0 in cluster mode" + ) + self.nodes.append(redis.cluster.ClusterNode( + host=url_options.get('host'), + port=url_options.get('port') + )) + self.client = RedisClusterNamespaceManager.clients.get( + urls, redis.cluster.RedisCluster, startup_nodes=self.nodes, **kwargs + ) + else: + self.client = urls + + def get_creation_lock(self, key): + return RedisClusterSynchronizer(self._format_key(key), self.client, self.nodes, **self.options) + + +class RedisClusterSynchronizer(RedisSynchronizer): + """Synchronizer based on redis cluster. + + Provided ``urls`` can be both multiple redis connection strings separated by a comma or + an already existing RedisCluster instance. + + Unlike a StrictRedis connection string, a RedisCluster one does not support + database indicators, it is zero by default. + + Example: ``redis://node-1:7001,redis://node-2:7002, + + This Synchronizer only supports 1 reader or 1 writer at time, not concurrent readers. + """ + + def __init__(self, identifier, urls, nodes=None, **kwargs): + super(RedisSynchronizer, self).__init__() + self.identifier = 'beaker_lock:%s' % identifier + if isinstance(urls, string_type): + self.client = RedisClusterNamespaceManager.clients.get( + urls, redis.cluster.RedisCluster, startup_nodes=nodes, **kwargs + ) + else: + self.client = urls diff --git a/tests/test_managers/test_ext_rediscluster.py b/tests/test_managers/test_ext_rediscluster.py new file mode 100644 index 00000000..5c19d549 --- /dev/null +++ b/tests/test_managers/test_ext_rediscluster.py @@ -0,0 +1,16 @@ +from beaker.cache import Cache +from . import base + + +class TestRedis(base.CacheManagerBaseTests): + CACHE_ARGS = { + 'type': 'ext:rediscluster', + 'url': 'redis://localhost:6379' + } + + def test_client_reuse(self): + cache1 = Cache('test1', **self.CACHE_ARGS) + cli1 = cache1.namespace.client + cache2 = Cache('test2', **self.CACHE_ARGS) + cli2 = cache2.namespace.client + self.assertTrue(cli1 is cli2) \ No newline at end of file