From b8fbc6fe9f836e6c262b64581037851b91b77bf8 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Wed, 10 Nov 2021 14:17:27 +0200 Subject: [PATCH 1/9] Fixing multiprocessing tests to respect redisurl ports --- tests/conftest.py | 20 +++++++++++++------- tests/test_connection_pool.py | 17 +++++++++-------- tests/test_multiprocessing.py | 11 +++++++---- tests/test_sentinel.py | 2 +- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b1a0f8cab8..3ba52a5f53 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,7 +11,6 @@ REDIS_INFO = {} default_redis_url = "redis://localhost:6379/9" -default_redismod_url = "redis://localhost:36379/9" default_redismod_url = "redis://localhost:36379" @@ -44,10 +43,13 @@ def pytest_sessionstart(session): REDIS_INFO["version"] = version REDIS_INFO["arch_bits"] = arch_bits - # module info - redismod_url = session.config.getoption("--redismod-url") - info = _get_info(redismod_url) - REDIS_INFO["modules"] = info["modules"] + # module info, if the second redis is running + try: + redismod_url = session.config.getoption("--redismod-url") + info = _get_info(redismod_url) + REDIS_INFO["modules"] = info["modules"] + except redis.exceptions.ConnectionError: + pass def skip_if_server_version_lt(min_version): @@ -72,7 +74,11 @@ def skip_unless_arch_bits(arch_bits): def skip_ifmodversion_lt(min_version: str, module_name: str): - modules = REDIS_INFO["modules"] + try: + modules = REDIS_INFO["modules"] + except KeyError: + return pytest.mark.skipif(True, + reason="No redismod-url server running") if modules == []: return pytest.mark.skipif(True, reason="No redis modules found") @@ -218,7 +224,7 @@ def mock_cluster_resp_slaves(request, **kwargs): def master_host(request): url = request.config.getoption("--redis-url") parts = urlparse(url) - yield parts.hostname + yield parts.hostname, parts.port def wait_for_command(client, monitor, command): diff --git a/tests/test_connection_pool.py b/tests/test_connection_pool.py index 8d2ad041a0..6fedec66fb 100644 --- a/tests/test_connection_pool.py +++ b/tests/test_connection_pool.py @@ -44,14 +44,14 @@ def test_connection_creation(self): assert connection.kwargs == connection_kwargs def test_multiple_connections(self, master_host): - connection_kwargs = {'host': master_host} + connection_kwargs = {'host': master_host[0], 'port': master_host[1]} pool = self.get_pool(connection_kwargs=connection_kwargs) c1 = pool.get_connection('_') c2 = pool.get_connection('_') assert c1 != c2 def test_max_connections(self, master_host): - connection_kwargs = {'host': master_host} + connection_kwargs = {'host': master_host[0], 'port': master_host[1]} pool = self.get_pool(max_connections=2, connection_kwargs=connection_kwargs) pool.get_connection('_') @@ -60,7 +60,7 @@ def test_max_connections(self, master_host): pool.get_connection('_') def test_reuse_previously_released_connection(self, master_host): - connection_kwargs = {'host': master_host} + connection_kwargs = {'host': master_host[0], 'port': master_host[1]} pool = self.get_pool(connection_kwargs=connection_kwargs) c1 = pool.get_connection('_') pool.release(c1) @@ -103,14 +103,15 @@ def get_pool(self, connection_kwargs=None, max_connections=10, timeout=20): return pool def test_connection_creation(self, master_host): - connection_kwargs = {'foo': 'bar', 'biz': 'baz', 'host': master_host} + connection_kwargs = {'foo': 'bar', 'biz': 'baz', + 'host': master_host[0], 'port': master_host[1]} pool = self.get_pool(connection_kwargs=connection_kwargs) connection = pool.get_connection('_') assert isinstance(connection, DummyConnection) assert connection.kwargs == connection_kwargs def test_multiple_connections(self, master_host): - connection_kwargs = {'host': master_host} + connection_kwargs = {'host': master_host[0], 'port': master_host[1]} pool = self.get_pool(connection_kwargs=connection_kwargs) c1 = pool.get_connection('_') c2 = pool.get_connection('_') @@ -118,7 +119,7 @@ def test_multiple_connections(self, master_host): def test_connection_pool_blocks_until_timeout(self, master_host): "When out of connections, block for timeout seconds, then raise" - connection_kwargs = {'host': master_host} + connection_kwargs = {'host': master_host[0], 'port': master_host[1]} pool = self.get_pool(max_connections=1, timeout=0.1, connection_kwargs=connection_kwargs) pool.get_connection('_') @@ -134,7 +135,7 @@ def test_connection_pool_blocks_until_conn_available(self, master_host): When out of connections, block until another connection is released to the pool """ - connection_kwargs = {'host': master_host} + connection_kwargs = {'host': master_host[0], 'port': master_host[1]} pool = self.get_pool(max_connections=1, timeout=2, connection_kwargs=connection_kwargs) c1 = pool.get_connection('_') @@ -149,7 +150,7 @@ def target(): assert time.time() - start >= 0.1 def test_reuse_previously_released_connection(self, master_host): - connection_kwargs = {'host': master_host} + connection_kwargs = {'host': master_host[0], 'port': master_host[1]} pool = self.get_pool(connection_kwargs=connection_kwargs) c1 = pool.get_connection('_') pool.release(c1) diff --git a/tests/test_multiprocessing.py b/tests/test_multiprocessing.py index 2d27c4e8bb..d0feef155f 100644 --- a/tests/test_multiprocessing.py +++ b/tests/test_multiprocessing.py @@ -35,7 +35,7 @@ def test_close_connection_in_child(self, master_host): A connection owned by a parent and closed by a child doesn't destroy the file descriptors so a parent can still use it. """ - conn = Connection(host=master_host) + conn = Connection(host=master_host[0], port=master_host[1]) conn.send_command('ping') assert conn.read_response() == b'PONG' @@ -61,7 +61,7 @@ def test_close_connection_in_parent(self, master_host): A connection owned by a parent is unusable by a child if the parent (the owning process) closes the connection. """ - conn = Connection(host=master_host) + conn = Connection(host=master_host[0], port=master_host[1]) conn.send_command('ping') assert conn.read_response() == b'PONG' @@ -89,7 +89,9 @@ def test_pool(self, max_connections, master_host): A child will create its own connections when using a pool created by a parent. """ - pool = ConnectionPool.from_url('redis://{}'.format(master_host), + pool = ConnectionPool.from_url('redis://{}:{}'.format(master_host[0], + master_host[1], + ), max_connections=max_connections) conn = pool.get_connection('ping') @@ -124,7 +126,8 @@ def test_close_pool_in_main(self, max_connections, master_host): A child process that uses the same pool as its parent isn't affected when the parent disconnects all connections within the pool. """ - pool = ConnectionPool.from_url('redis://{}'.format(master_host), + pool = ConnectionPool.from_url('redis://{}:{}'.format(master_host[0], + master_host[1]), max_connections=max_connections) conn = pool.get_connection('ping') diff --git a/tests/test_sentinel.py b/tests/test_sentinel.py index 54cf262c43..7f3ff0ae2a 100644 --- a/tests/test_sentinel.py +++ b/tests/test_sentinel.py @@ -10,7 +10,7 @@ @pytest.fixture(scope="module") def master_ip(master_host): - yield socket.gethostbyname(master_host) + yield socket.gethostbyname(master_host[0]) class SentinelTestClient: From 1dd6ffa5fb78ba62d7ec4d0be7d95041a058dd86 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Wed, 10 Nov 2021 14:19:30 +0200 Subject: [PATCH 2/9] test improvement: do not fill modules information if no modules present --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3ba52a5f53..0adec91aaa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -78,7 +78,7 @@ def skip_ifmodversion_lt(min_version: str, module_name: str): modules = REDIS_INFO["modules"] except KeyError: return pytest.mark.skipif(True, - reason="No redismod-url server running") + reason="Redis server does not have modules") if modules == []: return pytest.mark.skipif(True, reason="No redis modules found") From 463f4ca4285cd22f06812f75cf763d729fb154e8 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Wed, 10 Nov 2021 14:22:10 +0200 Subject: [PATCH 3/9] Marking module loader as redismod --- tests/test_connection.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_connection.py b/tests/test_connection.py index fa9a2b0c90..f2fc834158 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -18,12 +18,14 @@ def test_invalid_response(r): @skip_if_server_version_lt('4.0.0') +@pytest.mark.redismod def test_loaded_modules(r, modclient): assert r.loaded_modules == [] assert 'rejson' in modclient.loaded_modules.keys() @skip_if_server_version_lt('4.0.0') +@pytest.mark.redismod def test_loading_external_modules(r, modclient): def inner(): pass From 0108892738405c587806796e41dd9c231fc5c5ff Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Wed, 10 Nov 2021 14:26:33 +0200 Subject: [PATCH 4/9] database override support --- tests/test_monitor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_monitor.py b/tests/test_monitor.py index 1013202f22..bbb7fb75a4 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -9,11 +9,12 @@ def test_wait_command_not_found(self, r): assert response is None def test_response_values(self, r): + db = r.connection_pool.connection_kwargs.get('db', 0) with r.monitor() as m: r.ping() response = wait_for_command(r, m, 'PING') assert isinstance(response['time'], float) - assert response['db'] == 9 + assert response['db'] == db assert response['client_type'] in ('tcp', 'unix') assert isinstance(response['client_address'], str) assert isinstance(response['client_port'], str) From 139415b7c823aaeb3b70fd7e639fa1d7a51fc760 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Wed, 10 Nov 2021 15:36:48 +0200 Subject: [PATCH 5/9] pubsub test fix --- tests/test_pubsub.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_pubsub.py b/tests/test_pubsub.py index 4be6c7a305..cfc6e5e864 100644 --- a/tests/test_pubsub.py +++ b/tests/test_pubsub.py @@ -575,7 +575,8 @@ def exception_handler(ex, pubsub, thread): class TestPubSubDeadlock: @pytest.mark.timeout(30, method='thread') def test_pubsub_deadlock(self, master_host): - pool = redis.ConnectionPool(host=master_host) + pool = redis.ConnectionPool(host=master_host[0], + port=master_host[1]) r = redis.Redis(connection_pool=pool) for i in range(60): From 6a7ed23089c50c3afca0aec20798dae88c49008d Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Thu, 11 Nov 2021 10:47:54 +0200 Subject: [PATCH 6/9] more enterprise support --- tests/conftest.py | 12 ++++++++ tests/test_commands.py | 55 +++++++++++++++++++++++++++-------- tests/test_connection_pool.py | 12 +++++++- 3 files changed, 66 insertions(+), 13 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 0adec91aaa..b586c120a1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,6 +31,12 @@ def pytest_addoption(parser): def _get_info(redis_url): client = redis.Redis.from_url(redis_url) info = client.info() + try: + client.execute_command("CONFIG SET maxmemory 5555555") + client.execute_command("CONFIG SET maxmemory 0") + info["enterprise"] = False + except redis.exceptions.ResponseError: + info["enterprise"] = True client.connection_pool.disconnect() return info @@ -42,6 +48,7 @@ def pytest_sessionstart(session): arch_bits = info["arch_bits"] REDIS_INFO["version"] = version REDIS_INFO["arch_bits"] = arch_bits + REDIS_INFO["enterprise"] = info["enterprise"] # module info, if the second redis is running try: @@ -92,6 +99,11 @@ def skip_ifmodversion_lt(min_version: str, module_name: str): raise AttributeError("No redis module named {}".format(module_name)) +def skip_if_redis_enterprise(func): + return pytest.mark.skipif(REDIS_INFO["enterprise"] == True, + reason="Redis enterprise") + + def _get_client(cls, request, single_connection_client=True, flushdb=True, from_url=None, **kwargs): diff --git a/tests/test_commands.py b/tests/test_commands.py index 6d4ab008ab..61b90d9936 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -13,6 +13,7 @@ _get_client, skip_if_server_version_gte, skip_if_server_version_lt, + skip_if_redis_enterprise, skip_unless_arch_bits, ) @@ -80,6 +81,7 @@ def test_acl_cat_with_category(self, r): assert 'get' in commands @skip_if_server_version_lt("6.0.0") + @skip_if_redis_enterprise def test_acl_deluser(self, r, request): username = 'redis-py-user' @@ -104,6 +106,7 @@ def teardown(): assert r.acl_getuser(users[4]) is None @skip_if_server_version_lt("6.0.0") + @skip_if_redis_enterprise def test_acl_genpass(self, r): password = r.acl_genpass() assert isinstance(password, str) @@ -117,6 +120,7 @@ def test_acl_genpass(self, r): assert isinstance(password, str) @skip_if_server_version_lt("6.0.0") + @skip_if_redis_enterprise def test_acl_getuser_setuser(self, r, request): username = 'redis-py-user' @@ -210,6 +214,7 @@ def test_acl_help(self, r): assert len(res) != 0 @skip_if_server_version_lt("6.0.0") + @skip_if_redis_enterprise def test_acl_list(self, r, request): username = 'redis-py-user' @@ -222,6 +227,7 @@ def teardown(): assert len(users) == 2 @skip_if_server_version_lt("6.0.0") + @skip_if_redis_enterprise def test_acl_log(self, r, request): username = 'redis-py-user' @@ -257,6 +263,7 @@ def teardown(): assert r.acl_log_reset() @skip_if_server_version_lt("6.0.0") + @skip_if_redis_enterprise def test_acl_setuser_categories_without_prefix_fails(self, r, request): username = 'redis-py-user' @@ -268,6 +275,7 @@ def teardown(): r.acl_setuser(username, categories=['list']) @skip_if_server_version_lt("6.0.0") + @skip_if_redis_enterprise def test_acl_setuser_commands_without_prefix_fails(self, r, request): username = 'redis-py-user' @@ -279,6 +287,7 @@ def teardown(): r.acl_setuser(username, commands=['get']) @skip_if_server_version_lt("6.0.0") + @skip_if_redis_enterprise def test_acl_setuser_add_passwords_and_nopass_fails(self, r, request): username = 'redis-py-user' @@ -312,12 +321,17 @@ def test_client_info(self, r): assert 'addr' in info @skip_if_server_version_lt('5.0.0') - def test_client_list_type(self, r): + def test_client_list_types_not_replica(self, r): with pytest.raises(exceptions.RedisError): r.client_list(_type='not a client type') - for client_type in ['normal', 'master', 'replica', 'pubsub']: + for client_type in ['normal', 'master', 'pubsub']: clients = r.client_list(_type=client_type) assert isinstance(clients, list) + + @skip_if_redis_enterprise + def test_client_list_replica(self, r): + clients = r.client_list(_type='replica') + assert isinstance(clients, list) @skip_if_server_version_lt('6.2.0') def test_client_list_client_id(self, r, request): @@ -454,6 +468,7 @@ def test_client_kill_filter_by_laddr(self, r, r2): assert r.client_kill_filter(laddr=client_2_addr) @skip_if_server_version_lt('2.8.12') + @skip_if_redis_enterprise def test_client_kill_filter_by_user(self, r, request): killuser = 'user_to_kill' r.acl_setuser(killuser, enabled=True, reset=True, @@ -467,6 +482,7 @@ def test_client_kill_filter_by_user(self, r, request): r.acl_deluser(killuser) @skip_if_server_version_lt('2.9.50') + @skip_if_redis_enterprise def test_client_pause(self, r): assert r.client_pause(1) assert r.client_pause(timeout=1) @@ -474,6 +490,7 @@ def test_client_pause(self, r): r.client_pause(timeout='not an integer') @skip_if_server_version_lt('6.2.0') + @skip_if_redis_enterprise def test_client_unpause(self, r): assert r.client_unpause() == b'OK' @@ -491,15 +508,18 @@ def test_client_reply(self, r, r_timeout): assert r.get('foo') == b'bar' @skip_if_server_version_lt('6.0.0') + @skip_if_redis_enterprise def test_client_getredir(self, r): assert isinstance(r.client_getredir(), int) assert r.client_getredir() == -1 def test_config_get(self, r): data = r.config_get() - assert 'maxmemory' in data - assert data['maxmemory'].isdigit() + assert len(data.keys()) > 10 + # # assert 'maxmemory' in data + # assert data['maxmemory'].isdigit() + @skip_if_redis_enterprise def test_config_resetstat(self, r): r.ping() prior_commands_processed = int(r.info()['total_commands_processed']) @@ -509,13 +529,16 @@ def test_config_resetstat(self, r): assert reset_commands_processed < prior_commands_processed def test_config_set(self, r): - data = r.config_get() - rdbname = data['dbfilename'] - try: - assert r.config_set('dbfilename', 'redis_py_test.rdb') - assert r.config_get()['dbfilename'] == 'redis_py_test.rdb' - finally: - assert r.config_set('dbfilename', rdbname) + # data = r.config_get() + # rdbname = data['dbfilename'] + # try: + # assert r.config_set('dbfilename', 'redis_py_test.rdb') + # assert r.config_get()['dbfilename'] == 'redis_py_test.rdb' + # finally: + # assert r.config_set('dbfilename', rdbname) + assert r.config_set('list-max-ziplist-entriies', 1000) + assert r.config_get('list-max-ziplist-entries') == 1000 + r.config_set('list-max-ziplist-entriies', 0) def test_dbsize(self, r): r['a'] = 'foo' @@ -530,8 +553,10 @@ def test_info(self, r): r['b'] = 'bar' info = r.info() assert isinstance(info, dict) - assert info['db9']['keys'] == 2 + assert 'arch_bits' in info.keys() + assert 'redis_version' in info.keys() + @skip_if_redis_enterprise def test_lastsave(self, r): assert isinstance(r.lastsave(), datetime.datetime) @@ -625,6 +650,7 @@ def test_time(self, r): assert isinstance(t[0], int) assert isinstance(t[1], int) + @skip_if_redis_enterprise def test_bgsave(self, r): assert r.bgsave() time.sleep(0.3) @@ -2432,6 +2458,7 @@ def test_cluster_slaves(self, mock_cluster_resp_slaves): 'slaves', 'nodeid'), dict) @skip_if_server_version_lt('3.0.0') + @skip_if_redis_enterprise def test_readwrite(self, r): assert r.readwrite() @@ -3613,6 +3640,7 @@ def test_memory_usage(self, r): assert isinstance(r.memory_usage('foo'), int) @skip_if_server_version_lt('4.0.0') + @skip_if_redis_enterprise def test_module_list(self, r): assert isinstance(r.module_list(), list) for x in r.module_list(): @@ -3625,6 +3653,7 @@ def test_command_count(self, r): assert res >= 100 @skip_if_server_version_lt('4.0.0') + @skip_if_redis_enterprise def test_module(self, r): with pytest.raises(redis.exceptions.ModuleError) as excinfo: r.module_load('/some/fake/path') @@ -3677,6 +3706,7 @@ def test_restore(self, r): assert r.get(key) == b'blee!' @skip_if_server_version_lt('5.0.0') + @skip_if_redis_enterprise def test_replicaof(self, r): with pytest.raises(redis.ResponseError): @@ -3755,6 +3785,7 @@ def test_22_info(self, r): assert '6' in parsed['allocation_stats'] assert '>=256' in parsed['allocation_stats'] + @skip_if_redis_enterprise def test_large_responses(self, r): "The PythonParser has some special cases for return values > 1MB" # load up 5MB of data into a key diff --git a/tests/test_connection_pool.py b/tests/test_connection_pool.py index 6fedec66fb..d4ad4e7819 100644 --- a/tests/test_connection_pool.py +++ b/tests/test_connection_pool.py @@ -7,7 +7,11 @@ from threading import Thread from redis.connection import ssl_available, to_bool -from .conftest import skip_if_server_version_lt, _get_client +from .conftest import ( + skip_if_server_version_lt, + skip_if_redis_enterprise, + _get_client +) from .test_pubsub import wait_for_message @@ -481,6 +485,7 @@ def test_on_connect_error(self): assert not pool._available_connections[0]._sock @skip_if_server_version_lt('2.8.8') + @skip_if_redis_enterprise def test_busy_loading_disconnects_socket(self, r): """ If Redis raises a LOADING error, the connection should be @@ -491,6 +496,7 @@ def test_busy_loading_disconnects_socket(self, r): assert not r.connection._sock @skip_if_server_version_lt('2.8.8') + @skip_if_redis_enterprise def test_busy_loading_from_pipeline_immediate_command(self, r): """ BusyLoadingErrors should raise from Pipelines that execute a @@ -506,6 +512,7 @@ def test_busy_loading_from_pipeline_immediate_command(self, r): assert not pool._available_connections[0]._sock @skip_if_server_version_lt('2.8.8') + @skip_if_redis_enterprise def test_busy_loading_from_pipeline(self, r): """ BusyLoadingErrors should be raised from a pipeline execution @@ -521,6 +528,7 @@ def test_busy_loading_from_pipeline(self, r): assert not pool._available_connections[0]._sock @skip_if_server_version_lt('2.8.8') + @skip_if_redis_enterprise def test_read_only_error(self, r): "READONLY errors get turned in ReadOnlyError exceptions" with pytest.raises(redis.ReadOnlyError): @@ -546,6 +554,7 @@ def test_connect_from_url_unix(self): 'path=/path/to/socket,db=0', ) + @skip_if_redis_enterprise def test_connect_no_auth_supplied_when_required(self, r): """ AuthenticationError should be raised when the server requires a @@ -555,6 +564,7 @@ def test_connect_no_auth_supplied_when_required(self, r): r.execute_command('DEBUG', 'ERROR', 'ERR Client sent AUTH, but no password is set') + @skip_if_redis_enterprise def test_connect_invalid_password_supplied(self, r): "AuthenticationError should be raised when sending the wrong password" with pytest.raises(redis.AuthenticationError): From 28559b90006c5e293084d51c5b49e3baceae94b8 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Thu, 11 Nov 2021 12:17:55 +0200 Subject: [PATCH 7/9] test validation and skips --- tests/conftest.py | 10 ++++++++-- tests/test_commands.py | 17 ++++++----------- tests/test_connection_pool.py | 2 +- tests/test_monitor.py | 15 ++++++++++++++- tests/test_pubsub.py | 7 ++++++- tests/test_scripting.py | 11 ++++++++++- 6 files changed, 45 insertions(+), 17 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index b586c120a1..bb682f78ca 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -100,8 +100,14 @@ def skip_ifmodversion_lt(min_version: str, module_name: str): def skip_if_redis_enterprise(func): - return pytest.mark.skipif(REDIS_INFO["enterprise"] == True, - reason="Redis enterprise") + check = REDIS_INFO["enterprise"] is True + return pytest.mark.skipif(check, reason="Redis enterprise" + ) + + +def skip_ifnot_redis_enterprise(func): + check = REDIS_INFO["enterprise"] is False + return pytest.mark.skipif(check, reason="Redis enterprise") def _get_client(cls, request, single_connection_client=True, flushdb=True, diff --git a/tests/test_commands.py b/tests/test_commands.py index 61b90d9936..7b5db03783 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -327,7 +327,7 @@ def test_client_list_types_not_replica(self, r): for client_type in ['normal', 'master', 'pubsub']: clients = r.client_list(_type=client_type) assert isinstance(clients, list) - + @skip_if_redis_enterprise def test_client_list_replica(self, r): clients = r.client_list(_type='replica') @@ -528,17 +528,12 @@ def test_config_resetstat(self, r): reset_commands_processed = int(r.info()['total_commands_processed']) assert reset_commands_processed < prior_commands_processed + @skip_if_redis_enterprise def test_config_set(self, r): - # data = r.config_get() - # rdbname = data['dbfilename'] - # try: - # assert r.config_set('dbfilename', 'redis_py_test.rdb') - # assert r.config_get()['dbfilename'] == 'redis_py_test.rdb' - # finally: - # assert r.config_set('dbfilename', rdbname) - assert r.config_set('list-max-ziplist-entriies', 1000) - assert r.config_get('list-max-ziplist-entries') == 1000 - r.config_set('list-max-ziplist-entriies', 0) + r.config_set('timeout', 70) + assert r.config_get()['timeout'] == '70' + assert r.config_set('timeout', 0) + assert r.config_get()['timeout'] == '0' def test_dbsize(self, r): r['a'] = 'foo' diff --git a/tests/test_connection_pool.py b/tests/test_connection_pool.py index d4ad4e7819..521f520777 100644 --- a/tests/test_connection_pool.py +++ b/tests/test_connection_pool.py @@ -8,7 +8,7 @@ from threading import Thread from redis.connection import ssl_available, to_bool from .conftest import ( - skip_if_server_version_lt, + skip_if_server_version_lt, skip_if_redis_enterprise, _get_client ) diff --git a/tests/test_monitor.py b/tests/test_monitor.py index bbb7fb75a4..a8a535b59a 100644 --- a/tests/test_monitor.py +++ b/tests/test_monitor.py @@ -1,4 +1,8 @@ -from .conftest import wait_for_command +from .conftest import ( + skip_if_redis_enterprise, + skip_ifnot_redis_enterprise, + wait_for_command +) class TestMonitor: @@ -40,6 +44,7 @@ def test_command_with_escaped_data(self, r): response = wait_for_command(r, m, 'GET foo\\\\x92') assert response['command'] == 'GET foo\\\\x92' + @skip_if_redis_enterprise def test_lua_script(self, r): with r.monitor() as m: script = 'return redis.call("GET", "foo")' @@ -49,3 +54,11 @@ def test_lua_script(self, r): assert response['client_type'] == 'lua' assert response['client_address'] == 'lua' assert response['client_port'] == '' + + @skip_ifnot_redis_enterprise + def test_lua_script_in_enterprise(self, r): + with r.monitor() as m: + script = 'return redis.call("GET", "foo")' + assert r.eval(script, 0) is None + response = wait_for_command(r, m, 'GET foo') + assert response is None diff --git a/tests/test_pubsub.py b/tests/test_pubsub.py index cfc6e5e864..e2424592cd 100644 --- a/tests/test_pubsub.py +++ b/tests/test_pubsub.py @@ -7,7 +7,11 @@ import redis from redis.exceptions import ConnectionError -from .conftest import _get_client, skip_if_server_version_lt +from .conftest import ( + _get_client, + skip_if_redis_enterprise, + skip_if_server_version_lt +) def wait_for_message(pubsub, timeout=0.1, ignore_subscribe_messages=False): @@ -528,6 +532,7 @@ def test_send_pubsub_ping_message(self, r): class TestPubSubConnectionKilled: @skip_if_server_version_lt('3.0.0') + @skip_if_redis_enterprise def test_connection_error_raised_when_connection_dies(self, r): p = r.pubsub() p.subscribe('foo') diff --git a/tests/test_scripting.py b/tests/test_scripting.py index c3c2094d4a..352f3bae2e 100644 --- a/tests/test_scripting.py +++ b/tests/test_scripting.py @@ -2,6 +2,8 @@ from redis import exceptions +from tests.conftest import skip_if_server_version_lt + multiply_script = """ local value = redis.call('GET', KEYS[1]) @@ -30,7 +32,8 @@ def test_eval(self, r): # 2 * 3 == 6 assert r.eval(multiply_script, 1, 'a', 3) == 6 - def test_script_flush(self, r): + @skip_if_server_version_lt('6.2.0') + def test_script_flush_620(self, r): r.set('a', 2) r.script_load(multiply_script) r.script_flush('ASYNC') @@ -43,6 +46,12 @@ def test_script_flush(self, r): r.script_load(multiply_script) r.script_flush() + with pytest.raises(exceptions.DataError): + r.set('a', 2) + r.script_load(multiply_script) + r.script_flush("NOTREAL") + + def test_script_flush(self, r): r.set('a', 2) r.script_load(multiply_script) r.script_flush(None) From 924eb67f0e7510ff9461637736cd7539b8002afe Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Thu, 11 Nov 2021 16:23:35 +0200 Subject: [PATCH 8/9] skips for redis enterprise --- tests/test_commands.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_commands.py b/tests/test_commands.py index df561d4d5d..6cb1a78279 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -1208,6 +1208,12 @@ def test_stralgo_lcs(self, r): value1 = 'ohmytext' value2 = 'mynewtext' res = 'mytext' + + if skip_if_redis_enterprise(None).args[0] is True: + with pytest.raises(redis.exceptions.ResponseError): + assert r.stralgo('LCS', value1, value2) == res + return + # test LCS of strings assert r.stralgo('LCS', value1, value2) == res # test using keys @@ -1250,6 +1256,12 @@ def test_strlen(self, r): def test_substr(self, r): r['a'] = '0123456789' + + if skip_if_redis_enterprise(None).args[0] is True: + with pytest.raises(redis.exceptions.ResponseError): + assert r.substr('a', 0) == b'0123456789' + return + assert r.substr('a', 0) == b'0123456789' assert r.substr('a', 2) == b'23456789' assert r.substr('a', 3, 5) == b'345' @@ -3617,6 +3629,11 @@ def test_memory_doctor(self, r): @skip_if_server_version_lt('4.0.0') def test_memory_malloc_stats(self, r): + if skip_if_redis_enterprise(None).args[0] is True: + with pytest.raises(redis.exceptions.ResponseError): + assert r.memory_malloc_stats() + return + assert r.memory_malloc_stats() @skip_if_server_version_lt('4.0.0') @@ -3624,6 +3641,12 @@ def test_memory_stats(self, r): # put a key into the current db to make sure that "db." # has data r.set('foo', 'bar') + + if skip_if_redis_enterprise(None).args[0] is True: + with pytest.raises(redis.exceptions.ResponseError): + stats = r.memory_stats() + return + stats = r.memory_stats() assert isinstance(stats, dict) for key, value in stats.items(): From f55d7626861081a8bd54d5bc72e1e2c0dd234003 Mon Sep 17 00:00:00 2001 From: "Chayim I. Kirshen" Date: Sun, 14 Nov 2021 12:31:36 +0200 Subject: [PATCH 9/9] Adding support for redis command COMMAND fixing tests with enterprise --- redis/client.py | 41 +++++++++++++-------------- redis/commands/core.py | 3 ++ redis/commands/redismodules.py | 38 ++++++++++++------------- redis/commands/search/__init__.py | 3 +- redis/commands/timeseries/__init__.py | 3 +- tests/conftest.py | 10 +++---- tests/test_commands.py | 8 ++++++ tests/test_connection.py | 26 ++++++----------- 8 files changed, 65 insertions(+), 67 deletions(-) diff --git a/redis/client.py b/redis/client.py index f2f1eed08c..753770e3e2 100755 --- a/redis/client.py +++ b/redis/client.py @@ -703,7 +703,6 @@ class Redis(RedisModuleCommands, CoreCommands, object): 'CLUSTER SET-CONFIG-EPOCH': bool_ok, 'CLUSTER SETSLOT': bool_ok, 'CLUSTER SLAVES': parse_cluster_nodes, - 'COMMAND': int, 'COMMAND COUNT': int, 'CONFIG GET': parse_config_get, 'CONFIG RESETSTAT': bool_ok, @@ -891,6 +890,12 @@ def __init__(self, host='localhost', port=6379, self.response_callbacks = CaseInsensitiveDict( self.__class__.RESPONSE_CALLBACKS) + # preload our class with the available redis commands + try: + self.__redis_commands__() + except RedisError: + pass + def __repr__(self): return "%s<%s>" % (type(self).__name__, repr(self.connection_pool)) @@ -898,12 +903,12 @@ def set_response_callback(self, command, callback): "Set a custom Response Callback" self.response_callbacks[command] = callback - def load_external_module(self, modname, funcname, func): + def load_external_module(self, funcname, func, + ): """ This function can be used to add externally defined redis modules, and their namespaces to the redis client. - modname - A string containing the name of the redis module to look for - in the redis info block. + funcname - A string containing the name of the function to create func - The function, being added to this class. @@ -914,31 +919,25 @@ def load_external_module(self, modname, funcname, func): from redis import Redis from foomodule import F r = Redis() - r.load_external_module("foomod", "foo", F) + r.load_external_module("foo", F) r.foo().dothing('your', 'arguments') For a concrete example see the reimport of the redisjson module in tests/test_connection.py::test_loading_external_modules """ - mods = self.loaded_modules - if modname.lower() not in mods: - raise ModuleError("{} is not loaded in redis.".format(modname)) setattr(self, funcname, func) - @property - def loaded_modules(self): - key = '__redis_modules__' - mods = getattr(self, key, None) - if mods is not None: - return mods - + def __redis_commands__(self): + """Store the list of available commands, for our redis instance.""" + cmds = getattr(self, '__commands__', None) + if cmds is not None: + return cmds try: - mods = {f.get('name').lower(): f.get('ver') - for f in self.info().get('modules')} - except TypeError: - mods = [] - setattr(self, key, mods) - return mods + cmds = [c[0].upper().decode() for c in self.command()] + except AttributeError: # if encoded + cmds = [c[0].upper() for c in self.command()] + self.__commands__ = cmds + return cmds def pipeline(self, transaction=True, shard_hint=None): """ diff --git a/redis/commands/core.py b/redis/commands/core.py index 67f1bfa1be..516e7d9c83 100644 --- a/redis/commands/core.py +++ b/redis/commands/core.py @@ -3315,6 +3315,9 @@ def command_info(self): def command_count(self): return self.execute_command('COMMAND COUNT') + def command(self): + return self.execute_command('COMMAND') + class Script: "An executable Lua script object returned by ``register_script``" diff --git a/redis/commands/redismodules.py b/redis/commands/redismodules.py index 457a69e2a2..b3cbee1d87 100644 --- a/redis/commands/redismodules.py +++ b/redis/commands/redismodules.py @@ -8,41 +8,41 @@ class RedisModuleCommands: """ def json(self, encoder=JSONEncoder(), decoder=JSONDecoder()): - """Access the json namespace, providing support for redis json.""" - try: - modversion = self.loaded_modules['rejson'] - except IndexError: - raise ModuleError("rejson is not a loaded in the redis instance.") + """Access the json namespace, providing support for redis json. + """ + if 'JSON.SET' not in self.__commands__: + raise ModuleError("redisjson is not loaded in redis. " + "For more information visit " + "https://redisjson.io/") from .json import JSON jj = JSON( client=self, - version=modversion, encoder=encoder, decoder=decoder) return jj def ft(self, index_name="idx"): - """Access the search namespace, providing support for redis search.""" - try: - modversion = self.loaded_modules['search'] - except IndexError: - raise ModuleError("search is not a loaded in the redis instance.") + """Access the search namespace, providing support for redis search. + """ + if 'FT.INFO' not in self.__commands__: + raise ModuleError("redisearch is not loaded in redis. " + "For more information visit " + "https://redisearch.io/") from .search import Search - s = Search(client=self, version=modversion, index_name=index_name) + s = Search(client=self, index_name=index_name) return s - def ts(self, index_name="idx"): + def ts(self): """Access the timeseries namespace, providing support for redis timeseries data. """ - try: - modversion = self.loaded_modules['timeseries'] - except IndexError: - raise ModuleError("timeseries is not a loaded in " - "the redis instance.") + if 'TS.INFO' not in self.__commands__: + raise ModuleError("reditimeseries is not loaded in redis. " + "For more information visit " + "https://redistimeseries.io/") from .timeseries import TimeSeries - s = TimeSeries(client=self, version=modversion, index_name=index_name) + s = TimeSeries(client=self) return s diff --git a/redis/commands/search/__init__.py b/redis/commands/search/__init__.py index 425578eabd..8320ad4392 100644 --- a/redis/commands/search/__init__.py +++ b/redis/commands/search/__init__.py @@ -83,7 +83,7 @@ def commit(self): self.pipeline.execute() self.current_chunk = 0 - def __init__(self, client, version=None, index_name="idx"): + def __init__(self, client, index_name="idx"): """ Create a new Client for the given index_name. The default name is `idx` @@ -91,7 +91,6 @@ def __init__(self, client, version=None, index_name="idx"): If conn is not None, we employ an already existing redis connection """ self.client = client - self.MODULE_VERSION = version self.index_name = index_name self.execute_command = client.execute_command self.pipeline = client.pipeline diff --git a/redis/commands/timeseries/__init__.py b/redis/commands/timeseries/__init__.py index 83fa17082e..5ce538f675 100644 --- a/redis/commands/timeseries/__init__.py +++ b/redis/commands/timeseries/__init__.py @@ -34,7 +34,7 @@ class TimeSeries(TimeSeriesCommands): functionality. """ - def __init__(self, client=None, version=None, **kwargs): + def __init__(self, client=None, **kwargs): """Create a new RedisTimeSeries client.""" # Set the module commands' callbacks self.MODULE_CALLBACKS = { @@ -55,7 +55,6 @@ def __init__(self, client=None, version=None, **kwargs): self.client = client self.execute_command = client.execute_command - self.MODULE_VERSION = version for key, value in self.MODULE_CALLBACKS.items(): self.client.set_response_callback(key, value) diff --git a/tests/conftest.py b/tests/conftest.py index bb682f78ca..31d3fbd1a8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,12 +31,10 @@ def pytest_addoption(parser): def _get_info(redis_url): client = redis.Redis.from_url(redis_url) info = client.info() - try: - client.execute_command("CONFIG SET maxmemory 5555555") - client.execute_command("CONFIG SET maxmemory 0") - info["enterprise"] = False - except redis.exceptions.ResponseError: + if 'dping' in client.__commands__: info["enterprise"] = True + else: + info["enterprise"] = False client.connection_pool.disconnect() return info @@ -57,6 +55,8 @@ def pytest_sessionstart(session): REDIS_INFO["modules"] = info["modules"] except redis.exceptions.ConnectionError: pass + except KeyError: + pass def skip_if_server_version_lt(min_version): diff --git a/tests/test_commands.py b/tests/test_commands.py index 6cb1a78279..dbd04429b4 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -3671,6 +3671,14 @@ def test_command_count(self, r): assert isinstance(res, int) assert res >= 100 + @skip_if_server_version_lt('2.8.13') + def test_command(self, r): + res = r.command() + assert len(res) >= 100 + cmds = [c[0].decode() for c in res] + assert 'set' in cmds + assert 'get' in cmds + @skip_if_server_version_lt('4.0.0') @skip_if_redis_enterprise def test_module(self, r): diff --git a/tests/test_connection.py b/tests/test_connection.py index f2fc834158..7c44768150 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -2,7 +2,7 @@ import types import pytest -from redis.exceptions import InvalidResponse, ModuleError +from redis.exceptions import InvalidResponse from redis.utils import HIREDIS_AVAILABLE from .conftest import skip_if_server_version_lt @@ -19,30 +19,20 @@ def test_invalid_response(r): @skip_if_server_version_lt('4.0.0') @pytest.mark.redismod -def test_loaded_modules(r, modclient): - assert r.loaded_modules == [] - assert 'rejson' in modclient.loaded_modules.keys() - - -@skip_if_server_version_lt('4.0.0') -@pytest.mark.redismod -def test_loading_external_modules(r, modclient): +def test_loading_external_modules(modclient): def inner(): pass - with pytest.raises(ModuleError): - r.load_external_module('rejson', 'myfuncname', inner) - - modclient.load_external_module('rejson', 'myfuncname', inner) + modclient.load_external_module('myfuncname', inner) assert getattr(modclient, 'myfuncname') == inner assert isinstance(getattr(modclient, 'myfuncname'), types.FunctionType) # and call it from redis.commands import RedisModuleCommands j = RedisModuleCommands.json - modclient.load_external_module('rejson', 'sometestfuncname', j) + modclient.load_external_module('sometestfuncname', j) - d = {'hello': 'world!'} - mod = j(modclient) - mod.set("fookey", ".", d) - assert mod.get('fookey') == d + # d = {'hello': 'world!'} + # mod = j(modclient) + # mod.set("fookey", ".", d) + # assert mod.get('fookey') == d