Merge branch 'master' into fix-pool-repr

Author: petyaslavova

CHANGES

@@ -71,6 +71,7 @@
     * Close Unix sockets if the connection attempt fails. This prevents `ResourceWarning`s. (#3314)
     * Close SSL sockets if the connection attempt fails, or if validations fail. (#3317)
     * Eliminate mutable default arguments in the `redis.commands.core.Script` class. (#3332)
+    * Fix SSL verification with `ssl_cert_reqs="none"` and `ssl_check_hostname=True` by automatically setting `check_hostname=False` when `verify_mode=ssl.CERT_NONE` (#3635) 
     * Allow newer versions of PyJWT as dependency. (#3630)
 
 * 4.1.3 (Feb 8, 2022)

README.md

@@ -31,12 +31,17 @@ The Python interface to the Redis key-value store.
 
 ## Installation
 
-Start a redis via docker:
+Start a redis via docker (for Redis versions >= 8.0):
 
 ``` bash
-docker run -p 6379:6379 -it redis/redis-stack:latest
+docker run -p 6379:6379 -it redis:latest
 ```
 
+Start a redis via docker (for Redis versions < 8.0):
+
+``` bash
+docker run -p 6379:6379 -it redis/redis-stack:latest
+
 To install redis-py, simply:
 
 ``` bash
@@ -54,15 +59,16 @@ Looking for a high-level library to handle object mapping? See [redis-om-python]
 
 ## Supported Redis Versions
 
-The most recent version of this library supports redis version [5.0](https://github.com/redis/redis/blob/5.0/00-RELEASENOTES), [6.0](https://github.com/redis/redis/blob/6.0/00-RELEASENOTES), [6.2](https://github.com/redis/redis/blob/6.2/00-RELEASENOTES), [7.0](https://github.com/redis/redis/blob/7.0/00-RELEASENOTES), [7.2](https://github.com/redis/redis/blob/7.2/00-RELEASENOTES) and [7.4](https://github.com/redis/redis/blob/7.4/00-RELEASENOTES).
+The most recent version of this library supports Redis version [7.2](https://github.com/redis/redis/blob/7.2/00-RELEASENOTES), [7.4](https://github.com/redis/redis/blob/7.4/00-RELEASENOTES) and [8.0](https://github.com/redis/redis/blob/8.0/00-RELEASENOTES).
 
 The table below highlights version compatibility of the most-recent library versions and redis versions.
 
 | Library version | Supported redis versions |
 |-----------------|-------------------|
 | 3.5.3 | <= 6.2 Family of releases |
 | >= 4.5.0 | Version 5.0 to 7.0 |
-| >= 5.0.0 | Version 5.0 to current |
+| >= 5.0.0 | Version 5.0 to 7.4 |
+| >= 6.0.0 | Version 7.2 to current |
 
 
 ## Usage
@@ -152,8 +158,42 @@ The following example shows how to utilize [Redis Pub/Sub](https://redis.io/docs
 {'pattern': None, 'type': 'subscribe', 'channel': b'my-second-channel', 'data': 1}
 ```
 
+### Redis’ search and query capabilities default dialect
+
+Release 6.0.0 introduces a client-side default dialect for Redis’ search and query capabilities.
+By default, the client now overrides the server-side dialect with version 2, automatically appending *DIALECT 2* to commands like *FT.AGGREGATE* and *FT.SEARCH*.
 
---------------------------
+**Important**: Be aware that the query dialect may impact the results returned. If needed, you can revert to a different dialect version by configuring the client accordingly.
+
+``` python
+>>> from redis.commands.search.field import TextField
+>>> from redis.commands.search.query import Query
+>>> from redis.commands.search.index_definition import IndexDefinition
+>>> import redis
+
+>>> r = redis.Redis(host='localhost', port=6379, db=0)
+>>> r.ft().create_index(
+>>>     (TextField("name"), TextField("lastname")),
+>>>     definition=IndexDefinition(prefix=["test:"]),
+>>> )
+
+>>> r.hset("test:1", "name", "James")
+>>> r.hset("test:1", "lastname", "Brown")
+
+>>> # Query with default DIALECT 2
+>>> query = "@name: James Brown"
+>>> q = Query(query)
+>>> res = r.ft().search(q)
+
+>>> # Query with explicit DIALECT 1
+>>> query = "@name: James Brown"
+>>> q = Query(query).dialect(1)
+>>> res = r.ft().search(q)
+```
+
+You can find further details in the [query dialect documentation](https://redis.io/docs/latest/develop/interact/search-and-query/advanced-concepts/dialects/).
+
+---------------------------------------------
 
 ### Author
 
@@ -169,4 +209,4 @@ Special thanks to:
     system.
 -   Paul Hubbard for initial packaging support.
 
-[![Redis](./docs/_static/logo-redis.svg)](https://redis.io)
+[![Redis](./docs/_static/logo-redis.svg)](https://redis.io)

docker-compose.yml

@@ -1,9 +1,10 @@
 ---
+# image tag 8.0-RC2-pre is the one matching the 8.0 GA release
 x-client-libs-stack-image: &client-libs-stack-image
-  image: "redislabs/client-libs-test:${CLIENT_LIBS_TEST_STACK_IMAGE_TAG:-rs-7.4.0-v2}"
+  image: "redislabs/client-libs-test:${CLIENT_LIBS_TEST_STACK_IMAGE_TAG:-8.0-RC2-pre}"
 
 x-client-libs-image: &client-libs-image
-  image: "redislabs/client-libs-test:${CLIENT_LIBS_TEST_IMAGE_TAG:-7.4.2}"
+  image: "redislabs/client-libs-test:${CLIENT_LIBS_TEST_IMAGE_TAG:-8.0-RC2-pre}"
 
 services:
 

docs/examples/ssl_connection_examples.ipynb

@@ -37,7 +37,6 @@
     "    host='localhost',\n",
     "    port=6666,\n",
     "    ssl=True,\n",
-    "    ssl_check_hostname=False,\n",
     "    ssl_cert_reqs=\"none\",\n",
     ")\n",
     "r.ping()"
@@ -69,7 +68,7 @@
    "source": [
     "import redis\n",
     "\n",
-    "r = redis.from_url(\"rediss://localhost:6666?ssl_cert_reqs=none&ssl_check_hostname=False&decode_responses=True&health_check_interval=2\")\n",
+    "r = redis.from_url(\"rediss://localhost:6666?ssl_cert_reqs=none&decode_responses=True&health_check_interval=2\")\n",
     "r.ping()"
    ]
   },
@@ -103,7 +102,6 @@
     "    host=\"localhost\",\n",
     "    port=6666,\n",
     "    connection_class=redis.SSLConnection,\n",
-    "    ssl_check_hostname=False,\n",
     "    ssl_cert_reqs=\"none\",\n",
     ")\n",
     "\n",
@@ -143,7 +141,6 @@
     "    port=6666,\n",
     "    ssl=True,\n",
     "    ssl_min_version=ssl.TLSVersion.TLSv1_3,\n",
-    "    ssl_check_hostname=False,\n",
     "    ssl_cert_reqs=\"none\",\n",
     ")\n",
     "r.ping()"

redis/__init__.py

@@ -45,7 +45,7 @@ def int_or_str(value):
         return value
 
 
-__version__ = "5.2.1"
+__version__ = "6.1.0"
 VERSION = tuple(map(int_or_str, __version__.split(".")))
 
 

redis/asyncio/cluster.py

@@ -134,6 +134,14 @@ class RedisCluster(AbstractRedis, AbstractRedisCluster, AsyncRedisClusterCommand
         | Enable read from replicas in READONLY mode and defines the load balancing
           strategy that will be used for cluster node selection.
           The data read from replicas is eventually consistent with the data in primary nodes.
+    :param dynamic_startup_nodes:
+        | Set the RedisCluster's startup nodes to all the discovered nodes.
+          If true (default value), the cluster's discovered nodes will be used to
+          determine the cluster nodes-slots mapping in the next topology refresh.
+          It will remove the initial passed startup nodes if their endpoints aren't
+          listed in the CLUSTER SLOTS output.
+          If you use dynamic DNS endpoints for startup nodes but CLUSTER SLOTS lists
+          specific IP addresses, it is best to set it to false.
     :param reinitialize_steps:
         | Specifies the number of MOVED errors that need to occur before reinitializing
           the whole cluster topology. If a MOVED error occurs and the cluster does not
@@ -250,6 +258,7 @@ def __init__(
         require_full_coverage: bool = True,
         read_from_replicas: bool = False,
         load_balancing_strategy: Optional[LoadBalancingStrategy] = None,
+        dynamic_startup_nodes: bool = True,
         reinitialize_steps: int = 5,
         cluster_error_retry_attempts: int = 3,
         max_connections: int = 2**31,
@@ -388,6 +397,7 @@ def __init__(
             startup_nodes,
             require_full_coverage,
             kwargs,
+            dynamic_startup_nodes=dynamic_startup_nodes,
             address_remap=address_remap,
             event_dispatcher=self._event_dispatcher,
         )
@@ -1162,6 +1172,7 @@ async def _mock(self, error: RedisError):
 
 class NodesManager:
     __slots__ = (
+        "_dynamic_startup_nodes",
         "_moved_exception",
         "_event_dispatcher",
         "connection_kwargs",
@@ -1179,6 +1190,7 @@ def __init__(
         startup_nodes: List["ClusterNode"],
         require_full_coverage: bool,
         connection_kwargs: Dict[str, Any],
+        dynamic_startup_nodes: bool = True,
         address_remap: Optional[Callable[[Tuple[str, int]], Tuple[str, int]]] = None,
         event_dispatcher: Optional[EventDispatcher] = None,
     ) -> None:
@@ -1191,6 +1203,8 @@ def __init__(
         self.nodes_cache: Dict[str, "ClusterNode"] = {}
         self.slots_cache: Dict[int, List["ClusterNode"]] = {}
         self.read_load_balancer = LoadBalancer()
+
+        self._dynamic_startup_nodes: bool = dynamic_startup_nodes
         self._moved_exception: MovedError = None
         if event_dispatcher is None:
             self._event_dispatcher = EventDispatcher()
@@ -1433,8 +1447,10 @@ async def initialize(self) -> None:
         # Set the tmp variables to the real variables
         self.slots_cache = tmp_slots
         self.set_nodes(self.nodes_cache, tmp_nodes_cache, remove_old=True)
-        # Populate the startup nodes with all discovered nodes
-        self.set_nodes(self.startup_nodes, self.nodes_cache, remove_old=True)
+
+        if self._dynamic_startup_nodes:
+            # Populate the startup nodes with all discovered nodes
+            self.set_nodes(self.startup_nodes, self.nodes_cache, remove_old=True)
 
         # Set the default node
         self.default_node = self.get_nodes_by_server_type(PRIMARY)[0]

redis/asyncio/connection.py

@@ -868,7 +868,7 @@ def __init__(
         cert_reqs: Optional[Union[str, ssl.VerifyMode]] = None,
         ca_certs: Optional[str] = None,
         ca_data: Optional[str] = None,
-        check_hostname: bool = False,
+        check_hostname: bool = True,
         min_version: Optional[TLSVersion] = None,
         ciphers: Optional[str] = None,
     ):
@@ -893,7 +893,9 @@ def __init__(
         self.cert_reqs = cert_reqs
         self.ca_certs = ca_certs
         self.ca_data = ca_data
-        self.check_hostname = check_hostname
+        self.check_hostname = (
+            check_hostname if self.cert_reqs != ssl.CERT_NONE else False
+        )
         self.min_version = min_version
         self.ciphers = ciphers
         self.context: Optional[SSLContext] = None

redis/cluster.py

@@ -185,6 +185,7 @@ def parse_cluster_myshardid(resp, **options):
     "ssl_cert_reqs",
     "ssl_keyfile",
     "ssl_password",
+    "ssl_check_hostname",
     "unix_socket_path",
     "username",
     "cache",

redis/connection.py

@@ -1083,7 +1083,9 @@ def __init__(
         self.ca_certs = ssl_ca_certs
         self.ca_data = ssl_ca_data
         self.ca_path = ssl_ca_path
-        self.check_hostname = ssl_check_hostname
+        self.check_hostname = (
+            ssl_check_hostname if self.cert_reqs != ssl.CERT_NONE else False
+        )
         self.certificate_password = ssl_password
         self.ssl_validate_ocsp = ssl_validate_ocsp
         self.ssl_validate_ocsp_stapled = ssl_validate_ocsp_stapled

tests/test_asyncio/test_cluster.py

@@ -2723,6 +2723,27 @@ def cmd_init_mock(self, r: ClusterNode) -> None:
                     assert rc.get_node(host=default_host, port=7001) is not None
                     assert rc.get_node(host=default_host, port=7002) is not None
 
+    @pytest.mark.parametrize("dynamic_startup_nodes", [True, False])
+    async def test_init_slots_dynamic_startup_nodes(self, dynamic_startup_nodes):
+        rc = await get_mocked_redis_client(
+            host="my@DNS.com",
+            port=7000,
+            cluster_slots=default_cluster_slots,
+            dynamic_startup_nodes=dynamic_startup_nodes,
+        )
+        # Nodes are taken from default_cluster_slots
+        discovered_nodes = [
+            "127.0.0.1:7000",
+            "127.0.0.1:7001",
+            "127.0.0.1:7002",
+            "127.0.0.1:7003",
+        ]
+        startup_nodes = list(rc.nodes_manager.startup_nodes.keys())
+        if dynamic_startup_nodes is True:
+            assert sorted(startup_nodes) == sorted(discovered_nodes)
+        else:
+            assert startup_nodes == ["my@DNS.com:7000"]
+
 
 class TestClusterPipeline:
     """Tests for the ClusterPipeline class."""
@@ -3118,9 +3139,7 @@ async def test_ssl_with_invalid_cert(
     async def test_ssl_connection(
         self, create_client: Callable[..., Awaitable[RedisCluster]]
     ) -> None:
-        async with await create_client(
-            ssl=True, ssl_check_hostname=False, ssl_cert_reqs="none"
-        ) as rc:
+        async with await create_client(ssl=True, ssl_cert_reqs="none") as rc:
             assert await rc.ping()
 
     @pytest.mark.parametrize(
@@ -3136,7 +3155,6 @@ async def test_ssl_connection_tls12_custom_ciphers(
     ) -> None:
         async with await create_client(
             ssl=True,
-            ssl_check_hostname=False,
             ssl_cert_reqs="none",
             ssl_min_version=ssl.TLSVersion.TLSv1_2,
             ssl_ciphers=ssl_ciphers,
@@ -3148,7 +3166,6 @@ async def test_ssl_connection_tls12_custom_ciphers_invalid(
     ) -> None:
         async with await create_client(
             ssl=True,
-            ssl_check_hostname=False,
             ssl_cert_reqs="none",
             ssl_min_version=ssl.TLSVersion.TLSv1_2,
             ssl_ciphers="foo:bar",
@@ -3170,7 +3187,6 @@ async def test_ssl_connection_tls13_custom_ciphers(
         # TLSv1.3 does not support changing the ciphers
         async with await create_client(
             ssl=True,
-            ssl_check_hostname=False,
             ssl_cert_reqs="none",
             ssl_min_version=ssl.TLSVersion.TLSv1_2,
             ssl_ciphers=ssl_ciphers,

tests/test_asyncio/test_ssl.py

@@ -0,0 +1,56 @@
+from urllib.parse import urlparse
+import pytest
+import pytest_asyncio
+import redis.asyncio as redis
+
+# Skip test or not based on cryptography installation
+try:
+    import cryptography  # noqa
+
+    skip_if_cryptography = pytest.mark.skipif(False, reason="")
+    skip_if_nocryptography = pytest.mark.skipif(False, reason="")
+except ImportError:
+    skip_if_cryptography = pytest.mark.skipif(True, reason="cryptography not installed")
+    skip_if_nocryptography = pytest.mark.skipif(
+        True, reason="cryptography not installed"
+    )
+
+
+@pytest.mark.ssl
+class TestSSL:
+    """Tests for SSL connections in asyncio."""
+
+    @pytest_asyncio.fixture()
+    async def _get_client(self, request):
+        ssl_url = request.config.option.redis_ssl_url
+        p = urlparse(ssl_url)[1].split(":")
+        client = redis.Redis(host=p[0], port=p[1], ssl=True)
+        yield client
+        await client.aclose()
+
+    async def test_ssl_with_invalid_cert(self, _get_client):
+        """Test SSL connection with invalid certificate."""
+        pass
+
+    async def test_cert_reqs_none_with_check_hostname(self, request):
+        """Test that when ssl_cert_reqs=none is used with ssl_check_hostname=True,
+        the connection is created successfully with check_hostname internally set to False"""
+        ssl_url = request.config.option.redis_ssl_url
+        parsed_url = urlparse(ssl_url)
+        r = redis.Redis(
+            host=parsed_url.hostname,
+            port=parsed_url.port,
+            ssl=True,
+            ssl_cert_reqs="none",
+            # Check that ssl_check_hostname is ignored, when ssl_cert_reqs=none
+            ssl_check_hostname=True,
+        )
+        try:
+            # Connection should be successful
+            assert await r.ping()
+            # check_hostname should have been automatically set to False
+            assert r.connection_pool.connection_class == redis.SSLConnection
+            conn = r.connection_pool.make_connection()
+            assert conn.check_hostname is False
+        finally:
+            await r.aclose()