Detect blocking calls in coroutines using BlockBuster

aiohttp/client.py

@@ -607,11 +607,14 @@ async def _request(
                         if req_cookies:
                             all_cookies.load(req_cookies)
 
+                    proxy_: Optional[URL] = None
                     if proxy is not None:
-                        proxy = URL(proxy)
+                        proxy_ = URL(proxy)
                     elif self._trust_env:
                         with suppress(LookupError):
-                            proxy, proxy_auth = get_env_proxy_for_url(url)
+                            proxy_, proxy_auth = await asyncio.to_thread(
+                                get_env_proxy_for_url, url
+                            )
 
                     req = self._request_class(
                         method,
@@ -628,7 +631,7 @@ async def _request(
                         expect100=expect100,
                         loop=self._loop,
                         response_class=self._response_class,
-                        proxy=proxy,
+                        proxy=proxy_,
                         proxy_auth=proxy_auth,
                         timer=timer,
                         session=self,

requirements/test.in

@@ -1,5 +1,6 @@
 -r base.in
 
+blockbuster
 coverage
 freezegun
 mypy; implementation_name == "cpython"

requirements/test.txt

@@ -14,6 +14,8 @@ annotated-types==0.7.0
     # via pydantic
 async-timeout==5.0.1 ; python_version < "3.11"
     # via -r requirements/runtime-deps.in
+blockbuster==1.5.14
+    # via -r requirements/test.in
 brotli==1.1.0 ; platform_python_implementation == "CPython"
     # via -r requirements/runtime-deps.in
 cffi==1.17.1
@@ -33,6 +35,8 @@ exceptiongroup==1.2.2
     # via pytest
 execnet==2.1.1
     # via pytest-xdist
+forbiddenfruit==0.1.4
+    # via blockbuster
 freezegun==1.5.1
     # via -r requirements/test.in
 frozenlist==1.5.0

tests/conftest.py

@@ -12,6 +12,7 @@
 from uuid import uuid4
 
 import pytest
+from blockbuster import blockbuster_ctx
 
 from aiohttp.client_proto import ResponseHandler
 from aiohttp.http import WS_KEY
@@ -33,6 +34,25 @@
 IS_LINUX = sys.platform.startswith("linux")
 
 
+@pytest.fixture(autouse=True)
+def blockbuster() -> Iterator[None]:
+    with blockbuster_ctx("aiohttp") as bb:
+        bb.functions["io.TextIOWrapper.read"].can_block_in(
+            "aiohttp/client_reqrep.py", "update_auth"
+        )
+        bb.functions["os.stat"].can_block_in("aiohttp/client_reqrep.py", "update_auth")
+        bb.functions["os.stat"].can_block_in(
+            "asyncio/unix_events.py", "create_unix_server"
+        )
+        bb.functions["os.sendfile"].can_block_in(
+            "asyncio/unix_events.py", "_sock_sendfile_native_impl"
+        )
+        bb.functions["os.read"].can_block_in(
+            "asyncio/base_events.py", "subprocess_shell"
+        )
+        yield
+
+
 @pytest.fixture
 def tls_certificate_authority() -> trustme.CA:
     if not TRUSTME:

tests/test_client_functional.py

@@ -63,6 +63,11 @@ def fname(here: pathlib.Path) -> pathlib.Path:
     return here / "conftest.py"
 
 
+@pytest.fixture
+async def file_content(fname: pathlib.Path) -> bytes:
+    return await asyncio.to_thread(fname.read_bytes)
+
+
 async def test_keepalive_two_requests_success(aiohttp_client: AiohttpClient) -> None:
     async def handler(request: web.Request) -> web.Response:
         body = await request.read()
@@ -507,7 +512,7 @@ async def handler(request: web.Request) -> web.Response:
         assert ["file"] == list(post_data.keys())
         file_field = post_data["file"]
         assert isinstance(file_field, web.FileField)
-        assert data == file_field.file.read()
+        assert data == await asyncio.to_thread(file_field.file.read)
         return web.Response()
 
     app = web.Application()
@@ -1632,17 +1637,16 @@ async def handler(request: web.Request) -> web.Response:
     resp.close()
 
 
-async def test_POST_FILES(aiohttp_client: AiohttpClient, fname: pathlib.Path) -> None:
+async def test_POST_FILES(
+    aiohttp_client: AiohttpClient, fname: pathlib.Path, file_content: bytes
+) -> None:
     async def handler(request: web.Request) -> web.Response:
         data = await request.post()
         assert isinstance(data["some"], web.FileField)
-        assert data["some"].filename == fname.name
-        with fname.open("rb") as f:
-            content1 = f.read()
-        content2 = data["some"].file.read()
-        assert content1 == content2
+        content = await asyncio.to_thread(data["some"].file.read)
+        assert content == file_content
         assert isinstance(data["test"], web.FileField)
-        assert data["test"].file.read() == b"data"
+        assert await asyncio.to_thread(data["test"].file.read) == b"data"
         assert isinstance(data["some"], web.FileField)
         data["some"].file.close()
         data["test"].file.close()
@@ -1660,17 +1664,15 @@ async def handler(request: web.Request) -> web.Response:
 
 
 async def test_POST_FILES_DEFLATE(
-    aiohttp_client: AiohttpClient, fname: pathlib.Path
+    aiohttp_client: AiohttpClient, fname: pathlib.Path, file_content: bytes
 ) -> None:
     async def handler(request: web.Request) -> web.Response:
         data = await request.post()
         assert isinstance(data["some"], web.FileField)
         assert data["some"].filename == fname.name
-        with fname.open("rb") as f:
-            content1 = f.read()
-        content2 = data["some"].file.read()
+        content = await asyncio.to_thread(data["some"].file.read)
         data["some"].file.close()
-        assert content1 == content2
+        assert content == file_content
         return web.Response()
 
     app = web.Application()
@@ -1720,54 +1722,50 @@ async def handler(request: web.Request) -> web.Response:
 
 
 async def test_POST_FILES_STR(
-    aiohttp_client: AiohttpClient, fname: pathlib.Path
+    aiohttp_client: AiohttpClient, file_content: bytes
 ) -> None:
+    content1 = file_content.decode()
+
     async def handler(request: web.Request) -> web.Response:
         data = await request.post()
-        with fname.open("rb") as f:
-            content1 = f.read().decode()
         content2 = data["some"]
-        assert content1 == content2
+        assert content2 == content1
         return web.Response()
 
     app = web.Application()
     app.router.add_post("/", handler)
     client = await aiohttp_client(app)
 
-    with fname.open("rb") as f:
-        async with client.post("/", data={"some": f.read().decode()}) as resp:
-            assert 200 == resp.status
+    async with client.post("/", data={"some": content1}) as resp:
+        assert 200 == resp.status
 
 
 async def test_POST_FILES_STR_SIMPLE(
-    aiohttp_client: AiohttpClient, fname: pathlib.Path
+    aiohttp_client: AiohttpClient, file_content: bytes
 ) -> None:
+
     async def handler(request: web.Request) -> web.Response:
         data = await request.read()
-        with fname.open("rb") as f:
-            content = f.read()
-        assert content == data
+        assert data == file_content
         return web.Response()
 
     app = web.Application()
     app.router.add_post("/", handler)
     client = await aiohttp_client(app)
 
-    with fname.open("rb") as f:
-        async with client.post("/", data=f.read()) as resp:
-            assert 200 == resp.status
+    async with client.post("/", data=file_content) as resp:
+        assert 200 == resp.status
 
 
 async def test_POST_FILES_LIST(
-    aiohttp_client: AiohttpClient, fname: pathlib.Path
+    aiohttp_client: AiohttpClient, fname: pathlib.Path, file_content: bytes
 ) -> None:
+
     async def handler(request: web.Request) -> web.Response:
         data = await request.post()
         assert isinstance(data["some"], web.FileField)
         assert fname.name == data["some"].filename
-        with fname.open("rb") as f:
-            content = f.read()
-        assert content == data["some"].file.read()
+        assert await asyncio.to_thread(data["some"].file.read) == file_content
         data["some"].file.close()
         return web.Response()
 
@@ -1781,16 +1779,14 @@ async def handler(request: web.Request) -> web.Response:
 
 
 async def test_POST_FILES_CT(
-    aiohttp_client: AiohttpClient, fname: pathlib.Path
+    aiohttp_client: AiohttpClient, fname: pathlib.Path, file_content: bytes
 ) -> None:
     async def handler(request: web.Request) -> web.Response:
         data = await request.post()
         assert isinstance(data["some"], web.FileField)
         assert fname.name == data["some"].filename
         assert "text/plain" == data["some"].content_type
-        with fname.open("rb") as f:
-            content = f.read()
-        assert content == data["some"].file.read()
+        assert await asyncio.to_thread(data["some"].file.read) == file_content
         data["some"].file.close()
         return web.Response()
 
@@ -1806,13 +1802,11 @@ async def handler(request: web.Request) -> web.Response:
 
 
 async def test_POST_FILES_SINGLE(
-    aiohttp_client: AiohttpClient, fname: pathlib.Path
+    aiohttp_client: AiohttpClient, fname: pathlib.Path, file_content: bytes
 ) -> None:
     async def handler(request: web.Request) -> web.Response:
         data = await request.text()
-        with fname.open("rb") as f:
-            content = f.read().decode()
-            assert content == data
+        assert data == file_content.decode()
         # if system cannot determine 'text/x-python' MIME type
         # then use 'application/octet-stream' default
         assert request.content_type in [
@@ -1834,13 +1828,11 @@ async def handler(request: web.Request) -> web.Response:
 
 
 async def test_POST_FILES_SINGLE_content_disposition(
-    aiohttp_client: AiohttpClient, fname: pathlib.Path
+    aiohttp_client: AiohttpClient, fname: pathlib.Path, file_content: bytes
 ) -> None:
     async def handler(request: web.Request) -> web.Response:
         data = await request.text()
-        with fname.open("rb") as f:
-            content = f.read().decode()
-            assert content == data
+        assert data == file_content.decode()
         # if system cannot determine 'application/pgp-keys' MIME type
         # then use 'application/octet-stream' default
         assert request.content_type in [
@@ -1866,13 +1858,11 @@ async def handler(request: web.Request) -> web.Response:
 
 
 async def test_POST_FILES_SINGLE_BINARY(
-    aiohttp_client: AiohttpClient, fname: pathlib.Path
+    aiohttp_client: AiohttpClient, fname: pathlib.Path, file_content: bytes
 ) -> None:
     async def handler(request: web.Request) -> web.Response:
         data = await request.read()
-        with fname.open("rb") as f:
-            content = f.read()
-        assert content == data
+        assert data == file_content
         # if system cannot determine 'application/pgp-keys' MIME type
         # then use 'application/octet-stream' default
         assert request.content_type in [
@@ -1896,7 +1886,7 @@ async def test_POST_FILES_IO(aiohttp_client: AiohttpClient) -> None:
     async def handler(request: web.Request) -> web.Response:
         data = await request.post()
         assert isinstance(data["unknown"], web.FileField)
-        assert b"data" == data["unknown"].file.read()
+        assert b"data" == await asyncio.to_thread(data["unknown"].file.read)
         assert data["unknown"].content_type == "application/octet-stream"
         assert data["unknown"].filename == "unknown"
         data["unknown"].file.close()
@@ -1918,7 +1908,7 @@ async def handler(request: web.Request) -> web.Response:
         assert isinstance(data["unknown"], web.FileField)
         assert data["unknown"].content_type == "application/octet-stream"
         assert data["unknown"].filename == "unknown"
-        assert data["unknown"].file.read() == b"data"
+        assert await asyncio.to_thread(data["unknown"].file.read) == b"data"
         data["unknown"].file.close()
         assert data.getall("q") == ["t1", "t2"]
 
@@ -1937,7 +1927,7 @@ async def handler(request: web.Request) -> web.Response:
 
 
 async def test_POST_FILES_WITH_DATA(
-    aiohttp_client: AiohttpClient, fname: pathlib.Path
+    aiohttp_client: AiohttpClient, fname: pathlib.Path, file_content: bytes
 ) -> None:
     async def handler(request: web.Request) -> web.Response:
         data = await request.post()
@@ -1949,9 +1939,8 @@ async def handler(request: web.Request) -> web.Response:
             "application/octet-stream",
         ]
         assert data["some"].filename == fname.name
-        with fname.open("rb") as f:
-            assert data["some"].file.read() == f.read()
-            data["some"].file.close()
+        assert await asyncio.to_thread(data["some"].file.read) == file_content
+        data["some"].file.close()
 
         return web.Response()
 
@@ -1965,31 +1954,28 @@ async def handler(request: web.Request) -> web.Response:
 
 
 async def test_POST_STREAM_DATA(
-    aiohttp_client: AiohttpClient, fname: pathlib.Path
+    aiohttp_client: AiohttpClient, fname: pathlib.Path, file_content: bytes
 ) -> None:
     async def handler(request: web.Request) -> web.Response:
         assert request.content_type == "application/octet-stream"
         content = await request.read()
-        with fname.open("rb") as f:
-            expected = f.read()
-            assert request.content_length == len(expected)
-            assert content == expected
+        assert request.content_length == len(file_content)
+        assert content == file_content
 
         return web.Response()
 
     app = web.Application()
     app.router.add_post("/", handler)
     client = await aiohttp_client(app)
 
-    with fname.open("rb") as f:
-        data_size = len(f.read())
+    data_size = len(file_content)
 
     async def gen(fname: pathlib.Path) -> AsyncIterator[bytes]:
         with fname.open("rb") as f:
-            data = f.read(100)
+            data = await asyncio.to_thread(f.read, 100)
             while data:
                 yield data
-                data = f.read(100)
+                data = await asyncio.to_thread(f.read, 100)
 
     async with client.post(
         "/", data=gen(fname), headers={"Content-Length": str(data_size)}
@@ -3033,7 +3019,8 @@ async def close(self) -> None:
 def create_server_for_url_and_handler(
     aiohttp_server: AiohttpServer, tls_certificate_authority: trustme.CA
 ) -> Callable[[URL, Handler], Awaitable[TestServer]]:
-    def create(url: URL, srv: Handler) -> Awaitable[TestServer]:
+
+    async def create(url: URL, srv: Handler) -> TestServer:
         app = web.Application()
         app.router.add_route("GET", url.path, srv)
 
@@ -3043,9 +3030,9 @@ def create(url: URL, srv: Handler) -> Awaitable[TestServer]:
                 url.host, "localhost", "127.0.0.1"
             )
             ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
-            cert.configure_cert(ssl_ctx)
-            return aiohttp_server(app, ssl=ssl_ctx)
-        return aiohttp_server(app)
+            await asyncio.to_thread(cert.configure_cert, ssl_ctx)
+            return await aiohttp_server(app, ssl=ssl_ctx)
+        return await aiohttp_server(app)
 
     return create
 
@@ -4179,7 +4166,9 @@ async def not_ok_handler(request: web.Request) -> NoReturn:
 
     file_size_bytes = 1024 * 1024
     file_path = tmp_path / "uploaded.txt"
-    file_path.write_text("0" * file_size_bytes, encoding="utf8")
+    await asyncio.to_thread(
+        file_path.write_text, "0" * file_size_bytes, encoding="utf8"
+    )
 
     with open(file_path, "rb") as file:
         data = {"file": file}