Update compliance test suite.

Author: aaugustin

Makefile

@@ -8,8 +8,8 @@ build:
 	python setup.py build_ext --inplace
 
 style:
-	ruff format src tests
-	ruff check --fix src tests
+	ruff format compliance src tests
+	ruff check --fix compliance src tests
 
 types:
 	mypy --strict src

compliance/README.rst

@@ -4,47 +4,69 @@ Autobahn Testsuite
 General information and installation instructions are available at
 https://github.com/crossbario/autobahn-testsuite.
 
-To improve performance, you should compile the C extension first::
+Running the test suite
+----------------------
+
+All commands below must be run from the root directory of the repository.
+
+To get acceptable performance, compile the C extension first:
+
+.. code-block:: console
 
     $ python setup.py build_ext --inplace
 
-Running the test suite
-----------------------
+Run each command in a different shell. Testing takes several minutes to complete
+— wstest is the bottleneck. When clients finish, stop servers with Ctrl-C.
+
+You can exclude slow tests by modifying the configuration files as follows::
+
+    "exclude-cases": ["9.*", "12.*", "13.*"]
 
-All commands below must be run from the directory containing this file.
+The test server and client applications shouldn't display any exceptions.
 
-To test the server::
+To test the servers:
 
-    $ PYTHONPATH=.. python test_server.py
-    $ wstest -m fuzzingclient
+.. code-block:: console
 
-To test the client::
+    $ PYTHONPATH=src python compliance/asyncio/server.py
+    $ PYTHONPATH=src python compliance/sync/server.py
 
-    $ wstest -m fuzzingserver
-    $ PYTHONPATH=.. python test_client.py
+    $ docker run --interactive --tty --rm \
+        --volume "${PWD}/compliance/config:/config" \
+        --volume "${PWD}/compliance/reports:/reports" \
+        --name fuzzingclient \
+        crossbario/autobahn-testsuite \
+        wstest --mode fuzzingclient --spec /config/fuzzingclient.json
 
-Run the first command in a shell. Run the second command in another shell.
-It should take about ten minutes to complete — wstest is the bottleneck.
-Then kill the first one with Ctrl-C.
+    $ open reports/servers/index.html
 
-The test client or server shouldn't display any exceptions. The results are
-stored in reports/clients/index.html.
+To test the clients:
 
-Note that the Autobahn software only supports Python 2, while ``websockets``
-only supports Python 3; you need two different environments.
+.. code-block:: console
+    $ docker run --interactive --tty --rm \
+        --volume "${PWD}/compliance/config:/config" \
+        --volume "${PWD}/compliance/reports:/reports" \
+        --publish 9001:9001 \
+        --name fuzzingserver \
+        crossbario/autobahn-testsuite \
+        wstest --mode fuzzingserver --spec /config/fuzzingserver.json
+
+    $ PYTHONPATH=src python compliance/asyncio/client.py
+    $ PYTHONPATH=src python compliance/sync/client.py
+
+    $ open reports/clients/index.html
 
 Conformance notes
 -----------------
 
 Some test cases are more strict than the RFC. Given the implementation of the
-library and the test echo client or server, ``websockets`` gets a "Non-Strict"
-in these cases.
-
-In 3.2, 3.3, 4.1.3, 4.1.4, 4.2.3, 4.2.4, and 5.15 ``websockets`` notices the
-protocol error and closes the connection before it has had a chance to echo
-the previous frame.
+library and the test client and server applications, websockets passes with a
+"Non-Strict" result in these cases.
 
-In 6.4.3 and 6.4.4, even though it uses an incremental decoder, ``websockets``
-doesn't notice the invalid utf-8 fast enough to get a "Strict" pass. These
-tests are more strict than the RFC.
+In 3.2, 3.3, 4.1.3, 4.1.4, 4.2.3, 4.2.4, and 5.15 websockets notices the
+protocol error and closes the connection at the library level before the
+application gets a chance to echo the previous frame.
 
+In 6.4.3 and 6.4.4, even though it uses an incremental decoder, websockets
+doesn't notice the invalid utf-8 fast enough to get a "Strict" pass. These tests
+are more strict than the RFC.

compliance/asyncio/client.py

@@ -0,0 +1,52 @@
+import asyncio
+import json
+import logging
+
+from websockets.asyncio.client import connect
+from websockets.exceptions import WebSocketException
+
+
+logging.basicConfig(level=logging.WARNING)
+
+SERVER = "ws://127.0.0.1:9001"
+
+
+async def get_case_count():
+    async with connect(f"{SERVER}/getCaseCount") as ws:
+        return json.loads(await ws.recv())
+
+
+async def run_case(case):
+    async with connect(
+        f"{SERVER}/runCase?case={case}",
+        user_agent_header="websockets.asyncio",
+        max_size=2**25,
+    ) as ws:
+        async for msg in ws:
+            await ws.send(msg)
+
+
+async def update_reports():
+    async with connect(f"{SERVER}/updateReports", open_timeout=60):
+        pass
+
+
+async def main():
+    cases = await get_case_count()
+    for case in range(1, cases + 1):
+        print(f"Running test case {case:03d} / {cases}... ", end="\t")
+        try:
+            await run_case(case)
+        except WebSocketException as exc:
+            print(f"ERROR: {type(exc).__name__}: {exc}")
+        except Exception as exc:
+            print(f"FAIL: {type(exc).__name__}: {exc}")
+        else:
+            print("OK")
+    print("Ran {cases} test cases")
+    await update_reports()
+    print("Updated reports")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

compliance/asyncio/server.py

@@ -0,0 +1,32 @@
+import asyncio
+import logging
+
+from websockets.asyncio.server import serve
+
+
+logging.basicConfig(level=logging.WARNING)
+
+HOST, PORT = "0.0.0.0", 9002
+
+
+async def echo(ws):
+    async for msg in ws:
+        await ws.send(msg)
+
+
+async def main():
+    async with serve(
+        echo,
+        HOST,
+        PORT,
+        server_header="websockets.sync",
+        max_size=2**25,
+    ) as server:
+        try:
+            await server.serve_forever()
+        except KeyboardInterrupt:
+            pass
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

compliance/config/fuzzingclient.json

@@ -0,0 +1,11 @@
+
+{
+   "servers": [{
+      "url": "ws://host.docker.internal:9002"
+   }, {
+      "url": "ws://host.docker.internal:9003"
+   }],
+   "outdir": "./reports/servers",
+   "cases": ["*"],
+   "exclude-cases": []
+}

compliance/config/fuzzingserver.json

@@ -0,0 +1,7 @@
+
+{
+   "url": "ws://localhost:9001",
+   "outdir": "./reports/clients",
+   "cases": ["*"],
+   "exclude-cases": []
+}

compliance/fuzzingclient.json

@@ -0,0 +1,51 @@
+import json
+import logging
+
+from websockets.exceptions import WebSocketException
+from websockets.sync.client import connect
+
+
+logging.basicConfig(level=logging.WARNING)
+
+SERVER = "ws://127.0.0.1:9001"
+
+
+def get_case_count():
+    with connect(f"{SERVER}/getCaseCount") as ws:
+        return json.loads(ws.recv())
+
+
+def run_case(case):
+    with connect(
+        f"{SERVER}/runCase?case={case}",
+        user_agent_header="websockets.sync",
+        max_size=2**25,
+    ) as ws:
+        for msg in ws:
+            ws.send(msg)
+
+
+def update_reports():
+    with connect(f"{SERVER}/updateReports", open_timeout=60):
+        pass
+
+
+def main():
+    cases = get_case_count()
+    for case in range(1, cases + 1):
+        print(f"Running test case {case:03d} / {cases}... ", end="\t")
+        try:
+            run_case(case)
+        except WebSocketException as exc:
+            print(f"ERROR: {type(exc).__name__}: {exc}")
+        except Exception as exc:
+            print(f"FAIL: {type(exc).__name__}: {exc}")
+        else:
+            print("OK")
+    print("Ran {cases} test cases")
+    update_reports()
+    print("Updated reports")
+
+
+if __name__ == "__main__":
+    main()

compliance/fuzzingserver.json

@@ -0,0 +1,31 @@
+import logging
+
+from websockets.sync.server import serve
+
+
+logging.basicConfig(level=logging.WARNING)
+
+HOST, PORT = "0.0.0.0", 9003
+
+
+def echo(ws):
+    for msg in ws:
+        ws.send(msg)
+
+
+def main():
+    with serve(
+        echo,
+        HOST,
+        PORT,
+        server_header="websockets.asyncio",
+        max_size=2**25,
+    ) as server:
+        try:
+            server.serve_forever()
+        except KeyboardInterrupt:
+            pass
+
+
+if __name__ == "__main__":
+    main()