UI tests with Playwright (#413)

Co-authored-by: Philip Meier <github.pmeier@posteo.de>

Author: blakerosenthal

.github/actions/setup-env/action.yml

@@ -58,6 +58,10 @@ runs:
       shell: bash -el {0}
       run: mamba install --yes --channel conda-forge redis-server
 
+    - name: Install playwright
+      shell: bash -el {0}
+      run: playwright install
+
     - name: Install ragna
       shell: bash -el {0}
       run: |

.github/workflows/test.yml

@@ -68,10 +68,76 @@ jobs:
 
       - name: Run unit tests
         id: tests
-        run: pytest --junit-xml=test-results.xml --durations=25
+        run: |
+          pytest \
+          --ignore tests/deploy/ui \
+          --junit-xml=test-results.xml \
+          --durations=25
 
       - name: Surface failing tests
         if: steps.tests.outcome != 'success'
         uses: pmeier/pytest-results-action@v0.3.0
         with:
           path: test-results.xml
+
+  pytest-ui:
+    strategy:
+      matrix:
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macos-latest
+        browser:
+          - chromium
+          - firefox
+        python-version:
+          - "3.9"
+          - "3.10"
+          - "3.11"
+        exclude:
+          - python-version: "3.10"
+            os: windows-latest
+          - python-version: "3.11"
+            os: windows-latest
+          - python-version: "3.10"
+            os: macos-latest
+          - python-version: "3.11"
+            os: macos-latest
+        include:
+          - browser: webkit
+            os: macos-latest
+            python-version: "3.9"
+
+      fail-fast: false
+
+    runs-on: ${{ matrix.os }}
+
+    defaults:
+      run:
+        shell: bash -el {0}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup environment
+        uses: ./.github/actions/setup-env
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Run unit tests
+        id: tests
+        run: |
+          pytest tests/deploy/ui \
+          --browser ${{ matrix.browser }} \
+          --video=retain-on-failure
+
+      - name: Upload playwright video
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name:
+            playwright-${{ matrix.os }}-${{ matrix.python-version}}-${{ github.run_id }}
+          path: test-results

environment-dev.yml

@@ -10,6 +10,7 @@ dependencies:
       - pytest >=6
       - pytest-mock
       - pytest-asyncio
+      - pytest-playwright
       - mypy ==1.10.0
       - pre-commit
       - types-aiofiles

tests/deploy/api/test_batch_endpoints.py

@@ -3,8 +3,7 @@
 
 from ragna.deploy import Config
 from ragna.deploy._api import app
-
-from .utils import authenticate
+from tests.deploy.utils import authenticate_with_api
 
 
 def test_batch_sequential_upload_equivalence(tmp_local_root):
@@ -23,7 +22,7 @@ def test_batch_sequential_upload_equivalence(tmp_local_root):
     with TestClient(
         app(config=Config(), ignore_unavailable_components=False)
     ) as client:
-        authenticate(client)
+        authenticate_with_api(client)
 
         document1_upload = (
             client.post("/document", json={"name": document_path1.name})

tests/deploy/api/test_components.py

@@ -6,8 +6,7 @@
 from ragna.core import RagnaException
 from ragna.deploy import Config
 from ragna.deploy._api import app
-
-from .utils import authenticate
+from tests.deploy.utils import authenticate_with_api
 
 
 @pytest.mark.parametrize("ignore_unavailable_components", [True, False])
@@ -27,7 +26,7 @@ def test_ignore_unavailable_components(ignore_unavailable_components):
                 ignore_unavailable_components=ignore_unavailable_components,
             )
         ) as client:
-            authenticate(client)
+            authenticate_with_api(client)
 
             components = client.get("/components").raise_for_status().json()
             assert [assistant["title"] for assistant in components["assistants"]] == [
@@ -66,7 +65,7 @@ def test_unknown_component(tmp_local_root):
     with TestClient(
         app(config=Config(), ignore_unavailable_components=False)
     ) as client:
-        authenticate(client)
+        authenticate_with_api(client)
 
         document_upload = (
             client.post("/document", json={"name": document_path.name})

tests/deploy/api/test_e2e.py

@@ -1,29 +1,11 @@
 import json
-import time
 
 import pytest
 from fastapi.testclient import TestClient
 
-from ragna.assistants import RagnaDemoAssistant
 from ragna.deploy import Config
 from ragna.deploy._api import app
-
-from .utils import authenticate
-
-
-class TestAssistant(RagnaDemoAssistant):
-    def answer(self, prompt, sources, *, multiple_answer_chunks: bool):
-        # Simulate a "real" assistant through a small delay. See
-        # https://github.com/Quansight/ragna/pull/401#issuecomment-2095851440
-        # for why this is needed.
-        time.sleep(1e-3)
-        content = next(super().answer(prompt, sources))
-
-        if multiple_answer_chunks:
-            for chunk in content.split(" "):
-                yield f"{chunk} "
-        else:
-            yield content
+from tests.deploy.utils import TestAssistant, authenticate_with_api
 
 
 @pytest.mark.parametrize("multiple_answer_chunks", [True, False])
@@ -38,7 +20,7 @@ def test_e2e(tmp_local_root, multiple_answer_chunks, stream_answer):
         file.write("!\n")
 
     with TestClient(app(config=config, ignore_unavailable_components=False)) as client:
-        authenticate(client)
+        authenticate_with_api(client)
 
         assert client.get("/chats").raise_for_status().json() == []
 

tests/deploy/api/utils.py

@@ -0,0 +1,157 @@
+import socket
+import subprocess
+import sys
+import time
+
+import httpx
+import panel as pn
+import pytest
+from playwright.sync_api import Page, expect
+
+from ragna._utils import timeout_after
+from ragna.deploy import Config
+from tests.deploy.utils import TestAssistant
+
+
+def get_available_port():
+    with socket.socket() as s:
+        s.bind(("", 0))
+        return s.getsockname()[1]
+
+
+@pytest.fixture
+def config(
+    tmp_local_root,
+):
+    config = Config(
+        local_root=tmp_local_root,
+        assistants=[TestAssistant],
+        ui=dict(port=get_available_port()),
+        api=dict(port=get_available_port()),
+    )
+    path = tmp_local_root / "ragna.toml"
+    config.to_file(path)
+    return config
+
+
+class Server:
+    def __init__(self, config):
+        self.config = config
+        self.base_url = f"http://{config.ui.hostname}:{config.ui.port}"
+
+    def server_up(self):
+        try:
+            return httpx.get(self.base_url).is_success
+        except httpx.ConnectError:
+            return False
+
+    @timeout_after(60)
+    def start(self):
+        self.proc = subprocess.Popen(
+            [
+                sys.executable,
+                "-m",
+                "ragna",
+                "ui",
+                "--config",
+                self.config.local_root / "ragna.toml",
+                "--start-api",
+                "--ignore-unavailable-components",
+                "--no-open-browser",
+            ],
+            stdout=sys.stdout,
+            stderr=sys.stderr,
+        )
+
+        while not self.server_up():
+            time.sleep(1)
+
+    def stop(self):
+        self.proc.kill()
+        pn.state.kill_all_servers()
+
+    def __enter__(self):
+        self.start()
+        return self
+
+    def __exit__(self, *args):
+        self.stop()
+
+
+def test_health(config, page: Page) -> None:
+    with Server(config) as server:
+        health_url = f"{server.base_url}/health"
+        response = page.goto(health_url)
+        assert response.ok
+
+
+def test_start_chat(config, page: Page) -> None:
+    with Server(config) as server:
+        # Index page, no auth
+        index_url = server.base_url
+        page.goto(index_url)
+        expect(page.get_by_role("button", name="Sign In")).to_be_visible()
+
+        # Authorize with no credentials
+        page.get_by_role("button", name="Sign In").click()
+        expect(page.get_by_role("button", name=" New Chat")).to_be_visible()
+
+        # expect auth token to be set
+        cookies = page.context.cookies()
+        assert len(cookies) == 1
+        cookie = cookies[0]
+        assert cookie.get("name") == "auth_token"
+        auth_token = cookie.get("value")
+        assert auth_token is not None
+
+        # New page button
+        new_chat_button = page.get_by_role("button", name=" New Chat")
+        expect(new_chat_button).to_be_visible()
+        new_chat_button.click()
+
+        document_root = config.local_root / "documents"
+        document_root.mkdir()
+        document_name = "test.txt"
+        document_path = document_root / document_name
+        with open(document_path, "w") as file:
+            file.write("!\n")
+
+        # File upload selector
+        with page.expect_file_chooser() as fc_info:
+            page.locator(".fileUpload").click()
+        file_chooser = fc_info.value
+        file_chooser.set_files(document_path)
+
+        # Upload document and expect to see it listed
+        file_list = page.locator(".fileListContainer")
+        expect(file_list.first).to_have_text(str(document_name))
+
+        chat_dialog = page.get_by_role("dialog")
+        expect(chat_dialog).to_be_visible()
+        start_chat_button = page.get_by_role("button", name="Start Conversation")
+        expect(start_chat_button).to_be_visible()
+        time.sleep(0.5)  # hack while waiting for button to be fully clickable
+        start_chat_button.click(delay=5)
+
+        chat_box_row = page.locator(".chat-interface-input-row")
+        expect(chat_box_row).to_be_visible()
+
+        chat_box = chat_box_row.get_by_role("textbox")
+        expect(chat_box).to_be_visible()
+
+        # Document should be in the database
+        chats_url = f"http://{config.api.hostname}:{config.api.port}/chats"
+        chats = httpx.get(
+            chats_url, headers={"Authorization": f"Bearer {auth_token}"}
+        ).json()
+        assert len(chats) == 1
+        chat = chats[0]
+        chat_documents = chat["metadata"]["documents"]
+        assert len(chat_documents) == 1
+        assert chat_documents[0]["name"] == document_name
+
+        chat_box.fill("Tell me about the documents")
+
+        chat_button = chat_box_row.get_by_role("button")
+        expect(chat_button).to_be_visible()
+        chat_button.click()