diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml index 7deb55c2..60684969 100644 --- a/.github/actions/setup-env/action.yml +++ b/.github/actions/setup-env/action.yml @@ -58,6 +58,10 @@ runs: shell: bash -el {0} run: mamba install --yes --channel conda-forge redis-server + - name: Install playwright + shell: bash -el {0} + run: playwright install + - name: Install ragna shell: bash -el {0} run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b74f3907..5ff4147d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,10 +68,76 @@ jobs: - name: Run unit tests id: tests - run: pytest --junit-xml=test-results.xml --durations=25 + run: | + pytest \ + --ignore tests/deploy/ui \ + --junit-xml=test-results.xml \ + --durations=25 - name: Surface failing tests if: steps.tests.outcome != 'success' uses: pmeier/pytest-results-action@v0.3.0 with: path: test-results.xml + + pytest-ui: + strategy: + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + browser: + - chromium + - firefox + python-version: + - "3.9" + - "3.10" + - "3.11" + exclude: + - python-version: "3.10" + os: windows-latest + - python-version: "3.11" + os: windows-latest + - python-version: "3.10" + os: macos-latest + - python-version: "3.11" + os: macos-latest + include: + - browser: webkit + os: macos-latest + python-version: "3.9" + + fail-fast: false + + runs-on: ${{ matrix.os }} + + defaults: + run: + shell: bash -el {0} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup environment + uses: ./.github/actions/setup-env + with: + python-version: ${{ matrix.python-version }} + + - name: Run unit tests + id: tests + run: | + pytest tests/deploy/ui \ + --browser ${{ matrix.browser }} \ + --video=retain-on-failure + + - name: Upload playwright video + if: failure() + uses: actions/upload-artifact@v4 + with: + name: + playwright-${{ matrix.os }}-${{ matrix.python-version}}-${{ github.run_id }} + path: test-results diff --git a/environment-dev.yml b/environment-dev.yml index 2a7b6a03..9ee47a1b 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -10,6 +10,7 @@ dependencies: - pytest >=6 - pytest-mock - pytest-asyncio + - pytest-playwright - mypy ==1.10.0 - pre-commit - types-aiofiles diff --git a/tests/deploy/api/test_batch_endpoints.py b/tests/deploy/api/test_batch_endpoints.py index 94740750..3c85c77c 100644 --- a/tests/deploy/api/test_batch_endpoints.py +++ b/tests/deploy/api/test_batch_endpoints.py @@ -3,8 +3,7 @@ from ragna.deploy import Config from ragna.deploy._api import app - -from .utils import authenticate +from tests.deploy.utils import authenticate_with_api def test_batch_sequential_upload_equivalence(tmp_local_root): @@ -23,7 +22,7 @@ def test_batch_sequential_upload_equivalence(tmp_local_root): with TestClient( app(config=Config(), ignore_unavailable_components=False) ) as client: - authenticate(client) + authenticate_with_api(client) document1_upload = ( client.post("/document", json={"name": document_path1.name}) diff --git a/tests/deploy/api/test_components.py b/tests/deploy/api/test_components.py index 65f02209..b7fe464c 100644 --- a/tests/deploy/api/test_components.py +++ b/tests/deploy/api/test_components.py @@ -6,8 +6,7 @@ from ragna.core import RagnaException from ragna.deploy import Config from ragna.deploy._api import app - -from .utils import authenticate +from tests.deploy.utils import authenticate_with_api @pytest.mark.parametrize("ignore_unavailable_components", [True, False]) @@ -27,7 +26,7 @@ def test_ignore_unavailable_components(ignore_unavailable_components): ignore_unavailable_components=ignore_unavailable_components, ) ) as client: - authenticate(client) + authenticate_with_api(client) components = client.get("/components").raise_for_status().json() assert [assistant["title"] for assistant in components["assistants"]] == [ @@ -66,7 +65,7 @@ def test_unknown_component(tmp_local_root): with TestClient( app(config=Config(), ignore_unavailable_components=False) ) as client: - authenticate(client) + authenticate_with_api(client) document_upload = ( client.post("/document", json={"name": document_path.name}) diff --git a/tests/deploy/api/test_e2e.py b/tests/deploy/api/test_e2e.py index 41b154db..4e60127b 100644 --- a/tests/deploy/api/test_e2e.py +++ b/tests/deploy/api/test_e2e.py @@ -1,29 +1,11 @@ import json -import time import pytest from fastapi.testclient import TestClient -from ragna.assistants import RagnaDemoAssistant from ragna.deploy import Config from ragna.deploy._api import app - -from .utils import authenticate - - -class TestAssistant(RagnaDemoAssistant): - def answer(self, prompt, sources, *, multiple_answer_chunks: bool): - # Simulate a "real" assistant through a small delay. See - # https://github.com/Quansight/ragna/pull/401#issuecomment-2095851440 - # for why this is needed. - time.sleep(1e-3) - content = next(super().answer(prompt, sources)) - - if multiple_answer_chunks: - for chunk in content.split(" "): - yield f"{chunk} " - else: - yield content +from tests.deploy.utils import TestAssistant, authenticate_with_api @pytest.mark.parametrize("multiple_answer_chunks", [True, False]) @@ -38,7 +20,7 @@ def test_e2e(tmp_local_root, multiple_answer_chunks, stream_answer): file.write("!\n") with TestClient(app(config=config, ignore_unavailable_components=False)) as client: - authenticate(client) + authenticate_with_api(client) assert client.get("/chats").raise_for_status().json() == [] diff --git a/tests/deploy/api/utils.py b/tests/deploy/api/utils.py deleted file mode 100644 index abcf1411..00000000 --- a/tests/deploy/api/utils.py +++ /dev/null @@ -1,23 +0,0 @@ -import os - -from fastapi.testclient import TestClient - -from ragna.core._utils import default_user - - -def authenticate(client: TestClient) -> None: - username = default_user() - token = ( - client.post( - "/token", - data={ - "username": username, - "password": os.environ.get( - "RAGNA_DEMO_AUTHENTICATION_PASSWORD", username - ), - }, - ) - .raise_for_status() - .json() - ) - client.headers["Authorization"] = f"Bearer {token}" diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py new file mode 100644 index 00000000..85699278 --- /dev/null +++ b/tests/deploy/ui/test_ui.py @@ -0,0 +1,157 @@ +import socket +import subprocess +import sys +import time + +import httpx +import panel as pn +import pytest +from playwright.sync_api import Page, expect + +from ragna._utils import timeout_after +from ragna.deploy import Config +from tests.deploy.utils import TestAssistant + + +def get_available_port(): + with socket.socket() as s: + s.bind(("", 0)) + return s.getsockname()[1] + + +@pytest.fixture +def config( + tmp_local_root, +): + config = Config( + local_root=tmp_local_root, + assistants=[TestAssistant], + ui=dict(port=get_available_port()), + api=dict(port=get_available_port()), + ) + path = tmp_local_root / "ragna.toml" + config.to_file(path) + return config + + +class Server: + def __init__(self, config): + self.config = config + self.base_url = f"http://{config.ui.hostname}:{config.ui.port}" + + def server_up(self): + try: + return httpx.get(self.base_url).is_success + except httpx.ConnectError: + return False + + @timeout_after(60) + def start(self): + self.proc = subprocess.Popen( + [ + sys.executable, + "-m", + "ragna", + "ui", + "--config", + self.config.local_root / "ragna.toml", + "--start-api", + "--ignore-unavailable-components", + "--no-open-browser", + ], + stdout=sys.stdout, + stderr=sys.stderr, + ) + + while not self.server_up(): + time.sleep(1) + + def stop(self): + self.proc.kill() + pn.state.kill_all_servers() + + def __enter__(self): + self.start() + return self + + def __exit__(self, *args): + self.stop() + + +def test_health(config, page: Page) -> None: + with Server(config) as server: + health_url = f"{server.base_url}/health" + response = page.goto(health_url) + assert response.ok + + +def test_start_chat(config, page: Page) -> None: + with Server(config) as server: + # Index page, no auth + index_url = server.base_url + page.goto(index_url) + expect(page.get_by_role("button", name="Sign In")).to_be_visible() + + # Authorize with no credentials + page.get_by_role("button", name="Sign In").click() + expect(page.get_by_role("button", name=" New Chat")).to_be_visible() + + # expect auth token to be set + cookies = page.context.cookies() + assert len(cookies) == 1 + cookie = cookies[0] + assert cookie.get("name") == "auth_token" + auth_token = cookie.get("value") + assert auth_token is not None + + # New page button + new_chat_button = page.get_by_role("button", name=" New Chat") + expect(new_chat_button).to_be_visible() + new_chat_button.click() + + document_root = config.local_root / "documents" + document_root.mkdir() + document_name = "test.txt" + document_path = document_root / document_name + with open(document_path, "w") as file: + file.write("!\n") + + # File upload selector + with page.expect_file_chooser() as fc_info: + page.locator(".fileUpload").click() + file_chooser = fc_info.value + file_chooser.set_files(document_path) + + # Upload document and expect to see it listed + file_list = page.locator(".fileListContainer") + expect(file_list.first).to_have_text(str(document_name)) + + chat_dialog = page.get_by_role("dialog") + expect(chat_dialog).to_be_visible() + start_chat_button = page.get_by_role("button", name="Start Conversation") + expect(start_chat_button).to_be_visible() + time.sleep(0.5) # hack while waiting for button to be fully clickable + start_chat_button.click(delay=5) + + chat_box_row = page.locator(".chat-interface-input-row") + expect(chat_box_row).to_be_visible() + + chat_box = chat_box_row.get_by_role("textbox") + expect(chat_box).to_be_visible() + + # Document should be in the database + chats_url = f"http://{config.api.hostname}:{config.api.port}/chats" + chats = httpx.get( + chats_url, headers={"Authorization": f"Bearer {auth_token}"} + ).json() + assert len(chats) == 1 + chat = chats[0] + chat_documents = chat["metadata"]["documents"] + assert len(chat_documents) == 1 + assert chat_documents[0]["name"] == document_name + + chat_box.fill("Tell me about the documents") + + chat_button = chat_box_row.get_by_role("button") + expect(chat_button).to_be_visible() + chat_button.click() diff --git a/tests/deploy/utils.py b/tests/deploy/utils.py new file mode 100644 index 00000000..76638d4f --- /dev/null +++ b/tests/deploy/utils.py @@ -0,0 +1,44 @@ +import os +import time + +from fastapi.testclient import TestClient + +from ragna.assistants import RagnaDemoAssistant +from ragna.core._utils import default_user + + +class TestAssistant(RagnaDemoAssistant): + def answer(self, prompt, sources, *, multiple_answer_chunks: bool = True): + # Simulate a "real" assistant through a small delay. See + # https://github.com/Quansight/ragna/pull/401#issuecomment-2095851440 + # for why this is needed. + # + # Note: multiple_answer_chunks is given a default value here to satisfy + # the tests in deploy/ui/test_ui.py. This can be removed if TestAssistant + # is ever removed from that file. + time.sleep(1e-3) + content = next(super().answer(prompt, sources)) + + if multiple_answer_chunks: + for chunk in content.split(" "): + yield f"{chunk} " + else: + yield content + + +def authenticate_with_api(client: TestClient) -> None: + username = default_user() + token = ( + client.post( + "/token", + data={ + "username": username, + "password": os.environ.get( + "RAGNA_DEMO_AUTHENTICATION_PASSWORD", username + ), + }, + ) + .raise_for_status() + .json() + ) + client.headers["Authorization"] = f"Bearer {token}"