From 7c58c7f21b74e293560cefee3ec13158b168aa54 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Wed, 8 May 2024 12:12:40 -0700 Subject: [PATCH 01/39] wip --- environment-dev.yml | 2 + tests/deploy/ui/test_ui.py | 85 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 tests/deploy/ui/test_ui.py diff --git a/environment-dev.yml b/environment-dev.yml index 2a7b6a031..c4bcd6740 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -16,6 +16,8 @@ dependencies: - sqlalchemy-stubs - setuptools-scm - pip-tools + - pytest-playwright + - playwright # documentation - mkdocs - mkdocs-material diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py new file mode 100644 index 000000000..ce8dc1e97 --- /dev/null +++ b/tests/deploy/ui/test_ui.py @@ -0,0 +1,85 @@ +import os + +import pytest +from fastapi.testclient import TestClient +from playwright.sync_api import expect, sync_playwright + +from ragna.assistants import RagnaDemoAssistant +from ragna.core._utils import default_user +from ragna.deploy import Config +from ragna.deploy._api import app as api_app +from ragna.deploy._ui import app as ui_app + + +def authenticate(client: TestClient) -> None: + username = default_user() + token = ( + client.post( + "/token", + data={ + "username": username, + "password": os.environ.get( + "RAGNA_DEMO_AUTHENTICATION_PASSWORD", username + ), + }, + ) + .raise_for_status() + .json() + ) + client.headers["Authorization"] = f"Bearer {token}" + + +class TestAssistant(RagnaDemoAssistant): + def answer(self, prompt, sources, *, multiple_answer_chunks: bool): + content = next(super().answer(prompt, sources)) + + if multiple_answer_chunks: + for chunk in content.split(" "): + yield f"{chunk} " + else: + yield content + + +@pytest.fixture(scope="function") +def config(tmp_local_root): + return Config(local_root=tmp_local_root, assistants=[TestAssistant]) + + +@pytest.fixture(scope="function") +def server(config, open_browser=False): + document_root = config.local_root / "documents" + document_root.mkdir() + document_path = document_root / "test.txt" + with open(document_path, "w") as file: + file.write("!\n") + + with TestClient( + api_app(config=config, ignore_unavailable_components=True) + ) as client: + authenticate(client) + + server = ui_app(config=config, open_browser=open_browser) + server.serve() + + yield server + + +def test_ui(server) -> None: + with sync_playwright() as playwright: + browser = playwright.chromium.launch(headless=False) + context = browser.new_context() + page = context.new_page() + url = server.origins[0] + page.goto(url) + expect(page.get_by_role("button", name="Sign In")).to_be_visible() + page.get_by_role("button", name="Sign In").click() + expect(page.get_by_role("button", name=" New Chat")).to_be_visible() + # page.locator("div").filter( + # has_text=re.compile(r"^Source storageChromaLanceDBRagna/DemoSourceStorage$") + # ).get_by_role("combobox").select_option("LanceDB") + # page.get_by_role("button", name="Advanced Configurations ▶").click() + # page.locator("#fileUpload-p2365").click() + + # --------------------- + context.close() + browser.close() From e36f4b48dfbbd4ca4a327363f5aacc2e51d74bf4 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Sun, 12 May 2024 13:26:42 -0700 Subject: [PATCH 02/39] working health page --- environment-dev.yml | 5 ++-- tests/deploy/ui/test_ui.py | 47 ++++++++++++++++++++++++-------------- 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index c4bcd6740..4ebb63a71 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -10,14 +10,15 @@ dependencies: - pytest >=6 - pytest-mock - pytest-asyncio + - pytest-httpx + - pytest-playwright + - playwright - mypy ==1.10.0 - pre-commit - types-aiofiles - sqlalchemy-stubs - setuptools-scm - pip-tools - - pytest-playwright - - playwright # documentation - mkdocs - mkdocs-material diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index ce8dc1e97..a94357af6 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -3,6 +3,7 @@ import pytest from fastapi.testclient import TestClient from playwright.sync_api import expect, sync_playwright +from pytest_httpx import HTTPXMock from ragna.assistants import RagnaDemoAssistant from ragna.core._utils import default_user @@ -40,40 +41,52 @@ def answer(self, prompt, sources, *, multiple_answer_chunks: bool): yield content +@pytest.fixture(scope="session") +def headless_mode(pytestconfig): + return pytestconfig.getoption("headed") or False + + @pytest.fixture(scope="function") def config(tmp_local_root): return Config(local_root=tmp_local_root, assistants=[TestAssistant]) @pytest.fixture(scope="function") -def server(config, open_browser=False): - document_root = config.local_root / "documents" - document_root.mkdir() - document_path = document_root / "test.txt" - with open(document_path, "w") as file: - file.write("!\n") - +def api_client(config): with TestClient( - api_app(config=config, ignore_unavailable_components=True) + api_app(config=config, ignore_unavailable_components=True), + base_url=config.api.url, ) as client: authenticate(client) - server = ui_app(config=config, open_browser=open_browser) - server.serve() + yield client - yield server +@pytest.fixture(scope="function") +def server(config, api_client, httpx_mock: HTTPXMock, open_browser=False): + # httpx_mock.add_response(url=config.api.url) + server = ui_app(config=config, open_browser=open_browser) + return server -def test_ui(server) -> None: + +# @pytest.mark.asyncio +# async def test_ui(config, api_client, server, httpx_mock: HTTPXMock): +# httpx_mock.add_response() +# server.serve() + + +def test_health_page(config, server, headless_mode) -> None: with sync_playwright() as playwright: - browser = playwright.chromium.launch(headless=False) + server.serve() + browser = playwright.chromium.launch(headless=headless_mode) context = browser.new_context() page = context.new_page() - url = server.origins[0] + url = config.ui.origins[0] + "/health" page.goto(url) - expect(page.get_by_role("button", name="Sign In")).to_be_visible() - page.get_by_role("button", name="Sign In").click() - expect(page.get_by_role("button", name=" New Chat")).to_be_visible() + expect(page.get_by_role("heading", name="Ok")).to_be_visible() + # expect(page.get_by_role("button", name="Sign In")).to_be_visible() + # page.get_by_role("button", name="Sign In").click() + # expect(page.get_by_role("button", name=" New Chat")).to_be_visible() # page.locator("div").filter( # has_text=re.compile(r"^Source storageChromaLanceDBRagna/DemoSourceStorage$") # ).get_by_role("combobox").select_option("LanceDB") From 45a7f3342649a013d3a4a15fcaaf4254c9e62db3 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Mon, 13 May 2024 11:18:28 -0700 Subject: [PATCH 03/39] working api server --- tests/deploy/ui/test_ui.py | 66 +++++++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 15 deletions(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index a94357af6..75a9211a2 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -1,9 +1,12 @@ import os +import time +from multiprocessing import Process +import httpx import pytest +import uvicorn from fastapi.testclient import TestClient from playwright.sync_api import expect, sync_playwright -from pytest_httpx import HTTPXMock from ragna.assistants import RagnaDemoAssistant from ragna.core._utils import default_user @@ -62,29 +65,63 @@ def api_client(config): yield client +TEST_UI_HOSTNAME = "http://localhost" +TEST_API_PORT = "8769" + + @pytest.fixture(scope="function") -def server(config, api_client, httpx_mock: HTTPXMock, open_browser=False): - # httpx_mock.add_response(url=config.api.url) - server = ui_app(config=config, open_browser=open_browser) - return server +def api_server(config): + def start_server(): + uvicorn.run( + api_app( + config=config, + ignore_unavailable_components=True, + ), + host=config.api.hostname, + port=config.api.port, + ) + + def server_up(): + try: + return httpx.get(config.api.url).is_success + except httpx.ConnectError: + return False + + proc = Process(target=start_server, args=(), daemon=True) + proc.start() + + timeout = 5 + while timeout < 0 and not server_up(): + print(f"Waiting for API server to come up on {config.api.url}") + time.sleep(1) + timeout -= 1 + yield proc + proc.kill() -# @pytest.mark.asyncio -# async def test_ui(config, api_client, server, httpx_mock: HTTPXMock): -# httpx_mock.add_response() -# server.serve() + +@pytest.fixture(scope="function") +def ui_server(config, api_server, open_browser=False): + server = ui_app(config=config, open_browser=open_browser) + return server -def test_health_page(config, server, headless_mode) -> None: +def test_ui(config, ui_server, headless_mode) -> None: with sync_playwright() as playwright: - server.serve() + ui_server.serve() + browser = playwright.chromium.launch(headless=headless_mode) context = browser.new_context() page = context.new_page() - url = config.ui.origins[0] + "/health" - page.goto(url) + + health_url = config.ui.origins[0] + "/health" + page.goto(health_url) expect(page.get_by_role("heading", name="Ok")).to_be_visible() - # expect(page.get_by_role("button", name="Sign In")).to_be_visible() + + index_url = config.ui.origins[0] + page.goto(index_url) + expect(page.get_by_role("button", name="Sign In")).to_be_visible() + # page.get_by_role("button", name="Sign In").click() # expect(page.get_by_role("button", name=" New Chat")).to_be_visible() # page.locator("div").filter( @@ -93,6 +130,5 @@ def test_health_page(config, server, headless_mode) -> None: # page.get_by_role("button", name="Advanced Configurations ▶").click() # page.locator("#fileUpload-p2365").click() - # --------------------- context.close() browser.close() From b83aafe1a973c3c9e64cf2b01556306d9e84fb0f Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Mon, 13 May 2024 11:34:19 -0700 Subject: [PATCH 04/39] fix headless mode --- tests/deploy/ui/test_ui.py | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 75a9211a2..237136557 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -45,7 +45,7 @@ def answer(self, prompt, sources, *, multiple_answer_chunks: bool): @pytest.fixture(scope="session") -def headless_mode(pytestconfig): +def headed_mode(pytestconfig): return pytestconfig.getoption("headed") or False @@ -54,17 +54,6 @@ def config(tmp_local_root): return Config(local_root=tmp_local_root, assistants=[TestAssistant]) -@pytest.fixture(scope="function") -def api_client(config): - with TestClient( - api_app(config=config, ignore_unavailable_components=True), - base_url=config.api.url, - ) as client: - authenticate(client) - - yield client - - TEST_UI_HOSTNAME = "http://localhost" TEST_API_PORT = "8769" @@ -101,16 +90,16 @@ def server_up(): @pytest.fixture(scope="function") -def ui_server(config, api_server, open_browser=False): - server = ui_app(config=config, open_browser=open_browser) +def ui_server(config, api_server): + server = ui_app(config=config, open_browser=False) return server -def test_ui(config, ui_server, headless_mode) -> None: +def test_ui(config, ui_server, headed_mode) -> None: with sync_playwright() as playwright: ui_server.serve() - browser = playwright.chromium.launch(headless=headless_mode) + browser = playwright.chromium.launch(headless=not headed_mode) context = browser.new_context() page = context.new_page() From 463f564dbdb491c72fabc7c10582e3311391e291 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Mon, 13 May 2024 11:36:41 -0700 Subject: [PATCH 05/39] fix timeout logic --- tests/deploy/ui/test_ui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 237136557..d51b3baa6 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -80,7 +80,7 @@ def server_up(): proc.start() timeout = 5 - while timeout < 0 and not server_up(): + while timeout > 0 and not server_up(): print(f"Waiting for API server to come up on {config.api.url}") time.sleep(1) timeout -= 1 From c6f21537130e30aac1c7672c98e4690de875980b Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Mon, 13 May 2024 18:21:46 -0700 Subject: [PATCH 06/39] add more tests, auth header not working --- environment-dev.yml | 1 - tests/deploy/ui/test_ui.py | 130 +++++++++++++++++++++++++------------ 2 files changed, 88 insertions(+), 43 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index 4ebb63a71..41ba2138d 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -10,7 +10,6 @@ dependencies: - pytest >=6 - pytest-mock - pytest-asyncio - - pytest-httpx - pytest-playwright - playwright - mypy ==1.10.0 diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index d51b3baa6..b831b3f27 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -3,9 +3,9 @@ from multiprocessing import Process import httpx +import panel as pn import pytest import uvicorn -from fastapi.testclient import TestClient from playwright.sync_api import expect, sync_playwright from ragna.assistants import RagnaDemoAssistant @@ -15,24 +15,6 @@ from ragna.deploy._ui import app as ui_app -def authenticate(client: TestClient) -> None: - username = default_user() - token = ( - client.post( - "/token", - data={ - "username": username, - "password": os.environ.get( - "RAGNA_DEMO_AUTHENTICATION_PASSWORD", username - ), - }, - ) - .raise_for_status() - .json() - ) - client.headers["Authorization"] = f"Bearer {token}" - - class TestAssistant(RagnaDemoAssistant): def answer(self, prompt, sources, *, multiple_answer_chunks: bool): content = next(super().answer(prompt, sources)) @@ -49,7 +31,7 @@ def headed_mode(pytestconfig): return pytestconfig.getoption("headed") or False -@pytest.fixture(scope="function") +@pytest.fixture def config(tmp_local_root): return Config(local_root=tmp_local_root, assistants=[TestAssistant]) @@ -58,7 +40,7 @@ def config(tmp_local_root): TEST_API_PORT = "8769" -@pytest.fixture(scope="function") +@pytest.fixture def api_server(config): def start_server(): uvicorn.run( @@ -89,35 +71,99 @@ def server_up(): proc.kill() +def auth_header(base_url): + username = default_user() + token = ( + httpx.post( + base_url + "/token", + data={ + "username": username, + "password": os.environ.get( + "RAGNA_DEMO_AUTHENTICATION_PASSWORD", username + ), + }, + ) + .raise_for_status() + .json() + ) + + return f"Bearer {token}" + + @pytest.fixture(scope="function") -def ui_server(config, api_server): +def page(config, api_server, headed_mode): server = ui_app(config=config, open_browser=False) - return server - -def test_ui(config, ui_server, headed_mode) -> None: with sync_playwright() as playwright: - ui_server.serve() - + server.serve() browser = playwright.chromium.launch(headless=not headed_mode) context = browser.new_context() page = context.new_page() - health_url = config.ui.origins[0] + "/health" - page.goto(health_url) - expect(page.get_by_role("heading", name="Ok")).to_be_visible() - - index_url = config.ui.origins[0] - page.goto(index_url) - expect(page.get_by_role("button", name="Sign In")).to_be_visible() - - # page.get_by_role("button", name="Sign In").click() - # expect(page.get_by_role("button", name=" New Chat")).to_be_visible() - # page.locator("div").filter( - # has_text=re.compile(r"^Source storageChromaLanceDBRagna/DemoSourceStorage$") - # ).get_by_role("combobox").select_option("LanceDB") - # page.get_by_role("button", name="Advanced Configurations ▶").click() - # page.locator("#fileUpload-p2365").click() + yield page context.close() browser.close() + pn.state.kill_all_servers() + + +def test_health(config, page) -> None: + health_url = config.ui.origins[0] + "/health" + page.goto(health_url) + expect(page.get_by_role("heading", name="Ok")).to_be_visible() + + +def test_index_with_blank_credentials(config, page) -> None: + # Index page, no auth + index_url = config.ui.origins[0] + page.goto(index_url) + expect(page.get_by_role("button", name="Sign In")).to_be_visible() + + # Authorize with no credentials + page.get_by_role("button", name="Sign In").click() + expect(page.get_by_role("button", name=" New Chat")).to_be_visible() + + +def test_index_with_auth_header(config, page) -> None: + pn.state.headers["Authorization"] = auth_header(config.api.url) + + index_url = config.ui.origins[0] + page.goto(index_url) + expect(page.get_by_role("button", name=" New Chat")).to_be_visible() + + # page.locator("div").filter( + # has_text=re.compile(r"^Source storageChromaLanceDBRagna/DemoSourceStorage$") + # ).get_by_role("combobox").select_option("LanceDB") + # page.get_by_role("button", name="Advanced Configurations ▶").click() + # page.locator("#fileUpload-p2365").click() + + +# def test_ui_with_auth_headers(config, ui_server, headed_mode) -> None: +# with sync_playwright() as playwright: +# +# browser = playwright.chromium.launch(headless=not headed_mode) +# context = browser.new_context() +# page = context.new_page() +# +# # Health page +# health_url = config.ui.origins[0] + "/health" +# page.goto(health_url) +# expect(page.get_by_role("heading", name="Ok")).to_be_visible() +# +# # Index page, no auth +# index_url = config.ui.origins[0] +# page.goto(index_url) +# expect(page.get_by_role("button", name="Sign In")).to_be_visible() +# +# # Authorize with no credentials +# page.get_by_role("button", name="Sign In").click() +# expect(page.get_by_role("button", name=" New Chat")).to_be_visible() +# +# # page.locator("div").filter( +# # has_text=re.compile(r"^Source storageChromaLanceDBRagna/DemoSourceStorage$") +# # ).get_by_role("combobox").select_option("LanceDB") +# # page.get_by_role("button", name="Advanced Configurations ▶").click() +# # page.locator("#fileUpload-p2365").click() +# +# context.close() +# browser.close() From c5f3eee7ba641d8d3319154f7e3e8193be5feabe Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Tue, 14 May 2024 09:18:55 -0700 Subject: [PATCH 07/39] assign text ports and skip failing auth test --- ragna/deploy/__init__.py | 2 +- tests/deploy/ui/test_ui.py | 58 +++++++++++++------------------------- 2 files changed, 20 insertions(+), 40 deletions(-) diff --git a/ragna/deploy/__init__.py b/ragna/deploy/__init__.py index f3a86255c..d8b2aab89 100644 --- a/ragna/deploy/__init__.py +++ b/ragna/deploy/__init__.py @@ -5,7 +5,7 @@ ] from ._authentication import Authentication, RagnaDemoAuthentication -from ._config import Config +from ._config import ApiConfig, Config, UiConfig # isort: split diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index b831b3f27..31988f142 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -10,10 +10,13 @@ from ragna.assistants import RagnaDemoAssistant from ragna.core._utils import default_user -from ragna.deploy import Config +from ragna.deploy import ApiConfig, Config, UiConfig from ragna.deploy._api import app as api_app from ragna.deploy._ui import app as ui_app +TEST_API_PORT = "8769" +TEST_UI_PORT = "8770" + class TestAssistant(RagnaDemoAssistant): def answer(self, prompt, sources, *, multiple_answer_chunks: bool): @@ -33,11 +36,12 @@ def headed_mode(pytestconfig): @pytest.fixture def config(tmp_local_root): - return Config(local_root=tmp_local_root, assistants=[TestAssistant]) - - -TEST_UI_HOSTNAME = "http://localhost" -TEST_API_PORT = "8769" + return Config( + local_root=tmp_local_root, + assistants=[TestAssistant], + ui=UiConfig(port=TEST_UI_PORT), + api=ApiConfig(port=TEST_API_PORT), + ) @pytest.fixture @@ -121,11 +125,18 @@ def test_index_with_blank_credentials(config, page) -> None: # Authorize with no credentials page.get_by_role("button", name="Sign In").click() + expect(page.get_by_role("button", name=" New Chat")).to_be_visible() -def test_index_with_auth_header(config, page) -> None: - pn.state.headers["Authorization"] = auth_header(config.api.url) +@pytest.mark.skip(reason="Need to figure out how to set the auth token") +def test_index_with_auth_token(config, page) -> None: + auth = auth_header(config.api.url) + assert len(auth) > len("Bearer ") + + page.set_extra_http_headers({"Authorization": auth}) # this doesn't work + pn.state.cookies["auth_token"] = "" # or this + pn.state.headers["Authorization"] = auth # or this index_url = config.ui.origins[0] page.goto(index_url) @@ -136,34 +147,3 @@ def test_index_with_auth_header(config, page) -> None: # ).get_by_role("combobox").select_option("LanceDB") # page.get_by_role("button", name="Advanced Configurations ▶").click() # page.locator("#fileUpload-p2365").click() - - -# def test_ui_with_auth_headers(config, ui_server, headed_mode) -> None: -# with sync_playwright() as playwright: -# -# browser = playwright.chromium.launch(headless=not headed_mode) -# context = browser.new_context() -# page = context.new_page() -# -# # Health page -# health_url = config.ui.origins[0] + "/health" -# page.goto(health_url) -# expect(page.get_by_role("heading", name="Ok")).to_be_visible() -# -# # Index page, no auth -# index_url = config.ui.origins[0] -# page.goto(index_url) -# expect(page.get_by_role("button", name="Sign In")).to_be_visible() -# -# # Authorize with no credentials -# page.get_by_role("button", name="Sign In").click() -# expect(page.get_by_role("button", name=" New Chat")).to_be_visible() -# -# # page.locator("div").filter( -# # has_text=re.compile(r"^Source storageChromaLanceDBRagna/DemoSourceStorage$") -# # ).get_by_role("combobox").select_option("LanceDB") -# # page.get_by_role("button", name="Advanced Configurations ▶").click() -# # page.locator("#fileUpload-p2365").click() -# -# context.close() -# browser.close() From c9ccca886b5b15c7a2f048d3a6a2ce5b22c64f61 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Tue, 14 May 2024 10:28:18 -0700 Subject: [PATCH 08/39] placeholder test for ui chat --- tests/deploy/ui/test_ui.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 31988f142..32bf5e7d6 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -14,8 +14,8 @@ from ragna.deploy._api import app as api_app from ragna.deploy._ui import app as ui_app -TEST_API_PORT = "8769" -TEST_UI_PORT = "8770" +TEST_API_PORT = "38769" +TEST_UI_PORT = "38770" class TestAssistant(RagnaDemoAssistant): @@ -142,8 +142,29 @@ def test_index_with_auth_token(config, page) -> None: page.goto(index_url) expect(page.get_by_role("button", name=" New Chat")).to_be_visible() - # page.locator("div").filter( - # has_text=re.compile(r"^Source storageChromaLanceDBRagna/DemoSourceStorage$") - # ).get_by_role("combobox").select_option("LanceDB") - # page.get_by_role("button", name="Advanced Configurations ▶").click() - # page.locator("#fileUpload-p2365").click() + +@pytest.mark.skip(reason="TODO: figure out best locators") +def test_new_chat(config, page) -> None: + index_url = config.ui.origins[0] + page.goto(index_url) + expect(page.get_by_role("button", name="Sign In")).to_be_visible() + page.get_by_role("button", name="Sign In").click() + + expect(page.get_by_role("button", name=" New Chat")).to_be_visible() + page.get_by_role("button", name=" New Chat").click() + + expect(page.locator("#fileUpload-p4447")).to_be_visible() + page.locator("#fileUpload-p4447").click() + + page.locator("#fileUpload-p4447").set_input_files() + page.get_by_role("button", name="Start Conversation").click() + page.get_by_text("How can I help you with the").click() + page.get_by_placeholder("Ask a question about the").click() + page.get_by_placeholder("Ask a question about the").fill( + "Tell me about the documents" + ) + page.get_by_role("button", name="").click() + page.get_by_role("button", name=" Source Info").click() + page.locator("#main div").filter(has_text="Source Info ¶ This response").nth( + 3 + ).click() From 1a6f840311543c3fe673694704910b79aac6e63e Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Tue, 14 May 2024 11:42:09 -0700 Subject: [PATCH 09/39] install playwright to gh env --- .github/actions/setup-env/action.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml index 7deb55c29..99a8942cc 100644 --- a/.github/actions/setup-env/action.yml +++ b/.github/actions/setup-env/action.yml @@ -58,6 +58,11 @@ runs: shell: bash -el {0} run: mamba install --yes --channel conda-forge redis-server + - name: Install playwright + if: steps.cache.outputs.cache-hit != 'true' + shell: bash -el {0} + run: playwright install + - name: Install ragna shell: bash -el {0} run: | From 1dc7548cc4fafea4c77741a3c260f48c50a5c494 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Tue, 14 May 2024 12:54:39 -0700 Subject: [PATCH 10/39] make an api wrapper to try to get around the pickling issue on mac/windows --- tests/deploy/ui/test_ui.py | 47 ++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 32bf5e7d6..43aa35c22 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -44,35 +44,48 @@ def config(tmp_local_root): ) -@pytest.fixture -def api_server(config): - def start_server(): +class ApiServer: + def __init__(self, config): + self.config = config + + def start_server(self): uvicorn.run( api_app( - config=config, + config=self.config, ignore_unavailable_components=True, ), - host=config.api.hostname, - port=config.api.port, + host=self.config.api.hostname, + port=self.config.api.port, ) - def server_up(): + def server_up(self): try: - return httpx.get(config.api.url).is_success + return httpx.get(self.config.api.url).is_success except httpx.ConnectError: return False - proc = Process(target=start_server, args=(), daemon=True) - proc.start() + def start(self): + self.proc = Process(target=self.start_server, args=(), daemon=True) + self.proc.start() + + timeout = 5 + while timeout > 0 and not self.server_up(): + print(f"Waiting for API server to come up on {self.config.api.url}") + time.sleep(1) + timeout -= 1 - timeout = 5 - while timeout > 0 and not server_up(): - print(f"Waiting for API server to come up on {config.api.url}") - time.sleep(1) - timeout -= 1 + def stop(self): + self.proc.kill() - yield proc - proc.kill() + +@pytest.fixture +def api_server(config): + server = ApiServer(config) + try: + server.start() + yield server + finally: + server.stop() def auth_header(base_url): From 1c81128f8e64418e36cef793f7471f031b3f6638 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Tue, 14 May 2024 13:08:10 -0700 Subject: [PATCH 11/39] always install playwright --- .github/actions/setup-env/action.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml index 99a8942cc..606849699 100644 --- a/.github/actions/setup-env/action.yml +++ b/.github/actions/setup-env/action.yml @@ -59,7 +59,6 @@ runs: run: mamba install --yes --channel conda-forge redis-server - name: Install playwright - if: steps.cache.outputs.cache-hit != 'true' shell: bash -el {0} run: playwright install From cd68b85317520c56287fff5e3a84435396580db1 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Thu, 16 May 2024 11:47:21 -0700 Subject: [PATCH 12/39] some PR review fixes --- environment-dev.yml | 1 - ragna/deploy/__init__.py | 2 +- tests/deploy/ui/test_ui.py | 50 +++++++++++++++++++++++++------------- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/environment-dev.yml b/environment-dev.yml index 41ba2138d..9ee47a1b6 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -11,7 +11,6 @@ dependencies: - pytest-mock - pytest-asyncio - pytest-playwright - - playwright - mypy ==1.10.0 - pre-commit - types-aiofiles diff --git a/ragna/deploy/__init__.py b/ragna/deploy/__init__.py index d8b2aab89..f3a86255c 100644 --- a/ragna/deploy/__init__.py +++ b/ragna/deploy/__init__.py @@ -5,7 +5,7 @@ ] from ._authentication import Authentication, RagnaDemoAuthentication -from ._config import ApiConfig, Config, UiConfig +from ._config import Config # isort: split diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 43aa35c22..27093a8a5 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -1,4 +1,5 @@ import os +import socket import time from multiprocessing import Process @@ -8,15 +9,13 @@ import uvicorn from playwright.sync_api import expect, sync_playwright +from ragna._utils import timeout_after from ragna.assistants import RagnaDemoAssistant from ragna.core._utils import default_user -from ragna.deploy import ApiConfig, Config, UiConfig +from ragna.deploy import Config from ragna.deploy._api import app as api_app from ragna.deploy._ui import app as ui_app -TEST_API_PORT = "38769" -TEST_UI_PORT = "38770" - class TestAssistant(RagnaDemoAssistant): def answer(self, prompt, sources, *, multiple_answer_chunks: bool): @@ -29,6 +28,12 @@ def answer(self, prompt, sources, *, multiple_answer_chunks: bool): yield content +def get_available_port(): + with socket.socket() as s: + s.bind(("", 0)) + return s.getsockname()[1] + + @pytest.fixture(scope="session") def headed_mode(pytestconfig): return pytestconfig.getoption("headed") or False @@ -36,11 +41,13 @@ def headed_mode(pytestconfig): @pytest.fixture def config(tmp_local_root): + ui_port = get_available_port() + api_port = get_available_port() return Config( local_root=tmp_local_root, assistants=[TestAssistant], - ui=UiConfig(port=TEST_UI_PORT), - api=ApiConfig(port=TEST_API_PORT), + ui=dict(port=ui_port), + api=dict(port=api_port), ) @@ -64,15 +71,13 @@ def server_up(self): except httpx.ConnectError: return False + @timeout_after(5) def start(self): self.proc = Process(target=self.start_server, args=(), daemon=True) self.proc.start() - timeout = 5 - while timeout > 0 and not self.server_up(): - print(f"Waiting for API server to come up on {self.config.api.url}") + while not self.server_up(): time.sleep(1) - timeout -= 1 def stop(self): self.proc.kill() @@ -88,11 +93,22 @@ def api_server(config): server.stop() -def auth_header(base_url): +@pytest.fixture +def base_ui_url(config): + return f"http://{config.ui.hostname}:{config.ui.port}" + + +@pytest.fixture +def base_api_url(config): + return f"http://{config.api.hostname}:{config.api.port}" + + +@pytest.fixture +def auth_header(base_api_url): username = default_user() token = ( httpx.post( - base_url + "/token", + base_api_url + "/token", data={ "username": username, "password": os.environ.get( @@ -107,7 +123,7 @@ def auth_header(base_url): return f"Bearer {token}" -@pytest.fixture(scope="function") +@pytest.fixture def page(config, api_server, headed_mode): server = ui_app(config=config, open_browser=False) @@ -124,15 +140,15 @@ def page(config, api_server, headed_mode): pn.state.kill_all_servers() -def test_health(config, page) -> None: - health_url = config.ui.origins[0] + "/health" +def test_health(base_ui_url, page) -> None: + health_url = base_ui_url + "/health" page.goto(health_url) expect(page.get_by_role("heading", name="Ok")).to_be_visible() -def test_index_with_blank_credentials(config, page) -> None: +def test_index_with_blank_credentials(base_ui_url, page) -> None: # Index page, no auth - index_url = config.ui.origins[0] + index_url = base_ui_url page.goto(index_url) expect(page.get_by_role("button", name="Sign In")).to_be_visible() From fc61dbeea46acc6d136682a50492976c18cec89f Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Thu, 16 May 2024 15:21:40 -0700 Subject: [PATCH 13/39] move shared stuff to utils --- tests/deploy/api/test_batch_endpoints.py | 2 +- tests/deploy/api/test_components.py | 2 +- tests/deploy/api/test_e2e.py | 19 +------- tests/deploy/api/utils.py | 23 ---------- tests/deploy/ui/test_ui.py | 55 +----------------------- 5 files changed, 5 insertions(+), 96 deletions(-) delete mode 100644 tests/deploy/api/utils.py diff --git a/tests/deploy/api/test_batch_endpoints.py b/tests/deploy/api/test_batch_endpoints.py index 947407506..916ef7f00 100644 --- a/tests/deploy/api/test_batch_endpoints.py +++ b/tests/deploy/api/test_batch_endpoints.py @@ -4,7 +4,7 @@ from ragna.deploy import Config from ragna.deploy._api import app -from .utils import authenticate +from ..utils import authenticate def test_batch_sequential_upload_equivalence(tmp_local_root): diff --git a/tests/deploy/api/test_components.py b/tests/deploy/api/test_components.py index 65f02209f..e080ef9d7 100644 --- a/tests/deploy/api/test_components.py +++ b/tests/deploy/api/test_components.py @@ -7,7 +7,7 @@ from ragna.deploy import Config from ragna.deploy._api import app -from .utils import authenticate +from ..utils import authenticate @pytest.mark.parametrize("ignore_unavailable_components", [True, False]) diff --git a/tests/deploy/api/test_e2e.py b/tests/deploy/api/test_e2e.py index 41b154db4..8a958a705 100644 --- a/tests/deploy/api/test_e2e.py +++ b/tests/deploy/api/test_e2e.py @@ -1,29 +1,12 @@ import json -import time import pytest from fastapi.testclient import TestClient -from ragna.assistants import RagnaDemoAssistant from ragna.deploy import Config from ragna.deploy._api import app -from .utils import authenticate - - -class TestAssistant(RagnaDemoAssistant): - def answer(self, prompt, sources, *, multiple_answer_chunks: bool): - # Simulate a "real" assistant through a small delay. See - # https://github.com/Quansight/ragna/pull/401#issuecomment-2095851440 - # for why this is needed. - time.sleep(1e-3) - content = next(super().answer(prompt, sources)) - - if multiple_answer_chunks: - for chunk in content.split(" "): - yield f"{chunk} " - else: - yield content +from ..utils import TestAssistant, authenticate @pytest.mark.parametrize("multiple_answer_chunks", [True, False]) diff --git a/tests/deploy/api/utils.py b/tests/deploy/api/utils.py deleted file mode 100644 index abcf1411d..000000000 --- a/tests/deploy/api/utils.py +++ /dev/null @@ -1,23 +0,0 @@ -import os - -from fastapi.testclient import TestClient - -from ragna.core._utils import default_user - - -def authenticate(client: TestClient) -> None: - username = default_user() - token = ( - client.post( - "/token", - data={ - "username": username, - "password": os.environ.get( - "RAGNA_DEMO_AUTHENTICATION_PASSWORD", username - ), - }, - ) - .raise_for_status() - .json() - ) - client.headers["Authorization"] = f"Bearer {token}" diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 27093a8a5..fc876fd5c 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -1,4 +1,3 @@ -import os import socket import time from multiprocessing import Process @@ -10,22 +9,11 @@ from playwright.sync_api import expect, sync_playwright from ragna._utils import timeout_after -from ragna.assistants import RagnaDemoAssistant -from ragna.core._utils import default_user from ragna.deploy import Config from ragna.deploy._api import app as api_app from ragna.deploy._ui import app as ui_app - -class TestAssistant(RagnaDemoAssistant): - def answer(self, prompt, sources, *, multiple_answer_chunks: bool): - content = next(super().answer(prompt, sources)) - - if multiple_answer_chunks: - for chunk in content.split(" "): - yield f"{chunk} " - else: - yield content +from ..utils import TestAssistant def get_available_port(): @@ -80,7 +68,7 @@ def start(self): time.sleep(1) def stop(self): - self.proc.kill() + self.proc.terminate() @pytest.fixture @@ -98,31 +86,6 @@ def base_ui_url(config): return f"http://{config.ui.hostname}:{config.ui.port}" -@pytest.fixture -def base_api_url(config): - return f"http://{config.api.hostname}:{config.api.port}" - - -@pytest.fixture -def auth_header(base_api_url): - username = default_user() - token = ( - httpx.post( - base_api_url + "/token", - data={ - "username": username, - "password": os.environ.get( - "RAGNA_DEMO_AUTHENTICATION_PASSWORD", username - ), - }, - ) - .raise_for_status() - .json() - ) - - return f"Bearer {token}" - - @pytest.fixture def page(config, api_server, headed_mode): server = ui_app(config=config, open_browser=False) @@ -158,20 +121,6 @@ def test_index_with_blank_credentials(base_ui_url, page) -> None: expect(page.get_by_role("button", name=" New Chat")).to_be_visible() -@pytest.mark.skip(reason="Need to figure out how to set the auth token") -def test_index_with_auth_token(config, page) -> None: - auth = auth_header(config.api.url) - assert len(auth) > len("Bearer ") - - page.set_extra_http_headers({"Authorization": auth}) # this doesn't work - pn.state.cookies["auth_token"] = "" # or this - pn.state.headers["Authorization"] = auth # or this - - index_url = config.ui.origins[0] - page.goto(index_url) - expect(page.get_by_role("button", name=" New Chat")).to_be_visible() - - @pytest.mark.skip(reason="TODO: figure out best locators") def test_new_chat(config, page) -> None: index_url = config.ui.origins[0] From 6dc91eb0d0552c4b7ca4c9006fd6cf2c40135ef5 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Wed, 22 May 2024 07:35:07 -0700 Subject: [PATCH 14/39] wip --- tests/deploy/ui/test_ui.py | 62 +++++++++++++++++++++++++++++++++----- tests/deploy/utils.py | 40 ++++++++++++++++++++++++ 2 files changed, 94 insertions(+), 8 deletions(-) create mode 100644 tests/deploy/utils.py diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index fc876fd5c..c6c706755 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -87,39 +87,85 @@ def base_ui_url(config): @pytest.fixture -def page(config, api_server, headed_mode): +def context(config, api_server, headed_mode): server = ui_app(config=config, open_browser=False) with sync_playwright() as playwright: server.serve() browser = playwright.chromium.launch(headless=not headed_mode) context = browser.new_context() - page = context.new_page() - yield page + yield context context.close() browser.close() pn.state.kill_all_servers() -def test_health(base_ui_url, page) -> None: +def test_health(base_ui_url, context) -> None: + page = context.new_page() health_url = base_ui_url + "/health" - page.goto(health_url) - expect(page.get_by_role("heading", name="Ok")).to_be_visible() + response = page.goto(health_url) + assert response.ok -def test_index_with_blank_credentials(base_ui_url, page) -> None: +def test_index(base_ui_url, context, config) -> None: # Index page, no auth + page = context.new_page() index_url = base_ui_url page.goto(index_url) expect(page.get_by_role("button", name="Sign In")).to_be_visible() # Authorize with no credentials + # page.locator('input[type="text"]').fill("hello") + # page.locator('input[type="password"]').fill("hello") page.get_by_role("button", name="Sign In").click() - expect(page.get_by_role("button", name=" New Chat")).to_be_visible() + # expect auth token to be set + cookies = context.cookies() + assert len(cookies) == 1 + cookie = cookies[0] + assert cookie.get("name") == "auth_token" + auth_token = cookie.get("value") + assert auth_token is not None + + # New page button + new_chat_button = page.get_by_role("button", name=" New Chat") + expect(new_chat_button).to_be_visible() + new_chat_button.click() + + document_root = config.local_root / "documents" + document_root.mkdir() + document_name = "test.txt" + document_path = document_root / document_name + with open(document_path, "w") as file: + file.write("!\n") + + # File upload selector + with page.expect_file_chooser() as fc_info: + page.locator(".fileUpload").click() + file_chooser = fc_info.value + # file_chooser.set_files(document_path) + file_chooser.set_files( + files=[ + {"name": "test.txt", "mimeType": "text/plain", "buffer": b"this is a test"} + ] + ) + + # Upload file and expect to see it listed + file_list = page.locator(".fileListContainer") + expect(file_list.first).to_have_text(str(document_name)) + + start_chat_button = page.get_by_role("button", name="Start Conversation") + expect(start_chat_button).to_be_visible() + start_chat_button.click() + + breakpoint() + + # chat_box = page.get_by_placeholder("Ask a question about the") + # expect(chat_box).to_be_visible() + @pytest.mark.skip(reason="TODO: figure out best locators") def test_new_chat(config, page) -> None: diff --git a/tests/deploy/utils.py b/tests/deploy/utils.py new file mode 100644 index 000000000..35652935b --- /dev/null +++ b/tests/deploy/utils.py @@ -0,0 +1,40 @@ +import os +import time + +from fastapi.testclient import TestClient + +from ragna.assistants import RagnaDemoAssistant +from ragna.core._utils import default_user + + +class TestAssistant(RagnaDemoAssistant): + def answer(self, prompt, sources, *, multiple_answer_chunks: bool): + # Simulate a "real" assistant through a small delay. See + # https://github.com/Quansight/ragna/pull/401#issuecomment-2095851440 + # for why this is needed. + time.sleep(1e-3) + content = next(super().answer(prompt, sources)) + + if multiple_answer_chunks: + for chunk in content.split(" "): + yield f"{chunk} " + else: + yield content + + +def authenticate(client: TestClient) -> None: + username = default_user() + token = ( + client.post( + "/token", + data={ + "username": username, + "password": os.environ.get( + "RAGNA_DEMO_AUTHENTICATION_PASSWORD", username + ), + }, + ) + .raise_for_status() + .json() + ) + client.headers["Authorization"] = f"Bearer {token}" From d9f19cbbd4236731988909903a5049d18101bcb7 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Wed, 22 May 2024 09:59:04 -0700 Subject: [PATCH 15/39] wip: replace server with cli function --- tests/deploy/ui/test_ui.py | 118 +++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 64 deletions(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index c6c706755..a257d4f7b 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -1,17 +1,15 @@ import socket +import subprocess +import sys import time -from multiprocessing import Process import httpx import panel as pn import pytest -import uvicorn from playwright.sync_api import expect, sync_playwright from ragna._utils import timeout_after from ragna.deploy import Config -from ragna.deploy._api import app as api_app -from ragna.deploy._ui import app as ui_app from ..utils import TestAssistant @@ -27,53 +25,74 @@ def headed_mode(pytestconfig): return pytestconfig.getoption("headed") or False +@pytest.fixture(scope="session") +def ui_port(): + return get_available_port() + + +@pytest.fixture(scope="session") +def api_port(): + return get_available_port() + + @pytest.fixture -def config(tmp_local_root): - ui_port = get_available_port() - api_port = get_available_port() - return Config( +def config(tmp_local_root, ui_port, api_port): + config = Config( local_root=tmp_local_root, assistants=[TestAssistant], ui=dict(port=ui_port), api=dict(port=api_port), ) + path = tmp_local_root / "ragna.toml" + config.to_file(path) + return config -class ApiServer: - def __init__(self, config): +class Server: + def __init__(self, config, base_url): self.config = config - - def start_server(self): - uvicorn.run( - api_app( - config=self.config, - ignore_unavailable_components=True, - ), - host=self.config.api.hostname, - port=self.config.api.port, - ) + self.base_url = base_url def server_up(self): try: - return httpx.get(self.config.api.url).is_success + return httpx.get(self.base_url).is_success except httpx.ConnectError: return False @timeout_after(5) def start(self): - self.proc = Process(target=self.start_server, args=(), daemon=True) - self.proc.start() + self.proc = subprocess.Popen( + [ + sys.executable, + "-m", + "ragna", + "ui", + "--config", + self.config.local_root / "ragna.toml", + "--start-api", + "--ignore-unavailable-components", + "--no-open-browser", + ], + stdout=sys.stdout, + stderr=sys.stderr, + ) while not self.server_up(): time.sleep(1) def stop(self): - self.proc.terminate() + self.proc.kill() + pn.state.kill_all_servers() @pytest.fixture -def api_server(config): - server = ApiServer(config) +def base_ui_url(ui_port): + return f"http://127.0.0.1:{ui_port}" + + +@pytest.fixture +def server(config, base_ui_url): + server = Server(config, base_ui_url) try: server.start() yield server @@ -82,16 +101,8 @@ def api_server(config): @pytest.fixture -def base_ui_url(config): - return f"http://{config.ui.hostname}:{config.ui.port}" - - -@pytest.fixture -def context(config, api_server, headed_mode): - server = ui_app(config=config, open_browser=False) - +def context(server, headed_mode): with sync_playwright() as playwright: - server.serve() browser = playwright.chromium.launch(headless=not headed_mode) context = browser.new_context() @@ -99,7 +110,6 @@ def context(config, api_server, headed_mode): context.close() browser.close() - pn.state.kill_all_servers() def test_health(base_ui_url, context) -> None: @@ -117,8 +127,6 @@ def test_index(base_ui_url, context, config) -> None: expect(page.get_by_role("button", name="Sign In")).to_be_visible() # Authorize with no credentials - # page.locator('input[type="text"]').fill("hello") - # page.locator('input[type="password"]').fill("hello") page.get_by_role("button", name="Sign In").click() expect(page.get_by_role("button", name=" New Chat")).to_be_visible() @@ -166,29 +174,11 @@ def test_index(base_ui_url, context, config) -> None: # chat_box = page.get_by_placeholder("Ask a question about the") # expect(chat_box).to_be_visible() - -@pytest.mark.skip(reason="TODO: figure out best locators") -def test_new_chat(config, page) -> None: - index_url = config.ui.origins[0] - page.goto(index_url) - expect(page.get_by_role("button", name="Sign In")).to_be_visible() - page.get_by_role("button", name="Sign In").click() - - expect(page.get_by_role("button", name=" New Chat")).to_be_visible() - page.get_by_role("button", name=" New Chat").click() - - expect(page.locator("#fileUpload-p4447")).to_be_visible() - page.locator("#fileUpload-p4447").click() - - page.locator("#fileUpload-p4447").set_input_files() - page.get_by_role("button", name="Start Conversation").click() - page.get_by_text("How can I help you with the").click() - page.get_by_placeholder("Ask a question about the").click() - page.get_by_placeholder("Ask a question about the").fill( - "Tell me about the documents" - ) - page.get_by_role("button", name="").click() - page.get_by_role("button", name=" Source Info").click() - page.locator("#main div").filter(has_text="Source Info ¶ This response").nth( - 3 - ).click() + # page.get_by_placeholder("Ask a question about the").fill( + # "Tell me about the documents" + # ) + # page.get_by_role("button", name="").click() + # page.get_by_role("button", name=" Source Info").click() + # page.locator("#main div").filter(has_text="Source Info ¶ This response").nth( + # 3 + # ).click() From b7e59a5fb7c7ee2ab136982e20cfb3d36667311d Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Wed, 22 May 2024 10:03:01 -0700 Subject: [PATCH 16/39] remove breakpoint --- tests/deploy/ui/test_ui.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index a257d4f7b..7c98d9a08 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -169,8 +169,6 @@ def test_index(base_ui_url, context, config) -> None: expect(start_chat_button).to_be_visible() start_chat_button.click() - breakpoint() - # chat_box = page.get_by_placeholder("Ask a question about the") # expect(chat_box).to_be_visible() From d62bf5e92b149bb415bf66cf905bdf5ba6deb99a Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Thu, 23 May 2024 09:31:32 +0200 Subject: [PATCH 17/39] increase timeout --- tests/deploy/ui/test_ui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 7c98d9a08..9db537ca6 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -59,7 +59,7 @@ def server_up(self): except httpx.ConnectError: return False - @timeout_after(5) + @timeout_after(30) def start(self): self.proc = subprocess.Popen( [ From e6ab71bf2a3beff27119466080795a1c4665e689 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Fri, 24 May 2024 14:54:08 -0700 Subject: [PATCH 18/39] server mods --- tests/deploy/ui/test_ui.py | 76 +++++++++++++++----------------------- tests/deploy/utils.py | 6 ++- 2 files changed, 34 insertions(+), 48 deletions(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 9db537ca6..f9641b924 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -25,23 +25,13 @@ def headed_mode(pytestconfig): return pytestconfig.getoption("headed") or False -@pytest.fixture(scope="session") -def ui_port(): - return get_available_port() - - -@pytest.fixture(scope="session") -def api_port(): - return get_available_port() - - @pytest.fixture -def config(tmp_local_root, ui_port, api_port): +def config(tmp_local_root): config = Config( local_root=tmp_local_root, assistants=[TestAssistant], - ui=dict(port=ui_port), - api=dict(port=api_port), + ui=dict(port=get_available_port()), + api=dict(port=get_available_port()), ) path = tmp_local_root / "ragna.toml" config.to_file(path) @@ -49,9 +39,9 @@ def config(tmp_local_root, ui_port, api_port): class Server: - def __init__(self, config, base_url): + def __init__(self, config): self.config = config - self.base_url = base_url + self.base_url = f"http://{config.ui.hostname}:{config.ui.port}" def server_up(self): try: @@ -86,13 +76,8 @@ def stop(self): @pytest.fixture -def base_ui_url(ui_port): - return f"http://127.0.0.1:{ui_port}" - - -@pytest.fixture -def server(config, base_ui_url): - server = Server(config, base_ui_url) +def server(config): + server = Server(config) try: server.start() yield server @@ -101,7 +86,7 @@ def server(config, base_ui_url): @pytest.fixture -def context(server, headed_mode): +def context(headed_mode): with sync_playwright() as playwright: browser = playwright.chromium.launch(headless=not headed_mode) context = browser.new_context() @@ -112,17 +97,17 @@ def context(server, headed_mode): browser.close() -def test_health(base_ui_url, context) -> None: +def test_health(server, context) -> None: page = context.new_page() - health_url = base_ui_url + "/health" + health_url = server.base_url + "/health" response = page.goto(health_url) assert response.ok -def test_index(base_ui_url, context, config) -> None: +def test_index(server, context, config) -> None: # Index page, no auth page = context.new_page() - index_url = base_ui_url + index_url = server.base_url page.goto(index_url) expect(page.get_by_role("button", name="Sign In")).to_be_visible() @@ -154,29 +139,26 @@ def test_index(base_ui_url, context, config) -> None: with page.expect_file_chooser() as fc_info: page.locator(".fileUpload").click() file_chooser = fc_info.value - # file_chooser.set_files(document_path) - file_chooser.set_files( - files=[ - {"name": "test.txt", "mimeType": "text/plain", "buffer": b"this is a test"} - ] - ) + file_chooser.set_files(document_path) - # Upload file and expect to see it listed + # Upload document and expect to see it listed file_list = page.locator(".fileListContainer") expect(file_list.first).to_have_text(str(document_name)) start_chat_button = page.get_by_role("button", name="Start Conversation") expect(start_chat_button).to_be_visible() - start_chat_button.click() - - # chat_box = page.get_by_placeholder("Ask a question about the") - # expect(chat_box).to_be_visible() - - # page.get_by_placeholder("Ask a question about the").fill( - # "Tell me about the documents" - # ) - # page.get_by_role("button", name="").click() - # page.get_by_role("button", name=" Source Info").click() - # page.locator("#main div").filter(has_text="Source Info ¶ This response").nth( - # 3 - # ).click() + start_chat_button.click(delay=5) + + # TODO: Document should be in the database + + chat_box_row = page.locator(".chat-interface-input-row") + expect(chat_box_row).to_be_visible() + + chat_box = chat_box_row.get_by_role("textbox") + expect(chat_box).to_be_visible() + + chat_box.fill("Tell me about the documents") + + chat_button = chat_box_row.get_by_role("button") + expect(chat_button).to_be_visible() + chat_button.click() diff --git a/tests/deploy/utils.py b/tests/deploy/utils.py index 35652935b..5b4855e78 100644 --- a/tests/deploy/utils.py +++ b/tests/deploy/utils.py @@ -8,10 +8,14 @@ class TestAssistant(RagnaDemoAssistant): - def answer(self, prompt, sources, *, multiple_answer_chunks: bool): + def answer(self, prompt, sources, *, multiple_answer_chunks: bool = True): # Simulate a "real" assistant through a small delay. See # https://github.com/Quansight/ragna/pull/401#issuecomment-2095851440 # for why this is needed. + # + # Note: multiple_answer_chunks is given a default value here to satisfy + # the tests in deploy/ui/test_ui.py. This can be removed if TestAssistant + # is ever removed from that file. time.sleep(1e-3) content = next(super().answer(prompt, sources)) From 382d71b2cbe883ec2a58a7ec9d15dca52adaa0c5 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Fri, 24 May 2024 15:00:39 -0700 Subject: [PATCH 19/39] use playwright Page directly --- tests/deploy/ui/test_ui.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index f9641b924..82f7f01d9 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -6,7 +6,7 @@ import httpx import panel as pn import pytest -from playwright.sync_api import expect, sync_playwright +from playwright.sync_api import Page, expect from ragna._utils import timeout_after from ragna.deploy import Config @@ -85,28 +85,14 @@ def server(config): server.stop() -@pytest.fixture -def context(headed_mode): - with sync_playwright() as playwright: - browser = playwright.chromium.launch(headless=not headed_mode) - context = browser.new_context() - - yield context - - context.close() - browser.close() - - -def test_health(server, context) -> None: - page = context.new_page() +def test_health(server, page: Page) -> None: health_url = server.base_url + "/health" response = page.goto(health_url) assert response.ok -def test_index(server, context, config) -> None: +def test_index(server, config, page: Page) -> None: # Index page, no auth - page = context.new_page() index_url = server.base_url page.goto(index_url) expect(page.get_by_role("button", name="Sign In")).to_be_visible() @@ -116,7 +102,7 @@ def test_index(server, context, config) -> None: expect(page.get_by_role("button", name=" New Chat")).to_be_visible() # expect auth token to be set - cookies = context.cookies() + cookies = page.context.cookies() assert len(cookies) == 1 cookie = cookies[0] assert cookie.get("name") == "auth_token" From b378a5189841bb064a67b2ec7f84b1750723fa07 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Fri, 24 May 2024 15:42:06 -0700 Subject: [PATCH 20/39] check to make sure document is in database --- tests/deploy/ui/test_ui.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 82f7f01d9..3359b269a 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -91,6 +91,19 @@ def test_health(server, page: Page) -> None: assert response.ok +@timeout_after(10) +def get_chats(config, auth_token): + chats_url = f"http://{config.api.hostname}:{config.api.port}/chats" + chats = [] + while len(chats) == 0: + chats = httpx.get( + chats_url, headers={"Authorization": f"Bearer {auth_token}"} + ).json() + time.sleep(1) + + return chats + + def test_index(server, config, page: Page) -> None: # Index page, no auth index_url = server.base_url @@ -135,7 +148,13 @@ def test_index(server, config, page: Page) -> None: expect(start_chat_button).to_be_visible() start_chat_button.click(delay=5) - # TODO: Document should be in the database + # Document should be in the database + chats = get_chats(config, auth_token) + assert len(chats) == 1 + chat = chats[0] + chat_documents = chat["metadata"]["documents"] + assert len(chat_documents) == 1 + assert chat_documents[0]["name"] == document_name chat_box_row = page.locator(".chat-interface-input-row") expect(chat_box_row).to_be_visible() From 53b4c4fedd3470febcae8b4e535f31b9241e6904 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Fri, 24 May 2024 15:45:47 -0700 Subject: [PATCH 21/39] increase timeout --- tests/deploy/ui/test_ui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 3359b269a..91e70ae18 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -91,7 +91,7 @@ def test_health(server, page: Page) -> None: assert response.ok -@timeout_after(10) +@timeout_after(30) def get_chats(config, auth_token): chats_url = f"http://{config.api.hostname}:{config.api.port}/chats" chats = [] From 0d80801f2878ff5e4847ebfb42f85ce9e1d439c8 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Tue, 28 May 2024 14:42:08 -0700 Subject: [PATCH 22/39] increase timeout again --- tests/deploy/ui/test_ui.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 91e70ae18..88a92bf4c 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -49,7 +49,7 @@ def server_up(self): except httpx.ConnectError: return False - @timeout_after(30) + @timeout_after(60) def start(self): self.proc = subprocess.Popen( [ @@ -91,7 +91,7 @@ def test_health(server, page: Page) -> None: assert response.ok -@timeout_after(30) +@timeout_after(60) def get_chats(config, auth_token): chats_url = f"http://{config.api.hostname}:{config.api.port}/chats" chats = [] From aae2f224ffe74a91807ae49660f3f5592e45e47c Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Tue, 28 May 2024 15:13:25 -0700 Subject: [PATCH 23/39] use consistent ports --- tests/deploy/ui/test_ui.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 88a92bf4c..3ab40845b 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -20,18 +20,28 @@ def get_available_port(): return s.getsockname()[1] +@pytest.fixture(scope="session") +def api_port(): + return get_available_port() + + +@pytest.fixture(scope="session") +def ui_port(): + return get_available_port() + + @pytest.fixture(scope="session") def headed_mode(pytestconfig): return pytestconfig.getoption("headed") or False @pytest.fixture -def config(tmp_local_root): +def config(tmp_local_root, api_port, ui_port): config = Config( local_root=tmp_local_root, assistants=[TestAssistant], - ui=dict(port=get_available_port()), - api=dict(port=get_available_port()), + ui=dict(port=ui_port), + api=dict(port=api_port), ) path = tmp_local_root / "ragna.toml" config.to_file(path) From 36e6f436ab3e6584353d78847743114d679784dd Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Thu, 30 May 2024 13:52:39 -0700 Subject: [PATCH 24/39] try with slowmo --- .github/workflows/test.yml | 2 +- tests/deploy/ui/test_ui.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b74f39079..650783736 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,7 @@ jobs: - name: Run unit tests id: tests - run: pytest --junit-xml=test-results.xml --durations=25 + run: pytest --junit-xml=test-results.xml --durations=25 --slowmo 100 - name: Surface failing tests if: steps.tests.outcome != 'success' diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 3ab40845b..f4a229a63 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -36,12 +36,14 @@ def headed_mode(pytestconfig): @pytest.fixture -def config(tmp_local_root, api_port, ui_port): +def config( + tmp_local_root, +): config = Config( local_root=tmp_local_root, assistants=[TestAssistant], - ui=dict(port=ui_port), - api=dict(port=api_port), + ui=dict(port=get_available_port()), + api=dict(port=get_available_port()), ) path = tmp_local_root / "ragna.toml" config.to_file(path) @@ -157,6 +159,7 @@ def test_index(server, config, page: Page) -> None: start_chat_button = page.get_by_role("button", name="Start Conversation") expect(start_chat_button).to_be_visible() start_chat_button.click(delay=5) + # expect().to_be_visible() # Document should be in the database chats = get_chats(config, auth_token) From 050dc72023494de5f5ef1f41084c579510ec309a Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Thu, 30 May 2024 14:15:08 -0700 Subject: [PATCH 25/39] reorder element expectations; remove slowmo --- .github/workflows/test.yml | 2 +- tests/deploy/ui/test_ui.py | 33 ++++++++++++--------------------- 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 650783736..b74f39079 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,7 @@ jobs: - name: Run unit tests id: tests - run: pytest --junit-xml=test-results.xml --durations=25 --slowmo 100 + run: pytest --junit-xml=test-results.xml --durations=25 - name: Surface failing tests if: steps.tests.outcome != 'success' diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index f4a229a63..87bdf5460 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -103,19 +103,6 @@ def test_health(server, page: Page) -> None: assert response.ok -@timeout_after(60) -def get_chats(config, auth_token): - chats_url = f"http://{config.api.hostname}:{config.api.port}/chats" - chats = [] - while len(chats) == 0: - chats = httpx.get( - chats_url, headers={"Authorization": f"Bearer {auth_token}"} - ).json() - time.sleep(1) - - return chats - - def test_index(server, config, page: Page) -> None: # Index page, no auth index_url = server.base_url @@ -156,25 +143,29 @@ def test_index(server, config, page: Page) -> None: file_list = page.locator(".fileListContainer") expect(file_list.first).to_have_text(str(document_name)) + chat_dialog = page.get_by_role("dialog") + expect(chat_dialog).to_be_visible() start_chat_button = page.get_by_role("button", name="Start Conversation") expect(start_chat_button).to_be_visible() start_chat_button.click(delay=5) - # expect().to_be_visible() + + chat_box_row = page.locator(".chat-interface-input-row") + expect(chat_box_row).to_be_visible() + + chat_box = chat_box_row.get_by_role("textbox") + expect(chat_box).to_be_visible() # Document should be in the database - chats = get_chats(config, auth_token) + chats_url = f"http://{config.api.hostname}:{config.api.port}/chats" + chats = httpx.get( + chats_url, headers={"Authorization": f"Bearer {auth_token}"} + ).json() assert len(chats) == 1 chat = chats[0] chat_documents = chat["metadata"]["documents"] assert len(chat_documents) == 1 assert chat_documents[0]["name"] == document_name - chat_box_row = page.locator(".chat-interface-input-row") - expect(chat_box_row).to_be_visible() - - chat_box = chat_box_row.get_by_role("textbox") - expect(chat_box).to_be_visible() - chat_box.fill("Tell me about the documents") chat_button = chat_box_row.get_by_role("button") From a983307f097bfe0d32c0b7646b58e891338e8fc1 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Thu, 30 May 2024 14:24:41 -0700 Subject: [PATCH 26/39] hack --- tests/deploy/ui/test_ui.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 87bdf5460..78113d6fb 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -147,6 +147,7 @@ def test_index(server, config, page: Page) -> None: expect(chat_dialog).to_be_visible() start_chat_button = page.get_by_role("button", name="Start Conversation") expect(start_chat_button).to_be_visible() + time.sleep(0.5) # hack start_chat_button.click(delay=5) chat_box_row = page.locator(".chat-interface-input-row") From f49bdd82264498dd91daa168ef34148551d85165 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Fri, 31 May 2024 10:37:45 -0700 Subject: [PATCH 27/39] upload playwright video on failing tests --- .github/workflows/test.yml | 14 +++++++++++++- tests/deploy/ui/test_ui.py | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b74f39079..f59e7e515 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,19 @@ jobs: - name: Run unit tests id: tests - run: pytest --junit-xml=test-results.xml --durations=25 + run: | + pytest \ + --junit-xml=test-results.xml \ + --durations=25 \ + --video=retain-on-failure # playwright ui tests + + - name: Upload playwright video + - uses: actions/upload-artifact@v4 + with: + name: + playwright-${{ matrix.os }}-${{ matrix.python-version}}-${{ github.run_id }} + path: test-results + if-no-files-found: ignore - name: Surface failing tests if: steps.tests.outcome != 'success' diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 78113d6fb..0bb5c608b 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -147,7 +147,7 @@ def test_index(server, config, page: Page) -> None: expect(chat_dialog).to_be_visible() start_chat_button = page.get_by_role("button", name="Start Conversation") expect(start_chat_button).to_be_visible() - time.sleep(0.5) # hack + # time.sleep(0.5) # hack start_chat_button.click(delay=5) chat_box_row = page.locator(".chat-interface-input-row") From 648823d899434ae4570f8c824c6bde4171aabd85 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Fri, 31 May 2024 10:39:34 -0700 Subject: [PATCH 28/39] syntax fix --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f59e7e515..f928cbaf9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,7 +75,7 @@ jobs: --video=retain-on-failure # playwright ui tests - name: Upload playwright video - - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v4 with: name: playwright-${{ matrix.os }}-${{ matrix.python-version}}-${{ github.run_id }} From 7038f302db8fcf75b27c2ad054ad87e607490772 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Fri, 31 May 2024 10:44:22 -0700 Subject: [PATCH 29/39] run upload on failure --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f928cbaf9..ea76fd013 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,12 +75,12 @@ jobs: --video=retain-on-failure # playwright ui tests - name: Upload playwright video + if: failure() uses: actions/upload-artifact@v4 with: name: playwright-${{ matrix.os }}-${{ matrix.python-version}}-${{ github.run_id }} path: test-results - if-no-files-found: ignore - name: Surface failing tests if: steps.tests.outcome != 'success' From 855c0d3e55425ad3c34cd09883b54c29399fab59 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Fri, 31 May 2024 10:53:43 -0700 Subject: [PATCH 30/39] get tests to pass again --- tests/deploy/ui/test_ui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 0bb5c608b..4ae3c9d19 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -147,7 +147,7 @@ def test_index(server, config, page: Page) -> None: expect(chat_dialog).to_be_visible() start_chat_button = page.get_by_role("button", name="Start Conversation") expect(start_chat_button).to_be_visible() - # time.sleep(0.5) # hack + time.sleep(0.5) # hack while waiting for button to be fully clickable start_chat_button.click(delay=5) chat_box_row = page.locator(".chat-interface-input-row") From 50c0ae114a046750cbef7394642be2350e78dfc0 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Fri, 7 Jun 2024 09:45:18 -0700 Subject: [PATCH 31/39] separate ui and non-ui tests --- .github/workflows/test.yml | 59 +++++++++++++++++++++++++++++++++----- 1 file changed, 52 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ea76fd013..b97f18e74 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -70,9 +70,60 @@ jobs: id: tests run: | pytest \ + --ignore tests/deploy/ui \ --junit-xml=test-results.xml \ --durations=25 \ - --video=retain-on-failure # playwright ui tests + + - name: Surface failing tests + if: steps.tests.outcome != 'success' + uses: pmeier/pytest-results-action@v0.3.0 + with: + path: test-results.xml + + pytest-ui: + strategy: + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + browser: + - chromium + - firefox + python-version: ["3.9"] + include: + - os: macos-latest + browser: webkit + - os: ubuntu-latest + python-version: "3.10" + - os: ubuntu-latest + python-version: "3.11" + + fail-fast: false + + runs-on: ${{ matrix.os }} + + defaults: + run: + shell: bash -el {0} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup environment + uses: ./.github/actions/setup-env + with: + python-version: ${{ matrix.python-version }} + + - name: Run unit tests + id: tests + run: | + pytest tests/deploy/ui \ + --browser ${{ matrix.browser }} \ + --video=retain-on-failure - name: Upload playwright video if: failure() @@ -81,9 +132,3 @@ jobs: name: playwright-${{ matrix.os }}-${{ matrix.python-version}}-${{ github.run_id }} path: test-results - - - name: Surface failing tests - if: steps.tests.outcome != 'success' - uses: pmeier/pytest-results-action@v0.3.0 - with: - path: test-results.xml From 4dade557fdf58183c7ca77661030f6b160b27a25 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Fri, 7 Jun 2024 09:52:39 -0700 Subject: [PATCH 32/39] avoid relative imports; better func name --- tests/deploy/api/test_batch_endpoints.py | 5 ++--- tests/deploy/api/test_components.py | 7 +++---- tests/deploy/api/test_e2e.py | 5 ++--- tests/deploy/ui/test_ui.py | 3 +-- tests/deploy/utils.py | 2 +- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/tests/deploy/api/test_batch_endpoints.py b/tests/deploy/api/test_batch_endpoints.py index 916ef7f00..3c85c77c3 100644 --- a/tests/deploy/api/test_batch_endpoints.py +++ b/tests/deploy/api/test_batch_endpoints.py @@ -3,8 +3,7 @@ from ragna.deploy import Config from ragna.deploy._api import app - -from ..utils import authenticate +from tests.deploy.utils import authenticate_with_api def test_batch_sequential_upload_equivalence(tmp_local_root): @@ -23,7 +22,7 @@ def test_batch_sequential_upload_equivalence(tmp_local_root): with TestClient( app(config=Config(), ignore_unavailable_components=False) ) as client: - authenticate(client) + authenticate_with_api(client) document1_upload = ( client.post("/document", json={"name": document_path1.name}) diff --git a/tests/deploy/api/test_components.py b/tests/deploy/api/test_components.py index e080ef9d7..b7fe464c2 100644 --- a/tests/deploy/api/test_components.py +++ b/tests/deploy/api/test_components.py @@ -6,8 +6,7 @@ from ragna.core import RagnaException from ragna.deploy import Config from ragna.deploy._api import app - -from ..utils import authenticate +from tests.deploy.utils import authenticate_with_api @pytest.mark.parametrize("ignore_unavailable_components", [True, False]) @@ -27,7 +26,7 @@ def test_ignore_unavailable_components(ignore_unavailable_components): ignore_unavailable_components=ignore_unavailable_components, ) ) as client: - authenticate(client) + authenticate_with_api(client) components = client.get("/components").raise_for_status().json() assert [assistant["title"] for assistant in components["assistants"]] == [ @@ -66,7 +65,7 @@ def test_unknown_component(tmp_local_root): with TestClient( app(config=Config(), ignore_unavailable_components=False) ) as client: - authenticate(client) + authenticate_with_api(client) document_upload = ( client.post("/document", json={"name": document_path.name}) diff --git a/tests/deploy/api/test_e2e.py b/tests/deploy/api/test_e2e.py index 8a958a705..4e60127b0 100644 --- a/tests/deploy/api/test_e2e.py +++ b/tests/deploy/api/test_e2e.py @@ -5,8 +5,7 @@ from ragna.deploy import Config from ragna.deploy._api import app - -from ..utils import TestAssistant, authenticate +from tests.deploy.utils import TestAssistant, authenticate_with_api @pytest.mark.parametrize("multiple_answer_chunks", [True, False]) @@ -21,7 +20,7 @@ def test_e2e(tmp_local_root, multiple_answer_chunks, stream_answer): file.write("!\n") with TestClient(app(config=config, ignore_unavailable_components=False)) as client: - authenticate(client) + authenticate_with_api(client) assert client.get("/chats").raise_for_status().json() == [] diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 4ae3c9d19..a0e48ce2c 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -10,8 +10,7 @@ from ragna._utils import timeout_after from ragna.deploy import Config - -from ..utils import TestAssistant +from tests.deploy.utils import TestAssistant def get_available_port(): diff --git a/tests/deploy/utils.py b/tests/deploy/utils.py index 5b4855e78..76638d4f2 100644 --- a/tests/deploy/utils.py +++ b/tests/deploy/utils.py @@ -26,7 +26,7 @@ def answer(self, prompt, sources, *, multiple_answer_chunks: bool = True): yield content -def authenticate(client: TestClient) -> None: +def authenticate_with_api(client: TestClient) -> None: username = default_user() token = ( client.post( From 162e982f8ba47f19b09b0a08c8f3dd0faa84b61b Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Fri, 7 Jun 2024 09:56:09 -0700 Subject: [PATCH 33/39] use function scope for test ports --- tests/deploy/ui/test_ui.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index a0e48ce2c..27656f1e6 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -19,12 +19,12 @@ def get_available_port(): return s.getsockname()[1] -@pytest.fixture(scope="session") +@pytest.fixture def api_port(): return get_available_port() -@pytest.fixture(scope="session") +@pytest.fixture def ui_port(): return get_available_port() From 8c82a4d3e54e3b022cddfe95ac904492a9e5e3e5 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Fri, 7 Jun 2024 10:03:20 -0700 Subject: [PATCH 34/39] review nits --- tests/deploy/ui/test_ui.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 27656f1e6..1130e5e61 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -97,12 +97,12 @@ def server(config): def test_health(server, page: Page) -> None: - health_url = server.base_url + "/health" + health_url = f"{server.base_url}/health" response = page.goto(health_url) assert response.ok -def test_index(server, config, page: Page) -> None: +def test_start_chat(server, config, page: Page) -> None: # Index page, no auth index_url = server.base_url page.goto(index_url) From dcd4339dfa5156141ed255737c778e4925c02902 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Fri, 7 Jun 2024 10:09:31 -0700 Subject: [PATCH 35/39] make Server a context manager --- tests/deploy/ui/test_ui.py | 171 ++++++++++++++++++------------------- 1 file changed, 85 insertions(+), 86 deletions(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 1130e5e61..74e70bbc5 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -85,89 +85,88 @@ def stop(self): self.proc.kill() pn.state.kill_all_servers() - -@pytest.fixture -def server(config): - server = Server(config) - try: - server.start() - yield server - finally: - server.stop() - - -def test_health(server, page: Page) -> None: - health_url = f"{server.base_url}/health" - response = page.goto(health_url) - assert response.ok - - -def test_start_chat(server, config, page: Page) -> None: - # Index page, no auth - index_url = server.base_url - page.goto(index_url) - expect(page.get_by_role("button", name="Sign In")).to_be_visible() - - # Authorize with no credentials - page.get_by_role("button", name="Sign In").click() - expect(page.get_by_role("button", name=" New Chat")).to_be_visible() - - # expect auth token to be set - cookies = page.context.cookies() - assert len(cookies) == 1 - cookie = cookies[0] - assert cookie.get("name") == "auth_token" - auth_token = cookie.get("value") - assert auth_token is not None - - # New page button - new_chat_button = page.get_by_role("button", name=" New Chat") - expect(new_chat_button).to_be_visible() - new_chat_button.click() - - document_root = config.local_root / "documents" - document_root.mkdir() - document_name = "test.txt" - document_path = document_root / document_name - with open(document_path, "w") as file: - file.write("!\n") - - # File upload selector - with page.expect_file_chooser() as fc_info: - page.locator(".fileUpload").click() - file_chooser = fc_info.value - file_chooser.set_files(document_path) - - # Upload document and expect to see it listed - file_list = page.locator(".fileListContainer") - expect(file_list.first).to_have_text(str(document_name)) - - chat_dialog = page.get_by_role("dialog") - expect(chat_dialog).to_be_visible() - start_chat_button = page.get_by_role("button", name="Start Conversation") - expect(start_chat_button).to_be_visible() - time.sleep(0.5) # hack while waiting for button to be fully clickable - start_chat_button.click(delay=5) - - chat_box_row = page.locator(".chat-interface-input-row") - expect(chat_box_row).to_be_visible() - - chat_box = chat_box_row.get_by_role("textbox") - expect(chat_box).to_be_visible() - - # Document should be in the database - chats_url = f"http://{config.api.hostname}:{config.api.port}/chats" - chats = httpx.get( - chats_url, headers={"Authorization": f"Bearer {auth_token}"} - ).json() - assert len(chats) == 1 - chat = chats[0] - chat_documents = chat["metadata"]["documents"] - assert len(chat_documents) == 1 - assert chat_documents[0]["name"] == document_name - - chat_box.fill("Tell me about the documents") - - chat_button = chat_box_row.get_by_role("button") - expect(chat_button).to_be_visible() - chat_button.click() + def __enter__(self): + self.start() + return self + + def __exit__(self, *args): + self.stop() + + +def test_health(config, page: Page) -> None: + with Server(config) as server: + health_url = f"{server.base_url}/health" + response = page.goto(health_url) + assert response.ok + + +def test_start_chat(config, page: Page) -> None: + with Server(config) as server: + # Index page, no auth + index_url = server.base_url + page.goto(index_url) + expect(page.get_by_role("button", name="Sign In")).to_be_visible() + + # Authorize with no credentials + page.get_by_role("button", name="Sign In").click() + expect(page.get_by_role("button", name=" New Chat")).to_be_visible() + + # expect auth token to be set + cookies = page.context.cookies() + assert len(cookies) == 1 + cookie = cookies[0] + assert cookie.get("name") == "auth_token" + auth_token = cookie.get("value") + assert auth_token is not None + + # New page button + new_chat_button = page.get_by_role("button", name=" New Chat") + expect(new_chat_button).to_be_visible() + new_chat_button.click() + + document_root = config.local_root / "documents" + document_root.mkdir() + document_name = "test.txt" + document_path = document_root / document_name + with open(document_path, "w") as file: + file.write("!\n") + + # File upload selector + with page.expect_file_chooser() as fc_info: + page.locator(".fileUpload").click() + file_chooser = fc_info.value + file_chooser.set_files(document_path) + + # Upload document and expect to see it listed + file_list = page.locator(".fileListContainer") + expect(file_list.first).to_have_text(str(document_name)) + + chat_dialog = page.get_by_role("dialog") + expect(chat_dialog).to_be_visible() + start_chat_button = page.get_by_role("button", name="Start Conversation") + expect(start_chat_button).to_be_visible() + time.sleep(0.5) # hack while waiting for button to be fully clickable + start_chat_button.click(delay=5) + + chat_box_row = page.locator(".chat-interface-input-row") + expect(chat_box_row).to_be_visible() + + chat_box = chat_box_row.get_by_role("textbox") + expect(chat_box).to_be_visible() + + # Document should be in the database + chats_url = f"http://{config.api.hostname}:{config.api.port}/chats" + chats = httpx.get( + chats_url, headers={"Authorization": f"Bearer {auth_token}"} + ).json() + assert len(chats) == 1 + chat = chats[0] + chat_documents = chat["metadata"]["documents"] + assert len(chat_documents) == 1 + assert chat_documents[0]["name"] == document_name + + chat_box.fill("Tell me about the documents") + + chat_button = chat_box_row.get_by_role("button") + expect(chat_button).to_be_visible() + chat_button.click() From c8ba9dc24c9cc26624e60d019e5a42793155958b Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Fri, 7 Jun 2024 10:53:59 -0700 Subject: [PATCH 36/39] try to fix CI matrix --- .github/workflows/test.yml | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b97f18e74..b458e7413 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -90,14 +90,22 @@ jobs: browser: - chromium - firefox - python-version: ["3.9"] + python-version: + - 3.9 + - 3.10 + - 3.11 + exclude: + - python-version: 3.10 + os: windows-latest + - python-version: 3.11 + os: windows-latest + - python-version: 3.10 + os: macos-latest + - python-version: 3.11 + os: macos-latest include: - - os: macos-latest - browser: webkit - - os: ubuntu-latest - python-version: "3.10" - - os: ubuntu-latest - python-version: "3.11" + - browser: webkit + os: macos-latest fail-fast: false From d2dfccef311ffa35978d138c9687836d4e768cef Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Fri, 7 Jun 2024 10:57:44 -0700 Subject: [PATCH 37/39] attempt #2 --- .github/workflows/test.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b458e7413..f40f160f8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -91,21 +91,22 @@ jobs: - chromium - firefox python-version: - - 3.9 - - 3.10 - - 3.11 + - "3.9" + - "3.10" + - "3.11" exclude: - - python-version: 3.10 + - python-version: "3.10" os: windows-latest - - python-version: 3.11 + - python-version: "3.11" os: windows-latest - - python-version: 3.10 + - python-version: "3.10" os: macos-latest - - python-version: 3.11 + - python-version: "3.11" os: macos-latest include: - browser: webkit os: macos-latest + python-version: "3.9" fail-fast: false From 8a604305e36005a3f09733679f60db6f771c1f41 Mon Sep 17 00:00:00 2001 From: Philip Meier Date: Mon, 10 Jun 2024 11:40:21 +0200 Subject: [PATCH 38/39] nits --- .github/workflows/test.yml | 2 +- tests/deploy/ui/test_ui.py | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f40f160f8..5ff4147de 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,7 +72,7 @@ jobs: pytest \ --ignore tests/deploy/ui \ --junit-xml=test-results.xml \ - --durations=25 \ + --durations=25 - name: Surface failing tests if: steps.tests.outcome != 'success' diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index 74e70bbc5..a64cf34d7 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -19,16 +19,6 @@ def get_available_port(): return s.getsockname()[1] -@pytest.fixture -def api_port(): - return get_available_port() - - -@pytest.fixture -def ui_port(): - return get_available_port() - - @pytest.fixture(scope="session") def headed_mode(pytestconfig): return pytestconfig.getoption("headed") or False From 37e98122ee1a02f70cc092cc78d5afbb354bebe1 Mon Sep 17 00:00:00 2001 From: Blake Rosenthal Date: Mon, 10 Jun 2024 09:57:06 -0700 Subject: [PATCH 39/39] remove unused func --- tests/deploy/ui/test_ui.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/deploy/ui/test_ui.py b/tests/deploy/ui/test_ui.py index a64cf34d7..856992781 100644 --- a/tests/deploy/ui/test_ui.py +++ b/tests/deploy/ui/test_ui.py @@ -19,11 +19,6 @@ def get_available_port(): return s.getsockname()[1] -@pytest.fixture(scope="session") -def headed_mode(pytestconfig): - return pytestconfig.getoption("headed") or False - - @pytest.fixture def config( tmp_local_root,