From 8ea0abf9e0c5c2ed3f8bee157a6e4695ba729669 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Wed, 24 Jul 2024 13:23:20 +0300 Subject: [PATCH 01/16] add playwright and fix db port for running tests outside container --- .devcontainer/docker-compose.yaml | 3 +++ .gitignore | 2 ++ pyproject.toml | 4 ++-- requirements-dev.txt | 5 +++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml index cd408336..552bc0e7 100644 --- a/.devcontainer/docker-compose.yaml +++ b/.devcontainer/docker-compose.yaml @@ -27,6 +27,9 @@ services: POSTGRES_USER: admin POSTGRES_PASSWORD: postgres + ports: + - "5432:5432" + # For local developemnt, we need to forward the database port here too. # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. # (Adding the "ports" property to this file will not forward from a Codespace.) diff --git a/.gitignore b/.gitignore index 20dda899..18609cf6 100644 --- a/.gitignore +++ b/.gitignore @@ -146,3 +146,5 @@ npm-debug.log* node_modules static/ +# Playwright test trace +test-results/ diff --git a/pyproject.toml b/pyproject.toml index 5905aa19..1f168be0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,9 +11,9 @@ python_version = 3.12 exclude = [".venv/*"] [tool.pytest.ini_options] -addopts = "-ra --cov" +addopts = "-ra" testpaths = ["tests"] -pythonpath = ['src'] +pythonpath = ['src/backend'] filterwarnings = ["ignore::DeprecationWarning"] [[tool.mypy.overrides]] diff --git a/requirements-dev.txt b/requirements-dev.txt index 1d7ad271..5ebb470c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,9 +4,10 @@ mypy pre-commit pip-tools pip-compile-cross-platform +playwright pytest -pytest-cov pytest-asyncio +pytest-cov +pytest-playwright pytest-snapshot -mypy locust From b67dbc97f4b543d3bbec255a8e94471c8f3b2fa9 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Wed, 24 Jul 2024 14:03:16 +0300 Subject: [PATCH 02/16] change 'session_state' to 'sessionState' Change to conform with microsoft chat protocol --- src/backend/fastapi_app/api_models.py | 5 +++-- src/frontend/src/pages/chat/Chat.tsx | 3 ++- .../test_advanced_chat_flow/advanced_chat_flow_response.json | 2 +- .../advanced_chat_streaming_flow_response.jsonlines | 4 ++-- .../test_simple_chat_flow/simple_chat_flow_response.json | 2 +- .../simple_chat_streaming_flow_response.jsonlines | 4 ++-- 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/backend/fastapi_app/api_models.py b/src/backend/fastapi_app/api_models.py index c98ca76d..616275c2 100644 --- a/src/backend/fastapi_app/api_models.py +++ b/src/backend/fastapi_app/api_models.py @@ -37,6 +37,7 @@ class ChatRequestContext(BaseModel): class ChatRequest(BaseModel): messages: list[ChatCompletionMessageParam] context: ChatRequestContext + sessionState: Any | None = None class ThoughtStep(BaseModel): @@ -54,13 +55,13 @@ class RAGContext(BaseModel): class RetrievalResponse(BaseModel): message: Message context: RAGContext - session_state: Any | None = None + sessionState: Any | None = None class RetrievalResponseDelta(BaseModel): delta: Message | None = None context: RAGContext | None = None - session_state: Any | None = None + sessionState: Any | None = None class ItemPublic(BaseModel): diff --git a/src/frontend/src/pages/chat/Chat.tsx b/src/frontend/src/pages/chat/Chat.tsx index da0b6934..f583f012 100644 --- a/src/frontend/src/pages/chat/Chat.tsx +++ b/src/frontend/src/pages/chat/Chat.tsx @@ -108,7 +108,8 @@ const Chat = () => { prompt_template: promptTemplate.length === 0 ? undefined : promptTemplate, temperature: temperature } - } + }, + sessionState: answers.length ? answers[answers.length - 1][1].sessionState : null }; const chatClient: AIChatProtocolClient = new AIChatProtocolClient("/chat"); if (shouldStream) { diff --git a/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json b/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json index 2e9eb3ae..c7692bd1 100644 --- a/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json +++ b/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json @@ -64,5 +64,5 @@ ], "followup_questions": null }, - "session_state": null + "sessionState": null } \ No newline at end of file diff --git a/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines b/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines index 8b65342f..b7e4efa3 100644 --- a/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines +++ b/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines @@ -1,2 +1,2 @@ -{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":["{'role': 'system', 'content': 'Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\\nGenerate a search query based on the conversation and the new question.\\nIf the question is not in English, translate the question to English before generating the search query.\\nIf you cannot generate a search query, return the original user question.\\nDO NOT return anything besides the query.'}","{'role': 'user', 'content': 'What is the capital of France?'}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}},{"title":"Search using generated search arguments","description":"The capital of France is Paris. [Benefit_Options-2.pdf].","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"session_state":null} -{"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"session_state":null} +{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":["{'role': 'system', 'content': 'Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\\nGenerate a search query based on the conversation and the new question.\\nIf the question is not in English, translate the question to English before generating the search query.\\nIf you cannot generate a search query, return the original user question.\\nDO NOT return anything besides the query.'}","{'role': 'user', 'content': 'What is the capital of France?'}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}},{"title":"Search using generated search arguments","description":"The capital of France is Paris. [Benefit_Options-2.pdf].","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"sessionState":null} +{"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"sessionState":null} diff --git a/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json b/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json index d5ecba21..a73ff24d 100644 --- a/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json +++ b/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json @@ -52,5 +52,5 @@ ], "followup_questions": null }, - "session_state": null + "sessionState": null } \ No newline at end of file diff --git a/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines b/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines index 6251bd52..fc63aea9 100644 --- a/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines +++ b/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines @@ -1,2 +1,2 @@ -{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Search query for database","description":"What is the capital of France?","props":{"top":1,"vector_search":true,"text_search":true}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"session_state":null} -{"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"session_state":null} +{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Search query for database","description":"What is the capital of France?","props":{"top":1,"vector_search":true,"text_search":true}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"sessionState":null} +{"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"sessionState":null} From 367a86d852d60eac936db4b3423d01fe6d0b1193 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Wed, 24 Jul 2024 14:08:05 +0300 Subject: [PATCH 03/16] add playwright tests and workflows --- .github/workflows/app-tests.yaml | 14 ++- tests/e2e.py | 192 +++++++++++++++++++++++++++++++ 2 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 tests/e2e.py diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index ec3e5e1b..c1c14bd1 100755 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -73,4 +73,16 @@ jobs: - name: Run MyPy run: python3 -m mypy . - name: Run Pytest - run: python3 -m pytest + run: python3 -m pytest -s -vv --cov --cov-fail-under=85 + - name: Run E2E tests with Playwright + id: e2e + if: runner.os != 'Windows' + run: | + playwright install chromium --with-deps + python3 -m pytest tests/e2e.py --tracing=retain-on-failure + - name: Upload test artifacts + if: ${{ failure() && steps.e2e.conclusion == 'failure' }} + uses: actions/upload-artifact@v4 + with: + name: playwright-traces${{ matrix.python_version }} + path: test-results diff --git a/tests/e2e.py b/tests/e2e.py new file mode 100644 index 00000000..ace7f591 --- /dev/null +++ b/tests/e2e.py @@ -0,0 +1,192 @@ +import socket +import time +from collections.abc import Generator +from contextlib import closing +from multiprocessing import Process + +import pytest +import requests +import uvicorn +from playwright.sync_api import Page, Route, expect + +import fastapi_app as app + +expect.set_options(timeout=10_000) + + +def wait_for_server_ready(url: str, timeout: float = 10.0, check_interval: float = 0.5) -> bool: + """Make requests to provided url until it responds without error.""" + conn_error = None + for _ in range(int(timeout / check_interval)): + try: + requests.get(url) + except requests.ConnectionError as exc: + time.sleep(check_interval) + conn_error = str(exc) + else: + return True + raise RuntimeError(conn_error) + + +@pytest.fixture(scope="session") +def free_port() -> int: + """Returns a free port for the test server to bind.""" + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + s.bind(("", 0)) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return s.getsockname()[1] + + +def run_server(port: int): + uvicorn.run(app.create_app(testing=True), port=port) + + +@pytest.fixture() +def live_server_url(mock_session_env, free_port: int) -> Generator[str, None, None]: + proc = Process(target=run_server, args=(free_port,), daemon=True) + proc.start() + url = f"http://localhost:{free_port}/" + wait_for_server_ready(url, timeout=10.0, check_interval=0.5) + yield url + proc.kill() + + +@pytest.fixture(params=[(480, 800), (600, 1024), (768, 1024), (992, 1024), (1024, 768)]) +def sized_page(page: Page, request): + size = request.param + page.set_viewport_size({"width": size[0], "height": size[1]}) + yield page + + +def test_home(page: Page, live_server_url: str): + page.goto(live_server_url) + expect(page).to_have_title("RAG on PostgreSQL") + + +def test_chat(sized_page: Page, live_server_url: str): + page = sized_page + + # Set up a mock route to the /chat endpoint with streaming results + def handle(route: Route): + # Assert that session_state is specified in the request (None for now) + if route.request.post_data_json: + session_state = route.request.post_data_json["sessionState"] + assert session_state is None + # Read the JSONL from our snapshot results and return as the response + f = open( + "tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines" + ) + jsonl = f.read() + f.close() + route.fulfill(body=jsonl, status=200, headers={"Transfer-encoding": "Chunked"}) + + page.route("*/**/chat/stream", handle) + + # Check initial page state + page.goto(live_server_url) + expect(page).to_have_title("RAG on PostgreSQL") + expect(page.get_by_role("heading", name="Product chat")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_disabled() + expect(page.get_by_role("button", name="Developer settings")).to_be_enabled() + + # Ask a question and wait for the message to appear + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").click() + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").fill( + "Whats the dental plan?" + ) + page.get_by_role("button", name="Ask question button").click() + + expect(page.get_by_text("Whats the dental plan?")).to_be_visible() + expect(page.get_by_text("The capital of France is Paris.")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_enabled() + + # Show the thought process + page.get_by_label("Show thought process").click() + expect(page.get_by_title("Thought process")).to_be_visible() + expect(page.get_by_text("Prompt to generate search arguments")).to_be_visible() + + # Clear the chat + page.get_by_role("button", name="Clear chat").click() + expect(page.get_by_text("Whats the dental plan?")).not_to_be_visible() + expect(page.get_by_text("The capital of France is Paris.")).not_to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_disabled() + + +def test_chat_customization(page: Page, live_server_url: str): + # Set up a mock route to the /chat endpoint + def handle(route: Route): + if route.request.post_data_json: + overrides = route.request.post_data_json["context"]["overrides"] + assert overrides["use_advanced_flow"] is False + assert overrides["retrieval_mode"] == "vectors" + assert overrides["top"] == 1 + assert overrides["prompt_template"] == "You are a cat and only talk about tuna." + + # Read the JSON from our snapshot results and return as the response + f = open("tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json") + json = f.read() + f.close() + route.fulfill(body=json, status=200) + + page.route("*/**/chat", handle) + + # Check initial page state + page.goto(live_server_url) + expect(page).to_have_title("RAG on PostgreSQL") + + # Customize all the settings + page.get_by_role("button", name="Developer settings").click() + page.get_by_text( + "Use advanced flow with query rewriting and filter formulation. Not compatible with Ollama models." + ).click() + page.get_by_label("Retrieve this many matching rows:").click() + page.get_by_label("Retrieve this many matching rows:").fill("1") + page.get_by_text("Vectors + Text (Hybrid)").click() + page.get_by_role("option", name="Vectors", exact=True).click() + page.get_by_label("Override prompt template").click() + page.get_by_label("Override prompt template").fill("You are a cat and only talk about tuna.") + + page.get_by_text("Stream chat completion responses").click() + page.locator("button").filter(has_text="Close").click() + + # Ask a question and wait for the message to appear + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").click() + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").fill( + "Whats the dental plan?" + ) + page.get_by_role("button", name="Ask question button").click() + + expect(page.get_by_text("Whats the dental plan?")).to_be_visible() + expect(page.get_by_text("The capital of France is Paris.")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_enabled() + + +def test_chat_nonstreaming(page: Page, live_server_url: str): + # Set up a mock route to the /chat_stream endpoint + def handle(route: Route): + # Read the JSON from our snapshot results and return as the response + f = open("tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json") + json = f.read() + f.close() + route.fulfill(body=json, status=200) + + page.route("*/**/chat", handle) + + # Check initial page state + page.goto(live_server_url) + expect(page).to_have_title("RAG on PostgreSQL") + expect(page.get_by_role("button", name="Developer settings")).to_be_enabled() + page.get_by_role("button", name="Developer settings").click() + page.get_by_text("Stream chat completion responses").click() + page.locator("button").filter(has_text="Close").click() + + # Ask a question and wait for the message to appear + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").click() + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").fill( + "Whats the dental plan?" + ) + page.get_by_label("Ask question button").click() + + expect(page.get_by_text("Whats the dental plan?")).to_be_visible() + expect(page.get_by_text("The capital of France is Paris.")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_enabled() From 8f27e2396a5946b9841cce7302852eb9755d0dc1 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Wed, 24 Jul 2024 14:08:23 +0300 Subject: [PATCH 04/16] add types for mypy --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 5ebb470c..08ce71aa 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,7 @@ -r src/backend/requirements.txt ruff mypy +types-requests pre-commit pip-tools pip-compile-cross-platform From 4c7bcaa04c777467c9316a3559e5f11924cffaf9 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 24 Jul 2024 14:50:25 -0700 Subject: [PATCH 05/16] Update .devcontainer/docker-compose.yaml --- .devcontainer/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml index 552bc0e7..e1c04e14 100644 --- a/.devcontainer/docker-compose.yaml +++ b/.devcontainer/docker-compose.yaml @@ -29,7 +29,7 @@ services: ports: - "5432:5432" - # For local developemnt, we need to forward the database port here too. + # For local development, we need to forward the database port here too. # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. # (Adding the "ports" property to this file will not forward from a Codespace.) From 8d6abfd9003d1cec2d070372f30822a304458d69 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Thu, 25 Jul 2024 15:26:35 +0300 Subject: [PATCH 06/16] apply feedback from PR review --- .devcontainer/docker-compose.yaml | 3 --- tests/e2e.py | 11 +---------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml index e1c04e14..cd408336 100644 --- a/.devcontainer/docker-compose.yaml +++ b/.devcontainer/docker-compose.yaml @@ -27,9 +27,6 @@ services: POSTGRES_USER: admin POSTGRES_PASSWORD: postgres - ports: - - "5432:5432" - # For local development, we need to forward the database port here too. # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. # (Adding the "ports" property to this file will not forward from a Codespace.) diff --git a/tests/e2e.py b/tests/e2e.py index ace7f591..56f6023d 100644 --- a/tests/e2e.py +++ b/tests/e2e.py @@ -51,21 +51,12 @@ def live_server_url(mock_session_env, free_port: int) -> Generator[str, None, No proc.kill() -@pytest.fixture(params=[(480, 800), (600, 1024), (768, 1024), (992, 1024), (1024, 768)]) -def sized_page(page: Page, request): - size = request.param - page.set_viewport_size({"width": size[0], "height": size[1]}) - yield page - - def test_home(page: Page, live_server_url: str): page.goto(live_server_url) expect(page).to_have_title("RAG on PostgreSQL") -def test_chat(sized_page: Page, live_server_url: str): - page = sized_page - +def test_chat(page: Page, live_server_url: str): # Set up a mock route to the /chat endpoint with streaming results def handle(route: Route): # Assert that session_state is specified in the request (None for now) From 5c708e076671aa0edb0e0270dcde55541f98d265 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 29 Jul 2024 21:34:21 +0000 Subject: [PATCH 07/16] Try on windows --- .github/workflows/app-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index c1c14bd1..922f5943 100755 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -76,7 +76,7 @@ jobs: run: python3 -m pytest -s -vv --cov --cov-fail-under=85 - name: Run E2E tests with Playwright id: e2e - if: runner.os != 'Windows' + #if: runner.os != 'Windows' run: | playwright install chromium --with-deps python3 -m pytest tests/e2e.py --tracing=retain-on-failure From 76d1899f228372bda6a722561b7937cf80e7b775 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Wed, 24 Jul 2024 13:23:20 +0300 Subject: [PATCH 08/16] add playwright and fix db port for running tests outside container --- .devcontainer/docker-compose.yaml | 3 +++ .gitignore | 2 ++ pyproject.toml | 4 ++-- requirements-dev.txt | 5 +++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml index cd408336..552bc0e7 100644 --- a/.devcontainer/docker-compose.yaml +++ b/.devcontainer/docker-compose.yaml @@ -27,6 +27,9 @@ services: POSTGRES_USER: admin POSTGRES_PASSWORD: postgres + ports: + - "5432:5432" + # For local developemnt, we need to forward the database port here too. # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. # (Adding the "ports" property to this file will not forward from a Codespace.) diff --git a/.gitignore b/.gitignore index 20dda899..18609cf6 100644 --- a/.gitignore +++ b/.gitignore @@ -146,3 +146,5 @@ npm-debug.log* node_modules static/ +# Playwright test trace +test-results/ diff --git a/pyproject.toml b/pyproject.toml index 5905aa19..1f168be0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,9 +11,9 @@ python_version = 3.12 exclude = [".venv/*"] [tool.pytest.ini_options] -addopts = "-ra --cov" +addopts = "-ra" testpaths = ["tests"] -pythonpath = ['src'] +pythonpath = ['src/backend'] filterwarnings = ["ignore::DeprecationWarning"] [[tool.mypy.overrides]] diff --git a/requirements-dev.txt b/requirements-dev.txt index 1d7ad271..5ebb470c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,9 +4,10 @@ mypy pre-commit pip-tools pip-compile-cross-platform +playwright pytest -pytest-cov pytest-asyncio +pytest-cov +pytest-playwright pytest-snapshot -mypy locust From 2dda914e4c8455ac19a70a5db64b56d97e071602 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Wed, 24 Jul 2024 14:03:16 +0300 Subject: [PATCH 09/16] change 'session_state' to 'sessionState' Change to conform with microsoft chat protocol --- src/backend/fastapi_app/api_models.py | 5 +++-- src/frontend/src/pages/chat/Chat.tsx | 3 ++- .../test_advanced_chat_flow/advanced_chat_flow_response.json | 2 +- .../advanced_chat_streaming_flow_response.jsonlines | 4 ++-- .../test_simple_chat_flow/simple_chat_flow_response.json | 2 +- .../simple_chat_streaming_flow_response.jsonlines | 4 ++-- 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/backend/fastapi_app/api_models.py b/src/backend/fastapi_app/api_models.py index c98ca76d..616275c2 100644 --- a/src/backend/fastapi_app/api_models.py +++ b/src/backend/fastapi_app/api_models.py @@ -37,6 +37,7 @@ class ChatRequestContext(BaseModel): class ChatRequest(BaseModel): messages: list[ChatCompletionMessageParam] context: ChatRequestContext + sessionState: Any | None = None class ThoughtStep(BaseModel): @@ -54,13 +55,13 @@ class RAGContext(BaseModel): class RetrievalResponse(BaseModel): message: Message context: RAGContext - session_state: Any | None = None + sessionState: Any | None = None class RetrievalResponseDelta(BaseModel): delta: Message | None = None context: RAGContext | None = None - session_state: Any | None = None + sessionState: Any | None = None class ItemPublic(BaseModel): diff --git a/src/frontend/src/pages/chat/Chat.tsx b/src/frontend/src/pages/chat/Chat.tsx index da0b6934..f583f012 100644 --- a/src/frontend/src/pages/chat/Chat.tsx +++ b/src/frontend/src/pages/chat/Chat.tsx @@ -108,7 +108,8 @@ const Chat = () => { prompt_template: promptTemplate.length === 0 ? undefined : promptTemplate, temperature: temperature } - } + }, + sessionState: answers.length ? answers[answers.length - 1][1].sessionState : null }; const chatClient: AIChatProtocolClient = new AIChatProtocolClient("/chat"); if (shouldStream) { diff --git a/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json b/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json index 2e9eb3ae..c7692bd1 100644 --- a/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json +++ b/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json @@ -64,5 +64,5 @@ ], "followup_questions": null }, - "session_state": null + "sessionState": null } \ No newline at end of file diff --git a/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines b/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines index 8b65342f..b7e4efa3 100644 --- a/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines +++ b/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines @@ -1,2 +1,2 @@ -{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":["{'role': 'system', 'content': 'Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\\nGenerate a search query based on the conversation and the new question.\\nIf the question is not in English, translate the question to English before generating the search query.\\nIf you cannot generate a search query, return the original user question.\\nDO NOT return anything besides the query.'}","{'role': 'user', 'content': 'What is the capital of France?'}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}},{"title":"Search using generated search arguments","description":"The capital of France is Paris. [Benefit_Options-2.pdf].","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"session_state":null} -{"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"session_state":null} +{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":["{'role': 'system', 'content': 'Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\\nGenerate a search query based on the conversation and the new question.\\nIf the question is not in English, translate the question to English before generating the search query.\\nIf you cannot generate a search query, return the original user question.\\nDO NOT return anything besides the query.'}","{'role': 'user', 'content': 'What is the capital of France?'}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}},{"title":"Search using generated search arguments","description":"The capital of France is Paris. [Benefit_Options-2.pdf].","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"sessionState":null} +{"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"sessionState":null} diff --git a/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json b/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json index d5ecba21..a73ff24d 100644 --- a/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json +++ b/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json @@ -52,5 +52,5 @@ ], "followup_questions": null }, - "session_state": null + "sessionState": null } \ No newline at end of file diff --git a/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines b/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines index 6251bd52..fc63aea9 100644 --- a/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines +++ b/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines @@ -1,2 +1,2 @@ -{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Search query for database","description":"What is the capital of France?","props":{"top":1,"vector_search":true,"text_search":true}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"session_state":null} -{"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"session_state":null} +{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Search query for database","description":"What is the capital of France?","props":{"top":1,"vector_search":true,"text_search":true}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"sessionState":null} +{"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"sessionState":null} From 69b174184dcc13e209c35dcf264245794f98833f Mon Sep 17 00:00:00 2001 From: John Aziz Date: Wed, 24 Jul 2024 14:08:05 +0300 Subject: [PATCH 10/16] add playwright tests and workflows --- .github/workflows/app-tests.yaml | 14 ++- tests/e2e.py | 192 +++++++++++++++++++++++++++++++ 2 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 tests/e2e.py diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index ec3e5e1b..c1c14bd1 100755 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -73,4 +73,16 @@ jobs: - name: Run MyPy run: python3 -m mypy . - name: Run Pytest - run: python3 -m pytest + run: python3 -m pytest -s -vv --cov --cov-fail-under=85 + - name: Run E2E tests with Playwright + id: e2e + if: runner.os != 'Windows' + run: | + playwright install chromium --with-deps + python3 -m pytest tests/e2e.py --tracing=retain-on-failure + - name: Upload test artifacts + if: ${{ failure() && steps.e2e.conclusion == 'failure' }} + uses: actions/upload-artifact@v4 + with: + name: playwright-traces${{ matrix.python_version }} + path: test-results diff --git a/tests/e2e.py b/tests/e2e.py new file mode 100644 index 00000000..ace7f591 --- /dev/null +++ b/tests/e2e.py @@ -0,0 +1,192 @@ +import socket +import time +from collections.abc import Generator +from contextlib import closing +from multiprocessing import Process + +import pytest +import requests +import uvicorn +from playwright.sync_api import Page, Route, expect + +import fastapi_app as app + +expect.set_options(timeout=10_000) + + +def wait_for_server_ready(url: str, timeout: float = 10.0, check_interval: float = 0.5) -> bool: + """Make requests to provided url until it responds without error.""" + conn_error = None + for _ in range(int(timeout / check_interval)): + try: + requests.get(url) + except requests.ConnectionError as exc: + time.sleep(check_interval) + conn_error = str(exc) + else: + return True + raise RuntimeError(conn_error) + + +@pytest.fixture(scope="session") +def free_port() -> int: + """Returns a free port for the test server to bind.""" + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + s.bind(("", 0)) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return s.getsockname()[1] + + +def run_server(port: int): + uvicorn.run(app.create_app(testing=True), port=port) + + +@pytest.fixture() +def live_server_url(mock_session_env, free_port: int) -> Generator[str, None, None]: + proc = Process(target=run_server, args=(free_port,), daemon=True) + proc.start() + url = f"http://localhost:{free_port}/" + wait_for_server_ready(url, timeout=10.0, check_interval=0.5) + yield url + proc.kill() + + +@pytest.fixture(params=[(480, 800), (600, 1024), (768, 1024), (992, 1024), (1024, 768)]) +def sized_page(page: Page, request): + size = request.param + page.set_viewport_size({"width": size[0], "height": size[1]}) + yield page + + +def test_home(page: Page, live_server_url: str): + page.goto(live_server_url) + expect(page).to_have_title("RAG on PostgreSQL") + + +def test_chat(sized_page: Page, live_server_url: str): + page = sized_page + + # Set up a mock route to the /chat endpoint with streaming results + def handle(route: Route): + # Assert that session_state is specified in the request (None for now) + if route.request.post_data_json: + session_state = route.request.post_data_json["sessionState"] + assert session_state is None + # Read the JSONL from our snapshot results and return as the response + f = open( + "tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines" + ) + jsonl = f.read() + f.close() + route.fulfill(body=jsonl, status=200, headers={"Transfer-encoding": "Chunked"}) + + page.route("*/**/chat/stream", handle) + + # Check initial page state + page.goto(live_server_url) + expect(page).to_have_title("RAG on PostgreSQL") + expect(page.get_by_role("heading", name="Product chat")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_disabled() + expect(page.get_by_role("button", name="Developer settings")).to_be_enabled() + + # Ask a question and wait for the message to appear + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").click() + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").fill( + "Whats the dental plan?" + ) + page.get_by_role("button", name="Ask question button").click() + + expect(page.get_by_text("Whats the dental plan?")).to_be_visible() + expect(page.get_by_text("The capital of France is Paris.")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_enabled() + + # Show the thought process + page.get_by_label("Show thought process").click() + expect(page.get_by_title("Thought process")).to_be_visible() + expect(page.get_by_text("Prompt to generate search arguments")).to_be_visible() + + # Clear the chat + page.get_by_role("button", name="Clear chat").click() + expect(page.get_by_text("Whats the dental plan?")).not_to_be_visible() + expect(page.get_by_text("The capital of France is Paris.")).not_to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_disabled() + + +def test_chat_customization(page: Page, live_server_url: str): + # Set up a mock route to the /chat endpoint + def handle(route: Route): + if route.request.post_data_json: + overrides = route.request.post_data_json["context"]["overrides"] + assert overrides["use_advanced_flow"] is False + assert overrides["retrieval_mode"] == "vectors" + assert overrides["top"] == 1 + assert overrides["prompt_template"] == "You are a cat and only talk about tuna." + + # Read the JSON from our snapshot results and return as the response + f = open("tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json") + json = f.read() + f.close() + route.fulfill(body=json, status=200) + + page.route("*/**/chat", handle) + + # Check initial page state + page.goto(live_server_url) + expect(page).to_have_title("RAG on PostgreSQL") + + # Customize all the settings + page.get_by_role("button", name="Developer settings").click() + page.get_by_text( + "Use advanced flow with query rewriting and filter formulation. Not compatible with Ollama models." + ).click() + page.get_by_label("Retrieve this many matching rows:").click() + page.get_by_label("Retrieve this many matching rows:").fill("1") + page.get_by_text("Vectors + Text (Hybrid)").click() + page.get_by_role("option", name="Vectors", exact=True).click() + page.get_by_label("Override prompt template").click() + page.get_by_label("Override prompt template").fill("You are a cat and only talk about tuna.") + + page.get_by_text("Stream chat completion responses").click() + page.locator("button").filter(has_text="Close").click() + + # Ask a question and wait for the message to appear + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").click() + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").fill( + "Whats the dental plan?" + ) + page.get_by_role("button", name="Ask question button").click() + + expect(page.get_by_text("Whats the dental plan?")).to_be_visible() + expect(page.get_by_text("The capital of France is Paris.")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_enabled() + + +def test_chat_nonstreaming(page: Page, live_server_url: str): + # Set up a mock route to the /chat_stream endpoint + def handle(route: Route): + # Read the JSON from our snapshot results and return as the response + f = open("tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json") + json = f.read() + f.close() + route.fulfill(body=json, status=200) + + page.route("*/**/chat", handle) + + # Check initial page state + page.goto(live_server_url) + expect(page).to_have_title("RAG on PostgreSQL") + expect(page.get_by_role("button", name="Developer settings")).to_be_enabled() + page.get_by_role("button", name="Developer settings").click() + page.get_by_text("Stream chat completion responses").click() + page.locator("button").filter(has_text="Close").click() + + # Ask a question and wait for the message to appear + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").click() + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").fill( + "Whats the dental plan?" + ) + page.get_by_label("Ask question button").click() + + expect(page.get_by_text("Whats the dental plan?")).to_be_visible() + expect(page.get_by_text("The capital of France is Paris.")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_enabled() From b1e2e6b670af50c65d5479edfb72c9032a4389b8 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Wed, 24 Jul 2024 14:08:23 +0300 Subject: [PATCH 11/16] add types for mypy --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 5ebb470c..08ce71aa 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,7 @@ -r src/backend/requirements.txt ruff mypy +types-requests pre-commit pip-tools pip-compile-cross-platform From be1cc1511fa7fe863f7dd79fa846614d54b654e4 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 24 Jul 2024 14:50:25 -0700 Subject: [PATCH 12/16] Update .devcontainer/docker-compose.yaml --- .devcontainer/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml index 552bc0e7..e1c04e14 100644 --- a/.devcontainer/docker-compose.yaml +++ b/.devcontainer/docker-compose.yaml @@ -29,7 +29,7 @@ services: ports: - "5432:5432" - # For local developemnt, we need to forward the database port here too. + # For local development, we need to forward the database port here too. # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. # (Adding the "ports" property to this file will not forward from a Codespace.) From 1eddb92d14d566e092c20f66917f736fc9ecb030 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Thu, 25 Jul 2024 15:26:35 +0300 Subject: [PATCH 13/16] apply feedback from PR review --- .devcontainer/docker-compose.yaml | 3 --- tests/e2e.py | 11 +---------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml index e1c04e14..cd408336 100644 --- a/.devcontainer/docker-compose.yaml +++ b/.devcontainer/docker-compose.yaml @@ -27,9 +27,6 @@ services: POSTGRES_USER: admin POSTGRES_PASSWORD: postgres - ports: - - "5432:5432" - # For local development, we need to forward the database port here too. # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. # (Adding the "ports" property to this file will not forward from a Codespace.) diff --git a/tests/e2e.py b/tests/e2e.py index ace7f591..56f6023d 100644 --- a/tests/e2e.py +++ b/tests/e2e.py @@ -51,21 +51,12 @@ def live_server_url(mock_session_env, free_port: int) -> Generator[str, None, No proc.kill() -@pytest.fixture(params=[(480, 800), (600, 1024), (768, 1024), (992, 1024), (1024, 768)]) -def sized_page(page: Page, request): - size = request.param - page.set_viewport_size({"width": size[0], "height": size[1]}) - yield page - - def test_home(page: Page, live_server_url: str): page.goto(live_server_url) expect(page).to_have_title("RAG on PostgreSQL") -def test_chat(sized_page: Page, live_server_url: str): - page = sized_page - +def test_chat(page: Page, live_server_url: str): # Set up a mock route to the /chat endpoint with streaming results def handle(route: Route): # Assert that session_state is specified in the request (None for now) From c26d3f229e664b7ff4475e1b5c462df6a5c3a654 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Mon, 29 Jul 2024 21:34:21 +0000 Subject: [PATCH 14/16] Try on windows --- .github/workflows/app-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index c1c14bd1..922f5943 100755 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -76,7 +76,7 @@ jobs: run: python3 -m pytest -s -vv --cov --cov-fail-under=85 - name: Run E2E tests with Playwright id: e2e - if: runner.os != 'Windows' + #if: runner.os != 'Windows' run: | playwright install chromium --with-deps python3 -m pytest tests/e2e.py --tracing=retain-on-failure From 0ac27bcd3ee4f50b5a33bd3ac46091c357d255b2 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 1 Aug 2024 00:35:05 +0000 Subject: [PATCH 15/16] Get Ollama working --- src/backend/fastapi_app/dependencies.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/backend/fastapi_app/dependencies.py b/src/backend/fastapi_app/dependencies.py index 00837111..1cec6ad3 100644 --- a/src/backend/fastapi_app/dependencies.py +++ b/src/backend/fastapi_app/dependencies.py @@ -29,8 +29,8 @@ class FastAPIAppContext(BaseModel): openai_chat_model: str openai_embed_model: str openai_embed_dimensions: int - openai_chat_deployment: str - openai_embed_deployment: str + openai_chat_deployment: str | None + openai_embed_deployment: str | None async def common_parameters(): @@ -51,10 +51,10 @@ async def common_parameters(): openai_chat_deployment = os.getenv("AZURE_OPENAI_CHAT_DEPLOYMENT", "gpt-35-turbo") openai_chat_model = os.getenv("AZURE_OPENAI_CHAT_MODEL", "gpt-35-turbo") elif OPENAI_CHAT_HOST == "ollama": - openai_chat_deployment = "phi3:3.8b" + openai_chat_deployment = None openai_chat_model = os.getenv("OLLAMA_CHAT_MODEL", "phi3:3.8b") else: - openai_chat_deployment = "gpt-3.5-turbo" + openai_chat_deployment = None openai_chat_model = os.getenv("OPENAICOM_CHAT_MODEL", "gpt-3.5-turbo") return FastAPIAppContext( openai_chat_model=openai_chat_model, From 2093fd9f8e69d11c638e69981a566650e6a40220 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Thu, 1 Aug 2024 00:52:58 +0000 Subject: [PATCH 16/16] Disable Windows --- .github/workflows/app-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 922f5943..c1c14bd1 100755 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -76,7 +76,7 @@ jobs: run: python3 -m pytest -s -vv --cov --cov-fail-under=85 - name: Run E2E tests with Playwright id: e2e - #if: runner.os != 'Windows' + if: runner.os != 'Windows' run: | playwright install chromium --with-deps python3 -m pytest tests/e2e.py --tracing=retain-on-failure