|
| 1 | +import socket |
| 2 | +import time |
| 3 | +from collections.abc import Generator |
| 4 | +from contextlib import closing |
| 5 | +from multiprocessing import Process |
| 6 | + |
| 7 | +import pytest |
| 8 | +import requests |
| 9 | +import uvicorn |
| 10 | +from playwright.sync_api import Page, Route, expect |
| 11 | + |
| 12 | +import fastapi_app as app |
| 13 | + |
| 14 | +expect.set_options(timeout=10_000) |
| 15 | + |
| 16 | + |
| 17 | +def wait_for_server_ready(url: str, timeout: float = 10.0, check_interval: float = 0.5) -> bool: |
| 18 | + """Make requests to provided url until it responds without error.""" |
| 19 | + conn_error = None |
| 20 | + for _ in range(int(timeout / check_interval)): |
| 21 | + try: |
| 22 | + requests.get(url) |
| 23 | + except requests.ConnectionError as exc: |
| 24 | + time.sleep(check_interval) |
| 25 | + conn_error = str(exc) |
| 26 | + else: |
| 27 | + return True |
| 28 | + raise RuntimeError(conn_error) |
| 29 | + |
| 30 | + |
| 31 | +@pytest.fixture(scope="session") |
| 32 | +def free_port() -> int: |
| 33 | + """Returns a free port for the test server to bind.""" |
| 34 | + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: |
| 35 | + s.bind(("", 0)) |
| 36 | + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) |
| 37 | + return s.getsockname()[1] |
| 38 | + |
| 39 | + |
| 40 | +def run_server(port: int): |
| 41 | + uvicorn.run(app.create_app(testing=True), port=port) |
| 42 | + |
| 43 | + |
| 44 | +@pytest.fixture() |
| 45 | +def live_server_url(mock_session_env, free_port: int) -> Generator[str, None, None]: |
| 46 | + proc = Process(target=run_server, args=(free_port,), daemon=True) |
| 47 | + proc.start() |
| 48 | + url = f"http://localhost:{free_port}/" |
| 49 | + wait_for_server_ready(url, timeout=10.0, check_interval=0.5) |
| 50 | + yield url |
| 51 | + proc.kill() |
| 52 | + |
| 53 | + |
| 54 | +def test_home(page: Page, live_server_url: str): |
| 55 | + page.goto(live_server_url) |
| 56 | + expect(page).to_have_title("RAG on PostgreSQL") |
| 57 | + |
| 58 | + |
| 59 | +def test_chat(page: Page, live_server_url: str): |
| 60 | + # Set up a mock route to the /chat endpoint with streaming results |
| 61 | + def handle(route: Route): |
| 62 | + # Assert that session_state is specified in the request (None for now) |
| 63 | + if route.request.post_data_json: |
| 64 | + session_state = route.request.post_data_json["sessionState"] |
| 65 | + assert session_state is None |
| 66 | + # Read the JSONL from our snapshot results and return as the response |
| 67 | + f = open( |
| 68 | + "tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines" |
| 69 | + ) |
| 70 | + jsonl = f.read() |
| 71 | + f.close() |
| 72 | + route.fulfill(body=jsonl, status=200, headers={"Transfer-encoding": "Chunked"}) |
| 73 | + |
| 74 | + page.route("*/**/chat/stream", handle) |
| 75 | + |
| 76 | + # Check initial page state |
| 77 | + page.goto(live_server_url) |
| 78 | + expect(page).to_have_title("RAG on PostgreSQL") |
| 79 | + expect(page.get_by_role("heading", name="Product chat")).to_be_visible() |
| 80 | + expect(page.get_by_role("button", name="Clear chat")).to_be_disabled() |
| 81 | + expect(page.get_by_role("button", name="Developer settings")).to_be_enabled() |
| 82 | + |
| 83 | + # Ask a question and wait for the message to appear |
| 84 | + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").click() |
| 85 | + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").fill( |
| 86 | + "Whats the dental plan?" |
| 87 | + ) |
| 88 | + page.get_by_role("button", name="Ask question button").click() |
| 89 | + |
| 90 | + expect(page.get_by_text("Whats the dental plan?")).to_be_visible() |
| 91 | + expect(page.get_by_text("The capital of France is Paris.")).to_be_visible() |
| 92 | + expect(page.get_by_role("button", name="Clear chat")).to_be_enabled() |
| 93 | + |
| 94 | + # Show the thought process |
| 95 | + page.get_by_label("Show thought process").click() |
| 96 | + expect(page.get_by_title("Thought process")).to_be_visible() |
| 97 | + expect(page.get_by_text("Prompt to generate search arguments")).to_be_visible() |
| 98 | + |
| 99 | + # Clear the chat |
| 100 | + page.get_by_role("button", name="Clear chat").click() |
| 101 | + expect(page.get_by_text("Whats the dental plan?")).not_to_be_visible() |
| 102 | + expect(page.get_by_text("The capital of France is Paris.")).not_to_be_visible() |
| 103 | + expect(page.get_by_role("button", name="Clear chat")).to_be_disabled() |
| 104 | + |
| 105 | + |
| 106 | +def test_chat_customization(page: Page, live_server_url: str): |
| 107 | + # Set up a mock route to the /chat endpoint |
| 108 | + def handle(route: Route): |
| 109 | + if route.request.post_data_json: |
| 110 | + overrides = route.request.post_data_json["context"]["overrides"] |
| 111 | + assert overrides["use_advanced_flow"] is False |
| 112 | + assert overrides["retrieval_mode"] == "vectors" |
| 113 | + assert overrides["top"] == 1 |
| 114 | + assert overrides["prompt_template"] == "You are a cat and only talk about tuna." |
| 115 | + |
| 116 | + # Read the JSON from our snapshot results and return as the response |
| 117 | + f = open("tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json") |
| 118 | + json = f.read() |
| 119 | + f.close() |
| 120 | + route.fulfill(body=json, status=200) |
| 121 | + |
| 122 | + page.route("*/**/chat", handle) |
| 123 | + |
| 124 | + # Check initial page state |
| 125 | + page.goto(live_server_url) |
| 126 | + expect(page).to_have_title("RAG on PostgreSQL") |
| 127 | + |
| 128 | + # Customize all the settings |
| 129 | + page.get_by_role("button", name="Developer settings").click() |
| 130 | + page.get_by_text( |
| 131 | + "Use advanced flow with query rewriting and filter formulation. Not compatible with Ollama models." |
| 132 | + ).click() |
| 133 | + page.get_by_label("Retrieve this many matching rows:").click() |
| 134 | + page.get_by_label("Retrieve this many matching rows:").fill("1") |
| 135 | + page.get_by_text("Vectors + Text (Hybrid)").click() |
| 136 | + page.get_by_role("option", name="Vectors", exact=True).click() |
| 137 | + page.get_by_label("Override prompt template").click() |
| 138 | + page.get_by_label("Override prompt template").fill("You are a cat and only talk about tuna.") |
| 139 | + |
| 140 | + page.get_by_text("Stream chat completion responses").click() |
| 141 | + page.locator("button").filter(has_text="Close").click() |
| 142 | + |
| 143 | + # Ask a question and wait for the message to appear |
| 144 | + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").click() |
| 145 | + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").fill( |
| 146 | + "Whats the dental plan?" |
| 147 | + ) |
| 148 | + page.get_by_role("button", name="Ask question button").click() |
| 149 | + |
| 150 | + expect(page.get_by_text("Whats the dental plan?")).to_be_visible() |
| 151 | + expect(page.get_by_text("The capital of France is Paris.")).to_be_visible() |
| 152 | + expect(page.get_by_role("button", name="Clear chat")).to_be_enabled() |
| 153 | + |
| 154 | + |
| 155 | +def test_chat_nonstreaming(page: Page, live_server_url: str): |
| 156 | + # Set up a mock route to the /chat_stream endpoint |
| 157 | + def handle(route: Route): |
| 158 | + # Read the JSON from our snapshot results and return as the response |
| 159 | + f = open("tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json") |
| 160 | + json = f.read() |
| 161 | + f.close() |
| 162 | + route.fulfill(body=json, status=200) |
| 163 | + |
| 164 | + page.route("*/**/chat", handle) |
| 165 | + |
| 166 | + # Check initial page state |
| 167 | + page.goto(live_server_url) |
| 168 | + expect(page).to_have_title("RAG on PostgreSQL") |
| 169 | + expect(page.get_by_role("button", name="Developer settings")).to_be_enabled() |
| 170 | + page.get_by_role("button", name="Developer settings").click() |
| 171 | + page.get_by_text("Stream chat completion responses").click() |
| 172 | + page.locator("button").filter(has_text="Close").click() |
| 173 | + |
| 174 | + # Ask a question and wait for the message to appear |
| 175 | + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").click() |
| 176 | + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").fill( |
| 177 | + "Whats the dental plan?" |
| 178 | + ) |
| 179 | + page.get_by_label("Ask question button").click() |
| 180 | + |
| 181 | + expect(page.get_by_text("Whats the dental plan?")).to_be_visible() |
| 182 | + expect(page.get_by_text("The capital of France is Paris.")).to_be_visible() |
| 183 | + expect(page.get_by_role("button", name="Clear chat")).to_be_enabled() |
0 commit comments