Ground truth generation and evaluation

Author: pamelafox

.devcontainer/devcontainer.json

@@ -46,10 +46,6 @@
 						"source.fixAll": "explicit"
 					},
 					"editor.defaultFormatter": "charliermarsh.ruff"
-				},
-				"files.exclude": {
-					".ruff_cache": true,
-					".pytest_cache": true
 				}
 			}
 		}

.vscode/settings.json

@@ -27,5 +27,13 @@
         "tests"
     ],
     "python.testing.unittestEnabled": false,
-    "python.testing.pytestEnabled": true
+    "python.testing.pytestEnabled": true,
+    "files.exclude": {
+        ".ruff_cache": true,
+        ".pytest_cache": true,
+        "__pycache__": true,
+        "htmlcov": true,
+        ".mypy_cache": true,
+        ".coverage": true
+    }
 }

evals/README.md

@@ -0,0 +1,2 @@
+pip install git+https://github.com/Azure-Samples/ai-rag-chat-evaluator/@installable
+pip install psycopg2

evals/generate_ground_truth.py

@@ -0,0 +1,95 @@
+import logging
+import os
+from collections.abc import Generator
+from pathlib import Path
+
+from azure.identity import DefaultAzureCredential
+from dotenv import load_dotenv
+from evaltools.gen.generate import generate_test_qa_data
+from sqlalchemy import create_engine, select
+from sqlalchemy.orm import Session
+
+from fastapi_app.postgres_models import Item
+
+logger = logging.getLogger("ragapp")
+
+
+def source_retriever() -> Generator[dict, None, None]:
+    # Connect to the database
+    DBHOST = os.environ["POSTGRES_HOST"]
+    DBUSER = os.environ["POSTGRES_USERNAME"]
+    DBPASS = os.environ["POSTGRES_PASSWORD"]
+    DBNAME = os.environ["POSTGRES_DATABASE"]
+    DATABASE_URI = f"postgresql://{DBUSER}:{DBPASS}@{DBHOST}/{DBNAME}"
+    engine = create_engine(DATABASE_URI, echo=False)
+    with Session(engine) as session:
+        # Fetch all products for a particular type
+        item_types = session.scalars(select(Item.type).distinct())
+        for item_type in item_types:
+            records = list(session.scalars(select(Item).filter(Item.type == item_type).order_by(Item.id)))
+            logger.info(f"Processing database records for type: {item_type}")
+            yield {
+                "citations": " ".join([f"[{record.id}] - {record.name}" for record in records]),
+                "content": "\n\n".join([record.to_str_for_rag() for record in records]),
+            }
+        # Fetch each item individually
+        # records = session.scalars(select(Item).order_by(Item.id))
+        # for record in records:
+        #    logger.info(f"Processing database record: {record.name}")
+        #    yield {"id": [record.id], "content": record.to_str_for_rag()}
+
+
+def source_to_text(source) -> str:
+    return source["content"]
+
+
+def answer_formatter(answer, source) -> str:
+    return f"{answer} {source['citations']}"
+
+
+def get_openai_config_dict() -> dict:
+    """Return a dictionary with OpenAI configuration based on environment variables."""
+    OPENAI_CHAT_HOST = os.getenv("OPENAI_CHAT_HOST")
+    if OPENAI_CHAT_HOST == "azure":
+        if api_key := os.getenv("AZURE_OPENAI_KEY"):
+            logger.info("Using Azure OpenAI Service with API Key from AZURE_OPENAI_KEY")
+            api_key = os.environ["AZURE_OPENAI_KEY"]
+        else:
+            logger.info("Using Azure OpenAI Service with Azure Developer CLI Credential")
+            azure_credential = DefaultAzureCredential()
+            api_key = azure_credential.get_token("https://cognitiveservices.azure.com/.default").token
+        openai_config = {
+            "api_type": "azure",
+            "api_base": os.environ["AZURE_OPENAI_ENDPOINT"],
+            "api_key": api_key,
+            "api_version": os.environ["AZURE_OPENAI_VERSION"],
+            "deployment": os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT"],
+            "model": os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT"],
+        }
+    elif OPENAI_CHAT_HOST == "ollama":
+        raise NotImplementedError("Ollama OpenAI Service is not supported. Switch to Azure or OpenAI.com")
+    else:
+        logger.info("Using OpenAI Service with API Key from OPENAICOM_KEY")
+        openai_config = {
+            "api_type": "openai",
+            "api_key": os.environ["OPENAICOM_KEY"],
+            "model": os.environ["OPENAICOM_CHAT_MODEL"],
+            "deployment": "none-needed-for-openaicom",
+        }
+    return openai_config
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.WARNING)
+    logger.setLevel(logging.INFO)
+    load_dotenv(".env", override=True)
+
+    generate_test_qa_data(
+        get_openai_config_dict(),
+        10,
+        5,
+        Path(__file__).parent / "ground_truth.json",
+        source_retriever,
+        source_to_text,
+        answer_formatter,
+    )

evals/ground_truth.json

@@ -0,0 +1,10 @@
+{"question": "What features does the Trailblazer Bike Helmet by Green Equipment offer?", "truth": "The Trailblazer Bike Helmet by Green Equipment offers a lightweight and durable design with an adjustable fit system and ventilation for added comfort. It comes in a vibrant green color and sleek design, suitable for all types of cycling.\n ['[10] - Trailblazer Bike Helmet', '[25] - Stealth Lite Bike Helmet', '[34] - Velocity Red Bike Helmet', '[46] - Zenith Cycling Jersey', '[58] - VelociX 2000 Bike Helmet', '[65] - Sprint PRO Carbon Cycling Helmet', '[76] - Fusion Carbon Cycling Helmet', '[86] - ProVent Bike Helmet', '[95] - AeroLite Cycling Helmet']"}
+{"question": "What are the key features of the Daybird Stealth Lite Bike Helmet?", "truth": "The Daybird Stealth Lite Bike Helmet features a sleek matte silver color, lightweight in-mold construction, adjustable retention system, and multiple ventilation channels for optimal airflow. It is designed for cyclists who value safety and style.\n ['[10] - Trailblazer Bike Helmet', '[25] - Stealth Lite Bike Helmet', '[34] - Velocity Red Bike Helmet', '[46] - Zenith Cycling Jersey', '[58] - VelociX 2000 Bike Helmet', '[65] - Sprint PRO Carbon Cycling Helmet', '[76] - Fusion Carbon Cycling Helmet', '[86] - ProVent Bike Helmet', '[95] - AeroLite Cycling Helmet']"}
+{"question": "What are the standout features of the Zenith Cycling Jersey by B&R?", "truth": "The Zenith Cycling Jersey by B&R is made from moisture-wicking fabric with a full-length zipper, three rear pockets for storage, and reflective accents for increased visibility in low-light conditions. It is designed for high-performance cycling.\n ['[10] - Trailblazer Bike Helmet', '[25] - Stealth Lite Bike Helmet', '[34] - Velocity Red Bike Helmet', '[46] - Zenith Cycling Jersey', '[58] - VelociX 2000 Bike Helmet', '[65] - Sprint PRO Carbon Cycling Helmet', '[76] - Fusion Carbon Cycling Helmet', '[86] - ProVent Bike Helmet', '[95] - AeroLite Cycling Helmet']"}
+{"question": "What are the notable features of the VelociX 2000 Bike Helmet by Raptor Elite?", "truth": "The VelociX 2000 Bike Helmet by Raptor Elite features an aerodynamic design, adjustable fit system, detachable visor, and 14 ventilation channels for optimal airflow. It is suitable for road cycling, mountain biking, and various other cycling activities.\n ['[10] - Trailblazer Bike Helmet', '[25] - Stealth Lite Bike Helmet', '[34] - Velocity Red Bike Helmet', '[46] - Zenith Cycling Jersey', '[58] - VelociX 2000 Bike Helmet', '[65] - Sprint PRO Carbon Cycling Helmet', '[76] - Fusion Carbon Cycling Helmet', '[86] - ProVent Bike Helmet', '[95] - AeroLite Cycling Helmet']"}
+{"question": "What features does the Solstix Sprint PRO Carbon Cycling Helmet offer?", "truth": "The Solstix Sprint PRO Carbon Cycling Helmet is crafted from carbon fiber for optimal impact protection and durability. It features an aerodynamic design, adjustable fit system, and ventilation channels for cooling during long rides. The rainbow color with the Solstix emblem adds a stylish touch to the helmet. ['[10] - Trailblazer Bike Helmet', '[25] - Stealth Lite Bike Helmet', '[34] - Velocity Red Bike Helmet', '[46] - Zenith Cycling Jersey', '[58] - VelociX 2000 Bike Helmet', '[65] - Sprint PRO Carbon Cycling Helmet', '[76] - Fusion Carbon Cycling Helmet', '[86] - ProVent Bike Helmet', '[95] - AeroLite Cycling Helmet']"}
+{"question": "What features do the Alpine Fusion Goggles offer for skiing?", "truth": "The Alpine Fusion Goggles from WildRunner offer full UV protection and anti-fog lenses to keep vision clear on the slopes. They have a stylish silver frame and orange lenses for a standout look, with adjustable straps for a secure fit and soft foam padding for all-day comfort. ['[3] - Alpine Fusion Goggles', '[5] - Blizzard Rider Snowboard', '[12] - Powder Pro Snowboard', '[16] - Expedition 2022 Goggles', '[22] - Venture 2022 Snowboard', '[30] - Zero Gravity Ski Goggles', '[32] - Cosmic Purple Snowboard', '[39] - Midnight Blue Goggles', '[42] - Gravity 5000 All-Mountain Skis', '[43] - Glacier Frost Snowboard', '[53] - Raven Swift Snowboard', '[60] - SummitRider Snowboard Boots', '[62] - Shadow Black Snowboard', '[72] - GravityZone All-Mountain Skis', '[73] - Omni-Snow Dual Snowboard', '[82] - Maverick Pro Ski Goggles', '[83] - Blizzard Freestyle Snowboard', '[92] - Blizzard Snowboard']"}
+{"question": "What is the price and brand of the Blizzard Rider Snowboard?", "truth": "The price of the Blizzard Rider Snowboard by B&R is 299.99. The brand of this versatile snowboard is B&R. ['[3] - Alpine Fusion Goggles', '[5] - Blizzard Rider Snowboard', '[12] - Powder Pro Snowboard', '[16] - Expedition 2022 Goggles', '[22] - Venture 2022 Snowboard', '[30] - Zero Gravity Ski Goggles', '[32] - Cosmic Purple Snowboard', '[39] - Midnight Blue Goggles', '[42] - Gravity 5000 All-Mountain Skis', '[43] - Glacier Frost Snowboard', '[53] - Raven Swift Snowboard', '[60] - SummitRider Snowboard Boots', '[62] - Shadow Black Snowboard', '[72] - GravityZone All-Mountain Skis', '[73] - Omni-Snow Dual Snowboard', '[82] - Maverick Pro Ski Goggles', '[83] - Blizzard Freestyle Snowboard', '[92] - Blizzard Snowboard']"}
+{"question": "What are the key features of the Powder Pro Snowboard by Zephyr?", "truth": "The Powder Pro Snowboard by Zephyr is designed for deep snow with floating camber for effortless turns and carbon fiber construction for control at high speeds. It comes in a vibrant turquoise color and is priced at 399.0. ['[3] - Alpine Fusion Goggles', '[5] - Blizzard Rider Snowboard', '[12] - Powder Pro Snowboard', '[16] - Expedition 2022 Goggles', '[22] - Venture 2022 Snowboard', '[30] - Zero Gravity Ski Goggles', '[32] - Cosmic Purple Snowboard', '[39] - Midnight Blue Goggles', '[42] - Gravity 5000 All-Mountain Skis', '[43] - Glacier Frost Snowboard', '[53] - Raven Swift Snowboard', '[60] - SummitRider Snowboard Boots', '[62] - Shadow Black Snowboard', '[72] - GravityZone All-Mountain Skis', '[73] - Omni-Snow Dual Snowboard', '[82] - Maverick Pro Ski Goggles', '[83] - Blizzard Freestyle Snowboard', '[92] - Blizzard Snowboard']"}
+{"question": "Describe the features of the Zero Gravity Ski Goggles by Gravitator.", "truth": "The Gravitator Zero Gravity Ski Goggles offer style, performance, and comfort with a sleek white frame, red mirrored lenses, anti-fog coating, and 100% UV protection. They have an adjustable strap for a secure fit and are priced at 79.99. ['[3] - Alpine Fusion Goggles', '[5] - Blizzard Rider Snowboard', '[12] - Powder Pro Snowboard', '[16] - Expedition 2022 Goggles', '[22] - Venture 2022 Snowboard', '[30] - Zero Gravity Ski Goggles', '[32] - Cosmic Purple Snowboard', '[39] - Midnight Blue Goggles', '[42] - Gravity 5000 All-Mountain Skis', '[43] - Glacier Frost Snowboard', '[53] - Raven Swift Snowboard', '[60] - SummitRider Snowboard Boots', '[62] - Shadow Black Snowboard', '[72] - GravityZone All-Mountain Skis', '[73] - Omni-Snow Dual Snowboard', '[82] - Maverick Pro Ski Goggles', '[83] - Blizzard Freestyle Snowboard', '[92] - Blizzard Snowboard']"}
+{"question": "What are the standout features of the Gravity 5000 All-Mountain Skis by Gravitator?", "truth": "The Gravity 5000 All-Mountain Skis by Gravitator excel in all conditions with a lightweight wood core, carbon inserts for responsiveness, and ABS sidewalls for durability. They come in a striking blue color and are priced at 699.0. ['[3] - Alpine Fusion Goggles', '[5] - Blizzard Rider Snowboard', '[12] - Powder Pro Snowboard', '[16] - Expedition 2022 Goggles', '[22] - Venture 2022 Snowboard', '[30] - Zero Gravity Ski Goggles', '[32] - Cosmic Purple Snowboard', '[39] - Midnight Blue Goggles', '[42] - Gravity 5000 All-Mountain Skis', '[43] - Glacier Frost Snowboard', '[53] - Raven Swift Snowboard', '[60] - SummitRider Snowboard Boots', '[62] - Shadow Black Snowboard', '[72] - GravityZone All-Mountain Skis', '[73] - Omni-Snow Dual Snowboard', '[82] - Maverick Pro Ski Goggles', '[83] - Blizzard Freestyle Snowboard', '[92] - Blizzard Snowboard']"}