reranker_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install lancedb sentence-transformers openai PyPDF2 nltk transformers seaborn pandas matplotlib numpy renumics-spotlight\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lancedb\n",
    "from lancedb.pydantic import LanceModel, Vector\n",
    "from PyPDF2 import PdfReader\n",
    "from sentence_transformers import SentenceTransformer\n",
    "import nltk\n",
    "from nltk.tokenize import sent_tokenize\n",
    "\n",
    "PATH_TO_SOURCE_PDF = \"PUT YOUR PATH TO YOUR PDF HERE\"\n",
    "\n",
    "# Initialize SentenceTransformer model\n",
    "model = SentenceTransformer('BAAI/bge-m3').to(\"mps\")\n",
    "\n",
    "# Function to extract text from PDF\n",
    "def extract_text_from_pdf(pdf_path):\n",
    "    with open(pdf_path, 'rb') as file:\n",
    "        reader = PdfReader(file)\n",
    "        text = ''\n",
    "        for page in reader.pages:\n",
    "            text += page.extract_text()\n",
    "    return text\n",
    "\n",
    "# Function to split text into chunks of approximately 10 sentences\n",
    "def create_chunks(text, sentences_per_chunk=10):\n",
    "    sentences = sent_tokenize(text)\n",
    "    chunks = []\n",
    "    current_chunk = []\n",
    "    \n",
    "    for sentence in sentences:\n",
    "        current_chunk.append(sentence)\n",
    "        if len(current_chunk) >= sentences_per_chunk:\n",
    "            chunks.append(\" \".join(current_chunk))\n",
    "            current_chunk = []\n",
    "    \n",
    "    # Add any remaining sentences as the last chunk\n",
    "    if current_chunk:\n",
    "        chunks.append(\" \".join(current_chunk))\n",
    "    \n",
    "    return chunks\n",
    "\n",
    "# Function to create embeddings for chunks\n",
    "def create_chunk_embeddings(chunks):\n",
    "    embeddings = model.encode(chunks, show_progress_bar=True)\n",
    "    return list(zip(chunks, embeddings))\n",
    "\n",
    "# Set up LanceDB\n",
    "db = lancedb.connect(\"data/lancedb\")\n",
    "\n",
    "# Define the schema\n",
    "class ChunkEmbedding(LanceModel):\n",
    "    chunk: str\n",
    "    embedding: Vector(model.get_sentence_embedding_dimension())\n",
    "\n",
    "# Create or open the table\n",
    "table_name = \"masterarbeit_chunks\"\n",
    "table_created = False\n",
    "if table_name in db.table_names():\n",
    "    table = db.open_table(table_name)\n",
    "else:\n",
    "    table = db.create_table(table_name, schema=ChunkEmbedding)\n",
    "    table_created = True\n",
    "\n",
    "# Check if the table is empty\n",
    "if table_created:\n",
    "    # Process the PDF\n",
    "    text = extract_text_from_pdf(PATH_TO_SOURCE_PDF)\n",
    "    chunks = create_chunks(text)\n",
    "    chunk_embeddings = create_chunk_embeddings(chunks)\n",
    "\n",
    "    print(len(chunk_embeddings))\n",
    "\n",
    "    # Insert data into the table\n",
    "    data = [{\"chunk\": chunk, \"embedding\": emb.tolist()} for chunk, emb in chunk_embeddings]\n",
    "    table.add(data)\n",
    "else:\n",
    "    print(\"Chunk table already exists and is not empty. Skipping chunk creation.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Instantiate the reranker\n",
    "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
    "\n",
    "reranker_tokenizer = AutoTokenizer.from_pretrained('BAAI/bge-reranker-v2-m3')\n",
    "reranker_model = AutoModelForSequenceClassification.from_pretrained('BAAI/bge-reranker-v2-m3').to(\"mps\")\n",
    "reranker_model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "questions_with_answers = {\n",
    "    \"factoid\": [\n",
    "        {\n",
    "            \"question\": \"Example question 1 for factoid category?\",\n",
    "            \"answer\": \"Example answer 1 for factoid category.\"\n",
    "        },\n",
    "        {\n",
    "            \"question\": \"Example question 2 for factoid category?\",\n",
    "            \"answer\": \"Example answer 2 for factoid category.\"\n",
    "        }\n",
    "    ],\n",
    "    \"paraphrased_factoid\": [\n",
    "        {\n",
    "            \"question\": \"Example question 1 for paraphrased factoid category?\",\n",
    "            \"answer\": \"Example answer 1 for paraphrased factoid category.\"\n",
    "        },\n",
    "        {\n",
    "            \"question\": \"Example question 2 for paraphrased factoid category?\",\n",
    "            \"answer\": \"Example answer 2 for paraphrased factoid category.\"\n",
    "        }\n",
    "    ],\n",
    "    \"multi_source_question\": [\n",
    "        {\n",
    "            \"question\": \"Example question 1 for multi-source question category?\",\n",
    "            \"answer\": \"Example answer 1 for multi-source question category.\"\n",
    "        },\n",
    "        {\n",
    "            \"question\": \"Example question 2 for multi-source question category?\",\n",
    "            \"answer\": \"Example answer 2 for multi-source question category.\"\n",
    "        }\n",
    "    ],\n",
    "    \"summary_table_question\": [\n",
    "        {\n",
    "            \"question\": \"Example question 1 for summary table question category?\",\n",
    "            \"answer\": \"Example answer 1 for summary table question category.\"\n",
    "        },\n",
    "        {\n",
    "            \"question\": \"Example question 2 for summary table question category?\",\n",
    "            \"answer\": \"Example answer 2 for summary table question category.\"\n",
    "        }\n",
    "    ]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "import torch\n",
    "import numpy as np\n",
    "from typing import Literal\n",
    "from pydantic import BaseModel\n",
    "from openai import OpenAI\n",
    "from typing import List\n",
    "import seaborn as sns\n",
    "\n",
    "# Constants for configuration\n",
    "INITIAL_RESULTS = 50  # Number of initially retrieved results\n",
    "TOP_RESULTS = 10  # Number of top results to display\n",
    "PLOT_RESULTS = True  # Toggle for plotting\n",
    "\n",
    "# Initialize OpenAI client\n",
    "client = OpenAI()\n",
    "\n",
    "class ContextEvaluation(BaseModel):\n",
    "    # Information Content Changes\n",
    "    added_relevant_info: bool    \n",
    "    lost_relevant_info: bool     \n",
    "    improved_coverage: bool      \n",
    "    \n",
    "    # Ranking Changes\n",
    "    better_top_position: bool    \n",
    "    worse_top_position: bool     \n",
    "    \n",
    "    # Quality Changes\n",
    "    reduced_redundancy: bool     \n",
    "    more_noise: bool            \n",
    "    \n",
    "    explanation: str            \n",
    "\n",
    "def evaluate_contexts(query: str, answer: str, original_contexts: List[str], reranked_contexts: List[str]) -> ContextEvaluation:\n",
    "    prompt = f\"\"\"\n",
    "    Question: {query}\n",
    "\n",
    "    Ground Truth Answer: {answer}\n",
    "\n",
    "    Original Top 10 Results:\n",
    "    {'-' * 80}\n",
    "    {' '.join(original_contexts)}\n",
    "    \n",
    "    Reranked Top 10 Results:\n",
    "    {'-' * 80}\n",
    "    {' '.join(reranked_contexts)}\n",
    "\n",
    "    Compare these contexts and evaluate the following aspects (true/false):\n",
    "\n",
    "    1. Information Changes:\n",
    "    - added_relevant_info: Did new relevant information appear in the reranked context?\n",
    "    - lost_relevant_info: Was relevant information lost from the original context?\n",
    "    - improved_coverage: Does the reranked context cover more aspects of the answer?\n",
    "\n",
    "    2. Position Changes:\n",
    "    - better_top_position: Did the most relevant information move to earlier positions?\n",
    "    - worse_top_position: Did the most relevant information move to later positions?\n",
    "\n",
    "    3. Quality Changes:\n",
    "    - reduced_redundancy: Was repetitive information reduced?\n",
    "    - more_noise: Was more irrelevant information introduced?\n",
    "\n",
    "    Provide a brief explanation of your evaluation.\n",
    "\n",
    "    You must respond with valid JSON matching this schema:\n",
    "    {{\n",
    "        \"type\": \"object\",\n",
    "        \"properties\": {{\n",
    "            \"added_relevant_info\": {{\"type\": \"boolean\"}},\n",
    "            \"lost_relevant_info\": {{\"type\": \"boolean\"}},\n",
    "            \"improved_coverage\": {{\"type\": \"boolean\"}},\n",
    "            \"better_top_position\": {{\"type\": \"boolean\"}},\n",
    "            \"worse_top_position\": {{\"type\": \"boolean\"}},\n",
    "            \"reduced_redundancy\": {{\"type\": \"boolean\"}},\n",
    "            \"more_noise\": {{\"type\": \"boolean\"}},\n",
    "            \"explanation\": {{\"type\": \"string\"}}\n",
    "        }},\n",
    "        \"required\": [\"added_relevant_info\", \"lost_relevant_info\", \"improved_coverage\", \n",
    "                    \"better_top_position\", \"worse_top_position\", \"reduced_redundancy\", \n",
    "                    \"more_noise\", \"explanation\"]\n",
    "    }}\n",
    "    \"\"\"\n",
    "\n",
    "    response = client.beta.chat.completions.parse(\n",
    "        model=\"gpt-4o\",\n",
    "        response_format=ContextEvaluation,\n",
    "        messages=[\n",
    "            {\n",
    "                \"role\": \"system\", \n",
    "                \"content\": \"You are a helpful assistant that evaluates text relevance. Always provide responses in the specified JSON format.\"\n",
    "            },\n",
    "            {\"role\": \"user\", \"content\": prompt}\n",
    "        ]\n",
    "    )\n",
    "    \n",
    "    return response.choices[0].message.parsed\n",
    "\n",
    "def calculate_effectiveness_score(eval: ContextEvaluation) -> float:\n",
    "    return sum([\n",
    "        eval.added_relevant_info * 1.0,\n",
    "        -eval.lost_relevant_info * 1.0,\n",
    "        eval.improved_coverage * 0.8,\n",
    "        eval.better_top_position * 0.6,\n",
    "        -eval.worse_top_position * 0.6,\n",
    "        eval.reduced_redundancy * 0.4,\n",
    "        -eval.more_noise * 0.4\n",
    "    ])\n",
    "\n",
    "# Initialize a list to accumulate all question data\n",
    "all_questions_data = []\n",
    "\n",
    "def visualize_ranking(query, category):\n",
    "    print(f\"Category: {category}\")\n",
    "    print(f\"Question: {query}\")\n",
    "    print()\n",
    "\n",
    "    # Perform the query\n",
    "    query_embedding = model.encode([query])[0]\n",
    "    results = table.search(query_embedding).limit(INITIAL_RESULTS).to_pandas()\n",
    "\n",
    "    # Add original rank\n",
    "    results['original_rank'] = range(1, len(results) + 1)\n",
    "\n",
    "    # Prepare pairs for reranking\n",
    "    pairs = [[query, row['chunk']] for _, row in results.iterrows()]\n",
    "\n",
    "    # Rerank the results\n",
    "    with torch.no_grad():\n",
    "        inputs = reranker_tokenizer(pairs, padding=True, truncation=True, return_tensors='pt', max_length=512).to(\"mps\")\n",
    "        scores = reranker_model(**inputs, return_dict=True).logits.view(-1,).float()\n",
    "\n",
    "    # Add scores to the results DataFrame\n",
    "    results['rerank_score'] = scores.tolist()\n",
    "\n",
    "    # Sort results by rerank score and add new rank\n",
    "    reranked_results = results.sort_values('rerank_score', ascending=False).reset_index(drop=True)\n",
    "    reranked_results['reranked_rank'] = range(1, len(reranked_results) + 1)\n",
    "\n",
    "    # Get top contexts from both rankings\n",
    "    original_top = results.head(TOP_RESULTS)['chunk'].tolist()\n",
    "    reranked_top = reranked_results.head(TOP_RESULTS)['chunk'].tolist()\n",
    "    \n",
    "    # Find new contexts that were added to top results\n",
    "    added_contexts = [chunk for chunk in reranked_top if chunk not in original_top]\n",
    "    added_contexts_with_ranks = [\n",
    "        {\n",
    "            'chunk': chunk,\n",
    "            'new_rank': idx + 1,\n",
    "            'original_rank': results[results['chunk'] == chunk]['original_rank'].iloc[0]\n",
    "        }\n",
    "        for idx, chunk in enumerate(reranked_top) \n",
    "        if chunk not in original_top\n",
    "    ]\n",
    "\n",
    "    answer = next(item['answer'] for item in questions_with_answers[category] if item['question'] == query)\n",
    "    # Compare contexts using GPT-4\n",
    "    evaluation = evaluate_contexts(query, answer, original_top, reranked_top)\n",
    "    effectiveness_score = calculate_effectiveness_score(evaluation)\n",
    "\n",
    "    # Calculate standard metrics\n",
    "    rank_changes = reranked_results['original_rank'] - reranked_results['reranked_rank']\n",
    "    total_rank_change = np.sum(np.abs(rank_changes))\n",
    "    average_rank_change = np.mean(np.abs(rank_changes))\n",
    "    max_rank_change = np.max(np.abs(rank_changes))\n",
    "    \n",
    "    # Calculate neglected results metric\n",
    "    top_reranked = reranked_results[reranked_results['reranked_rank'] <= TOP_RESULTS]\n",
    "    neglected_results = np.sum(top_reranked['original_rank'] > TOP_RESULTS)\n",
    "\n",
    "    # Append data for the current question\n",
    "    question_data = {\n",
    "        \"category\": category,\n",
    "        \"question\": query,\n",
    "        \"total_rank_change\": total_rank_change,\n",
    "        \"average_rank_change\": average_rank_change,\n",
    "        \"max_rank_change\": max_rank_change,\n",
    "        \"num_results_changed\": np.sum(rank_changes != 0),\n",
    "        \"neglected_results\": neglected_results,\n",
    "        \"added_relevant_info\": evaluation.added_relevant_info,\n",
    "        \"lost_relevant_info\": evaluation.lost_relevant_info,\n",
    "        \"improved_coverage\": evaluation.improved_coverage,\n",
    "        \"better_top_position\": evaluation.better_top_position,\n",
    "        \"worse_top_position\": evaluation.worse_top_position,\n",
    "        \"reduced_redundancy\": evaluation.reduced_redundancy,\n",
    "        \"more_noise\": evaluation.more_noise,\n",
    "        \"effectiveness_score\": effectiveness_score,\n",
    "        \"explanation\": evaluation.explanation,\n",
    "        \"added_contexts\": added_contexts_with_ranks  # New field with detailed context information\n",
    "    }\n",
    "    all_questions_data.append(question_data)\n",
    "\n",
    "    if PLOT_RESULTS:\n",
    "        # After reranking but before printing results, add this visualization:\n",
    "        plt.figure(figsize=(8, 6))  # Reduced figure width\n",
    "        \n",
    "        # Get top N results after reranking\n",
    "        top_n = 10\n",
    "        top_reranked = reranked_results.head(top_n)\n",
    "        \n",
    "        # Create non-linear scale for ranks beyond top 10\n",
    "        def transform_rank(rank):\n",
    "            if rank <= 10:\n",
    "                return rank\n",
    "            else:\n",
    "                # Compress higher ranks logarithmically\n",
    "                return 10 + (rank - 10) / 4\n",
    "        \n",
    "        # Plot lines connecting original and new ranks with transformed scale\n",
    "        for _, row in top_reranked.iterrows():\n",
    "            orig_rank_transformed = transform_rank(row['original_rank'])\n",
    "            new_rank_transformed = transform_rank(row['reranked_rank'])\n",
    "            plt.plot([1, 2], [orig_rank_transformed, new_rank_transformed], '-o', \n",
    "                    alpha=0.5, linewidth=1.5)\n",
    "        \n",
    "        # Customize the plot\n",
    "        plt.xlim(0.8, 2.2)  # Reduced padding on sides\n",
    "        max_y = transform_rank(INITIAL_RESULTS)\n",
    "        plt.ylim(max_y + 1, -1)  # Reverse y-axis and set limit\n",
    "        \n",
    "        # Create custom y-ticks for better readability\n",
    "        y_ticks = list(range(1, 11)) + [20, 30, 40, 50]\n",
    "        y_ticks_transformed = [transform_rank(y) for y in y_ticks]\n",
    "        plt.yticks(y_ticks_transformed, y_ticks)\n",
    "        \n",
    "        plt.xticks([1, 2], ['Original Rank', 'Reranked'])\n",
    "        plt.ylabel('Rank')\n",
    "        plt.title(f'Rank Changes for Top {top_n} Results\\n{category}: {query}')\n",
    "        plt.grid(True, alpha=0.3)\n",
    "        plt.tight_layout()\n",
    "        plt.show()\n",
    "\n",
    "    # Print results\n",
    "    print(\"Context Evaluation:\")\n",
    "    print(f\"Effectiveness Score: {effectiveness_score:.2f}\")\n",
    "    print(\"\\nEvaluation Details:\")\n",
    "    for key, value in evaluation.dict().items():\n",
    "        if key != 'explanation':\n",
    "            print(f\"- {key}: {value}\")\n",
    "    print(f\"\\nExplanation: {evaluation.explanation}\")\n",
    "    print()\n",
    "\n",
    "    # Print information about added contexts\n",
    "    if added_contexts_with_ranks:\n",
    "        print(\"\\nNew Contexts Added to Top Results:\")\n",
    "        for ctx in added_contexts_with_ranks:\n",
    "            print(f\"- Moved from rank {ctx['original_rank']} to {ctx['new_rank']}:\")\n",
    "            print(f\"  {ctx['chunk'][:200]}...\")  # Print first 200 chars of each chunk\n",
    "    \n",
    "    print(\"\\nAggregate Information:\")\n",
    "    print(f\"Total absolute rank change: {total_rank_change}\")\n",
    "    print(f\"Average absolute rank change: {average_rank_change:.2f}\")\n",
    "    print(f\"Maximum rank change: {max_rank_change}\")\n",
    "    print(f\"Number of results that changed rank: {np.sum(rank_changes != 0)}\")\n",
    "    print(f\"Number of results moved into top {TOP_RESULTS}: {neglected_results}\")\n",
    "    print()\n",
    "\n",
    "# Iterate through all questions and generate visualizations\n",
    "for category, questions in questions_with_answers.items():\n",
    "    for item in questions:\n",
    "        visualize_ranking(item['question'], category)\n",
    "        print(\"\\n\" + \"=\"*80 + \"\\n\")  # Separator between questions\n",
    "\n",
    "# Create a DataFrame for all questions and save to a CSV file\n",
    "all_questions_df = pd.DataFrame(all_questions_data)\n",
    "all_questions_df.to_csv(\"all_questions_ranking_with_comparison.csv\", index=False)\n",
    "\n",
    "# Modified summary statistics\n",
    "summary_stats = all_questions_df.groupby('category').agg({\n",
    "    'neglected_results': ['mean', 'std'],\n",
    "    'average_rank_change': ['mean', 'std'],\n",
    "    'effectiveness_score': ['mean', 'std'],\n",
    "    'added_relevant_info': 'mean',\n",
    "    'lost_relevant_info': 'mean',\n",
    "    'improved_coverage': 'mean',\n",
    "    'better_top_position': 'mean',\n",
    "    'worse_top_position': 'mean',\n",
    "    'reduced_redundancy': 'mean',\n",
    "    'more_noise': 'mean'\n",
    "}).round(2)\n",
    "\n",
    "print(\"\\nSummary Statistics by Category:\")\n",
    "print(summary_stats)\n",
    "\n",
    "\n",
    "# Add visualization of evaluation metrics\n",
    "plt.figure(figsize=(15, 6))\n",
    "evaluation_metrics = ['added_relevant_info', 'lost_relevant_info', 'improved_coverage',\n",
    "                     'better_top_position', 'worse_top_position', 'reduced_redundancy', 'more_noise']\n",
    "metric_means = all_questions_df[evaluation_metrics].mean()\n",
    "\n",
    "sns.barplot(x=metric_means.index, y=metric_means.values)\n",
    "plt.xticks(rotation=45, ha='right')\n",
    "plt.title('Average Evaluation Metrics Across All Questions')\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "# Distribution of effectiveness scores by category\n",
    "plt.figure(figsize=(12, 6))\n",
    "sns.boxplot(data=all_questions_df, x='category', y='effectiveness_score')\n",
    "plt.xticks(rotation=45, ha='right')\n",
    "plt.title('Distribution of Effectiveness Scores by Category')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from renumics import spotlight\n",
    "\n",
    "spotlight.show(all_questions_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Create separate plots for each metric\n",
    "metrics = ['neglected_results', 'average_rank_change', 'total_rank_change']\n",
    "metric_titles = ['Neglected Results', 'Average Rank Change', 'Total Rank Change']\n",
    "\n",
    "for metric, title in zip(metrics, metric_titles):\n",
    "    plt.figure(figsize=(15, 5))\n",
    "    \n",
    "    # Get unique categories and determine axis limits\n",
    "    categories = all_questions_df['category'].unique()\n",
    "    num_categories = len(categories)\n",
    "    x_min = all_questions_df[metric].min()\n",
    "    x_max = all_questions_df[metric].max()\n",
    "    y_max = 0\n",
    "    \n",
    "    # First pass to determine maximum y value\n",
    "    for category in categories:\n",
    "        hist, _ = np.histogram(all_questions_df[all_questions_df['category'] == category][metric], bins=5)\n",
    "        y_max = max(y_max, hist.max())\n",
    "    \n",
    "    # Create subplots for each category\n",
    "    for idx, category in enumerate(categories, 1):\n",
    "        plt.subplot(1, num_categories, idx)\n",
    "        sns.histplot(data=all_questions_df[all_questions_df['category'] == category],\n",
    "                    x=metric,\n",
    "                    bins=5)\n",
    "        plt.title(f'{category}')\n",
    "        plt.xlabel(title)\n",
    "        plt.ylabel('Count')\n",
    "        \n",
    "        # Set consistent axis limits\n",
    "        plt.xlim(x_min, x_max)\n",
    "        plt.ylim(0, y_max + 1)  # Add 1 for some padding\n",
    "    \n",
    "    plt.suptitle(f'Distribution of {title} by Category', y=1.05)\n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "# Calculate and display summary statistics by category\n",
    "summary_stats = all_questions_df.groupby('category').agg({\n",
    "    'neglected_results': ['mean', 'std', 'min', 'max'],\n",
    "    'average_rank_change': ['mean', 'std', 'min', 'max'],\n",
    "    'total_rank_change': ['mean', 'std', 'min', 'max']\n",
    "}).round(2)\n",
    "\n",
    "print(\"\\nSummary Statistics by Category:\")\n",
    "print(summary_stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# Assuming all_questions_df is your DataFrame with columns:\n",
    "# 'category', 'average_rank_change', 'neglected_results'\n",
    "\n",
    "# Calculate means for each category and metric\n",
    "means = all_questions_df.groupby('category').agg({\n",
    "    'average_rank_change': 'mean',\n",
    "    'neglected_results': 'mean'\n",
    "}).reset_index()\n",
    "\n",
    "# Create figure\n",
    "plt.figure(figsize=(12, 6))\n",
    "\n",
    "# Set bar width and positions\n",
    "bar_width = 0.35\n",
    "x = np.arange(len(means['category']))\n",
    "\n",
    "# Create bars\n",
    "plt.bar(x - bar_width/2, means['average_rank_change'], bar_width, \n",
    "        label='Average Rank Change', color='skyblue')\n",
    "plt.bar(x + bar_width/2, means['neglected_results'], bar_width,\n",
    "        label='Neglected Results', color='lightcoral')\n",
    "\n",
    "# Customize plot\n",
    "plt.xlabel('Question Category')\n",
    "plt.ylabel('Value')\n",
    "plt.title('Average Rank Change and Neglected Results by Category')\n",
    "plt.xticks(x, means['category'], rotation=45, ha='right')\n",
    "plt.legend()\n",
    "\n",
    "# Add value labels on top of bars\n",
    "for i in x:\n",
    "    plt.text(i - bar_width/2, means['average_rank_change'].iloc[i], \n",
    "             f'{means[\"average_rank_change\"].iloc[i]:.1f}', \n",
    "             ha='center', va='bottom')\n",
    "    plt.text(i + bar_width/2, means['neglected_results'].iloc[i], \n",
    "             f'{means[\"neglected_results\"].iloc[i]:.1f}', \n",
    "             ha='center', va='bottom')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}