Updates

Author: Abby Hartman

scenarios/evaluate/Supported_Evaluation_Metrics/Document_Retrieval_Evaluation/Document_Retrieval_Evaluation.ipynb

@@ -266,7 +266,7 @@
     "    \"top\": search_top_k,\n",
     "    \"vector_queries\": [\n",
     "        {\n",
-    "            \"kind\": \"vector\",\n",
+    "            \"kind\": \"text\",\n",
     "            \"fields\": vector_field_name,\n",
     "            \"k_nearest_neighbors\": search_top_k\n",
     "        }\n",
@@ -283,12 +283,12 @@
     "    \"semantic_configuration_name\": \"en-semantic-config\",\n",
     "    \"vector_queries\": [\n",
     "        {\n",
-    "            \"kind\": \"vector\",\n",
+    "            \"kind\": \"text\",\n",
     "            \"fields\": vector_field_name,\n",
     "            \"k_nearest_neighbors\": search_top_k\n",
     "        }\n",
     "    ],\n",
-    "    \"score_field\": \"@search.reranker_score\"\n",
+    "    \"score_field\": \"@search.score\"\n",
     "}"
    ]
   },
@@ -613,7 +613,7 @@
     "        search_configuration.pop(\"vector_queries\")\n",
     "        \n",
     "    results = search_client.search(search_text=search_text, vector_queries=vector_queries, **search_configuration)\n",
-    "    return [{\"document_id\": result[\"doc_id\"], \"label\": result.get(score_field, None)} for result in results]\n",
+    "    return [{\"document_id\": result[\"doc_id\"], \"relevance_score\": result.get(score_field, None)} for result in results]\n",
     "\n",
     "def prepare_dataset(search_configuration):\n",
     "    # Load the queryset and qrels\n",
@@ -623,28 +623,28 @@
     "    # Drop negative qrels values and duplicates, and rename columns\n",
     "    qrels = qrels.loc[qrels[\"score\"] >= 0]\n",
     "    qrels.drop_duplicates(subset=[\"query-id\", \"corpus-id\"], inplace=True)\n",
-    "    qrels.rename(columns={\"corpus-id\": \"document_id\", \"score\": \"label\"}, inplace=True)\n",
+    "    qrels.rename(columns={\"corpus-id\": \"document_id\", \"score\": \"query_relevance_label\"}, inplace=True)\n",
     "    \n",
     "    # Group qrels by query ID and generate groundtruth set per query\n",
     "    qrels_grouped = qrels.groupby(\"query-id\")\n",
-    "    qrels_aggregated = qrels_grouped[[\"document_id\", \"label\"]].agg(lambda x: list(x))\n",
-    "    qrels_aggregated[\"groundtruth_documents_labels\"] = qrels_aggregated.apply(lambda x: json.dumps([{\"document_id\": doc_id, \"label\": label} for (doc_id, label) in zip(x[\"document_id\"], x[\"label\"])]), axis=1)\n",
+    "    qrels_aggregated = qrels_grouped[[\"document_id\", \"query_relevance_label\"]].agg(lambda x: list(x))\n",
+    "    qrels_aggregated[\"retrieval_ground_truth\"] = qrels_aggregated.apply(lambda x: json.dumps([{\"document_id\": doc_id, \"query_relevance_label\": label} for (doc_id, label) in zip(x[\"document_id\"], x[\"query_relevance_label\"])]), axis=1)\n",
     "    \n",
     "    # Join the queryset and qrels on query ID and doc ID\n",
     "    merged = queryset.merge(qrels_aggregated, left_on=\"_id\", right_on=\"query-id\")\n",
     "    \n",
     "    # Generate search results for each query\n",
     "    search_configuration_name = search_configuration.pop(\"name\")\n",
     "    score_field = search_configuration.pop(\"score_field\")\n",
-    "    merged[\"retrieved_documents_labels\"] = merged.apply(\n",
+    "    merged[\"retrieved_documents\"] = merged.apply(\n",
     "        lambda x: json.dumps(search(\n",
     "            query=x[\"text\"],\n",
     "            search_client=search_client,\n",
     "            score_field=score_field,\n",
     "            **search_configuration\n",
     "        )), axis=1)\n",
     "    \n",
-    "    merged_final = merged[[\"retrieved_documents_labels\", \"groundtruth_documents_labels\"]]\n",
+    "    merged_final = merged[[\"retrieved_documents\", \"retrieval_ground_truth\"]]\n",
     "    # Save final dataset to a local file in JSON-lines format\n",
     "    jsonl_path = os.path.join(\".\", f\"evaluate-beir-{search_configuration_name}.jsonl\")\n",
     "    merged_final.to_json(jsonl_path, lines=True, orient=\"records\")\n",
@@ -695,7 +695,7 @@
    "metadata": {},
    "source": [
     "## Run document retrieval evaluation\n",
-    "In the following cell, we will configure and run the document retrieval evaluator for our dataset.  The init params `groundtruth_min`, `groundtruth_max` and `groundtruth_step` help us to configure the qrels scaling for some metrics which depend on a count of labels, such as Fidelity.  In this case, the TREC-COVID dataset groundtruth set has 0, 1, and 2 as possible labels, so we set the values of those init params accordingly."
+    "In the following cell, we will configure and run the document retrieval evaluator for our dataset.  The init params `groundtruth_label_min` and `groundtruth_label_max` help us to configure the qrels scaling for some metrics which depend on a count of labels, such as Fidelity.  In this case, the TREC-COVID dataset groundtruth set has 0, 1, and 2 as possible labels, so we set the values of those init params accordingly."
    ]
   },
   {
@@ -715,13 +715,12 @@
     "            \"documentretrievalevaluator\": EvaluatorConfiguration(\n",
     "                id=DocumentRetrievalEvaluator().id,\n",
     "                data_mapping={\n",
-    "                    \"groundtruth_documents_labels\": \"${data.groundtruth_documents_labels}\",\n",
-    "                    \"retrieved_documents_labels\": \"${data.retrieved_documents_labels}\"\n",
+    "                    \"retrieval_ground_truth\": \"${data.retrieval_ground_truth}\",\n",
+    "                    \"retrieved_documents\": \"${data.retrieved_documents}\"\n",
     "                },\n",
     "                init_params={\n",
-    "                    \"groundtruth_min\": 0,\n",
-    "                    \"groundtruth_max\": 2,\n",
-    "                    \"groundtruth_step\": 1\n",
+    "                    \"groundtruth_label_min\": 0,\n",
+    "                    \"groundtruth_label_max\": 2\n",
     "                }\n",
     "            )\n",
     "        },\n",