Add support for FinMTEB benchmark

docs/adding_a_dataset.md

@@ -180,6 +180,7 @@ The domains follow the categories used in the [Universal Dependencies project](h
 | Religious     | Religious text e.g. bibles                                       |
 | Blog          | [Blogpost, weblog etc.](https://en.wikipedia.org/wiki/Blog)      |
 | Fiction       | Works of [fiction](https://en.wikipedia.org/wiki/Fiction)        |
+| Finance       | Financial documents, reports etc.                                |
 | Government    | Governmental communication, websites or similar                  |
 | Legal         | Legal documents, laws etc.                                       |
 | Medical       | doctors notes, medical procedures or similar                     |

mteb/abstasks/AbsTaskClassification.py

@@ -75,7 +75,7 @@ class AbsTaskClassification(AbsTask):
     n_experiments: int = 10
     k: int = 3
     train_split = "train"
-
+    sentence_column: str = "text"
     def evaluate(
         self,
         model: Encoder,
@@ -137,7 +137,7 @@ def _evaluate_subset(
             )
             # Bootstrap `self.samples_per_label` samples per label for each split
             X_sampled, y_sampled, idxs = self._undersample_data(
-                train_split["text"],  # type: ignore
+                train_split[self.sentence_column],  # type: ignore
                 train_split["label"],  # type: ignore
                 self.samples_per_label,
                 idxs,
@@ -146,14 +146,15 @@ def _evaluate_subset(
             evaluator = self.evaluator(
                 X_sampled,
                 y_sampled,
-                eval_split["text"],  # type: ignore
+                eval_split[self.sentence_column],  # type: ignore
                 eval_split["label"],  # type: ignore
                 task_name=self.metadata.name,
                 **params,
             )
             scores_exp, test_cache = evaluator(
                 model, encode_kwargs=encode_kwargs, test_cache=test_cache
             )
+
             scores.append(scores_exp)
 
         avg_scores: dict[str, Any] = {
@@ -183,20 +184,20 @@ def _calculate_metrics_from_split(
     ) -> ClassificationDescriptiveStatistics:
         train_text = []
         if hf_subset:
-            text = self.dataset[hf_subset][split]["text"]
+            text = self.dataset[hf_subset][split][self.sentence_column]
             label = self.dataset[hf_subset][split]["label"]
             if split != "train":
                 train_text = self.dataset[hf_subset]["train"]["text"]
         elif compute_overall:
             text = []
             label = []
             for hf_subset in self.metadata.eval_langs:
-                text.extend(self.dataset[hf_subset][split]["text"])
+                text.extend(self.dataset[hf_subset][split][self.sentence_column])
                 label.extend(self.dataset[hf_subset][split]["label"])
                 if split != "train":
                     train_text.extend(self.dataset[hf_subset]["train"]["text"])
         else:
-            text = self.dataset[split]["text"]
+            text = self.dataset[split][self.sentence_column]
             label = self.dataset[split]["label"]
             if split != "train":
                 train_text = self.dataset["train"]["text"]

mteb/abstasks/AbsTaskPairClassification.py

@@ -66,6 +66,8 @@ class AbsTaskPairClassification(AbsTask):
     """
 
     abstask_prompt = "Retrieve text that are semantically similar to the given text."
+    sentence_1_column: str = "sentence1"
+    sentence_2_column: str = "sentence2"
 
     def _evaluate_subset(
         self,
@@ -80,8 +82,8 @@ def _evaluate_subset(
             "sentence_transformers.evaluation.PairClassificationEvaluator"
         ).setLevel(logging.WARN)
         evaluator = PairClassificationEvaluator(
-            data_split["sentence1"],
-            data_split["sentence2"],
+            data_split[self.sentence_1_column],
+            data_split[self.sentence_2_column],
             data_split["labels"],
             task_name=self.metadata.name,
             **kwargs,
@@ -111,14 +113,14 @@ def _calculate_metrics_from_split(
             dataset = dataset[0]
 
         sentence1 = (
-            dataset["sentence1"][0]
-            if len(dataset["sentence1"]) == 1
-            else dataset["sentence1"]
+            dataset[self.sentence_1_column][0]
+            if len(dataset[self.sentence_1_column]) == 1
+            else dataset[self.sentence_1_column]
         )
         sentence2 = (
-            dataset["sentence2"][0]
-            if len(dataset["sentence2"]) == 1
-            else dataset["sentence2"]
+            dataset[self.sentence_2_column][0]
+            if len(dataset[self.sentence_2_column]) == 1
+            else dataset[self.sentence_2_column]
         )
         labels = (
             dataset["labels"][0] if len(dataset["labels"]) == 1 else dataset["labels"]

mteb/abstasks/AbsTaskSTS.py

@@ -61,6 +61,8 @@ class AbsTaskSTS(AbsTask):
     """
 
     abstask_prompt = "Retrieve semantically similar text."
+    sentence_1_column: str = "sentence1"
+    sentence_2_column: str = "sentence2"
     min_score: int
     max_score: int
 
@@ -72,8 +74,8 @@ def normalize(x):
 
         normalized_scores = list(map(normalize, data_split["score"]))
         evaluator = STSEvaluator(
-            data_split["sentence1"],
-            data_split["sentence2"],
+            data_split[self.sentence_1_column],
+            data_split[self.sentence_2_column],
             normalized_scores,
             task_name=self.metadata.name,
             **kwargs,
@@ -87,20 +89,20 @@ def _calculate_metrics_from_split(
         self, split: str, hf_subset: str | None = None, compute_overall: bool = False
     ) -> STSDescriptiveStatistics:
         if hf_subset:
-            sentence1 = self.dataset[hf_subset][split]["sentence1"]
-            sentence2 = self.dataset[hf_subset][split]["sentence2"]
+            sentence1 = self.dataset[hf_subset][split][self.sentence_1_column]
+            sentence2 = self.dataset[hf_subset][split][self.sentence_2_column]
             score = self.dataset[hf_subset][split]["score"]
         elif compute_overall:
             sentence1 = []
             sentence2 = []
             score = []
             for hf_subset in self.metadata.eval_langs:
-                sentence1.extend(self.dataset[hf_subset][split]["sentence1"])
-                sentence2.extend(self.dataset[hf_subset][split]["sentence2"])
+                sentence1.extend(self.dataset[hf_subset][split][self.sentence_1_column])
+                sentence2.extend(self.dataset[hf_subset][split][self.sentence_2_column])
                 score.extend(self.dataset[hf_subset][split]["score"])
         else:
-            sentence1 = self.dataset[split]["sentence1"]
-            sentence2 = self.dataset[split]["sentence2"]
+            sentence1 = self.dataset[split][self.sentence_1_column]
+            sentence2 = self.dataset[split][self.sentence_2_column]
             score = self.dataset[split]["score"]
 
         sentence1_len = [len(s) for s in sentence1]

mteb/abstasks/AbsTaskSummarization.py

@@ -82,6 +82,10 @@ class AbsTaskSummarization(AbsTask):
     abstask_prompt = (
         "Given a news summary, retrieve other semantically similar summaries."
     )
+
+    reference_summaries_column: str = "human_summaries"
+    generated_summaries_column: str = "machine_summaries"
+
     # SummEval has DeprecatedSummarizationEvaluator
     evaluator = SummarizationEvaluator
 
@@ -97,9 +101,10 @@ def _evaluate_subset(
             (np.array(x) - self.min_score) / (self.max_score - self.min_score)
             for x in data_split["relevance"]
         ]
+
         evaluator = self.evaluator(
-            machine_summaries=data_split["machine_summaries"],
-            human_summaries=data_split["human_summaries"],
+            machine_summaries=data_split[self.generated_summaries_column],
+            human_summaries=data_split[self.reference_summaries_column],
             texts=data_split["text"],
             gold_scores=normalized_scores,
             task_name=self.metadata.name,
@@ -114,8 +119,12 @@ def _calculate_metrics_from_split(
     ) -> SummarizationDescriptiveStatistics:
         if hf_subset:
             text = self.dataset[hf_subset][split]["text"]
-            human_summaries = self.dataset[hf_subset][split]["human_summaries"]
-            machine_summaries = self.dataset[hf_subset][split]["machine_summaries"]
+            human_summaries = self.dataset[hf_subset][split][
+                self.reference_summaries_column
+            ]
+            machine_summaries = self.dataset[hf_subset][split][
+                self.generated_summaries_column
+            ]
             relevance = self.dataset[hf_subset][split]["relevance"]
         elif compute_overall:
             text = []
@@ -126,16 +135,16 @@ def _calculate_metrics_from_split(
             for hf_subset in self.metadata.eval_langs:
                 text.extend(self.dataset[hf_subset][split]["text"])
                 human_summaries.extend(
-                    self.dataset[hf_subset][split]["human_summaries"]
+                    self.dataset[hf_subset][split][self.reference_summaries_column]
                 )
                 machine_summaries.extend(
-                    self.dataset[hf_subset][split]["machine_summaries"]
+                    self.dataset[hf_subset][split][self.generated_summaries_column]
                 )
                 relevance.extend(self.dataset[hf_subset][split]["relevance"])
         else:
             text = self.dataset[split]["text"]
-            human_summaries = self.dataset[split]["human_summaries"]
-            machine_summaries = self.dataset[split]["machine_summaries"]
+            human_summaries = self.dataset[split][self.reference_summaries_column]
+            machine_summaries = self.dataset[split][self.generated_summaries_column]
             relevance = self.dataset[split]["relevance"]
 
         all_human_summaries = []

mteb/abstasks/TaskMetadata.py

@@ -68,6 +68,7 @@
     "Constructed",
     "Encyclopaedic",
     "Fiction",
+    "Finance",
     "Government",
     "Legal",
     "Medical",
@@ -207,6 +208,7 @@
         "mpl-2.0",
         "msr-la-nc",
         "multiple",
+        "acm"
     ]
 )
 
@@ -274,7 +276,7 @@ class TaskMetadata(BaseModel):
             huggingface dataset contain different languages).
         main_score: The main score used for evaluation.
         date: The date when the data was collected. Specified as a tuple of two dates.
-        domains: The domains of the data. These includes "Non-fiction", "Social", "Fiction", "News", "Academic", "Blog", "Encyclopaedic",
+        domains: The domains of the data. These includes "Non-fiction", "Social", "Fiction", "Finance", "News", "Academic", "Blog", "Encyclopaedic",
             "Government", "Legal", "Medical", "Poetry", "Religious", "Reviews", "Web", "Spoken", "Written". A dataset can belong to multiple domains.
         task_subtypes: The subtypes of the task. E.g. includes "Sentiment/Hate speech", "Thematic Clustering". Feel free to update the list as needed.
         license: The license of the data specified as lowercase, e.g. "cc-by-nc-4.0". If the license is not specified, use "not specified". For custom licenses a URL is used.

mteb/benchmarks/benchmarks.py

@@ -14,7 +14,8 @@
 
 http_url_adapter = TypeAdapter(AnyUrl)
 UrlString = Annotated[
-    str, BeforeValidator(lambda value: str(http_url_adapter.validate_python(value)))
+    str,
+    BeforeValidator(lambda value: str(http_url_adapter.validate_python(value))),
 ]  # Allows the type to be a string, but ensures that the string is a URL
 
 
@@ -1471,3 +1472,86 @@ def load_results(
           url={https://arxiv.org/abs/2305.19840},
 }""",
 )
+
+FinMTEB = Benchmark(
+    name="MTEB(Finance)",
+    tasks=get_tasks(
+        languages=["eng", "zho"],
+        tasks=[
+            "FinancialPhrasebankClassification",
+            "FinSentClassification",
+            "FiQAClassification",
+            "SemEva2017Classification",
+            "FLSClassification",
+            "ESGClassification",
+            "FOMCClassification",
+            "FinancialFraudClassification",
+            "FinNSPClassification",
+            "FinChinaSentimentClassification",
+            "FinFEClassification",
+            "OpenFinDataSentimentClassification",
+            "Weibo21Classification",
+            "MInDS14EnClustering",
+            "ComplaintsClustering",
+            "PiiClustering",
+            "FinanceArxivS2SClustering",
+            "FinanceArxivP2PClustering",
+            "WikiCompany2IndustryClustering",
+            "MInDS14ZhClustering",
+            "FinNLClustering",
+            "CCKS2022Clustering",
+            "CCKS2020Clustering",
+            "CCKS2019Clustering",
+            "HeadlineACPairClassification",
+            "HeadlinePDDPairClassification",
+            "HeadlinePDUPairClassification",
+            "AFQMC",
+            "FinFactReranking",
+            "FiQA2018Reranking",
+            "HC3Reranking",
+            "FinEvaReranking",
+            "DISCFinLLMReranking",
+            "FiQA2018",
+            "FinanceBenchRetrieval",
+            "HC3Retrieval",
+            "Apple10KRetrieval",
+            "FinQARetrieval",
+            "TATQARetrieval",
+            "USNewsRetrieval",
+            "TradeTheEventEncyclopediaRetrieval",
+            "TradeTheEventNewsRetrieval",
+            "TheGoldmanEnRetrieval",
+            "FinTruthQARetrieval",
+            "FinEvaRetrieval",
+            "AlphaFinRetrieval",
+            "DISCFinLLMRetrieval",
+            "DISCFinLLMComputingRetrieval",
+            "DuEEFinRetrieval",
+            "SmoothNLPRetrieval",
+            "THUCNewsRetrieval",
+            "FinEvaEncyclopediaRetrieval",
+            "TheGoldmanZhRetrieval",
+            "FINAL",
+            "FinSTS",
+            "AFQMC",
+            "BQ",
+            "Ectsum",
+            "FINDsum",
+            "FNS2022sum",
+            "FiNNAsum",
+            "FinEvaHeadlinesum",
+            "FinEvasum",
+        ],
+    ),
+    description="FinMTEB is an embedding benchmark consists of 64 financial domain-specific text datasets, across English and Chinese.",
+    reference="https://arxiv.org/abs/2409.18511v1",
+    citation="""@misc{tang2024needdomainspecificembeddingmodels,
+          title={Do We Need Domain-Specific Embedding Models? An Empirical Investigation},
+          author={Yixuan Tang and Yi Yang},
+          year={2024},
+          eprint={2409.18511},
+          archivePrefix={arXiv},
+          primaryClass={cs.CL},
+          url={https://arxiv.org/abs/2409.18511},
+    }""",
+)

mteb/tasks/Classification/__init__.py

@@ -96,7 +96,13 @@
     Diversity5LegalBenchClassification,
     Diversity6LegalBenchClassification,
     EmotionClassification,
+    ESGClassification,
+    FinancialFraudClassification,
     FinancialPhrasebankClassification,
+    FinSentClassification,
+    FiQAClassification,
+    FLSClassification,
+    FOMCClassification,
     FrenkEnClassification,
     FunctionOfDecisionSectionLegalBenchClassification,
     ImdbClassification,
@@ -150,6 +156,7 @@
     SCDDVerificationLegalBenchClassification,
     SDSEyeProtectionClassification,
     SDSGlovesClassification,
+    SemEva2017Classification,
     TelemarketingSalesRuleLegalBenchClassification,
     TextualismToolDictionariesLegalBenchClassification,
     TextualismToolPlainLegalBenchClassification,
@@ -308,12 +315,17 @@
 from .urd import UrduRomanSentimentClassification
 from .vie import VieStudentFeedbackClassification
 from .zho import (
+    FinChinaSentimentClassification,
+    FinFEClassification,
+    FinNSPClassification,
     IFlyTek,
     JDReview,
     MultilingualSentiment,
     OnlineShopping,
+    OpenFinDataSentimentClassification,
     TNews,
     Waimai,
+    Weibo21Classification,
     YueOpenriceReviewClassification,
 )
 from .zul import IsiZuluNewsClassification
@@ -622,4 +634,16 @@
     "HindiDiscourseClassification",
     "SentimentAnalysisHindi",
     "MalayalamNewsClassification",
+    "ESGClassification",
+    "FLSClassification",
+    "FOMCClassification",
+    "FiQAClassification",
+    "FinSentClassification",
+    "FinancialFraudClassification",
+    "SemEva2017Classification",
+    "FinChinaSentimentClassification",
+    "FinFEClassification",
+    "FinNSPClassification",
+    "OpenFinDataSentimentClassification",
+    "Weibo21Classification",
 ]