clean: Minor cleanups (#200)

Author: KennethEnevoldsen

docs/create_cli_docs.py

@@ -2,6 +2,7 @@
 # intended to be used for anything other than a starting point.
 # at least we would need this issue fixed first:
 # https://github.com/explosion/radicli/issues/30
+from __future__ import annotations
 
 from pathlib import Path
 

docs/create_desc_stats.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from pathlib import Path
 
 import pandas as pd

docs/update_benchmark_tables.py

@@ -1,11 +1,11 @@
-"""
-Script for running the benchmark and pushing the results to Datawrapper.
+"""Script for running the benchmark and pushing the results to Datawrapper.
 
 Example:
-
     python update_benchmark_tables.py --data-wrapper-api-token <token>
 """
 
+from __future__ import annotations
+
 import argparse
 from collections import defaultdict
 from collections.abc import Sequence
@@ -69,7 +69,7 @@ def create_mdl_name_w_reference(mdl: seb.ModelMeta) -> str:
     return mdl_name
 
 
-def get_speed_results(model_meta: seb.ModelMeta) -> Optional[float]:
+def get_speed_results(model_meta: seb.ModelMeta) -> float | None:
     model = seb.get_model(model_meta.name)
     TOKENS_IN_UGLY_DUCKLING = 3591
 
@@ -218,9 +218,7 @@ def push_to_datawrapper(df: pd.DataFrame, chart_id: str, token: str):
 
 
 def compute_avg_rank(df: pd.DataFrame) -> pd.Series:
-    """
-    For each model in the dataset, for each task, compute the rank of the model and then compute the average rank.
-    """
+    """For each model in the dataset, for each task, compute the rank of the model and then compute the average rank."""
     df = df.drop(columns=["Average Score", "Open Source", "Embedding Size", "Model name", "WPS (CPU)"])
 
     ranks = df.rank(axis=0, ascending=False, na_option="bottom")
@@ -229,9 +227,7 @@ def compute_avg_rank(df: pd.DataFrame) -> pd.Series:
 
 
 def compute_avg_rank_bootstrap(df: pd.DataFrame, n_samples: int = 100) -> pd.Series:
-    """
-    For all models bootstrap a set of tasks and compute the average rank. Repeat this n_samples times.
-    """
+    """For all models bootstrap a set of tasks and compute the average rank. Repeat this n_samples times."""
     df = df.drop(columns=["Average Score", "Open Source", "Embedding Size", "Average Rank", "WPS (CPU)", "Model name"])
     tasks = np.array(df.columns.tolist())
     n_tasks = len(tasks)

pyproject.toml

@@ -8,6 +8,7 @@ version = "0.13.11"
 authors = [
   { name = "Kenneth Enevoldsen", email = "Kennethcenevoldsen@gmail.com" },
 ]
+license = { file = "LICENSE" }
 description = "Scandinavian Embedding Benchmark"
 classifiers = [
   "Operating System :: POSIX :: Linux",
@@ -29,11 +30,6 @@ dependencies = [
   "psutil>=7.0.0",
 ]
 
-
-[project.license]
-file = "LICENSE"
-name = "MIT"
-
 [project.optional-dependencies]
 mistral = [
   "transformers>=4.31.0", # lower bound required for mistral models (could potentially be lowered)
@@ -73,21 +69,25 @@ exclude = [".*venv*"]
 pythonPlatform = "Darwin"
 
 [tool.ruff]
-# extend-include = ["*.ipynb"]
 line-length = 150
+target-version = "py39"
 
-# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
-lint.select = [
+[tool.ruff.lint]
+select = [
+  "F",      # pyflakes rules,
+  "I",      # sorting for imports
+  "E",      # formatting for docs
+  "D",      # formatting for docs
+  "UP",     # upgrade to latest syntax if possible
+  "FA",     # Future annotations
+  "C4",     # cleaner comprehensions
   "A",
   "ANN",
   "ARG",
   "B",
-  "C4",
   "COM",
   "D417",
-  "E",
   "ERA",
-  "F",
   "I",
   "ICN",
   "NPY001",
@@ -105,7 +105,7 @@ lint.select = [
   "SIM",
   "W",
 ]
-lint.ignore = [
+ignore = [
   "ANN101",
   "ANN102",
   "ANN401",
@@ -116,36 +116,19 @@ lint.ignore = [
   "F841",
   "RET504",
   "COM812",
+  "D100",   # Missing docstring in public module
+  "D101",   # Missing docstring in public class
+  "D102",   # Missing docstring in public method
+  "D103",   # Missing docstring in public function
+  "D105",   # Missing docstring in magic method
+  "D104",   # Missing docstring in public package
+  "D107",   # Missing docstring in __init__
 ]
 # Allow autofix for all enabled rules (when `--fix`) is provided.
-lint.unfixable = ["ERA"]
-# Exclude a variety of commonly ignored directories.
-lint.exclude = [
-  ".bzr",
-  ".direnv",
-  ".eggs",
-  ".git",
-  ".hg",
-  ".nox",
-  ".pants.d",
-  ".pytype",
-  ".ruff_cache",
-  ".svn",
-  ".tox",
-  ".venv",
-  "__pypackages__",
-  "_build",
-  "buck-out",
-  "build",
-  "dist",
-  "node_modules",
-  "venv",
-  "__init__.py",
-  "docs/conf.py",
-]
+unfixable = ["ERA"]
+exclude = [".venv"]
 # Allow unused variables when underscore-prefixed.
-lint.dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
-target-version = "py39"
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
 
 [tool.ruff.lint.flake8-annotations]
 mypy-init-return = true
@@ -154,9 +137,8 @@ suppress-none-returning = true
 [tool.ruff.lint.pydocstyle]
 convention = "google"
 
-[tool.ruff.lint.mccabe]
-# Unlike Flake8, default to a complexity level of 10.
-max-complexity = 10
+[tool.ruff.lint.isort]
+required-imports = ["from __future__ import annotations"]
 
 [tool.semantic_release]
 branch = "main"
@@ -167,8 +149,8 @@ build_command = "python -m pip install build; python -m build"
 include-package-data = true
 
 [tool.uv]
-default-groups = ["dev", "tests", "docs"]
 conflicts = [[{ extra = "sonar" }, { extra = "arctic" }]]
+default-groups = ["dev", "tests", "docs"]
 no-build-isolation-package = ["xformers", "flash-attn"]
 
 [dependency-groups]

src/seb/cache/all-MiniLM-L6-v2/LCC.json

@@ -1 +1 @@
-{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.1","time_of_run":"2024-05-21T09:44:03.564974","scores":{"da":{"accuracy":0.3846666666666666,"f1":0.3650136884557438,"accuracy_stderr":0.03664241622309678,"f1_stderr":0.03540233062350939,"main_score":0.3846666666666666}},"main_score":"accuracy"}
+{"task_name":"LCC","task_description":"The leipzig corpora collection, annotated for sentiment","task_version":"1.1.1","time_of_run":"2025-05-23T16:02:19.308605","scores":{"da":{"accuracy":0.38533333333333325,"f1":0.3657168079255128,"accuracy_stderr":0.036490485822410684,"f1_stderr":0.03512881865293476,"main_score":0.38533333333333325}},"main_score":"accuracy"}

src/seb/interfaces/task.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from typing import Literal, Protocol, TypedDict, runtime_checkable
 
 import numpy as np
@@ -36,8 +38,7 @@ class DescriptiveDatasetStats(TypedDict):
 
 @runtime_checkable
 class Task(Protocol):
-    """
-    A task is a specific evaluation task for a sentence embedding model.
+    """A task is a specific evaluation task for a sentence embedding model.
 
     Attributes:
         name: The name of the task.
@@ -62,8 +63,7 @@ class Task(Protocol):
     description: str
 
     def evaluate(self, model: Encoder) -> TaskResult:
-        """
-        Evaluates a Sentence Embedding Model on the task.
+        """Evaluates a Sentence Embedding Model on the task.
 
         Args:
             model: A model with the encode method implemented.
@@ -74,8 +74,7 @@ def evaluate(self, model: Encoder) -> TaskResult:
         ...
 
     def get_documents(self) -> list[str]:
-        """
-        Get the documents for the task.
+        """Get the documents for the task.
 
         Returns:
             A list of strings.
@@ -95,8 +94,6 @@ def get_descriptive_stats(self) -> DescriptiveDatasetStats:
         )
 
     def name_to_path(self) -> str:
-        """
-        Convert a name to a path.
-        """
+        """Convert a name to a path."""
         name = self.name.replace("/", "__").replace(" ", "_")
         return name

src/seb/registered_models/cohere_models.py

@@ -27,7 +27,7 @@ def get_embedding_dim(self) -> int:
         return v.shape[1]
 
     def _embed(self, sentences: list[str], input_type: str) -> torch.Tensor:
-        import cohere
+        import cohere  # type: ignore[import]
 
         client = cohere.Client()
         response = client.embed(

src/seb/registered_models/llm2vec_models.py

@@ -1,20 +1,21 @@
 from __future__ import annotations
+
 import logging
+from collections.abc import Iterable, Sequence
 from datetime import date
 from functools import partial
-import torch
-from typing import Any, Optional, TypeVar, Union, List
-from collections.abc import Iterable, Sequence
-from tqdm import tqdm
 from itertools import islice
+from typing import Any, List, Optional, TypeVar, Union
+
 import numpy as np
+import torch
+from tqdm import tqdm
 
 import seb
-from seb.interfaces.model import LazyLoadEncoder, ModelMeta, SebModel, Encoder
+from seb.interfaces.model import Encoder, LazyLoadEncoder, ModelMeta, SebModel
 from seb.interfaces.task import Task
 from seb.registries import models
 
-
 logger = logging.getLogger(__name__)
 T = TypeVar("T")
 
@@ -95,12 +96,12 @@ def __init__(
     ):
         logger.info("Started loading LLM2Vec model")
         try:
-            from llm2vec import LLM2Vec
+            from llm2vec import LLM2Vec  # type: ignore[import]
         except ImportError:
             raise ImportError("To use the LLM2Vec models `llm2vec` is required. Please install it with `pip seb[llm2vec].")
         extra_kwargs = {}
         try:
-            import flash_attn  # noqa
+            import flash_attn  # type: ignore[import]
 
             extra_kwargs["attn_implementation"] = "flash_attention_2"
         except ImportError:
@@ -149,32 +150,6 @@ def encode(
         return torch.cat(batched_embeddings).numpy()
 
 
-@models.register("TTC-L2V-supervised-da-1")
-def create_llm2vec_da_mntp_ttc_supervised() -> SebModel:
-    base_model = "jealk/llm2vec-da-mntp"
-    peft_model = "jealk/TTC-L2V-supervised-1"
-    meta = ModelMeta(
-        name="TTC-L2V-supervised-da-1",
-        huggingface_name=peft_model,
-        reference=f"https://huggingface.co/{peft_model}",
-        languages=["da"],
-        open_source=True,
-        embedding_size=4096,
-        architecture="LLM2Vec",
-        release_date=date(2024, 12, 20),
-    )
-    partial_model = partial(
-        LLM2VecModel,
-        base_model_name_or_path=base_model,
-        peft_model_name_or_path=peft_model,
-        max_length=8192,
-    )
-    return SebModel(
-        encoder=LazyLoadEncoder(partial_model),
-        meta=meta,
-    )
-
-
 @models.register("TTC-L2V-unsupervised-da-1")
 def create_llm2vec_da_mntp_ttc_unsupervised() -> SebModel:
     base_model = "jealk/llm2vec-da-mntp"

tests/dummy_model.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import numpy as np
 
 import seb

tests/dummy_task.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from datetime import datetime
 from typing import Any
 
@@ -52,9 +54,7 @@ def evaluate(self, model: seb.Encoder) -> seb.TaskResult:
 
 
 def create_test_raise_error_task() -> seb.Task:
-    """
-    Note this task is not registered as it will cause errrors in other tests.
-    """
+    """Note this task is not registered as it will cause errrors in other tests."""
 
     class TestTaskWithError(TestTask):
         name = "test raise error task"