pyproject.toml

[build-system]
requires = ["setuptools>=42", "wheel"]
build-backend = "setuptools.build_meta"

[project]
name = "mteb"
version = "2.1.4"
description = "Massive Text Embedding Benchmark"
readme = "README.md"
authors = [
    { name = "MTEB Contributors", email = "niklas@huggingface.co" },
    { name = "Kenneth Enevoldsen", email = "kenneth.enevoldsen@cas.au.dk" },
    { name = "Nouamane Tazi", email = "nouamane@huggingface.co" },
    { name = "Nils Reimers", email = "info@nils-reimers.de" },
]
maintainers = [
    { name = "Kenneth Enevoldsen", email = "kenneth.enevoldsen@cas.au.dk" },
    { name = "Roman Solomatin", email = "risolomatin@gmail.com" },
    { name = "Isaac Chung", email = "chungisaac1217@gmail.com" },
]
license = "Apache-2.0"
license-files = ["LICENSE"]
keywords = ["deep learning", "text embeddings", "embeddings", "multimodal", "benchmark", "retrieval", "information retrieval"]
classifiers = [
    "Development Status :: 5 - Production/Stable",
    "Environment :: Console",
    "Intended Audience :: Developers",
    "Intended Audience :: Information Technology",
    "Operating System :: OS Independent",
    "Programming Language :: Python",
]
requires-python = ">=3.10,<3.14"
dependencies = [
    "datasets>=2.19.0",
    "numpy>=1.0.0,<3.0.0",
    "requests>=2.26.0",
    "scikit-learn>=1.4.0",
    "scipy>=0.0.0",
    "sentence_transformers>=3.0.0",
    "typing-extensions>=4.5.0",
    "torch>1.0.0",
    "tqdm>1.0.0",
    "rich>=0.0.0",
    "pytrec-eval-terrier>=0.5.6",
    "pydantic>=2.0.0",
    "polars>=0.20.22",
]


[project.urls]
Homepage = "https://github.com/embeddings-benchmark/mteb"
Documentation = "https://embeddings-benchmark.github.io/mteb/"
Repository = "https://github.com/embeddings-benchmark/mteb"
"Hugging Face Organization" = "https://huggingface.co/mteb"

[project.scripts]
mteb = "mteb.cli:main"

[project.optional-dependencies]
image = ["torchvision>0.2.1"]
codecarbon = ["codecarbon>=2.0.0,<3.0.0"]
leaderboard = [
    "gradio==5.35.0",
    "plotly>=5.24.0,<6.0.0",
    "cachetools>=5.2.0",
    "matplotlib>=3.9.4",
]

# model specific optional dependencies:
peft = ["peft>=0.11.0"]
flagembedding = ["FlagEmbedding==1.3.4"]
jina = ["einops>=0.8.0"]
jina-v4 = ["peft>=0.15.2", "transformers>=4.52.0", "torchvision>=0.22.1"]
flash_attention = ["flash-attn>=2.6.3"]
openai = ["openai>=1.41.0", "tiktoken>=0.8.0"]
model2vec = ["model2vec>=0.3.0"]
pylate = ["pylate>=1.3.1; python_version < '3.13'"]  # Required for sentence-transformers 5.0.0 compatibility (otherwise 1.1.6), pylate requires voyager, which in turn requires <=3.12
bm25s = ["bm25s>=0.2.6", "PyStemmer>=2.2.0.3"]
gritlm = ["gritlm>=1.0.2"]
xformers = ["xformers>=0.0.29"]
blip2 = ["salesforce-lavis>=1.0.2"]
voyageai = ["voyageai>0.3.0,<2.0.0"]
voyage_v = ["voyageai>0.3.0,<2.0.0", "tenacity>9.0.0"]
cohere = ["cohere==5.14.0"]
vertexai = ["vertexai==1.71.1"]
llm2vec = ["llm2vec>=0.2.3,<0.3.0"]
timm = ["timm>=1.0.15,<1.1.0"]
open_clip_torch = ["open_clip_torch==2.31.0"]
nomic = ["einops>=0.8.1"]
ark = ["volcengine-python-sdk[ark]==3.0.2", "tiktoken>=0.8.0"]
colpali_engine = ["colpali_engine>=0.3.12"]
xet = ["huggingface_hub>=0.32.0"]
youtu = ["tencentcloud-sdk-python-common>=3.0.1454", "tencentcloud-sdk-python-lkeap>=3.0.1451"]
llama-embed-nemotron = ["transformers==4.51.0"]
faiss-cpu = ["faiss-cpu>=1.12.0"]

[dependency-groups]
lint = [
    "ruff==0.14.0", # locked so we don't get PRs which fail only due to a lint update
    "pre-commit>=4.1.0",
    "bibtexparser>=1.4.3", # used for tests/test_citation_formatting.py
    "typos>1",
]
test = [
    "pytest>=8.3.4,<8.4.0",
    "pytest-xdist>=3.6.1,<3.7.0",
    "pytest-coverage>=0.0",
    "pytest-rerunfailures>=15.0,<16.0",
    "iso639>=0.1.4", # used for tests/scripts/test_generate_model_meta.py
    "GitPython>=3.0.0",
]
docs = [
    "mkdocs>=1.6.1",
    "mkdocs-material>=9.5.47",
    "mkdocstrings-python>=1.18.2",
    "mkdocs-bibtex>=2.16.2",
    "mkdocs-exclude>=1.0.2",
    "mkdocs-include-dir-to-nav>=1.2.0",
    "tabulate>=0.9.0",
]
typing = [
    "mypy>=1.18.1",
    "types-cachetools>=6.2.0.20250827",
    "types-pysocks>=1.7.1.20250828",
    "types-pyyaml>=6.0.12.20250822",
    "types-requests>=2.32.4.20250913",
    "types-simplejson>=3.20.0.20250822",
    "types-tqdm>=4.67.0.20250809",
    "types-tensorflow>=2.18.0.20250809",
    # stubs require python >=3.10
    # "pandas-stubs>=2.3.2.250827",
    # "scipy-stubs>=1.15.3.0",
]
dev = [
    {include-group = "lint"},
    {include-group = "test"},
    {include-group = "typing"},
]


[tool.coverage.report]
omit = ["tests/*", "mteb/tasks/**/*", "scripts"]

# Regexes for lines to exclude from consideration
exclude_also = [
    # Don't complain about missing debug-only code:
    "def __repr__",
    "if self\\.debug",

    # Don't complain if tests don't hit defensive assertion code:
    "raise AssertionError",
    "raise NotImplementedError",

    # Don't complain if non-runnable code isn't run:
    "if 0:",
    "if __name__ == .__main__.:",

    # Don't complain about abstract methods, they aren't run:
    "@(abc\\.)?abstractmethod",
]

[tool.setuptools.packages.find]
where = ["."]
include = ["mteb", "mteb.*"]
namespaces = false

[tool.setuptools.package-data]
"mteb" = [
    "languages/*.json",  # for languages
    "descriptive_stats/*/*.json",  # for descriptive_stats image
    "descriptive_stats/Image/*/*.json",  # for descriptive_stats image
]
"mteb.abstasks" = ["dataset_card_template.md"]
"mteb.tasks.Image.ZeroShotClassification.eng.templates" = ["*.txt"]

[tool.ruff]
target-version = "py310"


[tool.ruff.lint]
select = [
    "F",  # pyflakes rules,
    "I",  # sorting for imports
    "E",  # formatting for docs
    "D",  # formatting for docs
    "UP", # upgrade to latest syntax if possible
    "FA", # Future annotations
    "C4", # cleaner comprehensions
    "NPY", # numpy
    "N",       # PEP8 naming convention
    "RUF009",  # function-call-in-dataclass-default-argument
    "RUF008",  # mutable-dataclass-default
    "RUF013",  # implicit-optional
    "RUF012",  # mutable-class-default
    "RUF015",  # unnecessary-iterable-allocation-for-first-element
    "RUF016",  # invalid-index-type
    "RUF017",  # quadratic-list-summation
    "RUF018",  # assignment-in-assert
    "RUF019",  # unnecessary-key-check
    "RUF020",  # never-union
    "RUF021",  # parenthesize-chained-operators
    "RUF022",  # unsorted-dunder-all
    "RUF024",  # mutable-fromkeys-value
    "RUF026",  # default-factory-kwarg
    "RUF033",  # post-init-default
    "RUF034",  # useless-if-else
    "RUF041",  # unnecessary-nested-literal
    "RUF040",  # invalid-assert-message-literal-argument
    "RUF046",  # unnecessary-cast-to-int
    "RUF051",  # if-key-in-dict-del
    "RUF100",  # unused-noqa
    "RUF101",  # redirected-noqa
    "RUF200",  # invalid-pyproject-toml
    "PTH",     # use pathlib
    "TID",     # tidy-imports
    "D",       # pydocstyle
]

ignore = [
    "E501",   # line too long
    "E741",     # ambiguous variable name
    "D100",     # Missing docstring in public module
    "D105",     # Missing docstring in magic method
    "D104",     # Missing docstring in public package
    "D107",     # Missing docstring in __init__
    "D415",     # First line should end with a period
    "C408",     # don't use unecc. collection call, e.g. dict over {}
]

[tool.ruff.lint.per-file-ignores]
"scripts/*" = ["ALL"]
"docs/*" = ["D", "DOC"]
"tests/*" = ["RUF012", "D", "DOC"]
"mteb/tasks/*__init__.py" = ["F403"]  # undefined import `from .lang import *`
"mteb/tasks/*" = ["D", "DOC"]
"mteb/models/model_implementations/*" = ["D", "DOC"]

[tool.ruff.lint.flake8-implicit-str-concat]
allow-multiline = false

[tool.ruff.lint.pep8-naming]
ignore-names = [
    "F",  # torch.nn.functional
    "X_train",
    "X_test",
]

[tool.ruff.lint.pydocstyle]
convention = "google"

[tool.ruff.lint.flake8-annotations]
mypy-init-return = true
suppress-none-returning = true

[tool.semantic_release]
branch = "main"
version_toml = ["pyproject.toml:project.version"]
build_command = "python -m pip install build; python -m build"
tag_format = "{version}"

[tool.semantic_release.commit_parser_options]
minor_tags = ["minor", "feat"]
patch_tags = [
    "patch",
    "fix",
    "perf",
    "model",
    "dataset",
]

[tool.pytest.ini_options]
addopts = """
            --reruns 3
            --only-rerun requests.exceptions.ReadTimeout
            --only-rerun huggingface_hub.errors.HfHubHTTPError
            --only-rerun huggingface_hub.errors.LocalEntryNotFoundError
            --only-rerun FileNotFoundError
            --durations=5
            --reruns-delay 10
        """
# --reruns 3 -> # Retry failed tests 3 times
# requests.exceptions.ReadTimeout -> # HF Read timed out -> https://github.com/embeddings-benchmark/mteb/actions/runs/13275350693/job/37093688544
# huggingface_hub.errors.HfHubHTTPError -> # HF is unavailable, e.g. seen here: https://github.com/embeddings-benchmark/mteb/actions/runs/13275350693/job/37093688544
# huggingface_hub.errors.LocalEntryNotFoundError -> # Gateway Time-out from HF, e.g. seen here: https://github.com/embeddings-benchmark/mteb/actions/runs/13275350693/job/37093688544
# FileNotFoundError -> HF Cache is broken:  https://github.com/embeddings-benchmark/mteb/actions/runs/13302915091/job/37147507251?pr=2029
# --durations=5 -> Show the 5 slowest tests
# --reruns-delay 10 -> Delay between reruns in seconds to avoid running into the same issue again

markers = [
    "test_datasets: marks tests that tests dataset availability",
]

[tool.uv]
default-groups = ["test", "docs", "typing", "lint"]
override-dependencies = [
    "salesforce-lavis", # salesforce-lavis is not valid with sentence transformers >=3.0.0
]
conflicts = [
    [{ extra = "timm" }, { extra = "blip2" }],
    [{ extra = "llm2vec" }, { extra = "model2vec" }],
    [{ extra = "llm2vec" }, { extra = "pylate" }], # conflicting versions of transformers
    [{ extra = "llm2vec" }, { extra = "llama-embed-nemotron" }], # conflicting versions of transformers
    [{ extra = "colpali-engine" }, { extra = "pylate" }],
    [{ extra = "colpali-engine" }, { extra = "llm2vec" }],
    [{ extra = "colpali-engine" }, { extra = "llama-embed-nemotron" }], # conflicting version of transformers
    [{ extra = "jina-v4" }, { extra = "llm2vec" }],
    [{ extra = "jina-v4" }, { extra = "llama-embed-nemotron" }], # conflicting versions of transformers
]

[tool.uv.extra-build-dependencies]
flash-attn = [{ requirement = "torch", match-runtime = true }]

[tool.mypy]
plugins = ['pydantic.mypy']

[[tool.mypy.overrides]]
# these modules not typed and don't have stubs
module = [
    "datasets",
    "sklearn",
    "sklearn.*",
]
ignore_missing_imports = true

[[tool.mypy.overrides]]
# don't typecheck these modules (too many issues)
module = [
    "mteb.models.model_implementations.*",
    "mteb.tasks.*",
    "mteb.leaderboard.*",
]
ignore_errors = true

[[tool.mypy.overrides]]
# mypy can't resolve dataset dict
module = ["mteb.abstasks.*"]
disable_error_code = ["index"]

[tool.typos.default]
binary = false
locale = "en"
extend-ignore-words-re = [
    # authors
    "Stap",
    "Asai",
    "Tung",
    "Gool",
    "Winn",
    "STIL",
    "Sherif",
    "Mor",
    "Ond",
    "Mehta",
    "Parth",
    # languages/bibtext artifacts
    "ba",
    "creen",
    "Accademia",
]
extend-ignore-re = [
    "author.*",  # ignore authors in citations
]

[tool.typos.files]
extend-exclude = [
    "*.json",
    "*.jsonl",
    "*.cff",
    "*.txt",  # zero shot labels
    "docs/mmteb",
    "scripts/",
    "docs/references.bib",
    "mteb/models/model_implementations/gme_v_models.py",  # video_grid_thw `thw`
    "mteb/models/model_implementations/vista_models.py",  # self.normlized: in visual bge
    "tests/mock_tasks.py",  #  "denne her matche ikke den ovenstående",
    "mteb/models/model_implementations/kalm_models.py",  # prompt:  classify ist topic",
    "mteb/tasks/reranking/eng/built_bench_reranking.py",  # prompt:   descriptions from buit asset
    "mteb/tasks/retrieval/eng/built_bench_retrieval.py",
    "mteb/tasks/retrieval/eng/llava_it2t_retrieval.py",
]

[tool.typos.type.py]
extend-ignore-re = [
    # ignore lang identifiers
    "...-....",
    "..._....",
]

[tool.typos.type.py.extend-identifiers]
BulgarianStoreReviewSentimentClassfication = "BulgarianStoreReviewSentimentClassfication"

[tool.typos.type.py.extend-words]
wdth = "wdth"
Allo = "Allo"   # AlloProfClusteringS2S.v2
# langs/splits
som = "som"
yor = "yor"
Beng = "Beng"
als = "als"
tha = "tha"
mak = "mak"
Maka = "Maka"
Mape = "Mape"
Mian = "Mian"
Ono = "Ono"
Yau = "Yau"
Altas = "Altas"