lm-evaluation-harness/pyproject.toml

[build-system]
requires = ["setuptools>=64.0"]
build-backend = "setuptools.build_meta"

[project]
name = "lm_eval"
version = "0.4.14.dev0"
authors = [
    { name = "EleutherAI", email = "[email protected]" }
]
description = "A framework for evaluating language models"
readme = "README.md"
classifiers = [
    "Development Status :: 3 - Alpha",
    "Programming Language :: Python :: 3",
    "Operating System :: OS Independent",
]
requires-python = ">=3.10"
license = { text = "MIT" }
dependencies = [
    "datasets>=2.16.0",
    "numpy",
    "evaluate>=0.4.0",
    "jinja2",
    "pytablewriter",
    "rouge-score>=0.0.4",
    "sacrebleu>=1.5.0",
    "scikit-learn>=0.24.1",
    "sqlitedict",
    "dill",
    "word2number",
    "more_itertools",
    "typing_extensions",
    "tqdm>=4.59.0"
]

[tool.setuptools.packages.find]
include = ["lm_eval*"]

# required to include yaml files in pip installation
[tool.setuptools.package-data]
lm_eval = ["**/*.yaml", "tasks/**/*"]

[project.scripts]
lm-eval = "lm_eval.__main__:cli_evaluate"
lm_eval = "lm_eval.__main__:cli_evaluate"

[project.urls]
Homepage = "https://github.com/EleutherAI/lm-evaluation-harness"
Repository = "https://github.com/EleutherAI/lm-evaluation-harness"

[project.optional-dependencies]
# Model backend dependencies
api = ["requests", "aiohttp", "tenacity", "tqdm", "tiktoken"]
archiver = ["jsonlines", "zstandard"]
hf = ["transformers>=4.1","torch>=1.8", "accelerate>=0.26.0", "peft>=0.2.0",]
vllm = ["vllm>=0.18"]
gptq = ["auto-gptq[triton]>=0.6.0"]
gptqmodel = ["gptqmodel>=1.0.9"]
ipex = ["optimum-intel"]
ibm_watsonx_ai = ["ibm_watsonx_ai>=1.1.22", "python-dotenv"]
# ONNX Runtime GenAI backend. For non-CPU execution providers install the
# matching wheel instead of `onnxruntime-genai`, e.g. `onnxruntime-genai-cuda`
# or `onnxruntime-genai-directml` (these are mutually exclusive).
onnxruntime-genai = ["onnxruntime-genai", "transformers", "numpy"]
# Raw ONNX Runtime backend. For non-CPU execution providers install the matching
# wheel instead of `onnxruntime`, e.g. `onnxruntime-gpu` (CUDA) or
# `onnxruntime-rocm` (ROCm) - these are mutually exclusive. The runtime must be
# new enough to load the contrib-op schemas your Model Builder version emits.
onnxruntime = ["onnxruntime>=1.23", "transformers", "numpy"]
litellm = ["litellm>=1.60,<1.85", "aiohttp", "requests", "tenacity", "tqdm"]
# mamba requires CUDA (nvcc) - cannot build on macOS/CPU-only systems
# mamba = ["mamba_ssm", "causal-conv1d==1.0.2", "torch"]
# megatron_lm requires separate installation of Megatron-LM
# megatron_lm = ["torch>=1.8"]
optimum = ["optimum[openvino]"]
habana = ["optimum-habana"]
sparsify = ["sparsify"]
#sae_lens = ["sae_lens"] #  sae-lens requires datasets < 3.0
# Task specific dependencies
acpbench = ["lark>=1.1.9", "tarski[clingo]==0.8.2", "pddl==0.4.2", "kstar-planner==1.4.2"]
audiolm_qwen = ["librosa", "soundfile"]
dev = ["pytest>=9.0", "pytest-cov", "pytest-xdist", "requests", "aiohttp", "tenacity", "tqdm", "tiktoken", "sentencepiece", "pillow"]
ifeval = ["langdetect", "immutabledict", "nltk>=3.9.1"]
japanese_leaderboard = ["emoji==2.14.0", "neologdn==0.5.3", "fugashi[unidic-lite]", "rouge_score>=0.1.2"]
longbench = ["jieba", "fuzzywuzzy", "rouge"]
libra = ["pymorphy2"]
math = ["sympy>=1.12", "antlr4-python3-runtime==4.11", "math_verify[antlr4_11_0]"]
multilingual = ["nagisa>=0.2.7", "jieba>=0.42.1", "pycountry"]
ruler = ["nltk", "wonderwords", "scipy"]
sentencepiece = ["sentencepiece>=0.1.98"]
discrim_eval = ["statsmodels==0.14.4"]
unitxt = ["unitxt==1.22.0"]
trackio = ["trackio>=0.23.0"]
wandb = ["wandb>=0.16.3", "pandas", "numpy"]
zeno = ["pandas", "zeno-client"]
tasks = [
    "lm_eval[discrim_eval]",
    "lm_eval[ifeval]",
    "lm_eval[japanese_leaderboard]",
    "lm_eval[longbench]",
    "lm_eval[libra]",
    "lm_eval[math]",
    "lm_eval[multilingual]",
    "lm_eval[ruler]",
]

[dependency-groups]
dev = [
  {include-group = "dev1"},
]
dev1 = [
    "lm_eval[api]", "lm_eval[dev]", "lm_eval[hf]","sentencepiece"
]

[tool.uv]
conflicts = [
    [
      { extra = "acpbench" },
      { extra = "math" },
    ],
    [
      { extra = "acpbench" },
      { extra = "tasks" },
    ],
    [
      { extra = "gptq" },
      { extra = "vllm" },
    ],
]

[tool.pymarkdown]
plugins.md013.enabled = false # line-length
plugins.md024.allow_different_nesting = true # no-duplicate-headers
plugins.md025.enabled = false # single-header
plugins.md028.enabled = false # no-blanks-blockquote
plugins.md029.allow_extended_start_values = true # ol-prefix
plugins.md034.enabled = false # no-bare-urls

[tool.ruff.lint]
preview = true
explicit-preview-rules = true
extend-select = [
    # bugbear, comprehension, docstyle, pycodestyle, pyflakes
    "B", "C419", "D", "E", "F",
    # refurb, isort, typecheck, useless-if-else
    "FURB", "I", "TC", "RUF034",
     # flake8-bandit, simplify, pyupgrade, invalid-escape-sequence
    "S", "SIM", "S307", "UP", "W605",
]
fixable = [
     # bugbear, pydocstyle, pycodestyle, unused-import
    "B", "D", "E", "F401",
    # refurb, unsorted-imports, ruff-specific
    "FURB", "I001", "RUF",
    # simplify, type-checking, pyupgrade, invalid-escape
    "SIM", "TC", "UP", "W605",
]
ignore = [
    "D10", "D200", "D202", "D205",
    "D212", "D415", "D417",
    "E11", "E402", "E501", "E701",
    "E731", "E741", "SIM108",
    "SIM105", "SIM401", "SIM118", # flake8-simplify
    "S311", "S701",  "S101" # bandits, assert-used
]

[tool.ruff.lint.extend-per-file-ignores]
"__init__.py" = [
    "F401",   # unused-import
    "F402",   # import-shadowed-by-loop-var
    "F403",   # undefined-local-with-import-star
    "F405",   # undefined-local-with-import-star-usage
]

[tool.ruff.lint.isort]
combine-as-imports = true
known-first-party = ["lm_eval"]
lines-after-imports = 2

[tool.ruff.lint.pydocstyle]
convention = "google"
ignore-var-parameters = true

[tool.pytest] # syntax supported from pytest>9.0
testpaths = ["tests"]