mirror of
https://github.com/EleutherAI/lm-evaluation-harness.git
synced 2026-10-12 02:02:05 +00:00
Kiersten Stokes
7158f4f4ab
* Add markdown linter to pre-commit hooks * Reformat existing markdown (excluding lm_eval/tasks/*.md)
135 lines
3.8 KiB
TOML
135 lines
3.8 KiB
TOML
[build-system]
|
|
requires = ["setuptools>=40.8.0", "wheel"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[project]
|
|
name = "lm_eval"
|
|
version = "0.4.8"
|
|
authors = [
|
|
{name="EleutherAI", email="[email protected]"}
|
|
]
|
|
description = "A framework for evaluating language models"
|
|
readme = "README.md"
|
|
classifiers = [
|
|
"Development Status :: 3 - Alpha",
|
|
"Programming Language :: Python :: 3",
|
|
"License :: OSI Approved :: MIT License",
|
|
"Operating System :: OS Independent",
|
|
]
|
|
requires-python = ">=3.9"
|
|
license = { "text" = "MIT" }
|
|
dependencies = [
|
|
"accelerate>=0.26.0",
|
|
"evaluate",
|
|
"datasets>=2.16.0",
|
|
"evaluate>=0.4.0",
|
|
"jsonlines",
|
|
"numexpr",
|
|
"peft>=0.2.0",
|
|
"pybind11>=2.6.2",
|
|
"pytablewriter",
|
|
"rouge-score>=0.0.4",
|
|
"sacrebleu>=1.5.0",
|
|
"scikit-learn>=0.24.1",
|
|
"sqlitedict",
|
|
"torch>=1.8",
|
|
"tqdm-multiprocess",
|
|
"transformers>=4.1",
|
|
"zstandard",
|
|
"dill",
|
|
"word2number",
|
|
"more_itertools",
|
|
]
|
|
|
|
[tool.setuptools.packages.find]
|
|
include = ["lm_eval*"]
|
|
|
|
# required to include yaml files in pip installation
|
|
[tool.setuptools.package-data]
|
|
lm_eval = ["**/*.yaml", "tasks/**/*"]
|
|
|
|
[project.scripts]
|
|
lm-eval = "lm_eval.__main__:cli_evaluate"
|
|
lm_eval = "lm_eval.__main__:cli_evaluate"
|
|
|
|
[project.urls]
|
|
Homepage = "https://github.com/EleutherAI/lm-evaluation-harness"
|
|
Repository = "https://github.com/EleutherAI/lm-evaluation-harness"
|
|
|
|
[project.optional-dependencies]
|
|
api = ["requests", "aiohttp", "tenacity", "tqdm", "tiktoken"]
|
|
audiolm_qwen = ["librosa", "soundfile"]
|
|
deepsparse = ["deepsparse-nightly[llm]>=1.8.0.20240404"]
|
|
dev = ["pytest", "pytest-cov", "pytest-xdist", "pre-commit", "mypy", "unitxt"]
|
|
gptq = ["auto-gptq[triton]>=0.6.0"]
|
|
gptqmodel = ["gptqmodel>=1.0.9"]
|
|
hf_transfer = ["hf_transfer"]
|
|
ibm_watsonx_ai = ["ibm_watsonx_ai>=1.1.22", "python-dotenv"]
|
|
ifeval = ["langdetect", "immutabledict", "nltk>=3.9.1"]
|
|
ipex = ["optimum"]
|
|
japanese_leaderboard = ["emoji==2.14.0", "neologdn==0.5.3", "fugashi[unidic-lite]", "rouge_score>=0.1.2"]
|
|
longbench=["jeiba", "fuzzywuzzy", "rouge"]
|
|
mamba = ["mamba_ssm", "causal-conv1d==1.0.2"]
|
|
math = ["sympy>=1.12", "antlr4-python3-runtime==4.11", "math_verify[antlr4_11_0]"]
|
|
multilingual = ["nagisa>=0.2.7", "jieba>=0.42.1", "pycountry"]
|
|
neuronx = ["optimum[neuronx]"]
|
|
optimum = ["optimum[openvino]"]
|
|
promptsource = ["promptsource>=0.2.3"]
|
|
ruler = ["nltk", "wonderwords", "scipy"]
|
|
sae_lens = ["sae_lens"]
|
|
sentencepiece = ["sentencepiece>=0.1.98"]
|
|
sparseml = ["sparseml-nightly[llm]>=1.8.0.20240404"]
|
|
sparsify = ["sparsify"]
|
|
testing = ["pytest", "pytest-cov", "pytest-xdist"]
|
|
vllm = ["vllm>=0.4.2"]
|
|
wandb = ["wandb>=0.16.3", "pandas", "numpy"]
|
|
zeno = ["pandas", "zeno-client"]
|
|
all = [
|
|
"lm_eval[api]",
|
|
"lm_eval[audiolm_qwen]",
|
|
"lm_eval[deepsparse]",
|
|
"lm_eval[dev]",
|
|
"lm_eval[gptq]",
|
|
"lm_eval[gptqmodel]",
|
|
"lm_eval[hf_transfer]",
|
|
"lm_eval[ibm_watsonx_ai]",
|
|
"lm_eval[ifeval]",
|
|
"lm_eval[ipex]",
|
|
"lm_eval[japanese_leaderboard]",
|
|
"lm_eval[longbench]",
|
|
"lm_eval[mamba]",
|
|
"lm_eval[math]",
|
|
"lm_eval[multilingual]",
|
|
"lm_eval[neuronx]",
|
|
"lm_eval[optimum]",
|
|
"lm_eval[promptsource]",
|
|
"lm_eval[ruler]",
|
|
"lm_eval[sae_lens]",
|
|
"lm_eval[sentencepiece]",
|
|
"lm_eval[sparseml]",
|
|
"lm_eval[sparsify]",
|
|
"lm_eval[testing]",
|
|
"lm_eval[vllm]",
|
|
"lm_eval[wandb]",
|
|
"lm_eval[zeno]",
|
|
]
|
|
|
|
[tool.pymarkdown]
|
|
plugins.md013.enabled = false # line-length
|
|
plugins.md024.allow_different_nesting = true # no-duplicate-headers
|
|
plugins.md025.enabled = false # single-header
|
|
plugins.md028.enabled = false # no-blanks-blockquote
|
|
plugins.md029.allow_extended_start_values = true # ol-prefix
|
|
plugins.md034.enabled = false # no-bare-urls
|
|
|
|
[tool.ruff.lint]
|
|
extend-select = ["I"]
|
|
|
|
[tool.ruff.lint.isort]
|
|
lines-after-imports = 2
|
|
known-first-party = ["lm_eval"]
|
|
|
|
[tool.ruff.lint.extend-per-file-ignores]
|
|
"__init__.py" = ["F401","F402","F403"]
|
|
"utils.py" = ["F401"]
|