diff --git a/README.md b/README.md index d9d9251b0..76d494a86 100644 --- a/README.md +++ b/README.md @@ -177,6 +177,22 @@ lm_eval --model hf \ > [!Tip] > Ensure the tokenizer path points to a valid Hugging Face tokenizer directory (e.g., containing tokenizer_config.json, vocab.json, etc.). +#### Evaluating GGUF Models Served by llama.cpp + +The `gguf` model type evaluates models hosted by a [llama.cpp](https://github.com/ggml-org/llama.cpp) server (`llama-server`) through its OpenAI-compatible `/v1/completions` endpoint: + +```bash +lm_eval --model gguf \ + --model_args base_url=http://127.0.0.1:8080 \ + --tasks hellaswag +``` + +- Requires a llama.cpp release from December 2024 or newer, which returns logprobs in the modern OpenAI format (`logprobs.content`, see [llama.cpp#10783](https://github.com/ggml-org/llama.cpp/pull/10783)). The deprecated legacy format (`token_logprobs`) is not supported. +- If the server runs in router mode (multiple models), pass the model name or alias: `--model_args base_url=http://127.0.0.1:8080,model=my-model-alias`. +- Requests are issued concurrently. The default degree of parallelism is auto-detected from the server's slot count (`/props` → `total_slots`, i.e. llama-server's `--parallel` setting); override it with `parallel=`. `loglikelihood` requests are additionally pinned to slots via llama.cpp's `id_slot` parameter, with consecutive requests sharing a context (e.g. the candidate continuations of one multiple-choice question) assigned to the same slot so they reuse its cached prompt prefix. `generate_until` requests are left unpinned, since the server's dynamic idle-slot assignment load-balances variable-length generations better. +- `loglikelihood` is implemented via exact teacher forcing: llama.cpp ignores `echo` and never returns prompt logprobs, so the continuation is tokenized with the server's `/tokenize` endpoint (context and continuation encoded separately, matching the HF backend), and each continuation token is scored with its natural prefix as a token-id prompt plus a `logit_bias` that forces the server to sample it. llama.cpp reports the forced token's *pre-sampling* logprob and an unbiased `top_logprobs`, yielding loglikelihoods that match the HF backend to within quantization noise. (A naive alternative — forcing the continuation with a GBNF grammar — does *not* work: grammar-constrained decoding can pick a fragmented tokenization whose logprobs are not the natural-tokenization loglikelihoods, which systematically deflates scores.) +- `loglikelihood_rolling` (perplexity tasks such as wikitext) is not implemented for this model type. + #### Multi-GPU Evaluation with Hugging Face `accelerate` We support three main ways of using Hugging Face's [accelerate 🚀](https://github.com/huggingface/accelerate) library for multi-GPU evaluation. diff --git a/lm_eval/models/gguf.py b/lm_eval/models/gguf.py index 52aef0dee..ec33bde03 100644 --- a/lm_eval/models/gguf.py +++ b/lm_eval/models/gguf.py @@ -1,5 +1,7 @@ import logging +import threading import time +from concurrent.futures import ThreadPoolExecutor import requests from requests.exceptions import RequestException @@ -7,124 +9,297 @@ from tqdm import tqdm from lm_eval.api.model import LM from lm_eval.api.registry import register_model +from lm_eval.models.utils import normalize_gen_kwargs logger = logging.getLogger(__name__) - -def get_result(logprobs, context_length): - is_greedy = True - offsets = logprobs["text_offset"] - tokens = logprobs["tokens"] - tokens_logprobs = logprobs["token_logprobs"] - - idx = 0 - while offsets[idx] < context_length: - idx += 1 - continuation_logprobs = sum(tokens_logprobs[idx:-1]) - for i in range(idx, len(tokens)): - token = tokens[i] - top_tokens = logprobs["top_logprobs"][i] - top_token = max(top_tokens.keys(), key=lambda x: top_tokens[x]) - if top_token != token: - is_greedy = False - break - - return continuation_logprobs, is_greedy +# logit bias used to force the server to sample a specific token. llama.cpp +# still reports the token's *pre-sampling* logprob and an unbiased +# top_logprobs list, so this yields exact teacher-forced loglikelihoods. +_FORCE_TOKEN_BIAS = 100 @register_model("gguf", "ggml") class GGUFLM(LM): - def __init__(self, base_url=None, max_length=2048, **kwargs): - super().__init__() - self.base_url = base_url - assert self.base_url, "must pass `base_url` to use GGUF LM!" - self.logprobs = 10 - self.temperature = 0.0 - self.max_length = max_length + """Evaluate GGUF models served by a llama.cpp server (llama-server) through its OpenAI-compatible endpoints. - def gguf_completion( - self, context, continuation=None, stop=None, retries=3, delay=5, **kwargs + Requires a llama.cpp version that returns logprobs in the modern + OpenAI format (`logprobs.content`, see llama.cpp PR #10783; any release + from December 2024 onward). The deprecated legacy format + (`token_logprobs`/`text_offset`) is not supported. + + Pass `base_url` pointing at the server, e.g. + `--model gguf --model_args base_url=http://127.0.0.1:8080`. + When the server runs in router mode (multiple models), also pass + `model=` so requests are routed correctly. + + Requests are issued concurrently with a thread pool. By default the + degree of parallelism is auto-detected from the server's `/props` + endpoint (`total_slots`, i.e. llama-server's `--parallel` setting); + override it with `parallel=`. Note that llama-server only processes + as many requests simultaneously as it has slots, so values above the + server's slot count mostly add queueing on the server side. + """ + + def __init__( + self, + base_url=None, + model=None, + max_length=2048, + max_gen_toks=256, + timeout=300, + temperature=0.0, + parallel=None, + **kwargs, ): + super().__init__() + assert base_url, "must pass `base_url` to use GGUF LM!" + base_url = base_url.rstrip("/") + # derive the server root so /props and /tokenize can be queried + server_url = base_url + for suffix in ("/v1/completions", "/completions", "/v1"): + if server_url.endswith(suffix): + server_url = server_url[: -len(suffix)] + break + self.server_url = server_url + self.completions_url = server_url + "/v1/completions" + self.tokenize_url = server_url + "/tokenize" + self.model = model + self.temperature = temperature + self.max_length = max_length + self.max_gen_toks = int(max_gen_toks) + self.timeout = timeout + self.parallel = parallel + self._resolved_parallel = None + self._tok_cache = {} + self._tok_lock = threading.Lock() + + def _post_with_retries(self, url, payload, retries=3, delay=5): for _ in range(retries): try: - prompt = context - request = { - "prompt": prompt, - "logprobs": self.logprobs, - "temperature": self.temperature, - } - if continuation: - prompt += continuation - request.update({"prompt": prompt, "max_tokens": 1, "echo": True}) - if stop is not None: - request["stop"] = stop - response = requests.post( - f"{self.base_url}/v1/completions", json=request - ) + response = requests.post(url, json=payload, timeout=self.timeout) response.raise_for_status() return response.json() except RequestException as e: - logger.error(f"RequestException: {e}") + logger.error("RequestException: %s", e) time.sleep(delay) # wait before retrying - else: - raise RuntimeError( - f"Failed to get a valid response after {retries} retries." + raise RuntimeError(f"Failed to get a valid response after {retries} retries.") + + def _tokenize(self, text, add_special): + """Tokenize text with the server's own tokenizer (cached).""" + key = (text, add_special) + with self._tok_lock: + cached = self._tok_cache.get(key) + if cached is not None: + return cached + payload = {"content": text, "add_special": add_special} + if self.model is not None: + payload["model"] = self.model + result = self._post_with_retries(self.tokenize_url, payload) + ids = result["tokens"] + with self._tok_lock: + self._tok_cache[key] = ids + return ids + + def _detect_total_slots(self): + """Query the server's /props endpoint for its slot count (llama-server's `--parallel` setting). Returns None if unavailable.""" + try: + params = {"model": self.model} if self.model is not None else None + response = requests.get( + f"{self.server_url}/props", params=params, timeout=10 ) + response.raise_for_status() + total_slots = response.json().get("total_slots") + if isinstance(total_slots, int) and total_slots > 0: + return total_slots + except (RequestException, ValueError) as e: + logger.debug("Could not query /props for slot count: %s", e) + return None + + def _resolve_parallel(self): + if self._resolved_parallel is None: + if self.parallel is not None: + self._resolved_parallel = max(1, int(self.parallel)) + else: + total_slots = self._detect_total_slots() + self._resolved_parallel = total_slots or 1 + if total_slots: + logger.info( + "Auto-detected %s llama.cpp server slots; issuing up to %s concurrent requests. Override with `parallel=`.", + total_slots, + total_slots, + ) + return self._resolved_parallel + + def _map_requests(self, fn, items, disable_tqdm): + """Apply fn to each item, preserving order, using a thread pool when parallelism is enabled.""" + parallel = self._resolve_parallel() + if parallel <= 1 or len(items) <= 1: + return [fn(item) for item in tqdm(items, disable=disable_tqdm)] + with ThreadPoolExecutor(max_workers=parallel) as executor: + return list( + tqdm( + executor.map(fn, items), + total=len(items), + disable=disable_tqdm, + ) + ) + + def gguf_completion( + self, + context, + continuation=None, + stop=None, + max_tokens=None, + id_slot=None, + **kwargs, + ): + request = { + "prompt": context, + "temperature": self.temperature, + } + if self.model is not None: + request["model"] = self.model + if id_slot is not None: + # pin the request to a specific server slot so that prompts + # sharing a prefix hit the same per-slot KV cache + request["id_slot"] = id_slot + if max_tokens is not None: + request["max_tokens"] = max_tokens + if stop is not None: + request["stop"] = stop + return self._post_with_retries(self.completions_url, request) + + def _score_position(self, prompt_ids, target_id, id_slot=None): + """Score one continuation token given its natural preceding context. + + Sends the preceding tokens as a token-id array (exact control over + tokenization, matching the HF backend's encode-context-and- + continuation-separately convention) and forces the server to sample + `target_id` with a large logit bias. llama.cpp reports the sampled + token's *pre-sampling* logprob and an unbiased top_logprobs list, + so the returned values are the true teacher-forced logprob of the + target token and whether it was the argmax. + """ + request = { + "prompt": prompt_ids, + "temperature": 0, + "max_tokens": 1, + "logprobs": 2, + "logit_bias": [[target_id, _FORCE_TOKEN_BIAS]], + } + if self.model is not None: + request["model"] = self.model + if id_slot is not None: + request["id_slot"] = id_slot + response = self._post_with_retries(self.completions_url, request) + if not (response and response.get("choices")): + raise RuntimeError(f"Invalid scoring response: {response}") + content = response["choices"][0].get("logprobs", {}).get("content") or [] + if not content: + raise RuntimeError(f"Missing logprobs in scoring response: {response}") + entry = content[0] + if entry.get("id") != target_id: + raise RuntimeError( + f"Server did not sample the forced token id {target_id}: {entry}" + ) + top_logprobs = entry.get("top_logprobs") or [] + is_greedy = bool(top_logprobs) and ( + max(top_logprobs, key=lambda t: t["logprob"]).get("id") == target_id + ) + return entry["logprob"], is_greedy + + def _loglikelihood_one(self, item): + args, id_slot = item + context, continuation = args + # Migrate trailing whitespace from context to continuation (match HF backend) + n_spaces = len(context) - len(context.rstrip()) + if n_spaces > 0: + continuation = context[-n_spaces:] + continuation + context = context[:-n_spaces] + # Encode context + continuation together, then split at context boundary + # (BPE-correct: matches the HF backend's tok_encode approach) + whole_ids = self._tokenize(context + continuation, add_special=True) + context_ids = self._tokenize(context, add_special=True) + continuation_ids = whole_ids[len(context_ids) :] + if not continuation_ids: + return (0.0, True) + total = 0.0 + is_greedy = True + for j, target_id in enumerate(continuation_ids): + logprob, greedy = self._score_position( + context_ids + continuation_ids[:j], target_id, id_slot=id_slot + ) + total += logprob + is_greedy = is_greedy and greedy + return (total, is_greedy) + + @staticmethod + def _assign_slots_by_context(args_list, parallel): + """Assign a server slot id to each (context, continuation) pair. + + Consecutive requests sharing the same context (e.g. the candidate + continuations of one multiple-choice question) are pinned to the same + slot, so all but the first reuse the slot's cached prompt prefix. + Groups are round-robined across slots to keep them busy in parallel. + """ + slots = [] + group_idx = -1 + prev_context = None + for context, _ in args_list: + if context != prev_context: + group_idx += 1 + prev_context = context + slots.append(group_idx % parallel) + return slots def loglikelihood(self, requests, disable_tqdm: bool = False): if not requests: return [] - res = [] - for context, continuation in tqdm( - [req.args for req in requests], disable=disable_tqdm - ): - response = self.gguf_completion(context=context, continuation=continuation) - if response and "choices" in response and response["choices"]: - choice = response["choices"][0] - logprobs = choice.get("logprobs") - if ( - logprobs - and "token_logprobs" in logprobs - and logprobs["token_logprobs"] - ): - logprob, is_greedy = get_result(logprobs, len(context)) - res.append((logprob, is_greedy)) - else: - logger.warning( - "Invalid logprobs data. Expected 'logprobs' to contain 'token_logprobs' list." - ) + parallel = self._resolve_parallel() + args_list = [req.args for req in requests] + slots = self._assign_slots_by_context(args_list, parallel) + return self._map_requests( + self._loglikelihood_one, + list(zip(args_list, slots, strict=True)), + disable_tqdm, + ) + + def _generate_one(self, args): + inp, request_args = args + until = request_args.get("until", [""]) + max_gen_toks = normalize_gen_kwargs(request_args, self.max_gen_toks)[ + "max_gen_toks" + ] + # no id_slot pinning here: generation lengths vary widely, so the + # server's dynamic idle-slot assignment load-balances better than a + # static assignment (measured ~30% slower with pinning on gsm8k) + response = self.gguf_completion( + context=inp, stop=until, max_tokens=max_gen_toks + ) + if response and "choices" in response and response["choices"]: + choice = response["choices"][0] + if "text" in choice: + return choice["text"].strip() else: logger.error( - f"Invalid response for loglikelihood. Response: {response}" + "Invalid response for greedy_until. Response: %s", response ) - assert False - return res + return None # Add default value in case of error + else: + logger.error("Invalid response for greedy_until. Response: %s", response) + return None # Add default value in case of error def generate_until(self, requests, disable_tqdm: bool = False): if not requests: return [] - res = [] - for request in tqdm([req.args for req in requests], disable=disable_tqdm): - inp = request[0] - request_args = request[1] - until = request_args.get("until", [""]) - response = self.gguf_completion(context=inp, stop=until) - if response and "choices" in response and response["choices"]: - choice = response["choices"][0] - if "text" in choice: - generated_text = choice["text"].strip() - res.append(generated_text) - else: - logger.error( - f"Invalid response for greedy_until. Response: {response}" - ) - res.append(None) # Add default value in case of error - else: - logger.error(f"Invalid response for greedy_until. Response: {response}") - res.append(None) # Add default value in case of error - return res + return self._map_requests( + self._generate_one, + [req.args for req in requests], + disable_tqdm, + ) def loglikelihood_rolling(self, requests, disable_tqdm: bool = False): raise NotImplementedError( diff --git a/tests/models/test_gguf.py b/tests/models/test_gguf.py index b5e197e77..b83f2eec5 100644 --- a/tests/models/test_gguf.py +++ b/tests/models/test_gguf.py @@ -1,7 +1,3 @@ -import hashlib -import json -import os -import pickle import unittest from unittest.mock import patch @@ -9,143 +5,250 @@ from lm_eval.api.instance import Instance from lm_eval.models.gguf import GGUFLM -base_url = "https://matthoffner-ggml-llm-api.hf.space" +base_url = "http://fake-server:8080" -def gguf_completion_mock(base_url=None, **kwargs): - # Generate a hash from the parameters - hash_kwargs = {"base_url": base_url, **kwargs} - parameters_hash = hashlib.sha256( - json.dumps(hash_kwargs, sort_keys=True).encode("utf-8") - ).hexdigest() +class FakeResponse: + def __init__(self, data): + self._data = data - fname = f"./tests/testdata/gguf_test_{parameters_hash}.pkl" + def json(self): + return self._data - if os.path.exists(fname): - with open(fname, "rb") as fh: - return pickle.load(fh) - else: - print("The file does not exist, attempting to write...") - if "stop" in kwargs: - result = { - "choices": [ - { - "text": f"generated text until {kwargs['stop']}", - "logprobs": {"token_logprobs": [-1.2345], "text_offset": 0}, - "finish_reason": "length", - } - ] - } - else: - # generated with # curl -X 'POST' 'http://localhost:8000/v1/completions' -H 'accept: application/json' -H 'Content-Type: application/json' -d '{"prompt": "string", "logprobs": 10, "temperature": 0.0, "max_tokens": 1, "echo": true}' - result = { - "id": "cmpl-4023976b-bc6a-43b0-a5a9-629f4216c7f3", - "object": "text_completion", - "created": 1700511361, - "model": "../llama-2-7b.Q8_0.gguf", - "choices": [ - { - "text": "string(", - "index": 0, - "logprobs": { - "text_offset": [0, 7], - "token_logprobs": [None, -1.033263319857306], - "tokens": [" string", "("], - "top_logprobs": [ - None, - { - "(": -1.033263319857306, - "[]": -2.6530743779017394, - ".": -3.0377145947291324, - "\n": -3.0399156750513976, - "_": -3.510376089937872, - " =": -3.6957918347193663, - ",": -3.9309459866358702, - " of": -4.2834550083949035, - '("': -4.322762841112799, - "()": -4.426229113466925, - }, - ], - }, - "finish_reason": "length", - } - ], - "usage": { - "prompt_tokens": 2, - "completion_tokens": 1, - "total_tokens": 3, - }, - } + def raise_for_status(self): + pass - try: - os.makedirs(os.path.dirname(fname), exist_ok=True) - print("Writing file at", fname) - with open(fname, "wb") as fh: - pickle.dump(result, fh) - print("File written successfully") - except Exception as e: - print("File writing failed:", e) - return result +def make_fake_server(token_logprobs=None, top_token_id=None): + """Build a fake llama.cpp server. + + Tokenization is byte-level: each UTF-8 byte is a token id; a fake BOS + (id 1) is prepended when add_special is true. Scoring requests (prompt + given as a token-id array) return `token_logprobs[target_id]` as the + sampled logprob (default -1.0), and a top_logprobs list whose argmax is + `top_token_id` if given, else the target itself. + """ + calls = {"tokenize": [], "completions": []} + + def fake_post(url, json=None, timeout=None, **kwargs): + if url.endswith("/tokenize"): + calls["tokenize"].append(json) + ids = list(json["content"].encode("utf-8")) + if json.get("add_special"): + ids = [1] + ids + return FakeResponse({"tokens": ids}) + elif url.endswith("/completions"): + calls["completions"].append(json) + prompt = json["prompt"] + if isinstance(prompt, list): + # scoring request + target_id = json["logit_bias"][0][0] + lp = (token_logprobs or {}).get(target_id, -1.0) + top_id = top_token_id if top_token_id is not None else target_id + top_lp = lp + 1.0 if top_id != target_id else lp + entry = { + "id": target_id, + "token": "x", + "logprob": lp, + "top_logprobs": [ + {"id": top_id, "token": "y", "logprob": top_lp}, + {"id": target_id, "token": "x", "logprob": lp}, + ], + } + return FakeResponse( + {"choices": [{"text": "x", "logprobs": {"content": [entry]}}]} + ) + # generation request + stop = json.get("stop") + return FakeResponse({"choices": [{"text": f"generated text until {stop}"}]}) + raise AssertionError(f"unexpected url {url}") + + return fake_post, calls + + +def llm_instances(args_list, request_type="loglikelihood"): + return [ + Instance( + request_type=request_type, + doc=args, + arguments=args if request_type == "loglikelihood" else (args[0], args[1]), + idx=i, + ) + for i, args in enumerate(args_list) + ] class GGUFLMTest(unittest.TestCase): - @patch( - "lm_eval.models.gguf.GGUFLM.gguf_completion", side_effect=gguf_completion_mock - ) - def test_loglikelihood(self, gguf_completion_mock): - lm = GGUFLM(base_url) + def test_loglikelihood_scoring(self): + fake_post, calls = make_fake_server( + token_logprobs={97: -0.5, 98: -2.0} # 'a', 'b' + ) + with patch("lm_eval.models.gguf.requests.post", side_effect=fake_post): + lm = GGUFLM(base_url, parallel=1) + res = lm.loglikelihood(llm_instances([("x", "ab")])) + self.assertEqual(res, [(-2.5, True)]) - # Test loglikelihood - requests = [ - Instance( - request_type="loglikelihood", - doc=args, - arguments=args, - idx=i, - ) - for i, args in enumerate([("str", "ing"), ("str", "ing")]) + # context+continuation encoded together and split at the context + # boundary (matches the HF backend); context also encoded separately; + # both with special tokens (fake BOS prepended) + tok_calls = calls["tokenize"] + self.assertEqual( + {(c["content"], c["add_special"]) for c in tok_calls}, + {("x", True), ("xab", True)}, + ) + # one scoring request per continuation token, with correct prompts + # ([BOS, 'x'] ++ prefix) and logit bias on the target token + prompts = [c["prompt"] for c in calls["completions"]] + biases = [c["logit_bias"][0][0] for c in calls["completions"]] + self.assertEqual(prompts, [[1, ord("x")], [1, ord("x"), ord("a")]]) + self.assertEqual(biases, [ord("a"), ord("b")]) + + def test_loglikelihood_migrates_trailing_whitespace(self): + fake_post, calls = make_fake_server() + with patch("lm_eval.models.gguf.requests.post", side_effect=fake_post): + lm = GGUFLM(base_url, parallel=1) + res = lm.loglikelihood(llm_instances([("x ", "ab")])) + # trailing space moved from context into the continuation + self.assertEqual(res, [(-3.0, True)]) + biases = [c["logit_bias"][0][0] for c in calls["completions"]] + self.assertEqual(biases, [ord(" "), ord("a"), ord("b")]) + + def test_loglikelihood_empty_continuation(self): + fake_post, calls = make_fake_server() + with patch("lm_eval.models.gguf.requests.post", side_effect=fake_post): + lm = GGUFLM(base_url, parallel=1) + res = lm.loglikelihood(llm_instances([("x", ""), ("x", "a")])) + # empty continuation scores (0.0, True) with no scoring requests + self.assertEqual(res[0], (0.0, True)) + self.assertEqual(res[1], (-1.0, True)) + self.assertEqual(len(calls["completions"]), 1) + + def test_loglikelihood_non_greedy(self): + fake_post, _ = make_fake_server(top_token_id=999) + with patch("lm_eval.models.gguf.requests.post", side_effect=fake_post): + lm = GGUFLM(base_url, parallel=1) + res = lm.loglikelihood(llm_instances([("x", "ab")])) + self.assertEqual(res, [(-2.0, False)]) + + def test_loglikelihood_forced_token_mismatch_raises(self): + fake_post, _ = make_fake_server() + + def bad_post(url, json=None, timeout=None, **kwargs): + resp = fake_post(url, json, timeout) + if url.endswith("/completions") and isinstance(json["prompt"], list): + data = resp.json() + data["choices"][0]["logprobs"]["content"][0]["id"] = 424242 + return FakeResponse(data) + return resp + + with patch("lm_eval.models.gguf.requests.post", side_effect=bad_post): + lm = GGUFLM(base_url, parallel=1) + with self.assertRaises(RuntimeError): + lm.loglikelihood(llm_instances([("x", "a")])) + + def test_tokenize_cached(self): + fake_post, calls = make_fake_server() + with patch("lm_eval.models.gguf.requests.post", side_effect=fake_post): + lm = GGUFLM(base_url, parallel=1) + lm.loglikelihood(llm_instances([("x", "a"), ("x", "b")])) + ctx_calls = [c for c in calls["tokenize"] if c["content"] == "x"] + self.assertEqual(len(ctx_calls), 1) + + def test_generate_until(self): + fake_post, _ = make_fake_server() + with patch("lm_eval.models.gguf.requests.post", side_effect=fake_post): + lm = GGUFLM(base_url, parallel=1) + requests = [ + Instance( + request_type="generate_until", + doc={"input": doc}, + arguments=(doc, {"until": stop}), + idx=i, + ) + for i, (doc, stop) in enumerate( + [("input1", "stop1"), ("input2", "stop2")] + ) + ] + res = lm.generate_until(requests) + self.assertEqual( + res, ["generated text until stop1", "generated text until stop2"] + ) + + def test_generate_until_max_token_aliases(self): + fake_post, calls = make_fake_server() + with patch("lm_eval.models.gguf.requests.post", side_effect=fake_post): + lm = GGUFLM(base_url, parallel=1) + requests = [ + Instance( + request_type="generate_until", + doc={"input": doc}, + arguments=(doc, gen_kwargs), + idx=i, + ) + for i, (doc, gen_kwargs) in enumerate( + [ + ("a", {"max_tokens": "512"}), # CLI-style string alias + ("b", {"max_gen_toks": 8, "max_tokens": "512"}), + ("c", {}), + ] + ) + ] + lm.generate_until(requests) + max_tokens = [c.get("max_tokens") for c in calls["completions"]] + # alias resolved and int()-converted; max_gen_toks wins over aliases; + # backend default (256) when the task specifies nothing + self.assertEqual(max_tokens, [512, 8, 256]) + + def test_parallel_mapping_preserves_order(self): + lm = GGUFLM(base_url, parallel=3) + items = list(range(20)) + res = lm._map_requests(lambda x: x * 2, items, disable_tqdm=True) + self.assertEqual(res, [x * 2 for x in items]) + + def test_assign_slots_by_context(self): + # consecutive same-context requests share a slot; groups round-robin + args = [ + ("ctx1", "a"), + ("ctx1", "b"), + ("ctx2", "c"), + ("ctx3", "d"), + ("ctx2", "e"), ] - res = lm.loglikelihood(requests) + self.assertEqual(GGUFLM._assign_slots_by_context(args, 2), [0, 0, 1, 0, 1]) + # parallel=1 pins everything to slot 0 + self.assertEqual(GGUFLM._assign_slots_by_context(args, 1), [0, 0, 0, 0, 0]) - # Assert the loglikelihood response is correct - expected_res = [(logprob, True) for logprob in [0, 0]] - self.assertEqual(res, expected_res) - - @patch( - "lm_eval.models.gguf.GGUFLM.gguf_completion", side_effect=gguf_completion_mock - ) - def test_generate_until(self, gguf_completion_mock): + def test_detect_total_slots(self): lm = GGUFLM(base_url) + with patch("lm_eval.models.gguf.requests.get") as mock_get: + mock_get.return_value.json.return_value = {"total_slots": 4} + self.assertEqual(lm._detect_total_slots(), 4) + self.assertEqual(lm._resolve_parallel(), 4) - # Test generate_until - requests = [ - Instance( - request_type="generate_until", - doc={"input": doc}, - arguments=(doc, {"until": stop}), - idx=i, - ) - for i, (doc, stop) in enumerate([("input1", "stop1"), ("input2", "stop2")]) - ] + from requests.exceptions import RequestException - res = lm.generate_until(requests) + lm = GGUFLM(base_url) + with patch( + "lm_eval.models.gguf.requests.get", + side_effect=RequestException("no server"), + ): + self.assertIsNone(lm._detect_total_slots()) + # falls back to serial requests + self.assertEqual(lm._resolve_parallel(), 1) - # Assert the generate_until response is correct - expected_res = ["generated text until stop1", "generated text until stop2"] - self.assertEqual(res, expected_res) - - # @patch('lm_eval.models.gguf.GGUFLM.gguf_completion', side_effect=gguf_completion_mock) - # def test_loglikelihood_rolling(self, gguf_completion_mock): - # lm = GGUFLM(base_url) - - # # Test loglikelihood_rolling - # requests = ["input1", "input2"] - # res = lm.loglikelihood_rolling(requests) - - # # Assert the loglikelihood_rolling response is correct - # expected_res = [(-1.2345, True), (-1.2345, True)] - # self.assertEqual(res, expected_res) + def test_completions_url_derivation(self): + for base, expected in [ + ("http://localhost:8080", "http://localhost:8080/v1/completions"), + ("http://localhost:8080/", "http://localhost:8080/v1/completions"), + ("http://localhost:8080/v1", "http://localhost:8080/v1/completions"), + ( + "http://localhost:8080/v1/completions", + "http://localhost:8080/v1/completions", + ), + ]: + lm = GGUFLM(base, parallel=1) + self.assertEqual(lm.completions_url, expected) + self.assertEqual(lm.server_url, "http://localhost:8080") if __name__ == "__main__": diff --git a/tests/testdata/gguf_test_44e268d15decc4d2d0f99e57e1476269826cd3b54262f7a0981f75ddd45b25d0.pkl b/tests/testdata/gguf_test_44e268d15decc4d2d0f99e57e1476269826cd3b54262f7a0981f75ddd45b25d0.pkl deleted file mode 100644 index 55770de54..000000000 Binary files a/tests/testdata/gguf_test_44e268d15decc4d2d0f99e57e1476269826cd3b54262f7a0981f75ddd45b25d0.pkl and /dev/null differ diff --git a/tests/testdata/gguf_test_52ea409606de8755e03cf7c79f824101a4ce64bb6e6d3df556b8a4e7a5d92418.pkl b/tests/testdata/gguf_test_52ea409606de8755e03cf7c79f824101a4ce64bb6e6d3df556b8a4e7a5d92418.pkl deleted file mode 100644 index ee5803d9c..000000000 Binary files a/tests/testdata/gguf_test_52ea409606de8755e03cf7c79f824101a4ce64bb6e6d3df556b8a4e7a5d92418.pkl and /dev/null differ diff --git a/tests/testdata/gguf_test_8fcf3f2f52afeb2acd7c8e02c2cc3ce31a691b665d295f6c4e4bbd71c7caa1a2.pkl b/tests/testdata/gguf_test_8fcf3f2f52afeb2acd7c8e02c2cc3ce31a691b665d295f6c4e4bbd71c7caa1a2.pkl deleted file mode 100644 index 2284ed6ed..000000000 Binary files a/tests/testdata/gguf_test_8fcf3f2f52afeb2acd7c8e02c2cc3ce31a691b665d295f6c4e4bbd71c7caa1a2.pkl and /dev/null differ