diff --git a/docs/task_guide.md b/docs/task_guide.md index 8665ffa13..aa135dbd0 100644 --- a/docs/task_guide.md +++ b/docs/task_guide.md @@ -30,9 +30,10 @@ Dataset configuration options: Prompting / in-context formatting options: - **use_prompt** (`str`, *optional*) — Name of prompt in promptsource to use. if defined, will overwrite doc_to_text, doc_to_target, and doc_to_choice. -- **doc_to_text** (`Union[Callable, str]`, *optional*) — Jinja2, f-string, or function to process a sample into the appropriate input for the model -- **doc_to_target** (`Union[Callable, str]`, *optional*) — Jinja2, f-string, or function to process a sample into the appropriate target output for the model. For multiple choice tasks, this should return an index into -- **doc_to_choice** (`Union[Callable, str]`, *optional*) — Jinja2, f-string, or function to process a sample into a list of possible string choices for `multiple_choice` tasks. Left undefined for `generate_until` tasks. +- **description** (`str`, *optional*) — An optional prepended Jinja2 template or string which will be prepended to the few-shot examples passed into the model, often describing the task or providing instructions to a model, such as `"The following are questions (with answers) about {{subject}}.\n\n"`. No delimiters or spacing are inserted between the description and the first few-shot example. +- **doc_to_text** (`Union[Callable, str]`, *optional*) — Jinja2 template, string, or function to process a sample into the appropriate input for the model +- **doc_to_target** (`Union[Callable, str]`, *optional*) — Jinja2 template, string, or function to process a sample into the appropriate target output for the model. For multiple choice tasks, this should return an index into +- **doc_to_choice** (`Union[Callable, str]`, *optional*) — Jinja2 template, string, or function to process a sample into a list of possible string choices for `multiple_choice` tasks. Left undefined for `generate_until` tasks. - **fewshot_delimiter** (`str`, *optional*, defaults to "\n\n") — String to insert between few-shot examples. - **target_delimiter** (`str`, *optional*, defaults to `" "`) — String to insert between input and target output for the datapoint being tested. diff --git a/lm_eval/api/task.py b/lm_eval/api/task.py index fb32fede6..833ea6ce3 100644 --- a/lm_eval/api/task.py +++ b/lm_eval/api/task.py @@ -940,14 +940,14 @@ class ConfigurableTask(Task): :returns: str The fewshot context. """ + if description := self.config.description: + description = utils.apply_template(self.config.description, doc) if num_fewshot == 0: # always prepend the (possibly empty) task description - labeled_examples = self.config.description + labeled_examples = description else: - labeled_examples = self.config.description + self.sampler.get_context( - doc, num_fewshot - ) + labeled_examples = description + self.sampler.get_context(doc, num_fewshot) example = self.doc_to_text(doc) if self.multiple_input: