Release 0.0.8 (#122)

See release notes

README.md

@@ -1,6 +1,4 @@
-🚧 Under Active Development 🚧
-
-# 🦜💪 LangChain Benchmarks
+# 🦜💯 LangChain Benchmarks
 
 [![Release Notes](https://img.shields.io/github/release/langchain-ai/langchain-benchmarks)](https://github.com/langchain-ai/langchain-benchmarks/releases)
 [![CI](https://github.com/langchain-ai/langchain-benchmarks/actions/workflows/ci.yml/badge.svg)](https://github.com/langchain-ai/langchain-benchmarks/actions/workflows/ci.yml)
@@ -35,7 +33,7 @@ pip install -U langchain-benchmarks
 All the benchmarks come with an associated benchmark dataset stored in [LangSmith](https://smith.langchain.com). To take advantage of the eval and debugging experience, [sign up](https://smith.langchain.com), and set your API key in your environment:
 
 ```bash
-export LANGCHAIN_API_KEY=sk-...
+export LANGCHAIN_API_KEY=ls-...
 ```
 
 ## Repo Structure

docs/source/.gitignore

@@ -1 +1,3 @@
 chromadb/
+index.md
+Untitled.ipynb

docs/source/conf.py

@@ -74,12 +74,20 @@ exclude_patterns: List[str] = []
 html_theme = "sphinx_book_theme"
 
 html_theme_options = {
-    "path_to_docs": "docs",
+    "path_to_docs": "docs/source",
     "repository_url": "https://github.com/langchain-ai/langchain-benchmarks",
     "home_page_in_toc": True,
     "show_navbar_depth": 2,
     "use_sidenotes": True,
     "use_repository_button": True,
+    "use_issues_button": True,
+    "use_source_button": True,
+    "use_fullscreen_button": True,
+    "repository_branch": "main",
+    "launch_buttons": {
+        "notebook_interface": "jupyterlab",
+        "colab_url": "https://colab.research.google.com",
+    },
 }
 
 html_context = {

docs/source/notebooks/datasets.ipynb

@@ -182,7 +182,7 @@
                 "import os\n",
                 "\n",
                 "# Get from https://smith.langchain.com/settings\n",
-                "os.environ[\"LANGCHAIN_API_KEY\"] = \"sk-...\""
+                "os.environ[\"LANGCHAIN_API_KEY\"] = \"ls_...\""
             ]
         },
         {

docs/source/notebooks/extraction/intro.ipynb

@@ -1,122 +1,130 @@
 {
-    "cells": [
-        {
-            "cell_type": "markdown",
-            "id": "7e8fc49a-e8b2-404b-a059-e9f668c460e5",
-            "metadata": {},
-            "source": [
-                "# Extraction Tasks\n",
-                "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain-benchmarks/blob/main/docs/source/notebooks/extraction/intro.ipynb)\n",
-                "\n",
-                "These tasks refer to an LLM's ability to extract structured output from an unstructured source, such as emails, websites, or other text. Below are a list of supported datasets.\n"
-            ]
-        },
-        {
-            "cell_type": "code",
-            "execution_count": 4,
-            "id": "86912590-a90a-4351-8ab4-89192cdee1e7",
-            "metadata": {},
-            "outputs": [
-                {
-                    "data": {
-                        "text/html": [
-                            "<table>\n",
-                            "<thead>\n",
-                            "<tr><th>Name            </th><th>Type          </th><th>Dataset ID                                                                                                                                                 </th><th>Description  </th></tr>\n",
-                            "</thead>\n",
-                            "<tbody>\n",
-                            "<tr><td>Email Extraction</td><td>ExtractionTask</td><td><a href=\"https://smith.langchain.com/public/36bdfe7d-3cd1-4b36-b957-d12d95810a2b/d\" target=\"_blank\" rel=\"noopener\">36bdfe7d-3cd1-4b36-b957-d12d95810a2b</a></td><td>A dataset of 42 real emails deduped from a spam folder, with semantic HTML tags removed, as well as a script for initial extraction and formatting of other emails from an arbitrary .mbox file like the one exported by Gmail.\n",
-                            "\n",
-                            "Some additional cleanup of the data was done by hand after the initial pass.\n",
-                            "\n",
-                            "See https://github.com/jacoblee93/oss-model-extraction-evals.              </td></tr>\n",
-                            "</tbody>\n",
-                            "</table>"
-                        ],
-                        "text/plain": [
-                            "Registry(tasks=[ExtractionTask(name='Email Extraction', dataset_id='https://smith.langchain.com/public/36bdfe7d-3cd1-4b36-b957-d12d95810a2b/d', description='A dataset of 42 real emails deduped from a spam folder, with semantic HTML tags removed, as well as a script for initial extraction and formatting of other emails from an arbitrary .mbox file like the one exported by Gmail.\\n\\nSome additional cleanup of the data was done by hand after the initial pass.\\n\\nSee https://github.com/jacoblee93/oss-model-extraction-evals.\\n    ', schema=<class 'langchain_benchmarks.extraction.tasks.email_task.Email'>, instructions=ChatPromptTemplate(input_variables=['email'], messages=[SystemMessagePromptTemplate(prompt=PromptTemplate(input_variables=[], template='You are an expert researcher.')), HumanMessagePromptTemplate(prompt=PromptTemplate(input_variables=['email'], template='What can you tell me about the following email? Make sure to extract the question in the correct format. Here is the email:\\n ```\\n{email}\\n```'))]))])"
-                        ]
-                    },
-                    "execution_count": 4,
-                    "metadata": {},
-                    "output_type": "execute_result"
-                }
-            ],
-            "source": [
-                "from langchain_benchmarks import registry\n",
-                "\n",
-                "registry.filter(Type=\"ExtractionTask\")"
-            ]
-        },
-        {
-            "cell_type": "markdown",
-            "id": "e771e544-b8f5-4359-8fd7-c89b71fbe460",
-            "metadata": {},
-            "source": [
-                "### Task resources\n",
-                "\n",
-                "In addition to the dataset_id, name, and description, each extraction task provides the following:\n",
-                "\n",
-                "- `schema` - a pydantic base model defining the schema (or schemas) the model should extract\n",
-                "\n",
-                "\n",
-                "### Dataset schema\n",
-                "\n",
-                "Each task corresponds to a LangSmith dataset with the following schema:\n",
-                "\n",
-                "Inputs:\n",
-                "- `input: str` - the input text\n",
-                "\n",
-                "Outputs\n",
-                "- `output: str` - the expected extraction result, as a json object\n"
-            ]
-        },
-        {
-            "cell_type": "markdown",
-            "id": "d04e05f3-3f20-4fed-bb98-3eb072213bbd",
-            "metadata": {},
-            "source": [
-                "### Evaluation\n",
-                "\n",
-                "The extraction tasks also have an evaluation config, which defines default LangSmith evaluators to apply when benchmarking your architecture.\n",
-                "\n"
-            ]
-        },
-        {
-            "cell_type": "code",
-            "execution_count": 5,
-            "id": "9c7865bd-8251-4579-85a3-f9085d96f497",
-            "metadata": {},
-            "outputs": [],
-            "source": [
-                "from langchain.chat_models import ChatOpenAI\n",
-                "\n",
-                "from langchain_benchmarks.extraction import get_eval_config\n",
-                "\n",
-                "eval_llm = ChatOpenAI(model=\"gpt-4\", model_kwargs={\"seed\": 42})\n",
-                "eval_config = get_eval_config(eval_llm)"
-            ]
-        }
-    ],
-    "metadata": {
-        "kernelspec": {
-            "display_name": "Python 3 (ipykernel)",
-            "language": "python",
-            "name": "python3"
-        },
-        "language_info": {
-            "codemirror_mode": {
-                "name": "ipython",
-                "version": 3
-            },
-            "file_extension": ".py",
-            "mimetype": "text/x-python",
-            "name": "python",
-            "nbconvert_exporter": "python",
-            "pygments_lexer": "ipython3",
-            "version": "3.11.2"
-        }
-    },
-    "nbformat": 4,
-    "nbformat_minor": 5
-}
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "7e8fc49a-e8b2-404b-a059-e9f668c460e5",
+   "metadata": {},
+   "source": [
+    "# Introduction\n",
+    "\n",
+    "\n",
+    "\n",
+    "These tasks refer to an LLM's ability to extract structured output from an unstructured source, such as emails, websites, or other text. Below are a list of supported datasets.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "86912590-a90a-4351-8ab4-89192cdee1e7",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table>\n",
+       "<thead>\n",
+       "<tr><th>Name            </th><th>Type          </th><th>Dataset ID                                                                                                                                                 </th><th>Description  </th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "<tr><td>Email Extraction</td><td>ExtractionTask</td><td><a href=\"https://smith.langchain.com/public/a1742786-bde5-4f51-a1d8-e148e5251ddb/d\" target=\"_blank\" rel=\"noopener\">a1742786-bde5-4f51-a1d8-e148e5251ddb</a></td><td>A dataset of 42 real emails deduped from a spam folder, with semantic HTML tags removed, as well as a script for initial extraction and formatting of other emails from an arbitrary .mbox file like the one exported by Gmail.\n",
+       "\n",
+       "Some additional cleanup of the data was done by hand after the initial pass.\n",
+       "\n",
+       "See https://github.com/jacoblee93/oss-model-extraction-evals.              </td></tr>\n",
+       "<tr><td>Chat Extraction </td><td>ExtractionTask</td><td><a href=\"https://smith.langchain.com/public/00f4444c-9460-4a82-b87a-f50096f1cfef/d\" target=\"_blank\" rel=\"noopener\">00f4444c-9460-4a82-b87a-f50096f1cfef</a></td><td>A dataset meant to test the ability of an LLM to extract and infer\n",
+       "structured information from a dialogue. The dialogue is between a user and a support\n",
+       "engineer. Outputs should be structured as a JSON object and test both the ability\n",
+       "of the LLM to correctly structure the information and its ability to perform simple \n",
+       "classification tasks.              </td></tr>\n",
+       "</tbody>\n",
+       "</table>"
+      ],
+      "text/plain": [
+       "Registry(tasks=[ExtractionTask(name='Email Extraction', dataset_id='https://smith.langchain.com/public/a1742786-bde5-4f51-a1d8-e148e5251ddb/d', description='A dataset of 42 real emails deduped from a spam folder, with semantic HTML tags removed, as well as a script for initial extraction and formatting of other emails from an arbitrary .mbox file like the one exported by Gmail.\\n\\nSome additional cleanup of the data was done by hand after the initial pass.\\n\\nSee https://github.com/jacoblee93/oss-model-extraction-evals.\\n    ', schema=<class 'langchain_benchmarks.extraction.tasks.email_task.Email'>, instructions=ChatPromptTemplate(input_variables=['input'], messages=[SystemMessagePromptTemplate(prompt=PromptTemplate(input_variables=[], template='You are an expert researcher.')), HumanMessagePromptTemplate(prompt=PromptTemplate(input_variables=['input'], template='What can you tell me about the following email? Make sure to extract the question in the correct format. Here is the email:\\n ```\\n{input}\\n```'))])), ExtractionTask(name='Chat Extraction', dataset_id='https://smith.langchain.com/public/00f4444c-9460-4a82-b87a-f50096f1cfef/d', description='A dataset meant to test the ability of an LLM to extract and infer\\nstructured information from a dialogue. The dialogue is between a user and a support\\nengineer. Outputs should be structured as a JSON object and test both the ability\\nof the LLM to correctly structure the information and its ability to perform simple \\nclassification tasks.', schema=<class 'langchain_benchmarks.extraction.tasks.chat_extraction.schema.GenerateTicket'>, instructions=ChatPromptTemplate(input_variables=['dialogue'], messages=[SystemMessagePromptTemplate(prompt=PromptTemplate(input_variables=[], template='You are a helpdesk assistant responsible with extracting information and generating tickets. Dialogues are between a user and a support engineer.')), HumanMessagePromptTemplate(prompt=PromptTemplate(input_variables=['dialogue'], template='Generate a ticket for the following question-response pair:\\n<Dialogue>\\n{dialogue}\\n</Dialogue>'))]))])"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from langchain_benchmarks import registry\n",
+    "\n",
+    "registry.filter(Type=\"ExtractionTask\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e771e544-b8f5-4359-8fd7-c89b71fbe460",
+   "metadata": {},
+   "source": [
+    "### Task resources\n",
+    "\n",
+    "In addition to the dataset_id, name, and description, each extraction task provides the following:\n",
+    "\n",
+    "- `schema` - a pydantic base model defining the schema (or schemas) the model should extract\n",
+    "\n",
+    "\n",
+    "### Dataset schema\n",
+    "\n",
+    "Each task corresponds to a LangSmith dataset with the following schema:\n",
+    "\n",
+    "Inputs:\n",
+    "- `input: str` - the input text\n",
+    "\n",
+    "Outputs\n",
+    "- `output: str` - the expected extraction result, as a json object\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d04e05f3-3f20-4fed-bb98-3eb072213bbd",
+   "metadata": {},
+   "source": [
+    "### Evaluation\n",
+    "\n",
+    "The extraction tasks also have an evaluation config, which defines default LangSmith evaluators to apply when benchmarking your architecture.\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "9c7865bd-8251-4579-85a3-f9085d96f497",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.chat_models import ChatOpenAI\n",
+    "\n",
+    "from langchain_benchmarks.extraction import get_eval_config\n",
+    "\n",
+    "eval_llm = ChatOpenAI(model=\"gpt-4\", model_kwargs={\"seed\": 42})\n",
+    "eval_config = get_eval_config(eval_llm)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/source/notebooks/getting_started.ipynb

@@ -6,7 +6,7 @@
             "metadata": {},
             "source": [
                 "# Getting Started\n",
-                "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain-benchmarks/blob/main/docs/source/notebooks/getting_started.ipynb)\n",
+                "\n",
                 "\n",
                 "LLMs are powerful but can be hard to steer and prone to errors when deployed. At the same time, new models and techniques are being developed all the time. We want to make it easy for you to experiment with different techniques, understand their tradeoffs, and make informed decisions for your specific use case.\n",
                 "\n",

docs/source/notebooks/retrieval/comparing_techniques.ipynb

@@ -6,7 +6,6 @@
    "metadata": {},
    "source": [
     "# Evaluating RAG Architectures on Benchmark Tasks\n",
-    "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain-benchmarks/blob/main/docs/source/notebooks/retrieval/comparing_techniques.ipynb)\n",
     "\n",
     "\n",
     "#### Introduction\n",
@@ -73,6 +72,18 @@
     "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import uuid\n",
+    "\n",
+    "# Generate a unique run ID for these experiments\n",
+    "run_uid = uuid.uuid4().hex[:6]"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "2e8a666d-8bf5-4bfd-8b20-8b7defdb8cd5",
@@ -267,6 +278,12 @@
     "    dataset_name=langchain_docs.name,\n",
     "    llm_or_chain_factory=partial(chain_factory, retriever, llm=llm),\n",
     "    evaluation=RAG_EVALUATION,\n",
+    "    project_name=f\"claude-2 qa-chain simple-index {run_uid}\",\n",
+    "    project_metadata={\n",
+    "        \"index_method\": \"basic\",\n",
+    "        \"embedding_model\": \"thenlper/gte-base\",\n",
+    "        \"llm\": \"claude-2\",\n",
+    "    },\n",
     "    verbose=True,\n",
     ")"
    ]
@@ -338,8 +355,16 @@
    "source": [
     "chunked_results = client.run_on_dataset(\n",
     "    dataset_name=langchain_docs.name,\n",
-    "    llm_or_chain_factory=partial(chain_factory, retriever, llm=llm),\n",
+    "    llm_or_chain_factory=partial(chain_factory, chunked_retriever, llm=llm),\n",
     "    evaluation=RAG_EVALUATION,\n",
+    "    project_name=f\"claude-2 qa-chain chunked {run_uid}\",\n",
+    "    project_metadata={\n",
+    "        \"index_method\": \"basic\",\n",
+    "        \"chunk_size\": 4000,\n",
+    "        \"chunk_overlap\": 200,\n",
+    "        \"embedding_model\": \"thenlper/gte-base\",\n",
+    "        \"llm\": \"claude-2\",\n",
+    "    },\n",
     "    verbose=True,\n",
     ")"
    ]
@@ -399,6 +424,12 @@
     "    dataset_name=langchain_docs.name,\n",
     "    llm_or_chain_factory=partial(chain_factory, parent_doc_retriever, llm=llm),\n",
     "    evaluation=RAG_EVALUATION,\n",
+    "    project_name=f\"claude-2 qa-chain parent-doc {run_uid}\",\n",
+    "    project_metadata={\n",
+    "        \"index_method\": \"parent-doc\",\n",
+    "        \"embedding_model\": \"thenlper/gte-base\",\n",
+    "        \"llm\": \"claude-2\",\n",
+    "    },\n",
     "    verbose=True,\n",
     ")"
    ]
@@ -447,9 +478,15 @@
    "source": [
     "hyde_test_run = client.run_on_dataset(\n",
     "    dataset_name=langchain_docs.name,\n",
-    "    llm_or_chain_factory=partial(chain_factory, retriever),\n",
+    "    llm_or_chain_factory=partial(chain_factory, retriever=retriever, llm=llm),\n",
     "    evaluation=RAG_EVALUATION,\n",
     "    verbose=True,\n",
+    "    project_name=f\"claude-2 qa-chain HyDE {run_uid}\",\n",
+    "    project_metadata={\n",
+    "        \"index_method\": \"HyDE\",\n",
+    "        \"embedding_model\": \"thenlper/gte-base\",\n",
+    "        \"llm\": \"claude-2\",\n",
+    "    },\n",
     ")"
    ]
   },
@@ -515,6 +552,12 @@
     "    dataset_name=langchain_docs.name,\n",
     "    llm_or_chain_factory=partial(chain_factory, openai_retriever),\n",
     "    evaluation=RAG_EVALUATION,\n",
+    "    project_name=f\"claude-2 qa-chain oai-emb basic {run_uid}\",\n",
+    "    project_metadata={\n",
+    "        \"index_method\": \"basic\",\n",
+    "        \"embedding_model\": \"openai/text-embedding-ada-002\",\n",
+    "        \"llm\": \"claude-2\",\n",
+    "    },\n",
     "    verbose=True,\n",
     ")"
    ]
@@ -559,16 +602,6 @@
     "ollama = ChatOllama(model=\"yarn-llama2:7b-128k\")"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "256b75fe-9f0c-4820-8e6c-9c87df39f0a7",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "ollama.invoke(\"Hi there\")"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -593,6 +626,12 @@
     "    dataset_name=langchain_docs.name,\n",
     "    llm_or_chain_factory=partial(chain_factory, llm=ollama, retriever=retriever),\n",
     "    evaluation=RAG_EVALUATION,\n",
+    "    project_name=f\"yarn-llama2:7b-128k qa-chain basic {run_uid}\",\n",
+    "    project_metadata={\n",
+    "        \"index_method\": \"basic\",\n",
+    "        \"embedding_model\": \"thenlper/gte-base\",\n",
+    "        \"llm\": \"ollama/yarn-llama2:7b-128k\",\n",
+    "    },\n",
     "    verbose=True,\n",
     ")"
    ]
@@ -649,9 +688,16 @@
     "claude_simple_prompt_test_run = client.run_on_dataset(\n",
     "    dataset_name=langchain_docs.name,\n",
     "    llm_or_chain_factory=partial(\n",
-    "        chain_factory, response_generator=generator, retriever=retriever\n",
+    "        chain_factory, response_generator=generator, retriever=retriever, llm=llm\n",
     "    ),\n",
     "    evaluation=RAG_EVALUATION,\n",
+    "    project_name=f\"claude-2 qa-chain basic rag-simple {run_uid}\",\n",
+    "    project_metadata={\n",
+    "        \"index_method\": \"basic\",\n",
+    "        \"embedding_model\": \"thenlper/gte-base\",\n",
+    "        \"prompt\": \"wfh/rag-simple\",\n",
+    "        \"llm\": \"claude-2\",\n",
+    "    },\n",
     "    verbose=True,\n",
     ")"
    ]
@@ -771,6 +817,13 @@
     "    dataset_name=langchain_docs.name,\n",
     "    llm_or_chain_factory=agent_executor,\n",
     "    evaluation=RAG_EVALUATION,\n",
+    "    project_name=f\"oai-functions basic rag-simple {run_uid}\",\n",
+    "    project_metadata={\n",
+    "        \"index_method\": \"basic\",\n",
+    "        \"embedding_model\": \"thenlper/gte-base\",\n",
+    "        \"llm\": \"gpt-4-1106-preview\",\n",
+    "        \"architecture\": \"oai-functions-agent\",\n",
+    "    },\n",
     "    verbose=True,\n",
     ")"
    ]
@@ -839,6 +892,13 @@
     "    dataset_name=langchain_docs.name,\n",
     "    llm_or_chain_factory=assistant_exector,\n",
     "    evaluation=RAG_EVALUATION,\n",
+    "    project_name=f\"oai-assistant basic rag-simple {run_uid}\",\n",
+    "    project_metadata={\n",
+    "        \"index_method\": \"basic\",\n",
+    "        \"embedding_model\": \"thenlper/gte-base\",\n",
+    "        \"llm\": \"gpt-4-1106-preview\",\n",
+    "        \"architecture\": \"oai-assistant\",\n",
+    "    },\n",
     "    verbose=True,\n",
     ")"
    ]

docs/source/notebooks/retrieval/intro.ipynb

@@ -1,117 +1,118 @@
 {
-    "cells": [
-        {
-            "cell_type": "markdown",
-            "metadata": {},
-            "source": [
-                "# Retrieval Tasks\n",
-                "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain-benchmarks/blob/main/docs/source/notebooks/retrieval/intro.ipynb)\n",
-                "\n",
-                "These tasks are meant to test retrieval-augmented generation (RAG) architectures on various datasets.\n",
-                "\n",
-                "You can check an up-to-date list of retrieval tasks in the registry:"
-            ]
-        },
-        {
-            "cell_type": "code",
-            "execution_count": 2,
-            "metadata": {},
-            "outputs": [
-                {
-                    "data": {
-                        "text/html": [
-                            "<table>\n",
-                            "<thead>\n",
-                            "<tr><th>Name                    </th><th>Type         </th><th>Dataset ID                                                                                                                                                 </th><th>Description  </th></tr>\n",
-                            "</thead>\n",
-                            "<tbody>\n",
-                            "<tr><td>LangChain Docs Q&A      </td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d\" target=\"_blank\" rel=\"noopener\">452ccafc-18e1-4314-885b-edd735f17b9d</a></td><td>Questions and answers based on a snapshot of the LangChain python docs.\n",
-                            "\n",
-                            "The environment provides the documents and the retriever information.\n",
-                            "\n",
-                            "Each example is composed of a question and reference answer.\n",
-                            "\n",
-                            "Success is measured based on the accuracy of the answer relative to the reference answer.\n",
-                            "We also measure the faithfulness of the model's response relative to the retrieved documents (if any).              </td></tr>\n",
-                            "<tr><td>Semi-structured Earnings</td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d\" target=\"_blank\" rel=\"noopener\">c47d9617-ab99-4d6e-a6e6-92b8daf85a7d</a></td><td>Questions and answers based on PDFs containing tables and charts.\n",
-                            "\n",
-                            "The task provides the raw documents as well as factory methods to easily index them\n",
-                            "and create a retriever.\n",
-                            "\n",
-                            "Each example is composed of a question and reference answer.\n",
-                            "\n",
-                            "Success is measured based on the accuracy of the answer relative to the reference answer.\n",
-                            "We also measure the faithfulness of the model's response relative to the retrieved documents (if any).              </td></tr>\n",
-                            "</tbody>\n",
-                            "</table>"
-                        ],
-                        "text/plain": [
-                            "Registry(tasks=[RetrievalTask(name='LangChain Docs Q&A', dataset_id='https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d', description=\"Questions and answers based on a snapshot of the LangChain python docs.\\n\\nThe environment provides the documents and the retriever information.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x138367ec0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x138367f60>, 'hyde': <function _chroma_hyde_retriever_factory at 0x13838c040>}, architecture_factories={'conversational-retrieval-qa': <function default_response_chain at 0x11fa74fe0>}, get_docs=<function load_cached_docs at 0x101bfb240>), RetrievalTask(name='Semi-structured Earnings', dataset_id='https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d', description=\"Questions and answers based on PDFs containing tables and charts.\\n\\nThe task provides the raw documents as well as factory methods to easily index them\\nand create a retriever.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x13838c5e0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x13838c680>, 'hyde': <function _chroma_hyde_retriever_factory at 0x13838c720>}, architecture_factories={}, get_docs=<function load_docs at 0x13838c540>)])"
-                        ]
-                    },
-                    "execution_count": 2,
-                    "metadata": {},
-                    "output_type": "execute_result"
-                }
-            ],
-            "source": [
-                "from langchain_benchmarks import registry\n",
-                "\n",
-                "registry.filter(Type=\"RetrievalTask\")"
-            ]
-        },
-        {
-            "cell_type": "markdown",
-            "metadata": {},
-            "source": [
-                "### Task resources\n",
-                "\n",
-                "In addition to a name, daset_id, and description, each retrieval task provides a few helper functions you can use to configure your pipeline.\n",
-                "\n",
-                "- `get_docs: callable` - fetches the original `Document` objects from the cache. Each task may provide configurable parameters you can use to define how the original documents are fetched.\n",
-                "- `retriever_factories: Dict[str, callable]` - define some configurable pipelines you can use to transform the documents, embed them, and add them to a vectorstore (or other retriever object) for downstream use. They use LangChain's caching `index` API so you don't have to re-index for every evaluation. For custom transformations, we ask that you provide a `transformation_name` to isolate the cache and vectorstore namespace. Currently (2023/11/21) these all use Chroma as a vectorstore, but you can swap this out\n",
-                "- `chain_factories: Dict[str, callable]` - define some off-the-shelf architectures you can configure to evaluate.\n",
-                "\n",
-                "When evaluating, you don't have to use any of these factory methods. You can instead define your own custom architecture or ETl pipeline before evaluating. They are meant to facilitate evaluations and comparisons for specific design decisions.\n",
-                "\n",
-                "### Dataset schema\n",
-                "\n",
-                "Each task corresponds to a LangSmith dataset with the following schema:\n",
-                "\n",
-                "Inputs:\n",
-                "- question: str - the user question\n",
-                "\n",
-                "Outputs\n",
-                "- answer: str - the expected answer"
-            ]
-        },
-        {
-            "cell_type": "code",
-            "execution_count": null,
-            "metadata": {},
-            "outputs": [],
-            "source": []
-        }
-    ],
-    "metadata": {
-        "kernelspec": {
-            "display_name": "Python 3 (ipykernel)",
-            "language": "python",
-            "name": "python3"
-        },
-        "language_info": {
-            "codemirror_mode": {
-                "name": "ipython",
-                "version": 3
-            },
-            "file_extension": ".py",
-            "mimetype": "text/x-python",
-            "name": "python",
-            "nbconvert_exporter": "python",
-            "pygments_lexer": "ipython3",
-            "version": "3.11.2"
-        }
-    },
-    "nbformat": 4,
-    "nbformat_minor": 4
-}
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Introduction\n",
+    "\n",
+    "\n",
+    "\n",
+    "These tasks are meant to test retrieval-augmented generation (RAG) architectures on various datasets.\n",
+    "\n",
+    "You can check an up-to-date list of retrieval tasks in the registry:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table>\n",
+       "<thead>\n",
+       "<tr><th>Name                    </th><th>Type         </th><th>Dataset ID                                                                                                                                                 </th><th>Description  </th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "<tr><td>LangChain Docs Q&A      </td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d\" target=\"_blank\" rel=\"noopener\">452ccafc-18e1-4314-885b-edd735f17b9d</a></td><td>Questions and answers based on a snapshot of the LangChain python docs.\n",
+       "\n",
+       "The environment provides the documents and the retriever information.\n",
+       "\n",
+       "Each example is composed of a question and reference answer.\n",
+       "\n",
+       "Success is measured based on the accuracy of the answer relative to the reference answer.\n",
+       "We also measure the faithfulness of the model's response relative to the retrieved documents (if any).              </td></tr>\n",
+       "<tr><td>Semi-structured Earnings</td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d\" target=\"_blank\" rel=\"noopener\">c47d9617-ab99-4d6e-a6e6-92b8daf85a7d</a></td><td>Questions and answers based on PDFs containing tables and charts.\n",
+       "\n",
+       "The task provides the raw documents as well as factory methods to easily index them\n",
+       "and create a retriever.\n",
+       "\n",
+       "Each example is composed of a question and reference answer.\n",
+       "\n",
+       "Success is measured based on the accuracy of the answer relative to the reference answer.\n",
+       "We also measure the faithfulness of the model's response relative to the retrieved documents (if any).              </td></tr>\n",
+       "</tbody>\n",
+       "</table>"
+      ],
+      "text/plain": [
+       "Registry(tasks=[RetrievalTask(name='LangChain Docs Q&A', dataset_id='https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d', description=\"Questions and answers based on a snapshot of the LangChain python docs.\\n\\nThe environment provides the documents and the retriever information.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x138367ec0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x138367f60>, 'hyde': <function _chroma_hyde_retriever_factory at 0x13838c040>}, architecture_factories={'conversational-retrieval-qa': <function default_response_chain at 0x11fa74fe0>}, get_docs=<function load_cached_docs at 0x101bfb240>), RetrievalTask(name='Semi-structured Earnings', dataset_id='https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d', description=\"Questions and answers based on PDFs containing tables and charts.\\n\\nThe task provides the raw documents as well as factory methods to easily index them\\nand create a retriever.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x13838c5e0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x13838c680>, 'hyde': <function _chroma_hyde_retriever_factory at 0x13838c720>}, architecture_factories={}, get_docs=<function load_docs at 0x13838c540>)])"
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from langchain_benchmarks import registry\n",
+    "\n",
+    "registry.filter(Type=\"RetrievalTask\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Task resources\n",
+    "\n",
+    "In addition to a name, daset_id, and description, each retrieval task provides a few helper functions you can use to configure your pipeline.\n",
+    "\n",
+    "- `get_docs: callable` - fetches the original `Document` objects from the cache. Each task may provide configurable parameters you can use to define how the original documents are fetched.\n",
+    "- `retriever_factories: Dict[str, callable]` - define some configurable pipelines you can use to transform the documents, embed them, and add them to a vectorstore (or other retriever object) for downstream use. They use LangChain's caching `index` API so you don't have to re-index for every evaluation. For custom transformations, we ask that you provide a `transformation_name` to isolate the cache and vectorstore namespace. Currently (2023/11/21) these all use Chroma as a vectorstore, but you can swap this out\n",
+    "- `chain_factories: Dict[str, callable]` - define some off-the-shelf architectures you can configure to evaluate.\n",
+    "\n",
+    "When evaluating, you don't have to use any of these factory methods. You can instead define your own custom architecture or ETl pipeline before evaluating. They are meant to facilitate evaluations and comparisons for specific design decisions.\n",
+    "\n",
+    "### Dataset schema\n",
+    "\n",
+    "Each task corresponds to a LangSmith dataset with the following schema:\n",
+    "\n",
+    "Inputs:\n",
+    "- question: str - the user question\n",
+    "\n",
+    "Outputs\n",
+    "- answer: str - the expected answer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

docs/source/notebooks/retrieval/langchain_docs_qa.ipynb

@@ -6,7 +6,6 @@
    "metadata": {},
    "source": [
     "# Q&A over LangChain Docs\n",
-    "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain-benchmarks/blob/main/docs/source/notebooks/retrieval/langchain_docs_qa.ipynb)\n",
     "\n",
     "Let's evaluate your architecture on a Q&A dataset for the LangChain python docs. For more examples of how to test different embeddings, indexing strategies, and architectures, see the [Evaluating RAG Architectures on Benchmark Tasks](./comparing_techniques.ipynb) notebook.\n",
     "\n",
@@ -40,9 +39,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "62b518cf-99fb-44be-8acb-ee0a8ba62272",
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "import os\n",
@@ -53,9 +54,11 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "4292ee8c",
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [],
    "source": [
     "# Update these with your own API keys\n",
@@ -65,6 +68,19 @@
     "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "1ccbf17a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import uuid\n",
+    "\n",
+    "# Generate a unique run ID for this experiment\n",
+    "run_uid = uuid.uuid4().hex[:6]"
+   ]
+  },
   {
    "cell_type": "markdown",
    "id": "2e8a666d-8bf5-4bfd-8b20-8b7defdb8cd5",
@@ -72,12 +88,12 @@
    "source": [
     "## Review Q&A Tasks\n",
     "\n",
-    "The registry provides configurations to test out common architectures on curated datasets."
+    "The registry provides configurations to test out common architectures on curated datasets. You can review retrieval tasks by filtering on the Type."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "b39159d0-9ea1-414f-a9d8-4a7b22b3d2cc",
    "metadata": {
     "tags": []
@@ -89,12 +105,49 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 7,
    "id": "3644d211-382e-41aa-b282-21b01d28fc35",
    "metadata": {
     "tags": []
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table>\n",
+       "<thead>\n",
+       "<tr><th>Name                   </th><th>Type         </th><th>Dataset ID                                                                                                                                                 </th><th>Description  </th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "<tr><td>LangChain Docs Q&A     </td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d\" target=\"_blank\" rel=\"noopener\">452ccafc-18e1-4314-885b-edd735f17b9d</a></td><td>Questions and answers based on a snapshot of the LangChain python docs.\n",
+       "\n",
+       "The environment provides the documents and the retriever information.\n",
+       "\n",
+       "Each example is composed of a question and reference answer.\n",
+       "\n",
+       "Success is measured based on the accuracy of the answer relative to the reference answer.\n",
+       "We also measure the faithfulness of the model's response relative to the retrieved documents (if any).              </td></tr>\n",
+       "<tr><td>Semi-structured Reports</td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d\" target=\"_blank\" rel=\"noopener\">c47d9617-ab99-4d6e-a6e6-92b8daf85a7d</a></td><td>Questions and answers based on PDFs containing tables and charts.\n",
+       "\n",
+       "The task provides the raw documents as well as factory methods to easily index them\n",
+       "and create a retriever.\n",
+       "\n",
+       "Each example is composed of a question and reference answer.\n",
+       "\n",
+       "Success is measured based on the accuracy of the answer relative to the reference answer.\n",
+       "We also measure the faithfulness of the model's response relative to the retrieved documents (if any).              </td></tr>\n",
+       "</tbody>\n",
+       "</table>"
+      ],
+      "text/plain": [
+       "Registry(tasks=[RetrievalTask(name='LangChain Docs Q&A', dataset_id='https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d', description=\"Questions and answers based on a snapshot of the LangChain python docs.\\n\\nThe environment provides the documents and the retriever information.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x2b433fc40>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x2b433fce0>, 'hyde': <function _chroma_hyde_retriever_factory at 0x2b433fd80>}, architecture_factories={'conversational-retrieval-qa': <function default_response_chain at 0x2b3b59580>}, get_docs=<function load_cached_docs at 0x2b3b59080>), RetrievalTask(name='Semi-structured Reports', dataset_id='https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d', description=\"Questions and answers based on PDFs containing tables and charts.\\n\\nThe task provides the raw documents as well as factory methods to easily index them\\nand create a retriever.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x2b43644a0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x2b4364540>, 'hyde': <function _chroma_hyde_retriever_factory at 0x2b43645e0>}, architecture_factories={}, get_docs=<function load_docs at 0x2b4364400>)])"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "registry = registry.filter(Type=\"RetrievalTask\")\n",
     "registry"
@@ -102,56 +155,159 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 8,
    "id": "671282f8-c455-4390-b018-e53bbd833093",
    "metadata": {
     "tags": []
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table>\n",
+       "<tbody>\n",
+       "<tr><td>Name                  </td><td>LangChain Docs Q&A                                                                                                                                         </td></tr>\n",
+       "<tr><td>Type                  </td><td>RetrievalTask                                                                                                                                              </td></tr>\n",
+       "<tr><td>Dataset ID            </td><td><a href=\"https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d\" target=\"_blank\" rel=\"noopener\">452ccafc-18e1-4314-885b-edd735f17b9d</a></td></tr>\n",
+       "<tr><td>Description           </td><td>Questions and answers based on a snapshot of the LangChain python docs.\n",
+       "\n",
+       "The environment provides the documents and the retriever information.\n",
+       "\n",
+       "Each example is composed of a question and reference answer.\n",
+       "\n",
+       "Success is measured based on the accuracy of the answer relative to the reference answer.\n",
+       "We also measure the faithfulness of the model's response relative to the retrieved documents (if any).                                                                                                                                                            </td></tr>\n",
+       "<tr><td>Retriever Factories   </td><td>basic, parent-doc, hyde                                                                                                                                    </td></tr>\n",
+       "<tr><td>Architecture Factories</td><td>conversational-retrieval-qa                                                                                                                                </td></tr>\n",
+       "<tr><td>get_docs              </td><td><function load_cached_docs at 0x2b3b59080>                                                                                                                 </td></tr>\n",
+       "</tbody>\n",
+       "</table>"
+      ],
+      "text/plain": [
+       "RetrievalTask(name='LangChain Docs Q&A', dataset_id='https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d', description=\"Questions and answers based on a snapshot of the LangChain python docs.\\n\\nThe environment provides the documents and the retriever information.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x2b433fc40>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x2b433fce0>, 'hyde': <function _chroma_hyde_retriever_factory at 0x2b433fd80>}, architecture_factories={'conversational-retrieval-qa': <function default_response_chain at 0x2b3b59580>}, get_docs=<function load_cached_docs at 0x2b3b59080>)"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "langchain_docs = registry[\"LangChain Docs Q&A\"]\n",
     "langchain_docs"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "968f78bb-7e00-407f-b486-1e5b27f3287a",
+   "metadata": {},
+   "source": [
+    "## Clone the dataset\n",
+    "\n",
+    "Once you've selected the LangChain Docs Q&A task, clone the dataset to your LangSmith tenant. This step\n",
+    "requires that your LANGCHAIN_API_KEY be set above."
+   ]
+  },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "id": "70369f67-deb4-467a-801a-6d38c3d0460d",
    "metadata": {
     "tags": []
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Dataset LangChain Docs Q&A already exists. Skipping.\n",
+      "You can access the dataset at https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/3f29798f-5939-4643-bd99-008ca66b72ed.\n"
+     ]
+    }
+   ],
    "source": [
     "clone_public_dataset(langchain_docs.dataset_id, dataset_name=langchain_docs.name)"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c58247f5-b9bd-4cc5-9632-78bc21bb10b4",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
+   "cell_type": "markdown",
+   "id": "2867f083-1f36-40a9-8e97-74381d47afcd",
+   "metadata": {},
    "source": [
-    "from langchain.embeddings import HuggingFaceEmbeddings\n",
+    "## Create the index\n",
     "\n",
-    "embeddings = HuggingFaceEmbeddings(\n",
-    "    model_name=\"thenlper/gte-base\",\n",
-    "    model_kwargs={\"device\": 0},  # Comment out to use CPU\n",
-    ")\n",
+    "When creating a retrieval Q&A system, the first step is to prepare the retriever. How you construct the index significantly impact your system's performance. Before trying anything too tricky, it's good benchmark a reliable baseline.\n",
     "\n",
-    "docs = langchain_docs.get_docs()\n",
-    "retriever_factory = langchain_docs.retriever_factories[\"basic\"]\n",
-    "# Indexes the documents with the specified embeddings\n",
-    "# Note that this does not apply any chunking to the docs,\n",
-    "# which means the documents can be of arbitrary length\n",
-    "retriever = retriever_factory(embeddings, docs=docs)"
+    "In this case, our baseline will be to generate a single vector for each raw source document and store them directly in a vector store.\n",
+    "\n",
+    "Below, fetch the source docs from the cache in GCS. This cache was formed using an [ingestion script](https://github.com/langchain-ai/langchain-benchmarks/blob/30aa706d9cefbaebb219f1763c04fafab6d0ee78/langchain_benchmarks/rag/tasks/langchain_docs/_ingest_docs.py#L1) that scraped the LangChain documentation. To save time and to ensure that the dataset answers are still correct, we will use these source docs for all benchmark approaches."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 17,
+   "id": "3afe1e54-9354-4bbc-9fd8-0f9d769fc43d",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Document(page_content=\"LangChain cookbook | 🦜️🔗 Langchain\\n\\n[Skip to main content](#docusaurus_skip...\n"
+     ]
+    }
+   ],
+   "source": [
+    "docs = list(langchain_docs.get_docs())\n",
+    "print(repr(docs[0])[:100] + \"...\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f7e6830a-dd2f-4532-95e7-81353d18dcd2",
+   "metadata": {},
+   "source": [
+    "Now we will populate our vectorstore. We will use LangChain's indexing API to cache embeddings"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "id": "4980f8d4-d4e6-4b70-a04d-a7d9333545ee",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.embeddings import HuggingFaceEmbeddings\n",
+    "from langchain.vectorstores.chroma import Chroma\n",
+    "\n",
+    "embeddings = HuggingFaceEmbeddings(\n",
+    "    model_name=\"thenlper/gte-base\",\n",
+    "    # model_kwargs={\"device\": 0},  # Comment out to use CPU\n",
+    ")\n",
+    "\n",
+    "vectorstore = Chroma(\n",
+    "    collection_name=f\"lcbm-b-huggingface-gte-base\",\n",
+    "    embedding_function=embeddings,\n",
+    "    persist_directory=\"./chromadb\",\n",
+    ")\n",
+    "\n",
+    "vectorstore.add_documents(docs)\n",
+    "retriever = vectorstore.as_retriever(search_kwargs={\"k\": 6})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "54bac7ea-ab28-4b08-9d06-56ac869085dd",
+   "metadata": {},
+   "source": [
+    "## Define the response generator\n",
+    "\n",
+    "Halfway done with our RAG system. We've made the **R**etriever. Now time for the response **G**enerator.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
    "id": "41e64350-63a7-4e7d-8e03-7dc459c444cc",
    "metadata": {
     "tags": []
@@ -165,10 +321,11 @@
     "from langchain.prompts import ChatPromptTemplate\n",
     "from langchain.schema.document import Document\n",
     "from langchain.schema.output_parser import StrOutputParser\n",
-    "from langchain.schema.runnable import RunnableLambda\n",
     "from langchain.schema.runnable.passthrough import RunnableAssign\n",
     "\n",
     "\n",
+    "# After the retriever fetches documents, this\n",
+    "# function formats them in a string to present for the LLM\n",
     "def format_docs(docs: Sequence[Document]) -> str:\n",
     "    formatted_docs = []\n",
     "    for i, doc in enumerate(docs):\n",
@@ -199,6 +356,11 @@
     "response_generator = (prompt | llm | StrOutputParser()).with_config(\n",
     "    run_name=\"GenerateResponse\",\n",
     ")\n",
+    "\n",
+    "# This is the final response chain.\n",
+    "# It fetches the \"question\" key from the input dict,\n",
+    "# passes it to the retriever, then formats as a string.\n",
+    "\n",
     "chain = (\n",
     "    RunnableAssign(\n",
     "        {\n",
@@ -207,18 +369,33 @@
     "            )\n",
     "        }\n",
     "    )\n",
+    "    # The \"RunnableAssign\" above returns a dict with keys\n",
+    "    # question (from the original input) and\n",
+    "    # context: the string-formatted docs.\n",
+    "    # This is passed to the response_generator above\n",
     "    | response_generator\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 23,
    "id": "10a1fca9-d356-4cff-93a9-c4f63944e57d",
    "metadata": {
     "tags": []
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "' The LangChain Expression Language (LCEL) is a declarative way to easily compose chains of different components like prompts, models, parsers, etc. \\n\\nSome key things it provides:\\n\\n- Streaming support - Ability to get incremental outputs from chains rather than waiting for full completion. Useful for long-running chains.\\n\\n- Async support - Chains can be called synchronously (like in a notebook) or asynchronously (like in production). Same code works for both.\\n\\n- Optimized parallel execution - Steps that can run in parallel (like multiple retrievals) are automatically parallelized to minimize latency.\\n\\n- Retries and fallbacks - Help make chains more robust to failure.\\n\\n- Access to intermediate results - Useful for debugging or showing work-in-progress.\\n\\n- Input and output validation via schemas - Enables catching issues early.\\n\\n- Tracing - Automatic structured logging of all chain steps for observability.\\n\\n- Seamless deployment - LCEL chains can be easily deployed with LangServe.\\n\\nThe key idea is it makes it very easy to take a prototype LLM application made with components like prompts and models and turn it into a robust, scalable production application without changing any code.'"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "chain.invoke({\"question\": \"What's expression language?\"})"
    ]
@@ -228,14 +405,14 @@
    "id": "3821e4b0-8e67-418a-840c-470fcde42df0",
    "metadata": {},
    "source": [
-    "### Evaluate\n",
+    "## Evaluate\n",
     "\n",
     "Let's evaluate your RAG architecture on the dataset now."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 24,
    "id": "513042fe-2878-44f8-ae84-05b9d521c1de",
    "metadata": {
     "tags": []
@@ -249,12 +426,37 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 25,
    "id": "aab7514e-a6ef-4c21-b90f-d9cbefcf5af1",
    "metadata": {
     "tags": []
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "View the evaluation results for project 'claude-2 qa-chain simple-index 1bdbe5' at:\n",
+      "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/projects/p/3fe31959-95e8-4413-aa09-620bd49bd0d3?eval=true\n",
+      "\n",
+      "View all tests for Dataset LangChain Docs Q&A at:\n",
+      "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/3f29798f-5939-4643-bd99-008ca66b72ed\n",
+      "[------------------------------------------------->] 86/86\n",
+      " Eval quantiles:\n",
+      "                                0.25        0.5       0.75       mean  \\\n",
+      "embedding_cosine_distance   0.088025   0.115760   0.159969   0.129161   \n",
+      "score_string:accuracy       0.500000   0.700000   1.000000   0.645349   \n",
+      "faithfulness                0.700000   1.000000   1.000000   0.812791   \n",
+      "execution_time             27.098772  27.098772  27.098772  27.098772   \n",
+      "\n",
+      "                                mode  \n",
+      "embedding_cosine_distance   0.048622  \n",
+      "score_string:accuracy       0.700000  \n",
+      "faithfulness                1.000000  \n",
+      "execution_time             27.098772  \n"
+     ]
+    }
+   ],
    "source": [
     "client = Client()\n",
     "RAG_EVALUATION = get_eval_config()\n",
@@ -263,6 +465,10 @@
     "    dataset_name=langchain_docs.name,\n",
     "    llm_or_chain_factory=chain,\n",
     "    evaluation=RAG_EVALUATION,\n",
+    "    project_name=f\"claude-2 qa-chain simple-index {run_uid}\",\n",
+    "    project_metadata={\n",
+    "        \"index_method\": \"basic\",\n",
+    "    },\n",
     "    verbose=True,\n",
     ")"
    ]
@@ -270,7 +476,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "e86578d5-be5c-4bcd-9dcb-35280eeed3f9",
+   "id": "79d85a82-5ba7-479f-94e2-c9e3c7cb90cb",
    "metadata": {
     "tags": []
    },
@@ -279,75 +485,13 @@
     "test_run.get_aggregate_feedback()"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "id": "01811b97-cb28-42a6-920a-7a700f77f19d",
-   "metadata": {},
-   "source": [
-    "## Evaluate with a default factory\n",
-    "\n",
-    "The task can define default chain and retriever \"factories\", which provide a default architecture that you can modify by choosing the llms, prompts, etc. Let's try the `conversational-retrieval-qa` factory."
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "f4d2e139-2653-4f7b-944b-91ef52f43d3e",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "# Factory for creating a conversational retrieval QA chain\n",
-    "chain_factory = langchain_docs.architecture_factories[\"conversational-retrieval-qa\"]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "9f9be718-64f0-4706-9527-240a1cdb3ecb",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "from langchain.chat_models import ChatAnthropic\n",
-    "\n",
-    "# Example\n",
-    "llm = ChatAnthropic(model=\"claude-2\", temperature=1)\n",
-    "\n",
-    "\n",
-    "chain = chain_factory(retriever, llm=llm)\n",
-    "\n",
-    "chain.invoke({\"question\": \"What is expression language?\"})"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "e9c013e2-241a-4def-9aa6-ccb34273eeb9",
-   "metadata": {
-    "tags": []
-   },
-   "outputs": [],
-   "source": [
-    "test_run = client.run_on_dataset(\n",
-    "    dataset_name=langchain_docs.name,\n",
-    "    llm_or_chain_factory=chain,\n",
-    "    evaluation=RAG_EVALUATION,\n",
-    "    verbose=True,\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "8f3c1d2d",
+   "id": "230bf290-0422-4774-b82f-f78eb2e51918",
    "metadata": {},
    "outputs": [],
-   "source": [
-    "test_run.get_aggregate_feedback()"
-   ]
+   "source": []
   }
  ],
  "metadata": {

docs/source/notebooks/retrieval/multi_modal_benchmarking/multi_modal_eval_baseline.ipynb

@@ -0,0 +1,610 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "9fa3470d-9448-4792-9f65-6978fc58cf84",
+   "metadata": {},
+   "source": [
+    "# Multi-modal eval: Baseline\n",
+    "\n",
+    "`Multi-modal slide decks` is a public dataset that contains a dataset of question-answer pairs from slide decks with visual content.\n",
+    "\n",
+    "The question-answer pairs are derived from the visual content in the decks, testing the ability of RAG to perform visual reasoning.\n",
+    "\n",
+    "As a baseline, we evaluate this dataset using text-based RAG pipeline, below.\n",
+    "\n",
+    "This will not reason about visual content and will simply load the text from the slides. \n",
+    "\n",
+    "## Pre-requisites"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "47220461-d4e9-4f1d-9c57-672ca947ca0d",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# %pip install -U langchain langsmith langchain_benchmarks\n",
+    "# %pip install --quiet chromadb openai pypdf pandas"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "196de967-6de6-40da-aa75-e836923ab5e3",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import getpass\n",
+    "import os\n",
+    "\n",
+    "os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
+    "env_vars = [\"LANGCHAIN_API_KEY\", \"OPENAI_API_KEY\"]\n",
+    "for var in env_vars:\n",
+    "    if var not in os.environ:\n",
+    "        os.environ[var] = getpass.getpass(prompt=f\"Enter your {var}: \")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "10da8e11-6288-4131-bd60-d5aa86928acc",
+   "metadata": {},
+   "source": [
+    "## Dataset\n",
+    "\n",
+    "We can browse the available LangChain benchmark datasets for retrieval."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "2ff97905-14a6-413c-99be-58b7a9c8d4c1",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table>\n",
+       "<thead>\n",
+       "<tr><th>Name                   </th><th>Type         </th><th>Dataset ID                                                                                                                                                 </th><th>Description  </th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "<tr><td>LangChain Docs Q&A     </td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d\" target=\"_blank\" rel=\"noopener\">452ccafc-18e1-4314-885b-edd735f17b9d</a></td><td>Questions and answers based on a snapshot of the LangChain python docs.\n",
+       "\n",
+       "The environment provides the documents and the retriever information.\n",
+       "\n",
+       "Each example is composed of a question and reference answer.\n",
+       "\n",
+       "Success is measured based on the accuracy of the answer relative to the reference answer.\n",
+       "We also measure the faithfulness of the model's response relative to the retrieved documents (if any).              </td></tr>\n",
+       "<tr><td>Semi-structured Reports</td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d\" target=\"_blank\" rel=\"noopener\">c47d9617-ab99-4d6e-a6e6-92b8daf85a7d</a></td><td>Questions and answers based on PDFs containing tables and charts.\n",
+       "\n",
+       "The task provides the raw documents as well as factory methods to easily index them\n",
+       "and create a retriever.\n",
+       "\n",
+       "Each example is composed of a question and reference answer.\n",
+       "\n",
+       "Success is measured based on the accuracy of the answer relative to the reference answer.\n",
+       "We also measure the faithfulness of the model's response relative to the retrieved documents (if any).              </td></tr>\n",
+       "<tr><td>Multi-modal slide decks</td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/40afc8e7-9d7e-44ed-8971-2cae1eb59731/d\" target=\"_blank\" rel=\"noopener\">40afc8e7-9d7e-44ed-8971-2cae1eb59731</a></td><td>This public dataset is a work-in-progress and will be extended over time.\n",
+       "        \n",
+       "Questions and answers based on slide decks containing visual tables and charts.\n",
+       "\n",
+       "Each example is composed of a question and reference answer.\n",
+       "\n",
+       "Success is measured based on the accuracy of the answer relative to the reference answer.              </td></tr>\n",
+       "</tbody>\n",
+       "</table>"
+      ],
+      "text/plain": [
+       "Registry(tasks=[RetrievalTask(name='LangChain Docs Q&A', dataset_id='https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d', description=\"Questions and answers based on a snapshot of the LangChain python docs.\\n\\nThe environment provides the documents and the retriever information.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", get_docs=<function load_cached_docs at 0x104485800>, retriever_factories={'basic': <function _chroma_retriever_factory at 0x1360289a0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x136028a40>, 'hyde': <function _chroma_hyde_retriever_factory at 0x136028ae0>}, architecture_factories={'conversational-retrieval-qa': <function default_response_chain at 0x126ba2660>}), RetrievalTask(name='Semi-structured Reports', dataset_id='https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d', description=\"Questions and answers based on PDFs containing tables and charts.\\n\\nThe task provides the raw documents as well as factory methods to easily index them\\nand create a retriever.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", get_docs=<function load_docs at 0x136029620>, retriever_factories={'basic': <function _chroma_retriever_factory at 0x1360296c0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x136029760>, 'hyde': <function _chroma_hyde_retriever_factory at 0x136029800>}, architecture_factories={}), RetrievalTask(name='Multi-modal slide decks', dataset_id='https://smith.langchain.com/public/40afc8e7-9d7e-44ed-8971-2cae1eb59731/d', description='This public dataset is a work-in-progress and will be extended over time.\\n        \\nQuestions and answers based on slide decks containing visual tables and charts.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\n', get_docs={}, retriever_factories={}, architecture_factories={})])"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from langchain_benchmarks import clone_public_dataset, registry\n",
+    "\n",
+    "registry = registry.filter(Type=\"RetrievalTask\")\n",
+    "registry"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2fb7dc3d-28f1-4c28-b0d0-3784d04b81ce",
+   "metadata": {},
+   "source": [
+    "`Multi-modal slide decks` is the relevant dataset for our task."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "219a4141-4a5f-48e4-ae05-5a824e2193fd",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table>\n",
+       "<tbody>\n",
+       "<tr><td>Name                  </td><td>Multi-modal slide decks                                                                                                                                    </td></tr>\n",
+       "<tr><td>Type                  </td><td>RetrievalTask                                                                                                                                              </td></tr>\n",
+       "<tr><td>Dataset ID            </td><td><a href=\"https://smith.langchain.com/public/40afc8e7-9d7e-44ed-8971-2cae1eb59731/d\" target=\"_blank\" rel=\"noopener\">40afc8e7-9d7e-44ed-8971-2cae1eb59731</a></td></tr>\n",
+       "<tr><td>Description           </td><td>This public dataset is a work-in-progress and will be extended over time.\n",
+       "        \n",
+       "Questions and answers based on slide decks containing visual tables and charts.\n",
+       "\n",
+       "Each example is composed of a question and reference answer.\n",
+       "\n",
+       "Success is measured based on the accuracy of the answer relative to the reference answer.                                                                                                                                                            </td></tr>\n",
+       "<tr><td>Retriever Factories   </td><td>                                                                                                                                                           </td></tr>\n",
+       "<tr><td>Architecture Factories</td><td>                                                                                                                                                           </td></tr>\n",
+       "<tr><td>get_docs              </td><td>{}                                                                                                                                                         </td></tr>\n",
+       "</tbody>\n",
+       "</table>"
+      ],
+      "text/plain": [
+       "RetrievalTask(name='Multi-modal slide decks', dataset_id='https://smith.langchain.com/public/40afc8e7-9d7e-44ed-8971-2cae1eb59731/d', description='This public dataset is a work-in-progress and will be extended over time.\\n        \\nQuestions and answers based on slide decks containing visual tables and charts.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\n', get_docs={}, retriever_factories={}, architecture_factories={})"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "task = registry[\"Multi-modal slide decks\"]\n",
+    "task"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2d6569b5-e79a-41b7-9745-c2f8a1dd704e",
+   "metadata": {},
+   "source": [
+    "Clone the dataset so that it's available in our LangSmith datasets."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "d2caa086-9549-4c74-bba9-ba80d5a7b218",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Dataset Multi-modal slide decks already exists. Skipping.\n",
+      "You can access the dataset at https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/08a29acb-5ad6-42ce-a482-574c9e2e5306.\n"
+     ]
+    }
+   ],
+   "source": [
+    "clone_public_dataset(task.dataset_id, dataset_name=task.name)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "bf350917-a1e5-46f4-81cd-c1678ab9220f",
+   "metadata": {},
+   "source": [
+    "Fetch the associated PDFs from remote cache for the dataset so that we can perform ingestion."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "99ce6afb-2317-4bc1-9faf-4f828095ad91",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain_benchmarks.rag.tasks.multi_modal_slide_decks import get_file_names\n",
+    "\n",
+    "file_names = list(get_file_names())  # PosixPath"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "848a4cdb-6c08-4c01-81ce-16ab83a7fdff",
+   "metadata": {},
+   "source": [
+    "## Load\n",
+    "\n",
+    "Load and split the files for indexing."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "6ce85810-98a7-406e-b44e-ce860ac35986",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "There are 98 text elements in DDOG_Q3_earnings_deck.pdf\n"
+     ]
+    }
+   ],
+   "source": [
+    "from langchain.document_loaders import PyPDFLoader\n",
+    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "\n",
+    "\n",
+    "def load_and_split(file):\n",
+    "    \"\"\"\n",
+    "    Load and split PDF files\n",
+    "    :param file: PosixPath path for pdf\n",
+    "    :return: A list of text chunks\n",
+    "    \"\"\"\n",
+    "\n",
+    "    loader = PyPDFLoader(str(file))\n",
+    "    pdf_pages = loader.load()\n",
+    "\n",
+    "    text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(\n",
+    "        chunk_size=100, chunk_overlap=50\n",
+    "    )\n",
+    "\n",
+    "    # Get chunks\n",
+    "    docs = text_splitter.split_documents(pdf_pages)\n",
+    "    texts = [d.page_content for d in docs]\n",
+    "    print(f\"There are {len(texts)} text elements in {file.name}\")\n",
+    "    return texts\n",
+    "\n",
+    "\n",
+    "texts = []\n",
+    "for fi in file_names:\n",
+    "    texts.extend(load_and_split(fi))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "eb01925d-b7d1-47a1-bd90-805178d3c4a9",
+   "metadata": {},
+   "source": [
+    "## Index\n",
+    "\n",
+    "Embed (OpenAIEmbeddings) and store splits in a vectorstore (Chroma)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "ceb31f71-45fb-4b12-bc1c-31981de334bb",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.embeddings import OpenAIEmbeddings\n",
+    "from langchain.vectorstores import Chroma\n",
+    "\n",
+    "vectorstore_baseline = Chroma.from_texts(\n",
+    "    texts=texts, collection_name=\"baseline-multi-modal\", embedding=OpenAIEmbeddings()\n",
+    ")\n",
+    "\n",
+    "retriever_baseline = vectorstore_baseline.as_retriever()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e6dcbb01-f480-456d-b972-c732eb26c393",
+   "metadata": {},
+   "source": [
+    "## RAG\n",
+    "\n",
+    "Create a pipeline for retrieval of relevant chunks based on semantic similarity to the input question.\n",
+    "\n",
+    "Pass the images to GPT-4 for answer synthesis."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "ea233664-e527-42f1-a820-0c2271e16c20",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.chat_models import ChatOpenAI\n",
+    "from langchain.prompts import ChatPromptTemplate\n",
+    "from langchain.schema.output_parser import StrOutputParser\n",
+    "from langchain.schema.runnable import RunnablePassthrough\n",
+    "\n",
+    "\n",
+    "def rag_chain(retriever):\n",
+    "    \"\"\"\n",
+    "    RAG pipeline for the indexed presentations\n",
+    "    :param retriever: PosixPath path for pdf\n",
+    "    \"\"\"\n",
+    "\n",
+    "    # Prompt template\n",
+    "    template = \"\"\"Answer the question based only on the following context, which can include text and tables:\n",
+    "    {context}\n",
+    "    Question: {question}\n",
+    "    \"\"\"\n",
+    "    prompt = ChatPromptTemplate.from_template(template)\n",
+    "\n",
+    "    # LLM\n",
+    "    model = ChatOpenAI(temperature=0, model=\"gpt-4\")\n",
+    "\n",
+    "    # RAG pipeline\n",
+    "    chain = (\n",
+    "        {\n",
+    "            \"context\": retriever | (lambda x: \"\\n\\n\".join([i.page_content for i in x])),\n",
+    "            \"question\": RunnablePassthrough(),\n",
+    "        }\n",
+    "        | prompt\n",
+    "        | model\n",
+    "        | StrOutputParser()\n",
+    "    )\n",
+    "    return chain\n",
+    "\n",
+    "\n",
+    "# Create RAG chain\n",
+    "chain = rag_chain(retriever_baseline)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "95df1446-143d-4f4c-a15b-2a379266d8bf",
+   "metadata": {},
+   "source": [
+    "## Eval\n",
+    "\n",
+    "Run evaluation on our dataset:\n",
+    "\n",
+    "* `task.name` is the dataset of QA pairs that we cloned\n",
+    "* `eval_config` specifies the [LangSmith evaluator](https://docs.smith.langchain.com/evaluation/evaluator-implementations#correctness-qa-evaluation) for our dataset, which will use GPT-4 as a grader\n",
+    "* The grader will evaluate the chain-generated answer to each question relative to ground truth"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "479ce09d-642e-4b3b-9e4e-e9c2b7f0e9ca",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "View the evaluation results for project '866f-baseline' at:\n",
+      "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/08a29acb-5ad6-42ce-a482-574c9e2e5306/compare?selectedSessions=30199d47-50d7-4c5c-a55a-e74157e05951\n",
+      "\n",
+      "View all tests for Dataset Multi-modal slide decks at:\n",
+      "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/08a29acb-5ad6-42ce-a482-574c9e2e5306\n",
+      "[------------------------------------------------->] 10/10"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<h3>Experiment Results:</h3>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>output</th>\n",
+       "      <th>feedback.COT Contextual Accuracy</th>\n",
+       "      <th>error</th>\n",
+       "      <th>execution_time</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>count</th>\n",
+       "      <td>10</td>\n",
+       "      <td>10.000000</td>\n",
+       "      <td>0</td>\n",
+       "      <td>10.000000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>unique</th>\n",
+       "      <td>10</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>top</th>\n",
+       "      <td>Datadog has 20 total customers.</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>freq</th>\n",
+       "      <td>1</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>NaN</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>mean</th>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.200000</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>4.674478</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>std</th>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.421637</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.864273</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>min</th>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>3.307960</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25%</th>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>4.113816</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>50%</th>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>4.700962</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>75%</th>\n",
+       "      <td>NaN</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>5.018359</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>max</th>\n",
+       "      <td>NaN</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>6.188082</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                 output  feedback.COT Contextual Accuracy  \\\n",
+       "count                                10                         10.000000   \n",
+       "unique                               10                               NaN   \n",
+       "top     Datadog has 20 total customers.                               NaN   \n",
+       "freq                                  1                               NaN   \n",
+       "mean                                NaN                          0.200000   \n",
+       "std                                 NaN                          0.421637   \n",
+       "min                                 NaN                          0.000000   \n",
+       "25%                                 NaN                          0.000000   \n",
+       "50%                                 NaN                          0.000000   \n",
+       "75%                                 NaN                          0.000000   \n",
+       "max                                 NaN                          1.000000   \n",
+       "\n",
+       "       error  execution_time  \n",
+       "count      0       10.000000  \n",
+       "unique     0             NaN  \n",
+       "top      NaN             NaN  \n",
+       "freq     NaN             NaN  \n",
+       "mean     NaN        4.674478  \n",
+       "std      NaN        0.864273  \n",
+       "min      NaN        3.307960  \n",
+       "25%      NaN        4.113816  \n",
+       "50%      NaN        4.700962  \n",
+       "75%      NaN        5.018359  \n",
+       "max      NaN        6.188082  "
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import uuid\n",
+    "\n",
+    "from langchain.smith import RunEvalConfig\n",
+    "from langsmith.client import Client\n",
+    "\n",
+    "# Evaluator configuration\n",
+    "client = Client()\n",
+    "eval_config = RunEvalConfig(\n",
+    "    evaluators=[\"cot_qa\"],\n",
+    ")\n",
+    "\n",
+    "# Experiments\n",
+    "chain_map = {\n",
+    "    \"baseline\": chain,\n",
+    "}\n",
+    "\n",
+    "# Run evaluation\n",
+    "run_id = uuid.uuid4().hex[:4]\n",
+    "test_runs = {}\n",
+    "for project_name, chain in chain_map.items():\n",
+    "    test_runs[project_name] = client.run_on_dataset(\n",
+    "        dataset_name=task.name,\n",
+    "        llm_or_chain_factory=lambda: (lambda x: x[\"Question\"]) | chain,\n",
+    "        evaluation=eval_config,\n",
+    "        verbose=True,\n",
+    "        project_name=f\"{run_id}-{project_name}\",\n",
+    "        project_metadata={\"chain\": project_name},\n",
+    "    )"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/source/notebooks/retrieval/semi_structured.ipynb

@@ -6,7 +6,7 @@
    "metadata": {},
    "source": [
     "# Semi-structured RAG\n",
-    "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain-benchmarks/blob/main/docs/source/notebooks/retrieval/semi_structured.ipynb)\n",
+    "\n",
     "\n",
     "Let's evaluate your architecture on a small semi-structured Q&A dataset. This dataset is composed of QA pairs over pdfs that contain tables."
    ]

docs/source/notebooks/tool_usage/benchmark_all_tasks.ipynb

@@ -0,0 +1,98 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "1ba9f105-c48f-4d8c-8253-355ef13156b0",
+   "metadata": {},
+   "source": [
+    "# Benchmark All\n",
+    "\n",
+    "Here, we'll run benchmarking against all tool usage task.\n",
+    "\n",
+    "Expand the models list to benchmark against different models."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ddadb9ef-e76a-4b48-85e4-f62c3957f502",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import datetime\n",
+    "import uuid\n",
+    "\n",
+    "from langsmith.client import Client\n",
+    "\n",
+    "from langchain_benchmarks import clone_public_dataset, registry\n",
+    "from langchain_benchmarks.tool_usage import agents"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e6fbc3ef-7a3f-430f-8b79-45af5861b3ee",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "experiment_uuid = uuid.uuid4().hex[:4]\n",
+    "models = [\"gpt-3.5-turbo-16k\"]\n",
+    "client = Client()  # Launch langsmith client for cloning datasets\n",
+    "today = datetime.date.today().isoformat()\n",
+    "\n",
+    "for task in registry:\n",
+    "    dataset_name = task.name + f\"_benchmarking_{today}\"\n",
+    "    clone_public_dataset(task.dataset_id, dataset_name=dataset_name)\n",
+    "\n",
+    "    if task.type != \"ToolUsageTask\":\n",
+    "        continue\n",
+    "    for model in models:\n",
+    "        print()\n",
+    "        print(f\"Benchmarking {task.name} with model: {model}\")\n",
+    "        eval_config = task.get_eval_config()\n",
+    "        agent_factory = agents.OpenAIAgentFactory(task, model=model)\n",
+    "\n",
+    "        client.run_on_dataset(\n",
+    "            dataset_name=dataset_name,\n",
+    "            llm_or_chain_factory=agent_factory,\n",
+    "            evaluation=eval_config,\n",
+    "            verbose=False,\n",
+    "            project_name=f\"{dataset_name}-{model}-{experiment_uuid}\",\n",
+    "            tags=[model],\n",
+    "            concurrency_level=1,\n",
+    "            project_metadata={\n",
+    "                \"model\": model,\n",
+    "                \"id\": experiment_uuid,\n",
+    "                \"task\": task.name,\n",
+    "                \"date\": today,\n",
+    "            },\n",
+    "        )"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}