x

Add docs scaffolding

.github/actions/poetry_setup/action.yml

@@ -0,0 +1,91 @@
+# An action for setting up poetry install with caching.
+# Using a custom action since the default action does not
+# take poetry install groups into account.
+# Action code from:
+# https://github.com/actions/setup-python/issues/505#issuecomment-1273013236
+name: poetry-install-with-caching
+description: Poetry install with support for caching of dependency groups.
+
+inputs:
+  python-version:
+    description: Python version, supporting MAJOR.MINOR only
+    required: true
+
+  poetry-version:
+    description: Poetry version
+    required: true
+
+  cache-key:
+    description: Cache key to use for manual handling of caching
+    required: true
+
+  working-directory:
+    description: Directory whose poetry.lock file should be cached
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/setup-python@v4
+      name: Setup python ${{ inputs.python-version }}
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - uses: actions/cache@v3
+      id: cache-bin-poetry
+      name: Cache Poetry binary - Python ${{ inputs.python-version }}
+      env:
+        SEGMENT_DOWNLOAD_TIMEOUT_MIN: "1"
+      with:
+        path: |
+          /opt/pipx/venvs/poetry
+        # This step caches the poetry installation, so make sure it's keyed on the poetry version as well.
+        key: bin-poetry-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-${{ inputs.poetry-version }}
+
+    - name: Refresh shell hashtable and fixup softlinks
+      if: steps.cache-bin-poetry.outputs.cache-hit == 'true'
+      shell: bash
+      env:
+        POETRY_VERSION: ${{ inputs.poetry-version }}
+        PYTHON_VERSION: ${{ inputs.python-version }}
+      run: |
+        set -eux
+
+        # Refresh the shell hashtable, to ensure correct `which` output.
+        hash -r
+
+        # `actions/cache@v3` doesn't always seem able to correctly unpack softlinks.
+        # Delete and recreate the softlinks pipx expects to have.
+        rm /opt/pipx/venvs/poetry/bin/python
+        cd /opt/pipx/venvs/poetry/bin
+        ln -s "$(which "python$PYTHON_VERSION")" python
+        chmod +x python
+        cd /opt/pipx_bin/
+        ln -s /opt/pipx/venvs/poetry/bin/poetry poetry
+        chmod +x poetry
+
+        # Ensure everything got set up correctly.
+        /opt/pipx/venvs/poetry/bin/python --version
+        /opt/pipx_bin/poetry --version
+
+    - name: Install poetry
+      if: steps.cache-bin-poetry.outputs.cache-hit != 'true'
+      shell: bash
+      env:
+        POETRY_VERSION: ${{ inputs.poetry-version }}
+        PYTHON_VERSION: ${{ inputs.python-version }}
+      run: pipx install "poetry==$POETRY_VERSION" --python "python$PYTHON_VERSION" --verbose
+
+    - name: Restore pip and poetry cached dependencies
+      uses: actions/cache@v3
+      env:
+        SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4"
+        WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }}
+      with:
+        path: |
+          ~/.cache/pip
+          ~/.cache/pypoetry/virtualenvs
+          ~/.cache/pypoetry/cache
+          ~/.cache/pypoetry/artifacts
+          ${{ env.WORKDIR }}/.venv
+        key: py-deps-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-poetry-${{ inputs.poetry-version }}-${{ inputs.cache-key }}-${{ hashFiles(format('{0}/**/poetry.lock', env.WORKDIR)) }}

.github/workflows/_lint.yml

@@ -0,0 +1,83 @@
+name: lint
+
+on:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+
+env:
+  POETRY_VERSION: "1.6.1"
+  WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      # This number is set "by eye": we want it to be big enough
+      # so that it's bigger than the number of commits in any reasonable PR,
+      # and also as small as possible since increasing the number makes
+      # the initial `git fetch` slower.
+      FETCH_DEPTH: 50
+    strategy:
+      matrix:
+        # Only lint on the min and max supported Python versions.
+        # It's extremely unlikely that there's a lint issue on any version in between
+        # that doesn't show up on the min or max versions.
+        #
+        # GitHub rate-limits how many jobs can be running at any one time.
+        # Starting new jobs is also relatively slow,
+        # so linting on fewer versions makes CI faster.
+        python-version:
+          - "3.8"
+          - "3.11"
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
+          cache-key: lint-with-extras
+
+      - name: Check Poetry File
+        shell: bash
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          poetry check
+
+      - name: Check lock file
+        shell: bash
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          poetry lock --check
+
+      - name: Install dependencies
+        # Also installs dev/lint/test/typing dependencies, to ensure we have
+        # type hints for as many of our libraries as possible.
+        # This helps catch errors that require dependencies to be spotted, for example:
+        # https://github.com/langchain-ai/langchain/pull/10249/files#diff-935185cd488d015f026dcd9e19616ff62863e8cde8c0bee70318d3ccbca98341
+        #
+        # If you change this configuration, make sure to change the `cache-key`
+        # in the `poetry_setup` action above to stop using the old cache.
+        # It doesn't matter how you change it, any change will cause a cache-bust.
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          poetry install --with dev,lint,test,typing
+
+      - name: Get .mypy_cache to speed up mypy
+        uses: actions/cache@v3
+        env:
+          SEGMENT_DOWNLOAD_TIMEOUT_MIN: "2"
+        with:
+          path: |
+            ${{ env.WORKDIR }}/.mypy_cache
+          key: mypy-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', env.WORKDIR)) }}
+
+      - name: Analysing the code with our lint
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          make lint

.github/workflows/_pydantic_compatibility.yml

@@ -0,0 +1,94 @@
+name: pydantic v1/v2 compatibility
+
+on:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  build:
+    timeout-minutes: 5
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+    name: Pydantic v1/v2 compatibility - Python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
+          cache-key: pydantic-cross-compat
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install
+
+      - name: Install the opposite major version of pydantic
+        # If normal tests use pydantic v1, here we'll use v2, and vice versa.
+        shell: bash
+        run: |
+          # Determine the major part of pydantic version
+          REGULAR_VERSION=$(poetry run python -c "import pydantic; print(pydantic.__version__)" | cut -d. -f1)
+
+          if [[ "$REGULAR_VERSION" == "1" ]]; then
+            PYDANTIC_DEP=">=2.1,<3"
+            TEST_WITH_VERSION="2"
+          elif [[ "$REGULAR_VERSION" == "2" ]]; then
+            PYDANTIC_DEP="<2"
+            TEST_WITH_VERSION="1"
+          else
+            echo "Unexpected pydantic major version '$REGULAR_VERSION', cannot determine which version to use for cross-compatibility test."
+            exit 1
+          fi
+
+          # Install via `pip` instead of `poetry add` to avoid changing lockfile,
+          # which would prevent caching from working: the cache would get saved
+          # to a different key than where it gets loaded from.
+          poetry run pip install "pydantic${PYDANTIC_DEP}"
+
+          # Ensure that the correct pydantic is installed now.
+          echo "Checking pydantic version... Expecting ${TEST_WITH_VERSION}"
+
+          # Determine the major part of pydantic version
+          CURRENT_VERSION=$(poetry run python -c "import pydantic; print(pydantic.__version__)" | cut -d. -f1)
+
+          # Check that the major part of pydantic version is as expected, if not
+          # raise an error
+          if [[ "$CURRENT_VERSION" != "$TEST_WITH_VERSION" ]]; then
+            echo "Error: expected pydantic version ${CURRENT_VERSION} to have been installed, but found: ${TEST_WITH_VERSION}"
+            exit 1
+          fi
+          echo "Found pydantic version ${CURRENT_VERSION}, as expected"
+      - name: Run pydantic compatibility tests
+        shell: bash
+        run: make test
+
+      - name: Ensure the tests did not create any additional files
+        shell: bash
+        run: |
+          set -eu
+
+          STATUS="$(git status)"
+          echo "$STATUS"
+
+          # grep will exit non-zero if the target message isn't found,
+          # and `set -e` above will cause the step to fail.
+          echo "$STATUS" | grep 'nothing to commit, working tree clean'

.github/workflows/_release.yml

@@ -0,0 +1,63 @@
+name: release
+
+on:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  if_release:
+    # Disallow publishing from branches that aren't `main`.
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    permissions:
+      # This permission is used for trusted publishing:
+      # https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/
+      #
+      # Trusted publishing has to also be configured on PyPI for each package:
+      # https://docs.pypi.org/trusted-publishers/adding-a-publisher/
+      id-token: write
+
+      # This permission is needed by `ncipollo/release-action` to create the GitHub release.
+      contents: write
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: "3.10"
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
+          cache-key: release
+
+      - name: Build project for distribution
+        run: poetry build
+      - name: Check Version
+        id: check-version
+        run: |
+          echo version=$(poetry version --short) >> $GITHUB_OUTPUT
+      - name: Create Release
+        uses: ncipollo/release-action@v1
+        with:
+          artifacts: "dist/*"
+          token: ${{ secrets.GITHUB_TOKEN }}
+          draft: false
+          generateReleaseNotes: true
+          tag: v${{ steps.check-version.outputs.version }}
+          commit: main
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: ${{ inputs.working-directory }}/dist/
+          verbose: true
+          print-hash: true

.github/workflows/_test.yml

@@ -0,0 +1,57 @@
+name: test
+
+on:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  build:
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+    name: Python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
+          cache-key: core
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install
+
+      - name: Run core tests
+        shell: bash
+        run: make test
+
+      - name: Ensure the tests did not create any additional files
+        shell: bash
+        run: |
+          set -eu
+
+          STATUS="$(git status)"
+          echo "$STATUS"
+
+          # grep will exit non-zero if the target message isn't found,
+          # and `set -e` above will cause the step to fail.
+          echo "$STATUS" | grep 'nothing to commit, working tree clean'

.github/workflows/ci.yml

@@ -0,0 +1,84 @@
+---
+name: Run CI Tests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    paths-ignore:
+      - 'README.md'
+  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
+
+# If another push to the same PR or branch happens while this workflow is still running,
+# cancel the earlier run in favor of the next run.
+#
+# There's no point in testing an outdated version of the code. GitHub only allows
+# a limited number of job runners to be active at the same time, so it's better to cancel
+# pointless jobs early so that more useful jobs can run sooner.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  POETRY_VERSION: "1.5.1"
+  WORKDIR: "."
+
+jobs:
+  lint:
+    uses:
+      ./.github/workflows/_lint.yml
+    with:
+      working-directory: .
+    secrets: inherit
+
+  pydantic-compatibility:
+    uses:
+      ./.github/workflows/_pydantic_compatibility.yml
+    with:
+      working-directory: .
+    secrets: inherit
+  test:
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ env.WORKDIR }}
+    strategy:
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+    name: Python ${{ matrix.python-version }} tests
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: .
+          cache-key: benchmarks-all
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running tests, installing dependencies with poetry..."
+          poetry install --with test,lint,typing
+
+      - name: Run tests
+        run: make test
+
+      - name: Ensure the tests did not create any additional files
+        shell: bash
+        run: |
+          set -eu
+
+          STATUS="$(git status)"
+          echo "$STATUS"
+
+          # grep will exit non-zero if the target message isn't found,
+          # and `set -e` above will cause the step to fail.
+          echo "$STATUS" | grep 'nothing to commit, working tree clean'

.github/workflows/release.yml

@@ -0,0 +1,13 @@
+---
+name: Publish Package to PyPi
+
+on:
+  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
+
+jobs:
+  release:
+    uses:
+      ./.github/workflows/_release.yml
+    with:
+      working-directory: .
+    secrets: inherit

.gitignore

@@ -0,0 +1,162 @@
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+

Makefile

@@ -0,0 +1,62 @@
+.PHONY: all lint format test help
+
+# Default target executed when no arguments are given to make.
+all: help
+
+######################
+# TESTING AND COVERAGE
+######################
+
+# Define a variable for the test file path.
+TEST_FILE ?= tests/unit_tests/
+
+test:
+	poetry run pytest --disable-socket --allow-unix-socket $(TEST_FILE)
+
+test_watch:
+	poetry run ptw . -- $(TEST_FILE)
+
+build_docs:
+	poetry run sphinx-build "./docs/source" "./docs/build"
+
+clean_docs:
+	rm -rf ./docs/build
+
+######################
+# LINTING AND FORMATTING
+######################
+
+# Define a variable for Python and notebook files.
+lint format: PYTHON_FILES=.
+lint_diff format_diff: PYTHON_FILES=$(shell git diff --relative=. --name-only --diff-filter=d master | grep -E '\.py$$|\.ipynb$$')
+
+lint lint_diff:
+	[ "$(PYTHON_FILES)" = "" ] ||	poetry run ruff format $(PYTHON_FILES) --diff
+	# [ "$(PYTHON_FILES)" = "" ] || poetry run mypy $(PYTHON_FILES)
+
+format format_diff:
+	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff format $(PYTHON_FILES)
+	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff --select I --fix $(PYTHON_FILES)
+
+spell_check:
+	poetry run codespell --toml pyproject.toml
+
+spell_fix:
+	poetry run codespell --toml pyproject.toml -w
+
+######################
+# HELP
+######################
+
+help:
+	@echo '===================='
+	@echo '-- LINTING --'
+	@echo 'format                       - run code formatters'
+	@echo 'lint                         - run linters'
+	@echo 'spell_check                 	- run codespell on the project'
+	@echo 'spell_fix                		- run codespell on the project and fix the errors'
+	@echo '-- TESTS --'
+	@echo 'coverage                     - run unit tests and generate coverage report'
+	@echo 'test                         - run unit tests'
+	@echo 'test TEST_FILE=<test_file>   - run all tests in file'
+	@echo '-- DOCUMENTATION tasks are from the top-level Makefile --'

README.md

@@ -1,4 +1,4 @@
-# LangChain Benchmarks
+# 🦜💪 LangChain Benchmarks
 
 This repository shows how we benchmark some of our more popular chains and agents.
 The benchmarks are organized by end-to-end use cases.
@@ -13,4 +13,6 @@ We have several goals in open sourcing this:
 
 We currently include the following tasks:
 - [CSV Question Answering](csv-qa)
-- [Extraction](extraction)
+- [Extraction](extraction)
+- [Q&A over the LangChain docs](langchain-docs-benchmarking)
+- [Meta-evaluation of 'correctness' evaluators](meta-evals)

csv-qa/custom_agent.py

@@ -1,22 +1,25 @@
-from langchain.agents import OpenAIFunctionsAgent, AgentExecutor
-from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
-from langchain.tools import PythonAstREPLTool
 import pandas as pd
+from langchain.agents import AgentExecutor, OpenAIFunctionsAgent
+from langchain.agents.agent_toolkits.conversational_retrieval.tool import (
+    create_retriever_tool,
+)
 from langchain.chat_models import ChatOpenAI
-from langsmith import Client
-from langchain.smith import RunEvalConfig, run_on_dataset
-from pydantic import BaseModel, Field
 from langchain.embeddings import OpenAIEmbeddings
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain.smith import RunEvalConfig, run_on_dataset
+from langchain.tools import PythonAstREPLTool
 from langchain.vectorstores import FAISS
-from langchain.agents.agent_toolkits.conversational_retrieval.tool import create_retriever_tool
+from langsmith import Client
+from pydantic import BaseModel, Field
 
-
-pd.set_option('display.max_rows', 20)
-pd.set_option('display.max_columns', 20)
+pd.set_option("display.max_rows", 20)
+pd.set_option("display.max_columns", 20)
 
 embedding_model = OpenAIEmbeddings()
 vectorstore = FAISS.load_local("titanic_data", embedding_model)
-retriever_tool = create_retriever_tool(vectorstore.as_retriever(), "person_name_search", "Search for a person by name")
+retriever_tool = create_retriever_tool(
+    vectorstore.as_retriever(), "person_name_search", "Search for a person by name"
+)
 
 
 TEMPLATE = """You are working with a pandas dataframe in Python. The name of the dataframe is `df`.
@@ -42,7 +45,6 @@ For example:
 """
 
 
-
 class PythonInputs(BaseModel):
     query: str = Field(description="code snippet to run")
 
@@ -51,27 +53,33 @@ if __name__ == "__main__":
     df = pd.read_csv("titanic.csv")
     template = TEMPLATE.format(dhead=df.head().to_markdown())
 
-    prompt = ChatPromptTemplate.from_messages([
-        ("system", template),
-        MessagesPlaceholder(variable_name="agent_scratchpad"),
-        ("human", "{input}")
-    ])
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", template),
+            MessagesPlaceholder(variable_name="agent_scratchpad"),
+            ("human", "{input}"),
+        ]
+    )
 
     def get_chain():
-        repl = PythonAstREPLTool(locals={"df": df}, name="python_repl",
-                                 description="Runs code and returns the output of the final line",
-                                 args_schema=PythonInputs)
+        repl = PythonAstREPLTool(
+            locals={"df": df},
+            name="python_repl",
+            description="Runs code and returns the output of the final line",
+            args_schema=PythonInputs,
+        )
         tools = [repl, retriever_tool]
-        agent = OpenAIFunctionsAgent(llm=ChatOpenAI(temperature=0, model="gpt-4"), prompt=prompt, tools=tools)
-        agent_executor = AgentExecutor(agent=agent, tools=tools, max_iterations=5, early_stopping_method="generate")
+        agent = OpenAIFunctionsAgent(
+            llm=ChatOpenAI(temperature=0, model="gpt-4"), prompt=prompt, tools=tools
+        )
+        agent_executor = AgentExecutor(
+            agent=agent, tools=tools, max_iterations=5, early_stopping_method="generate"
+        )
         return agent_executor
 
-
     client = Client()
     eval_config = RunEvalConfig(
-        evaluators=[
-            "qa"
-        ],
+        evaluators=["qa"],
     )
     chain_results = run_on_dataset(
         client,

csv-qa/pandas_agent_gpt_35.py

@@ -1,9 +1,9 @@
 import pandas as pd
-from langchain.chat_models import ChatOpenAI
 from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
 from langchain.agents.agent_types import AgentType
-from langsmith import Client
+from langchain.chat_models import ChatOpenAI
 from langchain.smith import RunEvalConfig, run_on_dataset
+from langsmith import Client
 
 if __name__ == "__main__":
     df = pd.read_csv("titanic.csv")
@@ -18,20 +18,17 @@ if __name__ == "__main__":
             df,
             agent_type=AgentType.OPENAI_FUNCTIONS,
             agent_executor_kwargs=agent_executor_kwargs,
-            max_iterations=5
+            max_iterations=5,
         )
         return agent
 
-
     client = Client()
     eval_config = RunEvalConfig(
-        evaluators=[
-            "qa"
-        ],
+        evaluators=["qa"],
     )
     chain_results = run_on_dataset(
         client,
         dataset_name="Titanic CSV Data",
         llm_or_chain_factory=get_chain,
         evaluation=eval_config,
-    )
+    )

csv-qa/pandas_agent_gpt_4.py

@@ -1,14 +1,13 @@
 import pandas as pd
-from langchain.chat_models import ChatOpenAI
 from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
 from langchain.agents.agent_types import AgentType
-from langsmith import Client
+from langchain.chat_models import ChatOpenAI
 from langchain.smith import RunEvalConfig, run_on_dataset
+from langsmith import Client
 
 if __name__ == "__main__":
     df = pd.read_csv("titanic.csv")
 
-
     def get_chain():
         llm = ChatOpenAI(temperature=0, model="gpt-4")
         agent_executor_kwargs = {
@@ -19,20 +18,17 @@ if __name__ == "__main__":
             df,
             agent_type=AgentType.OPENAI_FUNCTIONS,
             agent_executor_kwargs=agent_executor_kwargs,
-            max_iterations=5
+            max_iterations=5,
         )
         return agent
 
-
     client = Client()
     eval_config = RunEvalConfig(
-        evaluators=[
-            "qa"
-        ],
+        evaluators=["qa"],
     )
     chain_results = run_on_dataset(
         client,
         dataset_name="Titanic CSV Data",
         llm_or_chain_factory=get_chain,
         evaluation=eval_config,
-    )
+    )

csv-qa/pandas_agent_instruct.py

@@ -0,0 +1,82 @@
+import pandas as pd
+from langchain.agents import AgentExecutor, ZeroShotAgent
+from langchain.agents.agent_toolkits.conversational_retrieval.tool import (
+    create_retriever_tool,
+)
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.llms import OpenAI
+from langchain.prompts import PromptTemplate
+from langchain.smith import RunEvalConfig, run_on_dataset
+from langchain.tools import PythonAstREPLTool
+from langchain.vectorstores import FAISS
+from langsmith import Client
+from pydantic import BaseModel, Field
+
+pd.set_option("display.max_rows", 20)
+pd.set_option("display.max_columns", 20)
+
+embedding_model = OpenAIEmbeddings()
+vectorstore = FAISS.load_local("titanic_data", embedding_model)
+retriever_tool = create_retriever_tool(
+    vectorstore.as_retriever(), "person_name_search", "Search for a person by name"
+)
+
+
+TEMPLATE = """You are working with a pandas dataframe in Python. The name of the dataframe is `df`.
+It is important to understand the attributes of the dataframe before working with it. This is the result of running `df.head().to_markdown()`
+
+<df>
+{dhead}
+</df>
+
+You are not meant to use only these rows to answer questions - they are meant as a way of telling you about the shape and schema of the dataframe.
+You also do not have use only the information here to answer questions - you can run intermediate queries to do exporatory data analysis to give you more information as needed.
+
+You have a tool called `person_name_search` through which you can lookup a person by name and find the records corresponding to people with similar name as the query.
+You should only really use this if your search term contains a persons name. Otherwise, try to solve it with code.
+
+For example:
+
+<question>How old is Jane?</question>
+<logic>Use `person_name_search` since you can use the query `Jane`</logic>
+
+<question>Who has id 320</question>
+<logic>Use `python_repl` since even though the question is about a person, you don't know their name so you can't include it.</logic>"""
+
+
+class PythonInputs(BaseModel):
+    query: str = Field(description="code snippet to run")
+
+
+if __name__ == "__main__":
+    df = pd.read_csv("titanic.csv")
+    template = TEMPLATE.format(dhead=df.head().to_markdown())
+
+    def get_chain():
+        repl = PythonAstREPLTool(
+            locals={"df": df},
+            name="python_repl",
+            description="Runs code and returns the output of the final line",
+            args_schema=PythonInputs,
+        )
+        tools = [repl, retriever_tool]
+        agent = ZeroShotAgent.from_llm_and_tools(
+            llm=OpenAI(temperature=0, model="gpt-3.5-turbo-instruct"),
+            tools=tools,
+            prefix=template,
+        )
+        agent_executor = AgentExecutor(
+            agent=agent, tools=tools, max_iterations=5, early_stopping_method="generate"
+        )
+        return agent_executor
+
+    client = Client()
+    eval_config = RunEvalConfig(
+        evaluators=["qa"],
+    )
+    chain_results = run_on_dataset(
+        client,
+        dataset_name="Titanic CSV Data",
+        llm_or_chain_factory=get_chain,
+        evaluation=eval_config,
+    )

csv-qa/pandas_ai.py

@@ -1,44 +1,47 @@
 import pandas as pd
-from langchain.chat_models import ChatOpenAI
 from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
 from langchain.agents.agent_types import AgentType
-from langsmith import Client
-from langchain.smith import RunEvalConfig, run_on_dataset
-import pandas as pd
-from pandasai import PandasAI
-
 from langchain.chat_models import ChatOpenAI
 from langchain.prompts import ChatPromptTemplate
 from langchain.schema.output_parser import StrOutputParser
+from langchain.smith import RunEvalConfig, run_on_dataset
+from langsmith import Client
+from pandasai import PandasAI
 
 if __name__ == "__main__":
     df = pd.read_csv("titanic.csv")
 
     pandas_ai = PandasAI(ChatOpenAI(temperature=0, model="gpt-4"), enable_cache=False)
-    prompt = ChatPromptTemplate.from_messages([
-        ("system",
-         "Answer the users question about some data. A data scientist will run some code and the results will be returned to you to use in your answer"),
-        ("human", "Question: {input}"),
-        ("human", "Data Scientist Result: {result}"),
-    ])
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            (
+                "system",
+                "Answer the users question about some data. A data scientist will run some code and the results will be returned to you to use in your answer",
+            ),
+            ("human", "Question: {input}"),
+            ("human", "Data Scientist Result: {result}"),
+        ]
+    )
 
     def get_chain():
-        chain = {
-                    "input": lambda x: x["input_question"],
-                    "result": lambda x: pandas_ai(df, prompt=x['input_question'])
-                } | prompt | ChatOpenAI(temperature=0, model="gpt-4") | StrOutputParser()
+        chain = (
+            {
+                "input": lambda x: x["input_question"],
+                "result": lambda x: pandas_ai(df, prompt=x["input_question"]),
+            }
+            | prompt
+            | ChatOpenAI(temperature=0, model="gpt-4")
+            | StrOutputParser()
+        )
         return chain
 
-
     client = Client()
     eval_config = RunEvalConfig(
-        evaluators=[
-            "qa"
-        ],
+        evaluators=["qa"],
     )
     chain_results = run_on_dataset(
         client,
         dataset_name="Titanic CSV Data",
         llm_or_chain_factory=get_chain,
         evaluation=eval_config,
-    )
+    )

csv-qa/streamlit_app.py

@@ -1,10 +1,10 @@
 import pandas as pd
 import streamlit as st
-from langchain.chat_models import ChatOpenAI
 from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
 from langchain.agents.agent_types import AgentType
+from langchain.chat_models import ChatOpenAI
 
-df = pd.read_csv('titanic.csv')
+df = pd.read_csv("titanic.csv")
 
 
 llm = ChatOpenAI(temperature=0)
@@ -12,31 +12,36 @@ agent = create_pandas_dataframe_agent(llm, df, agent_type=AgentType.OPENAI_FUNCT
 
 
 from langsmith import Client
+
 client = Client()
+
+
 def send_feedback(run_id, score):
     client.create_feedback(run_id, "user_score", score=score)
 
-st.set_page_config(page_title='🦜🔗 Ask the CSV App')
-st.title('🦜🔗 Ask the CSV App')
-st.info("Most 'question answering' applications run over unstructured text data. But a lot of the data in the world is tabular data! This is an attempt to create an application using [LangChain](https://github.com/langchain-ai/langchain) to let you ask questions of data in tabular format. For this demo application, we will use the Titanic Dataset. Please explore it [here](https://github.com/datasciencedojo/datasets/blob/master/titanic.csv) to get a sense for what questions you can ask. Please leave feedback on well the question is answered, and we will use that improve the application!")
 
-query_text = st.text_input('Enter your question:', placeholder = 'Who was in cabin C128?')
+st.set_page_config(page_title="🦜🔗 Ask the CSV App")
+st.title("🦜🔗 Ask the CSV App")
+st.info(
+    "Most 'question answering' applications run over unstructured text data. But a lot of the data in the world is tabular data! This is an attempt to create an application using [LangChain](https://github.com/langchain-ai/langchain) to let you ask questions of data in tabular format. For this demo application, we will use the Titanic Dataset. Please explore it [here](https://github.com/datasciencedojo/datasets/blob/master/titanic.csv) to get a sense for what questions you can ask. Please leave feedback on well the question is answered, and we will use that improve the application!"
+)
+
+query_text = st.text_input("Enter your question:", placeholder="Who was in cabin C128?")
 # Form input and query
 result = None
-with st.form('myform', clear_on_submit=True):
-	submitted = st.form_submit_button('Submit')
-	if submitted:
-		with st.spinner('Calculating...'):
-			response = agent({"input": query_text}, include_run_info=True)
-			result = response["output"]
-			run_id = response["__run"].run_id
+with st.form("myform", clear_on_submit=True):
+    submitted = st.form_submit_button("Submit")
+    if submitted:
+        with st.spinner("Calculating..."):
+            response = agent({"input": query_text}, include_run_info=True)
+            result = response["output"]
+            run_id = response["__run"].run_id
 if result is not None:
-	st.info(result)
-	col_blank, col_text, col1, col2 = st.columns([10, 2,1,1])
-	with col_text:
-		st.text("Feedback:")
-	with col1:
-		st.button("👍", on_click=send_feedback, args=(run_id, 1))
-	with col2:
-		st.button("👎", on_click=send_feedback, args=(run_id, 0))
-
+    st.info(result)
+    col_blank, col_text, col1, col2 = st.columns([10, 2, 1, 1])
+    with col_text:
+        st.text("Feedback:")
+    with col1:
+        st.button("👍", on_click=send_feedback, args=(run_id, 1))
+    with col2:
+        st.button("👎", on_click=send_feedback, args=(run_id, 0))

csv-qa/upload_data.py

@@ -8,5 +8,5 @@ if __name__ == "__main__":
         output_keys=["output_text"],
         name="Titanic CSV Data",
         description="QA over titanic data",
-        data_type = "kv"
+        data_type="kv",
     )

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/source/conf.py

@@ -0,0 +1,105 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+import pathlib
+import sys
+from typing import List
+
+import toml
+
+ROOT_FOLDER = str(pathlib.Path(__file__).parent.parent.parent)
+
+# Add the project root to the path
+sys.path.insert(0, ROOT_FOLDER)
+
+with open("../../pyproject.toml") as f:
+    data = toml.load(f)
+
+project = "LangChain Benchmarks"
+copyright = "2023, Langchain AI"
+author = "Langchain AI"
+
+version = data["tool"]["poetry"]["version"]
+release = version
+
+html_title = project + " " + version
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autodoc.typehints",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.viewcode",
+    "myst_nb",
+    "sphinx_copybutton",
+    "IPython.sphinxext.ipython_console_highlighting",
+]
+source_suffix = [".ipynb", ".html", ".md", ".rst"]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns: List[str] = []
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_book_theme"
+
+html_theme_options = {
+    "path_to_docs": "docs",
+    "repository_url": "https://github.com/langchain-ai/langchain-benchmarks",
+    "home_page_in_toc": True,
+    "show_navbar_depth": 2,
+    "use_sidenotes": True,
+    "use_repository_button": True,
+}
+
+html_context = {
+    "display_github": True,  # Integrate GitHub
+    "github_user": "langchain-ai",  # Username
+    "github_repo": "langchain-benchmarks",  # Repo name
+    "github_version": "main",  # Version
+    "conf_py_path": "/docs/",  # Path in the checkout to the docs root
+}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+
+# These paths are either relative to html_static_path
+# or fully qualified paths (eg. https://...)
+html_css_files = [
+    "css/custom.css",
+]
+
+nb_execution_mode = "off"
+autosummary_generate = True

docs/source/index.md

@@ -0,0 +1,9 @@
+# Introduction
+
+
+```{toctree}
+:maxdepth: 2
+:caption: Contents
+
+./notebooks/datasets
+```

docs/source/notebooks/datasets.ipynb

@@ -0,0 +1,197 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "033684fb-65b2-4586-a959-68c614741ca2",
+   "metadata": {},
+   "source": [
+    "# Datasets\n",
+    "\n",
+    "Here, we'll see how to work with LangSmith datasets."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "6d272fbf-710e-4a49-a0da-67e010541905",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain_benchmarks import clone_public_dataset, download_public_dataset"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "18ee0f96-e5c4-4ae9-aebf-7d8b88c51662",
+   "metadata": {},
+   "source": [
+    "Let's first download the dataset to the local file system"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "58b94f6d-0c91-4361-9b22-f758ffaa150a",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Fetching examples...\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "5a2fad8c0c3549ec96a3b38fe8a002b0",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/21 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Done fetching examples.\n"
+     ]
+    }
+   ],
+   "source": [
+    "download_public_dataset('https://api.smith.langchain.com/public/e95d45da-aaa3-44b3-ba2b-7c15ff6e46f5/examples')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "841db832-b0d3-4fd1-8531-1154ec9b3caa",
+   "metadata": {},
+   "source": [
+    "we can take a look at the first two examples"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "664e90fc-af84-4c5f-a3dd-5d9ffe649650",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[\n",
+      "  {\n",
+      "    \"created_at\": \"2023-11-15T15:26:53.511629\",\n",
+      "    \"dataset_id\": \"9f73165c-d333-4d14-8f59-bd7eede5db08\",\n",
+      "    \"id\": \"0703a989-2693-4039-a1f6-7281fc1b4cb0\",\n",
+      "    \"inputs\": {\n",
+      "      \"question\": \"do bob and alice live in the same city?\"\n",
+      "    },\n",
+      "    \"modified_at\": \"2023-11-15T15:26:53.511629\",\n",
+      "    \"outputs\": {\n",
+      "      \"expected_steps\": [\n",
+      "        \"find_users_by_name\",\n",
+      "        \"get_user_location\",\n",
+      "        \"get_city_for_location\",\n",
+      "        \"get_user_location\",\n",
+      "        \"get_city_for_location\"\n",
+      "      ],\n",
+      "      \"order_matters\": false,\n",
+      "      \"reference\": \"no\"\n",
+      "    },\n",
+      "    \"runs\": []\n",
+      "  },\n",
+      "  {\n",
+      "    \"created_at\": \"2023-11-15T15:26:53.491359\",\n",
+      "    \"dataset_id\": \"9f73165c-d333-4d14-8f59-bd7eede5db08\",\n",
+      "    \"id\": \"b258b95a-9524-4da7-b758-c5481109322d\",\n",
+      "    \"inputs\": {\n",
+      "      \"question\": \"Is it likely that Donna is outside with an umbrella at this time?\"\n",
+      "    },\n",
+      "    \"modified_at\": \"2023-11-15T15:26:53.491359\",\n",
+      "    \"outputs\": {\n",
+      "      \"expected_steps\": [\n",
+      "        \"find_users_by_name\",\n",
+      "        \"get_user_location\",\n",
+      "        \"get_current_time_for_location\",\n",
+      "        \"get_current_weather_for_location\"\n",
+      "      ],\n",
+      "      \"order_matters\": false,\n",
+      "      \"reference\": \"yes\"\n",
+      "    },\n",
+      "    \"runs\": []\n",
+      "  }\n",
+      "]\n"
+     ]
+    }
+   ],
+   "source": [
+    "import json\n",
+    "with open('./e95d45da-aaa3-44b3-ba2b-7c15ff6e46f5.json', 'r', encoding='utf-8') as f:\n",
+    "    print(json.dumps(json.load(f)[:2], indent=2, sort_keys=True))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2c6cf01f-466b-406d-b4c7-2395747780fd",
+   "metadata": {},
+   "source": [
+    "We can also clone the dataset to our local tenant"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "e4dea4df-2f1c-436b-a71c-49ffb2295ccc",
+   "metadata": {},
+   "source": [
+    "Executing this command will clone the dataset to your local tenant. \n",
+    "For this to work you must have a langsmith account set up."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "18d0b905-2a6a-4752-a7cb-8653bd9049e3",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "clone_public_dataset('https://api.smith.langchain.com/public/e95d45da-aaa3-44b3-ba2b-7c15ff6e46f5/examples', dataset_name='Agent Dataset')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

extraction/streamlit_app.py

@@ -1,18 +1,23 @@
 import streamlit as st
-from langsmith import Client
-from langchain.chat_models import ChatOpenAI
 from langchain.chains import create_extraction_chain
+from langchain.chat_models import ChatOpenAI
+from langsmith import Client
 
-st.set_page_config(page_title='🦜🔗 Text-to-graph extraction')
+st.set_page_config(page_title="🦜🔗 Text-to-graph extraction")
 client = Client()
+
+
 def send_feedback(run_id, score):
     client.create_feedback(run_id, "user_score", score=score)
 
-st.title('🦜🔗 Text-to-graph playground')
-st.info("This playground explores the use of [OpenAI functions](https://openai.com/blog/function-calling-and-other-api-updates) and [LangChain](https://github.com/langchain-ai/langchain) to build knowledge graphs from user-input text. It breaks down the user input text into knowledge graph triples of subject (primary entities or concepts in a sentence), predicate (actions or relationships that connect subjects to objects), and object (entities or concepts that interact with or are acted upon by the subjects).")
+
+st.title("🦜🔗 Text-to-graph playground")
+st.info(
+    "This playground explores the use of [OpenAI functions](https://openai.com/blog/function-calling-and-other-api-updates) and [LangChain](https://github.com/langchain-ai/langchain) to build knowledge graphs from user-input text. It breaks down the user input text into knowledge graph triples of subject (primary entities or concepts in a sentence), predicate (actions or relationships that connect subjects to objects), and object (entities or concepts that interact with or are acted upon by the subjects)."
+)
 
 # Input text (optional default)
-oppenheimer_text=''''Julius Robert Oppenheimer, often known as Robert or "Oppie", is heralded as the father of the atomic bomb. Emerging from a non-practicing Jewish family in New York, he made several breakthroughs, such as the early black hole theory, before the monumental Manhattan Project. His wife, Katherine “Kitty” Oppenheimer, was a German-born woman with a complex past, including connections to the Communist Party. Oppenheimer\'s journey was beset by political adversaries, notably Lewis Strauss, chairman of the U.S. Atomic Energy Commission, and William Borden, an executive director with hawkish nuclear ambitions. These tensions culminated in the famous 1954 security hearing. Influential figures like lieutenant general Leslie Groves, who had also overseen the Pentagon\'s creation, stood by Oppenheimer\'s side, having earlier chosen him for the Manhattan Project and the Los Alamos location. Intimate relationships, like that with Jean Tatlock, a Communist and the possible muse behind the Trinity test\'s name, and colleagues like Frank, Oppenheimer\'s physicist brother, intertwined with his professional life. Scientists such as Ernest Lawrence, Edward Teller, David Hill, Richard Feynman, and Hans Bethe were some of Oppenheimer\'s contemporaries, each contributing to and contesting the atomic age\'s directions. Boris Pash\'s investigations, and the perspectives of figures like Leo Szilard, Niels Bohr, Harry Truman, and others, framed the broader sociopolitical context. Meanwhile, individuals like Robert Serber, Enrico Fermi, Albert Einstein, and Isidor Isaac Rabi, among many others, each played their parts in this narrative, from naming the atomic bombs to pivotal scientific contributions and advisory roles. All these figures, together with the backdrop of World War II, McCarthyism, and the dawn of the nuclear age, presented a complex mosaic of ambitions, loyalties, betrayals, and ideologies.oppenheimer_short.txt'''
+oppenheimer_text = """'Julius Robert Oppenheimer, often known as Robert or "Oppie", is heralded as the father of the atomic bomb. Emerging from a non-practicing Jewish family in New York, he made several breakthroughs, such as the early black hole theory, before the monumental Manhattan Project. His wife, Katherine “Kitty” Oppenheimer, was a German-born woman with a complex past, including connections to the Communist Party. Oppenheimer\'s journey was beset by political adversaries, notably Lewis Strauss, chairman of the U.S. Atomic Energy Commission, and William Borden, an executive director with hawkish nuclear ambitions. These tensions culminated in the famous 1954 security hearing. Influential figures like lieutenant general Leslie Groves, who had also overseen the Pentagon\'s creation, stood by Oppenheimer\'s side, having earlier chosen him for the Manhattan Project and the Los Alamos location. Intimate relationships, like that with Jean Tatlock, a Communist and the possible muse behind the Trinity test\'s name, and colleagues like Frank, Oppenheimer\'s physicist brother, intertwined with his professional life. Scientists such as Ernest Lawrence, Edward Teller, David Hill, Richard Feynman, and Hans Bethe were some of Oppenheimer\'s contemporaries, each contributing to and contesting the atomic age\'s directions. Boris Pash\'s investigations, and the perspectives of figures like Leo Szilard, Niels Bohr, Harry Truman, and others, framed the broader sociopolitical context. Meanwhile, individuals like Robert Serber, Enrico Fermi, Albert Einstein, and Isidor Isaac Rabi, among many others, each played their parts in this narrative, from naming the atomic bombs to pivotal scientific contributions and advisory roles. All these figures, together with the backdrop of World War II, McCarthyism, and the dawn of the nuclear age, presented a complex mosaic of ambitions, loyalties, betrayals, and ideologies.oppenheimer_short.txt"""
 
 # Knowledge triplet schema
 default_schema = {
@@ -31,46 +36,44 @@ if len(user_input_text) > MAX_CHARS:
     st.warning(f"Text is too long. Processing only the first {MAX_CHARS} characters")
     user_input_text = user_input_text[:MAX_CHARS]
 
+
 # Output formatting of triples
 def json_to_markdown_table(json_list):
     if not json_list:
         return "No data available."
-    
+
     # Extract headers
     headers = json_list[0].keys()
     markdown_table = " | ".join(headers) + "\n"
     markdown_table += " | ".join(["---"] * len(headers)) + "\n"
-    
+
     # Extract rows
     for item in json_list:
         row = " | ".join([str(item[header]) for header in headers])
         markdown_table += row + "\n"
-    
+
     return markdown_table
 
+
 # Form input and query
 markdown_output = None
-with st.form('myform', clear_on_submit=True):
-
-    submitted = st.form_submit_button('Submit')
+with st.form("myform", clear_on_submit=True):
+    submitted = st.form_submit_button("Submit")
     if submitted:
-		
-        with st.spinner('Calculating...'):
-
+        with st.spinner("Calculating..."):
             llm = ChatOpenAI(temperature=0, model="gpt-3.5-turbo")
             chain = create_extraction_chain(default_schema, llm)
-            extraction_output = chain(user_input_text,include_run_info=True)
-            markdown_output = json_to_markdown_table(extraction_output['text'])
+            extraction_output = chain(user_input_text, include_run_info=True)
+            markdown_output = json_to_markdown_table(extraction_output["text"])
             run_id = extraction_output["__run"].run_id
-        
+
 # Feeback
 if markdown_output is not None:
     st.markdown(markdown_output)
-    col_blank, col_text, col1, col2 = st.columns([10, 2,1,1])
+    col_blank, col_text, col1, col2 = st.columns([10, 2, 1, 1])
     with col_text:
         st.text("Feedback:")
     with col1:
         st.button("👍", on_click=send_feedback, args=(run_id, 1))
     with col2:
         st.button("👎", on_click=send_feedback, args=(run_id, 0))
-	

langchain-docs-benchmarking/README.md

@@ -0,0 +1,89 @@
+# Benchmarking on LangChain Docs
+
+This directory contains code to benchmark your cognitive architecture on the public [LangChain Q&A docs evaluation benchmark](https://smith.langchain.com/public/e1bfd348-494a-4df5-899a-7c6c09233cc4/d).
+
+To one one of the existing configurations, activate your poetry environment, configure you LangSmith API key, and run the experiments.
+
+**Note:** this will benchmark chains on a _copy_ of the dataset and will not update the public leaderboard.
+
+## Running the published experiments
+
+The following steps will let you run pre-configured experiments:
+
+### 1. Install requirements
+
+```bash
+pip install poetry
+poetry shell
+poetry install
+```
+
+### 2. Configure API keys
+
+Create a [LangSmith account](https://smith.langchain.com/) and set your API key:
+
+```bash
+export LANGCHAIN_API_KEY=ls_your-api-key
+```
+
+The various cognitive architectures implemented already use Anthropic, [Fireworks.AI](https://www.fireworks.ai/), and OpenAI. Set the required API keys:
+
+```
+export OPENAI_API_KEY=your-api-key
+export ANTHROPIC_API_KEY=your-api-key
+export FIREWORKS_API_KEY=your-api-key
+```
+
+### 3. Run Experiments
+
+To run all experiments, run:
+
+```bash
+python run_experiments.py
+```
+
+If you want to only run certain experiments in the `run_experiments.py` file, use `--include` or `--exclude`
+
+Example:
+
+```bash
+python run_experiments --include mistral-7b-instruct-4k llama-v2-34b-code-instruct-w8a16
+```
+
+## Evaluating your custom cognitive architecture
+
+You can also evaluate your own custom cognitive architecture. To do so:
+
+1. Create a python file defining your architecture:
+
+```python
+# example_custom_chain.py
+
+...
+def load_runnable(config: dict) -> "Runnable":
+    # Load based on the config provided
+    return my_chain
+```
+
+2. Call `run_experiments.py` with a custom `--config my_config.json`
+
+```js
+{
+  // This specifies the path to your custom entrypoint followed by the loader function
+  "arch": "path/to/example_custom_chain.py::load_runnable",
+  "model_config": {
+    // This is passed to load_runnable() in example_custom_chain.py()
+    "chat_cls": "ChatOpenAI",
+    "model": "gpt-4"
+  },
+  "project_name": "example-custom-code" // This is the resulting test project name
+}
+```
+
+We have provided an example in [example_custom_chain.py](./packages/example/custom_example/example_custom_chain.py), which can be run by pointing `run_experiments` to the [example_custom_config.json](./example_custom_config.json) config file:
+
+```bash
+python run_experiments.py --config ./example_custom_config.json
+```
+
+Whenever you provide 1 or more `--config` files, the `--include` and `--exclude` arguments are ignored.

langchain-docs-benchmarking/app/server.py

@@ -0,0 +1,27 @@
+from chat_langchain.chain import chain
+from fastapi import FastAPI
+from openai_functions_agent import agent_executor as openai_functions_agent_chain
+
+from langserve import add_routes
+
+app = FastAPI()
+
+# Edit this to add the chain you want to add
+add_routes(
+    app,
+    chain,
+    path="/chat",
+    # include_callback_events=True, # TODO: Include when fixed
+)
+
+add_routes(app, openai_functions_agent_chain, path="/openai-functions-agent")
+
+
+def run_server(port: int = 1983):
+    import uvicorn
+
+    uvicorn.run(app, host="0.0.0.0", port=port)
+
+
+if __name__ == "__main__":
+    run_server()

langchain-docs-benchmarking/example_custom_config.json

@@ -0,0 +1,8 @@
+{
+  "arch": "packages/example/custom_example/example_custom_chain.py::create_runnable",
+  "model_config": {
+    "chat_cls": "ChatOpenAI",
+    "model": "gpt-4"
+  },
+  "project_name": "example-custom-code"
+}

langchain-docs-benchmarking/packages/anthropic-iterative-search/README.md

@@ -0,0 +1,69 @@
+
+# anthropic-iterative-search
+
+This template will create a virtual research assistant with the ability to search Wikipedia to find answers to your questions.
+
+It is heavily inspired by [this notebook](https://github.com/anthropics/anthropic-cookbook/blob/main/long_context/wikipedia-search-cookbook.ipynb).
+
+## Environment Setup
+
+Set the `ANTHROPIC_API_KEY` environment variable to access the Anthropic models.
+
+## Usage
+
+To use this package, you should first have the LangChain CLI installed:
+
+```shell
+pip install -U "langchain-cli[serve]"
+```
+
+To create a new LangChain project and install this as the only package, you can do:
+
+```shell
+langchain app new my-app --package anthropic-iterative-search
+```
+
+If you want to add this to an existing project, you can just run:
+
+```shell
+langchain app add anthropic-iterative-search
+```
+
+And add the following code to your `server.py` file:
+```python
+from anthropic_iterative_search import chain as anthropic_iterative_search_chain
+
+add_routes(app, anthropic_iterative_search_chain, path="/anthropic-iterative-search")
+```
+
+(Optional) Let's now configure LangSmith. 
+LangSmith will help us trace, monitor and debug LangChain applications. 
+LangSmith is currently in private beta, you can sign up [here](https://smith.langchain.com/). 
+If you don't have access, you can skip this section
+
+
+```shell
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_API_KEY=<your-api-key>
+export LANGCHAIN_PROJECT=<your-project>  # if not specified, defaults to "default"
+```
+
+If you are inside this directory, then you can spin up a LangServe instance directly by:
+
+```shell
+langchain serve
+```
+
+This will start the FastAPI app with a server is running locally at 
+[http://localhost:8000](http://localhost:8000)
+
+We can see all templates at [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs)
+We can access the playground at [http://127.0.0.1:8000/anthropic-iterative-search/playground](http://127.0.0.1:8000/anthropic-iterative-search/playground)  
+
+We can access the template from code with:
+
+```python
+from langserve.client import RemoteRunnable
+
+runnable = RemoteRunnable("http://localhost:8000/anthropic-iterative-search")
+```

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/__init__.py

@@ -0,0 +1,11 @@
+from langchain.schema.runnable import ConfigurableField
+
+from .chain import chain
+from .retriever_agent import executor
+
+final_chain = chain.configurable_alternatives(
+    ConfigurableField(id="chain"),
+    default_key="response",
+    # This adds a new option, with name `openai` that is equal to `ChatOpenAI()`
+    retrieve=executor,
+)

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/agent_scratchpad.py

@@ -0,0 +1,16 @@
+def _format_docs(docs):
+    result = "\n".join(
+        [
+            f'<item index="{i+1}">\n<page_content>\n{r}\n</page_content>\n</item>'
+            for i, r in enumerate(docs)
+        ]
+    )
+    return result
+
+
+def format_agent_scratchpad(intermediate_steps):
+    thoughts = ""
+    for action, observation in intermediate_steps:
+        thoughts += action.log
+        thoughts += "</search_query>" + _format_docs(observation)
+    return thoughts

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/chain.py

@@ -0,0 +1,29 @@
+from langchain.chat_models import ChatAnthropic
+from langchain.prompts import ChatPromptTemplate
+from langchain.pydantic_v1 import BaseModel
+from langchain.schema.output_parser import StrOutputParser
+from langchain.schema.runnable import RunnableLambda
+
+from .prompts import answer_prompt
+from .retriever_agent import executor
+
+prompt = ChatPromptTemplate.from_template(answer_prompt)
+
+model = ChatAnthropic(model="claude-2", temperature=0, max_tokens_to_sample=1000)
+
+chain = (
+    RunnableLambda(lambda x: {"query": x["question"]})
+    | {"query": lambda x: x["query"], "information": executor | (lambda x: x["output"])}
+    | prompt
+    | model
+    | StrOutputParser()
+)
+
+# Add typing for the inputs to be used in the playground
+
+
+class Inputs(BaseModel):
+    question: str
+
+
+chain = chain.with_types(input_type=Inputs)

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/output_parser.py

@@ -0,0 +1,37 @@
+import re
+
+from langchain.schema.agent import AgentAction, AgentFinish
+
+from .agent_scratchpad import _format_docs
+
+
+def extract_between_tags(tag: str, string: str, strip: bool = True) -> str:
+    ext_list = re.findall(f"<{tag}\s?>(.+?)</{tag}\s?>", string, re.DOTALL)
+    if strip:
+        ext_list = [e.strip() for e in ext_list]
+    if ext_list:
+        if len(ext_list) != 1:
+            raise ValueError
+        # Only return the first one
+        return ext_list[0]
+
+
+def parse_output(outputs):
+    partial_completion = outputs["partial_completion"]
+    steps = outputs["intermediate_steps"]
+    search_query = extract_between_tags(
+        "search_query", partial_completion + "</search_query>"
+    )
+    if search_query is None:
+        docs = []
+        str_output = ""
+        for action, observation in steps:
+            docs.extend(observation)
+            str_output += action.log
+            str_output += "</search_query>" + _format_docs(observation)
+        str_output += partial_completion
+        return AgentFinish({"docs": docs, "output": str_output}, log=partial_completion)
+    else:
+        return AgentAction(
+            tool="search", tool_input=search_query, log=partial_completion
+        )

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/prompts.py

@@ -0,0 +1,7 @@
+retrieval_prompt = """{retriever_description} Before beginning to research the user's question, first think for a moment inside <scratchpad> tags about what information is necessary for a well-informed answer. If the user's question is complex, you may need to decompose the query into multiple subqueries and execute them individually. Sometimes the search engine will return empty search results, or the search results may not contain the information you need. In such cases, feel free to try again with a different query. 
+
+After each call to the Search Engine Tool, reflect briefly inside <search_quality></search_quality> tags about whether you now have enough information to answer, or whether more information is needed. If you have all the relevant information, write it in <information></information> tags, WITHOUT actually answering the question. Otherwise, issue a new search.
+
+Here is the user's question: <question>{query}</question> Remind yourself to make short queries in your scratchpad as you plan out your strategy."""  # noqa: E501
+
+answer_prompt = "Here is a user query: <query>{query}</query>. Here is some relevant information: <information>{information}</information>. Please answer the question using the relevant information."  # noqa: E501

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/retriever.py

@@ -0,0 +1,17 @@
+from langchain.tools import tool
+from langchain_docs_retriever.retriever import get_retriever
+
+# This is used to tell the model how to best use the retriever.
+
+retriever_description = """You will be asked a question by a human user. You have access to the following tool to help answer the question. <tool_description> Search Engine Tool * The search engine will exclusively search over the LangChain documentation for pages similar to your query. It returns for each page its title and full page content. Use this tool if you want to get up-to-date and comprehensive information on a topic to help answer queries. Queries should be as atomic as possible -- they only need to address one part of the user's question. For example, if the user's query is "what is the color of a basketball?", your search query should be "basketball". Here's another example: if the user's question is "Who created the first neural network?", your first query should be "neural network". As you can see, these queries are quite short. Think keywords, not phrases. * At any time, you can make a call to the search engine using the following syntax: <search_query>query_word</search_query>. * You'll then get results back in <search_result> tags.</tool_description>"""  # noqa: E501
+
+retriever = get_retriever()
+
+# This should be the same as the function name below
+RETRIEVER_TOOL_NAME = "search"
+
+
+@tool
+def search(query, callbacks=None):
+    """Search the LangChain docs with the retriever."""
+    return retriever.get_relevant_documents(query, callbacks=callbacks)

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/retriever_agent.py

@@ -0,0 +1,41 @@
+from langchain.agents import AgentExecutor
+from langchain.chat_models import ChatAnthropic
+from langchain.prompts import ChatPromptTemplate
+from langchain.schema.output_parser import StrOutputParser
+from langchain.schema.runnable import RunnableMap, RunnablePassthrough
+
+from .agent_scratchpad import format_agent_scratchpad
+from .output_parser import parse_output
+from .prompts import retrieval_prompt
+from .retriever import retriever_description, search
+
+prompt = ChatPromptTemplate.from_messages(
+    [
+        ("user", retrieval_prompt),
+        ("ai", "{agent_scratchpad}"),
+    ]
+)
+prompt = prompt.partial(retriever_description=retriever_description)
+
+model = ChatAnthropic(model="claude-2", temperature=0, max_tokens_to_sample=1000)
+
+chain = (
+    RunnablePassthrough.assign(
+        agent_scratchpad=lambda x: format_agent_scratchpad(x["intermediate_steps"])
+    )
+    | prompt
+    | model.bind(stop_sequences=["</search_query>"])
+    | StrOutputParser()
+)
+
+agent_chain = (
+    RunnableMap(
+        {
+            "partial_completion": chain,
+            "intermediate_steps": lambda x: x["intermediate_steps"],
+        }
+    )
+    | parse_output
+)
+
+executor = AgentExecutor(agent=agent_chain, tools=[search])

langchain-docs-benchmarking/packages/anthropic-iterative-search/main.py

@@ -0,0 +1,12 @@
+from anthropic_iterative_search import final_chain
+
+if __name__ == "__main__":
+    query = (
+        "Which movie came out first: Oppenheimer, or "
+        "Are You There God It's Me Margaret?"
+    )
+    print(
+        final_chain.with_config(configurable={"chain": "retrieve"}).invoke(
+            {"query": query}
+        )
+    )

langchain-docs-benchmarking/packages/anthropic-iterative-search/pyproject.toml

@@ -0,0 +1,22 @@
+[tool.poetry]
+name = "anthropic-iterative-search"
+version = "0.0.1"
+description = ""
+authors = []
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.8.1,<4.0"
+langchain = ">=0.0.331,<0.1.0"
+anthropic = "^0.5.0"
+wikipedia = "^1.4.0"
+
+[tool.langserve]
+export_module = "anthropic_iterative_search"
+export_attr = "final_chain"
+
+[build-system]
+requires = [
+    "poetry-core",
+]
+build-backend = "poetry.core.masonry.api"

langchain-docs-benchmarking/packages/chat-langchain/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 LangChain, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

langchain-docs-benchmarking/packages/chat-langchain/README.md

@@ -0,0 +1,66 @@
+# chat-langchain
+
+TODO: What does this package do
+
+## Environment Setup
+
+TODO: What environment variables need to be set (if any)
+
+## Usage
+
+To use this package, you should first have the LangChain CLI installed:
+
+```shell
+pip install -U "langchain-cli[serve]"
+```
+
+To create a new LangChain project and install this as the only package, you can do:
+
+```shell
+langchain app new my-app --package chat-langchain
+```
+
+If you want to add this to an existing project, you can just run:
+
+```shell
+langchain app add chat-langchain
+```
+
+And add the following code to your `server.py` file:
+```python
+from chat_langchain import chain as chat_langchain_chain
+
+add_routes(app, chat_langchain_chain, path="/chat-langchain")
+```
+
+(Optional) Let's now configure LangSmith. 
+LangSmith will help us trace, monitor and debug LangChain applications. 
+LangSmith is currently in private beta, you can sign up [here](https://smith.langchain.com/). 
+If you don't have access, you can skip this section
+
+
+```shell
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_API_KEY=<your-api-key>
+export LANGCHAIN_PROJECT=<your-project>  # if not specified, defaults to "default"
+```
+
+If you are inside this directory, then you can spin up a LangServe instance directly by:
+
+```shell
+langchain serve
+```
+
+This will start the FastAPI app with a server is running locally at 
+[http://localhost:8000](http://localhost:8000)
+
+We can see all templates at [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs)
+We can access the playground at [http://127.0.0.1:8000/chat-langchain/playground](http://127.0.0.1:8000/chat-langchain/playground)  
+
+We can access the template from code with:
+
+```python
+from langserve.client import RemoteRunnable
+
+runnable = RemoteRunnable("http://localhost:8000/chat-langchain")
+```

langchain-docs-benchmarking/packages/chat-langchain/chat_langchain/__init__.py

@@ -0,0 +1,3 @@
+from chat_langchain.chain import chain
+
+__all__ = ["chain"]

langchain-docs-benchmarking/packages/chat-langchain/chat_langchain/chain.py

@@ -0,0 +1,183 @@
+"""Chat langchain 'engine'."""
+from operator import itemgetter
+from typing import Dict, List, Optional, Sequence
+
+from langchain.chat_models import ChatAnthropic, ChatFireworks, ChatOpenAI
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder, PromptTemplate
+from langchain.schema import Document
+from langchain.schema.language_model import BaseLanguageModel
+from langchain.schema.messages import AIMessage, HumanMessage
+from langchain.schema.output_parser import StrOutputParser
+from langchain.schema.retriever import BaseRetriever
+from langchain.schema.runnable import (
+    Runnable,
+    RunnableBranch,
+    RunnableLambda,
+    RunnableMap,
+)
+from langchain_docs_retriever.retriever import get_retriever
+from pydantic import BaseModel
+
+RESPONSE_TEMPLATE = """\
+You are an expert programmer and problem-solver, tasked with answering any question \
+about Langchain.
+
+Generate a comprehensive and informative answer of 80 words or less for the \
+given question based solely on the provided search results (URL and content). You must \
+only use information from the provided search results. Use an unbiased and \
+journalistic tone. Combine search results together into a coherent answer. Do not \
+repeat text. Cite search results using [${{number}}] notation. Only cite the most \
+relevant results that answer the question accurately. Place these citations at the end \
+of the sentence or paragraph that reference them - do not put them all at the end. If \
+different results refer to different entities within the same name, write separate \
+answers for each entity.
+
+You should use bullet points in your answer for readability. Put citations where they apply
+rather than putting them all at the end.
+
+If there is nothing in the context relevant to the question at hand, just say "Hmm, \
+I'm not sure." Don't try to make up an answer.
+
+Anything between the following `context`  html blocks is retrieved from a knowledge \
+bank, not part of the conversation with the user. 
+
+<context>
+    {context} 
+<context/>
+
+REMEMBER: If there is no relevant information within the context, just say "Hmm, I'm \
+not sure." Don't try to make up an answer. Anything between the preceding 'context' \
+html blocks is retrieved from a knowledge bank, not part of the conversation with the \
+user.\
+"""
+
+REPHRASE_TEMPLATE = """\
+Given the following conversation and a follow up question, rephrase the follow up \
+question to be a standalone question.
+
+Chat History:
+{chat_history}
+Follow Up Input: {question}
+Standalone Question:"""
+
+
+class ChatRequest(BaseModel):
+    question: str
+    chat_history: Optional[List[Dict[str, str]]]
+
+
+def create_retriever_chain(
+    llm: BaseLanguageModel, retriever: BaseRetriever
+) -> Runnable:
+    CONDENSE_QUESTION_PROMPT = PromptTemplate.from_template(REPHRASE_TEMPLATE)
+    condense_question_chain = (
+        CONDENSE_QUESTION_PROMPT | llm | StrOutputParser()
+    ).with_config(
+        run_name="CondenseQuestion",
+    )
+    conversation_chain = condense_question_chain | retriever
+    return RunnableBranch(
+        (
+            RunnableLambda(lambda x: bool(x.get("chat_history"))).with_config(
+                run_name="HasChatHistoryCheck"
+            ),
+            conversation_chain.with_config(run_name="RetrievalChainWithHistory"),
+        ),
+        (
+            RunnableLambda(itemgetter("question")).with_config(
+                run_name="Itemgetter:question"
+            )
+            | retriever
+        ).with_config(run_name="RetrievalChainWithNoHistory"),
+    ).with_config(run_name="RouteDependingOnChatHistory")
+
+
+def format_docs(docs: Sequence[Document]) -> str:
+    formatted_docs = []
+    for i, doc in enumerate(docs):
+        doc_string = f"<doc id='{i}'>{doc.page_content}</doc>"
+        formatted_docs.append(doc_string)
+    return "\n".join(formatted_docs)
+
+
+def serialize_history(request: ChatRequest):
+    chat_history = request.get("chat_history") or []
+    converted_chat_history = []
+    for message in chat_history:
+        if message.get("human") is not None:
+            converted_chat_history.append(HumanMessage(content=message["human"]))
+        if message.get("ai") is not None:
+            converted_chat_history.append(AIMessage(content=message["ai"]))
+    return converted_chat_history
+
+
+def create_response_chain(
+    llm: BaseLanguageModel,
+    retriever: BaseRetriever,
+) -> Runnable:
+    retriever_chain = create_retriever_chain(
+        llm,
+        retriever,
+    ).with_config(run_name="FindDocs")
+    _context = RunnableMap(
+        {
+            "context": retriever_chain | format_docs,
+            "question": itemgetter("question"),
+            "chat_history": itemgetter("chat_history"),
+        }
+    ).with_config(run_name="RetrieveDocs")
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", RESPONSE_TEMPLATE),
+            MessagesPlaceholder(variable_name="chat_history"),
+            ("human", "{question}"),
+        ]
+    )
+
+    response_synthesizer = (prompt | llm | StrOutputParser()).with_config(
+        run_name="GenerateResponse",
+    )
+    return (
+        {
+            "question": RunnableLambda(itemgetter("question")).with_config(
+                run_name="Itemgetter:question"
+            ),
+            "chat_history": RunnableLambda(serialize_history).with_config(
+                run_name="SerializeHistory"
+            ),
+        }
+        | _context
+        | response_synthesizer
+    )
+
+
+llm = ChatOpenAI(
+    model="gpt-3.5-turbo-16k",
+    streaming=True,
+    temperature=0,
+)
+
+retriever = get_retriever()
+chain = create_response_chain(
+    llm,
+    retriever,
+)
+chain = chain.with_types(input_type=ChatRequest)
+
+
+def create_chain(config: dict):
+    config_copy = config.copy()
+    chat_cls_name = config_copy.pop("chat_cls", "ChatOpenAI")
+
+    assert chat_cls_name in {"ChatOpenAI", "ChatFireworks", "ChatAnthropic"}
+    chat_cls = {
+        "ChatOpenAI": ChatOpenAI,
+        "ChatFireworks": ChatFireworks,
+        "ChatAnthropic": ChatAnthropic,
+    }[chat_cls_name]
+    model = chat_cls(**config_copy)
+    retriever = get_retriever(config.get("retriever_config", {}))
+    return create_response_chain(
+        model,
+        retriever,
+    )

langchain-docs-benchmarking/packages/chat-langchain/pyproject.toml

@@ -0,0 +1,34 @@
+[tool.poetry]
+name = "chat-langchain"
+version = "0.0.1"
+description = ""
+authors = []
+readme = "README.md"
+
+[tool.poetry.dependencies]
+openai = ">1,<2"
+python = "^3.10"
+fastapi = "^0.104.1"
+pydantic = "1.10"
+langchain = ">=0.0.327,<0.1.0"
+uvicorn = "^0.23.2"
+beautifulsoup4 = "^4.12.2"
+tiktoken = "^0.4.0"
+weaviate-client = "^3.23.2"
+psycopg2 = "^2.9.7"
+lxml = "^4.9.3"
+langserve = {extras = ["server"], version = ">=0.0.21,<0.1.0"}
+anthropic = "^0.5.0"
+
+[tool.poetry.group.dev.dependencies]
+langchain-cli = ">=0.0.4"
+fastapi = "^0.104.0"
+sse-starlette = "^1.6.5"
+
+[tool.langserve]
+export_module = "chat_langchain"
+export_attr = "chain"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

langchain-docs-benchmarking/packages/example/custom_example/example_custom_chain.py

@@ -0,0 +1,32 @@
+from langchain.chat_models import ChatAnthropic, ChatOpenAI
+from langchain.prompts import ChatPromptTemplate
+from langchain.schema.output_parser import StrOutputParser
+from langchain_docs_retriever.retriever import get_retriever
+
+
+def create_runnable(config: dict):
+    config_copy = config.copy()
+    chat_cls_name = config_copy.pop("chat_cls", "ChatOpenAI")
+
+    assert chat_cls_name in {"ChatOpenAI", "ChatAnthropic"}
+    chat_cls = {
+        "ChatOpenAI": ChatOpenAI,
+        "ChatAnthropic": ChatAnthropic,
+    }[chat_cls_name]
+    model = chat_cls(**config_copy)
+    retriever = get_retriever(config.get("retriever_config", {}))
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", "Answer the Q using the following docs\n{docs}"),
+            ("user", "Q: {question}"),
+        ]
+    )
+    return (
+        {
+            "question": lambda x: x["question"],
+            "docs": (lambda x: x["question"]) | retriever,
+        }
+        | prompt
+        | model
+        | StrOutputParser()
+    )

langchain-docs-benchmarking/packages/langchain-docs-retriever/README.md

@@ -0,0 +1,5 @@
+# LangChain Docs Retriever
+
+
+A simple vector store retriever over the LangChain python docs. Indexed
+simply using [ingest_docs.py](./ingest_docs.py).

langchain-docs-benchmarking/packages/langchain-docs-retriever/ingest_docs.py

@@ -0,0 +1,242 @@
+"""Load html from files, clean up, split, ingest into Weaviate."""
+import logging
+import os
+import re
+from typing import Generator
+
+from bs4 import BeautifulSoup, Doctype, NavigableString, SoupStrainer, Tag
+from langchain.document_loaders import RecursiveUrlLoader, SitemapLoader
+from langchain.embeddings import OpenAIEmbeddings, VoyageEmbeddings
+from langchain.indexes import SQLRecordManager, index
+from langchain.schema.embeddings import Embeddings
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.utils.html import PREFIXES_TO_IGNORE_REGEX, SUFFIXES_TO_IGNORE_REGEX
+from langchain.vectorstores.chroma import Chroma
+
+logger = logging.getLogger(__name__)
+directory = os.path.dirname(os.path.realpath(__file__))
+db_directory = os.path.join(directory, "langchain_docs_retriever", "db")
+
+
+def langchain_docs_extractor(soup: BeautifulSoup) -> str:
+    # Remove all the tags that are not meaningful for the extraction.
+    SCAPE_TAGS = ["nav", "footer", "aside", "script", "style"]
+    [tag.decompose() for tag in soup.find_all(SCAPE_TAGS)]
+
+    def get_text(tag: Tag) -> Generator[str, None, None]:
+        for child in tag.children:
+            if isinstance(child, Doctype):
+                continue
+
+            if isinstance(child, NavigableString):
+                yield child
+            elif isinstance(child, Tag):
+                if child.name in ["h1", "h2", "h3", "h4", "h5", "h6"]:
+                    yield f"{'#' * int(child.name[1:])} {child.get_text()}\n\n"
+                elif child.name == "a":
+                    yield f"[{child.get_text(strip=False)}]({child.get('href')})"
+                elif child.name == "img":
+                    yield f"![{child.get('alt', '')}]({child.get('src')})"
+                elif child.name in ["strong", "b"]:
+                    yield f"**{child.get_text(strip=False)}**"
+                elif child.name in ["em", "i"]:
+                    yield f"_{child.get_text(strip=False)}_"
+                elif child.name == "br":
+                    yield "\n"
+                elif child.name == "code":
+                    parent = child.find_parent()
+                    if parent is not None and parent.name == "pre":
+                        classes = parent.attrs.get("class", "")
+
+                        language = next(
+                            filter(lambda x: re.match(r"language-\w+", x), classes),
+                            None,
+                        )
+                        if language is None:
+                            language = ""
+                        else:
+                            language = language.split("-")[1]
+
+                        lines: list[str] = []
+                        for span in child.find_all("span", class_="token-line"):
+                            line_content = "".join(
+                                token.get_text() for token in span.find_all("span")
+                            )
+                            lines.append(line_content)
+
+                        code_content = "\n".join(lines)
+                        yield f"```{language}\n{code_content}\n```\n\n"
+                    else:
+                        yield f"`{child.get_text(strip=False)}`"
+
+                elif child.name == "p":
+                    yield from get_text(child)
+                    yield "\n\n"
+                elif child.name == "ul":
+                    for li in child.find_all("li", recursive=False):
+                        yield "- "
+                        yield from get_text(li)
+                        yield "\n\n"
+                elif child.name == "ol":
+                    for i, li in enumerate(child.find_all("li", recursive=False)):
+                        yield f"{i + 1}. "
+                        yield from get_text(li)
+                        yield "\n\n"
+                elif child.name == "div" and "tabs-container" in child.attrs.get(
+                    "class", [""]
+                ):
+                    tabs = child.find_all("li", {"role": "tab"})
+                    tab_panels = child.find_all("div", {"role": "tabpanel"})
+                    for tab, tab_panel in zip(tabs, tab_panels):
+                        tab_name = tab.get_text(strip=True)
+                        yield f"{tab_name}\n"
+                        yield from get_text(tab_panel)
+                elif child.name == "table":
+                    thead = child.find("thead")
+                    header_exists = isinstance(thead, Tag)
+                    if header_exists:
+                        headers = thead.find_all("th")
+                        if headers:
+                            yield "| "
+                            yield " | ".join(header.get_text() for header in headers)
+                            yield " |\n"
+                            yield "| "
+                            yield " | ".join("----" for _ in headers)
+                            yield " |\n"
+
+                    tbody = child.find("tbody")
+                    tbody_exists = isinstance(tbody, Tag)
+                    if tbody_exists:
+                        for row in tbody.find_all("tr"):
+                            yield "| "
+                            yield " | ".join(
+                                cell.get_text(strip=True) for cell in row.find_all("td")
+                            )
+                            yield " |\n"
+
+                    yield "\n\n"
+                elif child.name in ["button"]:
+                    continue
+                else:
+                    yield from get_text(child)
+
+    joined = "".join(get_text(soup))
+    return re.sub(r"\n\n+", "\n\n", joined).strip()
+
+
+RECORD_MANAGER_DB_URL = (
+    os.environ.get("RECORD_MANAGER_DB_URL") or "sqlite:///lcdocs_oai_record_manager.sql"
+)
+
+
+def metadata_extractor(meta: dict, soup: BeautifulSoup) -> dict:
+    title = soup.find("title")
+    description = soup.find("meta", attrs={"name": "description"})
+    html = soup.find("html")
+    return {
+        "source": meta["loc"] or "",
+        "title": (title.get_text() if title else "") or "",
+        "description": description.get("content") or "" if description else "",
+        "language": html.get("lang") or "" if html else "",
+        **{k: v or "" for k, v in meta.items()},
+    }
+
+
+def load_langchain_docs():
+    return SitemapLoader(
+        "https://python.langchain.com/sitemap.xml",
+        filter_urls=["https://python.langchain.com/"],
+        parsing_function=langchain_docs_extractor,
+        default_parser="lxml",
+        bs_kwargs={
+            "parse_only": SoupStrainer(
+                name=("article", "title", "html", "lang", "content")
+            ),
+        },
+        meta_function=metadata_extractor,
+    ).load()
+
+
+def simple_extractor(html: str) -> str:
+    soup = BeautifulSoup(html, "lxml")
+    return re.sub(r"\n\n+", "\n\n", soup.text).strip()
+
+
+def load_api_docs():
+    return RecursiveUrlLoader(
+        url="https://api.python.langchain.com/en/latest/",
+        max_depth=8,
+        extractor=simple_extractor,
+        prevent_outside=True,
+        use_async=True,
+        timeout=600,
+        # Drop trailing / to avoid duplicate pages.
+        link_regex=(
+            f"href=[\"']{PREFIXES_TO_IGNORE_REGEX}((?:{SUFFIXES_TO_IGNORE_REGEX}.)*?)"
+            r"(?:[\#'\"]|\/[\#'\"])"
+        ),
+        check_response_status=True,
+        exclude_dirs=(
+            "https://api.python.langchain.com/en/latest/_sources",
+            "https://api.python.langchain.com/en/latest/_modules",
+        ),
+    ).load()
+
+
+def get_embeddings_model() -> Embeddings:
+    if os.environ.get("VOYAGE_AI_URL") and os.environ.get("VOYAGE_AI_MODEL"):
+        return VoyageEmbeddings()
+    return OpenAIEmbeddings(chunk_size=200)
+
+
+CHROMA_COLLECTION_NAME = "langchain-docs"
+
+
+def ingest_docs():
+    docs_from_documentation = load_langchain_docs()
+    logger.info(f"Loaded {len(docs_from_documentation)} docs from documentation")
+    docs_from_api = load_api_docs()
+    logger.info(f"Loaded {len(docs_from_api)} docs from API")
+
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=4000, chunk_overlap=200)
+    docs_transformed = text_splitter.split_documents(
+        docs_from_documentation + docs_from_api
+    )
+
+    # We try to return 'source' and 'title' metadata when querying vector store and
+    # Chroma will error at query time if one of the attributes is missing from a
+    # retrieved document.
+    for doc in docs_transformed:
+        if "source" not in doc.metadata:
+            doc.metadata["source"] = ""
+        if "title" not in doc.metadata:
+            doc.metadata["title"] = ""
+        for k, v in doc.metadata.items():
+            if v is None:
+                doc.metadata[k] = ""
+
+    embedding = get_embeddings_model()
+    vectorstore = Chroma(
+        collection_name=CHROMA_COLLECTION_NAME,
+        embedding_function=embedding,
+        persist_directory=db_directory,
+    )
+
+    record_manager = SQLRecordManager(
+        f"chroma/{CHROMA_COLLECTION_NAME}", db_url=RECORD_MANAGER_DB_URL
+    )
+    record_manager.create_schema()
+
+    indexing_stats = index(
+        docs_transformed,
+        record_manager,
+        vectorstore,
+        cleanup="full",
+        source_id_key="source",
+    )
+
+    logger.info("Indexing stats: ", indexing_stats)
+
+
+if __name__ == "__main__":
+    ingest_docs()

langchain-docs-benchmarking/packages/langchain-docs-retriever/langchain_docs_retriever/download_db.py

@@ -0,0 +1,34 @@
+import os
+import zipfile
+
+import requests
+
+remote_url = "https://storage.googleapis.com/benchmarks-artifacts/langchain-docs-benchmarking/chroma_db.zip"
+directory = os.path.dirname(os.path.realpath(__file__))
+db_directory = os.path.join(directory, "db")
+
+
+def is_folder_populated(folder):
+    if os.path.exists(folder):
+        return any(os.scandir(folder))
+    return False
+
+
+def download_folder_from_gcs():
+    r = requests.get(remote_url, allow_redirects=True)
+    open("chroma_db.zip", "wb").write(r.content)
+
+    with zipfile.ZipFile("chroma_db.zip", "r") as zip_ref:
+        zip_ref.extractall(directory)
+
+    os.remove("chroma_db.zip")
+
+
+def fetch_langchain_docs_db():
+    if not is_folder_populated(db_directory):
+        print(f"Folder {db_directory} is not populated. Downloading from GCS...")
+        download_folder_from_gcs()
+
+
+if __name__ == "__main__":
+    fetch_langchain_docs_db()

langchain-docs-benchmarking/packages/langchain-docs-retriever/langchain_docs_retriever/retriever.py

@@ -0,0 +1,35 @@
+import os
+from typing import Optional
+
+from langchain.embeddings import OpenAIEmbeddings
+
+# from langchain_docs_retriever.voyage import VoyageEmbeddings
+from langchain.embeddings.voyageai import VoyageEmbeddings
+from langchain.schema.embeddings import Embeddings
+from langchain.schema.retriever import BaseRetriever
+from langchain.vectorstores.chroma import Chroma
+
+from .download_db import fetch_langchain_docs_db
+
+WEAVIATE_DOCS_INDEX_NAME = "LangChain_agent_docs"
+_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
+CHROMA_COLLECTION_NAME = "langchain-docs"
+_DB_DIRECTORY = os.path.join(_DIRECTORY, "db")
+
+
+def get_embeddings_model() -> Embeddings:
+    if os.environ.get("VOYAGE_AI_MODEL"):
+        return VoyageEmbeddings(model=os.environ["VOYAGE_AI_MODEL"], max_retries=20)
+    return OpenAIEmbeddings(chunk_size=200)
+
+
+def get_retriever(search_kwargs: Optional[dict] = None) -> BaseRetriever:
+    embedding_model = get_embeddings_model()
+    fetch_langchain_docs_db()
+    vectorstore = Chroma(
+        collection_name=CHROMA_COLLECTION_NAME,
+        embedding_function=embedding_model,
+        persist_directory=_DB_DIRECTORY,
+    )
+    search_kwargs = search_kwargs or dict(k=6)
+    return vectorstore.as_retriever(search_kwargs=search_kwargs)

langchain-docs-benchmarking/packages/langchain-docs-retriever/pyproject.toml

@@ -0,0 +1,31 @@
+[tool.poetry]
+name = "langchain-docs-retriever"
+version = "0.0.1"
+description = ""
+authors = []
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+fastapi = "^0.104.1"
+pydantic = "1.10"
+langchain = ">=0.0.331,<0.1.0"
+uvicorn = "^0.23.2"
+openai = ">1,<2"
+psycopg2 = "^2.9.7"
+lxml = "^4.9.3"
+langserve = {extras = ["server"], version = ">=0.0.23,<0.1.0"}
+chromadb = "^0.4.15"
+
+[tool.poetry.group.dev.dependencies]
+langchain-cli = ">=0.0.4"
+fastapi = "^0.104.0"
+sse-starlette = "^1.6.5"
+
+[tool.langserve]
+export_module = "chat_langchain"
+export_attr = "chain"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

langchain-docs-benchmarking/packages/oai-assistant/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 LangChain, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

langchain-docs-benchmarking/packages/oai-assistant/README.md

@@ -0,0 +1,66 @@
+# oai-assistant
+
+TODO: What does this package do
+
+## Environment Setup
+
+TODO: What environment variables need to be set (if any)
+
+## Usage
+
+To use this package, you should first have the LangChain CLI installed:
+
+```shell
+pip install -U "langchain-cli[serve]"
+```
+
+To create a new LangChain project and install this as the only package, you can do:
+
+```shell
+langchain app new my-app --package oai-assistant
+```
+
+If you want to add this to an existing project, you can just run:
+
+```shell
+langchain app add oai-assistant
+```
+
+And add the following code to your `server.py` file:
+```python
+from oai_assistant import chain as oai_assistant_chain
+
+add_routes(app, oai_assistant_chain, path="/oai-assistant")
+```
+
+(Optional) Let's now configure LangSmith. 
+LangSmith will help us trace, monitor and debug LangChain applications. 
+LangSmith is currently in private beta, you can sign up [here](https://smith.langchain.com/). 
+If you don't have access, you can skip this section
+
+
+```shell
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_API_KEY=<your-api-key>
+export LANGCHAIN_PROJECT=<your-project>  # if not specified, defaults to "default"
+```
+
+If you are inside this directory, then you can spin up a LangServe instance directly by:
+
+```shell
+langchain serve
+```
+
+This will start the FastAPI app with a server is running locally at 
+[http://localhost:8000](http://localhost:8000)
+
+We can see all templates at [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs)
+We can access the playground at [http://127.0.0.1:8000/oai-assistant/playground](http://127.0.0.1:8000/oai-assistant/playground)  
+
+We can access the template from code with:
+
+```python
+from langserve.client import RemoteRunnable
+
+runnable = RemoteRunnable("http://localhost:8000/oai-assistant")
+```

langchain-docs-benchmarking/packages/oai-assistant/oai_assistant/__init__.py

@@ -0,0 +1,3 @@
+from oai_assistant.chain import agent_executor
+
+__all__ = ["agent_executor"]

langchain-docs-benchmarking/packages/oai-assistant/oai_assistant/chain.py

@@ -0,0 +1,36 @@
+import json
+
+from langchain.agents import AgentExecutor
+from langchain.tools import tool
+from langchain_docs_retriever.retriever import get_retriever
+from langchain_experimental.openai_assistant import OpenAIAssistantRunnable
+
+# This is used to tell the model how to best use the retriever.
+
+
+_RETRIEVER = get_retriever()
+
+
+@tool
+def search(query, callbacks=None) -> str:
+    """Search the LangChain docs with the retriever."""
+    docs = _RETRIEVER.get_relevant_documents(query, callbacks=callbacks)
+    return json.dumps([doc.dict() for doc in docs])
+
+
+tools = [search]
+
+agent = OpenAIAssistantRunnable.create_assistant(
+    name="langchain docs assistant",
+    instructions="You are a helpful assistant tasked with answering technical questions about LangChain.",
+    tools=tools,
+    model="gpt-4-1106-preview",
+    as_agent=True,
+)
+
+
+agent_executor = (
+    (lambda x: {"content": x["question"]})
+    | AgentExecutor(agent=agent, tools=tools)
+    | (lambda x: x["output"])
+)

langchain-docs-benchmarking/packages/oai-assistant/pyproject.toml

@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "oai-assistant"
+version = "0.0.1"
+description = ""
+authors = []
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.8.1,<4.0"
+langchain = ">=0.0.332,<0.1.0"
+openai = ">1,<2"
+langchain-experimental = "^0.0.38"
+
+[tool.poetry.group.dev.dependencies]
+langchain-cli = ">=0.0.4"
+fastapi = "^0.104.0"
+sse-starlette = "^1.6.5"
+
+[tool.langserve]
+export_module = "oai_assistant"
+export_attr = "agent_executor"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

langchain-docs-benchmarking/packages/openai-functions-agent/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 LangChain, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

langchain-docs-benchmarking/packages/openai-functions-agent/README.md

@@ -0,0 +1,72 @@
+
+# openai-functions-agent
+
+This template creates an agent that uses OpenAI function calling to communicate its decisions on what actions to take. 
+
+This example creates an agent that can optionally look up information on the internet using Tavily's search engine.
+
+## Environment Setup
+
+The following environment variables need to be set:
+
+Set the `OPENAI_API_KEY` environment variable to access the OpenAI models.
+
+Set the `TAVILY_API_KEY` environment variable to access Tavily.
+
+## Usage
+
+To use this package, you should first have the LangChain CLI installed:
+
+```shell
+pip install -U "langchain-cli[serve]"
+```
+
+To create a new LangChain project and install this as the only package, you can do:
+
+```shell
+langchain app new my-app --package openai-functions-agent
+```
+
+If you want to add this to an existing project, you can just run:
+
+```shell
+langchain app add openai-functions-agent
+```
+
+And add the following code to your `server.py` file:
+```python
+from openai_functions_agent import chain as openai_functions_agent_chain
+
+add_routes(app, openai_functions_agent_chain, path="/openai-functions-agent")
+```
+
+(Optional) Let's now configure LangSmith. 
+LangSmith will help us trace, monitor and debug LangChain applications. 
+LangSmith is currently in private beta, you can sign up [here](https://smith.langchain.com/). 
+If you don't have access, you can skip this section
+
+```shell
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_API_KEY=<your-api-key>
+export LANGCHAIN_PROJECT=<your-project>  # if not specified, defaults to "default"
+```
+
+If you are inside this directory, then you can spin up a LangServe instance directly by:
+
+```shell
+langchain serve
+```
+
+This will start the FastAPI app with a server is running locally at 
+[http://localhost:8000](http://localhost:8000)
+
+We can see all templates at [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs)
+We can access the playground at [http://127.0.0.1:8000/openai-functions-agent/playground](http://127.0.0.1:8000/openai-functions-agent/playground)  
+
+We can access the template from code with:
+
+```python
+from langserve.client import RemoteRunnable
+
+runnable = RemoteRunnable("http://localhost:8000/openai-functions-agent")
+```

langchain-docs-benchmarking/packages/openai-functions-agent/main.py

@@ -0,0 +1,5 @@
+from openai_functions_agent.agent import agent_executor
+
+if __name__ == "__main__":
+    question = "who won the womens world cup in 2023?"
+    print(agent_executor.invoke({"input": question, "chat_history": []}))

langchain-docs-benchmarking/packages/openai-functions-agent/openai_functions_agent/__init__.py

@@ -0,0 +1,3 @@
+from openai_functions_agent.agent import agent_executor
+
+__all__ = ["agent_executor"]

langchain-docs-benchmarking/packages/openai-functions-agent/openai_functions_agent/agent.py

@@ -0,0 +1,85 @@
+from typing import List, Tuple
+
+from langchain.agents import AgentExecutor
+from langchain.agents.format_scratchpad import format_to_openai_functions
+from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain.pydantic_v1 import BaseModel, Field
+from langchain.schema.messages import AIMessage, HumanMessage
+from langchain.tools import tool
+from langchain.tools.render import format_tool_to_openai_function
+from langchain_docs_retriever.retriever import get_retriever
+
+# This is used to tell the model how to best use the retriever.
+
+
+_RETRIEVER = get_retriever()
+
+
+@tool
+def search(query, callbacks=None):
+    """Search the LangChain docs with the retriever."""
+    return _RETRIEVER.get_relevant_documents(query, callbacks=callbacks)
+
+
+tools = [search]
+
+llm = ChatOpenAI(model="gpt-3.5-turbo-16k", temperature=0)
+assistant_system_message = """You are a helpful assistant tasked with answering technical questions about LangChain. \
+Use tools (only if necessary) to best answer the users questions. Do not make up information if you cannot find the answer using your tools."""
+prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", assistant_system_message),
+        MessagesPlaceholder(variable_name="chat_history"),
+        ("user", "{input}"),
+        MessagesPlaceholder(variable_name="agent_scratchpad"),
+    ]
+)
+
+llm_with_tools = llm.bind(functions=[format_tool_to_openai_function(t) for t in tools])
+
+
+def _format_chat_history(chat_history: List[Tuple[str, str]]):
+    buffer = []
+    for human, ai in chat_history:
+        buffer.append(HumanMessage(content=human))
+        buffer.append(AIMessage(content=ai))
+    return buffer
+
+
+agent = (
+    {
+        "input": lambda x: x["input"],
+        "chat_history": lambda x: _format_chat_history(x["chat_history"]),
+        "agent_scratchpad": lambda x: format_to_openai_functions(
+            x["intermediate_steps"]
+        ),
+    }
+    | prompt
+    | llm_with_tools
+    | OpenAIFunctionsAgentOutputParser()
+)
+
+
+class AgentInput(BaseModel):
+    input: str
+    chat_history: List[Tuple[str, str]] = Field(..., extra={"widget": {"type": "chat"}})
+
+
+agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=False).with_types(
+    input_type=AgentInput
+)
+
+
+class ChainInput(BaseModel):
+    question: str
+
+
+def mapper(input: dict):
+    return {"input": input["question"], "chat_history": []}
+
+
+agent_executor = (mapper | agent_executor | (lambda x: x["output"])).with_types(
+    input_type=ChainInput
+)

langchain-docs-benchmarking/packages/openai-functions-agent/pyproject.toml

@@ -0,0 +1,24 @@
+[tool.poetry]
+name = "openai-functions-agent"
+version = "0.1.0"
+description = ""
+authors = [
+    "Lance Martin <lance@langchain.dev>",
+]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.8.1,<4.0"
+langchain = ">=0.0.327,<0.1.0"
+openai = ">=0.5.0"
+tavily-python = "^0.1.9"
+
+[tool.langserve]
+export_module = "openai_functions_agent"
+export_attr = "agent_executor"
+
+[build-system]
+requires = [
+    "poetry-core",
+]
+build-backend = "poetry.core.masonry.api"

langchain-docs-benchmarking/prepare_dataset.py

@@ -0,0 +1,26 @@
+"""Copy the public dataset to your own langsmith tenant."""
+from langsmith import Client
+from tqdm import tqdm
+
+DATASET_NAME = "LangChain Docs Q&A"
+PUBLIC_DATASET_TOKEN = "452ccafc-18e1-4314-885b-edd735f17b9d"
+client = Client()
+
+
+def create_langchain_docs_dataset(
+    dataset_name: str = DATASET_NAME, public_dataset_token: str = PUBLIC_DATASET_TOKEN
+):
+    if client.has_dataset(dataset_name=dataset_name):
+        return
+    ds = client.create_dataset(dataset_name=dataset_name)
+    examples = tqdm(list(client.list_shared_examples(public_dataset_token)))
+    client.create_examples(
+        inputs=[e.inputs for e in examples],
+        outputs=[e.outputs for e in examples],
+        dataset_id=ds.id,
+    )
+    print("Done creating dataset.")
+
+
+if __name__ == "__main__":
+    create_langchain_docs_dataset()

langchain-docs-benchmarking/pyproject.toml

@@ -0,0 +1,29 @@
+[tool.poetry]
+name = "langservehub-template"
+version = "0.1.0"
+description = ""
+authors = ["Your Name <you@example.com>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+langsmith = ">=0.0.64,<0.1.0"
+sse-starlette = "^1.6.5"
+tomli-w = "^1.0.0"
+uvicorn = "^0.23.2"
+fastapi = "^0.104"
+langserve = ">=0.0.16"
+chat-langchain = {path = "packages/chat-langchain", develop = true}
+langchain-docs-retriever = {path = "packages/langchain-docs-retriever", develop = true}
+anthropic-iterative-search = {path = "packages/anthropic-iterative-search", develop = true}
+oai-assistant = {path = "packages/oai-assistant", develop = true}
+openai-functions-agent = {path = "packages/openai-functions-agent", develop = true}
+
+[tool.poetry.group.dev.dependencies]
+uvicorn = "^0.23.2"
+pygithub = "^2.1.1"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

langchain-docs-benchmarking/run_evals.py

@@ -0,0 +1,121 @@
+import argparse
+import importlib.util
+import os
+import sys
+import uuid
+from functools import partial
+from typing import Callable, Optional
+
+from anthropic_iterative_search.chain import chain as anthropic_agent_chain
+from chat_langchain.chain import create_chain
+from langchain.chat_models import ChatOpenAI
+from langchain.schema.runnable import Runnable
+from langchain.smith import RunEvalConfig, run_on_dataset
+from langsmith import Client
+from oai_assistant.chain import agent_executor as openai_assistant_chain
+from openai_functions_agent import agent_executor as openai_functions_agent_chain
+
+ls_client = Client()
+
+
+def import_from_path(path_name: str):
+    func_name = "create_chain"
+    if "::" in path_name:
+        path_name, func_name = path_name.split("::")
+    spec = importlib.util.spec_from_file_location("module_name", path_name)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules["module_name"] = module
+    spec.loader.exec_module(module)
+    return getattr(module, func_name)
+
+
+def _get_chain_factory(arch: str) -> Callable:
+    _map = {
+        "chat": create_chain,
+        "anthropic-iterative-search": lambda _: anthropic_agent_chain,
+        "openai-functions-agent": lambda _: openai_functions_agent_chain,
+        "openai-assistant": lambda _: openai_assistant_chain,
+    }
+    if arch in _map:
+        return _map[arch]
+    else:
+        return import_from_path(arch)
+
+
+def create_runnable(
+    arch: str, model_config: Optional[dict], retry_config: Optional[dict] = None
+):
+    factory = _get_chain_factory(arch)
+    chain: Runnable = factory(model_config)
+    if retry_config:
+        return chain.with_retry(**retry_config)
+    return chain
+
+
+def get_eval_config():
+    accuracy_criteria = {
+        "accuracy": """
+Score 1: The answer is incorrect and unrelated to the question or reference document.
+Score 3: The answer shows slight relevance to the question or reference document but is largely incorrect.
+Score 5: The answer is partially correct but has significant errors or omissions.
+Score 7: The answer is mostly correct with minor errors or omissions, and aligns with the reference document.
+Score 10: The answer is correct, complete, and perfectly aligns with the reference document.
+
+If the reference answer contains multiple alternatives, the predicted answer must only match one of the alternatives to be considered correct.
+If the predicted answer contains additional helpful and accurate information that is not present in the reference answer, it should still be considered correct.
+"""  # noqa
+    }
+
+    eval_llm = ChatOpenAI(model="gpt-4", temperature=0.0)
+    return RunEvalConfig(
+        evaluators=[
+            RunEvalConfig.LabeledScoreString(
+                criteria=accuracy_criteria, llm=eval_llm, normalize_by=10.0
+            ),
+            # Mainly to compare with the above
+            # Suspected to be less reliable.
+            RunEvalConfig.EmbeddingDistance(),
+        ]
+    )
+
+
+def main(
+    arch: str,
+    dataset_name: str,
+    model_config: Optional[dict] = None,
+    max_concurrency: int = 5,
+    project_name: Optional[str] = None,
+    retry_config: Optional[dict] = None,
+):
+    eval_config = get_eval_config()
+    project_name = project_name or arch
+    project_name += f" {uuid.uuid4().hex[:4]}"
+    run_on_dataset(
+        client=ls_client,
+        dataset_name=dataset_name,
+        llm_or_chain_factory=partial(
+            create_runnable,
+            arch=arch,
+            model_config=model_config,
+            retry_config=retry_config,
+        ),
+        evaluation=eval_config,
+        concurrency_level=max_concurrency,
+        project_name=project_name,
+        project_metadata={"arch": arch, "model_config": model_config},
+    )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--url", type=str)
+    parser.add_argument("--dataset-name", type=str, default="Chat Langchain Pub")
+    parser.add_argument("--project-name", type=Optional[str], default=None)
+    parser.add_argument("--max-concurrency", type=int, default=5)
+    args = parser.parse_args()
+    main(
+        args.url,
+        args.dataset_name,
+        max_concurrency=args.max_concurrency,
+        project_name=args.project_name,
+    )

langchain-docs-benchmarking/run_experiments.py

@@ -0,0 +1,126 @@
+import argparse
+import json
+
+from prepare_dataset import create_langchain_docs_dataset
+from run_evals import main
+
+experiments = [
+    {
+        # "server_url": "http://localhost:1983/openai-functions-agent",
+        "arch": "openai-functions-agent",
+        "project_name": "openai-functions-agent",
+    },
+    {
+        # "server_url": "http://localhost:1983/anthropic_chat",
+        "arch": "chat",
+        "model_config": {
+            "chat_cls": "ChatAnthropic",
+            "model": "claude-2",
+            "temperature": 1.0,
+        },
+        "project_name": "anthropic-chat",
+    },
+    {
+        "arch": "chat",
+        "model_config": {
+            "chat_cls": "ChatOpenAI",
+            "model": "gpt-3.5-turbo-16k",
+        },
+        # "server_url": "http://localhost:1983/chat",
+        "project_name": "chat-gpt-3.5",
+    },
+    {
+        "arch": "chat",
+        "model_config": {
+            "chat_cls": "ChatFireworks",
+            "model": "accounts/fireworks/models/mistral-7b-instruct-4k",
+        },
+        "project_name": "mistral-7b-instruct-4k",
+    },
+    {
+        "arch": "chat",
+        "model_config": {
+            "chat_cls": "ChatFireworks",
+            "model": "accounts/fireworks/models/llama-v2-34b-code-instruct-w8a16",
+        },
+        "project_name": "llama-v2-34b-code-instruct-w8a16",
+    },
+    {
+        "arch": "chat",
+        "model_config": {
+            "chat_cls": "ChatFireworks",
+            "model": "accounts/fireworks/models/zephyr-7b-beta",
+        },
+        "project_name": "zephyr-7b-beta",
+    },
+    {
+        "arch": "chat",
+        "model_config": {
+            "chat_cls": "ChatOpenAI",
+            "model": "gpt-4",
+        },
+        "project_name": "gpt-4-chat",
+    },
+    {
+        "arch": "openai-assistant",
+        "model_config": {},
+        "project_name": "openai-assistant",
+        "max_concurrency": 2,  # Rate limit is VERY low right now.
+        "retry_config": {
+            "stop_after_attempt": 10,
+        },
+    },
+    # Not worth our time it's so bad and slow
+    {
+        # "server_url": "http://localhost:1983/anthropic_iterative_search",
+        "arch": "anthropic-iterative-search",
+        "max_concurrency": 2,
+        "project_name": "anthropic-iterative-search",
+    },
+]
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dataset-name", type=str, default="LangChain Docs Q&A")
+    parser.add_argument(
+        "--config",
+        type=str,
+        default=None,
+        nargs="*",
+        help="Path to a JSON file with experiment config."
+        " If specified, the include and exclude args are ignored",
+    )
+    parser.add_argument("--include", type=str, nargs="+", default=None)
+    parser.add_argument(
+        "--exclude",
+        type=str,
+        nargs="+",
+    )
+    args = parser.parse_args()
+    create_langchain_docs_dataset(dataset_name=args.dataset_name)
+    selected_experiments = experiments
+    if args.config:
+        selected_experiments = []
+        for config_path in args.config:
+            with open(config_path) as f:
+                selected_experiments.append(json.load(f))
+    elif args.include:
+        selected_experiments = [
+            e for e in selected_experiments if e["project_name"] in args.include
+        ]
+    to_exclude = args.exclude or []
+    if args.include and not to_exclude:
+        to_exclude = [
+            "anthropic-iterative-search",
+            "openai-assistant",
+        ]
+    if args.exclude:
+        selected_experiments = [
+            e for e in selected_experiments if e["project_name"] not in args.exclude
+        ]
+
+    for experiment in selected_experiments:
+        main(
+            **experiment,
+            dataset_name=args.dataset_name,
+        )

langchain_benchmarks/__init__.py

@@ -0,0 +1,7 @@
+from langchain_benchmarks.utils._langsmith import (
+    clone_public_dataset,
+    download_public_dataset,
+)
+
+# Please keep this list sorted!
+__all__ = ["clone_public_dataset", "download_public_dataset"]

langchain_benchmarks/tool_usage/README.md

@@ -0,0 +1,19 @@
+# Testing Agents
+
+This directory contains environments that can be used to test agent's ability
+to use tools and make decisions.
+
+## Environments
+
+Environments are named in the style of e[env_number]_[name].py.
+
+### e01_alpha
+
+* Consists of 3 relational tables of users, locations and foods.
+* Defines a set of tools that can be used these tables.
+* Agent should use the given tools to answer questions.
+
+## Running Evaluation
+
+Please refer to the following example to see how to set up and run evaluation
+for agents using [LangSmith](https://github.com/langchain-ai/langsmith-cookbook/blob/main/testing-examples/agent_steps/evaluating_agents.ipynb).

langchain_benchmarks/tool_usage/__init__.py

@@ -0,0 +1 @@
+"""Package for helping to evaluate agent runs."""

langchain_benchmarks/tool_usage/environments/e01_alpha.py

@@ -0,0 +1,401 @@
+"""A simple environment for evaluating an agent.
+
+A simple environment to evaluate an agent's ability to use a set of given tools
+to reference questions.
+
+The environment contains fake data about users and their locations and favorite foods.
+
+The environment defines a set of tools that the agent can use to access the data.
+
+Agent performance should be evaluated solely based on the agent's ability to use
+the tools to reference questions.
+"""
+from typing import Callable, List, TypedDict
+
+from langchain.tools import BaseTool, tool
+
+USER_DATA = [
+    # IDs are not consecutive to prevent agents from guessing the ID
+    {
+        "id": 1,
+        "name": "Alice",
+        "email": "alice@gmail.com",
+        "location": 1,
+        "favorite_color": "red",
+        "favorite_foods": [1, 2, 3],
+    },
+    {
+        "id": 21,
+        "name": "Bob",
+        "email": "bob@hotmail.com",
+        "location": 2,
+        "favorite_color": "orange",
+        "favorite_foods": [4, 5, 6],
+    },
+    {
+        "id": 35,
+        "name": "Charlie",
+        "email": "charlie@yahoo.com",
+        "location": 3,
+        "favorite_color": "yellow",
+        "favorite_foods": [3, 7, 2],
+    },
+    {
+        "id": 41,
+        "name": "Donna",
+        "email": "donna@example.com",
+        "location": 4,
+        "favorite_color": "green",
+        "favorite_foods": [6, 1, 4],
+    },
+    {
+        "id": 42,
+        "name": "Eve",
+        "email": "eve@example.org",
+        "location": 5,
+        "favorite_color": "blue",
+        "favorite_foods": [5, 7, 4],
+    },
+    {
+        "id": 43,
+        "name": "Frank The Cat",
+        "email": "frank.the.cat@langchain.dev",
+        "location": 5,
+        "favorite_color": "yellow",
+        "favorite_foods": [3],
+    },
+]
+
+# Create a list of JSON data for locations with "current_weather" as a single string
+LOCATION_DATA = [
+    {
+        "id": 1,
+        "city": "New York",
+        "current_time": "2023-11-14 10:30 AM",
+        "current_weather": "Partly Cloudy, Temperature: 68°F",  # Example weather string
+    },
+    {
+        "id": 2,
+        "city": "Los Angeles",
+        "current_time": "2023-11-14 7:45 AM",
+        "current_weather": "Sunny, Temperature: 75°F",  # Example weather string
+    },
+    {
+        "id": 3,
+        "city": "Chicago",
+        "current_time": "2023-11-14 11:15 AM",
+        "current_weather": "Mostly Cloudy, Temperature: 60°F",  # Example weather string
+    },
+    {
+        "id": 4,
+        "city": "Houston",
+        "current_time": "2023-11-14 12:00 PM",
+        "current_weather": "Rainy, Temperature: 55°F",  # Example weather string
+    },
+    {
+        "id": 5,
+        "city": "Miami",
+        "current_time": "2023-11-14 1:20 PM",
+        "current_weather": "Partly Cloudy, Temperature: 80°F",  # Example weather string
+    },
+]
+
+FOOD_DATA = [
+    {
+        "id": 1,
+        "name": "Pizza",
+        "calories": 285,  # Calories per serving
+        "allergic_ingredients": ["Gluten", "Dairy"],
+    },
+    {
+        "id": 2,
+        "name": "Chocolate",
+        "calories": 50,  # Calories per serving
+        "allergic_ingredients": ["Milk", "Soy"],
+    },
+    {
+        "id": 3,
+        "name": "Sushi",
+        "calories": 300,  # Calories per serving
+        "allergic_ingredients": ["Fish", "Soy"],
+    },
+    {
+        "id": 4,
+        "name": "Burger",
+        "calories": 350,  # Calories per serving
+        "allergic_ingredients": ["Gluten", "Dairy"],
+    },
+    {
+        "id": 5,
+        "name": "Ice Cream",
+        "calories": 200,  # Calories per serving
+        "allergic_ingredients": ["Dairy"],
+    },
+    {
+        "id": 6,
+        "name": "Pasta",
+        "calories": 180,  # Calories per serving
+        "allergic_ingredients": ["Gluten"],
+    },
+    {
+        "id": 7,
+        "name": "Salad",
+        "calories": 50,  # Calories per serving
+        "allergic_ingredients": [],
+    },
+]
+
+
+class SearchHit(TypedDict, total=False):
+    """A search hit."""
+
+    id: str
+
+
+def _similarity_search(data: List[dict], query: str, key: str) -> List[SearchHit]:
+    """Return a list of data that matches the given query.
+
+    Similarity score is jaccard similarity based on the number of shared
+    characters between the query and the data.
+
+    Args:
+        data: The data to search.
+        query: The query to search for.
+        key: The key to search in.
+
+    Returns:
+        The list of matching data.
+    """
+
+    def _score_function(x: str) -> float:
+        """Calculate the similarity score between the query and the given string."""
+        return len(set(x) & set(query)) / len(set(x) | set(query))
+
+    re_ranked_data = sorted(data, key=lambda x: _score_function(x[key]), reverse=True)
+    return [{"id": d["id"], key: d[key]} for d in re_ranked_data]
+
+
+def _get_user(id: int) -> dict:
+    """Find the user with the given user ID.
+
+    Args:
+        id: The user's ID.
+
+    Returns:
+        The user's data.
+    """
+    for user in USER_DATA:
+        if user["id"] == id:
+            return user
+    raise ValueError(f"User ID {id} cannot be resolved")
+
+
+def _get_location(id: int) -> dict:
+    """Find the location with the given location ID.
+
+    Args:
+        id: The location's ID.
+
+    Returns:
+        The location's data.
+    """
+    for location in LOCATION_DATA:
+        if location["id"] == id:
+            return location
+    raise ValueError(f"Location ID {id} cannot be resolved")
+
+
+def _get_food(food_id: int) -> dict:
+    """Find the food with the given food ID.
+
+    Args:
+        food_id: The food's ID.
+
+    Returns:
+        The food's data.
+    """
+    for food in FOOD_DATA:
+        if food["id"] == food_id:
+            return food
+    raise ValueError(f"Food ID {food_id} cannot be resolved")
+
+
+def get_available_functions() -> List[Callable]:
+    """Get all the available functions."""
+
+    def get_user_name(user_id: int) -> str:
+        """Get the name of the user with the given user ID.
+
+        Args:
+            user_id: The user's ID.
+
+        Returns:
+            The user's name.
+        """
+        return _get_user(user_id)["name"]
+
+    def list_user_ids() -> List[str]:
+        """List all the user IDs."""
+        return [user["id"] for user in USER_DATA]
+
+    def find_users_by_name(name: str) -> List[SearchHit]:
+        """Find users with the given name.
+
+        Args:
+            name: The name to search for.
+
+        Returns:
+            The list of matching users.
+        """
+        return _similarity_search(USER_DATA, name, "name")
+
+    def find_locations_by_name(city: str) -> List[SearchHit]:
+        """Find locations with the given city name."""
+        return _similarity_search(LOCATION_DATA, city, "city")
+
+    def find_foods_by_name(food: str) -> List[SearchHit]:
+        """Find foods with the given name."""
+        return _similarity_search(FOOD_DATA, food, "name")
+
+    def get_user_email(user_id: int) -> str:
+        """Get the email of the user with the given user ID.
+
+        Args:
+            user_id: The user's ID.
+
+        Returns:
+            The user's email.
+        """
+        return _get_user(user_id)["email"]
+
+    def get_user_location(user_id: int) -> int:
+        """Get the location ID of the user with the given user ID.
+
+        Args:
+            user_id: The user's ID.
+
+        Returns:
+            The user's location ID.
+        """
+        return _get_user(user_id)["location"]
+
+    def get_user_favorite_color(user_id: int) -> str:
+        """Get the favorite color of the user with the given user ID.
+
+        Args:
+            user_id: The user's ID.
+
+        Returns:
+            The user's favorite color.
+        """
+        return _get_user(user_id)["favorite_color"]
+
+    def get_user_favorite_foods(user_id: int) -> List[int]:
+        """Get the list of favorite foods of the user with the given user ID.
+
+        Args:
+            user_id: The user's ID.
+
+        Returns:
+            The list of favorite foods.
+        """
+        return _get_user(user_id)["favorite_foods"]
+
+    def get_weather_at_location(location_id: int) -> str:
+        """Get the current weather at the location with the given location ID.
+
+        Args:
+            location_id: The location's ID.
+
+        Returns:
+            The current weather at the location.
+        """
+        return _get_location(location_id)["current_weather"]
+
+    def get_city_for_location(location_id: int) -> str:
+        """Get the city for the location with the given location ID.
+
+        Args:
+            location_id: The location's ID.
+
+        Returns:
+            The city name for the location.
+        """
+        return _get_location(location_id)["city"]
+
+    def get_current_time_for_location(location_id: int) -> str:
+        """Get the current time for the location with the given location ID.
+
+        Args:
+            location_id: The location's ID.
+
+        Returns:
+            The current time for the location.
+        """
+        return _get_location(location_id)["current_time"]
+
+    def get_current_weather_for_location(location_id: int) -> str:
+        """Get the current weather for the location with the given location ID.
+
+        Args:
+            location_id: The location's ID.
+
+        Returns:
+            The current weather for the location.
+        """
+        return _get_location(location_id)["current_weather"]
+
+    def get_food_name(food_id: int) -> str:
+        """Get the name of the food with the given food ID.
+
+        Args:
+            food_id: The food's ID.
+
+        Returns:
+            The name of the food.
+        """
+        return _get_food(food_id)["name"]
+
+    def get_food_calories(food_id: int) -> int:
+        """Get the calories per serving for the food with the given food ID.
+
+        Args:
+            food_id: The food's ID.
+
+        Returns:
+            The calories per serving of the food.
+        """
+        return _get_food(food_id)["calories"]
+
+    def get_food_allergic_ingredients(food_id: int) -> List[str]:
+        """Get the list of allergic ingredients for the food with the given food ID.
+
+        Args:
+            food_id: The food's ID.
+
+        Returns:
+            The list of allergic ingredients.
+        """
+        return _get_food(food_id)["allergic_ingredients"]
+
+    def get_current_user_id() -> int:
+        """Get the current user's ID.
+
+        Returns:
+            The current user's ID.
+        """
+        return 35
+
+    # Get all the functions defined in the scope of this function
+    functions = [f for f in locals().values() if callable(f)]
+    return functions
+
+
+def get_tools() -> List[BaseTool]:
+    """Get all the available tools."""
+    functions = get_available_functions()
+    return [tool(f) for f in functions]
+
+
+# ID of a dataset that contains the questions and references
+DATASET_ID = "9f73165c-d333-4d14-8f59-bd7eede5db08"  # ID of Agent Gym: E01 Alpha

langchain_benchmarks/tool_usage/evaluators.py

@@ -0,0 +1,73 @@
+"""Module contains standard evaluators for agents.
+
+Requirements:
+
+* Agents must output "intermediate_steps" in their run outputs.
+* The dataset must have "expected_steps" in its outputs.
+"""
+from typing import Optional
+
+from langchain.evaluation import EvaluatorType
+from langchain.smith import RunEvalConfig
+from langsmith.evaluation.evaluator import (
+    EvaluationResult,
+    EvaluationResults,
+    RunEvaluator,
+)
+from langsmith.schemas import Example, Run
+
+
+class AgentTrajectoryEvaluator(RunEvaluator):
+    """An evaluator that can be used in conjunction with a standard agent interface."""
+
+    def evaluate_run(
+        self, run: Run, example: Optional[Example] = None
+    ) -> EvaluationResults:
+        if run.outputs is None:
+            raise ValueError("Run outputs cannot be None")
+        # This is the output of each run
+        intermediate_steps = run.outputs["intermediate_steps"]
+        # Since we are comparing to the tool names, we now need to get that
+        # Intermediate steps is a Tuple[AgentAction, Any]
+        # The first element is the action taken
+        # The second element is the observation from taking that action
+        trajectory = [action.tool for action, _ in intermediate_steps]
+        # This is what we uploaded to the dataset
+        if example is None:
+            raise ValueError("Example cannot be None")
+        expected_trajectory = example.outputs["expected_steps"]
+
+        # Just score it based on whether it is correct or not
+        score = int(trajectory == expected_trajectory)
+        step_fraction = len(trajectory) / len(expected_trajectory)
+
+        return {
+            "results": [
+                EvaluationResult(
+                    key="Intermediate steps correctness",
+                    score=score,
+                ),
+                EvaluationResult(
+                    key="# steps / # expected steps",
+                    score=step_fraction,
+                ),
+            ]
+        }
+
+
+STANDARD_AGENT_EVALUATOR = RunEvalConfig(
+    # Evaluators can either be an evaluator type
+    # (e.g., "qa", "criteria", "embedding_distance", etc.) or a
+    # configuration for that evaluator
+    evaluators=[
+        # Measures whether a QA response is "Correct", based on a reference answer
+        # You can also select via the raw string "qa"
+        EvaluatorType.QA
+    ],
+    # You can add custom StringEvaluator or RunEvaluator objects
+    # here as well, which will automatically be
+    # applied to each prediction. Check out the docs for examples.
+    custom_evaluators=[AgentTrajectoryEvaluator()],
+    # We now need to specify this because we have multiple outputs in our dataset
+    reference_key="reference",
+)

langchain_benchmarks/utils/_langsmith.py

@@ -0,0 +1,122 @@
+"""Copy the public dataset to your own langsmith tenant."""
+import json
+import urllib.parse
+from pathlib import Path
+from typing import Union, Optional, Tuple
+from uuid import UUID
+
+from langsmith import Client
+from langsmith.utils import LangSmithNotFoundError
+from tqdm import auto
+
+WEB_API_URL = "https://web.smith.langchain.com/"
+
+
+def _parse_token_or_url(url_or_token: str, api_url: str) -> Tuple[str, Optional[str]]:
+    """Parse a public dataset URL or share token."""
+    try:
+        UUID(url_or_token)
+        return api_url, url_or_token
+    except ValueError:
+        pass
+
+    # Then it's a URL
+    parsed_url = urllib.parse.urlparse(url_or_token)
+    # Extract the UUID from the path
+    path_parts = parsed_url.path.split("/")
+    uuid = path_parts[-2] if len(path_parts) >= 2 else None
+    return WEB_API_URL, uuid
+
+
+# PUBLIC API
+
+
+def clone_public_dataset(
+    token_or_url: str,
+    *,
+    dataset_name: Optional[str] = None,
+    source_api_url: str = WEB_API_URL,
+) -> None:
+    """Clone a public dataset to your own langsmith tenant.
+
+    This operation is idempotent. If you already have a dataset with the given name,
+    this function will do nothing.
+
+    Args:
+        token_or_url (str): The token of the public dataset to clone.
+        dataset_name (str): The name of the dataset to create in your tenant.
+        source_api_url: The URL of the langsmith server where the data is hosted:w
+    """
+    if dataset_name is None:
+        raise NotImplementedError(
+            "Automatic dataset name generation is not implemented yet"
+        )
+    client = Client()  # Client used to write to langsmith
+    try:
+        dataset = client.read_dataset(dataset_name=dataset_name)
+
+        if dataset:
+            print(f"Dataset {dataset_name} already exists. Skipping.")
+            print(f"You can access the dataset at {dataset.url}.")
+            return
+    except LangSmithNotFoundError:
+        pass
+
+    source_api_url, uuid = _parse_token_or_url(token_or_url, source_api_url)
+    source_client = Client(api_url=source_api_url, api_key="placeholder")
+    try:
+        # Fetch examples first
+        examples = auto.tqdm(list(source_client.list_shared_examples(uuid)))
+        print("Finished fetching examples. Creating dataset...")
+        dataset = client.create_dataset(dataset_name=dataset_name)
+        print(f"New dataset created you can access it at {dataset.url}.")
+        try:
+            client.create_examples(
+                inputs=[e.inputs for e in examples],
+                outputs=[e.outputs for e in examples],
+                dataset_id=dataset.id,
+            )
+        except BaseException as e:
+            # Let's not do automatic clean up for now in case there might be
+            # some other reasons why create_examples fails (i.e., not network issue or
+            # keyboard interrupt).
+            # The risk is that this is an existing dataset that has valid examples
+            # populated from another source so we don't want to delete it.
+            print(
+                f"An error occurred while creating dataset {dataset_name}. "
+                "You should delete it manually."
+            )
+            raise e
+
+        print("Done creating dataset.")
+    finally:
+        del source_client
+        del client
+
+
+def download_public_dataset(
+    token_or_url: str,
+    *,
+    path: Optional[Union[str, Path]] = None,
+    api_url: str = WEB_API_URL,
+) -> None:
+    """Download a public dataset."""
+    api_url, uuid = _parse_token_or_url(token_or_url, api_url)
+    _path = str(path) if path else f"{uuid}.json"
+    if not _path.endswith(".json"):
+        raise ValueError(f"Path must end with .json got: {_path}")
+
+    # This the client where the source data lives
+    # The destination for the dataset is the local filesystem
+    source_client = Client(api_url=api_url, api_key="placeholder")
+
+    try:
+        # Fetch examples first
+        print("Fetching examples...")
+        examples = auto.tqdm(list(source_client.list_shared_examples(uuid)))
+        with open(str(_path), mode="w", encoding="utf-8") as f:
+            jsonifable_examples = [json.loads(example.json()) for example in examples]
+            json.dump(jsonifable_examples, f, indent=2)
+        print("Done fetching examples.")
+    finally:
+        del source_client

meta-evals/README.md

@@ -0,0 +1,4 @@
+# Meta-Evaluations
+
+
+This folder holds some scripts/tests for evaluating some of LangChain's default evaluators.

meta-evals/correctness/README.md

@@ -0,0 +1,17 @@
+# Correctness Meta-Evals
+
+This folder contains a test script to check the aggregate performance of the "correctness"-related evaluators.
+
+To upload the dataset to LangSmith, run:
+
+```bash
+python meta-evals/correctness/_upload_dataset.py
+```
+
+To test, run:
+
+```bash
+pytest --capture=no meta-evals/correctness/test_correctness_evaluator.py
+```
+
+Then navigate to the Web Q&A dataset to review the results.

meta-evals/correctness/_upload_dataset.py

@@ -0,0 +1,36 @@
+import json
+import logging
+from pathlib import Path
+
+from langsmith import Client
+
+logging.basicConfig(level=logging.INFO)
+
+# Synthetic dataset adapted from https://aclanthology.org/D13-1160/
+
+_DATA_REPO = Path(__file__).parent / "data"
+_CLIENT = Client()
+
+
+def _upload_dataset(path: str):
+    with open(path, "r") as f:
+        data = json.load(f)
+        dataset_name = data["name"]
+        examples = data["examples"]
+        try:
+            dataset = _CLIENT.create_dataset(dataset_name)
+        except Exception:
+            logging.warning(f"Skipping {dataset_name}")
+            return
+        logging.info(f"Uploading dataset: {dataset_name}")
+        _CLIENT.create_examples(
+            inputs=[example["inputs"] for example in examples],
+            outputs=[example["outputs"] for example in examples],
+            dataset_id=dataset.id,
+        )
+
+
+if __name__ == "__main__":
+    for dataset in _DATA_REPO.glob("*.json"):
+        print("Uploading dataset:", dataset)
+        _upload_dataset(dataset)

meta-evals/correctness/data/Web-Q&A-Dataset-Correct.json

@@ -0,0 +1,805 @@
+{
+  "name": "Web Q&A Dataset - Correct",
+  "examples": [
+    {
+      "inputs": {
+        "input": "who does the author stephenie meyer play in the movie twilight?",
+        "input_answer": "The answer to the question 'who does the author stephenie meyer play in the movie twilight?' is Woman at Wedding",
+        "input_prediction": "Woman at Wedding"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where is jefferson davis buried?",
+        "input_answer": "The answer to the question 'where is jefferson davis buried?' is Hollywood Cemetery",
+        "input_prediction": "Hollywood Cemetery"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where do the ny rangers play?",
+        "input_answer": "The answer to the question 'where do the ny rangers play?' is Madison Square Garden",
+        "input_prediction": "Madison Square Garden"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who is robert downey jr wife?",
+        "input_answer": "The answer to the question 'who is robert downey jr wife?' is Susan Downey",
+        "input_prediction": "Susan Downey"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what kind of money should i take to jamaica?",
+        "input_answer": "The answer to the question 'what kind of money should i take to jamaica?' is Jamaican dollar",
+        "input_prediction": "Jamaican dollar"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who founded the roanoke settlement?",
+        "input_answer": "The answer to the question 'who founded the roanoke settlement?' is walter raleigh",
+        "input_prediction": "Sir Walter Raleigh"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is charles darwin famous for?",
+        "input_answer": "The answer to the question 'what is charles darwin famous for?' is Evolution",
+        "input_prediction": "Evolution"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who miley cyrus engaged to?",
+        "input_answer": "The answer to the question 'who miley cyrus engaged to?' is Liam Hemsworth",
+        "input_prediction": "Liam Hemsworth"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what sea does the yangtze river flow into?",
+        "input_answer": "The answer to the question 'what sea does the yangtze river flow into?' is East China Sea",
+        "input_prediction": "East China Sea"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where did timothy mcveigh go to prison?",
+        "input_answer": "The answer to the question 'where did timothy mcveigh go to prison?' is Terre Haute",
+        "input_prediction": "Terre Haute"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "which part of korea is communist?",
+        "input_answer": "The answer to the question 'which part of korea is communist?' is North Korea",
+        "input_prediction": "North Korea"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where is santa clara university?",
+        "input_answer": "The answer to the question 'where is santa clara university?' is Santa Clara, California",
+        "input_prediction": "Santa Clara, California, USA"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what language do the speak in pakistan?",
+        "input_answer": "The answer to the question 'what language do they speak in pakistan?' is English and Urdu",
+        "input_prediction": "English and Urdu are the official languages. Other languages include punjabi and pashto"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where is jack daniels factory?",
+        "input_answer": "The answer to the question 'where is jack daniels factory?' is Tennessee",
+        "input_prediction": "Tennessee"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what was woodrow wilson's major accomplishments?",
+        "input_answer": "The answer to the question 'what was woodrow wilson's major accomplishments?' is being the United States President and starting the league of nations",
+        "input_prediction": "Starting the league of nations as President of the USA"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is the official language of china 2010?",
+        "input_answer": "The answer to the question 'what is the official language of china 2010?' is Chinese language",
+        "input_prediction": "Chinese language"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "when did the new york knicks win a championship?",
+        "input_answer": "The answer to the question 'when did the new york knicks win a championship?' is 1970 NBA Finals",
+        "input_prediction": "1970 NBA Finals"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what time zone am i in michigan?",
+        "input_answer": "The answer to the question 'what time zone am i in michigan?' is North American Eastern Time Zone",
+        "input_prediction": "North American Eastern Time Zone"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where to get married in jacksonville nc?",
+        "input_answer": "One nearby place to try is Atlantic Beach",
+        "input_prediction": "Atlantic Beach is a great weddling location you can check out"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who was vincent van gogh inspired by?",
+        "input_answer": "The answer to the question 'who was vincent van gogh inspired by?' is Jozef Isra\u00ebls",
+        "input_prediction": "Jozef Isra\u00ebls"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where did france colonise?",
+        "input_answer": "The answer to the question 'where did france colonise?' is R\u00e9union",
+        "input_prediction": "R\u00e9union"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what school did joan miro attend?",
+        "input_answer": "The answer to the question 'what school did joan miro attend?' is Cercle Art\u00edstic de Sant Lluc",
+        "input_prediction": "Cercle Art\u00edstic de Sant Lluc"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where to go in london england?",
+        "input_answer": "The answer to the question 'where to go in london england?' is Regent's Park",
+        "input_prediction": "Regent's Park"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what college did matt lauer attend?",
+        "input_answer": "The answer to the question 'what college did matt lauer attend?' is Ohio University",
+        "input_prediction": "Ohio University"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who was peter parker's first girlfriend?",
+        "input_answer": "The answer to the question 'who was peter parker's first girlfriend?' is Gwen",
+        "input_prediction": "gwen"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "when did the colorado rockies go to the world series?",
+        "input_answer": "The answer to the question 'when did the colorado rockies go to the world series?' is 2007 ",
+        "input_prediction": "The rockies went to the world series in 2007, where they also won the National League Championship Series"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what form of government was practiced in sparta?",
+        "input_answer": "The answer to the question 'what form of government was practiced in sparta?' is Monarchy",
+        "input_prediction": "Monarchy"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where does jackie french live?",
+        "input_answer": "The answer to the question 'where does jackie french live?' is Australia",
+        "input_prediction": "Australia"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what character did brian austin green play on 90210?",
+        "input_answer": "The answer to the question 'what character did brian austin green play on 90210?' is David Silver",
+        "input_prediction": "David Silver"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what timezone is denver co?",
+        "input_answer": "The answer to the question 'what timezone is denver co?' is Mountain Time Zone",
+        "input_prediction": "Mountain Time Zone"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what tv show did joey lawrence play on?",
+        "input_answer": "The answer to the question 'what tv show did joey lawrence play on?' is Blossom",
+        "input_prediction": "Blossom"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "when is summer in australia sydney?",
+        "input_answer": "The answer to the question 'when is summer in australia sydney?' is from November to March",
+        "input_prediction": "Nov-March"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what style of music did louis armstrong play?",
+        "input_answer": "The answer to the question 'what style of music did louis armstrong play?' is Jazz",
+        "input_prediction": "Jazz"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who does maggie grace play in taken?",
+        "input_answer": "The answer to the question 'who does maggie grace play in taken?' is Kim",
+        "input_prediction": "Kim"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what timezone is australia gmt?",
+        "input_answer": "The answer to the question 'what timezone is australia gmt?' is UTC+8",
+        "input_prediction": "UTC+8"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what was tupac name in juice?",
+        "input_answer": "The answer to the question 'what was tupac name in juice?' is Bishop",
+        "input_prediction": "Bishop"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who is in the band bush?",
+        "input_answer": "The answer to the question 'who is in the band bush?' is Dave Parsons",
+        "input_prediction": "Dave Parsons"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "name one of the names of walt disney kids?",
+        "input_answer": "The answer to the question 'name of of the names of walt disney kids?' is Diane Disney Miller",
+        "input_prediction": "Diane Disney Miller is one of Walt Disney's children"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "Which county is Flemburg, PA in?",
+        "input_answer": "The answer to the question 'Which county is Flemburg, PA in?' is Cumberland County",
+        "input_prediction": "Cumberland"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "Name a country where people speak portuguese?",
+        "input_answer": "The answer to the question is Brazil",
+        "input_prediction": "Brazil"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where does lena river begin?",
+        "input_answer": "The answer to the question 'where does lena river begin?' is Baikal Mountains",
+        "input_prediction": "Baikal Mountains"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what type of car does Charmandr Beaton drive?",
+        "input_answer": "The answer to the question 'what type of car does Charmandr Beaton drive?' is Wishcraft",
+        "input_prediction": "Wishcraft is the type of car"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is the first book sherlock holmes appeared in?",
+        "input_answer": "The answer to the question 'what is the first book sherlock holmes appeared in?' is A Study in Scarlet",
+        "input_prediction": "A Study in Scarlet"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what did sally field get in math class in 4th grade?",
+        "input_answer": "The answer to the question 'what did sally field get in math class in 4th grade?' is a B+",
+        "input_prediction": "Sally got a B+"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who was queen victoria's father called?",
+        "input_answer": "The answer to the question 'who was queen victoria's father called?' is Prince Edward, Duke of Kent and Strathearn",
+        "input_prediction": "Prince Edward, Duke of Kent and Strathearn"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what was the name of frederick douglass book?",
+        "input_answer": "The answer to the question 'what was the name of frederick douglass book?' is Narrative of the Life of Frederick Douglass, an American Slave",
+        "input_prediction": "Narrative of the Life of Frederick Douglass, an American Slave"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what year did Maureen Jenson run for office?",
+        "input_answer": "The answer to the question 'what year did Maureen Jenson run for office?' is 2018 in Flibbings, NJ",
+        "input_prediction": "2018"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where do the san francisco giants play home games?",
+        "input_answer": "The answer to the question 'where do the san francisco giants play home games?' is AT&T Park",
+        "input_prediction": "AT&T Park"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what season did tony soprano get shot?",
+        "input_answer": "The answer to the question 'what season did tony soprano get shot?' is the seventh season",
+        "input_prediction": "season 7"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "What were the action items for this meeting?",
+        "input_answer": "James was to send out the minutes, Anna was to share the reports with Avi, and Jane agreed to share the api keys for the deployment with Nora",
+        "input_prediction": "- Anna will send the reports to Avi\nJane will give Nora the information about the deployment's API keys\nJames will send the minutes "
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what movie is josh hutcherson in?",
+        "input_answer": "The answer to the question 'what movie is josh hutcherson in?' is Cirque du Freak: The Vampire's Assistant",
+        "input_prediction": "Cirque du Freak: The Vampire's Assistant"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where did dmitri mendeleev study science?",
+        "input_answer": "The answer to the question 'where did dmitri mendeleev study science?' is Saint Petersburg State University",
+        "input_prediction": "Saint Petersburg State University"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what county is heathrow airport in?",
+        "input_answer": "The answer to the question 'what county is heathrow airport in?' is London",
+        "input_prediction": "London"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what did william shakespeare do for a living?",
+        "input_answer": "The answer to the question 'what did william shakespeare do for a living?' is Poet",
+        "input_prediction": "Poet"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is the name of the first harry potter novel?",
+        "input_answer": "The answer to the question 'what is the name of the first harry potter novel?' is Harry Potter and the Philosopher's Stone",
+        "input_prediction": "Harry Potter and the Philosopher's Stone"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what county is the city of hampton va in?",
+        "input_answer": "The answer to the question 'what county is the city of hampton va in?' is Hampton",
+        "input_prediction": "Hampton"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is Project Ambiance??",
+        "input_answer": "The answer to the question 'waht is Project Ambiance?' is: the project is a secret engineering undertaking by HP focused on modernizing their fiber investments",
+        "input_prediction": "Project Ambiance is the code-name for the secret project to improve HP's fiber infrastructure investments"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what party was Sean?",
+        "input_answer": "The answer to the question 'what party was Sean?' is the purble eggs party",
+        "input_prediction": "He was part of PEP, the purple eggs party"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is shana cooper doing now?",
+        "input_answer": "The answer to the question 'what is shana cooper doing now?' is that she is an author",
+        "input_prediction": "an author"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is cher's son's name?",
+        "input_answer": "The answer to the question 'what is cher's son's name?' is Elijah Blue Allman",
+        "input_prediction": "Elijah Blue Allman"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "when is the new series of the only way is essex starting?",
+        "input_answer": "The answer to the question 'when is the new series of the only way is essex starting?' is next month",
+        "input_prediction": "next month"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is Egroeg known for?",
+        "input_answer": "The answer to the question 'what is Egroeg known for?' is that he is a rapper",
+        "input_prediction": "Egroeg is a rapper and musician"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what sport do the toronto maple leafs play?",
+        "input_answer": "The answer to the question 'what sport do the toronto maple leafs play?' is Ice Hockey",
+        "input_prediction": "Ice Hockey"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what city did esther live in?",
+        "input_answer": "The answer to the question 'what city did esther live in?' is Susa",
+        "input_prediction": "Susa"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "which department is Cemalitryx known for?",
+        "input_answer": "The answer to the question 'which department is Cemalitryx known for?' is their legal team.",
+        "input_prediction": "The attorneys at Cemalitryx are renown."
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is the president of brazil?",
+        "input_answer": "The answer to the question 'what is the president of brazil?' is Dilma Rousseff",
+        "input_prediction": "Dilma Rousseff"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where are the netherlands on a world map? What is its name?",
+        "input_answer": "The answer to the question 'where are the netherlands on a world map?' is west of germany. The full name is the Kingdom of the Netherlands",
+        "input_prediction": "Northern europe, west of geremany. The full name is Kingdom of the Netherlands"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what was malcolm x trying to accomplish?",
+        "input_answer": "The answer to the question 'what was malcolm x trying to accomplish?' is African Americans' rights",
+        "input_prediction": "African Americans' rights"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who was ishmael's mom?",
+        "input_answer": "The answer to the question 'who was ishmael's mom?' is Hagar",
+        "input_prediction": "Hagar"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "which wife did king henry behead?",
+        "input_answer": "The answer to the question 'which wife did king henry behead?' is Anne",
+        "input_prediction": "Anne"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what does thai mean?",
+        "input_answer": "The answer to the question 'what does thai mean?' is Language",
+        "input_prediction": "Language"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "how much did adriana lima earn in the month after the Spike award show ?",
+        "input_answer": "The answer to the question 'how much did adriana lima earn in the month after the Spike award show ?' is 32 grand ",
+        "input_prediction": "$32,000"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what did the german revolution lead to?",
+        "input_answer": "The answer to the question 'what did the german revolution lead to?' is German Democratic Republic",
+        "input_prediction": "German Democratic Republic"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what year lebron james came to the nba?",
+        "input_answer": "The answer to the question 'what year lebron james came to the nba?' is 2003\u201304 NBA season",
+        "input_prediction": "2003\u201304 NBA season"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what movies does taylor lautner play in?",
+        "input_answer": "The answer to the question 'what movies does taylor lautner play in?' is Abduction",
+        "input_prediction": "Abduction"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what form of government did Jen propose yesterday?",
+        "input_answer": "The answer to the question 'what form of government did Jen propose yesterday?' is Constitutional republic",
+        "input_prediction": "Constitutional republic"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what high school did president bill clinton attend?",
+        "input_answer": "The answer to the question 'what high school did president bill clinton attend?' is Hot Springs High School",
+        "input_prediction": "Hot Springs High School"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what to see near sedona arizona?",
+        "input_answer": "The answer to the question 'what to see near sedona arizona?' is Sedona Airport",
+        "input_prediction": "Sedona Airport"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who is the prime minister of ethiopia?",
+        "input_answer": "The answer to the question 'who is the prime minister of ethiopia?' is Hailemariam Desalegn",
+        "input_prediction": "Hailemariam Desalegn"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where to fly into bali?",
+        "input_answer": "The answer to the question 'where to fly into bali?' is Ngurah Rai Airport",
+        "input_prediction": "Ngurah Rai Airport"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who did the philippines gain independence from?",
+        "input_answer": "The answer to the question 'who did the philippines gain independence from?' is United States of America",
+        "input_prediction": "United States of America"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what books did beverly cleary right?",
+        "input_answer": "The answer to the question 'what books did beverly cleary right?' is Ramona's World",
+        "input_prediction": "Ramona's World"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "which kennedy died first?",
+        "input_answer": "The answer to the question 'which kennedy died first?' is John F. Kennedy",
+        "input_prediction": "John F. Kennedy"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what country does iceland belong to?",
+        "input_answer": "The answer to the question 'what country does iceland belong to?' is Iceland",
+        "input_prediction": "Iceland is its own country and doesn't belong to any other country"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is nina dobrev nationality?",
+        "input_answer": "The answer to the question 'what is nina dobrev nationality?' is Bulgaria",
+        "input_prediction": "Bulgaria"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where is rome italy located on a map?",
+        "input_answer": "The answer to the question 'where is rome italy located on a map?' is central italy",
+        "input_prediction": "in the center of italy"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "which countries border the us?",
+        "input_answer": "The answer to the question 'which countries border the us?' is mexico and Canada",
+        "input_prediction": "Canada and mexico"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who did draco malloy end up marrying?",
+        "input_answer": "The answer to the question 'who did draco malloy end up marrying?' is Astoria Greengrass",
+        "input_prediction": "Astoria Greengrass"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what time zone am i in cleveland ohio?",
+        "input_answer": "The answer to the question 'what time zone am i in cleveland ohio?' is North American Eastern Time Zone",
+        "input_prediction": "North American Eastern Time Zone"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what two countries invaded poland in the beginning of ww2?",
+        "input_answer": "The answer to the question 'what two countries invaded poland in the beginning of ww2?' is Germany and russia",
+        "input_prediction": "Germany + russia"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "how old is sacha andra smith?",
+        "input_answer": "The answer to the question 'how old is sacha andra smith?' is 32, found on http://www.buzznet.com/sacha-andra-age",
+        "input_prediction": "32"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where did saki live?",
+        "input_answer": "The answer to the question 'where did saki live?' is United Kingdom",
+        "input_prediction": "United Kingdom"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where are the nfl redskins from?",
+        "input_answer": "The answer to the question 'where are the nfl redskins from?' is Washington Redskins",
+        "input_prediction": "Washington Redskins"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who does joakim noah play for?",
+        "input_answer": "The answer to the question 'who does joakim noah play for?' is Chicago Bulls",
+        "input_prediction": "Chicago Bulls"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what character did john noble play in lord of the rings?",
+        "input_answer": "The answer to the question 'what character did john noble play in lord of the rings?' is Denethor II",
+        "input_prediction": "Denethor II"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what kind of money to take to bahamas?",
+        "input_answer": "The answer to the question 'what kind of money to take to bahamas?' is Bahamian dollar",
+        "input_prediction": "Bahamian dollar"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what country is the grand bahama island in?",
+        "input_answer": "The answer to the question 'what country is the grand bahama island in?' is Bahamas",
+        "input_prediction": "Bahamas"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what state does selena gomez?",
+        "input_answer": "The answer to the question 'what state does selena gomez?' is New York",
+        "input_prediction": "New York"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what character did natalie portman play in star wars?",
+        "input_answer": "The answer to the question 'what character did natalie portman play in star wars?' is Padm\u00e9 Amidala",
+        "input_prediction": "Padm\u00e9 Amidala"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is the name of justin bieber sister?",
+        "input_answer": "The answer to the question 'what is the name of justin bieber sister?' is Jazmyn Bieber",
+        "input_prediction": "Jazmyn Bieber"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    }
+  ]
+}

meta-evals/correctness/test_correctness_evaluator.py

@@ -0,0 +1,206 @@
+# Tests for the criteria evaluator
+from typing import Tuple
+from uuid import uuid4
+
+import pytest
+from langchain import chat_models, hub, llms
+from langchain.evaluation import load_evaluator
+from langchain.schema import runnable
+from langchain.smith import RunEvalConfig, run_on_dataset
+from langsmith import Client, EvaluationResult
+from langsmith.evaluation import RunEvaluator
+from langsmith.schemas import Example, Run
+
+
+class ExactScoreMatch(RunEvaluator):
+    def evaluate_run(self, run: Run, example: Example) -> EvaluationResult:
+        predicted_score = run.outputs["score"]
+        return EvaluationResult(
+            key="exact_score_match",
+            score=predicted_score == example.outputs["output_correctness_score"],
+        )
+
+
+class AbsDistanceEvaluator(RunEvaluator):
+    def __init__(self) -> None:
+        super().__init__()
+
+    def evaluate_run(self, run: Run, example: Example) -> EvaluationResult:
+        predicted_score = run.outputs["score"]
+        if predicted_score is None:
+            return EvaluationResult(key="absolute_distance", score=None)
+        return EvaluationResult(
+            key="absolute_distance",
+            score=abs(predicted_score - example.outputs["output_correctness_score"]),
+        )
+
+
+class NullScore(RunEvaluator):
+    def __init__(self) -> None:
+        super().__init__()
+
+    def evaluate_run(self, run: Run, example: Example) -> EvaluationResult:
+        predicted_score = run.outputs["score"]
+        null_score = 1 if predicted_score is None else 0
+        return EvaluationResult(key="null_score", score=null_score)
+
+
+@pytest.fixture(scope="session")
+def uid() -> str:
+    return uuid4().hex[:8]
+
+
+def _check_dataset(
+    loader_kwargs: dict,
+    dataset_name: str,
+    project_name: str,
+    model_provider: str,
+    model_name: str,
+    tags: list,
+    metadata: dict,
+) -> Tuple[float, float]:
+    client = Client()
+    match model_provider:
+        case "openai":
+            llm = chat_models.ChatOpenAI(model_name=model_name, temperature=0)
+        case "openai-completion":
+            llm = llms.OpenAI(model_name=model_name, temperature=0)
+        case "anthropic":
+            llm = chat_models.ChatAnthropic(
+                model_name=model_name, max_tokens=1000, temperature=0
+            )
+    eval_chain = load_evaluator(**loader_kwargs, llm=llm)
+
+    def to_evaluate(input: dict, config: runnable.RunnableConfig) -> dict:
+        return eval_chain.evaluate_strings(
+            input=input["input"],
+            prediction=input["input_prediction"],
+            reference=input["input_answer"],
+            **config,
+        )
+
+    res = run_on_dataset(
+        dataset_name=dataset_name,
+        llm_or_chain_factory=runnable.RunnableLambda(to_evaluate).with_config(
+            {"metadata": metadata}
+        ),
+        evaluation=RunEvalConfig(
+            custom_evaluators=[
+                ExactScoreMatch(),
+                #   AbsDistanceEvaluator(),
+                NullScore(),
+            ],
+        ),
+        concurrency_level=8,
+        client=client,
+        verbose=True,
+        project_name=project_name,
+        tags=["int-test"] + tags,
+    )
+    df = res.to_dataframe()
+    feedback_cols = [
+        col for col in df.columns if col not in ["input", "output", "reference"]
+    ]
+    # Return averages
+    return df[feedback_cols].mean()
+
+
+def _get_project_name(
+    loader_kwargs: dict, uid: str, dataset_name: str, model: Tuple[str]
+) -> str:
+    other_args = "-".join(f"[{k}={v}]" for k, v in loader_kwargs.items())
+    return f"{model[0]}.{model[1]}-{loader_kwargs['evaluator']}{other_args} - {dataset_name} - {uid}"
+
+
+# prompt = "wfh/criteria_candidates"
+# commits = [
+#     "f470538b",
+#     "c92fcf90",
+# ]
+
+# anthropic_prompt = "wfh/criteria_candidates_anthropic"
+
+prompt_list = [
+    {
+        "openai": "wfh/criteria_candidates:f470538b",
+        "anthropic": "wfh/criteria_candidates_anthropic:fb037730",
+    },  # Here we'll try the anthropic one that inserts the "reasoning" step in the AI's mouth
+    {
+        "openai": "wfh/criteria_candidates:c92fcf90",
+    },  # It's the same as the openai one
+]
+
+
+@pytest.mark.parametrize(
+    "dataset_name",
+    [
+        "Web Q&A Dataset - Incorrect",
+        "Carb-IE-Test INCORRECT",
+        "Opus100 - Incorrect",
+    ]
+    + [
+        "Web Q&A Dataset - Correct",
+        "Opus100 - Correct",
+        "Carb-IE-Test CORRECT",
+    ],
+)
+@pytest.mark.parametrize(
+    "model",
+    [
+        ("openai", "gpt-4"),
+        ("openai", "gpt-3.5-turbo"),
+        ("openai-completion", "gpt-3.5-turbo-instruct"),
+        ("anthropic", "claude-2"),
+    ],
+)
+@pytest.mark.parametrize(
+    "loader_kwargs",
+    [
+        {"evaluator": "cot_qa"},
+        {"evaluator": "qa"},
+    ]
+    + [
+        {
+            "evaluator": "labeled_criteria",
+            "criteria": "correctness",
+            "prompt_lookup": pl,
+        }
+        for pl in prompt_list
+    ],
+)
+@pytest.mark.asyncio
+async def test_metaeval_correctness(
+    loader_kwargs: dict, uid: str, dataset_name: str, model: Tuple[str, str]
+):
+    project_name = _get_project_name(loader_kwargs, uid, dataset_name, model)
+    tags = ["test_metaeval_correctness", loader_kwargs["evaluator"]]
+    metadata = {
+        "model_provider": model[0],
+        "model_name": model[1],
+        "dataset_name": dataset_name,
+        "evaluator": loader_kwargs["evaluator"],
+        "uid": str(uid),
+    }
+    if "prompt_lookup" in loader_kwargs:
+        prompt_lookup = loader_kwargs.get("prompt_lookup")
+        prompt_repo = prompt_lookup.get(
+            model[0], prompt_lookup.get("openai")
+        )  # Fall back on openai prompt
+        commit = prompt_repo.split(":")[-1]
+        prompt = hub.pull(prompt_repo)
+        loader_kwargs["prompt"] = prompt
+        tags += [commit]
+        print("Using prompt:", prompt_repo, commit)
+        metadata["prompt"] = prompt_repo
+        metadata["commit"] = commit
+    scores = _check_dataset(
+        loader_kwargs,
+        dataset_name,
+        project_name,
+        model_provider=model[0],
+        model_name=model[1],
+        tags=tags,
+        metadata=metadata,
+    )
+    score = scores["exact_score_match"]
+    assert score >= 0.95

pyproject.toml

@@ -0,0 +1,86 @@
+[tool.poetry]
+name = "langchain-benchmarks"
+version = "0.0.1"
+description = "Flex them feathers! 🦜💪"
+authors = ["LangChain AI"]
+license = "MIT"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.8.1"
+langchain = ">=0.0.333"
+langsmith = "^0.0.64"
+tqdm = "^4.66.1"
+ipywidgets = "^8.1.1"
+
+[tool.poetry.group.dev.dependencies]
+jupyterlab = "^3.6.1"
+
+[tool.poetry.group.typing.dependencies]
+mypy = "^1.7.0"
+
+[tool.poetry.group.lint.dependencies]
+ruff = "^0.1.5"
+
+[tool.poetry.group.docs.dependencies]
+nbsphinx = "^0.8.9"
+sphinx = "^4.5.0"
+sphinx-autobuild = "^2021.3.14"
+sphinx_book_theme = "^1.0.0"
+myst-nb = "^0.17.1"
+linkchecker = "^10.2.1"
+toml = "^0.10.2"
+sphinx-copybutton = "^0.5.1"
+
+
+[tool.poetry.group.test.dependencies]
+pytest = "^7.2.1"
+pytest-cov = "^4.0.0"
+pytest-asyncio = "^0.21.1"
+pytest-mock = "^3.11.1"
+pytest-socket = "^0.6.0"
+pytest-watch = "^4.2.0"
+pytest-timeout = "^2.2.0"
+
+
+[tool.ruff]
+select = [
+    "E",  # pycodestyle
+    "F",  # pyflakes
+    "I",  # isort
+]
+
+# Same as Black.
+line-length = 88
+
+[tool.ruff.isort]
+# TODO(Team): Temporary to make isort work with examples.
+# examples assume langserve is available as a 3rd party package
+# For simplicity we'll define it as first party for now can update later.
+known-first-party = ["langserve"]
+
+[tool.mypy]
+disallow_untyped_defs = "True"
+ignore_missing_imports = "True"
+
+[tool.coverage.run]
+omit = [
+    "tests/*",
+]
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.pytest.ini_options]
+# --strict-markers will raise errors on unknown marks.
+# https://docs.pytest.org/en/7.1.x/how-to/mark.html#raising-errors-on-unknown-marks
+#
+# https://docs.pytest.org/en/7.1.x/reference/reference.html
+# --strict-config       any warnings encountered while parsing the `pytest`
+#                       section of the configuration file raise errors.
+addopts = "--strict-markers --strict-config --durations=5 -vv"
+# Global timeout for all tests. There shuold be a good reason for a test to
+# take more than 5 second
+timeout = 5

tests/unit_tests/test_public_api.py

@@ -0,0 +1,8 @@
+from langchain_benchmarks import __all__
+
+
+def test_public_api() -> None:
+    """Test that the public API is correct."""
+    # This test will also fail if __all__ is not sorted.
+    # Please keep it sorted!
+    assert __all__ == sorted(["clone_public_dataset", "download_public_dataset"])