Add collab link script

Faster for notebooks, etc.

.github/actions/poetry_setup/action.yml

@@ -0,0 +1,91 @@
+# An action for setting up poetry install with caching.
+# Using a custom action since the default action does not
+# take poetry install groups into account.
+# Action code from:
+# https://github.com/actions/setup-python/issues/505#issuecomment-1273013236
+name: poetry-install-with-caching
+description: Poetry install with support for caching of dependency groups.
+
+inputs:
+  python-version:
+    description: Python version, supporting MAJOR.MINOR only
+    required: true
+
+  poetry-version:
+    description: Poetry version
+    required: true
+
+  cache-key:
+    description: Cache key to use for manual handling of caching
+    required: true
+
+  working-directory:
+    description: Directory whose poetry.lock file should be cached
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/setup-python@v4
+      name: Setup python ${{ inputs.python-version }}
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - uses: actions/cache@v3
+      id: cache-bin-poetry
+      name: Cache Poetry binary - Python ${{ inputs.python-version }}
+      env:
+        SEGMENT_DOWNLOAD_TIMEOUT_MIN: "1"
+      with:
+        path: |
+          /opt/pipx/venvs/poetry
+        # This step caches the poetry installation, so make sure it's keyed on the poetry version as well.
+        key: bin-poetry-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-${{ inputs.poetry-version }}
+
+    - name: Refresh shell hashtable and fixup softlinks
+      if: steps.cache-bin-poetry.outputs.cache-hit == 'true'
+      shell: bash
+      env:
+        POETRY_VERSION: ${{ inputs.poetry-version }}
+        PYTHON_VERSION: ${{ inputs.python-version }}
+      run: |
+        set -eux
+
+        # Refresh the shell hashtable, to ensure correct `which` output.
+        hash -r
+
+        # `actions/cache@v3` doesn't always seem able to correctly unpack softlinks.
+        # Delete and recreate the softlinks pipx expects to have.
+        rm /opt/pipx/venvs/poetry/bin/python
+        cd /opt/pipx/venvs/poetry/bin
+        ln -s "$(which "python$PYTHON_VERSION")" python
+        chmod +x python
+        cd /opt/pipx_bin/
+        ln -s /opt/pipx/venvs/poetry/bin/poetry poetry
+        chmod +x poetry
+
+        # Ensure everything got set up correctly.
+        /opt/pipx/venvs/poetry/bin/python --version
+        /opt/pipx_bin/poetry --version
+
+    - name: Install poetry
+      if: steps.cache-bin-poetry.outputs.cache-hit != 'true'
+      shell: bash
+      env:
+        POETRY_VERSION: ${{ inputs.poetry-version }}
+        PYTHON_VERSION: ${{ inputs.python-version }}
+      run: pipx install "poetry==$POETRY_VERSION" --python "python$PYTHON_VERSION" --verbose
+
+    - name: Restore pip and poetry cached dependencies
+      uses: actions/cache@v3
+      env:
+        SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4"
+        WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }}
+      with:
+        path: |
+          ~/.cache/pip
+          ~/.cache/pypoetry/virtualenvs
+          ~/.cache/pypoetry/cache
+          ~/.cache/pypoetry/artifacts
+          ${{ env.WORKDIR }}/.venv
+        key: py-deps-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-poetry-${{ inputs.poetry-version }}-${{ inputs.cache-key }}-${{ hashFiles(format('{0}/**/poetry.lock', env.WORKDIR)) }}

.github/workflows/_lint.yml

@@ -0,0 +1,83 @@
+name: lint
+
+on:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+
+env:
+  POETRY_VERSION: "1.6.1"
+  WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      # This number is set "by eye": we want it to be big enough
+      # so that it's bigger than the number of commits in any reasonable PR,
+      # and also as small as possible since increasing the number makes
+      # the initial `git fetch` slower.
+      FETCH_DEPTH: 50
+    strategy:
+      matrix:
+        # Only lint on the min and max supported Python versions.
+        # It's extremely unlikely that there's a lint issue on any version in between
+        # that doesn't show up on the min or max versions.
+        #
+        # GitHub rate-limits how many jobs can be running at any one time.
+        # Starting new jobs is also relatively slow,
+        # so linting on fewer versions makes CI faster.
+        python-version:
+          - "3.8"
+          - "3.11"
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
+          cache-key: lint-with-extras
+
+      - name: Check Poetry File
+        shell: bash
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          poetry check
+
+      - name: Check lock file
+        shell: bash
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          poetry lock --check
+
+      - name: Install dependencies
+        # Also installs dev/lint/test/typing dependencies, to ensure we have
+        # type hints for as many of our libraries as possible.
+        # This helps catch errors that require dependencies to be spotted, for example:
+        # https://github.com/langchain-ai/langchain/pull/10249/files#diff-935185cd488d015f026dcd9e19616ff62863e8cde8c0bee70318d3ccbca98341
+        #
+        # If you change this configuration, make sure to change the `cache-key`
+        # in the `poetry_setup` action above to stop using the old cache.
+        # It doesn't matter how you change it, any change will cause a cache-bust.
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          poetry install --with dev,lint,test,typing
+
+      - name: Get .mypy_cache to speed up mypy
+        uses: actions/cache@v3
+        env:
+          SEGMENT_DOWNLOAD_TIMEOUT_MIN: "2"
+        with:
+          path: |
+            ${{ env.WORKDIR }}/.mypy_cache
+          key: mypy-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', env.WORKDIR)) }}
+
+      - name: Analysing the code with our lint
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          make lint

.github/workflows/_pydantic_compatibility.yml

@@ -0,0 +1,94 @@
+name: pydantic v1/v2 compatibility
+
+on:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  build:
+    timeout-minutes: 5
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+    name: Pydantic v1/v2 compatibility - Python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
+          cache-key: pydantic-cross-compat
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install
+
+      - name: Install the opposite major version of pydantic
+        # If normal tests use pydantic v1, here we'll use v2, and vice versa.
+        shell: bash
+        run: |
+          # Determine the major part of pydantic version
+          REGULAR_VERSION=$(poetry run python -c "import pydantic; print(pydantic.__version__)" | cut -d. -f1)
+
+          if [[ "$REGULAR_VERSION" == "1" ]]; then
+            PYDANTIC_DEP=">=2.1,<3"
+            TEST_WITH_VERSION="2"
+          elif [[ "$REGULAR_VERSION" == "2" ]]; then
+            PYDANTIC_DEP="<2"
+            TEST_WITH_VERSION="1"
+          else
+            echo "Unexpected pydantic major version '$REGULAR_VERSION', cannot determine which version to use for cross-compatibility test."
+            exit 1
+          fi
+
+          # Install via `pip` instead of `poetry add` to avoid changing lockfile,
+          # which would prevent caching from working: the cache would get saved
+          # to a different key than where it gets loaded from.
+          poetry run pip install "pydantic${PYDANTIC_DEP}"
+
+          # Ensure that the correct pydantic is installed now.
+          echo "Checking pydantic version... Expecting ${TEST_WITH_VERSION}"
+
+          # Determine the major part of pydantic version
+          CURRENT_VERSION=$(poetry run python -c "import pydantic; print(pydantic.__version__)" | cut -d. -f1)
+
+          # Check that the major part of pydantic version is as expected, if not
+          # raise an error
+          if [[ "$CURRENT_VERSION" != "$TEST_WITH_VERSION" ]]; then
+            echo "Error: expected pydantic version ${CURRENT_VERSION} to have been installed, but found: ${TEST_WITH_VERSION}"
+            exit 1
+          fi
+          echo "Found pydantic version ${CURRENT_VERSION}, as expected"
+      - name: Run pydantic compatibility tests
+        shell: bash
+        run: make test
+
+      - name: Ensure the tests did not create any additional files
+        shell: bash
+        run: |
+          set -eu
+
+          STATUS="$(git status)"
+          echo "$STATUS"
+
+          # grep will exit non-zero if the target message isn't found,
+          # and `set -e` above will cause the step to fail.
+          echo "$STATUS" | grep 'nothing to commit, working tree clean'

.github/workflows/_release.yml

@@ -0,0 +1,63 @@
+name: release
+
+on:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  if_release:
+    # Disallow publishing from branches that aren't `main`.
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    permissions:
+      # This permission is used for trusted publishing:
+      # https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/
+      #
+      # Trusted publishing has to also be configured on PyPI for each package:
+      # https://docs.pypi.org/trusted-publishers/adding-a-publisher/
+      id-token: write
+
+      # This permission is needed by `ncipollo/release-action` to create the GitHub release.
+      contents: write
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: "3.10"
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
+          cache-key: release
+
+      - name: Build project for distribution
+        run: poetry build
+      - name: Check Version
+        id: check-version
+        run: |
+          echo version=$(poetry version --short) >> $GITHUB_OUTPUT
+      - name: Create Release
+        uses: ncipollo/release-action@v1
+        with:
+          artifacts: "dist/*"
+          token: ${{ secrets.GITHUB_TOKEN }}
+          draft: false
+          generateReleaseNotes: true
+          tag: v${{ steps.check-version.outputs.version }}
+          commit: main
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: ${{ inputs.working-directory }}/dist/
+          verbose: true
+          print-hash: true

.github/workflows/_test.yml

@@ -0,0 +1,57 @@
+name: test
+
+on:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  build:
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+    name: Python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
+          cache-key: core
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install
+
+      - name: Run core tests
+        shell: bash
+        run: make test
+
+      - name: Ensure the tests did not create any additional files
+        shell: bash
+        run: |
+          set -eu
+
+          STATUS="$(git status)"
+          echo "$STATUS"
+
+          # grep will exit non-zero if the target message isn't found,
+          # and `set -e` above will cause the step to fail.
+          echo "$STATUS" | grep 'nothing to commit, working tree clean'

.github/workflows/ci.yml

@@ -0,0 +1,150 @@
+---
+name: Run CI Tests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    paths-ignore:
+      - 'README.md'
+  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
+
+# If another push to the same PR or branch happens while this workflow is still running,
+# cancel the earlier run in favor of the next run.
+#
+# There's no point in testing an outdated version of the code. GitHub only allows
+# a limited number of job runners to be active at the same time, so it's better to cancel
+# pointless jobs early so that more useful jobs can run sooner.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  POETRY_VERSION: "1.5.1"
+  WORKDIR: "."
+
+jobs:
+  lint:
+    uses:
+      ./.github/workflows/_lint.yml
+    with:
+      working-directory: .
+    secrets: inherit
+
+  pydantic-compatibility:
+    uses:
+      ./.github/workflows/_pydantic_compatibility.yml
+    with:
+      working-directory: .
+    secrets: inherit
+  test:
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ env.WORKDIR }}
+    strategy:
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+    name: Python ${{ matrix.python-version }} tests
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: .
+          cache-key: benchmarks-all
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running tests, installing dependencies with poetry..."
+          poetry install --with test,lint,typing,docs
+
+      - name: Run tests
+        run: make test
+
+      - name: Ensure the tests did not create any additional files
+        shell: bash
+        run: |
+          set -eu
+
+          STATUS="$(git status)"
+          echo "$STATUS"
+
+          # grep will exit non-zero if the target message isn't found,
+          # and `set -e` above will cause the step to fail.
+          echo "$STATUS" | grep 'nothing to commit, working tree clean'
+  test_docs:
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ env.WORKDIR }}
+    strategy:
+      matrix:
+        python-version:
+          - "3.11"
+    name: Documentation Build for Python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: .
+          cache-key: benchmarks-all
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running tests, installing dependencies with poetry..."
+          poetry install --with test,lint,typing,docs
+
+      - name: Test Sphinx Docs
+        shell: bash
+        run: |
+          echo "Attempting to build docs..."
+          make build_docs
+  test_datasets:
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ env.WORKDIR }}
+    strategy:
+      matrix:
+        python-version:
+          - "3.11"
+    name: Validate Public Datasets
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: .
+          cache-key: benchmarks-all
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running tests, installing dependencies with poetry..."
+          poetry install --with test,lint,typing,docs
+
+      - name: Request datasets
+        shell: bash
+        run: |
+          echo "Attempting to build docs..."
+          poetry run python -m scripts.check_datasets

.github/workflows/doc_publish.yaml

@@ -0,0 +1,44 @@
+name: Publish Docs
+on: [workflow_dispatch]
+permissions:
+    contents: write
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  docs:
+    strategy:
+      matrix:
+        python-version:
+          - "3.11"
+    runs-on: ubuntu-latest
+    name: Documentation Publish
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: .
+          cache-key: benchmarks-all
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running tests, installing dependencies with poetry..."
+          poetry install --with test,lint,typing,docs
+
+      - name: Sphinx build
+        shell: bash
+        run: |
+          make build_docs
+      - name: Publish Docs
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          publish_branch: gh-pages
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs/build
+          force_orphan: true

.github/workflows/release.yml

@@ -0,0 +1,13 @@
+---
+name: Publish Package to PyPi
+
+on:
+  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
+
+jobs:
+  release:
+    uses:
+      ./.github/workflows/_release.yml
+    with:
+      working-directory: .
+    secrets: inherit

.gitignore

@@ -0,0 +1,162 @@
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+.DS_Store

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Langchain AI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,66 @@
+.PHONY: all lint format test help
+
+# Default target executed when no arguments are given to make.
+all: help
+
+######################
+# TESTING AND COVERAGE
+######################
+
+# Define a variable for the test file path.
+TEST_FILE ?= tests/unit_tests/
+
+test:
+	poetry run pytest --disable-socket --allow-unix-socket $(TEST_FILE)
+
+test_watch:
+	poetry run ptw . -- $(TEST_FILE)
+
+build_docs:
+	# Copy README.md to docs/index.md
+	cp README.md ./docs/source/index.md
+	# Append to the table of contents the contents of the file
+	cat ./docs/source/toc.segment >> ./docs/source/index.md
+	poetry run sphinx-build "./docs/source" "./docs/build"
+
+clean_docs:
+	rm -rf ./docs/build
+
+######################
+# LINTING AND FORMATTING
+######################
+
+# Define a variable for Python and notebook files.
+lint format: PYTHON_FILES=.
+lint_diff format_diff: PYTHON_FILES=$(shell git diff --relative=. --name-only --diff-filter=d master | grep -E '\.py$$|\.ipynb$$')
+
+lint lint_diff:
+	[ "$(PYTHON_FILES)" = "" ] ||	poetry run ruff format $(PYTHON_FILES) --diff
+	# [ "$(PYTHON_FILES)" = "" ] || poetry run mypy $(PYTHON_FILES)
+
+format format_diff:
+	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff format $(PYTHON_FILES)
+	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff --select I --fix $(PYTHON_FILES)
+
+spell_check:
+	poetry run codespell --toml pyproject.toml
+
+spell_fix:
+	poetry run codespell --toml pyproject.toml -w
+
+######################
+# HELP
+######################
+
+help:
+	@echo '===================='
+	@echo '-- LINTING --'
+	@echo 'format                       - run code formatters'
+	@echo 'lint                         - run linters'
+	@echo 'spell_check                 	- run codespell on the project'
+	@echo 'spell_fix                		- run codespell on the project and fix the errors'
+	@echo '-- TESTS --'
+	@echo 'coverage                     - run unit tests and generate coverage report'
+	@echo 'test                         - run unit tests'
+	@echo 'test TEST_FILE=<test_file>   - run all tests in file'
+	@echo '-- DOCUMENTATION tasks are from the top-level Makefile --'

README.md

@@ -1,8 +1,21 @@
-# LangChain Benchmarks
+🚧 Under Active Development 🚧
 
-This repository shows how we benchmark some of our more popular chains and agents.
-The benchmarks are organized by end-to-end use cases.
-They utilize [LangSmith](https://smith.langchain.com/) heavily.
+# 🦜💪 LangChain Benchmarks
+
+[![Release Notes](https://img.shields.io/github/release/langchain-ai/langchain-benchmarks)](https://github.com/langchain-ai/langchain-benchmarks/releases)
+[![CI](https://github.com/langchain-ai/langchain-benchmarks/actions/workflows/ci.yml/badge.svg)](https://github.com/langchain-ai/langchain-benchmarks/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Twitter](https://img.shields.io/twitter/url/https/twitter.com/langchainai.svg?style=social&label=Follow%20%40LangChainAI)](https://twitter.com/langchainai)
+[![](https://dcbadge.vercel.app/api/server/6adMQxSpJS?compact=true&style=flat)](https://discord.gg/6adMQxSpJS)
+[![Open Issues](https://img.shields.io/github/issues-raw/langchain-ai/langchain-benchmarks)](https://github.com/langchain-ai/langchain-benchmarks/issues)
+
+
+[📖 Documentation](https://langchain-ai.github.io/langchain-benchmarks/index.html)
+
+A package to help benchmark various LLM related tasks.
+
+The benchmarks are organized by end-to-end use cases, and
+utilize [LangSmith](https://smith.langchain.com/) heavily.
 
 We have several goals in open sourcing this:
 
@@ -11,6 +24,39 @@ We have several goals in open sourcing this:
 - Showing how we evaluate each task
 - Encouraging others to benchmark their solutions on these tasks (we are always looking for better ways of doing things!)
 
-We currently include the following tasks:
-- [CSV Question Answering](csv-qa)
-- [Extraction](extraction)
+## Installation
+
+To install the packages, run the following command:
+
+```bash
+pip install -U langchain-benchmarks
+```
+
+All the benchmarks come with an associated benchmark dataset stored in [LangSmith](https://smith.langchain.com). To take advantage of the eval and debugging experience, [sign up](https://smith.langchain.com), and set your API key in your environment:
+
+```bash
+export LANGCHAIN_API_KEY=sk-...
+```
+
+## Repo Structure
+
+The package is located within [langchain_benchmarks](./langchain_benchmarks/). Check out the [docs](https://langchain-ai.github.io/langchain-benchmarks/index.html) for information on how to get starte.
+
+The other directories are legacy and may be moved in the future.
+
+
+## Archived
+
+Below are archived benchmarks that require cloning this repo to run.
+
+- [CSV Question Answering](https://github.com/langchain-ai/langchain-benchmarks/tree/main/csv-qa)
+- [Extraction](https://github.com/langchain-ai/langchain-benchmarks/tree/main/extraction)
+- [Q&A over the LangChain docs](https://github.com/langchain-ai/langchain-benchmarks/tree/main/langchain-docs-benchmarking)
+- [Meta-evaluation of 'correctness' evaluators](https://github.com/langchain-ai/langchain-benchmarks/tree/main/meta-evals)
+
+
+## Related
+
+- For cookbooks on other ways to test, debug, monitor, and improve your LLM applications, check out the [LangSmith docs](https://docs.smith.langchain.com/)
+- For information on building with LangChain, check out the [python documentation](https://python.langchain.com/docs/get_started/introduction) or [JS documentation](https://js.langchain.com/docs/get_started/introduction)
+

_scripts/add_collab_links.py

@@ -0,0 +1,70 @@
+import argparse
+import json
+import os
+
+from git import Git, Repo
+
+
+def get_repo_base_url(directory: str) -> str:
+    """Retrieves the base URL of the repository."""
+    try:
+        repo = Repo(directory, search_parent_directories=True)
+        remote_url = repo.remotes.origin.url
+        if remote_url.endswith(".git"):
+            remote_url = remote_url[:-4]
+        result = (
+            remote_url.replace("git@", "https://").replace("https://github.com:", "")
+            + "/blob/main/"
+        )
+        print(result)
+        return result
+    except Exception as e:
+        print("Error retrieving repository URL:", e)
+        return ""
+
+
+def add_collab_link(cell_content: list, filepath: str, repo_base_url: str) -> list:
+    """Inserts the 'Open In Collab' link into the cell content if it doesn't exist."""
+
+    if repo_base_url:
+        collab_link = f"[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/{repo_base_url}{filepath})".replace(
+            "/./", "/"
+        )
+        if collab_link not in "\n".join(cell_content):
+            cell_content = cell_content[:1] + [collab_link+"\n"] + cell_content[1:]
+
+    return cell_content
+
+
+def process_directory(directory: str) -> None:
+    """Traverses the directory and updates .ipynb files if necessary."""
+    repo_base_url = get_repo_base_url(directory)
+    for root, _, files in os.walk(directory):
+        for file in files:
+            if file.endswith(".ipynb"):
+                print("Checking", file)
+                filepath = os.path.join(root, file)
+                with open(filepath, "r", encoding="utf-8") as ipynb_file:
+                    ipynb_data = json.load(ipynb_file)
+
+                try:
+                    first_cell_content = ipynb_data["cells"][0]["source"]
+                except Exception as e:
+                    print("Skipping", filepath, e)
+                    continue
+                modified_content = add_collab_link(
+                    first_cell_content, filepath, repo_base_url
+                )
+                if modified_content != first_cell_content:
+                    print("Inserting link into", filepath)
+                    ipynb_data["cells"][0]["source"] = modified_content
+
+                    with open(filepath, "w", encoding="utf-8") as ipynb_file:
+                        json.dump(ipynb_data, ipynb_file, ensure_ascii=False, indent=4)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--directory", type=str, default=".")
+    args = parser.parse_args()
+    process_directory(args.directory)

csv-qa/custom_agent.py

@@ -1,22 +1,25 @@
-from langchain.agents import OpenAIFunctionsAgent, AgentExecutor
-from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
-from langchain.tools import PythonAstREPLTool
 import pandas as pd
+from langchain.agents import AgentExecutor, OpenAIFunctionsAgent
+from langchain.agents.agent_toolkits.conversational_retrieval.tool import (
+    create_retriever_tool,
+)
 from langchain.chat_models import ChatOpenAI
-from langsmith import Client
-from langchain.smith import RunEvalConfig, run_on_dataset
-from pydantic import BaseModel, Field
 from langchain.embeddings import OpenAIEmbeddings
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain.smith import RunEvalConfig, run_on_dataset
+from langchain.tools import PythonAstREPLTool
 from langchain.vectorstores import FAISS
-from langchain.agents.agent_toolkits.conversational_retrieval.tool import create_retriever_tool
+from langsmith import Client
+from pydantic import BaseModel, Field
 
-
-pd.set_option('display.max_rows', 20)
-pd.set_option('display.max_columns', 20)
+pd.set_option("display.max_rows", 20)
+pd.set_option("display.max_columns", 20)
 
 embedding_model = OpenAIEmbeddings()
 vectorstore = FAISS.load_local("titanic_data", embedding_model)
-retriever_tool = create_retriever_tool(vectorstore.as_retriever(), "person_name_search", "Search for a person by name")
+retriever_tool = create_retriever_tool(
+    vectorstore.as_retriever(), "person_name_search", "Search for a person by name"
+)
 
 
 TEMPLATE = """You are working with a pandas dataframe in Python. The name of the dataframe is `df`.
@@ -42,7 +45,6 @@ For example:
 """
 
 
-
 class PythonInputs(BaseModel):
     query: str = Field(description="code snippet to run")
 
@@ -51,27 +53,33 @@ if __name__ == "__main__":
     df = pd.read_csv("titanic.csv")
     template = TEMPLATE.format(dhead=df.head().to_markdown())
 
-    prompt = ChatPromptTemplate.from_messages([
-        ("system", template),
-        MessagesPlaceholder(variable_name="agent_scratchpad"),
-        ("human", "{input}")
-    ])
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", template),
+            MessagesPlaceholder(variable_name="agent_scratchpad"),
+            ("human", "{input}"),
+        ]
+    )
 
     def get_chain():
-        repl = PythonAstREPLTool(locals={"df": df}, name="python_repl",
-                                 description="Runs code and returns the output of the final line",
-                                 args_schema=PythonInputs)
+        repl = PythonAstREPLTool(
+            locals={"df": df},
+            name="python_repl",
+            description="Runs code and returns the output of the final line",
+            args_schema=PythonInputs,
+        )
         tools = [repl, retriever_tool]
-        agent = OpenAIFunctionsAgent(llm=ChatOpenAI(temperature=0, model="gpt-4"), prompt=prompt, tools=tools)
-        agent_executor = AgentExecutor(agent=agent, tools=tools, max_iterations=5, early_stopping_method="generate")
+        agent = OpenAIFunctionsAgent(
+            llm=ChatOpenAI(temperature=0, model="gpt-4"), prompt=prompt, tools=tools
+        )
+        agent_executor = AgentExecutor(
+            agent=agent, tools=tools, max_iterations=5, early_stopping_method="generate"
+        )
         return agent_executor
 
-
     client = Client()
     eval_config = RunEvalConfig(
-        evaluators=[
-            "qa"
-        ],
+        evaluators=["qa"],
     )
     chain_results = run_on_dataset(
         client,

csv-qa/pandas_agent_gpt_35.py

@@ -1,9 +1,9 @@
 import pandas as pd
-from langchain.chat_models import ChatOpenAI
 from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
 from langchain.agents.agent_types import AgentType
-from langsmith import Client
+from langchain.chat_models import ChatOpenAI
 from langchain.smith import RunEvalConfig, run_on_dataset
+from langsmith import Client
 
 if __name__ == "__main__":
     df = pd.read_csv("titanic.csv")
@@ -18,20 +18,17 @@ if __name__ == "__main__":
             df,
             agent_type=AgentType.OPENAI_FUNCTIONS,
             agent_executor_kwargs=agent_executor_kwargs,
-            max_iterations=5
+            max_iterations=5,
         )
         return agent
 
-
     client = Client()
     eval_config = RunEvalConfig(
-        evaluators=[
-            "qa"
-        ],
+        evaluators=["qa"],
     )
     chain_results = run_on_dataset(
         client,
         dataset_name="Titanic CSV Data",
         llm_or_chain_factory=get_chain,
         evaluation=eval_config,
-    )
+    )

csv-qa/pandas_agent_gpt_4.py

@@ -1,14 +1,13 @@
 import pandas as pd
-from langchain.chat_models import ChatOpenAI
 from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
 from langchain.agents.agent_types import AgentType
-from langsmith import Client
+from langchain.chat_models import ChatOpenAI
 from langchain.smith import RunEvalConfig, run_on_dataset
+from langsmith import Client
 
 if __name__ == "__main__":
     df = pd.read_csv("titanic.csv")
 
-
     def get_chain():
         llm = ChatOpenAI(temperature=0, model="gpt-4")
         agent_executor_kwargs = {
@@ -19,20 +18,17 @@ if __name__ == "__main__":
             df,
             agent_type=AgentType.OPENAI_FUNCTIONS,
             agent_executor_kwargs=agent_executor_kwargs,
-            max_iterations=5
+            max_iterations=5,
         )
         return agent
 
-
     client = Client()
     eval_config = RunEvalConfig(
-        evaluators=[
-            "qa"
-        ],
+        evaluators=["qa"],
     )
     chain_results = run_on_dataset(
         client,
         dataset_name="Titanic CSV Data",
         llm_or_chain_factory=get_chain,
         evaluation=eval_config,
-    )
+    )

csv-qa/pandas_agent_instruct.py

@@ -1,22 +1,24 @@
-from langchain.agents import ZeroShotAgent, AgentExecutor
-from langchain.prompts import PromptTemplate
-from langchain.tools import PythonAstREPLTool
 import pandas as pd
-from langchain.llms import OpenAI
-from langsmith import Client
-from langchain.smith import RunEvalConfig, run_on_dataset
-from pydantic import BaseModel, Field
+from langchain.agents import AgentExecutor, ZeroShotAgent
+from langchain.agents.agent_toolkits.conversational_retrieval.tool import (
+    create_retriever_tool,
+)
 from langchain.embeddings import OpenAIEmbeddings
+from langchain.llms import OpenAI
+from langchain.smith import RunEvalConfig, run_on_dataset
+from langchain.tools import PythonAstREPLTool
 from langchain.vectorstores import FAISS
-from langchain.agents.agent_toolkits.conversational_retrieval.tool import create_retriever_tool
+from langsmith import Client
+from pydantic import BaseModel, Field
 
-
-pd.set_option('display.max_rows', 20)
-pd.set_option('display.max_columns', 20)
+pd.set_option("display.max_rows", 20)
+pd.set_option("display.max_columns", 20)
 
 embedding_model = OpenAIEmbeddings()
 vectorstore = FAISS.load_local("titanic_data", embedding_model)
-retriever_tool = create_retriever_tool(vectorstore.as_retriever(), "person_name_search", "Search for a person by name")
+retriever_tool = create_retriever_tool(
+    vectorstore.as_retriever(), "person_name_search", "Search for a person by name"
+)
 
 
 TEMPLATE = """You are working with a pandas dataframe in Python. The name of the dataframe is `df`.
@@ -41,7 +43,6 @@ For example:
 <logic>Use `python_repl` since even though the question is about a person, you don't know their name so you can't include it.</logic>"""
 
 
-
 class PythonInputs(BaseModel):
     query: str = Field(description="code snippet to run")
 
@@ -50,22 +51,27 @@ if __name__ == "__main__":
     df = pd.read_csv("titanic.csv")
     template = TEMPLATE.format(dhead=df.head().to_markdown())
 
-
     def get_chain():
-        repl = PythonAstREPLTool(locals={"df": df}, name="python_repl",
-                                 description="Runs code and returns the output of the final line",
-                                 args_schema=PythonInputs)
+        repl = PythonAstREPLTool(
+            locals={"df": df},
+            name="python_repl",
+            description="Runs code and returns the output of the final line",
+            args_schema=PythonInputs,
+        )
         tools = [repl, retriever_tool]
-        agent = ZeroShotAgent.from_llm_and_tools(llm=OpenAI(temperature=0, model="gpt-3.5-turbo-instruct"), tools=tools, prefix=template)
-        agent_executor = AgentExecutor(agent=agent, tools=tools, max_iterations=5, early_stopping_method="generate")
+        agent = ZeroShotAgent.from_llm_and_tools(
+            llm=OpenAI(temperature=0, model="gpt-3.5-turbo-instruct"),
+            tools=tools,
+            prefix=template,
+        )
+        agent_executor = AgentExecutor(
+            agent=agent, tools=tools, max_iterations=5, early_stopping_method="generate"
+        )
         return agent_executor
 
-
     client = Client()
     eval_config = RunEvalConfig(
-        evaluators=[
-            "qa"
-        ],
+        evaluators=["qa"],
     )
     chain_results = run_on_dataset(
         client,

csv-qa/pandas_ai.py

@@ -1,44 +1,45 @@
 import pandas as pd
-from langchain.chat_models import ChatOpenAI
-from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
-from langchain.agents.agent_types import AgentType
-from langsmith import Client
-from langchain.smith import RunEvalConfig, run_on_dataset
-import pandas as pd
-from pandasai import PandasAI
-
 from langchain.chat_models import ChatOpenAI
 from langchain.prompts import ChatPromptTemplate
 from langchain.schema.output_parser import StrOutputParser
+from langchain.smith import RunEvalConfig, run_on_dataset
+from langsmith import Client
+from pandasai import PandasAI
 
 if __name__ == "__main__":
     df = pd.read_csv("titanic.csv")
 
     pandas_ai = PandasAI(ChatOpenAI(temperature=0, model="gpt-4"), enable_cache=False)
-    prompt = ChatPromptTemplate.from_messages([
-        ("system",
-         "Answer the users question about some data. A data scientist will run some code and the results will be returned to you to use in your answer"),
-        ("human", "Question: {input}"),
-        ("human", "Data Scientist Result: {result}"),
-    ])
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            (
+                "system",
+                "Answer the users question about some data. A data scientist will run some code and the results will be returned to you to use in your answer",
+            ),
+            ("human", "Question: {input}"),
+            ("human", "Data Scientist Result: {result}"),
+        ]
+    )
 
     def get_chain():
-        chain = {
-                    "input": lambda x: x["input_question"],
-                    "result": lambda x: pandas_ai(df, prompt=x['input_question'])
-                } | prompt | ChatOpenAI(temperature=0, model="gpt-4") | StrOutputParser()
+        chain = (
+            {
+                "input": lambda x: x["input_question"],
+                "result": lambda x: pandas_ai(df, prompt=x["input_question"]),
+            }
+            | prompt
+            | ChatOpenAI(temperature=0, model="gpt-4")
+            | StrOutputParser()
+        )
         return chain
 
-
     client = Client()
     eval_config = RunEvalConfig(
-        evaluators=[
-            "qa"
-        ],
+        evaluators=["qa"],
     )
     chain_results = run_on_dataset(
         client,
         dataset_name="Titanic CSV Data",
         llm_or_chain_factory=get_chain,
         evaluation=eval_config,
-    )
+    )

csv-qa/streamlit_app.py

@@ -1,10 +1,10 @@
 import pandas as pd
 import streamlit as st
-from langchain.chat_models import ChatOpenAI
 from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
 from langchain.agents.agent_types import AgentType
+from langchain.chat_models import ChatOpenAI
 
-df = pd.read_csv('titanic.csv')
+df = pd.read_csv("titanic.csv")
 
 
 llm = ChatOpenAI(temperature=0)
@@ -12,31 +12,36 @@ agent = create_pandas_dataframe_agent(llm, df, agent_type=AgentType.OPENAI_FUNCT
 
 
 from langsmith import Client
+
 client = Client()
+
+
 def send_feedback(run_id, score):
     client.create_feedback(run_id, "user_score", score=score)
 
-st.set_page_config(page_title='🦜🔗 Ask the CSV App')
-st.title('🦜🔗 Ask the CSV App')
-st.info("Most 'question answering' applications run over unstructured text data. But a lot of the data in the world is tabular data! This is an attempt to create an application using [LangChain](https://github.com/langchain-ai/langchain) to let you ask questions of data in tabular format. For this demo application, we will use the Titanic Dataset. Please explore it [here](https://github.com/datasciencedojo/datasets/blob/master/titanic.csv) to get a sense for what questions you can ask. Please leave feedback on well the question is answered, and we will use that improve the application!")
 
-query_text = st.text_input('Enter your question:', placeholder = 'Who was in cabin C128?')
+st.set_page_config(page_title="🦜🔗 Ask the CSV App")
+st.title("🦜🔗 Ask the CSV App")
+st.info(
+    "Most 'question answering' applications run over unstructured text data. But a lot of the data in the world is tabular data! This is an attempt to create an application using [LangChain](https://github.com/langchain-ai/langchain) to let you ask questions of data in tabular format. For this demo application, we will use the Titanic Dataset. Please explore it [here](https://github.com/datasciencedojo/datasets/blob/master/titanic.csv) to get a sense for what questions you can ask. Please leave feedback on well the question is answered, and we will use that improve the application!"
+)
+
+query_text = st.text_input("Enter your question:", placeholder="Who was in cabin C128?")
 # Form input and query
 result = None
-with st.form('myform', clear_on_submit=True):
-	submitted = st.form_submit_button('Submit')
-	if submitted:
-		with st.spinner('Calculating...'):
-			response = agent({"input": query_text}, include_run_info=True)
-			result = response["output"]
-			run_id = response["__run"].run_id
+with st.form("myform", clear_on_submit=True):
+    submitted = st.form_submit_button("Submit")
+    if submitted:
+        with st.spinner("Calculating..."):
+            response = agent({"input": query_text}, include_run_info=True)
+            result = response["output"]
+            run_id = response["__run"].run_id
 if result is not None:
-	st.info(result)
-	col_blank, col_text, col1, col2 = st.columns([10, 2,1,1])
-	with col_text:
-		st.text("Feedback:")
-	with col1:
-		st.button("👍", on_click=send_feedback, args=(run_id, 1))
-	with col2:
-		st.button("👎", on_click=send_feedback, args=(run_id, 0))
-
+    st.info(result)
+    col_blank, col_text, col1, col2 = st.columns([10, 2, 1, 1])
+    with col_text:
+        st.text("Feedback:")
+    with col1:
+        st.button("👍", on_click=send_feedback, args=(run_id, 1))
+    with col2:
+        st.button("👎", on_click=send_feedback, args=(run_id, 0))

csv-qa/upload_data.py

@@ -8,5 +8,5 @@ if __name__ == "__main__":
         output_keys=["output_text"],
         name="Titanic CSV Data",
         description="QA over titanic data",
-        data_type = "kv"
+        data_type="kv",
     )

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd

docs/source/.gitignore

@@ -0,0 +1 @@
+chromadb/

docs/source/conf.py

@@ -0,0 +1,105 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+import pathlib
+import sys
+from typing import List
+
+import toml
+
+ROOT_FOLDER = str(pathlib.Path(__file__).parent.parent.parent)
+
+# Add the project root to the path
+sys.path.insert(0, ROOT_FOLDER)
+
+with open("../../pyproject.toml") as f:
+    data = toml.load(f)
+
+project = "LangChain Benchmarks"
+copyright = "2023, Langchain AI"
+author = "Langchain AI"
+
+version = data["tool"]["poetry"]["version"]
+release = version
+
+html_title = project + " " + version
+
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.autodoc.typehints",
+    "sphinx.ext.autosummary",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.viewcode",
+    "myst_nb",
+    "sphinx_copybutton",
+    "IPython.sphinxext.ipython_console_highlighting",
+]
+source_suffix = [".ipynb", ".html", ".md", ".rst"]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ["_templates"]
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns: List[str] = []
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "sphinx_book_theme"
+
+html_theme_options = {
+    "path_to_docs": "docs",
+    "repository_url": "https://github.com/langchain-ai/langchain-benchmarks",
+    "home_page_in_toc": True,
+    "show_navbar_depth": 2,
+    "use_sidenotes": True,
+    "use_repository_button": True,
+}
+
+html_context = {
+    "display_github": True,  # Integrate GitHub
+    "github_user": "langchain-ai",  # Username
+    "github_repo": "langchain-benchmarks",  # Repo name
+    "github_version": "main",  # Version
+    "conf_py_path": "/docs/",  # Path in the checkout to the docs root
+}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+
+# These paths are either relative to html_static_path
+# or fully qualified paths (eg. https://...)
+html_css_files = [
+    "css/custom.css",
+]
+
+nb_execution_mode = "off"
+autosummary_generate = True

docs/source/notebooks/datasets.ipynb

@@ -0,0 +1,225 @@
+{
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "id": "033684fb-65b2-4586-a959-68c614741ca2",
+            "metadata": {},
+            "source": [
+                "# Datasets\n",
+                "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain-benchmarks/blob/main/docs/source/notebooks/datasets.ipynb)\n",
+                "\n",
+                "Here, we'll see how to work with LangSmith datasets."
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "%pip install -U langchain-benchmarks"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 1,
+            "id": "6d272fbf-710e-4a49-a0da-67e010541905",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [],
+            "source": [
+                "from langchain_benchmarks import clone_public_dataset, download_public_dataset"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "18ee0f96-e5c4-4ae9-aebf-7d8b88c51662",
+            "metadata": {},
+            "source": [
+                "Let's first download the dataset to the local file system"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 2,
+            "id": "58b94f6d-0c91-4361-9b22-f758ffaa150a",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "Fetching examples...\n"
+                    ]
+                },
+                {
+                    "data": {
+                        "application/vnd.jupyter.widget-view+json": {
+                            "model_id": "5a2fad8c0c3549ec96a3b38fe8a002b0",
+                            "version_major": 2,
+                            "version_minor": 0
+                        },
+                        "text/plain": [
+                            "  0%|          | 0/21 [00:00<?, ?it/s]"
+                        ]
+                    },
+                    "metadata": {},
+                    "output_type": "display_data"
+                },
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "Done fetching examples.\n"
+                    ]
+                }
+            ],
+            "source": [
+                "download_public_dataset(\n",
+                "    \"https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/examples\"\n",
+                ")"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "841db832-b0d3-4fd1-8531-1154ec9b3caa",
+            "metadata": {},
+            "source": [
+                "we can take a look at the first two examples"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 3,
+            "id": "664e90fc-af84-4c5f-a3dd-5d9ffe649650",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "[\n",
+                        "  {\n",
+                        "    \"created_at\": \"2023-11-15T15:26:53.511629\",\n",
+                        "    \"dataset_id\": \"9f73165c-d333-4d14-8f59-bd7eede5db08\",\n",
+                        "    \"id\": \"0703a989-2693-4039-a1f6-7281fc1b4cb0\",\n",
+                        "    \"inputs\": {\n",
+                        "      \"question\": \"do bob and alice live in the same city?\"\n",
+                        "    },\n",
+                        "    \"modified_at\": \"2023-11-15T15:26:53.511629\",\n",
+                        "    \"outputs\": {\n",
+                        "      \"expected_steps\": [\n",
+                        "        \"find_users_by_name\",\n",
+                        "        \"get_user_location\",\n",
+                        "        \"get_city_for_location\",\n",
+                        "        \"get_user_location\",\n",
+                        "        \"get_city_for_location\"\n",
+                        "      ],\n",
+                        "      \"order_matters\": false,\n",
+                        "      \"reference\": \"no\"\n",
+                        "    },\n",
+                        "    \"runs\": []\n",
+                        "  },\n",
+                        "  {\n",
+                        "    \"created_at\": \"2023-11-15T15:26:53.491359\",\n",
+                        "    \"dataset_id\": \"9f73165c-d333-4d14-8f59-bd7eede5db08\",\n",
+                        "    \"id\": \"b258b95a-9524-4da7-b758-c5481109322d\",\n",
+                        "    \"inputs\": {\n",
+                        "      \"question\": \"Is it likely that Donna is outside with an umbrella at this time?\"\n",
+                        "    },\n",
+                        "    \"modified_at\": \"2023-11-15T15:26:53.491359\",\n",
+                        "    \"outputs\": {\n",
+                        "      \"expected_steps\": [\n",
+                        "        \"find_users_by_name\",\n",
+                        "        \"get_user_location\",\n",
+                        "        \"get_current_time_for_location\",\n",
+                        "        \"get_current_weather_for_location\"\n",
+                        "      ],\n",
+                        "      \"order_matters\": false,\n",
+                        "      \"reference\": \"yes\"\n",
+                        "    },\n",
+                        "    \"runs\": []\n",
+                        "  }\n",
+                        "]\n"
+                    ]
+                }
+            ],
+            "source": [
+                "import json\n",
+                "\n",
+                "with open(\"./e95d45da-aaa3-44b3-ba2b-7c15ff6e46f5.json\", \"r\", encoding=\"utf-8\") as f:\n",
+                "    print(json.dumps(json.load(f)[:2], indent=2, sort_keys=True))"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "2c6cf01f-466b-406d-b4c7-2395747780fd",
+            "metadata": {},
+            "source": [
+                "We can also clone the dataset to our local tenant"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "e4dea4df-2f1c-436b-a71c-49ffb2295ccc",
+            "metadata": {},
+            "source": [
+                "Executing this command will clone the dataset to your own LangSmith tenant. \n",
+                "For this to work you must have a [LangSmith account](https://smith.langchain.com/) set up."
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "import os\n",
+                "\n",
+                "# Get from https://smith.langchain.com/settings\n",
+                "os.environ[\"LANGCHAIN_API_KEY\"] = \"sk-...\""
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "id": "18d0b905-2a6a-4752-a7cb-8653bd9049e3",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [],
+            "source": [
+                "clone_public_dataset(\n",
+                "    \"https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/examples\",\n",
+                "    dataset_name=\"Agent Dataset\",\n",
+                ")"
+            ]
+        }
+    ],
+    "metadata": {
+        "kernelspec": {
+            "display_name": "Python 3 (ipykernel)",
+            "language": "python",
+            "name": "python3"
+        },
+        "language_info": {
+            "codemirror_mode": {
+                "name": "ipython",
+                "version": 3
+            },
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.11.2"
+        }
+    },
+    "nbformat": 4,
+    "nbformat_minor": 5
+}

docs/source/notebooks/extraction/email.ipynb

@@ -0,0 +1,439 @@
+{
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "id": "60bb467d-861d-4b07-a48d-8e5aa177c969",
+            "metadata": {},
+            "source": [
+                "# Email Extraction\n",
+                "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain-benchmarks/blob/main/docs/source/notebooks/extraction/email.ipynb)\n",
+                "\n",
+                "Let's evaluate an LLM on its ability to extract structured information from email texts."
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "id": "47de0d20-d20b-44be-9e41-d2275f0866e8",
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "# %pip install -U langchain langchain_benchmarks openai"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 1,
+            "id": "c401de19-814e-4bd7-bb9c-7ea6e217985c",
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "import os\n",
+                "\n",
+                "# Get your API key from https://smith.langchain.com/settings\n",
+                "os.environ[\"LANGCHAIN_API_KEY\"] = \"sk-...\"\n",
+                "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\""
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 2,
+            "id": "b39159d0-9ea1-414f-a9d8-4a7b22b3d2cc",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [],
+            "source": [
+                "from langchain_benchmarks import clone_public_dataset, registry"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "03488ab1-31ed-41c2-8da2-46b02599b181",
+            "metadata": {},
+            "source": [
+                "For this code to work, please configure LangSmith environment variables with your credentials."
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 3,
+            "id": "60f22779-a948-4833-8e8c-ace9ef17f56f",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "data": {
+                        "text/html": [
+                            "<table>\n",
+                            "<tbody>\n",
+                            "<tr><td>Name       </td><td>Email Extraction                                                                                                                                           </td></tr>\n",
+                            "<tr><td>Type       </td><td>ExtractionTask                                                                                                                                             </td></tr>\n",
+                            "<tr><td>Dataset ID </td><td><a href=\"https://smith.langchain.com/public/a1742786-bde5-4f51-a1d8-e148e5251ddb/d\" target=\"_blank\" rel=\"noopener\">a1742786-bde5-4f51-a1d8-e148e5251ddb</a></td></tr>\n",
+                            "<tr><td>Description</td><td>A dataset of 42 real emails deduped from a spam folder, with semantic HTML tags removed, as well as a script for initial extraction and formatting of other emails from an arbitrary .mbox file like the one exported by Gmail.\n",
+                            "\n",
+                            "Some additional cleanup of the data was done by hand after the initial pass.\n",
+                            "\n",
+                            "See https://github.com/jacoblee93/oss-model-extraction-evals.                                                                                                                                                            </td></tr>\n",
+                            "</tbody>\n",
+                            "</table>"
+                        ],
+                        "text/plain": [
+                            "ExtractionTask(name='Email Extraction', dataset_id='https://smith.langchain.com/public/a1742786-bde5-4f51-a1d8-e148e5251ddb/d', description='A dataset of 42 real emails deduped from a spam folder, with semantic HTML tags removed, as well as a script for initial extraction and formatting of other emails from an arbitrary .mbox file like the one exported by Gmail.\\n\\nSome additional cleanup of the data was done by hand after the initial pass.\\n\\nSee https://github.com/jacoblee93/oss-model-extraction-evals.\\n    ', schema=<class 'langchain_benchmarks.extraction.tasks.email_task.Email'>, instructions=ChatPromptTemplate(input_variables=['input'], messages=[SystemMessagePromptTemplate(prompt=PromptTemplate(input_variables=[], template='You are an expert researcher.')), HumanMessagePromptTemplate(prompt=PromptTemplate(input_variables=['input'], template='What can you tell me about the following email? Make sure to extract the question in the correct format. Here is the email:\\n ```\\n{input}\\n```'))]))"
+                        ]
+                    },
+                    "execution_count": 3,
+                    "metadata": {},
+                    "output_type": "execute_result"
+                }
+            ],
+            "source": [
+                "task = registry[\"Email Extraction\"]\n",
+                "task"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 4,
+            "id": "49be36d2-343e-49df-8369-dd5bac405d5e",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "A dataset of 42 real emails deduped from a spam folder, with semantic HTML tags removed, as well as a script for initial extraction and formatting of other emails from an arbitrary .mbox file like the one exported by Gmail.\n",
+                        "\n",
+                        "Some additional cleanup of the data was done by hand after the initial pass.\n",
+                        "\n",
+                        "See https://github.com/jacoblee93/oss-model-extraction-evals.\n",
+                        "    \n"
+                    ]
+                }
+            ],
+            "source": [
+                "print(task.description)"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "bc33a639-3caf-4314-8ea7-1c7c8b1d114d",
+            "metadata": {},
+            "source": [
+                "Clone the dataset associated with this task"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 5,
+            "id": "70369f67-deb4-467a-801a-6d38c3d0460d",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "Dataset Email Extraction already exists. Skipping.\n",
+                        "You can access the dataset at https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/309a2fce-ce68-43aa-befb-67f94d0c3570.\n"
+                    ]
+                }
+            ],
+            "source": [
+                "clone_public_dataset(task.dataset_id, dataset_name=task.name)"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 6,
+            "id": "12e302e6-9b3d-42a4-b612-d672c591e8f0",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "{'definitions': {'ToneEnum': {'description': 'The tone of the email.',\n",
+                        "                              'enum': ['positive', 'negative'],\n",
+                        "                              'title': 'ToneEnum',\n",
+                        "                              'type': 'string'}},\n",
+                        " 'description': 'Relevant information about an email.',\n",
+                        " 'properties': {'action_items': {'description': 'A list of action items '\n",
+                        "                                                'requested by the email',\n",
+                        "                                 'items': {'type': 'string'},\n",
+                        "                                 'title': 'Action Items',\n",
+                        "                                 'type': 'array'},\n",
+                        "                'sender': {'description': \"The sender's name, if available\",\n",
+                        "                           'title': 'Sender',\n",
+                        "                           'type': 'string'},\n",
+                        "                'sender_address': {'description': \"The sender's address, if \"\n",
+                        "                                                  'available',\n",
+                        "                                   'title': 'Sender Address',\n",
+                        "                                   'type': 'string'},\n",
+                        "                'sender_phone_number': {'description': \"The sender's phone \"\n",
+                        "                                                       'number, if available',\n",
+                        "                                        'title': 'Sender Phone Number',\n",
+                        "                                        'type': 'string'},\n",
+                        "                'tone': {'allOf': [{'$ref': '#/definitions/ToneEnum'}],\n",
+                        "                         'description': 'The tone of the email.'},\n",
+                        "                'topic': {'description': 'High level description of what the '\n",
+                        "                                         'email is about',\n",
+                        "                          'title': 'Topic',\n",
+                        "                          'type': 'string'}},\n",
+                        " 'required': ['action_items', 'topic', 'tone'],\n",
+                        " 'title': 'Email',\n",
+                        " 'type': 'object'}\n"
+                    ]
+                }
+            ],
+            "source": [
+                "import pprint\n",
+                "\n",
+                "pprint.pprint(task.schema.schema())"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "b462f7b8-fd42-4613-ab5f-5f3cbbc37d28",
+            "metadata": {},
+            "source": [
+                "## Define an extraction chain\n",
+                "\n",
+                "Let's build the extraction chain that we can use to get structured information from the emails."
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 10,
+            "id": "b7536a5b-0140-4c38-88c6-50921307677d",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [],
+            "source": [
+                "from langchain.chat_models import ChatOpenAI\n",
+                "\n",
+                "from langchain_benchmarks.extraction.implementations import (\n",
+                "    create_openai_function_based_extractor,\n",
+                ")"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 11,
+            "id": "ade7077c-4602-4e5b-ad6d-3eb43cbd0247",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [],
+            "source": [
+                "extraction_chain = create_openai_function_based_extractor(\n",
+                "    task.instructions, ChatOpenAI(model=\"gpt-3.5-turbo-16k\", temperature=0), task.schema\n",
+                ")"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 12,
+            "id": "f66ed218-e1db-49b5-bde3-40ebec961723",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "data": {
+                        "text/plain": [
+                            "{'output': {'sender': 'Unknown',\n",
+                            "  'sender_phone_number': '000-1212-1111',\n",
+                            "  'sender_address': '12345 My Gold Way',\n",
+                            "  'action_items': ['Buy an envelope',\n",
+                            "   'Put gold inside',\n",
+                            "   'Close the envelope',\n",
+                            "   \"Mail it to sender's address\"],\n",
+                            "  'topic': 'Request to send gold',\n",
+                            "  'tone': 'positive'}}"
+                        ]
+                    },
+                    "execution_count": 12,
+                    "metadata": {},
+                    "output_type": "execute_result"
+                }
+            ],
+            "source": [
+                "extraction_chain.invoke(\n",
+                "    {\n",
+                "        \"input\": \"Hello Dear MR. I want you to send me gold to get rich.\"\n",
+                "        \" First buy an envelope. Then open it and put some gold inside. \"\n",
+                "        \"Then close it and finally mail it to my address at 12345 My Gold Way.\"\n",
+                "        \" You can call me any time at 000-1212-1111.\"\n",
+                "    }\n",
+                ")"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "87a64f76-65ae-4367-b43f-f2be3431e7af",
+            "metadata": {},
+            "source": [
+                "Now it's time to measure our chain's effectiveness!"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "3821e4b0-8e67-418a-840c-470fcde42df0",
+            "metadata": {},
+            "source": [
+                "## Evaluate\n",
+                "\n",
+                "Let's evaluate the chain now."
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 13,
+            "id": "513042fe-2878-44f8-ae84-05b9d521c1de",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [],
+            "source": [
+                "from langsmith.client import Client\n",
+                "\n",
+                "from langchain_benchmarks.extraction import get_eval_config"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 14,
+            "id": "2bedd9d1-fc06-4066-9f89-b874ae818d82",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [],
+            "source": [
+                "client = Client()"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 17,
+            "id": "6826a2c6-8443-4215-9e15-b6f4bb570405",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [],
+            "source": [
+                "eval_llm = ChatOpenAI(model=\"gpt-4\", model_kwargs={\"seed\": 42})\n",
+                "eval_config = get_eval_config(eval_llm)"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "id": "aab7514e-a6ef-4c21-b90f-d9cbefcf5af1",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "View the evaluation results for project 'test-notable-cake-39' at:\n",
+                        "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/projects/p/9950f779-8f98-4ca0-90ab-30e4f9f7af6c?eval=true\n",
+                        "\n",
+                        "View all tests for Dataset Email Extraction at:\n",
+                        "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/309a2fce-ce68-43aa-befb-67f94d0c3570\n",
+                        "[------------------------------------------------->] 42/42"
+                    ]
+                }
+            ],
+            "source": [
+                "test_run = client.run_on_dataset(\n",
+                "    dataset_name=task.name,\n",
+                "    llm_or_chain_factory=extraction_chain,\n",
+                "    evaluation=eval_config,\n",
+                "    verbose=True,\n",
+                "    tags=[\"openai-functions\"],\n",
+                ")"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "1b039225-01cf-481a-87a6-4e880e9b1dcd",
+            "metadata": {},
+            "source": [
+                "## Inspect\n",
+                "\n",
+                "Here, we'll take a look at the underlying results a little bit.\n",
+                "\n",
+                "A few things to note:\n",
+                "\n",
+                "* The correctness is 66% (so it's messing up a lot!)\n",
+                "* The number of tool invocations made by the agent can be very large; e.g., 15 invocations, when only a single invocation was actually needed."
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "id": "6eb19db1-43b8-4866-a3d2-f211ba92ab8b",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [],
+            "source": [
+                "import pandas as pd\n",
+                "\n",
+                "df = test_run.to_dataframe()\n",
+                "df = pd.json_normalize(df.to_dict(orient=\"records\"))"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "id": "7ab5a8b9-a937-4537-b879-704284df4494",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [],
+            "source": [
+                "df"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "416dce43-7e76-431c-b556-55abef32f393",
+            "metadata": {},
+            "source": [
+                "An example of a poorly behaving agent that seems to have gotten stuck in a loop!"
+            ]
+        }
+    ],
+    "metadata": {
+        "kernelspec": {
+            "display_name": "Python 3 (ipykernel)",
+            "language": "python",
+            "name": "python3"
+        },
+        "language_info": {
+            "codemirror_mode": {
+                "name": "ipython",
+                "version": 3
+            },
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.11.2"
+        }
+    },
+    "nbformat": 4,
+    "nbformat_minor": 5
+}

docs/source/notebooks/extraction/intro.ipynb

@@ -0,0 +1,122 @@
+{
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "id": "7e8fc49a-e8b2-404b-a059-e9f668c460e5",
+            "metadata": {},
+            "source": [
+                "# Extraction Tasks\n",
+                "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain-benchmarks/blob/main/docs/source/notebooks/extraction/intro.ipynb)\n",
+                "\n",
+                "These tasks refer to an LLM's ability to extract structured output from an unstructured source, such as emails, websites, or other text. Below are a list of supported datasets.\n"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 4,
+            "id": "86912590-a90a-4351-8ab4-89192cdee1e7",
+            "metadata": {},
+            "outputs": [
+                {
+                    "data": {
+                        "text/html": [
+                            "<table>\n",
+                            "<thead>\n",
+                            "<tr><th>Name            </th><th>Type          </th><th>Dataset ID                                                                                                                                                 </th><th>Description  </th></tr>\n",
+                            "</thead>\n",
+                            "<tbody>\n",
+                            "<tr><td>Email Extraction</td><td>ExtractionTask</td><td><a href=\"https://smith.langchain.com/public/36bdfe7d-3cd1-4b36-b957-d12d95810a2b/d\" target=\"_blank\" rel=\"noopener\">36bdfe7d-3cd1-4b36-b957-d12d95810a2b</a></td><td>A dataset of 42 real emails deduped from a spam folder, with semantic HTML tags removed, as well as a script for initial extraction and formatting of other emails from an arbitrary .mbox file like the one exported by Gmail.\n",
+                            "\n",
+                            "Some additional cleanup of the data was done by hand after the initial pass.\n",
+                            "\n",
+                            "See https://github.com/jacoblee93/oss-model-extraction-evals.              </td></tr>\n",
+                            "</tbody>\n",
+                            "</table>"
+                        ],
+                        "text/plain": [
+                            "Registry(tasks=[ExtractionTask(name='Email Extraction', dataset_id='https://smith.langchain.com/public/36bdfe7d-3cd1-4b36-b957-d12d95810a2b/d', description='A dataset of 42 real emails deduped from a spam folder, with semantic HTML tags removed, as well as a script for initial extraction and formatting of other emails from an arbitrary .mbox file like the one exported by Gmail.\\n\\nSome additional cleanup of the data was done by hand after the initial pass.\\n\\nSee https://github.com/jacoblee93/oss-model-extraction-evals.\\n    ', schema=<class 'langchain_benchmarks.extraction.tasks.email_task.Email'>, instructions=ChatPromptTemplate(input_variables=['email'], messages=[SystemMessagePromptTemplate(prompt=PromptTemplate(input_variables=[], template='You are an expert researcher.')), HumanMessagePromptTemplate(prompt=PromptTemplate(input_variables=['email'], template='What can you tell me about the following email? Make sure to extract the question in the correct format. Here is the email:\\n ```\\n{email}\\n```'))]))])"
+                        ]
+                    },
+                    "execution_count": 4,
+                    "metadata": {},
+                    "output_type": "execute_result"
+                }
+            ],
+            "source": [
+                "from langchain_benchmarks import registry\n",
+                "\n",
+                "registry.filter(Type=\"ExtractionTask\")"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "e771e544-b8f5-4359-8fd7-c89b71fbe460",
+            "metadata": {},
+            "source": [
+                "### Task resources\n",
+                "\n",
+                "In addition to the dataset_id, name, and description, each extraction task provides the following:\n",
+                "\n",
+                "- `schema` - a pydantic base model defining the schema (or schemas) the model should extract\n",
+                "\n",
+                "\n",
+                "### Dataset schema\n",
+                "\n",
+                "Each task corresponds to a LangSmith dataset with the following schema:\n",
+                "\n",
+                "Inputs:\n",
+                "- `input: str` - the input text\n",
+                "\n",
+                "Outputs\n",
+                "- `output: str` - the expected extraction result, as a json object\n"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "d04e05f3-3f20-4fed-bb98-3eb072213bbd",
+            "metadata": {},
+            "source": [
+                "### Evaluation\n",
+                "\n",
+                "The extraction tasks also have an evaluation config, which defines default LangSmith evaluators to apply when benchmarking your architecture.\n",
+                "\n"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 5,
+            "id": "9c7865bd-8251-4579-85a3-f9085d96f497",
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "from langchain.chat_models import ChatOpenAI\n",
+                "\n",
+                "from langchain_benchmarks.extraction import get_eval_config\n",
+                "\n",
+                "eval_llm = ChatOpenAI(model=\"gpt-4\", model_kwargs={\"seed\": 42})\n",
+                "eval_config = get_eval_config(eval_llm)"
+            ]
+        }
+    ],
+    "metadata": {
+        "kernelspec": {
+            "display_name": "Python 3 (ipykernel)",
+            "language": "python",
+            "name": "python3"
+        },
+        "language_info": {
+            "codemirror_mode": {
+                "name": "ipython",
+                "version": 3
+            },
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.11.2"
+        }
+    },
+    "nbformat": 4,
+    "nbformat_minor": 5
+}

docs/source/notebooks/retrieval/comparing_techniques.ipynb

@@ -0,0 +1,886 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "e2af633b-4d0f-4b80-b090-2d6429f22e90",
+   "metadata": {},
+   "source": [
+    "# Evaluating RAG Architectures on Benchmark Tasks\n",
+    "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain-benchmarks/blob/main/docs/source/notebooks/retrieval/comparing_techniques.ipynb)\n",
+    "\n",
+    "\n",
+    "#### Introduction\n",
+    "\n",
+    "If you ever wanted to compare different approaches to Q&A over docs, you'll find this notebook helpful to get started evaluating different configurations and common RAG architectures on benchmark tasks. The goal is to make it easy for you to experiment with different techniques, understand their tradeoffs, and make informed decisions for your specific use case.\n",
+    "\n",
+    "#### What is RAG?\n",
+    "\n",
+    "LLMs have a knowledge cutoff. For them to accurately respond to user queries, they need access to relevant information. Retrieval Augmented Generation (RAG) (aka \"give an LLM a search engine\") is a common design pattern to address this. The key components are:\n",
+    "\n",
+    "- Retriever: fetches information from a knowledge base, which can be a vector search engine, a database, or any search engine.\n",
+    "- Generator: synthesizes responses using a blend of learned knowledge and the retrieved information.\n",
+    "\n",
+    "The overall quality of the system depends on both components.\n",
+    "\n",
+    "\n",
+    "#### Benchmark Tasks and Datasets (As of 2023/11/21)\n",
+    "\n",
+    "The following datasets are currently available:\n",
+    "\n",
+    "- LangChain Docs Q&A - technical questions based on the LangChain python documentation\n",
+    "- Semi-structured Earnings - financial questions and answers on financial PDFs containing tables and graphs\n",
+    "\n",
+    "Each task comes with a labeled dataset of questions and answers. They also provide configurable factory functions for easy customization of chunking and indexing for the relevant source documents.\n",
+    "\n",
+    "And with that, let's get started!\n",
+    "\n",
+    "## Pre-requisites\n",
+    "\n",
+    "We will install quite a few prerequisites for this example since we are comparing many techniques and models.\n",
+    "\n",
+    "We will be using LangSmith to capture the evaluation traces. You can make a free account at [smith.langchain.com](https://smith.langchain.com/). Once you've done so, you can make an API key and set it below.\n",
+    "\n",
+    "We are comparing many methods throughout this notebook, so the list of dependencies we will install is long."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9f44b59b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pip install -U --quiet langchain langsmith langchainhub langchain_benchmarks\n",
+    "%pip install --quiet chromadb openai huggingface pandas langchain_experimental sentence_transformers pyarrow anthropic tiktoken"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "62b518cf-99fb-44be-8acb-ee0a8ba62272",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
+    "os.environ[\"LANGCHAIN_API_KEY\"] = \"sk-...\"  # Your API key\n",
+    "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"  # Your OpenAI API key\n",
+    "os.environ[\"ANTHROPIC_API_KEY\"] = \"sk-...\"  # Your Anthropic API key\n",
+    "# Silence warnings from HuggingFace\n",
+    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2e8a666d-8bf5-4bfd-8b20-8b7defdb8cd5",
+   "metadata": {},
+   "source": [
+    "## Review Q&A tasks\n",
+    "\n",
+    "The registry provides configurations to test out common architectures on curated datasets.\n",
+    "Below is a list of the available tasks at the time of writing."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "b39159d0-9ea1-414f-a9d8-4a7b22b3d2cc",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain_benchmarks import clone_public_dataset, registry"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "3644d211-382e-41aa-b282-21b01d28fc35",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table>\n",
+       "<thead>\n",
+       "<tr><th>Name                   </th><th>Type         </th><th>Dataset ID                                                                                                                                                 </th><th>Description  </th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "<tr><td>LangChain Docs Q&A     </td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d\" target=\"_blank\" rel=\"noopener\">452ccafc-18e1-4314-885b-edd735f17b9d</a></td><td>Questions and answers based on a snapshot of the LangChain python docs.\n",
+       "\n",
+       "The environment provides the documents and the retriever information.\n",
+       "\n",
+       "Each example is composed of a question and reference answer.\n",
+       "\n",
+       "Success is measured based on the accuracy of the answer relative to the reference answer.\n",
+       "We also measure the faithfulness of the model's response relative to the retrieved documents (if any).              </td></tr>\n",
+       "<tr><td>Semi-structured Reports</td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d\" target=\"_blank\" rel=\"noopener\">c47d9617-ab99-4d6e-a6e6-92b8daf85a7d</a></td><td>Questions and answers based on PDFs containing tables and charts.\n",
+       "\n",
+       "The task provides the raw documents as well as factory methods to easily index them\n",
+       "and create a retriever.\n",
+       "\n",
+       "Each example is composed of a question and reference answer.\n",
+       "\n",
+       "Success is measured based on the accuracy of the answer relative to the reference answer.\n",
+       "We also measure the faithfulness of the model's response relative to the retrieved documents (if any).              </td></tr>\n",
+       "</tbody>\n",
+       "</table>"
+      ],
+      "text/plain": [
+       "Registry(tasks=[RetrievalTask(name='LangChain Docs Q&A', dataset_id='https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d', description=\"Questions and answers based on a snapshot of the LangChain python docs.\\n\\nThe environment provides the documents and the retriever information.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x12aae2840>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x12aae28e0>, 'hyde': <function _chroma_hyde_retriever_factory at 0x12aae2980>}, architecture_factories={'conversational-retrieval-qa': <function default_response_chain at 0x12a1be020>}, get_docs=<function load_cached_docs at 0x12a1bdb20>), RetrievalTask(name='Semi-structured Reports', dataset_id='https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d', description=\"Questions and answers based on PDFs containing tables and charts.\\n\\nThe task provides the raw documents as well as factory methods to easily index them\\nand create a retriever.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x12aae3060>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x12aae3100>, 'hyde': <function _chroma_hyde_retriever_factory at 0x12aae31a0>}, architecture_factories={}, get_docs=<function load_docs at 0x12aae2fc0>)])"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "registry.filter(Type=\"RetrievalTask\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "671282f8-c455-4390-b018-e53bbd833093",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "langchain_docs = registry[\"LangChain Docs Q&A\"]\n",
+    "langchain_docs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "70369f67-deb4-467a-801a-6d38c3d0460d",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "clone_public_dataset(langchain_docs.dataset_id, dataset_name=langchain_docs.name)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "02011398-1a6f-42c1-b586-9d01c78e3ee4",
+   "metadata": {},
+   "source": [
+    "## Basic Vector Retrieval\n",
+    "\n",
+    "For our first example, we will generate a single embedding for each document in the dataset,\n",
+    "without chunking or indexing, and then provide that retriever to an LLM for inference."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c58247f5-b9bd-4cc5-9632-78bc21bb10b4",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.embeddings import HuggingFaceEmbeddings\n",
+    "\n",
+    "embeddings = HuggingFaceEmbeddings(\n",
+    "    model_name=\"thenlper/gte-base\",\n",
+    "    model_kwargs={\"device\": 0},  # Comment out to use CPU\n",
+    ")\n",
+    "\n",
+    "retriever_factory = langchain_docs.retriever_factories[\"basic\"]\n",
+    "# Indexes the documents with the specified embeddings\n",
+    "# Note that this does not apply any chunking to the docs,\n",
+    "# which means the documents can be of arbitrary length\n",
+    "retriever = retriever_factory(embeddings)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f4d2e139-2653-4f7b-944b-91ef52f43d3e",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Factory for creating a conversational retrieval QA chain\n",
+    "\n",
+    "chain_factory = langchain_docs.architecture_factories[\"conversational-retrieval-qa\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9f9be718-64f0-4706-9527-240a1cdb3ecb",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.chat_models import ChatAnthropic\n",
+    "\n",
+    "# Example\n",
+    "llm = ChatAnthropic(model=\"claude-2\", temperature=1)\n",
+    "\n",
+    "chain_factory(retriever, llm=llm).invoke({\"question\": \"what's lcel?\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "513042fe-2878-44f8-ae84-05b9d521c1de",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from functools import partial\n",
+    "\n",
+    "from langsmith.client import Client\n",
+    "\n",
+    "from langchain_benchmarks.rag import get_eval_config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aab7514e-a6ef-4c21-b90f-d9cbefcf5af1",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "client = Client()\n",
+    "RAG_EVALUATION = get_eval_config()\n",
+    "\n",
+    "test_run = client.run_on_dataset(\n",
+    "    dataset_name=langchain_docs.name,\n",
+    "    llm_or_chain_factory=partial(chain_factory, retriever, llm=llm),\n",
+    "    evaluation=RAG_EVALUATION,\n",
+    "    verbose=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e86578d5-be5c-4bcd-9dcb-35280eeed3f9",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "test_run.get_aggregate_feedback()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ee992f87-4137-49b1-a1f1-0cc7be0e32d8",
+   "metadata": {},
+   "source": [
+    "# Comparing with other indexing strategies\n",
+    "\n",
+    "The index used above retrieves the raw documents based on a single vector per document. It doesn't perform any additional chunking. You can try changing the chunking parameters when generating the index.\n",
+    "\n",
+    "## Customizing Chunking\n",
+    "\n",
+    "The simplest change you can make to the index is configure how you split the "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e72030d4-c201-44b8-85cd-903afa313f11",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
+    "\n",
+    "\n",
+    "def transform_docs(docs):\n",
+    "    splitter = RecursiveCharacterTextSplitter(chunk_size=4000, chunk_overlap=200)\n",
+    "    yield from splitter.split_documents(docs)\n",
+    "\n",
+    "\n",
+    "# Used for the cache\n",
+    "transformation_name = \"recursive-text-cs4k-ol200\"\n",
+    "\n",
+    "retriever_factory = langchain_docs.retriever_factories[\"basic\"]\n",
+    "\n",
+    "chunked_retriever = retriever_factory(\n",
+    "    embeddings,\n",
+    "    transform_docs=transform_docs,\n",
+    "    transformation_name=transformation_name,\n",
+    "    search_kwargs={\"k\": 4},\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d74f12f9-1ba6-4bf7-a850-4073fb0994f9",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "chunked_results = client.run_on_dataset(\n",
+    "    dataset_name=langchain_docs.name,\n",
+    "    llm_or_chain_factory=partial(chain_factory, retriever, llm=llm),\n",
+    "    evaluation=RAG_EVALUATION,\n",
+    "    verbose=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d6d825f1-9a91-429d-bf3e-a9b9c2785a69",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "chunked_results.get_aggregate_feedback()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5a7a62ec-9a9c-4d7a-ab90-97020d855ee7",
+   "metadata": {},
+   "source": [
+    "## Parent Document Retriever\n",
+    "\n",
+    "This indexing technique chunks documents and generates 1 vector per chunk.\n",
+    "At retrieval time, the K \"most similar\" chunks are fetched, then the full parent documents are returned for the LLM to reason over.\n",
+    "\n",
+    "This ensures the chunk is surfaced in its full natural context. It also can potentially improve the initial retrieval quality since the similarity scores are scoped to individual chunks.\n",
+    "\n",
+    "Let's see if this technique is effective in our case."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1398f5e3-b7fe-4693-bcc0-c6c6f75c8234",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "retriever_factory = langchain_docs.retriever_factories[\"parent-doc\"]\n",
+    "\n",
+    "# Indexes the documents with the specified embeddings\n",
+    "parent_doc_retriever = retriever_factory(embeddings)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7b1f4b5d-143a-44ce-95f4-d0b5782ada74",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "parent_doc_test_run = client.run_on_dataset(\n",
+    "    dataset_name=langchain_docs.name,\n",
+    "    llm_or_chain_factory=partial(chain_factory, parent_doc_retriever, llm=llm),\n",
+    "    evaluation=RAG_EVALUATION,\n",
+    "    verbose=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3cef0410-47ec-4830-9b75-621eb85240ed",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "parent_doc_test_run.get_aggregate_feedback()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "d4b27dd0-f0df-4551-a972-1a6c0df5ffb9",
+   "metadata": {},
+   "source": [
+    "## HyDE\n",
+    "\n",
+    "HyDE (Hypothetical document embeddings) refers to the technique of using an LLM\n",
+    "to generate example queries that my be used to retrieve a doc. By doing so, the resulting embeddings are automatically \"more aligned\" with the embeddings generated from the query. This comes with an additional indexing cost, since each document requires an additoinal call to an LLM while indexing."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9c92d2c2-f410-43cc-9c9f-abc22ef48353",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "retriever_factory = langchain_docs.retriever_factories[\"hyde\"]\n",
+    "\n",
+    "retriever = retriever_factory(embeddings)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2179cf29-2d75-4a04-bbb5-b8f22028fa34",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "hyde_test_run = client.run_on_dataset(\n",
+    "    dataset_name=langchain_docs.name,\n",
+    "    llm_or_chain_factory=partial(chain_factory, retriever),\n",
+    "    evaluation=RAG_EVALUATION,\n",
+    "    verbose=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "94a04f21-0308-4b00-a6f1-694d98ba7109",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "hyde_test_run.get_aggregate_feedback()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2c8af309-d0c4-4562-a5f0-30ca9f9fd861",
+   "metadata": {},
+   "source": [
+    "# Comparing Embeddings\n",
+    "\n",
+    "We've been using off-the-shelf GTE-Base embeddings so far to retrieve the docs, but\n",
+    "you may get better results with other embeddings. You could even try fine-tuning embedddings on your own documentation and evaluating here.\n",
+    "\n",
+    "Let's compare our results so far to OpenAI's embeddings."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4e0b2395-c07e-4eae-bb21-afdda3961cc2",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.embeddings.openai import OpenAIEmbeddings\n",
+    "\n",
+    "openai_embeddings = OpenAIEmbeddings()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "14a5edab-9a3a-4864-b69f-69bc1c9e7816",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "openai_retriever = langchain_docs.retriever_factories[\"basic\"](openai_embeddings)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6757c411-aaa5-42ad-824c-7c0b5b942e40",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "openai_embeddings_test_run = client.run_on_dataset(\n",
+    "    dataset_name=langchain_docs.name,\n",
+    "    llm_or_chain_factory=partial(chain_factory, openai_retriever),\n",
+    "    evaluation=RAG_EVALUATION,\n",
+    "    verbose=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a8ae7cbe-a8eb-4b40-aeae-f9c7f4bf335f",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "openai_embeddings_test_run.get_aggregate_feedback()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3ef164b9-7124-4907-b2b4-0595bf3b3441",
+   "metadata": {},
+   "source": [
+    "## Comparing Models\n",
+    "\n",
+    "We used Anthropic's Claude-2 model in our previous tests, but lets try with some other models.\n",
+    "\n",
+    "You can swap in any LangChain LLM within the response generator below.\n",
+    "We'll try a long-context llama 2 model first (using Ollama)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "402c86c7-9754-4527-a1a9-a89beba437b4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.chat_models import ChatOllama\n",
+    "\n",
+    "# A llama2-based model with 128k context\n",
+    "# (in theory) In practice, we will see how well\n",
+    "# it actually leverages that context.\n",
+    "ollama = ChatOllama(model=\"yarn-llama2:7b-128k\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "256b75fe-9f0c-4820-8e6c-9c87df39f0a7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ollama.invoke(\"Hi there\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fc7dff86-2b93-490a-81ab-72e757e8f1b3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# We'll go back to the GTE embeddings for now\n",
+    "\n",
+    "retriever_factory = langchain_docs.retriever_factories[\"basic\"]\n",
+    "retriever = retriever_factory(embeddings)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "eaa47085-e383-4cc5-9018-5491700c6f71",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "ollama_test_run = client.run_on_dataset(\n",
+    "    dataset_name=langchain_docs.name,\n",
+    "    llm_or_chain_factory=partial(chain_factory, llm=ollama, retriever=retriever),\n",
+    "    evaluation=RAG_EVALUATION,\n",
+    "    verbose=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "98edcf42-a405-400b-882e-04de2559359c",
+   "metadata": {},
+   "source": [
+    "## Changing the prompt in the response generator\n",
+    "\n",
+    "The default prompt was tested primariily on OpenAI's gpt-3.5 model. When switching models, you may get better results if you modify the prompt. Let's try a simple one."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "69d3b36f-68aa-4005-9bb2-de228491ef86",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain import hub\n",
+    "from langchain.schema.output_parser import StrOutputParser"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "64caac6f-888d-432c-9329-5c4b97ad859d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "prompt = hub.pull(\"wfh/rag-simple\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3c0e6762-1e50-4eef-833a-a4a2bf8883ba",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "generator = prompt | ChatAnthropic(model=\"claude-2\", temperature=1) | StrOutputParser()\n",
+    "new_chain = chain_factory(response_generator=generator, retriever=openai_retriever)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "96886de0-a653-4875-a68f-5a11efcb200b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "claude_simple_prompt_test_run = client.run_on_dataset(\n",
+    "    dataset_name=langchain_docs.name,\n",
+    "    llm_or_chain_factory=partial(\n",
+    "        chain_factory, response_generator=generator, retriever=retriever\n",
+    "    ),\n",
+    "    evaluation=RAG_EVALUATION,\n",
+    "    verbose=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "daffaf28-902e-4466-b3b9-25441d45585d",
+   "metadata": {},
+   "source": [
+    "## Testing Agents\n",
+    "\n",
+    "Agents use an LLM to decide actions and generate responses. There are two obvious ways they could potentially succeed where the approaches above fail:\n",
+    "- The above chains do not \"rephrase\" the user query. It could be that the rephrased question will result in more relevant documents.\n",
+    "- The above chains must respond based on a single retrieval step. Agents can iteratively query the retriever or subdivide the query into different parts to synthesize at the end. Our dataset has a number of questions that require information from different documents - if the\n",
+    "\n",
+    "Let's evaluate to see whether the \"plausible\" statements above are worth the tradeoffs. We will use the basic retriever as a tool for them."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c31c19c4-f8d6-41b3-9389-e89abd4b5f04",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from typing import List, Tuple\n",
+    "\n",
+    "from langchain.agents import AgentExecutor\n",
+    "from langchain.agents.format_scratchpad import format_to_openai_functions\n",
+    "from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser\n",
+    "from langchain.chat_models import ChatOpenAI\n",
+    "from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder\n",
+    "from langchain.pydantic_v1 import BaseModel, Field\n",
+    "from langchain.schema.messages import AIMessage, HumanMessage\n",
+    "from langchain.tools import tool\n",
+    "from langchain.tools.render import format_tool_to_openai_function\n",
+    "\n",
+    "# This is used to tell the model how to best use the retriever.\n",
+    "\n",
+    "\n",
+    "@tool\n",
+    "def search(query, callbacks=None):\n",
+    "    \"\"\"Search the LangChain docs with the retriever.\"\"\"\n",
+    "    return retriever.get_relevant_documents(query, callbacks=callbacks)\n",
+    "\n",
+    "\n",
+    "tools = [search]\n",
+    "\n",
+    "llm = ChatOpenAI(model=\"gpt-4-1106-preview\", temperature=0)\n",
+    "assistant_system_message = \"\"\"You are a helpful assistant tasked with answering technical questions about LangChain. \\\n",
+    "Use tools (only if necessary) to best answer the users questions. Do not make up information if you cannot find the answer using your tools.\"\"\"\n",
+    "prompt = ChatPromptTemplate.from_messages(\n",
+    "    [\n",
+    "        (\"system\", assistant_system_message),\n",
+    "        MessagesPlaceholder(variable_name=\"chat_history\"),\n",
+    "        (\"user\", \"{input}\"),\n",
+    "        MessagesPlaceholder(variable_name=\"agent_scratchpad\"),\n",
+    "    ]\n",
+    ")\n",
+    "\n",
+    "llm_with_tools = llm.bind(functions=[format_tool_to_openai_function(t) for t in tools])\n",
+    "\n",
+    "\n",
+    "def _format_chat_history(chat_history: List[Tuple[str, str]]):\n",
+    "    buffer = []\n",
+    "    for human, ai in chat_history:\n",
+    "        buffer.append(HumanMessage(content=human))\n",
+    "        buffer.append(AIMessage(content=ai))\n",
+    "    return buffer\n",
+    "\n",
+    "\n",
+    "agent = (\n",
+    "    {\n",
+    "        \"input\": lambda x: x[\"input\"],\n",
+    "        \"chat_history\": lambda x: _format_chat_history(x[\"chat_history\"]),\n",
+    "        \"agent_scratchpad\": lambda x: format_to_openai_functions(\n",
+    "            x[\"intermediate_steps\"]\n",
+    "        ),\n",
+    "    }\n",
+    "    | prompt\n",
+    "    | llm_with_tools\n",
+    "    | OpenAIFunctionsAgentOutputParser()\n",
+    ")\n",
+    "\n",
+    "\n",
+    "class AgentInput(BaseModel):\n",
+    "    input: str\n",
+    "    chat_history: List[Tuple[str, str]] = Field(..., extra={\"widget\": {\"type\": \"chat\"}})\n",
+    "\n",
+    "\n",
+    "agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=False).with_types(\n",
+    "    input_type=AgentInput\n",
+    ")\n",
+    "\n",
+    "\n",
+    "class ChainInput(BaseModel):\n",
+    "    question: str\n",
+    "\n",
+    "\n",
+    "def mapper(input: dict):\n",
+    "    return {\"input\": input[\"question\"], \"chat_history\": []}\n",
+    "\n",
+    "\n",
+    "agent_executor = (mapper | agent_executor | (lambda x: x[\"output\"])).with_types(\n",
+    "    input_type=ChainInput\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5a1c09c2-a983-450b-a531-c6871a9b27ae",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "oai_functions_test_run = client.run_on_dataset(\n",
+    "    dataset_name=langchain_docs.name,\n",
+    "    llm_or_chain_factory=agent_executor,\n",
+    "    evaluation=RAG_EVALUATION,\n",
+    "    verbose=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "73ffd740-fde1-479d-b84c-7dd8f65716a6",
+   "metadata": {},
+   "source": [
+    "## Assistant\n",
+    "\n",
+    "OpenAI provides a hosted agent service through their Assistants API. \n",
+    "\n",
+    "You can connect your LangChain retriever to an OpenAI's Assistant API and evaluate its performance. Let's test below:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3de4d42b-e34a-4980-97eb-9b2c78a24089",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "\n",
+    "from langchain.agents import AgentExecutor\n",
+    "from langchain.tools import tool\n",
+    "from langchain_experimental.openai_assistant import OpenAIAssistantRunnable\n",
+    "\n",
+    "\n",
+    "@tool\n",
+    "def search(query, callbacks=None) -> str:\n",
+    "    \"\"\"Search the LangChain docs with the retriever.\"\"\"\n",
+    "    docs = retriever.get_relevant_documents(query, callbacks=callbacks)\n",
+    "    return json.dumps([doc.dict() for doc in docs])\n",
+    "\n",
+    "\n",
+    "tools = [search]\n",
+    "\n",
+    "agent = OpenAIAssistantRunnable.create_assistant(\n",
+    "    name=\"langchain docs assistant\",\n",
+    "    instructions=\"You are a helpful assistant tasked with answering technical questions about LangChain.\",\n",
+    "    tools=tools,\n",
+    "    model=\"gpt-4-1106-preview\",\n",
+    "    as_agent=True,\n",
+    ")\n",
+    "\n",
+    "\n",
+    "assistant_exector = (\n",
+    "    (lambda x: {\"content\": x[\"question\"]})\n",
+    "    | AgentExecutor(agent=agent, tools=tools)\n",
+    "    | (lambda x: x[\"output\"])\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cd6baea1-ac90-43aa-a21c-98aa5ca23732",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assistant_test_run = client.run_on_dataset(\n",
+    "    dataset_name=langchain_docs.name,\n",
+    "    llm_or_chain_factory=assistant_exector,\n",
+    "    evaluation=RAG_EVALUATION,\n",
+    "    verbose=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a5ac5fad-0a74-4403-a917-7145be6d7d1a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "assistant_test_run.get_aggregate_feedback()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6a04b3e4-b5df-4075-9089-8aa10ef63348",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/source/notebooks/retrieval/intro.ipynb

@@ -0,0 +1,117 @@
+{
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "# Retrieval Tasks\n",
+                "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain-benchmarks/blob/main/docs/source/notebooks/retrieval/intro.ipynb)\n",
+                "\n",
+                "These tasks are meant to test retrieval-augmented generation (RAG) architectures on various datasets.\n",
+                "\n",
+                "You can check an up-to-date list of retrieval tasks in the registry:"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 2,
+            "metadata": {},
+            "outputs": [
+                {
+                    "data": {
+                        "text/html": [
+                            "<table>\n",
+                            "<thead>\n",
+                            "<tr><th>Name                    </th><th>Type         </th><th>Dataset ID                                                                                                                                                 </th><th>Description  </th></tr>\n",
+                            "</thead>\n",
+                            "<tbody>\n",
+                            "<tr><td>LangChain Docs Q&A      </td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d\" target=\"_blank\" rel=\"noopener\">452ccafc-18e1-4314-885b-edd735f17b9d</a></td><td>Questions and answers based on a snapshot of the LangChain python docs.\n",
+                            "\n",
+                            "The environment provides the documents and the retriever information.\n",
+                            "\n",
+                            "Each example is composed of a question and reference answer.\n",
+                            "\n",
+                            "Success is measured based on the accuracy of the answer relative to the reference answer.\n",
+                            "We also measure the faithfulness of the model's response relative to the retrieved documents (if any).              </td></tr>\n",
+                            "<tr><td>Semi-structured Earnings</td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d\" target=\"_blank\" rel=\"noopener\">c47d9617-ab99-4d6e-a6e6-92b8daf85a7d</a></td><td>Questions and answers based on PDFs containing tables and charts.\n",
+                            "\n",
+                            "The task provides the raw documents as well as factory methods to easily index them\n",
+                            "and create a retriever.\n",
+                            "\n",
+                            "Each example is composed of a question and reference answer.\n",
+                            "\n",
+                            "Success is measured based on the accuracy of the answer relative to the reference answer.\n",
+                            "We also measure the faithfulness of the model's response relative to the retrieved documents (if any).              </td></tr>\n",
+                            "</tbody>\n",
+                            "</table>"
+                        ],
+                        "text/plain": [
+                            "Registry(tasks=[RetrievalTask(name='LangChain Docs Q&A', dataset_id='https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d', description=\"Questions and answers based on a snapshot of the LangChain python docs.\\n\\nThe environment provides the documents and the retriever information.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x138367ec0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x138367f60>, 'hyde': <function _chroma_hyde_retriever_factory at 0x13838c040>}, architecture_factories={'conversational-retrieval-qa': <function default_response_chain at 0x11fa74fe0>}, get_docs=<function load_cached_docs at 0x101bfb240>), RetrievalTask(name='Semi-structured Earnings', dataset_id='https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d', description=\"Questions and answers based on PDFs containing tables and charts.\\n\\nThe task provides the raw documents as well as factory methods to easily index them\\nand create a retriever.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x13838c5e0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x13838c680>, 'hyde': <function _chroma_hyde_retriever_factory at 0x13838c720>}, architecture_factories={}, get_docs=<function load_docs at 0x13838c540>)])"
+                        ]
+                    },
+                    "execution_count": 2,
+                    "metadata": {},
+                    "output_type": "execute_result"
+                }
+            ],
+            "source": [
+                "from langchain_benchmarks import registry\n",
+                "\n",
+                "registry.filter(Type=\"RetrievalTask\")"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "metadata": {},
+            "source": [
+                "### Task resources\n",
+                "\n",
+                "In addition to a name, daset_id, and description, each retrieval task provides a few helper functions you can use to configure your pipeline.\n",
+                "\n",
+                "- `get_docs: callable` - fetches the original `Document` objects from the cache. Each task may provide configurable parameters you can use to define how the original documents are fetched.\n",
+                "- `retriever_factories: Dict[str, callable]` - define some configurable pipelines you can use to transform the documents, embed them, and add them to a vectorstore (or other retriever object) for downstream use. They use LangChain's caching `index` API so you don't have to re-index for every evaluation. For custom transformations, we ask that you provide a `transformation_name` to isolate the cache and vectorstore namespace. Currently (2023/11/21) these all use Chroma as a vectorstore, but you can swap this out\n",
+                "- `chain_factories: Dict[str, callable]` - define some off-the-shelf architectures you can configure to evaluate.\n",
+                "\n",
+                "When evaluating, you don't have to use any of these factory methods. You can instead define your own custom architecture or ETl pipeline before evaluating. They are meant to facilitate evaluations and comparisons for specific design decisions.\n",
+                "\n",
+                "### Dataset schema\n",
+                "\n",
+                "Each task corresponds to a LangSmith dataset with the following schema:\n",
+                "\n",
+                "Inputs:\n",
+                "- question: str - the user question\n",
+                "\n",
+                "Outputs\n",
+                "- answer: str - the expected answer"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "metadata": {},
+            "outputs": [],
+            "source": []
+        }
+    ],
+    "metadata": {
+        "kernelspec": {
+            "display_name": "Python 3 (ipykernel)",
+            "language": "python",
+            "name": "python3"
+        },
+        "language_info": {
+            "codemirror_mode": {
+                "name": "ipython",
+                "version": 3
+            },
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.11.2"
+        }
+    },
+    "nbformat": 4,
+    "nbformat_minor": 4
+}

docs/source/notebooks/retrieval/langchain_docs_qa.ipynb

@@ -0,0 +1,374 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "60bb467d-861d-4b07-a48d-8e5aa177c969",
+   "metadata": {},
+   "source": [
+    "# Q&A over LangChain Docs\n",
+    "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain-benchmarks/blob/main/docs/source/notebooks/retrieval/langchain_docs_qa.ipynb)\n",
+    "\n",
+    "Let's evaluate your architecture on a Q&A dataset for the LangChain python docs. For more examples of how to test different embeddings, indexing strategies, and architectures, see the [Evaluating RAG Architectures on Benchmark Tasks](./comparing_techniques.ipynb) notebook.\n",
+    "\n",
+    "## Pre-requisites\n",
+    "\n",
+    "We will install quite a few prerequisites for this example since we are comparing many techniques and models.\n",
+    "\n",
+    "We will be using LangSmith to capture the evaluation traces. You can make a free account at [smith.langchain.com](https://smith.langchain.com/). Once you've done so, you can make an API key and set it below."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9f44b59b",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "%pip install -U --quiet langchain langsmith langchainhub langchain_benchmarks\n",
+    "%pip install --quiet chromadb openai huggingface pandas langchain_experimental sentence_transformers pyarrow anthropic tiktoken"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0aae13f6-cd40-41e6-bd02-bd683e91cbff",
+   "metadata": {},
+   "source": [
+    "For this code to work, please configure LangSmith environment variables with your credentials."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "62b518cf-99fb-44be-8acb-ee0a8ba62272",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
+    "os.environ[\"LANGCHAIN_API_KEY\"] = \"ls_...\"  # Your API key"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4292ee8c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Update these with your own API keys\n",
+    "os.environ[\"ANTHROPIC_API_KEY\"] = \"sk-...\"\n",
+    "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"\n",
+    "# Silence warnings from HuggingFace\n",
+    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2e8a666d-8bf5-4bfd-8b20-8b7defdb8cd5",
+   "metadata": {},
+   "source": [
+    "## Review Q&A Tasks\n",
+    "\n",
+    "The registry provides configurations to test out common architectures on curated datasets."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b39159d0-9ea1-414f-a9d8-4a7b22b3d2cc",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain_benchmarks import clone_public_dataset, registry"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3644d211-382e-41aa-b282-21b01d28fc35",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "registry = registry.filter(Type=\"RetrievalTask\")\n",
+    "registry"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "671282f8-c455-4390-b018-e53bbd833093",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "langchain_docs = registry[\"LangChain Docs Q&A\"]\n",
+    "langchain_docs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "70369f67-deb4-467a-801a-6d38c3d0460d",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "clone_public_dataset(langchain_docs.dataset_id, dataset_name=langchain_docs.name)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c58247f5-b9bd-4cc5-9632-78bc21bb10b4",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.embeddings import HuggingFaceEmbeddings\n",
+    "\n",
+    "embeddings = HuggingFaceEmbeddings(\n",
+    "    model_name=\"thenlper/gte-base\",\n",
+    "    model_kwargs={\"device\": 0},  # Comment out to use CPU\n",
+    ")\n",
+    "\n",
+    "docs = langchain_docs.get_docs()\n",
+    "retriever_factory = langchain_docs.retriever_factories[\"basic\"]\n",
+    "# Indexes the documents with the specified embeddings\n",
+    "# Note that this does not apply any chunking to the docs,\n",
+    "# which means the documents can be of arbitrary length\n",
+    "retriever = retriever_factory(embeddings, docs=docs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "41e64350-63a7-4e7d-8e03-7dc459c444cc",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from operator import itemgetter\n",
+    "from typing import Sequence\n",
+    "\n",
+    "from langchain.chat_models import ChatAnthropic\n",
+    "from langchain.prompts import ChatPromptTemplate\n",
+    "from langchain.schema.document import Document\n",
+    "from langchain.schema.output_parser import StrOutputParser\n",
+    "from langchain.schema.runnable import RunnableLambda\n",
+    "from langchain.schema.runnable.passthrough import RunnableAssign\n",
+    "\n",
+    "\n",
+    "def format_docs(docs: Sequence[Document]) -> str:\n",
+    "    formatted_docs = []\n",
+    "    for i, doc in enumerate(docs):\n",
+    "        doc_string = (\n",
+    "            f\"<document index='{i}'>\\n\"\n",
+    "            f\"<source>{doc.metadata.get('source')}</source>\\n\"\n",
+    "            f\"<doc_content>{doc.page_content}</doc_content>\\n\"\n",
+    "            \"</document>\"\n",
+    "        )\n",
+    "        formatted_docs.append(doc_string)\n",
+    "    formatted_str = \"\\n\".join(formatted_docs)\n",
+    "    return f\"<documents>\\n{formatted_str}\\n</documents>\"\n",
+    "\n",
+    "\n",
+    "prompt = ChatPromptTemplate.from_messages(\n",
+    "    [\n",
+    "        (\n",
+    "            \"system\",\n",
+    "            \"You are an AI assistant answering questions about LangChain.\"\n",
+    "            \"\\n{context}\\n\"\n",
+    "            \"Respond solely based on the document content.\",\n",
+    "        ),\n",
+    "        (\"human\", \"{question}\"),\n",
+    "    ]\n",
+    ")\n",
+    "llm = ChatAnthropic(model=\"claude-2.1\", temperature=1)\n",
+    "\n",
+    "response_generator = (prompt | llm | StrOutputParser()).with_config(\n",
+    "    run_name=\"GenerateResponse\",\n",
+    ")\n",
+    "chain = (\n",
+    "    RunnableAssign(\n",
+    "        {\n",
+    "            \"context\": (itemgetter(\"question\") | retriever | format_docs).with_config(\n",
+    "                run_name=\"FormatDocs\"\n",
+    "            )\n",
+    "        }\n",
+    "    )\n",
+    "    | response_generator\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "10a1fca9-d356-4cff-93a9-c4f63944e57d",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "chain.invoke({\"question\": \"What's expression language?\"})"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3821e4b0-8e67-418a-840c-470fcde42df0",
+   "metadata": {},
+   "source": [
+    "### Evaluate\n",
+    "\n",
+    "Let's evaluate your RAG architecture on the dataset now."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "513042fe-2878-44f8-ae84-05b9d521c1de",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langsmith.client import Client\n",
+    "\n",
+    "from langchain_benchmarks.rag import get_eval_config"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aab7514e-a6ef-4c21-b90f-d9cbefcf5af1",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "client = Client()\n",
+    "RAG_EVALUATION = get_eval_config()\n",
+    "\n",
+    "test_run = client.run_on_dataset(\n",
+    "    dataset_name=langchain_docs.name,\n",
+    "    llm_or_chain_factory=chain,\n",
+    "    evaluation=RAG_EVALUATION,\n",
+    "    verbose=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e86578d5-be5c-4bcd-9dcb-35280eeed3f9",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "test_run.get_aggregate_feedback()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "01811b97-cb28-42a6-920a-7a700f77f19d",
+   "metadata": {},
+   "source": [
+    "## Evaluate with a default factory\n",
+    "\n",
+    "The task can define default chain and retriever \"factories\", which provide a default architecture that you can modify by choosing the llms, prompts, etc. Let's try the `conversational-retrieval-qa` factory."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f4d2e139-2653-4f7b-944b-91ef52f43d3e",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# Factory for creating a conversational retrieval QA chain\n",
+    "chain_factory = langchain_docs.architecture_factories[\"conversational-retrieval-qa\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9f9be718-64f0-4706-9527-240a1cdb3ecb",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain.chat_models import ChatAnthropic\n",
+    "\n",
+    "# Example\n",
+    "llm = ChatAnthropic(model=\"claude-2\", temperature=1)\n",
+    "\n",
+    "\n",
+    "chain = chain_factory(retriever, llm=llm)\n",
+    "\n",
+    "chain.invoke({\"question\": \"What is expression language?\"})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e9c013e2-241a-4def-9aa6-ccb34273eeb9",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "test_run = client.run_on_dataset(\n",
+    "    dataset_name=langchain_docs.name,\n",
+    "    llm_or_chain_factory=chain,\n",
+    "    evaluation=RAG_EVALUATION,\n",
+    "    verbose=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8f3c1d2d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "test_run.get_aggregate_feedback()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/source/notebooks/retrieval/semi_structured.ipynb

@@ -0,0 +1,647 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "60bb467d-861d-4b07-a48d-8e5aa177c969",
+   "metadata": {},
+   "source": [
+    "# Semi-structured RAG\n",
+    "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain-benchmarks/blob/main/docs/source/notebooks/retrieval/semi_structured.ipynb)\n",
+    "\n",
+    "Let's evaluate your architecture on a small semi-structured Q&A dataset. This dataset is composed of QA pairs over pdfs that contain tables."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f49db759-7ce6-4ab7-a58f-7fc3a6a7c8ec",
+   "metadata": {},
+   "source": [
+    "## Pre-requisites\n",
+    "\n",
+    "We will install quite a few prerequisites for this example since we are comparing various techinques and models."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "9f44b59b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%pip install -U langchain langsmith langchainhub  langchain_benchmarks langchain_experimental\n",
+    "%pip install --quiet chromadb openai huggingface pandas \"unstructured[all-docs]\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0aae13f6-cd40-41e6-bd02-bd683e91cbff",
+   "metadata": {},
+   "source": [
+    "For this code to work, please configure LangSmith environment variables with your credentials."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "62b518cf-99fb-44be-8acb-ee0a8ba62272",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "\n",
+    "os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n",
+    "os.environ[\"LANGCHAIN_API_KEY\"] = \"sk-...\"  # Your API key\n",
+    "\n",
+    "# Silence warnings from HuggingFace\n",
+    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2e8a666d-8bf5-4bfd-8b20-8b7defdb8cd5",
+   "metadata": {},
+   "source": [
+    "## Review Q&A Tasks\n",
+    "\n",
+    "The registry provides configurations to test out common architectures on curated datasets."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "id": "b39159d0-9ea1-414f-a9d8-4a7b22b3d2cc",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from langchain_benchmarks import clone_public_dataset, registry"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "3644d211-382e-41aa-b282-21b01d28fc35",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table>\n",
+       "<thead>\n",
+       "<tr><th>Name                   </th><th>Type         </th><th>Dataset ID                                                                                                                                                 </th><th>Description  </th></tr>\n",
+       "</thead>\n",
+       "<tbody>\n",
+       "<tr><td>LangChain Docs Q&A     </td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d\" target=\"_blank\" rel=\"noopener\">452ccafc-18e1-4314-885b-edd735f17b9d</a></td><td>Questions and answers based on a snapshot of the LangChain python docs.\n",
+       "\n",
+       "The environment provides the documents and the retriever information.\n",
+       "\n",
+       "Each example is composed of a question and reference answer.\n",
+       "\n",
+       "Success is measured based on the accuracy of the answer relative to the reference answer.\n",
+       "We also measure the faithfulness of the model's response relative to the retrieved documents (if any).              </td></tr>\n",
+       "<tr><td>Semi-structured Reports</td><td>RetrievalTask</td><td><a href=\"https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d\" target=\"_blank\" rel=\"noopener\">c47d9617-ab99-4d6e-a6e6-92b8daf85a7d</a></td><td>Questions and answers based on PDFs containing tables and charts.\n",
+       "\n",
+       "The task provides the raw documents as well as factory methods to easily index them\n",
+       "and create a retriever.\n",
+       "\n",
+       "Each example is composed of a question and reference answer.\n",
+       "\n",
+       "Success is measured based on the accuracy of the answer relative to the reference answer.\n",
+       "We also measure the faithfulness of the model's response relative to the retrieved documents (if any).              </td></tr>\n",
+       "</tbody>\n",
+       "</table>"
+      ],
+      "text/plain": [
+       "Registry(tasks=[RetrievalTask(name='LangChain Docs Q&A', dataset_id='https://smith.langchain.com/public/452ccafc-18e1-4314-885b-edd735f17b9d/d', description=\"Questions and answers based on a snapshot of the LangChain python docs.\\n\\nThe environment provides the documents and the retriever information.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x122c7c4a0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x122c7c540>, 'hyde': <function _chroma_hyde_retriever_factory at 0x122c7c5e0>}, architecture_factories={'conversational-retrieval-qa': <function default_response_chain at 0x12233be20>}, get_docs=<function load_cached_docs at 0x12233b920>), RetrievalTask(name='Semi-structured Reports', dataset_id='https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d', description=\"Questions and answers based on PDFs containing tables and charts.\\n\\nThe task provides the raw documents as well as factory methods to easily index them\\nand create a retriever.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x122c7ccc0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x122c7cd60>, 'hyde': <function _chroma_hyde_retriever_factory at 0x122c7ce00>}, architecture_factories={}, get_docs=<function load_docs at 0x122c7cc20>)])"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "registry = registry.filter(Type=\"RetrievalTask\")\n",
+    "registry"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "id": "671282f8-c455-4390-b018-e53bbd833093",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<table>\n",
+       "<tbody>\n",
+       "<tr><td>Name                  </td><td>Semi-structured Reports                                                                                                                                    </td></tr>\n",
+       "<tr><td>Type                  </td><td>RetrievalTask                                                                                                                                              </td></tr>\n",
+       "<tr><td>Dataset ID            </td><td><a href=\"https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d\" target=\"_blank\" rel=\"noopener\">c47d9617-ab99-4d6e-a6e6-92b8daf85a7d</a></td></tr>\n",
+       "<tr><td>Description           </td><td>Questions and answers based on PDFs containing tables and charts.\n",
+       "\n",
+       "The task provides the raw documents as well as factory methods to easily index them\n",
+       "and create a retriever.\n",
+       "\n",
+       "Each example is composed of a question and reference answer.\n",
+       "\n",
+       "Success is measured based on the accuracy of the answer relative to the reference answer.\n",
+       "We also measure the faithfulness of the model's response relative to the retrieved documents (if any).                                                                                                                                                            </td></tr>\n",
+       "<tr><td>Retriever Factories   </td><td>basic, parent-doc, hyde                                                                                                                                    </td></tr>\n",
+       "<tr><td>Architecture Factories</td><td>                                                                                                                                                           </td></tr>\n",
+       "<tr><td>get_docs              </td><td><function load_docs at 0x122c7cc20>                                                                                                                        </td></tr>\n",
+       "</tbody>\n",
+       "</table>"
+      ],
+      "text/plain": [
+       "RetrievalTask(name='Semi-structured Reports', dataset_id='https://smith.langchain.com/public/c47d9617-ab99-4d6e-a6e6-92b8daf85a7d/d', description=\"Questions and answers based on PDFs containing tables and charts.\\n\\nThe task provides the raw documents as well as factory methods to easily index them\\nand create a retriever.\\n\\nEach example is composed of a question and reference answer.\\n\\nSuccess is measured based on the accuracy of the answer relative to the reference answer.\\nWe also measure the faithfulness of the model's response relative to the retrieved documents (if any).\\n\", retriever_factories={'basic': <function _chroma_retriever_factory at 0x122c7ccc0>, 'parent-doc': <function _chroma_parent_document_retriever_factory at 0x122c7cd60>, 'hyde': <function _chroma_hyde_retriever_factory at 0x122c7ce00>}, architecture_factories={}, get_docs=<function load_docs at 0x122c7cc20>)"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "task = registry[\"Semi-structured Reports\"]\n",
+    "task"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "id": "70369f67-deb4-467a-801a-6d38c3d0460d",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Dataset Semi-structured Reports already exists. Skipping.\n",
+      "You can access the dataset at https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/f8f24935-cf57-4cb3-a30f-8df303a46962.\n"
+     ]
+    }
+   ],
+   "source": [
+    "clone_public_dataset(task.dataset_id, dataset_name=task.name)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "4b4fafb2-63d0-40b4-b803-0095c5b22ca6",
+   "metadata": {},
+   "source": [
+    "### Now, index the documents\n",
+    "\n",
+    "You can see the raw filepaths, or use unstructured to process the pdfs."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "c9657b27-4e10-4ba5-ab20-1f05f22fdbd4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain_benchmarks.rag.tasks.semi_structured_reports import get_file_names\n",
+    "\n",
+    "# If you want to completely customize the document processing, you can use the files directly\n",
+    "file_names = list(get_file_names())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "id": "f5ad4b23-fbd4-4ebc-b5a5-d3d05efd0b9c",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of the model checkpoint at microsoft/table-transformer-structure-recognition were not used when initializing TableTransformerForObjectDetection: ['model.backbone.conv_encoder.model.layer2.0.downsample.1.num_batches_tracked', 'model.backbone.conv_encoder.model.layer3.0.downsample.1.num_batches_tracked', 'model.backbone.conv_encoder.model.layer4.0.downsample.1.num_batches_tracked']\n",
+      "- This IS expected if you are initializing TableTransformerForObjectDetection from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing TableTransformerForObjectDetection from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b8a52c9983274c21a713ac8742e9c99b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/26 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from langchain.embeddings import HuggingFaceEmbeddings\n",
+    "\n",
+    "embeddings = HuggingFaceEmbeddings(\n",
+    "    model_name=\"thenlper/gte-base\",\n",
+    "    model_kwargs={\"device\": 0},  # Comment out to use CPU\n",
+    ")\n",
+    "\n",
+    "# Arguments to pass to partition_pdf\n",
+    "unstructured_config = {\n",
+    "    # Unstructured first finds embedded image blocks\n",
+    "    \"extract_images_in_pdf\": False,\n",
+    "    # Use layout model (YOLOX) to get bounding boxes (for tables) and find titles\n",
+    "    # Titles are any sub-section of the document\n",
+    "    \"infer_table_structure\": True,\n",
+    "    # Post processing to aggregate text once we have the title\n",
+    "    \"chunking_strategy\": \"by_title\",\n",
+    "    # Chunking params to aggregate text blocks\n",
+    "    # Attempt to create a new chunk 3800 chars\n",
+    "    # Attempt to keep chunks > 2000 chars\n",
+    "    \"max_characters\": 4000,\n",
+    "    \"new_after_n_chars\": 3800,\n",
+    "    \"combine_text_under_n_chars\": 2000,\n",
+    "}\n",
+    "docs = list(task.get_docs(unstructured_config=unstructured_config))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 42,
+   "id": "fe0a05ad-5b57-40b0-aac4-e2d9cd9e6b4b",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Chroma/semi-structured-earnings-b_Chroma_HuggingFaceEmbeddings_raw\n",
+      "[]\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "06a11e1a4d50416596d9dd953fdabafa",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/26 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "retriever_factory = task.retriever_factories[\"basic\"]\n",
+    "# Indexes the documents with the specified embeddings\n",
+    "retriever = retriever_factory(embeddings, docs=docs)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "57efac89-12f9-47e3-b60f-65d9279ebc1e",
+   "metadata": {},
+   "source": [
+    "### Time to evaluate\n",
+    "\n",
+    "We will compose our retriever with a simple Llama based LLM."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "id": "8d1bc360-d822-43a8-b6b7-ff66dc27caf4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langchain.chat_models import ChatAnthropic\n",
+    "from langchain.prompts import ChatPromptTemplate\n",
+    "from langchain.schema.output_parser import StrOutputParser\n",
+    "from langchain.schema.runnable.passthrough import RunnableAssign\n",
+    "\n",
+    "\n",
+    "def create_chain(retriever):\n",
+    "    prompt = ChatPromptTemplate.from_messages(\n",
+    "        [\n",
+    "            (\n",
+    "                \"system\",\n",
+    "                \"Answer based solely on the retrieved documents below:\\n\\n<Documents>\\n{docs}</Documents>\",\n",
+    "            ),\n",
+    "            (\"user\", \"{question}\"),\n",
+    "        ]\n",
+    "    )\n",
+    "    llm = ChatAnthropic(model=\"claude-2\")\n",
+    "    return (\n",
+    "        RunnableAssign({\"docs\": (lambda x: next(iter(x.values()))) | retriever})\n",
+    "        | prompt\n",
+    "        | llm\n",
+    "        | StrOutputParser()\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "id": "935cacd9-e841-4c76-ac16-f3f0cf18df62",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "View the evaluation results for project 'cold-attachment-88' at:\n",
+      "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/projects/p/d8e512b7-b63d-4eb5-8d73-d95f7fa7ffc2?eval=true\n",
+      "\n",
+      "View all tests for Dataset Semi-structured Reports at:\n",
+      "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/f8f24935-cf57-4cb3-a30f-8df303a46962\n",
+      "[------------------------------------------------->] 5/5\n",
+      " Eval quantiles:\n",
+      "                                          inputs.question  \\\n",
+      "count                                                   5   \n",
+      "unique                                                  5   \n",
+      "top     Analyzing the operating expenses for Q3 2023, ...   \n",
+      "freq                                                    1   \n",
+      "mean                                                  NaN   \n",
+      "std                                                   NaN   \n",
+      "min                                                   NaN   \n",
+      "25%                                                   NaN   \n",
+      "50%                                                   NaN   \n",
+      "75%                                                   NaN   \n",
+      "max                                                   NaN   \n",
+      "\n",
+      "        feedback.embedding_cosine_distance  feedback.faithfulness  \\\n",
+      "count                             5.000000                    5.0   \n",
+      "unique                                 NaN                    NaN   \n",
+      "top                                    NaN                    NaN   \n",
+      "freq                                   NaN                    NaN   \n",
+      "mean                              0.137066                    1.0   \n",
+      "std                               0.011379                    0.0   \n",
+      "min                               0.123112                    1.0   \n",
+      "25%                               0.129089                    1.0   \n",
+      "50%                               0.137871                    1.0   \n",
+      "75%                               0.143398                    1.0   \n",
+      "max                               0.151860                    1.0   \n",
+      "\n",
+      "        feedback.score_string:accuracy error  execution_time  \n",
+      "count                              5.0     0        5.000000  \n",
+      "unique                             NaN     0             NaN  \n",
+      "top                                NaN   NaN             NaN  \n",
+      "freq                               NaN   NaN             NaN  \n",
+      "mean                               0.1   NaN        7.940625  \n",
+      "std                                0.0   NaN        1.380190  \n",
+      "min                                0.1   NaN        6.416387  \n",
+      "25%                                0.1   NaN        7.272528  \n",
+      "50%                                0.1   NaN        7.324673  \n",
+      "75%                                0.1   NaN        8.831243  \n",
+      "max                                0.1   NaN        9.858293  \n"
+     ]
+    }
+   ],
+   "source": [
+    "from functools import partial\n",
+    "\n",
+    "from langsmith.client import Client\n",
+    "\n",
+    "from langchain_benchmarks.rag import get_eval_config\n",
+    "\n",
+    "client = Client()\n",
+    "RAG_EVALUATION = get_eval_config()\n",
+    "chain = create_chain(retriever)\n",
+    "test_run = client.run_on_dataset(\n",
+    "    dataset_name=task.name,\n",
+    "    llm_or_chain_factory=chain,\n",
+    "    evaluation=RAG_EVALUATION,\n",
+    "    verbose=True,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3b54ef6c-0194-410a-aae9-f30c2097548a",
+   "metadata": {},
+   "source": [
+    "## Example processing the docs\n",
+    "\n",
+    "RAG apps are as good as the information they are able to retrieve. Let's try indexing the tables' summaries to\n",
+    "improve the likelihood that they are retrieved whenever a user asks a relevant question.\n",
+    "\n",
+    "We will use unstructured's `partition_pdf` functionality and generate summaries using an LLM.\n",
+    "\n",
+    "You can define your own indexing pipeline to see how it impacts the downstream performance."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "3378eddb-0a8d-4179-8e9c-54343469eef6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from operator import itemgetter\n",
+    "\n",
+    "from langchain.chat_models import ChatOpenAI\n",
+    "from langchain.prompts import ChatPromptTemplate\n",
+    "from langchain.schema.document import Document\n",
+    "from langchain.schema.output_parser import StrOutputParser\n",
+    "from langchain.schema.runnable.passthrough import RunnableAssign\n",
+    "\n",
+    "# Prompt\n",
+    "prompt = ChatPromptTemplate.from_messages(\n",
+    "    [\n",
+    "        (\n",
+    "            \"system\",\n",
+    "            \"You are summarizing semi-structured tables or text in a pdf.\\n\\n```document\\n{doc}\\n```\",\n",
+    "        ),\n",
+    "        (\"user\", \"Write a concise summary.\"),\n",
+    "    ]\n",
+    ")\n",
+    "\n",
+    "# Summary chain\n",
+    "model = ChatOpenAI(temperature=0, model=\"gpt-3.5-turbo-16k\")\n",
+    "\n",
+    "\n",
+    "def create_doc(x) -> Document:\n",
+    "    return Document(\n",
+    "        page_content=x[\"output\"],\n",
+    "        metadata=x[\"doc\"].metadata,\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "summarize_chain = (\n",
+    "    {\"doc\": lambda x: x}\n",
+    "    | RunnableAssign({\"prompt\": prompt})\n",
+    "    | {\n",
+    "        \"output\": itemgetter(\"prompt\") | model | StrOutputParser(),\n",
+    "        \"doc\": itemgetter(\"doc\"),\n",
+    "    }\n",
+    "    | create_doc\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "07a2f070-3b5a-4de0-b3da-ddfb6e6f8c2b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "summaries = summarize_chain.batch(\n",
+    "    [doc for doc in docs if doc.metadata[\"element_type\"] == \"table\"]\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "22dc0bf8-fa50-4be3-8d23-04f6129548e0",
+   "metadata": {},
+   "source": [
+    "Index the documents and create the retriever. We will re"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 44,
+   "id": "35a1ccf6-2c2f-46f2-838e-5a5bf89515f5",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "029921ed3d7c4f389c666583a7192144",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/36 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Indexes the documents with the specified embeddings\n",
+    "retriever_with_summaries = retriever_factory(\n",
+    "    embeddings,\n",
+    "    docs=docs + summaries,\n",
+    "    # Specify a unique transformation name to avoid local cache collisions with other indices.\n",
+    "    transformation_name=\"docs-with_summaries\",\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3821e4b0-8e67-418a-840c-470fcde42df0",
+   "metadata": {},
+   "source": [
+    "### Evaluate\n",
+    "\n",
+    "We'll evaluate the new chain on the same dataset."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 45,
+   "id": "e6f08c4c-a738-4449-9190-5a4f0b65b99a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "View the evaluation results for project 'crazy-harmony-39' at:\n",
+      "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/projects/p/b69d796f-6ba4-4cde-822f-db363cf81f8f?eval=true\n",
+      "\n",
+      "View all tests for Dataset Semi-structured Reports at:\n",
+      "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/f8f24935-cf57-4cb3-a30f-8df303a46962\n",
+      "[------------------------------------------------->] 5/5\n",
+      " Eval quantiles:\n",
+      "                                          inputs.question  \\\n",
+      "count                                                   5   \n",
+      "unique                                                  5   \n",
+      "top     Analyzing the operating expenses for Q3 2023, ...   \n",
+      "freq                                                    1   \n",
+      "mean                                                  NaN   \n",
+      "std                                                   NaN   \n",
+      "min                                                   NaN   \n",
+      "25%                                                   NaN   \n",
+      "50%                                                   NaN   \n",
+      "75%                                                   NaN   \n",
+      "max                                                   NaN   \n",
+      "\n",
+      "        feedback.score_string:accuracy  feedback.faithfulness  \\\n",
+      "count                         5.000000                    5.0   \n",
+      "unique                             NaN                    NaN   \n",
+      "top                                NaN                    NaN   \n",
+      "freq                               NaN                    NaN   \n",
+      "mean                          0.720000                    1.0   \n",
+      "std                           0.408656                    0.0   \n",
+      "min                           0.100000                    1.0   \n",
+      "25%                           0.500000                    1.0   \n",
+      "50%                           1.000000                    1.0   \n",
+      "75%                           1.000000                    1.0   \n",
+      "max                           1.000000                    1.0   \n",
+      "\n",
+      "        feedback.embedding_cosine_distance error  execution_time  \n",
+      "count                             5.000000     0        5.000000  \n",
+      "unique                                 NaN     0             NaN  \n",
+      "top                                    NaN   NaN             NaN  \n",
+      "freq                                   NaN   NaN             NaN  \n",
+      "mean                              0.069363   NaN        8.659120  \n",
+      "std                               0.023270   NaN        2.611724  \n",
+      "min                               0.039593   NaN        6.283505  \n",
+      "25%                               0.050176   NaN        6.723136  \n",
+      "50%                               0.078912   NaN        7.441743  \n",
+      "75%                               0.084389   NaN       10.673265  \n",
+      "max                               0.093747   NaN       12.173952  \n"
+     ]
+    }
+   ],
+   "source": [
+    "chain_2 = create_chain(retriever_with_summaries)\n",
+    "\n",
+    "test_run_with_summaries = client.run_on_dataset(\n",
+    "    dataset_name=task.name,\n",
+    "    llm_or_chain_factory=chain_2,\n",
+    "    evaluation=RAG_EVALUATION,\n",
+    "    verbose=True,\n",
+    ")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/source/notebooks/tool_usage/intro.ipynb

@@ -0,0 +1,176 @@
+{
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "id": "6728b05f-e3bb-487a-8818-e0d5d18b5501",
+            "metadata": {},
+            "source": [
+                "# Agent Tool Usage Tasks\n",
+                "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain-benchmarks/blob/main/docs/source/notebooks/tool_usage/intro.ipynb)\n",
+                "\n",
+                "These tasks are meant to grade your agent's effectiveness at using tools to accomplish tasks.\n",
+                "\n",
+                "You can check an up-to-date list of tool usage tasks in the registry:"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 3,
+            "id": "a57e65d7-dbd6-4128-8260-f6ee3b43157c",
+            "metadata": {},
+            "outputs": [
+                {
+                    "data": {
+                        "text/html": [
+                            "<table>\n",
+                            "<thead>\n",
+                            "<tr><th>Name                              </th><th>Type         </th><th>Dataset ID                                                                                                                                                 </th><th>Description  </th></tr>\n",
+                            "</thead>\n",
+                            "<tbody>\n",
+                            "<tr><td>Tool Usage - Typewriter (1 tool)  </td><td>ToolUsageTask</td><td><a href=\"https://smith.langchain.com/public/59577193-8938-4ccf-92a7-e8a96bcf4f86/d\" target=\"_blank\" rel=\"noopener\">59577193-8938-4ccf-92a7-e8a96bcf4f86</a></td><td>Environment with a single tool that accepts a single letter as input, and prints it on a piece of virtual paper.\n",
+                            "\n",
+                            "The objective of this task is to evaluate the ability of the model to use the provided tools to repeat a given input string.\n",
+                            "\n",
+                            "For example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\n",
+                            "\n",
+                            "The dataset includes examples of varying difficulty. The difficulty is measured by the length of the string.              </td></tr>\n",
+                            "<tr><td>Tool Usage - Typewriter (26 tools)</td><td>ToolUsageTask</td><td><a href=\"https://smith.langchain.com/public/128af05e-aa00-4e3b-a958-d166dd450581/d\" target=\"_blank\" rel=\"noopener\">128af05e-aa00-4e3b-a958-d166dd450581</a></td><td>Environment with 26 tools each tool represents a letter of the alphabet.\n",
+                            "\n",
+                            "The objective of this task is to evaluate the model's ability the use tools\n",
+                            "for a simple repetition task.\n",
+                            "\n",
+                            "For example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\n",
+                            "\n",
+                            "The dataset includes examples of varying difficulty. The difficulty is measured by the length of the string.\n",
+                            "\n",
+                            "This is a variation of the typer writer task, where 26 parameterless tools are\n",
+                            "given instead of a single tool that takes a letter as an argument.              </td></tr>\n",
+                            "<tr><td>Tool Usage - Relational Data      </td><td>ToolUsageTask</td><td><a href=\"https://smith.langchain.com/public/1d89f4b3-5f73-48cf-a127-2fdeb22f6d84/d\" target=\"_blank\" rel=\"noopener\">1d89f4b3-5f73-48cf-a127-2fdeb22f6d84</a></td><td>Environment with fake data about users and their locations and favorite foods.\n",
+                            "\n",
+                            "The environment provides a set of tools that can be used to query the data.\n",
+                            "\n",
+                            "The objective of this task is to evaluate the ability to use the provided tools to answer questions about relational data.\n",
+                            "\n",
+                            "The dataset contains 21 examples of varying difficulty. The difficulty is measured by the number of tools that need to be used to answer the question.\n",
+                            "\n",
+                            "Each example is composed of a question, a reference answer, and information about the sequence in which tools should be used to answer the question.\n",
+                            "\n",
+                            "Success is measured by the ability to answer the question correctly, and efficiently.              </td></tr>\n",
+                            "<tr><td>Multiverse Math                   </td><td>ToolUsageTask</td><td><a href=\"https://smith.langchain.com/public/594f9f60-30a0-49bf-b075-f44beabf546a/d\" target=\"_blank\" rel=\"noopener\">594f9f60-30a0-49bf-b075-f44beabf546a</a></td><td>An environment that contains a few basic math operations, but with altered results.\n",
+                            "\n",
+                            "For example, multiplication of 5*3 will be re-interpreted as 5*3*1.1. The basic operations retain some basic properties, such as commutativity, associativity, and distributivity; however, the results are different than expected.\n",
+                            "\n",
+                            "The objective of this task is to evaluate the ability to use the provided tools to solve simple math questions and ignore any innate knowledge about math.              </td></tr>\n",
+                            "</tbody>\n",
+                            "</table>"
+                        ],
+                        "text/plain": [
+                            "Registry(tasks=[ToolUsageTask(name='Tool Usage - Typewriter (1 tool)', dataset_id='https://smith.langchain.com/public/59577193-8938-4ccf-92a7-e8a96bcf4f86/d', description=\"Environment with a single tool that accepts a single letter as input, and prints it on a piece of virtual paper.\\n\\nThe objective of this task is to evaluate the ability of the model to use the provided tools to repeat a given input string.\\n\\nFor example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\\n\\nThe dataset includes examples of varying difficulty. The difficulty is measured by the length of the string.\\n\", create_environment=<function get_environment at 0x132f9cea0>, instructions=\"Repeat the given string using the provided tools. Do not write anything else or provide any explanations. For example, if the string is 'abc', you must print the letters 'a', 'b', and 'c' one at a time and in that order. \"), ToolUsageTask(name='Tool Usage - Typewriter (26 tools)', dataset_id='https://smith.langchain.com/public/128af05e-aa00-4e3b-a958-d166dd450581/d', description=\"Environment with 26 tools each tool represents a letter of the alphabet.\\n\\nThe objective of this task is to evaluate the model's ability the use tools\\nfor a simple repetition task.\\n\\nFor example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\\n\\nThe dataset includes examples of varying difficulty. The difficulty is measured by the length of the string.\\n\\nThis is a variation of the typer writer task, where 26 parameterless tools are\\ngiven instead of a single tool that takes a letter as an argument.\\n\", create_environment=<function get_environment at 0x132f9d3a0>, instructions=\"Repeat the given string by using the provided tools. Do not write anything else or provide any explanations. For example, if the string is 'abc', you must invoke the tools 'a', 'b', and 'c' in that order. Please invoke the functions without any arguments.\"), ToolUsageTask(name='Tool Usage - Relational Data', dataset_id='https://smith.langchain.com/public/1d89f4b3-5f73-48cf-a127-2fdeb22f6d84/d', description='Environment with fake data about users and their locations and favorite foods.\\n\\nThe environment provides a set of tools that can be used to query the data.\\n\\nThe objective of this task is to evaluate the ability to use the provided tools to answer questions about relational data.\\n\\nThe dataset contains 21 examples of varying difficulty. The difficulty is measured by the number of tools that need to be used to answer the question.\\n\\nEach example is composed of a question, a reference answer, and information about the sequence in which tools should be used to answer the question.\\n\\nSuccess is measured by the ability to answer the question correctly, and efficiently.\\n', create_environment=<function get_environment at 0x132f9c9a0>, instructions=\"Please answer the user's question by using the tools provided. Do not guess the answer. Keep in mind that entities like users,foods and locations have both a name and an ID, which are not the same.\"), ToolUsageTask(name='Multiverse Math', dataset_id='https://smith.langchain.com/public/594f9f60-30a0-49bf-b075-f44beabf546a/d', description='An environment that contains a few basic math operations, but with altered results.\\n\\nFor example, multiplication of 5*3 will be re-interpreted as 5*3*1.1. The basic operations retain some basic properties, such as commutativity, associativity, and distributivity; however, the results are different than expected.\\n\\nThe objective of this task is to evaluate the ability to use the provided tools to solve simple math questions and ignore any innate knowledge about math.\\n', create_environment=<function get_environment at 0x132f9c2c0>, instructions='You are requested to solve math questions in an alternate mathematical universe. The operations have been altered to yield different results than expected. Do not guess the answer or rely on your  innate knowledge of math. Use the provided tools to answer the question. While associativity and commutativity apply, distributivity does not. Answer the question using the fewest possible tools. Only include the numeric response without any clarifications.')])"
+                        ]
+                    },
+                    "execution_count": 3,
+                    "metadata": {},
+                    "output_type": "execute_result"
+                }
+            ],
+            "source": [
+                "from langchain_benchmarks import registry\n",
+                "\n",
+                "registry.filter(Type=\"ToolUsageTask\")"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "9f54cdd3-67f6-43ba-a929-1a6ed1b01296",
+            "metadata": {},
+            "source": [
+                "### Task resources\n",
+                "\n",
+                "In addition to a name, daset_id, and description, the `tool_use` directory also has a shared agent factory you can use to get started:"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "id": "3363e86d-3c86-4297-81b6-f73899be48b0",
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "from langchain_benchmarks.tool_usage import agents\n",
+                "\n",
+                "agent_factory = agents.OpenAIAgentFactory(task, model=\"gpt-3.5-turbo-16k\")"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "994bb145-9b12-4a60-87da-003f44dd13e5",
+            "metadata": {},
+            "source": [
+                "They also define a `create_environment` method that returns a ToolUsageEnvironment object:\n",
+                "\n",
+                "```python\n",
+                "class ToolUsageEnvironment:\n",
+                "    \"\"\"An instance of an environment for tool usage.\"\"\"\n",
+                "\n",
+                "    tools: List[BaseTool]\n",
+                "    \"\"\"The tools that can be used in the environment.\"\"\"\n",
+                "\n",
+                "    read_state: Optional[Callable[[], Any]] = None\n",
+                "    \"\"\"A function that returns the current state of the environment.\"\"\"\n",
+                "```\n",
+                "\n",
+                "This is used to define the available tools for a given dataset and to let any evaluators read the world state grade the agent."
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "3d5e48c4-d5d0-4d19-9bab-61c01a512f21",
+            "metadata": {},
+            "source": [
+                "### Dataset schema\n",
+                "\n",
+                "Each task corresponds to a LangSmith dataset with the following schema:\n",
+                "\n",
+                "Inputs:\n",
+                "- `question: str` - the user question\n",
+                "\n",
+                "Outputs\n",
+                "- `expected_steps: list` - the expected order of tools used\n",
+                "- `reference: str` - the expected answer\n",
+                "\n",
+                "There may be additional output keys, such as:\n",
+                "- `order_matters`: bool - whether the order of tool invocations matters\n",
+                "- `state: any` - the end 'state' the environment should be in after a given data point"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "id": "cd9dea3f-68b5-47f7-9a12-c7b5eafc4a37",
+            "metadata": {},
+            "outputs": [],
+            "source": []
+        }
+    ],
+    "metadata": {
+        "kernelspec": {
+            "display_name": "Python 3 (ipykernel)",
+            "language": "python",
+            "name": "python3"
+        },
+        "language_info": {
+            "codemirror_mode": {
+                "name": "ipython",
+                "version": 3
+            },
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.11.2"
+        }
+    },
+    "nbformat": 4,
+    "nbformat_minor": 5
+}

docs/source/notebooks/tool_usage/multiverse_math.ipynb

@@ -0,0 +1,481 @@
+{
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "id": "60bb467d-861d-4b07-a48d-8e5aa177c969",
+            "metadata": {
+                "tags": []
+            },
+            "source": [
+                "# Multiverse Math\n",
+                "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain-benchmarks/blob/main/docs/source/notebooks/tool_usage/multiverse_math.ipynb)\n",
+                "\n",
+                "Let's see how to evaluate an agent's ability to use tools.\n",
+                "\n",
+                "    Solve basic math question using the provided tools.\n",
+                "\n",
+                "    Must use the provided tools to solve the math question.\n",
+                "\n",
+                "    To make sure that innate knowledge is not used, the math operations have been altered to yield different results than expected.\n",
+                "\n",
+                "    The modified operations should yield different results, but still retain appropriate properties. For example, the modified multiplication operation should still be commutative.\n",
+                "\n",
+                "    Please note that the modified operations are not guaranteed to even make sense in the real world since not all properties will be retained (e.g., distributive property)."
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "03488ab1-31ed-41c2-8da2-46b02599b181",
+            "metadata": {},
+            "source": [
+                "For this code to work, please configure LangSmith environment variables with your credentials."
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 1,
+            "id": "1615b8ff-688a-4447-8c4c-d64ad02818ed",
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "import os\n",
+                "\n",
+                "os.environ[\"LANGCHAIN_API_KEY\"] = \"sk-...\"  # Your LangSmith API key"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 18,
+            "id": "b39159d0-9ea1-414f-a9d8-4a7b22b3d2cc",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [],
+            "source": [
+                "from langchain_benchmarks import clone_public_dataset, registry"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 3,
+            "id": "1aef2b32-a5df-421f-8be3-a2ef27372ece",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "data": {
+                        "text/html": [
+                            "<table>\n",
+                            "<tbody>\n",
+                            "<tr><td>Name       </td><td>Multiverse Math                                                          </td></tr>\n",
+                            "<tr><td>Type       </td><td>ToolUsageTask                                                            </td></tr>\n",
+                            "<tr><td>Dataset ID </td><td>https://smith.langchain.com/public/594f9f60-30a0-49bf-b075-f44beabf546a/d</td></tr>\n",
+                            "<tr><td>Description</td><td>An environment that contains a few basic math operations, but with altered results.\n",
+                            "\n",
+                            "For example, mu...                                                                          </td></tr>\n",
+                            "</tbody>\n",
+                            "</table>"
+                        ],
+                        "text/plain": [
+                            "ToolUsageTask(name='Multiverse Math', dataset_id='https://smith.langchain.com/public/594f9f60-30a0-49bf-b075-f44beabf546a/d', description='An environment that contains a few basic math operations, but with altered results.\\n\\nFor example, multiplication of 5*3 will be re-interpreted as 5*3*1.1. The basic operations retain some basic properties, such as commutativity, associativity, and distributivity; however, the results are different than expected.\\n\\nThe objective of this task is to evaluate the ability to use the provided tools to solve simple math questions and ignore any innate knowledge about math.\\n', create_environment=<function get_environment at 0x7fae28d9f310>, instructions='You are requested to solve math questions in an alternate mathematical universe. The operations have been altered to yield different results than expected. Do not guess the answer or rely on your  innate knowledge of math. Use the provided tools to answer the question. While associativity and commutativity apply, distributivity does not. Answer the question using the fewest possible tools. Only include the numeric response without any clarifications.')"
+                        ]
+                    },
+                    "execution_count": 3,
+                    "metadata": {},
+                    "output_type": "execute_result"
+                }
+            ],
+            "source": [
+                "task = registry[\"Multiverse Math\"]\n",
+                "task"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "bc33a639-3caf-4314-8ea7-1c7c8b1d114d",
+            "metadata": {},
+            "source": [
+                "Clone the dataset associaetd with this task"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 6,
+            "id": "70369f67-deb4-467a-801a-6d38c3d0460d",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "Dataset Multiverse Math already exists. Skipping.\n",
+                        "You can access the dataset at https://smith.langchain.com/o/e081f11e-fbd2-41b4-9fa8-5d76c76ef854/datasets/ddca73f1-ceda-4562-8c49-7ee0a9df2a01.\n"
+                    ]
+                }
+            ],
+            "source": [
+                "clone_public_dataset(task.dataset_id, dataset_name=task.name)"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "b462f7b8-fd42-4613-ab5f-5f3cbbc37d28",
+            "metadata": {},
+            "source": [
+                "Let's build an agent that we can use for evaluation."
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 10,
+            "id": "6142cf4e-862c-47a3-aa75-81d7d3231308",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "data": {
+                        "text/plain": [
+                            "{'question': 'how much is 3 + 5',\n",
+                            " 'output': 'In this alternate mathematical universe, the result of adding 3 and 5 is 9.2.',\n",
+                            " 'intermediate_steps': [(AgentActionMessageLog(tool='add', tool_input={'a': 3, 'b': 5}, log=\"\\nInvoking: `add` with `{'a': 3, 'b': 5}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'arguments': '{\\n  \"a\": 3,\\n  \"b\": 5\\n}', 'name': 'add'}})]),\n",
+                            "   9.2)]}"
+                        ]
+                    },
+                    "execution_count": 10,
+                    "metadata": {},
+                    "output_type": "execute_result"
+                }
+            ],
+            "source": [
+                "from langchain_benchmarks.tool_usage import agents\n",
+                "\n",
+                "agent_factory = agents.OpenAIAgentFactory(task, model=\"gpt-3.5-turbo-16k\")\n",
+                "\n",
+                "# Let's test that our agent works\n",
+                "agent = agent_factory.create()\n",
+                "agent.invoke({\"question\": \"how much is 3 + 5\"})"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "3821e4b0-8e67-418a-840c-470fcde42df0",
+            "metadata": {},
+            "source": [
+                "## Eval\n",
+                "\n",
+                "Let's evaluate an agent now"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 13,
+            "id": "fb32763c-79ab-426a-8fc6-bf8ebb0dd432",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "View the evaluation results for project 'test-excellent-potato-37' at:\n",
+                        "https://smith.langchain.com/o/e081f11e-fbd2-41b4-9fa8-5d76c76ef854/projects/p/e350cda0-4e1d-49eb-8483-574172d1c635?eval=true\n",
+                        "\n",
+                        "View all tests for Dataset Multiverse Math at:\n",
+                        "https://smith.langchain.com/o/e081f11e-fbd2-41b4-9fa8-5d76c76ef854/datasets/ddca73f1-ceda-4562-8c49-7ee0a9df2a01\n",
+                        "[------------------------------------------------->] 10/10\n",
+                        " Eval quantiles:\n",
+                        "                                    0.25       0.5      0.75      mean  \\\n",
+                        "Intermediate steps correctness   0.00000   0.00000   0.00000   0.10000   \n",
+                        "# steps / # expected steps       5.00000   7.50000   8.62500   7.75000   \n",
+                        "correctness                      0.00000   0.00000   0.00000   0.10000   \n",
+                        "execution_time                  38.76436  38.76436  38.76436  38.76436   \n",
+                        "\n",
+                        "                                    mode  \n",
+                        "Intermediate steps correctness   0.00000  \n",
+                        "# steps / # expected steps       5.00000  \n",
+                        "correctness                      0.00000  \n",
+                        "execution_time                  38.76436  \n"
+                    ]
+                }
+            ],
+            "source": [
+                "from langsmith.client import Client\n",
+                "\n",
+                "from langchain_benchmarks.tool_usage import STANDARD_AGENT_EVALUATOR\n",
+                "\n",
+                "client = Client()\n",
+                "\n",
+                "test_run = client.run_on_dataset(\n",
+                "    dataset_name=task.name,\n",
+                "    llm_or_chain_factory=agent_factory.create,\n",
+                "    evaluation=STANDARD_AGENT_EVALUATOR,\n",
+                "    verbose=True,\n",
+                "    tags=[\"gpt-3.5-turbo-16k\"],\n",
+                ")"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "1b039225-01cf-481a-87a6-4e880e9b1dcd",
+            "metadata": {},
+            "source": [
+                "# Inspect\n",
+                "\n",
+                "You can take a look at the underlying results."
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 14,
+            "id": "6eb19db1-43b8-4866-a3d2-f211ba92ab8b",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [],
+            "source": [
+                "import pandas as pd\n",
+                "\n",
+                "df = test_run.to_dataframe()\n",
+                "df = pd.json_normalize(df.to_dict(orient=\"records\"))"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 15,
+            "id": "7ab5a8b9-a937-4537-b879-704284df4494",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "data": {
+                        "text/plain": [
+                            "0.1"
+                        ]
+                    },
+                    "execution_count": 15,
+                    "metadata": {},
+                    "output_type": "execute_result"
+                }
+            ],
+            "source": [
+                "df[\"correctness\"].mean()"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 16,
+            "id": "ab7516ed-36b1-4c16-bf4a-cc49077460ad",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [],
+            "source": [
+                "df[\"num_expected_steps\"] = df[\"reference.expected_steps\"].apply(len)\n",
+                "df[\"actual_number_of_steps\"] = df[\"output.intermediate_steps\"].apply(len)"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 17,
+            "id": "50d7590d-20de-4768-ac90-adcdbfa70068",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "data": {
+                        "text/html": [
+                            "<div>\n",
+                            "<style scoped>\n",
+                            "    .dataframe tbody tr th:only-of-type {\n",
+                            "        vertical-align: middle;\n",
+                            "    }\n",
+                            "\n",
+                            "    .dataframe tbody tr th {\n",
+                            "        vertical-align: top;\n",
+                            "    }\n",
+                            "\n",
+                            "    .dataframe thead th {\n",
+                            "        text-align: right;\n",
+                            "    }\n",
+                            "</style>\n",
+                            "<table border=\"1\" class=\"dataframe\">\n",
+                            "  <thead>\n",
+                            "    <tr style=\"text-align: right;\">\n",
+                            "      <th></th>\n",
+                            "      <th>Intermediate steps correctness</th>\n",
+                            "      <th># steps / # expected steps</th>\n",
+                            "      <th>correctness</th>\n",
+                            "      <th>execution_time</th>\n",
+                            "      <th>input.question</th>\n",
+                            "      <th>output.question</th>\n",
+                            "      <th>output.output</th>\n",
+                            "      <th>output.intermediate_steps</th>\n",
+                            "      <th>reference.reference</th>\n",
+                            "      <th>reference.expected_steps</th>\n",
+                            "      <th>num_expected_steps</th>\n",
+                            "      <th>actual_number_of_steps</th>\n",
+                            "    </tr>\n",
+                            "  </thead>\n",
+                            "  <tbody>\n",
+                            "    <tr>\n",
+                            "      <th>0</th>\n",
+                            "      <td>0</td>\n",
+                            "      <td>15.0</td>\n",
+                            "      <td>0</td>\n",
+                            "      <td>38.76436</td>\n",
+                            "      <td>Add 2 and 3</td>\n",
+                            "      <td>Add 2 and 3</td>\n",
+                            "      <td>Agent stopped due to iteration limit or time l...</td>\n",
+                            "      <td>[(tool='add' tool_input={'a': 2, 'b': 3} log=\"...</td>\n",
+                            "      <td>6.20</td>\n",
+                            "      <td>[add]</td>\n",
+                            "      <td>1</td>\n",
+                            "      <td>15</td>\n",
+                            "    </tr>\n",
+                            "    <tr>\n",
+                            "      <th>1</th>\n",
+                            "      <td>0</td>\n",
+                            "      <td>15.0</td>\n",
+                            "      <td>0</td>\n",
+                            "      <td>38.76436</td>\n",
+                            "      <td>Subtract 3 from 2</td>\n",
+                            "      <td>Subtract 3 from 2</td>\n",
+                            "      <td>Agent stopped due to iteration limit or time l...</td>\n",
+                            "      <td>[(tool='subtract' tool_input={'a': 2, 'b': 3} ...</td>\n",
+                            "      <td>-4.00</td>\n",
+                            "      <td>[subtract]</td>\n",
+                            "      <td>1</td>\n",
+                            "      <td>15</td>\n",
+                            "    </tr>\n",
+                            "    <tr>\n",
+                            "      <th>2</th>\n",
+                            "      <td>0</td>\n",
+                            "      <td>9.0</td>\n",
+                            "      <td>1</td>\n",
+                            "      <td>38.76436</td>\n",
+                            "      <td>What is -5 if evaluated using the negate funct...</td>\n",
+                            "      <td>What is -5 if evaluated using the negate funct...</td>\n",
+                            "      <td>-5.0\\n-5.0</td>\n",
+                            "      <td>[(tool='negate' tool_input={'a': -5} log=\"\\nIn...</td>\n",
+                            "      <td>-5.00</td>\n",
+                            "      <td>[negate]</td>\n",
+                            "      <td>1</td>\n",
+                            "      <td>9</td>\n",
+                            "    </tr>\n",
+                            "    <tr>\n",
+                            "      <th>3</th>\n",
+                            "      <td>1</td>\n",
+                            "      <td>1.0</td>\n",
+                            "      <td>0</td>\n",
+                            "      <td>38.76436</td>\n",
+                            "      <td>what is the result of 2 to the power of 3?</td>\n",
+                            "      <td>what is the result of 2 to the power of 3?</td>\n",
+                            "      <td>The result of 2 to the power of 3 is 32.</td>\n",
+                            "      <td>[(tool='power' tool_input={'a': 2, 'b': 3} log...</td>\n",
+                            "      <td>32.00</td>\n",
+                            "      <td>[power]</td>\n",
+                            "      <td>1</td>\n",
+                            "      <td>1</td>\n",
+                            "    </tr>\n",
+                            "    <tr>\n",
+                            "      <th>4</th>\n",
+                            "      <td>0</td>\n",
+                            "      <td>7.5</td>\n",
+                            "      <td>0</td>\n",
+                            "      <td>38.76436</td>\n",
+                            "      <td>I ate 1 apple and 2 oranges every day for 7 da...</td>\n",
+                            "      <td>I ate 1 apple and 2 oranges every day for 7 da...</td>\n",
+                            "      <td>Agent stopped due to iteration limit or time l...</td>\n",
+                            "      <td>[(tool='add' tool_input={'a': 1, 'b': 2} log=\"...</td>\n",
+                            "      <td>32.34</td>\n",
+                            "      <td>[multiply, add]</td>\n",
+                            "      <td>2</td>\n",
+                            "      <td>15</td>\n",
+                            "    </tr>\n",
+                            "  </tbody>\n",
+                            "</table>\n",
+                            "</div>"
+                        ],
+                        "text/plain": [
+                            "   Intermediate steps correctness  # steps / # expected steps  correctness  \\\n",
+                            "0                               0                        15.0            0   \n",
+                            "1                               0                        15.0            0   \n",
+                            "2                               0                         9.0            1   \n",
+                            "3                               1                         1.0            0   \n",
+                            "4                               0                         7.5            0   \n",
+                            "\n",
+                            "   execution_time                                     input.question  \\\n",
+                            "0        38.76436                                        Add 2 and 3   \n",
+                            "1        38.76436                                  Subtract 3 from 2   \n",
+                            "2        38.76436  What is -5 if evaluated using the negate funct...   \n",
+                            "3        38.76436         what is the result of 2 to the power of 3?   \n",
+                            "4        38.76436  I ate 1 apple and 2 oranges every day for 7 da...   \n",
+                            "\n",
+                            "                                     output.question  \\\n",
+                            "0                                        Add 2 and 3   \n",
+                            "1                                  Subtract 3 from 2   \n",
+                            "2  What is -5 if evaluated using the negate funct...   \n",
+                            "3         what is the result of 2 to the power of 3?   \n",
+                            "4  I ate 1 apple and 2 oranges every day for 7 da...   \n",
+                            "\n",
+                            "                                       output.output  \\\n",
+                            "0  Agent stopped due to iteration limit or time l...   \n",
+                            "1  Agent stopped due to iteration limit or time l...   \n",
+                            "2                                         -5.0\\n-5.0   \n",
+                            "3           The result of 2 to the power of 3 is 32.   \n",
+                            "4  Agent stopped due to iteration limit or time l...   \n",
+                            "\n",
+                            "                           output.intermediate_steps  reference.reference  \\\n",
+                            "0  [(tool='add' tool_input={'a': 2, 'b': 3} log=\"...                 6.20   \n",
+                            "1  [(tool='subtract' tool_input={'a': 2, 'b': 3} ...                -4.00   \n",
+                            "2  [(tool='negate' tool_input={'a': -5} log=\"\\nIn...                -5.00   \n",
+                            "3  [(tool='power' tool_input={'a': 2, 'b': 3} log...                32.00   \n",
+                            "4  [(tool='add' tool_input={'a': 1, 'b': 2} log=\"...                32.34   \n",
+                            "\n",
+                            "  reference.expected_steps  num_expected_steps  actual_number_of_steps  \n",
+                            "0                    [add]                   1                      15  \n",
+                            "1               [subtract]                   1                      15  \n",
+                            "2                 [negate]                   1                       9  \n",
+                            "3                  [power]                   1                       1  \n",
+                            "4          [multiply, add]                   2                      15  "
+                        ]
+                    },
+                    "execution_count": 17,
+                    "metadata": {},
+                    "output_type": "execute_result"
+                }
+            ],
+            "source": [
+                "df.head()"
+            ]
+        }
+    ],
+    "metadata": {
+        "kernelspec": {
+            "display_name": "Python 3 (ipykernel)",
+            "language": "python",
+            "name": "python3"
+        },
+        "language_info": {
+            "codemirror_mode": {
+                "name": "ipython",
+                "version": 3
+            },
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.11.2"
+        }
+    },
+    "nbformat": 4,
+    "nbformat_minor": 5
+}

docs/source/notebooks/tool_usage/typewriter_1.ipynb

@@ -0,0 +1,486 @@
+{
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "id": "60bb467d-861d-4b07-a48d-8e5aa177c969",
+            "metadata": {
+                "tags": []
+            },
+            "source": [
+                "# Typewriter: Single Tool\n",
+                "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain-benchmarks/blob/main/docs/source/notebooks/tool_usage/typewriter_1.ipynb)\n",
+                "\n",
+                "Let's see how to evaluate an agent's ability to use tools.\n",
+                "\n",
+                "    A task where the agent must type a given string one letter at a time.\n",
+                "\n",
+                "    In this variation of the task, the agent is given a single function,\n",
+                "    that takes a letter as an argument."
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 1,
+            "id": "b39159d0-9ea1-414f-a9d8-4a7b22b3d2cc",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [],
+            "source": [
+                "from langchain_benchmarks import clone_public_dataset, registry"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 2,
+            "id": "1aef2b32-a5df-421f-8be3-a2ef27372ece",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "data": {
+                        "text/html": [
+                            "<table>\n",
+                            "<tbody>\n",
+                            "<tr><td>Name       </td><td>Tool Usage - Typewriter (1 tool)                                                                                                                           </td></tr>\n",
+                            "<tr><td>Type       </td><td>ToolUsageTask                                                                                                                                              </td></tr>\n",
+                            "<tr><td>Dataset ID </td><td><a href=\"https://smith.langchain.com/public/59577193-8938-4ccf-92a7-e8a96bcf4f86/d\" target=\"_blank\" rel=\"noopener\">59577193-8938-4ccf-92a7-e8a96bcf4f86</a></td></tr>\n",
+                            "<tr><td>Description</td><td>Environment with a single tool that accepts a single letter as input, and prints it on a piece of virtual paper.\n",
+                            "\n",
+                            "The objective of this task is to evaluate the ability of the model to use the provided tools to repeat a given input string.\n",
+                            "\n",
+                            "For example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\n",
+                            "\n",
+                            "The dataset includes examples of varying difficulty. The difficulty is measured by the length of the string.                                                                                                                                                            </td></tr>\n",
+                            "</tbody>\n",
+                            "</table>"
+                        ],
+                        "text/plain": [
+                            "ToolUsageTask(name='Tool Usage - Typewriter (1 tool)', dataset_id='https://smith.langchain.com/public/59577193-8938-4ccf-92a7-e8a96bcf4f86/d', description=\"Environment with a single tool that accepts a single letter as input, and prints it on a piece of virtual paper.\\n\\nThe objective of this task is to evaluate the ability of the model to use the provided tools to repeat a given input string.\\n\\nFor example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\\n\\nThe dataset includes examples of varying difficulty. The difficulty is measured by the length of the string.\\n\", create_environment=<function get_environment at 0x12438d760>, instructions=\"Repeat the given string using the provided tools. Do not write anything else or provide any explanations. For example, if the string is 'abc', you must print the letters 'a', 'b', and 'c' one at a time and in that order. \")"
+                        ]
+                    },
+                    "execution_count": 2,
+                    "metadata": {},
+                    "output_type": "execute_result"
+                }
+            ],
+            "source": [
+                "task = registry[\"Tool Usage - Typewriter (1 tool)\"]\n",
+                "task"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "bc33a639-3caf-4314-8ea7-1c7c8b1d114d",
+            "metadata": {},
+            "source": [
+                "Clone the dataset associaetd with this task"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 3,
+            "id": "70369f67-deb4-467a-801a-6d38c3d0460d",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "Dataset Tool Usage - Typewriter (1 tool) already exists. Skipping.\n",
+                        "You can access the dataset at https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/82ca6840-cf23-4bb0-a9be-55237ebbe9d3.\n"
+                    ]
+                }
+            ],
+            "source": [
+                "clone_public_dataset(task.dataset_id, dataset_name=task.name)"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "b462f7b8-fd42-4613-ab5f-5f3cbbc37d28",
+            "metadata": {},
+            "source": [
+                "Let's build an agent that we can use for evaluation."
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 4,
+            "id": "6142cf4e-862c-47a3-aa75-81d7d3231308",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "data": {
+                        "text/plain": [
+                            "{'question': 'abc',\n",
+                            " 'output': 'a, b, c',\n",
+                            " 'intermediate_steps': [(AgentActionMessageLog(tool='type_letter', tool_input={'letter': 'a'}, log=\"\\nInvoking: `type_letter` with `{'letter': 'a'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'arguments': '{\\n  \"letter\": \"a\"\\n}', 'name': 'type_letter'}})]),\n",
+                            "   'OK'),\n",
+                            "  (AgentActionMessageLog(tool='type_letter', tool_input={'letter': 'b'}, log=\"\\nInvoking: `type_letter` with `{'letter': 'b'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'arguments': '{\\n  \"letter\": \"b\"\\n}', 'name': 'type_letter'}})]),\n",
+                            "   'OK'),\n",
+                            "  (AgentActionMessageLog(tool='type_letter', tool_input={'letter': 'c'}, log=\"\\nInvoking: `type_letter` with `{'letter': 'c'}`\\n\\n\\n\", message_log=[AIMessage(content='', additional_kwargs={'function_call': {'arguments': '{\\n  \"letter\": \"c\"\\n}', 'name': 'type_letter'}})]),\n",
+                            "   'OK')],\n",
+                            " 'state': 'abc'}"
+                        ]
+                    },
+                    "execution_count": 4,
+                    "metadata": {},
+                    "output_type": "execute_result"
+                }
+            ],
+            "source": [
+                "from langchain_benchmarks.tool_usage import agents\n",
+                "\n",
+                "agent_factory = agents.OpenAIAgentFactory(task, model=\"gpt-3.5-turbo-16k\")\n",
+                "\n",
+                "# Let's test that our agent works\n",
+                "agent = agent_factory.create()\n",
+                "agent.invoke({\"question\": \"abc\"})"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "3821e4b0-8e67-418a-840c-470fcde42df0",
+            "metadata": {},
+            "source": [
+                "## Eval\n",
+                "\n",
+                "Let's evaluate an agent now"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 5,
+            "id": "fb32763c-79ab-426a-8fc6-bf8ebb0dd432",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "View the evaluation results for project 'test-fresh-whip-11' at:\n",
+                        "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/projects/p/c0c32118-d413-409f-ac01-088632c0e6ab?eval=true\n",
+                        "\n",
+                        "View all tests for Dataset Tool Usage - Typewriter (1 tool) at:\n",
+                        "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/82ca6840-cf23-4bb0-a9be-55237ebbe9d3\n",
+                        "[------------------------------------------------->] 20/20\n",
+                        " Eval quantiles:\n",
+                        "                                0.25  0.5  0.75  mean  mode\n",
+                        "Intermediate steps correctness  1.00  1.0   1.0  0.95   1.0\n",
+                        "# steps / # expected steps      1.00  1.0   1.0  1.70   1.0\n",
+                        "Correct Final State             1.00  1.0   1.0  0.95   1.0\n",
+                        "correctness                     0.75  1.0   1.0  0.75   1.0\n"
+                    ]
+                }
+            ],
+            "source": [
+                "from langsmith.client import Client\n",
+                "\n",
+                "from langchain_benchmarks.tool_usage import STANDARD_AGENT_EVALUATOR\n",
+                "\n",
+                "client = Client()\n",
+                "\n",
+                "test_run = client.run_on_dataset(\n",
+                "    dataset_name=task.name,\n",
+                "    llm_or_chain_factory=agent_factory.create,\n",
+                "    evaluation=STANDARD_AGENT_EVALUATOR,\n",
+                "    verbose=True,\n",
+                "    tags=[\"gpt-3.5-turbo-16k\"],\n",
+                ")"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "1b039225-01cf-481a-87a6-4e880e9b1dcd",
+            "metadata": {},
+            "source": [
+                "# Inspect\n",
+                "\n",
+                "You can take a look at the underlying results."
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 12,
+            "id": "6eb19db1-43b8-4866-a3d2-f211ba92ab8b",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [],
+            "source": [
+                "import pandas as pd\n",
+                "\n",
+                "df = test_run.to_dataframe()\n",
+                "df = pd.json_normalize(df.to_dict(orient=\"records\"))"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 13,
+            "id": "7ab5a8b9-a937-4537-b879-704284df4494",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "data": {
+                        "text/plain": [
+                            "0.75"
+                        ]
+                    },
+                    "execution_count": 13,
+                    "metadata": {},
+                    "output_type": "execute_result"
+                }
+            ],
+            "source": [
+                "df[\"correctness\"].mean()"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 14,
+            "id": "ab7516ed-36b1-4c16-bf4a-cc49077460ad",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [],
+            "source": [
+                "df[\"num_expected_steps\"] = df[\"reference.expected_steps\"].apply(len)\n",
+                "df[\"actual_number_of_steps\"] = df[\"output.intermediate_steps\"].apply(len)"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 15,
+            "id": "50d7590d-20de-4768-ac90-adcdbfa70068",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "data": {
+                        "text/html": [
+                            "<div>\n",
+                            "<style scoped>\n",
+                            "    .dataframe tbody tr th:only-of-type {\n",
+                            "        vertical-align: middle;\n",
+                            "    }\n",
+                            "\n",
+                            "    .dataframe tbody tr th {\n",
+                            "        vertical-align: top;\n",
+                            "    }\n",
+                            "\n",
+                            "    .dataframe thead th {\n",
+                            "        text-align: right;\n",
+                            "    }\n",
+                            "</style>\n",
+                            "<table border=\"1\" class=\"dataframe\">\n",
+                            "  <thead>\n",
+                            "    <tr style=\"text-align: right;\">\n",
+                            "      <th></th>\n",
+                            "      <th>Intermediate steps correctness</th>\n",
+                            "      <th># steps / # expected steps</th>\n",
+                            "      <th>Correct Final State</th>\n",
+                            "      <th>correctness</th>\n",
+                            "      <th>input.question</th>\n",
+                            "      <th>output.question</th>\n",
+                            "      <th>output.output</th>\n",
+                            "      <th>output.intermediate_steps</th>\n",
+                            "      <th>output.state</th>\n",
+                            "      <th>reference.state</th>\n",
+                            "      <th>reference.reference</th>\n",
+                            "      <th>reference.expected_steps</th>\n",
+                            "      <th>num_expected_steps</th>\n",
+                            "      <th>actual_number_of_steps</th>\n",
+                            "    </tr>\n",
+                            "  </thead>\n",
+                            "  <tbody>\n",
+                            "    <tr>\n",
+                            "      <th>0</th>\n",
+                            "      <td>1</td>\n",
+                            "      <td>1.0</td>\n",
+                            "      <td>1</td>\n",
+                            "      <td>1</td>\n",
+                            "      <td>communication</td>\n",
+                            "      <td>communication</td>\n",
+                            "      <td>communication</td>\n",
+                            "      <td>[(tool='type_letter' tool_input={'letter': 'c'...</td>\n",
+                            "      <td>communication</td>\n",
+                            "      <td>communication</td>\n",
+                            "      <td>communication</td>\n",
+                            "      <td>[type_letter, type_letter, type_letter, type_l...</td>\n",
+                            "      <td>13</td>\n",
+                            "      <td>13</td>\n",
+                            "    </tr>\n",
+                            "    <tr>\n",
+                            "      <th>1</th>\n",
+                            "      <td>1</td>\n",
+                            "      <td>1.0</td>\n",
+                            "      <td>1</td>\n",
+                            "      <td>1</td>\n",
+                            "      <td>information</td>\n",
+                            "      <td>information</td>\n",
+                            "      <td>information</td>\n",
+                            "      <td>[(tool='type_letter' tool_input={'letter': 'i'...</td>\n",
+                            "      <td>information</td>\n",
+                            "      <td>information</td>\n",
+                            "      <td>information</td>\n",
+                            "      <td>[type_letter, type_letter, type_letter, type_l...</td>\n",
+                            "      <td>11</td>\n",
+                            "      <td>11</td>\n",
+                            "    </tr>\n",
+                            "    <tr>\n",
+                            "      <th>2</th>\n",
+                            "      <td>1</td>\n",
+                            "      <td>1.0</td>\n",
+                            "      <td>1</td>\n",
+                            "      <td>1</td>\n",
+                            "      <td>dictionary</td>\n",
+                            "      <td>dictionary</td>\n",
+                            "      <td>dictionary</td>\n",
+                            "      <td>[(tool='type_letter' tool_input={'letter': 'd'...</td>\n",
+                            "      <td>dictionary</td>\n",
+                            "      <td>dictionary</td>\n",
+                            "      <td>dictionary</td>\n",
+                            "      <td>[type_letter, type_letter, type_letter, type_l...</td>\n",
+                            "      <td>10</td>\n",
+                            "      <td>10</td>\n",
+                            "    </tr>\n",
+                            "    <tr>\n",
+                            "      <th>3</th>\n",
+                            "      <td>1</td>\n",
+                            "      <td>1.0</td>\n",
+                            "      <td>1</td>\n",
+                            "      <td>1</td>\n",
+                            "      <td>university</td>\n",
+                            "      <td>university</td>\n",
+                            "      <td>u\\nn\\ni\\nv\\ne\\nr\\ns\\ni\\nt\\ny</td>\n",
+                            "      <td>[(tool='type_letter' tool_input={'letter': 'u'...</td>\n",
+                            "      <td>university</td>\n",
+                            "      <td>university</td>\n",
+                            "      <td>university</td>\n",
+                            "      <td>[type_letter, type_letter, type_letter, type_l...</td>\n",
+                            "      <td>10</td>\n",
+                            "      <td>10</td>\n",
+                            "    </tr>\n",
+                            "    <tr>\n",
+                            "      <th>4</th>\n",
+                            "      <td>1</td>\n",
+                            "      <td>1.0</td>\n",
+                            "      <td>1</td>\n",
+                            "      <td>1</td>\n",
+                            "      <td>keyboard</td>\n",
+                            "      <td>keyboard</td>\n",
+                            "      <td>keyboard</td>\n",
+                            "      <td>[(tool='type_letter' tool_input={'letter': 'k'...</td>\n",
+                            "      <td>keyboard</td>\n",
+                            "      <td>keyboard</td>\n",
+                            "      <td>keyboard</td>\n",
+                            "      <td>[type_letter, type_letter, type_letter, type_l...</td>\n",
+                            "      <td>8</td>\n",
+                            "      <td>8</td>\n",
+                            "    </tr>\n",
+                            "  </tbody>\n",
+                            "</table>\n",
+                            "</div>"
+                        ],
+                        "text/plain": [
+                            "   Intermediate steps correctness  # steps / # expected steps  \\\n",
+                            "0                               1                         1.0   \n",
+                            "1                               1                         1.0   \n",
+                            "2                               1                         1.0   \n",
+                            "3                               1                         1.0   \n",
+                            "4                               1                         1.0   \n",
+                            "\n",
+                            "   Correct Final State  correctness input.question output.question  \\\n",
+                            "0                    1            1  communication   communication   \n",
+                            "1                    1            1    information     information   \n",
+                            "2                    1            1     dictionary      dictionary   \n",
+                            "3                    1            1     university      university   \n",
+                            "4                    1            1       keyboard        keyboard   \n",
+                            "\n",
+                            "                  output.output  \\\n",
+                            "0                 communication   \n",
+                            "1                   information   \n",
+                            "2                    dictionary   \n",
+                            "3  u\\nn\\ni\\nv\\ne\\nr\\ns\\ni\\nt\\ny   \n",
+                            "4                      keyboard   \n",
+                            "\n",
+                            "                           output.intermediate_steps   output.state  \\\n",
+                            "0  [(tool='type_letter' tool_input={'letter': 'c'...  communication   \n",
+                            "1  [(tool='type_letter' tool_input={'letter': 'i'...    information   \n",
+                            "2  [(tool='type_letter' tool_input={'letter': 'd'...     dictionary   \n",
+                            "3  [(tool='type_letter' tool_input={'letter': 'u'...     university   \n",
+                            "4  [(tool='type_letter' tool_input={'letter': 'k'...       keyboard   \n",
+                            "\n",
+                            "  reference.state reference.reference  \\\n",
+                            "0   communication       communication   \n",
+                            "1     information         information   \n",
+                            "2      dictionary          dictionary   \n",
+                            "3      university          university   \n",
+                            "4        keyboard            keyboard   \n",
+                            "\n",
+                            "                            reference.expected_steps  num_expected_steps  \\\n",
+                            "0  [type_letter, type_letter, type_letter, type_l...                  13   \n",
+                            "1  [type_letter, type_letter, type_letter, type_l...                  11   \n",
+                            "2  [type_letter, type_letter, type_letter, type_l...                  10   \n",
+                            "3  [type_letter, type_letter, type_letter, type_l...                  10   \n",
+                            "4  [type_letter, type_letter, type_letter, type_l...                   8   \n",
+                            "\n",
+                            "   actual_number_of_steps  \n",
+                            "0                      13  \n",
+                            "1                      11  \n",
+                            "2                      10  \n",
+                            "3                      10  \n",
+                            "4                       8  "
+                        ]
+                    },
+                    "execution_count": 15,
+                    "metadata": {},
+                    "output_type": "execute_result"
+                }
+            ],
+            "source": [
+                "df.head()"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "id": "62bcf6c2-6449-4967-a4f4-2f3d90657a52",
+            "metadata": {},
+            "outputs": [],
+            "source": []
+        }
+    ],
+    "metadata": {
+        "kernelspec": {
+            "display_name": "Python 3 (ipykernel)",
+            "language": "python",
+            "name": "python3"
+        },
+        "language_info": {
+            "codemirror_mode": {
+                "name": "ipython",
+                "version": 3
+            },
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.11.2"
+        }
+    },
+    "nbformat": 4,
+    "nbformat_minor": 5
+}

docs/source/notebooks/tool_usage/typewriter_26.ipynb

@@ -0,0 +1,284 @@
+{
+    "cells": [
+        {
+            "cell_type": "markdown",
+            "id": "60bb467d-861d-4b07-a48d-8e5aa177c969",
+            "metadata": {
+                "tags": []
+            },
+            "source": [
+                "# Typewriter: 26 Tools\n",
+                "[![Open In Collab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/langchain-ai/langchain-benchmarks/blob/main/docs/source/notebooks/tool_usage/typewriter_26.ipynb)\n",
+                "\n",
+                "Let's see how to evaluate an agent's ability to use tools.\n",
+                "\n",
+                "    A task where the agent must type a given string one letter at a time.\n",
+                "\n",
+                "    In this variation of the task, the agent is given access to 26 parameterless functions,\n",
+                "    each representing a letter of the alphabet."
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": null,
+            "id": "845c77a6-9da6-494c-973f-0ee1dac67b19",
+            "metadata": {},
+            "outputs": [],
+            "source": [
+                "import os\n",
+                "\n",
+                "os.environ[\"LANGCHAIN_API_KEY\"] = \"sk-...\"  # Your api key."
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 1,
+            "id": "b39159d0-9ea1-414f-a9d8-4a7b22b3d2cc",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [],
+            "source": [
+                "from langchain_benchmarks import clone_public_dataset, registry"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 2,
+            "id": "1aef2b32-a5df-421f-8be3-a2ef27372ece",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "data": {
+                        "text/html": [
+                            "<table>\n",
+                            "<tbody>\n",
+                            "<tr><td>Name       </td><td>Tool Usage - Typewriter (26 tools)                                                                                                                         </td></tr>\n",
+                            "<tr><td>Type       </td><td>ToolUsageTask                                                                                                                                              </td></tr>\n",
+                            "<tr><td>Dataset ID </td><td><a href=\"https://smith.langchain.com/public/128af05e-aa00-4e3b-a958-d166dd450581/d\" target=\"_blank\" rel=\"noopener\">128af05e-aa00-4e3b-a958-d166dd450581</a></td></tr>\n",
+                            "<tr><td>Description</td><td>Environment with 26 tools each tool represents a letter of the alphabet.\n",
+                            "\n",
+                            "The objective of this task is to evaluate the model's ability the use tools\n",
+                            "for a simple repetition task.\n",
+                            "\n",
+                            "For example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\n",
+                            "\n",
+                            "The dataset includes examples of varying difficulty. The difficulty is measured by the length of the string.\n",
+                            "\n",
+                            "This is a variation of the typer writer task, where 26 parameterless tools are\n",
+                            "given instead of a single tool that takes a letter as an argument.                                                                                                                                                            </td></tr>\n",
+                            "</tbody>\n",
+                            "</table>"
+                        ],
+                        "text/plain": [
+                            "ToolUsageTask(name='Tool Usage - Typewriter (26 tools)', dataset_id='https://smith.langchain.com/public/128af05e-aa00-4e3b-a958-d166dd450581/d', description=\"Environment with 26 tools each tool represents a letter of the alphabet.\\n\\nThe objective of this task is to evaluate the model's ability the use tools\\nfor a simple repetition task.\\n\\nFor example, if the string is 'abc', the tools 'a', 'b', and 'c' must be invoked in that order.\\n\\nThe dataset includes examples of varying difficulty. The difficulty is measured by the length of the string.\\n\\nThis is a variation of the typer writer task, where 26 parameterless tools are\\ngiven instead of a single tool that takes a letter as an argument.\\n\", create_environment=<function get_environment at 0x12788dd00>, instructions=\"Repeat the given string by using the provided tools. Do not write anything else or provide any explanations. For example, if the string is 'abc', you must invoke the tools 'a', 'b', and 'c' in that order. Please invoke the functions without any arguments.\")"
+                        ]
+                    },
+                    "execution_count": 2,
+                    "metadata": {},
+                    "output_type": "execute_result"
+                }
+            ],
+            "source": [
+                "task = registry[\"Tool Usage - Typewriter (26 tools)\"]\n",
+                "task"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "bc33a639-3caf-4314-8ea7-1c7c8b1d114d",
+            "metadata": {},
+            "source": [
+                "Clone the dataset associaetd with this task"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 3,
+            "id": "70369f67-deb4-467a-801a-6d38c3d0460d",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "Dataset Tool Usage - Typewriter (26 tools) already exists. Skipping.\n",
+                        "You can access the dataset at https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/2f462c7a-f9b9-46e7-b96b-7469e965f478.\n"
+                    ]
+                }
+            ],
+            "source": [
+                "clone_public_dataset(task.dataset_id, dataset_name=task.name)"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "b462f7b8-fd42-4613-ab5f-5f3cbbc37d28",
+            "metadata": {},
+            "source": [
+                "Let's build an agent that we can use for evaluation."
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 4,
+            "id": "61535a75-24f6-4727-9549-f76c263e9153",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [],
+            "source": [
+                "env = task.create_environment()"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 5,
+            "id": "6142cf4e-862c-47a3-aa75-81d7d3231308",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "data": {
+                        "text/plain": [
+                            "{'question': 'foo',\n",
+                            " 'output': \"Could not parse tool input: {'arguments': '', 'name': 'f'} because the `arguments` is not valid JSON.\",\n",
+                            " 'intermediate_steps': [(AgentAction(tool='_Exception', tool_input='Invalid or incomplete response', log=\"Could not parse tool input: {'arguments': 'f', 'name': 'f'} because the `arguments` is not valid JSON.\"),\n",
+                            "   'Invalid or incomplete response'),\n",
+                            "  (AgentAction(tool='_Exception', tool_input='Invalid or incomplete response', log=\"Could not parse tool input: {'arguments': '', 'name': 'f'} because the `arguments` is not valid JSON.\"),\n",
+                            "   'Invalid or incomplete response'),\n",
+                            "  (AgentAction(tool='_Exception', tool_input='Invalid or incomplete response', log=\"Could not parse tool input: {'arguments': '', 'name': 'f'} because the `arguments` is not valid JSON.\"),\n",
+                            "   'Invalid or incomplete response'),\n",
+                            "  (AgentAction(tool='_Exception', tool_input='Invalid or incomplete response', log=\"Could not parse tool input: {'arguments': '', 'name': 'f'} because the `arguments` is not valid JSON.\"),\n",
+                            "   'Invalid or incomplete response'),\n",
+                            "  (AgentAction(tool='_Exception', tool_input='Invalid or incomplete response', log=\"Could not parse tool input: {'arguments': '', 'name': 'f'} because the `arguments` is not valid JSON.\"),\n",
+                            "   'Invalid or incomplete response')],\n",
+                            " 'state': ''}"
+                        ]
+                    },
+                    "execution_count": 5,
+                    "metadata": {},
+                    "output_type": "execute_result"
+                }
+            ],
+            "source": [
+                "from langchain_benchmarks.tool_usage import agents\n",
+                "\n",
+                "agent_factory = agents.OpenAIAgentFactory(task, model=\"gpt-3.5-turbo-16k\")\n",
+                "\n",
+                "# Let's test that our agent works\n",
+                "agent = agent_factory()\n",
+                "agent.invoke({\"question\": \"foo\"})"
+            ]
+        },
+        {
+            "cell_type": "markdown",
+            "id": "3821e4b0-8e67-418a-840c-470fcde42df0",
+            "metadata": {},
+            "source": [
+                "## Eval\n",
+                "\n",
+                "Let's evaluate an agent now"
+            ]
+        },
+        {
+            "cell_type": "code",
+            "execution_count": 6,
+            "id": "fb32763c-79ab-426a-8fc6-bf8ebb0dd432",
+            "metadata": {
+                "tags": []
+            },
+            "outputs": [
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "View the evaluation results for project 'test-notable-artist-76' at:\n",
+                        "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/projects/p/5c828160-9f7f-4f01-84ea-05f8a498d031?eval=true\n",
+                        "\n",
+                        "View all tests for Dataset Tool Usage - Typewriter (26 tools) at:\n",
+                        "https://smith.langchain.com/o/ebbaf2eb-769b-4505-aca2-d11de10372a4/datasets/2f462c7a-f9b9-46e7-b96b-7469e965f478\n",
+                        "[>                                                 ] 0/20"
+                    ]
+                },
+                {
+                    "name": "stderr",
+                    "output_type": "stream",
+                    "text": [
+                        "Chain failed for example 2d4e99fc-8495-468e-8429-6c25a2d176f3 with inputs {'question': 'keyboard'}\n",
+                        "Error Type: InternalServerError, Message: Error code: 500 - {'error': {'message': 'The server had an error processing your request. Sorry about that! You can retry your request, or contact us through our help center at help.openai.com if you keep seeing this error. (Please include the request ID b658bca90fb852f4d236fc368bc65bcc in your email.)', 'type': 'server_error', 'param': None, 'code': None}}\n"
+                    ]
+                },
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "[------------------->                              ] 8/20"
+                    ]
+                },
+                {
+                    "name": "stderr",
+                    "output_type": "stream",
+                    "text": [
+                        "Chain failed for example 8af5bd36-fc11-4b23-9019-f642cfaf8a01 with inputs {'question': 'horse'}\n",
+                        "Error Type: InternalServerError, Message: Error code: 500 - {'error': {'message': 'The server had an error processing your request. Sorry about that! You can retry your request, or contact us through our help center at help.openai.com if you keep seeing this error. (Please include the request ID 3c40664804cb6e8c84e0e8796dbc0a6d in your email.)', 'type': 'server_error', 'param': None, 'code': None}}\n"
+                    ]
+                },
+                {
+                    "name": "stdout",
+                    "output_type": "stream",
+                    "text": [
+                        "[------------------------------------------------->] 20/20\n",
+                        " Eval quantiles:\n",
+                        "                                    0.25   0.5   0.75      mean  mode\n",
+                        "Intermediate steps correctness  0.000000  0.00  0.000  0.000000  0.00\n",
+                        "# steps / # expected steps      0.703571  0.75  1.375  1.007551  0.75\n",
+                        "Correct Final State             0.000000  0.00  0.000  0.055556  0.00\n",
+                        "correctness                     0.000000  0.00  0.000  0.111111  0.00\n"
+                    ]
+                }
+            ],
+            "source": [
+                "from langsmith.client import Client\n",
+                "\n",
+                "from langchain_benchmarks.tool_usage import STANDARD_AGENT_EVALUATOR\n",
+                "\n",
+                "client = Client()\n",
+                "\n",
+                "test_run = client.run_on_dataset(\n",
+                "    dataset_name=task.name,\n",
+                "    llm_or_chain_factory=agent_factory.create,\n",
+                "    evaluation=STANDARD_AGENT_EVALUATOR,\n",
+                "    verbose=True,\n",
+                "    tags=[\"gpt-3.5-turbo-16k\"],\n",
+                ")"
+            ]
+        }
+    ],
+    "metadata": {
+        "kernelspec": {
+            "display_name": "Python 3 (ipykernel)",
+            "language": "python",
+            "name": "python3"
+        },
+        "language_info": {
+            "codemirror_mode": {
+                "name": "ipython",
+                "version": 3
+            },
+            "file_extension": ".py",
+            "mimetype": "text/x-python",
+            "name": "python",
+            "nbconvert_exporter": "python",
+            "pygments_lexer": "ipython3",
+            "version": "3.11.2"
+        }
+    },
+    "nbformat": 4,
+    "nbformat_minor": 5
+}

docs/source/toc.segment

@@ -0,0 +1,37 @@
+```{toctree}
+:maxdepth: 2
+:caption: Introduction
+
+./notebooks/getting_started
+./notebooks/datasets
+```
+
+
+```{toctree}
+:maxdepth: 0
+:caption: Tool Usage
+
+./notebooks/tool_usage/intro
+./notebooks/tool_usage/relational_data
+./notebooks/tool_usage/multiverse_math
+./notebooks/tool_usage/typewriter_1
+./notebooks/tool_usage/typewriter_26
+```
+
+```{toctree}
+:maxdepth: 0
+:caption: Extraction
+
+./notebooks/extraction/intro
+./notebooks/extraction/email
+```
+
+```{toctree}
+:maxdepth: 2
+:caption: RAG
+
+./notebooks/retrieval/intro
+./notebooks/retrieval/langchain_docs_qa
+./notebooks/retrieval/semi_structured
+./notebooks/retrieval/comparing_techniques
+```

extraction/streamlit_app.py

@@ -1,18 +1,23 @@
 import streamlit as st
-from langsmith import Client
-from langchain.chat_models import ChatOpenAI
 from langchain.chains import create_extraction_chain
+from langchain.chat_models import ChatOpenAI
+from langsmith import Client
 
-st.set_page_config(page_title='🦜🔗 Text-to-graph extraction')
+st.set_page_config(page_title="🦜🔗 Text-to-graph extraction")
 client = Client()
+
+
 def send_feedback(run_id, score):
     client.create_feedback(run_id, "user_score", score=score)
 
-st.title('🦜🔗 Text-to-graph playground')
-st.info("This playground explores the use of [OpenAI functions](https://openai.com/blog/function-calling-and-other-api-updates) and [LangChain](https://github.com/langchain-ai/langchain) to build knowledge graphs from user-input text. It breaks down the user input text into knowledge graph triples of subject (primary entities or concepts in a sentence), predicate (actions or relationships that connect subjects to objects), and object (entities or concepts that interact with or are acted upon by the subjects).")
+
+st.title("🦜🔗 Text-to-graph playground")
+st.info(
+    "This playground explores the use of [OpenAI functions](https://openai.com/blog/function-calling-and-other-api-updates) and [LangChain](https://github.com/langchain-ai/langchain) to build knowledge graphs from user-input text. It breaks down the user input text into knowledge graph triples of subject (primary entities or concepts in a sentence), predicate (actions or relationships that connect subjects to objects), and object (entities or concepts that interact with or are acted upon by the subjects)."
+)
 
 # Input text (optional default)
-oppenheimer_text=''''Julius Robert Oppenheimer, often known as Robert or "Oppie", is heralded as the father of the atomic bomb. Emerging from a non-practicing Jewish family in New York, he made several breakthroughs, such as the early black hole theory, before the monumental Manhattan Project. His wife, Katherine “Kitty” Oppenheimer, was a German-born woman with a complex past, including connections to the Communist Party. Oppenheimer\'s journey was beset by political adversaries, notably Lewis Strauss, chairman of the U.S. Atomic Energy Commission, and William Borden, an executive director with hawkish nuclear ambitions. These tensions culminated in the famous 1954 security hearing. Influential figures like lieutenant general Leslie Groves, who had also overseen the Pentagon\'s creation, stood by Oppenheimer\'s side, having earlier chosen him for the Manhattan Project and the Los Alamos location. Intimate relationships, like that with Jean Tatlock, a Communist and the possible muse behind the Trinity test\'s name, and colleagues like Frank, Oppenheimer\'s physicist brother, intertwined with his professional life. Scientists such as Ernest Lawrence, Edward Teller, David Hill, Richard Feynman, and Hans Bethe were some of Oppenheimer\'s contemporaries, each contributing to and contesting the atomic age\'s directions. Boris Pash\'s investigations, and the perspectives of figures like Leo Szilard, Niels Bohr, Harry Truman, and others, framed the broader sociopolitical context. Meanwhile, individuals like Robert Serber, Enrico Fermi, Albert Einstein, and Isidor Isaac Rabi, among many others, each played their parts in this narrative, from naming the atomic bombs to pivotal scientific contributions and advisory roles. All these figures, together with the backdrop of World War II, McCarthyism, and the dawn of the nuclear age, presented a complex mosaic of ambitions, loyalties, betrayals, and ideologies.oppenheimer_short.txt'''
+oppenheimer_text = """'Julius Robert Oppenheimer, often known as Robert or "Oppie", is heralded as the father of the atomic bomb. Emerging from a non-practicing Jewish family in New York, he made several breakthroughs, such as the early black hole theory, before the monumental Manhattan Project. His wife, Katherine “Kitty” Oppenheimer, was a German-born woman with a complex past, including connections to the Communist Party. Oppenheimer\'s journey was beset by political adversaries, notably Lewis Strauss, chairman of the U.S. Atomic Energy Commission, and William Borden, an executive director with hawkish nuclear ambitions. These tensions culminated in the famous 1954 security hearing. Influential figures like lieutenant general Leslie Groves, who had also overseen the Pentagon\'s creation, stood by Oppenheimer\'s side, having earlier chosen him for the Manhattan Project and the Los Alamos location. Intimate relationships, like that with Jean Tatlock, a Communist and the possible muse behind the Trinity test\'s name, and colleagues like Frank, Oppenheimer\'s physicist brother, intertwined with his professional life. Scientists such as Ernest Lawrence, Edward Teller, David Hill, Richard Feynman, and Hans Bethe were some of Oppenheimer\'s contemporaries, each contributing to and contesting the atomic age\'s directions. Boris Pash\'s investigations, and the perspectives of figures like Leo Szilard, Niels Bohr, Harry Truman, and others, framed the broader sociopolitical context. Meanwhile, individuals like Robert Serber, Enrico Fermi, Albert Einstein, and Isidor Isaac Rabi, among many others, each played their parts in this narrative, from naming the atomic bombs to pivotal scientific contributions and advisory roles. All these figures, together with the backdrop of World War II, McCarthyism, and the dawn of the nuclear age, presented a complex mosaic of ambitions, loyalties, betrayals, and ideologies.oppenheimer_short.txt"""
 
 # Knowledge triplet schema
 default_schema = {
@@ -31,46 +36,44 @@ if len(user_input_text) > MAX_CHARS:
     st.warning(f"Text is too long. Processing only the first {MAX_CHARS} characters")
     user_input_text = user_input_text[:MAX_CHARS]
 
+
 # Output formatting of triples
 def json_to_markdown_table(json_list):
     if not json_list:
         return "No data available."
-    
+
     # Extract headers
     headers = json_list[0].keys()
     markdown_table = " | ".join(headers) + "\n"
     markdown_table += " | ".join(["---"] * len(headers)) + "\n"
-    
+
     # Extract rows
     for item in json_list:
         row = " | ".join([str(item[header]) for header in headers])
         markdown_table += row + "\n"
-    
+
     return markdown_table
 
+
 # Form input and query
 markdown_output = None
-with st.form('myform', clear_on_submit=True):
-
-    submitted = st.form_submit_button('Submit')
+with st.form("myform", clear_on_submit=True):
+    submitted = st.form_submit_button("Submit")
     if submitted:
-		
-        with st.spinner('Calculating...'):
-
+        with st.spinner("Calculating..."):
             llm = ChatOpenAI(temperature=0, model="gpt-3.5-turbo")
             chain = create_extraction_chain(default_schema, llm)
-            extraction_output = chain(user_input_text,include_run_info=True)
-            markdown_output = json_to_markdown_table(extraction_output['text'])
+            extraction_output = chain(user_input_text, include_run_info=True)
+            markdown_output = json_to_markdown_table(extraction_output["text"])
             run_id = extraction_output["__run"].run_id
-        
+
 # Feeback
 if markdown_output is not None:
     st.markdown(markdown_output)
-    col_blank, col_text, col1, col2 = st.columns([10, 2,1,1])
+    col_blank, col_text, col1, col2 = st.columns([10, 2, 1, 1])
     with col_text:
         st.text("Feedback:")
     with col1:
         st.button("👍", on_click=send_feedback, args=(run_id, 1))
     with col2:
         st.button("👎", on_click=send_feedback, args=(run_id, 0))
-	

langchain-docs-benchmarking/README.md

@@ -0,0 +1,89 @@
+# Benchmarking on LangChain Docs
+
+This directory contains code to benchmark your cognitive architecture on the public [LangChain Q&A docs evaluation benchmark](https://smith.langchain.com/public/e1bfd348-494a-4df5-899a-7c6c09233cc4/d).
+
+To one one of the existing configurations, activate your poetry environment, configure you LangSmith API key, and run the experiments.
+
+**Note:** this will benchmark chains on a _copy_ of the dataset and will not update the public leaderboard.
+
+## Running the published experiments
+
+The following steps will let you run pre-configured experiments:
+
+### 1. Install requirements
+
+```bash
+pip install poetry
+poetry shell
+poetry install
+```
+
+### 2. Configure API keys
+
+Create a [LangSmith account](https://smith.langchain.com/) and set your API key:
+
+```bash
+export LANGCHAIN_API_KEY=ls_your-api-key
+```
+
+The various cognitive architectures implemented already use Anthropic, [Fireworks.AI](https://www.fireworks.ai/), and OpenAI. Set the required API keys:
+
+```
+export OPENAI_API_KEY=your-api-key
+export ANTHROPIC_API_KEY=your-api-key
+export FIREWORKS_API_KEY=your-api-key
+```
+
+### 3. Run Experiments
+
+To run all experiments, run:
+
+```bash
+python run_experiments.py
+```
+
+If you want to only run certain experiments in the `run_experiments.py` file, use `--include` or `--exclude`
+
+Example:
+
+```bash
+python run_experiments --include mistral-7b-instruct-4k llama-v2-34b-code-instruct-w8a16
+```
+
+## Evaluating your custom cognitive architecture
+
+You can also evaluate your own custom cognitive architecture. To do so:
+
+1. Create a python file defining your architecture:
+
+```python
+# example_custom_chain.py
+
+...
+def load_runnable(config: dict) -> "Runnable":
+    # Load based on the config provided
+    return my_chain
+```
+
+2. Call `run_experiments.py` with a custom `--config my_config.json`
+
+```js
+{
+  // This specifies the path to your custom entrypoint followed by the loader function
+  "arch": "path/to/example_custom_chain.py::load_runnable",
+  "model_config": {
+    // This is passed to load_runnable() in example_custom_chain.py()
+    "chat_cls": "ChatOpenAI",
+    "model": "gpt-4"
+  },
+  "project_name": "example-custom-code" // This is the resulting test project name
+}
+```
+
+We have provided an example in [example_custom_chain.py](./packages/example/custom_example/example_custom_chain.py), which can be run by pointing `run_experiments` to the [example_custom_config.json](./example_custom_config.json) config file:
+
+```bash
+python run_experiments.py --config ./example_custom_config.json
+```
+
+Whenever you provide 1 or more `--config` files, the `--include` and `--exclude` arguments are ignored.

langchain-docs-benchmarking/app/server.py

@@ -0,0 +1,27 @@
+from chat_langchain.chain import chain
+from fastapi import FastAPI
+from openai_functions_agent import agent_executor as openai_functions_agent_chain
+
+from langserve import add_routes
+
+app = FastAPI()
+
+# Edit this to add the chain you want to add
+add_routes(
+    app,
+    chain,
+    path="/chat",
+    # include_callback_events=True, # TODO: Include when fixed
+)
+
+add_routes(app, openai_functions_agent_chain, path="/openai-functions-agent")
+
+
+def run_server(port: int = 1983):
+    import uvicorn
+
+    uvicorn.run(app, host="0.0.0.0", port=port)
+
+
+if __name__ == "__main__":
+    run_server()

langchain-docs-benchmarking/example_custom_config.json

@@ -0,0 +1,8 @@
+{
+  "arch": "packages/example/custom_example/example_custom_chain.py::create_runnable",
+  "model_config": {
+    "chat_cls": "ChatOpenAI",
+    "model": "gpt-4"
+  },
+  "project_name": "example-custom-code"
+}

langchain-docs-benchmarking/packages/anthropic-iterative-search/README.md

@@ -0,0 +1,69 @@
+
+# anthropic-iterative-search
+
+This template will create a virtual research assistant with the ability to search Wikipedia to find answers to your questions.
+
+It is heavily inspired by [this notebook](https://github.com/anthropics/anthropic-cookbook/blob/main/long_context/wikipedia-search-cookbook.ipynb).
+
+## Environment Setup
+
+Set the `ANTHROPIC_API_KEY` environment variable to access the Anthropic models.
+
+## Usage
+
+To use this package, you should first have the LangChain CLI installed:
+
+```shell
+pip install -U "langchain-cli[serve]"
+```
+
+To create a new LangChain project and install this as the only package, you can do:
+
+```shell
+langchain app new my-app --package anthropic-iterative-search
+```
+
+If you want to add this to an existing project, you can just run:
+
+```shell
+langchain app add anthropic-iterative-search
+```
+
+And add the following code to your `server.py` file:
+```python
+from anthropic_iterative_search import chain as anthropic_iterative_search_chain
+
+add_routes(app, anthropic_iterative_search_chain, path="/anthropic-iterative-search")
+```
+
+(Optional) Let's now configure LangSmith. 
+LangSmith will help us trace, monitor and debug LangChain applications. 
+LangSmith is currently in private beta, you can sign up [here](https://smith.langchain.com/). 
+If you don't have access, you can skip this section
+
+
+```shell
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_API_KEY=<your-api-key>
+export LANGCHAIN_PROJECT=<your-project>  # if not specified, defaults to "default"
+```
+
+If you are inside this directory, then you can spin up a LangServe instance directly by:
+
+```shell
+langchain serve
+```
+
+This will start the FastAPI app with a server is running locally at 
+[http://localhost:8000](http://localhost:8000)
+
+We can see all templates at [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs)
+We can access the playground at [http://127.0.0.1:8000/anthropic-iterative-search/playground](http://127.0.0.1:8000/anthropic-iterative-search/playground)  
+
+We can access the template from code with:
+
+```python
+from langserve.client import RemoteRunnable
+
+runnable = RemoteRunnable("http://localhost:8000/anthropic-iterative-search")
+```

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/__init__.py

@@ -0,0 +1,11 @@
+from langchain.schema.runnable import ConfigurableField
+
+from .chain import chain
+from .retriever_agent import executor
+
+final_chain = chain.configurable_alternatives(
+    ConfigurableField(id="chain"),
+    default_key="response",
+    # This adds a new option, with name `openai` that is equal to `ChatOpenAI()`
+    retrieve=executor,
+)

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/agent_scratchpad.py

@@ -0,0 +1,16 @@
+def _format_docs(docs):
+    result = "\n".join(
+        [
+            f'<item index="{i+1}">\n<page_content>\n{r}\n</page_content>\n</item>'
+            for i, r in enumerate(docs)
+        ]
+    )
+    return result
+
+
+def format_agent_scratchpad(intermediate_steps):
+    thoughts = ""
+    for action, observation in intermediate_steps:
+        thoughts += action.log
+        thoughts += "</search_query>" + _format_docs(observation)
+    return thoughts

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/chain.py

@@ -0,0 +1,29 @@
+from langchain.chat_models import ChatAnthropic
+from langchain.prompts import ChatPromptTemplate
+from langchain.pydantic_v1 import BaseModel
+from langchain.schema.output_parser import StrOutputParser
+from langchain.schema.runnable import RunnableLambda
+
+from .prompts import answer_prompt
+from .retriever_agent import executor
+
+prompt = ChatPromptTemplate.from_template(answer_prompt)
+
+model = ChatAnthropic(model="claude-2", temperature=0, max_tokens_to_sample=1000)
+
+chain = (
+    RunnableLambda(lambda x: {"query": x["question"]})
+    | {"query": lambda x: x["query"], "information": executor | (lambda x: x["output"])}
+    | prompt
+    | model
+    | StrOutputParser()
+)
+
+# Add typing for the inputs to be used in the playground
+
+
+class Inputs(BaseModel):
+    question: str
+
+
+chain = chain.with_types(input_type=Inputs)

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/output_parser.py

@@ -0,0 +1,37 @@
+import re
+
+from langchain.schema.agent import AgentAction, AgentFinish
+
+from .agent_scratchpad import _format_docs
+
+
+def extract_between_tags(tag: str, string: str, strip: bool = True) -> str:
+    ext_list = re.findall(f"<{tag}\s?>(.+?)</{tag}\s?>", string, re.DOTALL)
+    if strip:
+        ext_list = [e.strip() for e in ext_list]
+    if ext_list:
+        if len(ext_list) != 1:
+            raise ValueError
+        # Only return the first one
+        return ext_list[0]
+
+
+def parse_output(outputs):
+    partial_completion = outputs["partial_completion"]
+    steps = outputs["intermediate_steps"]
+    search_query = extract_between_tags(
+        "search_query", partial_completion + "</search_query>"
+    )
+    if search_query is None:
+        docs = []
+        str_output = ""
+        for action, observation in steps:
+            docs.extend(observation)
+            str_output += action.log
+            str_output += "</search_query>" + _format_docs(observation)
+        str_output += partial_completion
+        return AgentFinish({"docs": docs, "output": str_output}, log=partial_completion)
+    else:
+        return AgentAction(
+            tool="search", tool_input=search_query, log=partial_completion
+        )

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/prompts.py

@@ -0,0 +1,7 @@
+retrieval_prompt = """{retriever_description} Before beginning to research the user's question, first think for a moment inside <scratchpad> tags about what information is necessary for a well-informed answer. If the user's question is complex, you may need to decompose the query into multiple subqueries and execute them individually. Sometimes the search engine will return empty search results, or the search results may not contain the information you need. In such cases, feel free to try again with a different query. 
+
+After each call to the Search Engine Tool, reflect briefly inside <search_quality></search_quality> tags about whether you now have enough information to answer, or whether more information is needed. If you have all the relevant information, write it in <information></information> tags, WITHOUT actually answering the question. Otherwise, issue a new search.
+
+Here is the user's question: <question>{query}</question> Remind yourself to make short queries in your scratchpad as you plan out your strategy."""  # noqa: E501
+
+answer_prompt = "Here is a user query: <query>{query}</query>. Here is some relevant information: <information>{information}</information>. Please answer the question using the relevant information."  # noqa: E501

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/retriever.py

@@ -0,0 +1,17 @@
+from langchain.tools import tool
+from langchain_docs_retriever.retriever import get_retriever
+
+# This is used to tell the model how to best use the retriever.
+
+retriever_description = """You will be asked a question by a human user. You have access to the following tool to help answer the question. <tool_description> Search Engine Tool * The search engine will exclusively search over the LangChain documentation for pages similar to your query. It returns for each page its title and full page content. Use this tool if you want to get up-to-date and comprehensive information on a topic to help answer queries. Queries should be as atomic as possible -- they only need to address one part of the user's question. For example, if the user's query is "what is the color of a basketball?", your search query should be "basketball". Here's another example: if the user's question is "Who created the first neural network?", your first query should be "neural network". As you can see, these queries are quite short. Think keywords, not phrases. * At any time, you can make a call to the search engine using the following syntax: <search_query>query_word</search_query>. * You'll then get results back in <search_result> tags.</tool_description>"""  # noqa: E501
+
+retriever = get_retriever()
+
+# This should be the same as the function name below
+RETRIEVER_TOOL_NAME = "search"
+
+
+@tool
+def search(query, callbacks=None):
+    """Search the LangChain docs with the retriever."""
+    return retriever.get_relevant_documents(query, callbacks=callbacks)

langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/retriever_agent.py

@@ -0,0 +1,41 @@
+from langchain.agents import AgentExecutor
+from langchain.chat_models import ChatAnthropic
+from langchain.prompts import ChatPromptTemplate
+from langchain.schema.output_parser import StrOutputParser
+from langchain.schema.runnable import RunnableMap, RunnablePassthrough
+
+from .agent_scratchpad import format_agent_scratchpad
+from .output_parser import parse_output
+from .prompts import retrieval_prompt
+from .retriever import retriever_description, search
+
+prompt = ChatPromptTemplate.from_messages(
+    [
+        ("user", retrieval_prompt),
+        ("ai", "{agent_scratchpad}"),
+    ]
+)
+prompt = prompt.partial(retriever_description=retriever_description)
+
+model = ChatAnthropic(model="claude-2", temperature=0, max_tokens_to_sample=1000)
+
+chain = (
+    RunnablePassthrough.assign(
+        agent_scratchpad=lambda x: format_agent_scratchpad(x["intermediate_steps"])
+    )
+    | prompt
+    | model.bind(stop_sequences=["</search_query>"])
+    | StrOutputParser()
+)
+
+agent_chain = (
+    RunnableMap(
+        {
+            "partial_completion": chain,
+            "intermediate_steps": lambda x: x["intermediate_steps"],
+        }
+    )
+    | parse_output
+)
+
+executor = AgentExecutor(agent=agent_chain, tools=[search])

langchain-docs-benchmarking/packages/anthropic-iterative-search/main.py

@@ -0,0 +1,12 @@
+from anthropic_iterative_search import final_chain
+
+if __name__ == "__main__":
+    query = (
+        "Which movie came out first: Oppenheimer, or "
+        "Are You There God It's Me Margaret?"
+    )
+    print(
+        final_chain.with_config(configurable={"chain": "retrieve"}).invoke(
+            {"query": query}
+        )
+    )

langchain-docs-benchmarking/packages/anthropic-iterative-search/pyproject.toml

@@ -0,0 +1,22 @@
+[tool.poetry]
+name = "anthropic-iterative-search"
+version = "0.0.1"
+description = ""
+authors = []
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.8.1,<4.0"
+langchain = ">=0.0.331,<0.1.0"
+anthropic = "^0.5.0"
+wikipedia = "^1.4.0"
+
+[tool.langserve]
+export_module = "anthropic_iterative_search"
+export_attr = "final_chain"
+
+[build-system]
+requires = [
+    "poetry-core",
+]
+build-backend = "poetry.core.masonry.api"

langchain-docs-benchmarking/packages/chat-langchain/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 LangChain, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

langchain-docs-benchmarking/packages/chat-langchain/README.md

@@ -0,0 +1,66 @@
+# chat-langchain
+
+TODO: What does this package do
+
+## Environment Setup
+
+TODO: What environment variables need to be set (if any)
+
+## Usage
+
+To use this package, you should first have the LangChain CLI installed:
+
+```shell
+pip install -U "langchain-cli[serve]"
+```
+
+To create a new LangChain project and install this as the only package, you can do:
+
+```shell
+langchain app new my-app --package chat-langchain
+```
+
+If you want to add this to an existing project, you can just run:
+
+```shell
+langchain app add chat-langchain
+```
+
+And add the following code to your `server.py` file:
+```python
+from chat_langchain import chain as chat_langchain_chain
+
+add_routes(app, chat_langchain_chain, path="/chat-langchain")
+```
+
+(Optional) Let's now configure LangSmith. 
+LangSmith will help us trace, monitor and debug LangChain applications. 
+LangSmith is currently in private beta, you can sign up [here](https://smith.langchain.com/). 
+If you don't have access, you can skip this section
+
+
+```shell
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_API_KEY=<your-api-key>
+export LANGCHAIN_PROJECT=<your-project>  # if not specified, defaults to "default"
+```
+
+If you are inside this directory, then you can spin up a LangServe instance directly by:
+
+```shell
+langchain serve
+```
+
+This will start the FastAPI app with a server is running locally at 
+[http://localhost:8000](http://localhost:8000)
+
+We can see all templates at [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs)
+We can access the playground at [http://127.0.0.1:8000/chat-langchain/playground](http://127.0.0.1:8000/chat-langchain/playground)  
+
+We can access the template from code with:
+
+```python
+from langserve.client import RemoteRunnable
+
+runnable = RemoteRunnable("http://localhost:8000/chat-langchain")
+```

langchain-docs-benchmarking/packages/chat-langchain/chat_langchain/__init__.py

@@ -0,0 +1,3 @@
+from chat_langchain.chain import chain
+
+__all__ = ["chain"]

langchain-docs-benchmarking/packages/chat-langchain/chat_langchain/chain.py

@@ -0,0 +1,183 @@
+"""Chat langchain 'engine'."""
+from operator import itemgetter
+from typing import Dict, List, Optional, Sequence
+
+from langchain.chat_models import ChatAnthropic, ChatFireworks, ChatOpenAI
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder, PromptTemplate
+from langchain.schema import Document
+from langchain.schema.language_model import BaseLanguageModel
+from langchain.schema.messages import AIMessage, HumanMessage
+from langchain.schema.output_parser import StrOutputParser
+from langchain.schema.retriever import BaseRetriever
+from langchain.schema.runnable import (
+    Runnable,
+    RunnableBranch,
+    RunnableLambda,
+    RunnableMap,
+)
+from langchain_docs_retriever.retriever import get_retriever
+from pydantic import BaseModel
+
+RESPONSE_TEMPLATE = """\
+You are an expert programmer and problem-solver, tasked with answering any question \
+about Langchain.
+
+Generate a comprehensive and informative answer of 80 words or less for the \
+given question based solely on the provided search results (URL and content). You must \
+only use information from the provided search results. Use an unbiased and \
+journalistic tone. Combine search results together into a coherent answer. Do not \
+repeat text. Cite search results using [${{number}}] notation. Only cite the most \
+relevant results that answer the question accurately. Place these citations at the end \
+of the sentence or paragraph that reference them - do not put them all at the end. If \
+different results refer to different entities within the same name, write separate \
+answers for each entity.
+
+You should use bullet points in your answer for readability. Put citations where they apply
+rather than putting them all at the end.
+
+If there is nothing in the context relevant to the question at hand, just say "Hmm, \
+I'm not sure." Don't try to make up an answer.
+
+Anything between the following `context`  html blocks is retrieved from a knowledge \
+bank, not part of the conversation with the user. 
+
+<context>
+    {context} 
+<context/>
+
+REMEMBER: If there is no relevant information within the context, just say "Hmm, I'm \
+not sure." Don't try to make up an answer. Anything between the preceding 'context' \
+html blocks is retrieved from a knowledge bank, not part of the conversation with the \
+user.\
+"""
+
+REPHRASE_TEMPLATE = """\
+Given the following conversation and a follow up question, rephrase the follow up \
+question to be a standalone question.
+
+Chat History:
+{chat_history}
+Follow Up Input: {question}
+Standalone Question:"""
+
+
+class ChatRequest(BaseModel):
+    question: str
+    chat_history: Optional[List[Dict[str, str]]]
+
+
+def create_retriever_chain(
+    llm: BaseLanguageModel, retriever: BaseRetriever
+) -> Runnable:
+    CONDENSE_QUESTION_PROMPT = PromptTemplate.from_template(REPHRASE_TEMPLATE)
+    condense_question_chain = (
+        CONDENSE_QUESTION_PROMPT | llm | StrOutputParser()
+    ).with_config(
+        run_name="CondenseQuestion",
+    )
+    conversation_chain = condense_question_chain | retriever
+    return RunnableBranch(
+        (
+            RunnableLambda(lambda x: bool(x.get("chat_history"))).with_config(
+                run_name="HasChatHistoryCheck"
+            ),
+            conversation_chain.with_config(run_name="RetrievalChainWithHistory"),
+        ),
+        (
+            RunnableLambda(itemgetter("question")).with_config(
+                run_name="Itemgetter:question"
+            )
+            | retriever
+        ).with_config(run_name="RetrievalChainWithNoHistory"),
+    ).with_config(run_name="RouteDependingOnChatHistory")
+
+
+def format_docs(docs: Sequence[Document]) -> str:
+    formatted_docs = []
+    for i, doc in enumerate(docs):
+        doc_string = f"<doc id='{i}'>{doc.page_content}</doc>"
+        formatted_docs.append(doc_string)
+    return "\n".join(formatted_docs)
+
+
+def serialize_history(request: ChatRequest):
+    chat_history = request.get("chat_history") or []
+    converted_chat_history = []
+    for message in chat_history:
+        if message.get("human") is not None:
+            converted_chat_history.append(HumanMessage(content=message["human"]))
+        if message.get("ai") is not None:
+            converted_chat_history.append(AIMessage(content=message["ai"]))
+    return converted_chat_history
+
+
+def create_response_chain(
+    llm: BaseLanguageModel,
+    retriever: BaseRetriever,
+) -> Runnable:
+    retriever_chain = create_retriever_chain(
+        llm,
+        retriever,
+    ).with_config(run_name="FindDocs")
+    _context = RunnableMap(
+        {
+            "context": retriever_chain | format_docs,
+            "question": itemgetter("question"),
+            "chat_history": itemgetter("chat_history"),
+        }
+    ).with_config(run_name="RetrieveDocs")
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", RESPONSE_TEMPLATE),
+            MessagesPlaceholder(variable_name="chat_history"),
+            ("human", "{question}"),
+        ]
+    )
+
+    response_generator = (prompt | llm | StrOutputParser()).with_config(
+        run_name="GenerateResponse",
+    )
+    return (
+        {
+            "question": RunnableLambda(itemgetter("question")).with_config(
+                run_name="Itemgetter:question"
+            ),
+            "chat_history": RunnableLambda(serialize_history).with_config(
+                run_name="SerializeHistory"
+            ),
+        }
+        | _context
+        | response_generator
+    )
+
+
+llm = ChatOpenAI(
+    model="gpt-3.5-turbo-16k",
+    streaming=True,
+    temperature=0,
+)
+
+retriever = get_retriever()
+chain = create_response_chain(
+    llm,
+    retriever,
+)
+chain = chain.with_types(input_type=ChatRequest)
+
+
+def create_chain(config: dict):
+    config_copy = config.copy()
+    chat_cls_name = config_copy.pop("chat_cls", "ChatOpenAI")
+
+    assert chat_cls_name in {"ChatOpenAI", "ChatFireworks", "ChatAnthropic"}
+    chat_cls = {
+        "ChatOpenAI": ChatOpenAI,
+        "ChatFireworks": ChatFireworks,
+        "ChatAnthropic": ChatAnthropic,
+    }[chat_cls_name]
+    model = chat_cls(**config_copy)
+    retriever = get_retriever(config.get("retriever_config", {}))
+    return create_response_chain(
+        model,
+        retriever,
+    )

langchain-docs-benchmarking/packages/chat-langchain/pyproject.toml

@@ -0,0 +1,34 @@
+[tool.poetry]
+name = "chat-langchain"
+version = "0.0.1"
+description = ""
+authors = []
+readme = "README.md"
+
+[tool.poetry.dependencies]
+openai = ">1,<2"
+python = "^3.10"
+fastapi = "^0.104.1"
+pydantic = "1.10"
+langchain = ">=0.0.327,<0.1.0"
+uvicorn = "^0.23.2"
+beautifulsoup4 = "^4.12.2"
+tiktoken = "^0.4.0"
+weaviate-client = "^3.23.2"
+psycopg2 = "^2.9.7"
+lxml = "^4.9.3"
+langserve = {extras = ["server"], version = ">=0.0.21,<0.1.0"}
+anthropic = "^0.5.0"
+
+[tool.poetry.group.dev.dependencies]
+langchain-cli = ">=0.0.4"
+fastapi = "^0.104.0"
+sse-starlette = "^1.6.5"
+
+[tool.langserve]
+export_module = "chat_langchain"
+export_attr = "chain"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

langchain-docs-benchmarking/packages/example/custom_example/example_custom_chain.py

@@ -0,0 +1,32 @@
+from langchain.chat_models import ChatAnthropic, ChatOpenAI
+from langchain.prompts import ChatPromptTemplate
+from langchain.schema.output_parser import StrOutputParser
+from langchain_docs_retriever.retriever import get_retriever
+
+
+def create_runnable(config: dict):
+    config_copy = config.copy()
+    chat_cls_name = config_copy.pop("chat_cls", "ChatOpenAI")
+
+    assert chat_cls_name in {"ChatOpenAI", "ChatAnthropic"}
+    chat_cls = {
+        "ChatOpenAI": ChatOpenAI,
+        "ChatAnthropic": ChatAnthropic,
+    }[chat_cls_name]
+    model = chat_cls(**config_copy)
+    retriever = get_retriever(config.get("retriever_config", {}))
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", "Answer the Q using the following docs\n{docs}"),
+            ("user", "Q: {question}"),
+        ]
+    )
+    return (
+        {
+            "question": lambda x: x["question"],
+            "docs": (lambda x: x["question"]) | retriever,
+        }
+        | prompt
+        | model
+        | StrOutputParser()
+    )

langchain-docs-benchmarking/packages/langchain-docs-retriever/README.md

@@ -0,0 +1,5 @@
+# LangChain Docs Retriever
+
+
+A simple vector store retriever over the LangChain python docs. Indexed
+simply using [ingest_docs.py](./ingest_docs.py).

langchain-docs-benchmarking/packages/langchain-docs-retriever/ingest_docs.py

@@ -0,0 +1,242 @@
+"""Load html from files, clean up, split, ingest into Weaviate."""
+import logging
+import os
+import re
+from typing import Generator
+
+from bs4 import BeautifulSoup, Doctype, NavigableString, SoupStrainer, Tag
+from langchain.document_loaders import RecursiveUrlLoader, SitemapLoader
+from langchain.embeddings import OpenAIEmbeddings, VoyageEmbeddings
+from langchain.indexes import SQLRecordManager, index
+from langchain.schema.embeddings import Embeddings
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.utils.html import PREFIXES_TO_IGNORE_REGEX, SUFFIXES_TO_IGNORE_REGEX
+from langchain.vectorstores.chroma import Chroma
+
+logger = logging.getLogger(__name__)
+directory = os.path.dirname(os.path.realpath(__file__))
+db_directory = os.path.join(directory, "langchain_docs_retriever", "db")
+
+
+def langchain_docs_extractor(soup: BeautifulSoup) -> str:
+    # Remove all the tags that are not meaningful for the extraction.
+    SCAPE_TAGS = ["nav", "footer", "aside", "script", "style"]
+    [tag.decompose() for tag in soup.find_all(SCAPE_TAGS)]
+
+    def get_text(tag: Tag) -> Generator[str, None, None]:
+        for child in tag.children:
+            if isinstance(child, Doctype):
+                continue
+
+            if isinstance(child, NavigableString):
+                yield child
+            elif isinstance(child, Tag):
+                if child.name in ["h1", "h2", "h3", "h4", "h5", "h6"]:
+                    yield f"{'#' * int(child.name[1:])} {child.get_text()}\n\n"
+                elif child.name == "a":
+                    yield f"[{child.get_text(strip=False)}]({child.get('href')})"
+                elif child.name == "img":
+                    yield f"![{child.get('alt', '')}]({child.get('src')})"
+                elif child.name in ["strong", "b"]:
+                    yield f"**{child.get_text(strip=False)}**"
+                elif child.name in ["em", "i"]:
+                    yield f"_{child.get_text(strip=False)}_"
+                elif child.name == "br":
+                    yield "\n"
+                elif child.name == "code":
+                    parent = child.find_parent()
+                    if parent is not None and parent.name == "pre":
+                        classes = parent.attrs.get("class", "")
+
+                        language = next(
+                            filter(lambda x: re.match(r"language-\w+", x), classes),
+                            None,
+                        )
+                        if language is None:
+                            language = ""
+                        else:
+                            language = language.split("-")[1]
+
+                        lines: list[str] = []
+                        for span in child.find_all("span", class_="token-line"):
+                            line_content = "".join(
+                                token.get_text() for token in span.find_all("span")
+                            )
+                            lines.append(line_content)
+
+                        code_content = "\n".join(lines)
+                        yield f"```{language}\n{code_content}\n```\n\n"
+                    else:
+                        yield f"`{child.get_text(strip=False)}`"
+
+                elif child.name == "p":
+                    yield from get_text(child)
+                    yield "\n\n"
+                elif child.name == "ul":
+                    for li in child.find_all("li", recursive=False):
+                        yield "- "
+                        yield from get_text(li)
+                        yield "\n\n"
+                elif child.name == "ol":
+                    for i, li in enumerate(child.find_all("li", recursive=False)):
+                        yield f"{i + 1}. "
+                        yield from get_text(li)
+                        yield "\n\n"
+                elif child.name == "div" and "tabs-container" in child.attrs.get(
+                    "class", [""]
+                ):
+                    tabs = child.find_all("li", {"role": "tab"})
+                    tab_panels = child.find_all("div", {"role": "tabpanel"})
+                    for tab, tab_panel in zip(tabs, tab_panels):
+                        tab_name = tab.get_text(strip=True)
+                        yield f"{tab_name}\n"
+                        yield from get_text(tab_panel)
+                elif child.name == "table":
+                    thead = child.find("thead")
+                    header_exists = isinstance(thead, Tag)
+                    if header_exists:
+                        headers = thead.find_all("th")
+                        if headers:
+                            yield "| "
+                            yield " | ".join(header.get_text() for header in headers)
+                            yield " |\n"
+                            yield "| "
+                            yield " | ".join("----" for _ in headers)
+                            yield " |\n"
+
+                    tbody = child.find("tbody")
+                    tbody_exists = isinstance(tbody, Tag)
+                    if tbody_exists:
+                        for row in tbody.find_all("tr"):
+                            yield "| "
+                            yield " | ".join(
+                                cell.get_text(strip=True) for cell in row.find_all("td")
+                            )
+                            yield " |\n"
+
+                    yield "\n\n"
+                elif child.name in ["button"]:
+                    continue
+                else:
+                    yield from get_text(child)
+
+    joined = "".join(get_text(soup))
+    return re.sub(r"\n\n+", "\n\n", joined).strip()
+
+
+RECORD_MANAGER_DB_URL = (
+    os.environ.get("RECORD_MANAGER_DB_URL") or "sqlite:///lcdocs_oai_record_manager.sql"
+)
+
+
+def metadata_extractor(meta: dict, soup: BeautifulSoup) -> dict:
+    title = soup.find("title")
+    description = soup.find("meta", attrs={"name": "description"})
+    html = soup.find("html")
+    return {
+        "source": meta["loc"] or "",
+        "title": (title.get_text() if title else "") or "",
+        "description": description.get("content") or "" if description else "",
+        "language": html.get("lang") or "" if html else "",
+        **{k: v or "" for k, v in meta.items()},
+    }
+
+
+def load_langchain_docs():
+    return SitemapLoader(
+        "https://python.langchain.com/sitemap.xml",
+        filter_urls=["https://python.langchain.com/"],
+        parsing_function=langchain_docs_extractor,
+        default_parser="lxml",
+        bs_kwargs={
+            "parse_only": SoupStrainer(
+                name=("article", "title", "html", "lang", "content")
+            ),
+        },
+        meta_function=metadata_extractor,
+    ).load()
+
+
+def simple_extractor(html: str) -> str:
+    soup = BeautifulSoup(html, "lxml")
+    return re.sub(r"\n\n+", "\n\n", soup.text).strip()
+
+
+def load_api_docs():
+    return RecursiveUrlLoader(
+        url="https://api.python.langchain.com/en/latest/",
+        max_depth=8,
+        extractor=simple_extractor,
+        prevent_outside=True,
+        use_async=True,
+        timeout=600,
+        # Drop trailing / to avoid duplicate pages.
+        link_regex=(
+            f"href=[\"']{PREFIXES_TO_IGNORE_REGEX}((?:{SUFFIXES_TO_IGNORE_REGEX}.)*?)"
+            r"(?:[\#'\"]|\/[\#'\"])"
+        ),
+        check_response_status=True,
+        exclude_dirs=(
+            "https://api.python.langchain.com/en/latest/_sources",
+            "https://api.python.langchain.com/en/latest/_modules",
+        ),
+    ).load()
+
+
+def get_embeddings_model() -> Embeddings:
+    if os.environ.get("VOYAGE_AI_URL") and os.environ.get("VOYAGE_AI_MODEL"):
+        return VoyageEmbeddings()
+    return OpenAIEmbeddings(chunk_size=200)
+
+
+CHROMA_COLLECTION_NAME = "langchain-docs"
+
+
+def ingest_docs():
+    docs_from_documentation = load_langchain_docs()
+    logger.info(f"Loaded {len(docs_from_documentation)} docs from documentation")
+    docs_from_api = load_api_docs()
+    logger.info(f"Loaded {len(docs_from_api)} docs from API")
+
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=4000, chunk_overlap=200)
+    docs_transformed = text_splitter.split_documents(
+        docs_from_documentation + docs_from_api
+    )
+
+    # We try to return 'source' and 'title' metadata when querying vector store and
+    # Chroma will error at query time if one of the attributes is missing from a
+    # retrieved document.
+    for doc in docs_transformed:
+        if "source" not in doc.metadata:
+            doc.metadata["source"] = ""
+        if "title" not in doc.metadata:
+            doc.metadata["title"] = ""
+        for k, v in doc.metadata.items():
+            if v is None:
+                doc.metadata[k] = ""
+
+    embedding = get_embeddings_model()
+    vectorstore = Chroma(
+        collection_name=CHROMA_COLLECTION_NAME,
+        embedding_function=embedding,
+        persist_directory=db_directory,
+    )
+
+    record_manager = SQLRecordManager(
+        f"chroma/{CHROMA_COLLECTION_NAME}", db_url=RECORD_MANAGER_DB_URL
+    )
+    record_manager.create_schema()
+
+    indexing_stats = index(
+        docs_transformed,
+        record_manager,
+        vectorstore,
+        cleanup="full",
+        source_id_key="source",
+    )
+
+    logger.info("Indexing stats: ", indexing_stats)
+
+
+if __name__ == "__main__":
+    ingest_docs()

langchain-docs-benchmarking/packages/langchain-docs-retriever/langchain_docs_retriever/download_db.py

@@ -0,0 +1,34 @@
+import os
+import zipfile
+
+import requests
+
+remote_url = "https://storage.googleapis.com/benchmarks-artifacts/langchain-docs-benchmarking/chroma_db.zip"
+directory = os.path.dirname(os.path.realpath(__file__))
+db_directory = os.path.join(directory, "db")
+
+
+def is_folder_populated(folder):
+    if os.path.exists(folder):
+        return any(os.scandir(folder))
+    return False
+
+
+def download_folder_from_gcs():
+    r = requests.get(remote_url, allow_redirects=True)
+    open("chroma_db.zip", "wb").write(r.content)
+
+    with zipfile.ZipFile("chroma_db.zip", "r") as zip_ref:
+        zip_ref.extractall(directory)
+
+    os.remove("chroma_db.zip")
+
+
+def fetch_langchain_docs_db():
+    if not is_folder_populated(db_directory):
+        print(f"Folder {db_directory} is not populated. Downloading from GCS...")
+        download_folder_from_gcs()
+
+
+if __name__ == "__main__":
+    fetch_langchain_docs_db()

langchain-docs-benchmarking/packages/langchain-docs-retriever/langchain_docs_retriever/retriever.py

@@ -0,0 +1,35 @@
+import os
+from typing import Optional
+
+from langchain.embeddings import OpenAIEmbeddings
+
+# from langchain_docs_retriever.voyage import VoyageEmbeddings
+from langchain.embeddings.voyageai import VoyageEmbeddings
+from langchain.schema.embeddings import Embeddings
+from langchain.schema.retriever import BaseRetriever
+from langchain.vectorstores.chroma import Chroma
+
+from .download_db import fetch_langchain_docs_db
+
+WEAVIATE_DOCS_INDEX_NAME = "LangChain_agent_docs"
+_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
+CHROMA_COLLECTION_NAME = "langchain-docs"
+_DB_DIRECTORY = os.path.join(_DIRECTORY, "db")
+
+
+def get_embeddings_model() -> Embeddings:
+    if os.environ.get("VOYAGE_AI_MODEL"):
+        return VoyageEmbeddings(model=os.environ["VOYAGE_AI_MODEL"], max_retries=20)
+    return OpenAIEmbeddings(chunk_size=200)
+
+
+def get_retriever(search_kwargs: Optional[dict] = None) -> BaseRetriever:
+    embedding_model = get_embeddings_model()
+    fetch_langchain_docs_db()
+    vectorstore = Chroma(
+        collection_name=CHROMA_COLLECTION_NAME,
+        embedding_function=embedding_model,
+        persist_directory=_DB_DIRECTORY,
+    )
+    search_kwargs = search_kwargs or dict(k=6)
+    return vectorstore.as_retriever(search_kwargs=search_kwargs)

langchain-docs-benchmarking/packages/langchain-docs-retriever/pyproject.toml

@@ -0,0 +1,31 @@
+[tool.poetry]
+name = "langchain-docs-retriever"
+version = "0.0.1"
+description = ""
+authors = []
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+fastapi = "^0.104.1"
+pydantic = "1.10"
+langchain = ">=0.0.331,<0.1.0"
+uvicorn = "^0.23.2"
+openai = ">1,<2"
+psycopg2 = "^2.9.7"
+lxml = "^4.9.3"
+langserve = {extras = ["server"], version = ">=0.0.23,<0.1.0"}
+chromadb = "^0.4.15"
+
+[tool.poetry.group.dev.dependencies]
+langchain-cli = ">=0.0.4"
+fastapi = "^0.104.0"
+sse-starlette = "^1.6.5"
+
+[tool.langserve]
+export_module = "chat_langchain"
+export_attr = "chain"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

langchain-docs-benchmarking/packages/oai-assistant/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 LangChain, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

langchain-docs-benchmarking/packages/oai-assistant/README.md

@@ -0,0 +1,66 @@
+# oai-assistant
+
+TODO: What does this package do
+
+## Environment Setup
+
+TODO: What environment variables need to be set (if any)
+
+## Usage
+
+To use this package, you should first have the LangChain CLI installed:
+
+```shell
+pip install -U "langchain-cli[serve]"
+```
+
+To create a new LangChain project and install this as the only package, you can do:
+
+```shell
+langchain app new my-app --package oai-assistant
+```
+
+If you want to add this to an existing project, you can just run:
+
+```shell
+langchain app add oai-assistant
+```
+
+And add the following code to your `server.py` file:
+```python
+from oai_assistant import chain as oai_assistant_chain
+
+add_routes(app, oai_assistant_chain, path="/oai-assistant")
+```
+
+(Optional) Let's now configure LangSmith. 
+LangSmith will help us trace, monitor and debug LangChain applications. 
+LangSmith is currently in private beta, you can sign up [here](https://smith.langchain.com/). 
+If you don't have access, you can skip this section
+
+
+```shell
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_API_KEY=<your-api-key>
+export LANGCHAIN_PROJECT=<your-project>  # if not specified, defaults to "default"
+```
+
+If you are inside this directory, then you can spin up a LangServe instance directly by:
+
+```shell
+langchain serve
+```
+
+This will start the FastAPI app with a server is running locally at 
+[http://localhost:8000](http://localhost:8000)
+
+We can see all templates at [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs)
+We can access the playground at [http://127.0.0.1:8000/oai-assistant/playground](http://127.0.0.1:8000/oai-assistant/playground)  
+
+We can access the template from code with:
+
+```python
+from langserve.client import RemoteRunnable
+
+runnable = RemoteRunnable("http://localhost:8000/oai-assistant")
+```

langchain-docs-benchmarking/packages/oai-assistant/oai_assistant/__init__.py

@@ -0,0 +1,3 @@
+from oai_assistant.chain import agent_executor
+
+__all__ = ["agent_executor"]

langchain-docs-benchmarking/packages/oai-assistant/oai_assistant/chain.py

@@ -0,0 +1,36 @@
+import json
+
+from langchain.agents import AgentExecutor
+from langchain.tools import tool
+from langchain_docs_retriever.retriever import get_retriever
+from langchain_experimental.openai_assistant import OpenAIAssistantRunnable
+
+# This is used to tell the model how to best use the retriever.
+
+
+_RETRIEVER = get_retriever()
+
+
+@tool
+def search(query, callbacks=None) -> str:
+    """Search the LangChain docs with the retriever."""
+    docs = _RETRIEVER.get_relevant_documents(query, callbacks=callbacks)
+    return json.dumps([doc.dict() for doc in docs])
+
+
+tools = [search]
+
+agent = OpenAIAssistantRunnable.create_assistant(
+    name="langchain docs assistant",
+    instructions="You are a helpful assistant tasked with answering technical questions about LangChain.",
+    tools=tools,
+    model="gpt-4-1106-preview",
+    as_agent=True,
+)
+
+
+agent_executor = (
+    (lambda x: {"content": x["question"]})
+    | AgentExecutor(agent=agent, tools=tools)
+    | (lambda x: x["output"])
+)

langchain-docs-benchmarking/packages/oai-assistant/pyproject.toml

@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "oai-assistant"
+version = "0.0.1"
+description = ""
+authors = []
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.8.1,<4.0"
+langchain = ">=0.0.332,<0.1.0"
+openai = ">1,<2"
+langchain-experimental = "^0.0.38"
+
+[tool.poetry.group.dev.dependencies]
+langchain-cli = ">=0.0.4"
+fastapi = "^0.104.0"
+sse-starlette = "^1.6.5"
+
+[tool.langserve]
+export_module = "oai_assistant"
+export_attr = "agent_executor"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

langchain-docs-benchmarking/packages/openai-functions-agent/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 LangChain, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

langchain-docs-benchmarking/packages/openai-functions-agent/README.md

@@ -0,0 +1,72 @@
+
+# openai-functions-agent
+
+This template creates an agent that uses OpenAI function calling to communicate its decisions on what actions to take. 
+
+This example creates an agent that can optionally look up information on the internet using Tavily's search engine.
+
+## Environment Setup
+
+The following environment variables need to be set:
+
+Set the `OPENAI_API_KEY` environment variable to access the OpenAI models.
+
+Set the `TAVILY_API_KEY` environment variable to access Tavily.
+
+## Usage
+
+To use this package, you should first have the LangChain CLI installed:
+
+```shell
+pip install -U "langchain-cli[serve]"
+```
+
+To create a new LangChain project and install this as the only package, you can do:
+
+```shell
+langchain app new my-app --package openai-functions-agent
+```
+
+If you want to add this to an existing project, you can just run:
+
+```shell
+langchain app add openai-functions-agent
+```
+
+And add the following code to your `server.py` file:
+```python
+from openai_functions_agent import chain as openai_functions_agent_chain
+
+add_routes(app, openai_functions_agent_chain, path="/openai-functions-agent")
+```
+
+(Optional) Let's now configure LangSmith. 
+LangSmith will help us trace, monitor and debug LangChain applications. 
+LangSmith is currently in private beta, you can sign up [here](https://smith.langchain.com/). 
+If you don't have access, you can skip this section
+
+```shell
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_API_KEY=<your-api-key>
+export LANGCHAIN_PROJECT=<your-project>  # if not specified, defaults to "default"
+```
+
+If you are inside this directory, then you can spin up a LangServe instance directly by:
+
+```shell
+langchain serve
+```
+
+This will start the FastAPI app with a server is running locally at 
+[http://localhost:8000](http://localhost:8000)
+
+We can see all templates at [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs)
+We can access the playground at [http://127.0.0.1:8000/openai-functions-agent/playground](http://127.0.0.1:8000/openai-functions-agent/playground)  
+
+We can access the template from code with:
+
+```python
+from langserve.client import RemoteRunnable
+
+runnable = RemoteRunnable("http://localhost:8000/openai-functions-agent")
+```

langchain-docs-benchmarking/packages/openai-functions-agent/main.py

@@ -0,0 +1,5 @@
+from openai_functions_agent.agent import agent_executor
+
+if __name__ == "__main__":
+    question = "who won the womens world cup in 2023?"
+    print(agent_executor.invoke({"input": question, "chat_history": []}))

langchain-docs-benchmarking/packages/openai-functions-agent/openai_functions_agent/__init__.py

@@ -0,0 +1,3 @@
+from openai_functions_agent.agent import agent_executor
+
+__all__ = ["agent_executor"]

langchain-docs-benchmarking/packages/openai-functions-agent/openai_functions_agent/agent.py

@@ -0,0 +1,85 @@
+from typing import List, Tuple
+
+from langchain.agents import AgentExecutor
+from langchain.agents.format_scratchpad import format_to_openai_functions
+from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain.pydantic_v1 import BaseModel, Field
+from langchain.schema.messages import AIMessage, HumanMessage
+from langchain.tools import tool
+from langchain.tools.render import format_tool_to_openai_function
+from langchain_docs_retriever.retriever import get_retriever
+
+# This is used to tell the model how to best use the retriever.
+
+
+_RETRIEVER = get_retriever()
+
+
+@tool
+def search(query, callbacks=None):
+    """Search the LangChain docs with the retriever."""
+    return _RETRIEVER.get_relevant_documents(query, callbacks=callbacks)
+
+
+tools = [search]
+
+llm = ChatOpenAI(model="gpt-3.5-turbo-16k", temperature=0)
+assistant_system_message = """You are a helpful assistant tasked with answering technical questions about LangChain. \
+Use tools (only if necessary) to best answer the users questions. Do not make up information if you cannot find the answer using your tools."""
+prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", assistant_system_message),
+        MessagesPlaceholder(variable_name="chat_history"),
+        ("user", "{input}"),
+        MessagesPlaceholder(variable_name="agent_scratchpad"),
+    ]
+)
+
+llm_with_tools = llm.bind(functions=[format_tool_to_openai_function(t) for t in tools])
+
+
+def _format_chat_history(chat_history: List[Tuple[str, str]]):
+    buffer = []
+    for human, ai in chat_history:
+        buffer.append(HumanMessage(content=human))
+        buffer.append(AIMessage(content=ai))
+    return buffer
+
+
+agent = (
+    {
+        "input": lambda x: x["input"],
+        "chat_history": lambda x: _format_chat_history(x["chat_history"]),
+        "agent_scratchpad": lambda x: format_to_openai_functions(
+            x["intermediate_steps"]
+        ),
+    }
+    | prompt
+    | llm_with_tools
+    | OpenAIFunctionsAgentOutputParser()
+)
+
+
+class AgentInput(BaseModel):
+    input: str
+    chat_history: List[Tuple[str, str]] = Field(..., extra={"widget": {"type": "chat"}})
+
+
+agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=False).with_types(
+    input_type=AgentInput
+)
+
+
+class ChainInput(BaseModel):
+    question: str
+
+
+def mapper(input: dict):
+    return {"input": input["question"], "chat_history": []}
+
+
+agent_executor = (mapper | agent_executor | (lambda x: x["output"])).with_types(
+    input_type=ChainInput
+)

langchain-docs-benchmarking/packages/openai-functions-agent/pyproject.toml

@@ -0,0 +1,24 @@
+[tool.poetry]
+name = "openai-functions-agent"
+version = "0.1.0"
+description = ""
+authors = [
+    "Lance Martin <lance@langchain.dev>",
+]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.8.1,<4.0"
+langchain = ">=0.0.327,<0.1.0"
+openai = ">=0.5.0"
+tavily-python = "^0.1.9"
+
+[tool.langserve]
+export_module = "openai_functions_agent"
+export_attr = "agent_executor"
+
+[build-system]
+requires = [
+    "poetry-core",
+]
+build-backend = "poetry.core.masonry.api"

langchain-docs-benchmarking/prepare_dataset.py

@@ -0,0 +1,57 @@
+"""Copy the public dataset to your own langsmith tenant."""
+from typing import Optional
+
+from langsmith import Client
+
+DATASET_NAME = "LangChain Docs Q&A"
+PUBLIC_DATASET_TOKEN = "452ccafc-18e1-4314-885b-edd735f17b9d"
+
+
+def create_langchain_docs_dataset(
+    dataset_name: str = DATASET_NAME,
+    public_dataset_token: str = PUBLIC_DATASET_TOKEN,
+    client: Optional[Client] = None,
+):
+    shared_client = Client(
+        api_url="https://api.smith.langchain.com", api_key="placeholder"
+    )
+    examples = list(shared_client.list_shared_examples(public_dataset_token))
+    client = client or Client()
+    if client.has_dataset(dataset_name=dataset_name):
+        loaded_examples = list(client.list_examples(dataset_name=dataset_name))
+        if len(loaded_examples) == len(examples):
+            return
+        else:
+            ds = client.read_dataset(dataset_name=dataset_name)
+    else:
+        ds = client.create_dataset(dataset_name=dataset_name)
+    client.create_examples(
+        inputs=[e.inputs for e in examples],
+        outputs=[e.outputs for e in examples],
+        dataset_id=ds.id,
+    )
+    print("Done creating dataset.")
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--target-api-key", type=str, required=False)
+    parser.add_argument("--target-endpoint", type=str, required=False)
+    parser.add_argument("--dataset-name", type=str, default=DATASET_NAME)
+    parser.add_argument(
+        "--public-dataset-token", type=str, default=PUBLIC_DATASET_TOKEN
+    )
+    args = parser.parse_args()
+    client = None
+    if args.target_api_key or args.target_endpoint:
+        client = Client(
+            api_key=args.target_api_key,
+            api_url=args.target_endpoint,
+        )
+    create_langchain_docs_dataset(
+        dataset_name=args.dataset_name,
+        public_dataset_token=args.public_dataset_token,
+        client=client,
+    )

langchain-docs-benchmarking/pyproject.toml

@@ -0,0 +1,29 @@
+[tool.poetry]
+name = "langservehub-template"
+version = "0.1.0"
+description = ""
+authors = ["Your Name <you@example.com>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+langsmith = ">=0.0.64,<0.1.0"
+sse-starlette = "^1.6.5"
+tomli-w = "^1.0.0"
+uvicorn = "^0.23.2"
+fastapi = "^0.104"
+langserve = ">=0.0.16"
+chat-langchain = {path = "packages/chat-langchain", develop = true}
+langchain-docs-retriever = {path = "packages/langchain-docs-retriever", develop = true}
+anthropic-iterative-search = {path = "packages/anthropic-iterative-search", develop = true}
+oai-assistant = {path = "packages/oai-assistant", develop = true}
+openai-functions-agent = {path = "packages/openai-functions-agent", develop = true}
+
+[tool.poetry.group.dev.dependencies]
+uvicorn = "^0.23.2"
+pygithub = "^2.1.1"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

langchain-docs-benchmarking/run_evals.py

@@ -0,0 +1,120 @@
+import argparse
+import importlib.util
+import sys
+import uuid
+from functools import partial
+from typing import Callable, Optional
+
+from anthropic_iterative_search.chain import chain as anthropic_agent_chain
+from chat_langchain.chain import create_chain
+from langchain.chat_models import ChatOpenAI
+from langchain.schema.runnable import Runnable
+from langchain.smith import RunEvalConfig, run_on_dataset
+from langsmith import Client
+from oai_assistant.chain import agent_executor as openai_assistant_chain
+from openai_functions_agent import agent_executor as openai_functions_agent_chain
+
+ls_client = Client()
+
+
+def import_from_path(path_name: str):
+    func_name = "create_chain"
+    if "::" in path_name:
+        path_name, func_name = path_name.split("::")
+    spec = importlib.util.spec_from_file_location("module_name", path_name)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules["module_name"] = module
+    spec.loader.exec_module(module)
+    return getattr(module, func_name)
+
+
+def _get_chain_factory(arch: str) -> Callable:
+    _map = {
+        "chat": create_chain,
+        "anthropic-iterative-search": lambda _: anthropic_agent_chain,
+        "openai-functions-agent": lambda _: openai_functions_agent_chain,
+        "openai-assistant": lambda _: openai_assistant_chain,
+    }
+    if arch in _map:
+        return _map[arch]
+    else:
+        return import_from_path(arch)
+
+
+def create_runnable(
+    arch: str, model_config: Optional[dict], retry_config: Optional[dict] = None
+):
+    factory = _get_chain_factory(arch)
+    chain: Runnable = factory(model_config)
+    if retry_config:
+        return chain.with_retry(**retry_config)
+    return chain
+
+
+def get_eval_config():
+    accuracy_criteria = {
+        "accuracy": """
+Score 1: The answer is incorrect and unrelated to the question or reference document.
+Score 3: The answer shows slight relevance to the question or reference document but is largely incorrect.
+Score 5: The answer is partially correct but has significant errors or omissions.
+Score 7: The answer is mostly correct with minor errors or omissions, and aligns with the reference document.
+Score 10: The answer is correct, complete, and perfectly aligns with the reference document.
+
+If the reference answer contains multiple alternatives, the predicted answer must only match one of the alternatives to be considered correct.
+If the predicted answer contains additional helpful and accurate information that is not present in the reference answer, it should still be considered correct.
+"""  # noqa
+    }
+
+    eval_llm = ChatOpenAI(model="gpt-4", temperature=0.0)
+    return RunEvalConfig(
+        evaluators=[
+            RunEvalConfig.LabeledScoreString(
+                criteria=accuracy_criteria, llm=eval_llm, normalize_by=10.0
+            ),
+            # Mainly to compare with the above
+            # Suspected to be less reliable.
+            RunEvalConfig.EmbeddingDistance(),
+        ]
+    )
+
+
+def main(
+    arch: str,
+    dataset_name: str,
+    model_config: Optional[dict] = None,
+    max_concurrency: int = 5,
+    project_name: Optional[str] = None,
+    retry_config: Optional[dict] = None,
+):
+    eval_config = get_eval_config()
+    project_name = project_name or arch
+    project_name += f" {uuid.uuid4().hex[:4]}"
+    run_on_dataset(
+        client=ls_client,
+        dataset_name=dataset_name,
+        llm_or_chain_factory=partial(
+            create_runnable,
+            arch=arch,
+            model_config=model_config,
+            retry_config=retry_config,
+        ),
+        evaluation=eval_config,
+        concurrency_level=max_concurrency,
+        project_name=project_name,
+        project_metadata={"arch": arch, "model_config": model_config},
+    )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--url", type=str)
+    parser.add_argument("--dataset-name", type=str, default="Chat Langchain Pub")
+    parser.add_argument("--project-name", type=Optional[str], default=None)
+    parser.add_argument("--max-concurrency", type=int, default=5)
+    args = parser.parse_args()
+    main(
+        args.url,
+        args.dataset_name,
+        max_concurrency=args.max_concurrency,
+        project_name=args.project_name,
+    )

langchain-docs-benchmarking/run_experiments.py

@@ -0,0 +1,127 @@
+import argparse
+import json
+
+from prepare_dataset import create_langchain_docs_dataset
+from run_evals import main
+
+experiments = [
+    {
+        # "server_url": "http://localhost:1983/openai-functions-agent",
+        "arch": "openai-functions-agent",
+        "project_name": "openai-functions-agent",
+    },
+    {
+        # "server_url": "http://localhost:1983/anthropic_chat",
+        "arch": "chat",
+        "model_config": {
+            "chat_cls": "ChatAnthropic",
+            "model": "claude-2",
+            "temperature": 1.0,
+        },
+        "project_name": "anthropic-chat",
+    },
+    {
+        "arch": "chat",
+        "model_config": {
+            "chat_cls": "ChatOpenAI",
+            "model": "gpt-3.5-turbo-16k",
+        },
+        # "server_url": "http://localhost:1983/chat",
+        "project_name": "chat-gpt-3.5",
+    },
+    {
+        "arch": "chat",
+        "model_config": {
+            "chat_cls": "ChatFireworks",
+            "model": "accounts/fireworks/models/mistral-7b-instruct-4k",
+        },
+        "project_name": "mistral-7b-instruct-4k",
+    },
+    {
+        "arch": "chat",
+        "model_config": {
+            "chat_cls": "ChatFireworks",
+            "model": "accounts/fireworks/models/llama-v2-34b-code-instruct-w8a16",
+        },
+        "project_name": "llama-v2-34b-code-instruct-w8a16",
+    },
+    {
+        "arch": "chat",
+        "model_config": {
+            "chat_cls": "ChatFireworks",
+            "model": "accounts/fireworks/models/zephyr-7b-beta",
+        },
+        "project_name": "zephyr-7b-beta",
+    },
+    {
+        "arch": "chat",
+        "model_config": {
+            "chat_cls": "ChatOpenAI",
+            "model": "gpt-4",
+        },
+        "project_name": "gpt-4-chat",
+    },
+    {
+        "arch": "openai-assistant",
+        "model_config": {},
+        "project_name": "openai-assistant",
+        "max_concurrency": 2,  # Rate limit is VERY low right now.
+        "retry_config": {
+            "stop_after_attempt": 10,
+        },
+    },
+    # Not worth our time it's so bad and slow
+    {
+        # "server_url": "http://localhost:1983/anthropic_iterative_search",
+        "arch": "anthropic-iterative-search",
+        "max_concurrency": 2,
+        "project_name": "anthropic-iterative-search",
+    },
+]
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dataset-name", type=str, default="LangChain Docs Q&A")
+    parser.add_argument(
+        "--config",
+        type=str,
+        default=None,
+        nargs="*",
+        help="Path to a JSON file with experiment config."
+        " If specified, the include and exclude args are ignored",
+    )
+    parser.add_argument("--include", type=str, nargs="+", default=None)
+    parser.add_argument(
+        "--exclude",
+        type=str,
+        nargs="+",
+    )
+    args = parser.parse_args()
+    create_langchain_docs_dataset(dataset_name=args.dataset_name)
+    selected_experiments = experiments
+    if args.config:
+        selected_experiments = []
+        for config_path in args.config:
+            with open(config_path) as f:
+                selected_experiments.append(json.load(f))
+    elif args.include:
+        selected_experiments = [
+            e for e in selected_experiments if e["project_name"] in args.include
+        ]
+    to_exclude = args.exclude or []
+    if args.include and not to_exclude:
+        to_exclude = [
+            "anthropic-iterative-search",
+            "openai-assistant",
+        ]
+    if args.exclude:
+        selected_experiments = [
+            e for e in selected_experiments if e["project_name"] not in args.exclude
+        ]
+
+    for experiment in selected_experiments:
+        print("Running experiment:", experiment)
+        main(
+            **experiment,
+            dataset_name=args.dataset_name,
+        )

langchain_benchmarks/.gitignore

@@ -0,0 +1 @@
+.sql

langchain_benchmarks/__init__.py

@@ -0,0 +1,8 @@
+from langchain_benchmarks.registration import registry
+from langchain_benchmarks.utils._langsmith import (
+    clone_public_dataset,
+    download_public_dataset,
+)
+
+# Please keep this list sorted!
+__all__ = ["clone_public_dataset", "download_public_dataset", "registry"]

langchain_benchmarks/extraction/__init__.py

@@ -0,0 +1,10 @@
+from langchain_benchmarks.extraction.evaluators import get_eval_config
+from langchain_benchmarks.extraction.implementations import (
+    create_openai_function_based_extractor,
+)
+
+# Keep this sorted
+__all__ = [
+    "get_eval_config",
+    "create_openai_function_based_extractor",
+]

langchain_benchmarks/extraction/evaluators.py

@@ -0,0 +1,27 @@
+from langchain.chat_models.base import BaseChatModel
+from langchain.smith import RunEvalConfig
+
+
+def get_eval_config(eval_llm: BaseChatModel) -> RunEvalConfig:
+    """Get the evaluation configuration for the email task."""
+    return RunEvalConfig(
+        evaluators=[
+            RunEvalConfig.LabeledScoreString(
+                criteria={
+                    "accuracy": """
+    Score 1: The answer is incorrect and unrelated to the question or reference document.
+    Score 3: The answer is partially correct but has more than one omission or major errors.
+    Score 5: The answer is mostly correct but has more than one omission or major error.
+    Score 7: The answer is mostly correct but has at most one omission or major error.
+    Score 9: The answer is mostly correct with no omissions and only minor errors, and aligns with the reference document.
+    Score 10: The answer is correct, complete, and aligns with the reference document. Extra information is acceptable if it is sensible.
+
+    If the reference answer contains multiple alternatives, the predicted answer must only match one of the alternatives to be considered correct.
+    If the predicted answer contains additional helpful and accurate information that is not present in the reference answer, it should still be considered correct and not be penalized.
+    """  # noqa
+                },
+                llm=eval_llm,
+                normalize_by=10.0,
+            ),
+        ],
+    )

langchain_benchmarks/extraction/implementations.py

@@ -0,0 +1,73 @@
+"""Default implementations of LLMs that can be used for extraction."""
+from typing import Any, Dict, List, Optional, Type
+
+from langchain.chains.openai_functions import convert_to_openai_function
+from langchain.chat_models import ChatOpenAI
+from langchain.output_parsers.openai_functions import JsonOutputFunctionsParser
+from langchain.prompts import ChatPromptTemplate
+from langchain.schema.runnable import Runnable
+from langsmith.client import Client
+from pydantic import BaseModel
+
+from langchain_benchmarks.extraction.evaluators import get_eval_config
+from langchain_benchmarks.schema import ExtractionTask
+
+# PUBLIC API
+
+
+def create_openai_function_based_extractor(
+    prompt: ChatPromptTemplate,
+    llm: Runnable,
+    schema: Type[BaseModel],
+) -> Runnable[dict, dict]:
+    """Create an extraction chain that uses an LLM to extract a schema.
+
+    The underlying functionality is exclusively for LLMs that support
+    extraction using openai functions format.
+
+    Args:
+        prompt: The prompt to use for extraction.
+        llm: The LLM to use for extraction.
+        schema: The schema to extract.
+
+    Returns:
+        An llm that will extract the schema
+    """
+    openai_functions = [convert_to_openai_function(schema)]
+    llm_kwargs = {
+        "functions": openai_functions,
+        "function_call": {"name": openai_functions[0]["name"]},
+    }
+    output_parser = JsonOutputFunctionsParser()
+    extraction_chain = (
+        prompt | llm.bind(**llm_kwargs) | output_parser | (lambda x: {"output": x})
+    )
+    return extraction_chain
+
+
+def run_on_dataset(
+    task: ExtractionTask,
+    llm: Runnable,
+    *,
+    tags: Optional[List[str]] = None,
+    **kwargs: Any,
+) -> Dict[str, Any]:
+    """Run an LLM on a dataset.
+
+    Args:
+        task: The task to run on.
+        llm: The LLM to run.
+        tags: The tags to use for the run.
+        kwargs: Additional arguments to pass to the client.
+    """
+    client = Client()
+    eval_llm = ChatOpenAI(model="gpt-4", temperature=0.0, model_kwargs={"seed": 42})
+    return client.run_on_dataset(
+        dataset_name=task.name,
+        llm_or_chain_factory=create_openai_function_based_extractor(
+            task.instructions, llm, task.schema
+        ),
+        evaluation=get_eval_config(eval_llm),
+        tags=tags,
+        **kwargs,
+    )

langchain_benchmarks/extraction/tasks/email_task.py

@@ -0,0 +1,63 @@
+from enum import Enum
+from typing import List, Optional
+
+from langchain.prompts import ChatPromptTemplate
+from langchain.pydantic_v1 import BaseModel, Field
+
+from langchain_benchmarks.schema import ExtractionTask
+
+
+class ToneEnum(str, Enum):
+    """The tone of the email."""
+
+    positive = "positive"
+    negative = "negative"
+
+
+class Email(BaseModel):
+    """Relevant information about an email."""
+
+    sender: Optional[str] = Field(None, description="The sender's name, if available")
+    sender_phone_number: Optional[str] = Field(
+        None, description="The sender's phone number, if available"
+    )
+    sender_address: Optional[str] = Field(
+        None, description="The sender's address, if available"
+    )
+    action_items: List[str] = Field(
+        ..., description="A list of action items requested by the email"
+    )
+    topic: str = Field(
+        ..., description="High level description of what the email is about"
+    )
+    tone: ToneEnum = Field(..., description="The tone of the email.")
+
+
+# This is a default prompt that works for chat models.
+DEFAULT_CHAT_MODEL_PROMPT = ChatPromptTemplate.from_messages(
+    [
+        ("system", "You are an expert researcher."),
+        (
+            "human",
+            "What can you tell me about the following email? Make sure to "
+            "extract the question in the correct format. "
+            "Here is the email:\n ```\n{input}\n```",
+        ),
+    ]
+)
+
+EMAIL_EXTRACTION_TASK = ExtractionTask(
+    name="Email Extraction",
+    dataset_id="https://smith.langchain.com/public/a1742786-bde5-4f51-a1d8-e148e5251ddb/d",
+    schema=Email,
+    description="""\
+A dataset of 42 real emails deduped from a spam folder, with semantic HTML tags removed, \
+as well as a script for initial extraction and formatting of other emails from \
+an arbitrary .mbox file like the one exported by Gmail.
+
+Some additional cleanup of the data was done by hand after the initial pass.
+
+See https://github.com/jacoblee93/oss-model-extraction-evals.
+    """,
+    instructions=DEFAULT_CHAT_MODEL_PROMPT,
+)

langchain_benchmarks/rag/.gitignore

@@ -0,0 +1 @@
+*.sql