Update notebooks, model registry and make release (#131)

see release notes

.github/actions/poetry_setup/action.yml

@@ -0,0 +1,91 @@
+# An action for setting up poetry install with caching.
+# Using a custom action since the default action does not
+# take poetry install groups into account.
+# Action code from:
+# https://github.com/actions/setup-python/issues/505#issuecomment-1273013236
+name: poetry-install-with-caching
+description: Poetry install with support for caching of dependency groups.
+
+inputs:
+  python-version:
+    description: Python version, supporting MAJOR.MINOR only
+    required: true
+
+  poetry-version:
+    description: Poetry version
+    required: true
+
+  cache-key:
+    description: Cache key to use for manual handling of caching
+    required: true
+
+  working-directory:
+    description: Directory whose poetry.lock file should be cached
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/setup-python@v4
+      name: Setup python ${{ inputs.python-version }}
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - uses: actions/cache@v3
+      id: cache-bin-poetry
+      name: Cache Poetry binary - Python ${{ inputs.python-version }}
+      env:
+        SEGMENT_DOWNLOAD_TIMEOUT_MIN: "1"
+      with:
+        path: |
+          /opt/pipx/venvs/poetry
+        # This step caches the poetry installation, so make sure it's keyed on the poetry version as well.
+        key: bin-poetry-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-${{ inputs.poetry-version }}
+
+    - name: Refresh shell hashtable and fixup softlinks
+      if: steps.cache-bin-poetry.outputs.cache-hit == 'true'
+      shell: bash
+      env:
+        POETRY_VERSION: ${{ inputs.poetry-version }}
+        PYTHON_VERSION: ${{ inputs.python-version }}
+      run: |
+        set -eux
+
+        # Refresh the shell hashtable, to ensure correct `which` output.
+        hash -r
+
+        # `actions/cache@v3` doesn't always seem able to correctly unpack softlinks.
+        # Delete and recreate the softlinks pipx expects to have.
+        rm /opt/pipx/venvs/poetry/bin/python
+        cd /opt/pipx/venvs/poetry/bin
+        ln -s "$(which "python$PYTHON_VERSION")" python
+        chmod +x python
+        cd /opt/pipx_bin/
+        ln -s /opt/pipx/venvs/poetry/bin/poetry poetry
+        chmod +x poetry
+
+        # Ensure everything got set up correctly.
+        /opt/pipx/venvs/poetry/bin/python --version
+        /opt/pipx_bin/poetry --version
+
+    - name: Install poetry
+      if: steps.cache-bin-poetry.outputs.cache-hit != 'true'
+      shell: bash
+      env:
+        POETRY_VERSION: ${{ inputs.poetry-version }}
+        PYTHON_VERSION: ${{ inputs.python-version }}
+      run: pipx install "poetry==$POETRY_VERSION" --python "python$PYTHON_VERSION" --verbose
+
+    - name: Restore pip and poetry cached dependencies
+      uses: actions/cache@v3
+      env:
+        SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4"
+        WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }}
+      with:
+        path: |
+          ~/.cache/pip
+          ~/.cache/pypoetry/virtualenvs
+          ~/.cache/pypoetry/cache
+          ~/.cache/pypoetry/artifacts
+          ${{ env.WORKDIR }}/.venv
+        key: py-deps-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-poetry-${{ inputs.poetry-version }}-${{ inputs.cache-key }}-${{ hashFiles(format('{0}/**/poetry.lock', env.WORKDIR)) }}

.github/workflows/_lint.yml

@@ -0,0 +1,83 @@
+name: lint
+
+on:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+
+env:
+  POETRY_VERSION: "1.6.1"
+  WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      # This number is set "by eye": we want it to be big enough
+      # so that it's bigger than the number of commits in any reasonable PR,
+      # and also as small as possible since increasing the number makes
+      # the initial `git fetch` slower.
+      FETCH_DEPTH: 50
+    strategy:
+      matrix:
+        # Only lint on the min and max supported Python versions.
+        # It's extremely unlikely that there's a lint issue on any version in between
+        # that doesn't show up on the min or max versions.
+        #
+        # GitHub rate-limits how many jobs can be running at any one time.
+        # Starting new jobs is also relatively slow,
+        # so linting on fewer versions makes CI faster.
+        python-version:
+          - "3.8"
+          - "3.11"
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
+          cache-key: lint-with-extras
+
+      - name: Check Poetry File
+        shell: bash
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          poetry check
+
+      - name: Check lock file
+        shell: bash
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          poetry lock --check
+
+      - name: Install dependencies
+        # Also installs dev/lint/test/typing dependencies, to ensure we have
+        # type hints for as many of our libraries as possible.
+        # This helps catch errors that require dependencies to be spotted, for example:
+        # https://github.com/langchain-ai/langchain/pull/10249/files#diff-935185cd488d015f026dcd9e19616ff62863e8cde8c0bee70318d3ccbca98341
+        #
+        # If you change this configuration, make sure to change the `cache-key`
+        # in the `poetry_setup` action above to stop using the old cache.
+        # It doesn't matter how you change it, any change will cause a cache-bust.
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          poetry install --with dev,lint,test,typing
+
+      - name: Get .mypy_cache to speed up mypy
+        uses: actions/cache@v3
+        env:
+          SEGMENT_DOWNLOAD_TIMEOUT_MIN: "2"
+        with:
+          path: |
+            ${{ env.WORKDIR }}/.mypy_cache
+          key: mypy-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', env.WORKDIR)) }}
+
+      - name: Analysing the code with our lint
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          make lint

.github/workflows/_pydantic_compatibility.yml

@@ -0,0 +1,94 @@
+name: pydantic v1/v2 compatibility
+
+on:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  build:
+    timeout-minutes: 5
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+    name: Pydantic v1/v2 compatibility - Python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
+          cache-key: pydantic-cross-compat
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install
+
+      - name: Install the opposite major version of pydantic
+        # If normal tests use pydantic v1, here we'll use v2, and vice versa.
+        shell: bash
+        run: |
+          # Determine the major part of pydantic version
+          REGULAR_VERSION=$(poetry run python -c "import pydantic; print(pydantic.__version__)" | cut -d. -f1)
+
+          if [[ "$REGULAR_VERSION" == "1" ]]; then
+            PYDANTIC_DEP=">=2.1,<3"
+            TEST_WITH_VERSION="2"
+          elif [[ "$REGULAR_VERSION" == "2" ]]; then
+            PYDANTIC_DEP="<2"
+            TEST_WITH_VERSION="1"
+          else
+            echo "Unexpected pydantic major version '$REGULAR_VERSION', cannot determine which version to use for cross-compatibility test."
+            exit 1
+          fi
+
+          # Install via `pip` instead of `poetry add` to avoid changing lockfile,
+          # which would prevent caching from working: the cache would get saved
+          # to a different key than where it gets loaded from.
+          poetry run pip install "pydantic${PYDANTIC_DEP}"
+
+          # Ensure that the correct pydantic is installed now.
+          echo "Checking pydantic version... Expecting ${TEST_WITH_VERSION}"
+
+          # Determine the major part of pydantic version
+          CURRENT_VERSION=$(poetry run python -c "import pydantic; print(pydantic.__version__)" | cut -d. -f1)
+
+          # Check that the major part of pydantic version is as expected, if not
+          # raise an error
+          if [[ "$CURRENT_VERSION" != "$TEST_WITH_VERSION" ]]; then
+            echo "Error: expected pydantic version ${CURRENT_VERSION} to have been installed, but found: ${TEST_WITH_VERSION}"
+            exit 1
+          fi
+          echo "Found pydantic version ${CURRENT_VERSION}, as expected"
+      - name: Run pydantic compatibility tests
+        shell: bash
+        run: make test
+
+      - name: Ensure the tests did not create any additional files
+        shell: bash
+        run: |
+          set -eu
+
+          STATUS="$(git status)"
+          echo "$STATUS"
+
+          # grep will exit non-zero if the target message isn't found,
+          # and `set -e` above will cause the step to fail.
+          echo "$STATUS" | grep 'nothing to commit, working tree clean'

.github/workflows/_release.yml

@@ -0,0 +1,63 @@
+name: release
+
+on:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  if_release:
+    # Disallow publishing from branches that aren't `main`.
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    permissions:
+      # This permission is used for trusted publishing:
+      # https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/
+      #
+      # Trusted publishing has to also be configured on PyPI for each package:
+      # https://docs.pypi.org/trusted-publishers/adding-a-publisher/
+      id-token: write
+
+      # This permission is needed by `ncipollo/release-action` to create the GitHub release.
+      contents: write
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: "3.10"
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
+          cache-key: release
+
+      - name: Build project for distribution
+        run: poetry build
+      - name: Check Version
+        id: check-version
+        run: |
+          echo version=$(poetry version --short) >> $GITHUB_OUTPUT
+      - name: Create Release
+        uses: ncipollo/release-action@v1
+        with:
+          artifacts: "dist/*"
+          token: ${{ secrets.GITHUB_TOKEN }}
+          draft: false
+          generateReleaseNotes: true
+          tag: v${{ steps.check-version.outputs.version }}
+          commit: main
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: ${{ inputs.working-directory }}/dist/
+          verbose: true
+          print-hash: true

.github/workflows/_test.yml

@@ -0,0 +1,57 @@
+name: test
+
+on:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  build:
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+    name: Python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
+          cache-key: core
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install
+
+      - name: Run core tests
+        shell: bash
+        run: make test
+
+      - name: Ensure the tests did not create any additional files
+        shell: bash
+        run: |
+          set -eu
+
+          STATUS="$(git status)"
+          echo "$STATUS"
+
+          # grep will exit non-zero if the target message isn't found,
+          # and `set -e` above will cause the step to fail.
+          echo "$STATUS" | grep 'nothing to commit, working tree clean'

.github/workflows/ci.yml

@@ -0,0 +1,150 @@
+---
+name: Run CI Tests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    paths-ignore:
+      - 'README.md'
+  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
+
+# If another push to the same PR or branch happens while this workflow is still running,
+# cancel the earlier run in favor of the next run.
+#
+# There's no point in testing an outdated version of the code. GitHub only allows
+# a limited number of job runners to be active at the same time, so it's better to cancel
+# pointless jobs early so that more useful jobs can run sooner.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  POETRY_VERSION: "1.5.1"
+  WORKDIR: "."
+
+jobs:
+  lint:
+    uses:
+      ./.github/workflows/_lint.yml
+    with:
+      working-directory: .
+    secrets: inherit
+
+  pydantic-compatibility:
+    uses:
+      ./.github/workflows/_pydantic_compatibility.yml
+    with:
+      working-directory: .
+    secrets: inherit
+  test:
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ env.WORKDIR }}
+    strategy:
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+    name: Python ${{ matrix.python-version }} tests
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: .
+          cache-key: benchmarks-all
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running tests, installing dependencies with poetry..."
+          poetry install --with test,lint,typing,docs
+
+      - name: Run tests
+        run: make test
+
+      - name: Ensure the tests did not create any additional files
+        shell: bash
+        run: |
+          set -eu
+
+          STATUS="$(git status)"
+          echo "$STATUS"
+
+          # grep will exit non-zero if the target message isn't found,
+          # and `set -e` above will cause the step to fail.
+          echo "$STATUS" | grep 'nothing to commit, working tree clean'
+  test_docs:
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ env.WORKDIR }}
+    strategy:
+      matrix:
+        python-version:
+          - "3.11"
+    name: Documentation Build for Python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: .
+          cache-key: benchmarks-all
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running tests, installing dependencies with poetry..."
+          poetry install --with test,lint,typing,docs
+
+      - name: Test Sphinx Docs
+        shell: bash
+        run: |
+          echo "Attempting to build docs..."
+          make build_docs
+  test_datasets:
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ env.WORKDIR }}
+    strategy:
+      matrix:
+        python-version:
+          - "3.11"
+    name: Validate Public Datasets
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: .
+          cache-key: benchmarks-all
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running tests, installing dependencies with poetry..."
+          poetry install --with test,lint,typing,docs
+
+      - name: Request datasets
+        shell: bash
+        run: |
+          echo "Attempting to build docs..."
+          poetry run python -m scripts.check_datasets

.github/workflows/doc_publish.yaml

@@ -0,0 +1,44 @@
+name: Publish Docs
+on: [workflow_dispatch]
+permissions:
+    contents: write
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  docs:
+    strategy:
+      matrix:
+        python-version:
+          - "3.11"
+    runs-on: ubuntu-latest
+    name: Documentation Publish
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: .
+          cache-key: benchmarks-all
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running tests, installing dependencies with poetry..."
+          poetry install --with test,lint,typing,docs
+
+      - name: Sphinx build
+        shell: bash
+        run: |
+          make build_docs
+      - name: Publish Docs
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          publish_branch: gh-pages
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs/build
+          force_orphan: true

.github/workflows/release.yml

@@ -0,0 +1,13 @@
+---
+name: Publish Package to PyPi
+
+on:
+  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
+
+jobs:
+  release:
+    uses:
+      ./.github/workflows/_release.yml
+    with:
+      working-directory: .
+    secrets: inherit

.gitignore

@@ -0,0 +1,162 @@
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+.DS_Store

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Langchain AI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,66 @@
+.PHONY: all lint format test help
+
+# Default target executed when no arguments are given to make.
+all: help
+
+######################
+# TESTING AND COVERAGE
+######################
+
+# Define a variable for the test file path.
+TEST_FILE ?= tests/unit_tests/
+
+test:
+	poetry run pytest --disable-socket --allow-unix-socket $(TEST_FILE)
+
+test_watch:
+	poetry run ptw . -- $(TEST_FILE)
+
+build_docs:
+	# Copy README.md to docs/index.md
+	cp README.md ./docs/source/index.md
+	# Append to the table of contents the contents of the file
+	cat ./docs/source/toc.segment >> ./docs/source/index.md
+	poetry run sphinx-build "./docs/source" "./docs/build"
+
+clean_docs:
+	rm -rf ./docs/build
+
+######################
+# LINTING AND FORMATTING
+######################
+
+# Define a variable for Python and notebook files.
+lint format: PYTHON_FILES=.
+lint_diff format_diff: PYTHON_FILES=$(shell git diff --relative=. --name-only --diff-filter=d master | grep -E '\.py$$|\.ipynb$$')
+
+lint lint_diff:
+	[ "$(PYTHON_FILES)" = "" ] ||	poetry run ruff format $(PYTHON_FILES) --diff
+	# [ "$(PYTHON_FILES)" = "" ] || poetry run mypy $(PYTHON_FILES)
+
+format format_diff:
+	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff format $(PYTHON_FILES)
+	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff --select I --fix $(PYTHON_FILES)
+
+spell_check:
+	poetry run codespell --toml pyproject.toml
+
+spell_fix:
+	poetry run codespell --toml pyproject.toml -w
+
+######################
+# HELP
+######################
+
+help:
+	@echo '===================='
+	@echo '-- LINTING --'
+	@echo 'format                       - run code formatters'
+	@echo 'lint                         - run linters'
+	@echo 'spell_check                 	- run codespell on the project'
+	@echo 'spell_fix                		- run codespell on the project and fix the errors'
+	@echo '-- TESTS --'
+	@echo 'coverage                     - run unit tests and generate coverage report'
+	@echo 'test                         - run unit tests'
+	@echo 'test TEST_FILE=<test_file>   - run all tests in file'
+	@echo '-- DOCUMENTATION tasks are from the top-level Makefile --'

README.md

@@ -1,8 +1,19 @@
-# LangChain Benchmarks
+# 🦜💯 LangChain Benchmarks
 
-This repository shows how we benchmark some of our more popular chains and agents.
-The benchmarks are organized by end-to-end use cases.
-They utilize [LangSmith](https://smith.langchain.com/) heavily.
+[![Release Notes](https://img.shields.io/github/release/langchain-ai/langchain-benchmarks)](https://github.com/langchain-ai/langchain-benchmarks/releases)
+[![CI](https://github.com/langchain-ai/langchain-benchmarks/actions/workflows/ci.yml/badge.svg)](https://github.com/langchain-ai/langchain-benchmarks/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Twitter](https://img.shields.io/twitter/url/https/twitter.com/langchainai.svg?style=social&label=Follow%20%40LangChainAI)](https://twitter.com/langchainai)
+[![](https://dcbadge.vercel.app/api/server/6adMQxSpJS?compact=true&style=flat)](https://discord.gg/6adMQxSpJS)
+[![Open Issues](https://img.shields.io/github/issues-raw/langchain-ai/langchain-benchmarks)](https://github.com/langchain-ai/langchain-benchmarks/issues)
+
+
+[📖 Documentation](https://langchain-ai.github.io/langchain-benchmarks/index.html)
+
+A package to help benchmark various LLM related tasks.
+
+The benchmarks are organized by end-to-end use cases, and
+utilize [LangSmith](https://smith.langchain.com/) heavily.
 
 We have several goals in open sourcing this:
 
@@ -11,6 +22,39 @@ We have several goals in open sourcing this:
 - Showing how we evaluate each task
 - Encouraging others to benchmark their solutions on these tasks (we are always looking for better ways of doing things!)
 
-We currently include the following tasks:
-- [CSV Question Answering](csv-qa)
-- [Extraction](extraction)
+## Installation
+
+To install the packages, run the following command:
+
+```bash
+pip install -U langchain-benchmarks
+```
+
+All the benchmarks come with an associated benchmark dataset stored in [LangSmith](https://smith.langchain.com). To take advantage of the eval and debugging experience, [sign up](https://smith.langchain.com), and set your API key in your environment:
+
+```bash
+export LANGCHAIN_API_KEY=ls-...
+```
+
+## Repo Structure
+
+The package is located within [langchain_benchmarks](./langchain_benchmarks/). Check out the [docs](https://langchain-ai.github.io/langchain-benchmarks/index.html) for information on how to get starte.
+
+The other directories are legacy and may be moved in the future.
+
+
+## Archived
+
+Below are archived benchmarks that require cloning this repo to run.
+
+- [CSV Question Answering](https://github.com/langchain-ai/langchain-benchmarks/tree/main/csv-qa)
+- [Extraction](https://github.com/langchain-ai/langchain-benchmarks/tree/main/extraction)
+- [Q&A over the LangChain docs](https://github.com/langchain-ai/langchain-benchmarks/tree/main/langchain-docs-benchmarking)
+- [Meta-evaluation of 'correctness' evaluators](https://github.com/langchain-ai/langchain-benchmarks/tree/main/meta-evals)
+
+
+## Related
+
+- For cookbooks on other ways to test, debug, monitor, and improve your LLM applications, check out the [LangSmith docs](https://docs.smith.langchain.com/)
+- For information on building with LangChain, check out the [python documentation](https://python.langchain.com/docs/get_started/introduction) or [JS documentation](https://js.langchain.com/docs/get_started/introduction)
+

archived/csv-qa/custom_agent.py

@@ -1,22 +1,25 @@
-from langchain.agents import OpenAIFunctionsAgent, AgentExecutor
-from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
-from langchain.tools import PythonAstREPLTool
 import pandas as pd
+from langchain.agents import AgentExecutor, OpenAIFunctionsAgent
+from langchain.agents.agent_toolkits.conversational_retrieval.tool import (
+    create_retriever_tool,
+)
 from langchain.chat_models import ChatOpenAI
-from langsmith import Client
-from langchain.smith import RunEvalConfig, run_on_dataset
-from pydantic import BaseModel, Field
 from langchain.embeddings import OpenAIEmbeddings
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain.smith import RunEvalConfig, run_on_dataset
+from langchain.tools import PythonAstREPLTool
 from langchain.vectorstores import FAISS
-from langchain.agents.agent_toolkits.conversational_retrieval.tool import create_retriever_tool
+from langsmith import Client
+from pydantic import BaseModel, Field
 
-
-pd.set_option('display.max_rows', 20)
-pd.set_option('display.max_columns', 20)
+pd.set_option("display.max_rows", 20)
+pd.set_option("display.max_columns", 20)
 
 embedding_model = OpenAIEmbeddings()
 vectorstore = FAISS.load_local("titanic_data", embedding_model)
-retriever_tool = create_retriever_tool(vectorstore.as_retriever(), "person_name_search", "Search for a person by name")
+retriever_tool = create_retriever_tool(
+    vectorstore.as_retriever(), "person_name_search", "Search for a person by name"
+)
 
 
 TEMPLATE = """You are working with a pandas dataframe in Python. The name of the dataframe is `df`.
@@ -42,7 +45,6 @@ For example:
 """
 
 
-
 class PythonInputs(BaseModel):
     query: str = Field(description="code snippet to run")
 
@@ -51,27 +53,33 @@ if __name__ == "__main__":
     df = pd.read_csv("titanic.csv")
     template = TEMPLATE.format(dhead=df.head().to_markdown())
 
-    prompt = ChatPromptTemplate.from_messages([
-        ("system", template),
-        MessagesPlaceholder(variable_name="agent_scratchpad"),
-        ("human", "{input}")
-    ])
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", template),
+            MessagesPlaceholder(variable_name="agent_scratchpad"),
+            ("human", "{input}"),
+        ]
+    )
 
     def get_chain():
-        repl = PythonAstREPLTool(locals={"df": df}, name="python_repl",
-                                 description="Runs code and returns the output of the final line",
-                                 args_schema=PythonInputs)
+        repl = PythonAstREPLTool(
+            locals={"df": df},
+            name="python_repl",
+            description="Runs code and returns the output of the final line",
+            args_schema=PythonInputs,
+        )
         tools = [repl, retriever_tool]
-        agent = OpenAIFunctionsAgent(llm=ChatOpenAI(temperature=0, model="gpt-4"), prompt=prompt, tools=tools)
-        agent_executor = AgentExecutor(agent=agent, tools=tools, max_iterations=5, early_stopping_method="generate")
+        agent = OpenAIFunctionsAgent(
+            llm=ChatOpenAI(temperature=0, model="gpt-4"), prompt=prompt, tools=tools
+        )
+        agent_executor = AgentExecutor(
+            agent=agent, tools=tools, max_iterations=5, early_stopping_method="generate"
+        )
         return agent_executor
 
-
     client = Client()
     eval_config = RunEvalConfig(
-        evaluators=[
-            "qa"
-        ],
+        evaluators=["qa"],
     )
     chain_results = run_on_dataset(
         client,

archived/csv-qa/pandas_agent_gpt_35.py

@@ -1,9 +1,9 @@
 import pandas as pd
-from langchain.chat_models import ChatOpenAI
 from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
 from langchain.agents.agent_types import AgentType
-from langsmith import Client
+from langchain.chat_models import ChatOpenAI
 from langchain.smith import RunEvalConfig, run_on_dataset
+from langsmith import Client
 
 if __name__ == "__main__":
     df = pd.read_csv("titanic.csv")
@@ -18,20 +18,17 @@ if __name__ == "__main__":
             df,
             agent_type=AgentType.OPENAI_FUNCTIONS,
             agent_executor_kwargs=agent_executor_kwargs,
-            max_iterations=5
+            max_iterations=5,
         )
         return agent
 
-
     client = Client()
     eval_config = RunEvalConfig(
-        evaluators=[
-            "qa"
-        ],
+        evaluators=["qa"],
     )
     chain_results = run_on_dataset(
         client,
         dataset_name="Titanic CSV Data",
         llm_or_chain_factory=get_chain,
         evaluation=eval_config,
-    )
+    )

archived/csv-qa/pandas_agent_gpt_4.py

@@ -1,14 +1,13 @@
 import pandas as pd
-from langchain.chat_models import ChatOpenAI
 from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
 from langchain.agents.agent_types import AgentType
-from langsmith import Client
+from langchain.chat_models import ChatOpenAI
 from langchain.smith import RunEvalConfig, run_on_dataset
+from langsmith import Client
 
 if __name__ == "__main__":
     df = pd.read_csv("titanic.csv")
 
-
     def get_chain():
         llm = ChatOpenAI(temperature=0, model="gpt-4")
         agent_executor_kwargs = {
@@ -19,20 +18,17 @@ if __name__ == "__main__":
             df,
             agent_type=AgentType.OPENAI_FUNCTIONS,
             agent_executor_kwargs=agent_executor_kwargs,
-            max_iterations=5
+            max_iterations=5,
         )
         return agent
 
-
     client = Client()
     eval_config = RunEvalConfig(
-        evaluators=[
-            "qa"
-        ],
+        evaluators=["qa"],
     )
     chain_results = run_on_dataset(
         client,
         dataset_name="Titanic CSV Data",
         llm_or_chain_factory=get_chain,
         evaluation=eval_config,
-    )
+    )

archived/csv-qa/pandas_agent_instruct.py

@@ -0,0 +1,81 @@
+import pandas as pd
+from langchain.agents import AgentExecutor, ZeroShotAgent
+from langchain.agents.agent_toolkits.conversational_retrieval.tool import (
+    create_retriever_tool,
+)
+from langchain.embeddings import OpenAIEmbeddings
+from langchain.llms import OpenAI
+from langchain.smith import RunEvalConfig, run_on_dataset
+from langchain.tools import PythonAstREPLTool
+from langchain.vectorstores import FAISS
+from langsmith import Client
+from pydantic import BaseModel, Field
+
+pd.set_option("display.max_rows", 20)
+pd.set_option("display.max_columns", 20)
+
+embedding_model = OpenAIEmbeddings()
+vectorstore = FAISS.load_local("titanic_data", embedding_model)
+retriever_tool = create_retriever_tool(
+    vectorstore.as_retriever(), "person_name_search", "Search for a person by name"
+)
+
+
+TEMPLATE = """You are working with a pandas dataframe in Python. The name of the dataframe is `df`.
+It is important to understand the attributes of the dataframe before working with it. This is the result of running `df.head().to_markdown()`
+
+<df>
+{dhead}
+</df>
+
+You are not meant to use only these rows to answer questions - they are meant as a way of telling you about the shape and schema of the dataframe.
+You also do not have use only the information here to answer questions - you can run intermediate queries to do exporatory data analysis to give you more information as needed.
+
+You have a tool called `person_name_search` through which you can lookup a person by name and find the records corresponding to people with similar name as the query.
+You should only really use this if your search term contains a persons name. Otherwise, try to solve it with code.
+
+For example:
+
+<question>How old is Jane?</question>
+<logic>Use `person_name_search` since you can use the query `Jane`</logic>
+
+<question>Who has id 320</question>
+<logic>Use `python_repl` since even though the question is about a person, you don't know their name so you can't include it.</logic>"""
+
+
+class PythonInputs(BaseModel):
+    query: str = Field(description="code snippet to run")
+
+
+if __name__ == "__main__":
+    df = pd.read_csv("titanic.csv")
+    template = TEMPLATE.format(dhead=df.head().to_markdown())
+
+    def get_chain():
+        repl = PythonAstREPLTool(
+            locals={"df": df},
+            name="python_repl",
+            description="Runs code and returns the output of the final line",
+            args_schema=PythonInputs,
+        )
+        tools = [repl, retriever_tool]
+        agent = ZeroShotAgent.from_llm_and_tools(
+            llm=OpenAI(temperature=0, model="gpt-3.5-turbo-instruct"),
+            tools=tools,
+            prefix=template,
+        )
+        agent_executor = AgentExecutor(
+            agent=agent, tools=tools, max_iterations=5, early_stopping_method="generate"
+        )
+        return agent_executor
+
+    client = Client()
+    eval_config = RunEvalConfig(
+        evaluators=["qa"],
+    )
+    chain_results = run_on_dataset(
+        client,
+        dataset_name="Titanic CSV Data",
+        llm_or_chain_factory=get_chain,
+        evaluation=eval_config,
+    )

archived/csv-qa/pandas_ai.py

@@ -0,0 +1,45 @@
+import pandas as pd
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts import ChatPromptTemplate
+from langchain.schema.output_parser import StrOutputParser
+from langchain.smith import RunEvalConfig, run_on_dataset
+from langsmith import Client
+from pandasai import PandasAI
+
+if __name__ == "__main__":
+    df = pd.read_csv("titanic.csv")
+
+    pandas_ai = PandasAI(ChatOpenAI(temperature=0, model="gpt-4"), enable_cache=False)
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            (
+                "system",
+                "Answer the users question about some data. A data scientist will run some code and the results will be returned to you to use in your answer",
+            ),
+            ("human", "Question: {input}"),
+            ("human", "Data Scientist Result: {result}"),
+        ]
+    )
+
+    def get_chain():
+        chain = (
+            {
+                "input": lambda x: x["input_question"],
+                "result": lambda x: pandas_ai(df, prompt=x["input_question"]),
+            }
+            | prompt
+            | ChatOpenAI(temperature=0, model="gpt-4")
+            | StrOutputParser()
+        )
+        return chain
+
+    client = Client()
+    eval_config = RunEvalConfig(
+        evaluators=["qa"],
+    )
+    chain_results = run_on_dataset(
+        client,
+        dataset_name="Titanic CSV Data",
+        llm_or_chain_factory=get_chain,
+        evaluation=eval_config,
+    )

archived/csv-qa/streamlit_app.py

@@ -0,0 +1,47 @@
+import pandas as pd
+import streamlit as st
+from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
+from langchain.agents.agent_types import AgentType
+from langchain.chat_models import ChatOpenAI
+
+df = pd.read_csv("titanic.csv")
+
+
+llm = ChatOpenAI(temperature=0)
+agent = create_pandas_dataframe_agent(llm, df, agent_type=AgentType.OPENAI_FUNCTIONS)
+
+
+from langsmith import Client
+
+client = Client()
+
+
+def send_feedback(run_id, score):
+    client.create_feedback(run_id, "user_score", score=score)
+
+
+st.set_page_config(page_title="🦜🔗 Ask the CSV App")
+st.title("🦜🔗 Ask the CSV App")
+st.info(
+    "Most 'question answering' applications run over unstructured text data. But a lot of the data in the world is tabular data! This is an attempt to create an application using [LangChain](https://github.com/langchain-ai/langchain) to let you ask questions of data in tabular format. For this demo application, we will use the Titanic Dataset. Please explore it [here](https://github.com/datasciencedojo/datasets/blob/master/titanic.csv) to get a sense for what questions you can ask. Please leave feedback on well the question is answered, and we will use that improve the application!"
+)
+
+query_text = st.text_input("Enter your question:", placeholder="Who was in cabin C128?")
+# Form input and query
+result = None
+with st.form("myform", clear_on_submit=True):
+    submitted = st.form_submit_button("Submit")
+    if submitted:
+        with st.spinner("Calculating..."):
+            response = agent({"input": query_text}, include_run_info=True)
+            result = response["output"]
+            run_id = response["__run"].run_id
+if result is not None:
+    st.info(result)
+    col_blank, col_text, col1, col2 = st.columns([10, 2, 1, 1])
+    with col_text:
+        st.text("Feedback:")
+    with col1:
+        st.button("👍", on_click=send_feedback, args=(run_id, 1))
+    with col2:
+        st.button("👎", on_click=send_feedback, args=(run_id, 0))

archived/csv-qa/upload_data.py

@@ -8,5 +8,5 @@ if __name__ == "__main__":
         output_keys=["output_text"],
         name="Titanic CSV Data",
         description="QA over titanic data",
-        data_type = "kv"
+        data_type="kv",
     )

archived/extraction/streamlit_app.py

@@ -0,0 +1,79 @@
+import streamlit as st
+from langchain.chains import create_extraction_chain
+from langchain.chat_models import ChatOpenAI
+from langsmith import Client
+
+st.set_page_config(page_title="🦜🔗 Text-to-graph extraction")
+client = Client()
+
+
+def send_feedback(run_id, score):
+    client.create_feedback(run_id, "user_score", score=score)
+
+
+st.title("🦜🔗 Text-to-graph playground")
+st.info(
+    "This playground explores the use of [OpenAI functions](https://openai.com/blog/function-calling-and-other-api-updates) and [LangChain](https://github.com/langchain-ai/langchain) to build knowledge graphs from user-input text. It breaks down the user input text into knowledge graph triples of subject (primary entities or concepts in a sentence), predicate (actions or relationships that connect subjects to objects), and object (entities or concepts that interact with or are acted upon by the subjects)."
+)
+
+# Input text (optional default)
+oppenheimer_text = """'Julius Robert Oppenheimer, often known as Robert or "Oppie", is heralded as the father of the atomic bomb. Emerging from a non-practicing Jewish family in New York, he made several breakthroughs, such as the early black hole theory, before the monumental Manhattan Project. His wife, Katherine “Kitty” Oppenheimer, was a German-born woman with a complex past, including connections to the Communist Party. Oppenheimer\'s journey was beset by political adversaries, notably Lewis Strauss, chairman of the U.S. Atomic Energy Commission, and William Borden, an executive director with hawkish nuclear ambitions. These tensions culminated in the famous 1954 security hearing. Influential figures like lieutenant general Leslie Groves, who had also overseen the Pentagon\'s creation, stood by Oppenheimer\'s side, having earlier chosen him for the Manhattan Project and the Los Alamos location. Intimate relationships, like that with Jean Tatlock, a Communist and the possible muse behind the Trinity test\'s name, and colleagues like Frank, Oppenheimer\'s physicist brother, intertwined with his professional life. Scientists such as Ernest Lawrence, Edward Teller, David Hill, Richard Feynman, and Hans Bethe were some of Oppenheimer\'s contemporaries, each contributing to and contesting the atomic age\'s directions. Boris Pash\'s investigations, and the perspectives of figures like Leo Szilard, Niels Bohr, Harry Truman, and others, framed the broader sociopolitical context. Meanwhile, individuals like Robert Serber, Enrico Fermi, Albert Einstein, and Isidor Isaac Rabi, among many others, each played their parts in this narrative, from naming the atomic bombs to pivotal scientific contributions and advisory roles. All these figures, together with the backdrop of World War II, McCarthyism, and the dawn of the nuclear age, presented a complex mosaic of ambitions, loyalties, betrayals, and ideologies.oppenheimer_short.txt"""
+
+# Knowledge triplet schema
+default_schema = {
+    "properties": {
+        "subject": {"type": "string"},
+        "predicate": {"type": "string"},
+        "object": {"type": "string"},
+    },
+    "required": ["subject", "predicate", "object"],
+}
+
+# Create a text_area, set the default value to oppenheimer_text
+MAX_CHARS = 2000  # Maximum number of characters
+user_input_text = st.text_area("Enter your text (<2000 characters):", height=200)
+if len(user_input_text) > MAX_CHARS:
+    st.warning(f"Text is too long. Processing only the first {MAX_CHARS} characters")
+    user_input_text = user_input_text[:MAX_CHARS]
+
+
+# Output formatting of triples
+def json_to_markdown_table(json_list):
+    if not json_list:
+        return "No data available."
+
+    # Extract headers
+    headers = json_list[0].keys()
+    markdown_table = " | ".join(headers) + "\n"
+    markdown_table += " | ".join(["---"] * len(headers)) + "\n"
+
+    # Extract rows
+    for item in json_list:
+        row = " | ".join([str(item[header]) for header in headers])
+        markdown_table += row + "\n"
+
+    return markdown_table
+
+
+# Form input and query
+markdown_output = None
+with st.form("myform", clear_on_submit=True):
+    submitted = st.form_submit_button("Submit")
+    if submitted:
+        with st.spinner("Calculating..."):
+            llm = ChatOpenAI(temperature=0, model="gpt-3.5-turbo")
+            chain = create_extraction_chain(default_schema, llm)
+            extraction_output = chain(user_input_text, include_run_info=True)
+            markdown_output = json_to_markdown_table(extraction_output["text"])
+            run_id = extraction_output["__run"].run_id
+
+# Feeback
+if markdown_output is not None:
+    st.markdown(markdown_output)
+    col_blank, col_text, col1, col2 = st.columns([10, 2, 1, 1])
+    with col_text:
+        st.text("Feedback:")
+    with col1:
+        st.button("👍", on_click=send_feedback, args=(run_id, 1))
+    with col2:
+        st.button("👎", on_click=send_feedback, args=(run_id, 0))

archived/langchain-docs-benchmarking/README.md

@@ -0,0 +1,89 @@
+# Benchmarking on LangChain Docs
+
+This directory contains code to benchmark your cognitive architecture on the public [LangChain Q&A docs evaluation benchmark](https://smith.langchain.com/public/e1bfd348-494a-4df5-899a-7c6c09233cc4/d).
+
+To one one of the existing configurations, activate your poetry environment, configure you LangSmith API key, and run the experiments.
+
+**Note:** this will benchmark chains on a _copy_ of the dataset and will not update the public leaderboard.
+
+## Running the published experiments
+
+The following steps will let you run pre-configured experiments:
+
+### 1. Install requirements
+
+```bash
+pip install poetry
+poetry shell
+poetry install
+```
+
+### 2. Configure API keys
+
+Create a [LangSmith account](https://smith.langchain.com/) and set your API key:
+
+```bash
+export LANGCHAIN_API_KEY=ls_your-api-key
+```
+
+The various cognitive architectures implemented already use Anthropic, [Fireworks.AI](https://www.fireworks.ai/), and OpenAI. Set the required API keys:
+
+```
+export OPENAI_API_KEY=your-api-key
+export ANTHROPIC_API_KEY=your-api-key
+export FIREWORKS_API_KEY=your-api-key
+```
+
+### 3. Run Experiments
+
+To run all experiments, run:
+
+```bash
+python run_experiments.py
+```
+
+If you want to only run certain experiments in the `run_experiments.py` file, use `--include` or `--exclude`
+
+Example:
+
+```bash
+python run_experiments --include mistral-7b-instruct-4k llama-v2-34b-code-instruct-w8a16
+```
+
+## Evaluating your custom cognitive architecture
+
+You can also evaluate your own custom cognitive architecture. To do so:
+
+1. Create a python file defining your architecture:
+
+```python
+# example_custom_chain.py
+
+...
+def load_runnable(config: dict) -> "Runnable":
+    # Load based on the config provided
+    return my_chain
+```
+
+2. Call `run_experiments.py` with a custom `--config my_config.json`
+
+```js
+{
+  // This specifies the path to your custom entrypoint followed by the loader function
+  "arch": "path/to/example_custom_chain.py::load_runnable",
+  "model_config": {
+    // This is passed to load_runnable() in example_custom_chain.py()
+    "chat_cls": "ChatOpenAI",
+    "model": "gpt-4"
+  },
+  "project_name": "example-custom-code" // This is the resulting test project name
+}
+```
+
+We have provided an example in [example_custom_chain.py](./packages/example/custom_example/example_custom_chain.py), which can be run by pointing `run_experiments` to the [example_custom_config.json](./example_custom_config.json) config file:
+
+```bash
+python run_experiments.py --config ./example_custom_config.json
+```
+
+Whenever you provide 1 or more `--config` files, the `--include` and `--exclude` arguments are ignored.

archived/langchain-docs-benchmarking/app/server.py

@@ -0,0 +1,26 @@
+from chat_langchain.chain import chain
+from fastapi import FastAPI
+from langserve import add_routes
+from openai_functions_agent import agent_executor as openai_functions_agent_chain
+
+app = FastAPI()
+
+# Edit this to add the chain you want to add
+add_routes(
+    app,
+    chain,
+    path="/chat",
+    # include_callback_events=True, # TODO: Include when fixed
+)
+
+add_routes(app, openai_functions_agent_chain, path="/openai-functions-agent")
+
+
+def run_server(port: int = 1983):
+    import uvicorn
+
+    uvicorn.run(app, host="0.0.0.0", port=port)
+
+
+if __name__ == "__main__":
+    run_server()

archived/langchain-docs-benchmarking/example_custom_config.json

@@ -0,0 +1,8 @@
+{
+  "arch": "packages/example/custom_example/example_custom_chain.py::create_runnable",
+  "model_config": {
+    "chat_cls": "ChatOpenAI",
+    "model": "gpt-4"
+  },
+  "project_name": "example-custom-code"
+}

archived/langchain-docs-benchmarking/packages/anthropic-iterative-search/README.md

@@ -0,0 +1,69 @@
+
+# anthropic-iterative-search
+
+This template will create a virtual research assistant with the ability to search Wikipedia to find answers to your questions.
+
+It is heavily inspired by [this notebook](https://github.com/anthropics/anthropic-cookbook/blob/main/long_context/wikipedia-search-cookbook.ipynb).
+
+## Environment Setup
+
+Set the `ANTHROPIC_API_KEY` environment variable to access the Anthropic models.
+
+## Usage
+
+To use this package, you should first have the LangChain CLI installed:
+
+```shell
+pip install -U "langchain-cli[serve]"
+```
+
+To create a new LangChain project and install this as the only package, you can do:
+
+```shell
+langchain app new my-app --package anthropic-iterative-search
+```
+
+If you want to add this to an existing project, you can just run:
+
+```shell
+langchain app add anthropic-iterative-search
+```
+
+And add the following code to your `server.py` file:
+```python
+from anthropic_iterative_search import chain as anthropic_iterative_search_chain
+
+add_routes(app, anthropic_iterative_search_chain, path="/anthropic-iterative-search")
+```
+
+(Optional) Let's now configure LangSmith. 
+LangSmith will help us trace, monitor and debug LangChain applications. 
+LangSmith is currently in private beta, you can sign up [here](https://smith.langchain.com/). 
+If you don't have access, you can skip this section
+
+
+```shell
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_API_KEY=<your-api-key>
+export LANGCHAIN_PROJECT=<your-project>  # if not specified, defaults to "default"
+```
+
+If you are inside this directory, then you can spin up a LangServe instance directly by:
+
+```shell
+langchain serve
+```
+
+This will start the FastAPI app with a server is running locally at 
+[http://localhost:8000](http://localhost:8000)
+
+We can see all templates at [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs)
+We can access the playground at [http://127.0.0.1:8000/anthropic-iterative-search/playground](http://127.0.0.1:8000/anthropic-iterative-search/playground)  
+
+We can access the template from code with:
+
+```python
+from langserve.client import RemoteRunnable
+
+runnable = RemoteRunnable("http://localhost:8000/anthropic-iterative-search")
+```

archived/langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/__init__.py

@@ -0,0 +1,11 @@
+from langchain.schema.runnable import ConfigurableField
+
+from .chain import chain
+from .retriever_agent import executor
+
+final_chain = chain.configurable_alternatives(
+    ConfigurableField(id="chain"),
+    default_key="response",
+    # This adds a new option, with name `openai` that is equal to `ChatOpenAI()`
+    retrieve=executor,
+)

archived/langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/agent_scratchpad.py

@@ -0,0 +1,16 @@
+def _format_docs(docs):
+    result = "\n".join(
+        [
+            f'<item index="{i+1}">\n<page_content>\n{r}\n</page_content>\n</item>'
+            for i, r in enumerate(docs)
+        ]
+    )
+    return result
+
+
+def format_agent_scratchpad(intermediate_steps):
+    thoughts = ""
+    for action, observation in intermediate_steps:
+        thoughts += action.log
+        thoughts += "</search_query>" + _format_docs(observation)
+    return thoughts

archived/langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/chain.py

@@ -0,0 +1,29 @@
+from langchain.chat_models import ChatAnthropic
+from langchain.prompts import ChatPromptTemplate
+from langchain.pydantic_v1 import BaseModel
+from langchain.schema.output_parser import StrOutputParser
+from langchain.schema.runnable import RunnableLambda
+
+from .prompts import answer_prompt
+from .retriever_agent import executor
+
+prompt = ChatPromptTemplate.from_template(answer_prompt)
+
+model = ChatAnthropic(model="claude-2", temperature=0, max_tokens_to_sample=1000)
+
+chain = (
+    RunnableLambda(lambda x: {"query": x["question"]})
+    | {"query": lambda x: x["query"], "information": executor | (lambda x: x["output"])}
+    | prompt
+    | model
+    | StrOutputParser()
+)
+
+# Add typing for the inputs to be used in the playground
+
+
+class Inputs(BaseModel):
+    question: str
+
+
+chain = chain.with_types(input_type=Inputs)

archived/langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/output_parser.py

@@ -0,0 +1,37 @@
+import re
+
+from langchain.schema.agent import AgentAction, AgentFinish
+
+from .agent_scratchpad import _format_docs
+
+
+def extract_between_tags(tag: str, string: str, strip: bool = True) -> str:
+    ext_list = re.findall(f"<{tag}\s?>(.+?)</{tag}\s?>", string, re.DOTALL)
+    if strip:
+        ext_list = [e.strip() for e in ext_list]
+    if ext_list:
+        if len(ext_list) != 1:
+            raise ValueError
+        # Only return the first one
+        return ext_list[0]
+
+
+def parse_output(outputs):
+    partial_completion = outputs["partial_completion"]
+    steps = outputs["intermediate_steps"]
+    search_query = extract_between_tags(
+        "search_query", partial_completion + "</search_query>"
+    )
+    if search_query is None:
+        docs = []
+        str_output = ""
+        for action, observation in steps:
+            docs.extend(observation)
+            str_output += action.log
+            str_output += "</search_query>" + _format_docs(observation)
+        str_output += partial_completion
+        return AgentFinish({"docs": docs, "output": str_output}, log=partial_completion)
+    else:
+        return AgentAction(
+            tool="search", tool_input=search_query, log=partial_completion
+        )

archived/langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/prompts.py

@@ -0,0 +1,7 @@
+retrieval_prompt = """{retriever_description} Before beginning to research the user's question, first think for a moment inside <scratchpad> tags about what information is necessary for a well-informed answer. If the user's question is complex, you may need to decompose the query into multiple subqueries and execute them individually. Sometimes the search engine will return empty search results, or the search results may not contain the information you need. In such cases, feel free to try again with a different query. 
+
+After each call to the Search Engine Tool, reflect briefly inside <search_quality></search_quality> tags about whether you now have enough information to answer, or whether more information is needed. If you have all the relevant information, write it in <information></information> tags, WITHOUT actually answering the question. Otherwise, issue a new search.
+
+Here is the user's question: <question>{query}</question> Remind yourself to make short queries in your scratchpad as you plan out your strategy."""  # noqa: E501
+
+answer_prompt = "Here is a user query: <query>{query}</query>. Here is some relevant information: <information>{information}</information>. Please answer the question using the relevant information."  # noqa: E501

archived/langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/retriever.py

@@ -0,0 +1,17 @@
+from langchain.tools import tool
+from langchain_docs_retriever.retriever import get_retriever
+
+# This is used to tell the model how to best use the retriever.
+
+retriever_description = """You will be asked a question by a human user. You have access to the following tool to help answer the question. <tool_description> Search Engine Tool * The search engine will exclusively search over the LangChain documentation for pages similar to your query. It returns for each page its title and full page content. Use this tool if you want to get up-to-date and comprehensive information on a topic to help answer queries. Queries should be as atomic as possible -- they only need to address one part of the user's question. For example, if the user's query is "what is the color of a basketball?", your search query should be "basketball". Here's another example: if the user's question is "Who created the first neural network?", your first query should be "neural network". As you can see, these queries are quite short. Think keywords, not phrases. * At any time, you can make a call to the search engine using the following syntax: <search_query>query_word</search_query>. * You'll then get results back in <search_result> tags.</tool_description>"""  # noqa: E501
+
+retriever = get_retriever()
+
+# This should be the same as the function name below
+RETRIEVER_TOOL_NAME = "search"
+
+
+@tool
+def search(query, callbacks=None):
+    """Search the LangChain docs with the retriever."""
+    return retriever.get_relevant_documents(query, callbacks=callbacks)

archived/langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/retriever_agent.py

@@ -0,0 +1,41 @@
+from langchain.agents import AgentExecutor
+from langchain.chat_models import ChatAnthropic
+from langchain.prompts import ChatPromptTemplate
+from langchain.schema.output_parser import StrOutputParser
+from langchain.schema.runnable import RunnableMap, RunnablePassthrough
+
+from .agent_scratchpad import format_agent_scratchpad
+from .output_parser import parse_output
+from .prompts import retrieval_prompt
+from .retriever import retriever_description, search
+
+prompt = ChatPromptTemplate.from_messages(
+    [
+        ("user", retrieval_prompt),
+        ("ai", "{agent_scratchpad}"),
+    ]
+)
+prompt = prompt.partial(retriever_description=retriever_description)
+
+model = ChatAnthropic(model="claude-2", temperature=0, max_tokens_to_sample=1000)
+
+chain = (
+    RunnablePassthrough.assign(
+        agent_scratchpad=lambda x: format_agent_scratchpad(x["intermediate_steps"])
+    )
+    | prompt
+    | model.bind(stop_sequences=["</search_query>"])
+    | StrOutputParser()
+)
+
+agent_chain = (
+    RunnableMap(
+        {
+            "partial_completion": chain,
+            "intermediate_steps": lambda x: x["intermediate_steps"],
+        }
+    )
+    | parse_output
+)
+
+executor = AgentExecutor(agent=agent_chain, tools=[search])

archived/langchain-docs-benchmarking/packages/anthropic-iterative-search/main.py

@@ -0,0 +1,12 @@
+from anthropic_iterative_search import final_chain
+
+if __name__ == "__main__":
+    query = (
+        "Which movie came out first: Oppenheimer, or "
+        "Are You There God It's Me Margaret?"
+    )
+    print(
+        final_chain.with_config(configurable={"chain": "retrieve"}).invoke(
+            {"query": query}
+        )
+    )

archived/langchain-docs-benchmarking/packages/anthropic-iterative-search/pyproject.toml

@@ -0,0 +1,22 @@
+[tool.poetry]
+name = "anthropic-iterative-search"
+version = "0.0.1"
+description = ""
+authors = []
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.8.1,<4.0"
+langchain = ">=0.0.331,<0.1.0"
+anthropic = "^0.5.0"
+wikipedia = "^1.4.0"
+
+[tool.langserve]
+export_module = "anthropic_iterative_search"
+export_attr = "final_chain"
+
+[build-system]
+requires = [
+    "poetry-core",
+]
+build-backend = "poetry.core.masonry.api"

archived/langchain-docs-benchmarking/packages/chat-langchain/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 LangChain, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

archived/langchain-docs-benchmarking/packages/chat-langchain/README.md

@@ -0,0 +1,66 @@
+# chat-langchain
+
+TODO: What does this package do
+
+## Environment Setup
+
+TODO: What environment variables need to be set (if any)
+
+## Usage
+
+To use this package, you should first have the LangChain CLI installed:
+
+```shell
+pip install -U "langchain-cli[serve]"
+```
+
+To create a new LangChain project and install this as the only package, you can do:
+
+```shell
+langchain app new my-app --package chat-langchain
+```
+
+If you want to add this to an existing project, you can just run:
+
+```shell
+langchain app add chat-langchain
+```
+
+And add the following code to your `server.py` file:
+```python
+from chat_langchain import chain as chat_langchain_chain
+
+add_routes(app, chat_langchain_chain, path="/chat-langchain")
+```
+
+(Optional) Let's now configure LangSmith. 
+LangSmith will help us trace, monitor and debug LangChain applications. 
+LangSmith is currently in private beta, you can sign up [here](https://smith.langchain.com/). 
+If you don't have access, you can skip this section
+
+
+```shell
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_API_KEY=<your-api-key>
+export LANGCHAIN_PROJECT=<your-project>  # if not specified, defaults to "default"
+```
+
+If you are inside this directory, then you can spin up a LangServe instance directly by:
+
+```shell
+langchain serve
+```
+
+This will start the FastAPI app with a server is running locally at 
+[http://localhost:8000](http://localhost:8000)
+
+We can see all templates at [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs)
+We can access the playground at [http://127.0.0.1:8000/chat-langchain/playground](http://127.0.0.1:8000/chat-langchain/playground)  
+
+We can access the template from code with:
+
+```python
+from langserve.client import RemoteRunnable
+
+runnable = RemoteRunnable("http://localhost:8000/chat-langchain")
+```

archived/langchain-docs-benchmarking/packages/chat-langchain/chat_langchain/__init__.py

@@ -0,0 +1,3 @@
+from chat_langchain.chain import chain
+
+__all__ = ["chain"]

archived/langchain-docs-benchmarking/packages/chat-langchain/chat_langchain/chain.py

@@ -0,0 +1,183 @@
+"""Chat langchain 'engine'."""
+from operator import itemgetter
+from typing import Dict, List, Optional, Sequence
+
+from langchain.chat_models import ChatAnthropic, ChatFireworks, ChatOpenAI
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder, PromptTemplate
+from langchain.schema import Document
+from langchain.schema.language_model import BaseLanguageModel
+from langchain.schema.messages import AIMessage, HumanMessage
+from langchain.schema.output_parser import StrOutputParser
+from langchain.schema.retriever import BaseRetriever
+from langchain.schema.runnable import (
+    Runnable,
+    RunnableBranch,
+    RunnableLambda,
+    RunnableMap,
+)
+from langchain_docs_retriever.retriever import get_retriever
+from pydantic import BaseModel
+
+RESPONSE_TEMPLATE = """\
+You are an expert programmer and problem-solver, tasked with answering any question \
+about Langchain.
+
+Generate a comprehensive and informative answer of 80 words or less for the \
+given question based solely on the provided search results (URL and content). You must \
+only use information from the provided search results. Use an unbiased and \
+journalistic tone. Combine search results together into a coherent answer. Do not \
+repeat text. Cite search results using [${{number}}] notation. Only cite the most \
+relevant results that answer the question accurately. Place these citations at the end \
+of the sentence or paragraph that reference them - do not put them all at the end. If \
+different results refer to different entities within the same name, write separate \
+answers for each entity.
+
+You should use bullet points in your answer for readability. Put citations where they apply
+rather than putting them all at the end.
+
+If there is nothing in the context relevant to the question at hand, just say "Hmm, \
+I'm not sure." Don't try to make up an answer.
+
+Anything between the following `context`  html blocks is retrieved from a knowledge \
+bank, not part of the conversation with the user. 
+
+<context>
+    {context} 
+<context/>
+
+REMEMBER: If there is no relevant information within the context, just say "Hmm, I'm \
+not sure." Don't try to make up an answer. Anything between the preceding 'context' \
+html blocks is retrieved from a knowledge bank, not part of the conversation with the \
+user.\
+"""
+
+REPHRASE_TEMPLATE = """\
+Given the following conversation and a follow up question, rephrase the follow up \
+question to be a standalone question.
+
+Chat History:
+{chat_history}
+Follow Up Input: {question}
+Standalone Question:"""
+
+
+class ChatRequest(BaseModel):
+    question: str
+    chat_history: Optional[List[Dict[str, str]]]
+
+
+def create_retriever_chain(
+    llm: BaseLanguageModel, retriever: BaseRetriever
+) -> Runnable:
+    CONDENSE_QUESTION_PROMPT = PromptTemplate.from_template(REPHRASE_TEMPLATE)
+    condense_question_chain = (
+        CONDENSE_QUESTION_PROMPT | llm | StrOutputParser()
+    ).with_config(
+        run_name="CondenseQuestion",
+    )
+    conversation_chain = condense_question_chain | retriever
+    return RunnableBranch(
+        (
+            RunnableLambda(lambda x: bool(x.get("chat_history"))).with_config(
+                run_name="HasChatHistoryCheck"
+            ),
+            conversation_chain.with_config(run_name="RetrievalChainWithHistory"),
+        ),
+        (
+            RunnableLambda(itemgetter("question")).with_config(
+                run_name="Itemgetter:question"
+            )
+            | retriever
+        ).with_config(run_name="RetrievalChainWithNoHistory"),
+    ).with_config(run_name="RouteDependingOnChatHistory")
+
+
+def format_docs(docs: Sequence[Document]) -> str:
+    formatted_docs = []
+    for i, doc in enumerate(docs):
+        doc_string = f"<doc id='{i}'>{doc.page_content}</doc>"
+        formatted_docs.append(doc_string)
+    return "\n".join(formatted_docs)
+
+
+def serialize_history(request: ChatRequest):
+    chat_history = request.get("chat_history") or []
+    converted_chat_history = []
+    for message in chat_history:
+        if message.get("human") is not None:
+            converted_chat_history.append(HumanMessage(content=message["human"]))
+        if message.get("ai") is not None:
+            converted_chat_history.append(AIMessage(content=message["ai"]))
+    return converted_chat_history
+
+
+def create_response_chain(
+    llm: BaseLanguageModel,
+    retriever: BaseRetriever,
+) -> Runnable:
+    retriever_chain = create_retriever_chain(
+        llm,
+        retriever,
+    ).with_config(run_name="FindDocs")
+    _context = RunnableMap(
+        {
+            "context": retriever_chain | format_docs,
+            "question": itemgetter("question"),
+            "chat_history": itemgetter("chat_history"),
+        }
+    ).with_config(run_name="RetrieveDocs")
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", RESPONSE_TEMPLATE),
+            MessagesPlaceholder(variable_name="chat_history"),
+            ("human", "{question}"),
+        ]
+    )
+
+    response_generator = (prompt | llm | StrOutputParser()).with_config(
+        run_name="GenerateResponse",
+    )
+    return (
+        {
+            "question": RunnableLambda(itemgetter("question")).with_config(
+                run_name="Itemgetter:question"
+            ),
+            "chat_history": RunnableLambda(serialize_history).with_config(
+                run_name="SerializeHistory"
+            ),
+        }
+        | _context
+        | response_generator
+    )
+
+
+llm = ChatOpenAI(
+    model="gpt-3.5-turbo-16k",
+    streaming=True,
+    temperature=0,
+)
+
+retriever = get_retriever()
+chain = create_response_chain(
+    llm,
+    retriever,
+)
+chain = chain.with_types(input_type=ChatRequest)
+
+
+def create_chain(config: dict):
+    config_copy = config.copy()
+    chat_cls_name = config_copy.pop("chat_cls", "ChatOpenAI")
+
+    assert chat_cls_name in {"ChatOpenAI", "ChatFireworks", "ChatAnthropic"}
+    chat_cls = {
+        "ChatOpenAI": ChatOpenAI,
+        "ChatFireworks": ChatFireworks,
+        "ChatAnthropic": ChatAnthropic,
+    }[chat_cls_name]
+    model = chat_cls(**config_copy)
+    retriever = get_retriever(config.get("retriever_config", {}))
+    return create_response_chain(
+        model,
+        retriever,
+    )

archived/langchain-docs-benchmarking/packages/chat-langchain/pyproject.toml

@@ -0,0 +1,34 @@
+[tool.poetry]
+name = "chat-langchain"
+version = "0.0.1"
+description = ""
+authors = []
+readme = "README.md"
+
+[tool.poetry.dependencies]
+openai = ">1,<2"
+python = "^3.10"
+fastapi = "^0.104.1"
+pydantic = "1.10"
+langchain = ">=0.0.327,<0.1.0"
+uvicorn = "^0.23.2"
+beautifulsoup4 = "^4.12.2"
+tiktoken = "^0.4.0"
+weaviate-client = "^3.23.2"
+psycopg2 = "^2.9.7"
+lxml = "^4.9.3"
+langserve = {extras = ["server"], version = ">=0.0.21,<0.1.0"}
+anthropic = "^0.5.0"
+
+[tool.poetry.group.dev.dependencies]
+langchain-cli = ">=0.0.4"
+fastapi = "^0.104.0"
+sse-starlette = "^1.6.5"
+
+[tool.langserve]
+export_module = "chat_langchain"
+export_attr = "chain"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

archived/langchain-docs-benchmarking/packages/example/custom_example/example_custom_chain.py

@@ -0,0 +1,32 @@
+from langchain.chat_models import ChatAnthropic, ChatOpenAI
+from langchain.prompts import ChatPromptTemplate
+from langchain.schema.output_parser import StrOutputParser
+from langchain_docs_retriever.retriever import get_retriever
+
+
+def create_runnable(config: dict):
+    config_copy = config.copy()
+    chat_cls_name = config_copy.pop("chat_cls", "ChatOpenAI")
+
+    assert chat_cls_name in {"ChatOpenAI", "ChatAnthropic"}
+    chat_cls = {
+        "ChatOpenAI": ChatOpenAI,
+        "ChatAnthropic": ChatAnthropic,
+    }[chat_cls_name]
+    model = chat_cls(**config_copy)
+    retriever = get_retriever(config.get("retriever_config", {}))
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", "Answer the Q using the following docs\n{docs}"),
+            ("user", "Q: {question}"),
+        ]
+    )
+    return (
+        {
+            "question": lambda x: x["question"],
+            "docs": (lambda x: x["question"]) | retriever,
+        }
+        | prompt
+        | model
+        | StrOutputParser()
+    )

archived/langchain-docs-benchmarking/packages/langchain-docs-retriever/README.md

@@ -0,0 +1,5 @@
+# LangChain Docs Retriever
+
+
+A simple vector store retriever over the LangChain python docs. Indexed
+simply using [ingest_docs.py](./ingest_docs.py).

archived/langchain-docs-benchmarking/packages/langchain-docs-retriever/ingest_docs.py

@@ -0,0 +1,242 @@
+"""Load html from files, clean up, split, ingest into Weaviate."""
+import logging
+import os
+import re
+from typing import Generator
+
+from bs4 import BeautifulSoup, Doctype, NavigableString, SoupStrainer, Tag
+from langchain.document_loaders import RecursiveUrlLoader, SitemapLoader
+from langchain.embeddings import OpenAIEmbeddings, VoyageEmbeddings
+from langchain.indexes import SQLRecordManager, index
+from langchain.schema.embeddings import Embeddings
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.utils.html import PREFIXES_TO_IGNORE_REGEX, SUFFIXES_TO_IGNORE_REGEX
+from langchain.vectorstores.chroma import Chroma
+
+logger = logging.getLogger(__name__)
+directory = os.path.dirname(os.path.realpath(__file__))
+db_directory = os.path.join(directory, "langchain_docs_retriever", "db")
+
+
+def langchain_docs_extractor(soup: BeautifulSoup) -> str:
+    # Remove all the tags that are not meaningful for the extraction.
+    SCAPE_TAGS = ["nav", "footer", "aside", "script", "style"]
+    [tag.decompose() for tag in soup.find_all(SCAPE_TAGS)]
+
+    def get_text(tag: Tag) -> Generator[str, None, None]:
+        for child in tag.children:
+            if isinstance(child, Doctype):
+                continue
+
+            if isinstance(child, NavigableString):
+                yield child
+            elif isinstance(child, Tag):
+                if child.name in ["h1", "h2", "h3", "h4", "h5", "h6"]:
+                    yield f"{'#' * int(child.name[1:])} {child.get_text()}\n\n"
+                elif child.name == "a":
+                    yield f"[{child.get_text(strip=False)}]({child.get('href')})"
+                elif child.name == "img":
+                    yield f"![{child.get('alt', '')}]({child.get('src')})"
+                elif child.name in ["strong", "b"]:
+                    yield f"**{child.get_text(strip=False)}**"
+                elif child.name in ["em", "i"]:
+                    yield f"_{child.get_text(strip=False)}_"
+                elif child.name == "br":
+                    yield "\n"
+                elif child.name == "code":
+                    parent = child.find_parent()
+                    if parent is not None and parent.name == "pre":
+                        classes = parent.attrs.get("class", "")
+
+                        language = next(
+                            filter(lambda x: re.match(r"language-\w+", x), classes),
+                            None,
+                        )
+                        if language is None:
+                            language = ""
+                        else:
+                            language = language.split("-")[1]
+
+                        lines: list[str] = []
+                        for span in child.find_all("span", class_="token-line"):
+                            line_content = "".join(
+                                token.get_text() for token in span.find_all("span")
+                            )
+                            lines.append(line_content)
+
+                        code_content = "\n".join(lines)
+                        yield f"```{language}\n{code_content}\n```\n\n"
+                    else:
+                        yield f"`{child.get_text(strip=False)}`"
+
+                elif child.name == "p":
+                    yield from get_text(child)
+                    yield "\n\n"
+                elif child.name == "ul":
+                    for li in child.find_all("li", recursive=False):
+                        yield "- "
+                        yield from get_text(li)
+                        yield "\n\n"
+                elif child.name == "ol":
+                    for i, li in enumerate(child.find_all("li", recursive=False)):
+                        yield f"{i + 1}. "
+                        yield from get_text(li)
+                        yield "\n\n"
+                elif child.name == "div" and "tabs-container" in child.attrs.get(
+                    "class", [""]
+                ):
+                    tabs = child.find_all("li", {"role": "tab"})
+                    tab_panels = child.find_all("div", {"role": "tabpanel"})
+                    for tab, tab_panel in zip(tabs, tab_panels):
+                        tab_name = tab.get_text(strip=True)
+                        yield f"{tab_name}\n"
+                        yield from get_text(tab_panel)
+                elif child.name == "table":
+                    thead = child.find("thead")
+                    header_exists = isinstance(thead, Tag)
+                    if header_exists:
+                        headers = thead.find_all("th")
+                        if headers:
+                            yield "| "
+                            yield " | ".join(header.get_text() for header in headers)
+                            yield " |\n"
+                            yield "| "
+                            yield " | ".join("----" for _ in headers)
+                            yield " |\n"
+
+                    tbody = child.find("tbody")
+                    tbody_exists = isinstance(tbody, Tag)
+                    if tbody_exists:
+                        for row in tbody.find_all("tr"):
+                            yield "| "
+                            yield " | ".join(
+                                cell.get_text(strip=True) for cell in row.find_all("td")
+                            )
+                            yield " |\n"
+
+                    yield "\n\n"
+                elif child.name in ["button"]:
+                    continue
+                else:
+                    yield from get_text(child)
+
+    joined = "".join(get_text(soup))
+    return re.sub(r"\n\n+", "\n\n", joined).strip()
+
+
+RECORD_MANAGER_DB_URL = (
+    os.environ.get("RECORD_MANAGER_DB_URL") or "sqlite:///lcdocs_oai_record_manager.sql"
+)
+
+
+def metadata_extractor(meta: dict, soup: BeautifulSoup) -> dict:
+    title = soup.find("title")
+    description = soup.find("meta", attrs={"name": "description"})
+    html = soup.find("html")
+    return {
+        "source": meta["loc"] or "",
+        "title": (title.get_text() if title else "") or "",
+        "description": description.get("content") or "" if description else "",
+        "language": html.get("lang") or "" if html else "",
+        **{k: v or "" for k, v in meta.items()},
+    }
+
+
+def load_langchain_docs():
+    return SitemapLoader(
+        "https://python.langchain.com/sitemap.xml",
+        filter_urls=["https://python.langchain.com/"],
+        parsing_function=langchain_docs_extractor,
+        default_parser="lxml",
+        bs_kwargs={
+            "parse_only": SoupStrainer(
+                name=("article", "title", "html", "lang", "content")
+            ),
+        },
+        meta_function=metadata_extractor,
+    ).load()
+
+
+def simple_extractor(html: str) -> str:
+    soup = BeautifulSoup(html, "lxml")
+    return re.sub(r"\n\n+", "\n\n", soup.text).strip()
+
+
+def load_api_docs():
+    return RecursiveUrlLoader(
+        url="https://api.python.langchain.com/en/latest/",
+        max_depth=8,
+        extractor=simple_extractor,
+        prevent_outside=True,
+        use_async=True,
+        timeout=600,
+        # Drop trailing / to avoid duplicate pages.
+        link_regex=(
+            f"href=[\"']{PREFIXES_TO_IGNORE_REGEX}((?:{SUFFIXES_TO_IGNORE_REGEX}.)*?)"
+            r"(?:[\#'\"]|\/[\#'\"])"
+        ),
+        check_response_status=True,
+        exclude_dirs=(
+            "https://api.python.langchain.com/en/latest/_sources",
+            "https://api.python.langchain.com/en/latest/_modules",
+        ),
+    ).load()
+
+
+def get_embeddings_model() -> Embeddings:
+    if os.environ.get("VOYAGE_AI_URL") and os.environ.get("VOYAGE_AI_MODEL"):
+        return VoyageEmbeddings()
+    return OpenAIEmbeddings(chunk_size=200)
+
+
+CHROMA_COLLECTION_NAME = "langchain-docs"
+
+
+def ingest_docs():
+    docs_from_documentation = load_langchain_docs()
+    logger.info(f"Loaded {len(docs_from_documentation)} docs from documentation")
+    docs_from_api = load_api_docs()
+    logger.info(f"Loaded {len(docs_from_api)} docs from API")
+
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=4000, chunk_overlap=200)
+    docs_transformed = text_splitter.split_documents(
+        docs_from_documentation + docs_from_api
+    )
+
+    # We try to return 'source' and 'title' metadata when querying vector store and
+    # Chroma will error at query time if one of the attributes is missing from a
+    # retrieved document.
+    for doc in docs_transformed:
+        if "source" not in doc.metadata:
+            doc.metadata["source"] = ""
+        if "title" not in doc.metadata:
+            doc.metadata["title"] = ""
+        for k, v in doc.metadata.items():
+            if v is None:
+                doc.metadata[k] = ""
+
+    embedding = get_embeddings_model()
+    vectorstore = Chroma(
+        collection_name=CHROMA_COLLECTION_NAME,
+        embedding_function=embedding,
+        persist_directory=db_directory,
+    )
+
+    record_manager = SQLRecordManager(
+        f"chroma/{CHROMA_COLLECTION_NAME}", db_url=RECORD_MANAGER_DB_URL
+    )
+    record_manager.create_schema()
+
+    indexing_stats = index(
+        docs_transformed,
+        record_manager,
+        vectorstore,
+        cleanup="full",
+        source_id_key="source",
+    )
+
+    logger.info("Indexing stats: ", indexing_stats)
+
+
+if __name__ == "__main__":
+    ingest_docs()

archived/langchain-docs-benchmarking/packages/langchain-docs-retriever/langchain_docs_retriever/download_db.py

@@ -0,0 +1,34 @@
+import os
+import zipfile
+
+import requests
+
+remote_url = "https://storage.googleapis.com/benchmarks-artifacts/langchain-docs-benchmarking/chroma_db.zip"
+directory = os.path.dirname(os.path.realpath(__file__))
+db_directory = os.path.join(directory, "db")
+
+
+def is_folder_populated(folder):
+    if os.path.exists(folder):
+        return any(os.scandir(folder))
+    return False
+
+
+def download_folder_from_gcs():
+    r = requests.get(remote_url, allow_redirects=True)
+    open("chroma_db.zip", "wb").write(r.content)
+
+    with zipfile.ZipFile("chroma_db.zip", "r") as zip_ref:
+        zip_ref.extractall(directory)
+
+    os.remove("chroma_db.zip")
+
+
+def fetch_langchain_docs_db():
+    if not is_folder_populated(db_directory):
+        print(f"Folder {db_directory} is not populated. Downloading from GCS...")
+        download_folder_from_gcs()
+
+
+if __name__ == "__main__":
+    fetch_langchain_docs_db()

archived/langchain-docs-benchmarking/packages/langchain-docs-retriever/langchain_docs_retriever/retriever.py

@@ -0,0 +1,35 @@
+import os
+from typing import Optional
+
+from langchain.embeddings import OpenAIEmbeddings
+
+# from langchain_docs_retriever.voyage import VoyageEmbeddings
+from langchain.embeddings.voyageai import VoyageEmbeddings
+from langchain.schema.embeddings import Embeddings
+from langchain.schema.retriever import BaseRetriever
+from langchain.vectorstores.chroma import Chroma
+
+from .download_db import fetch_langchain_docs_db
+
+WEAVIATE_DOCS_INDEX_NAME = "LangChain_agent_docs"
+_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
+CHROMA_COLLECTION_NAME = "langchain-docs"
+_DB_DIRECTORY = os.path.join(_DIRECTORY, "db")
+
+
+def get_embeddings_model() -> Embeddings:
+    if os.environ.get("VOYAGE_AI_MODEL"):
+        return VoyageEmbeddings(model=os.environ["VOYAGE_AI_MODEL"], max_retries=20)
+    return OpenAIEmbeddings(chunk_size=200)
+
+
+def get_retriever(search_kwargs: Optional[dict] = None) -> BaseRetriever:
+    embedding_model = get_embeddings_model()
+    fetch_langchain_docs_db()
+    vectorstore = Chroma(
+        collection_name=CHROMA_COLLECTION_NAME,
+        embedding_function=embedding_model,
+        persist_directory=_DB_DIRECTORY,
+    )
+    search_kwargs = search_kwargs or dict(k=6)
+    return vectorstore.as_retriever(search_kwargs=search_kwargs)

archived/langchain-docs-benchmarking/packages/langchain-docs-retriever/pyproject.toml

@@ -0,0 +1,31 @@
+[tool.poetry]
+name = "langchain-docs-retriever"
+version = "0.0.1"
+description = ""
+authors = []
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+fastapi = "^0.104.1"
+pydantic = "1.10"
+langchain = ">=0.0.331,<0.1.0"
+uvicorn = "^0.23.2"
+openai = ">1,<2"
+psycopg2 = "^2.9.7"
+lxml = "^4.9.3"
+langserve = {extras = ["server"], version = ">=0.0.23,<0.1.0"}
+chromadb = "^0.4.15"
+
+[tool.poetry.group.dev.dependencies]
+langchain-cli = ">=0.0.4"
+fastapi = "^0.104.0"
+sse-starlette = "^1.6.5"
+
+[tool.langserve]
+export_module = "chat_langchain"
+export_attr = "chain"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

archived/langchain-docs-benchmarking/packages/oai-assistant/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 LangChain, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

archived/langchain-docs-benchmarking/packages/oai-assistant/README.md

@@ -0,0 +1,66 @@
+# oai-assistant
+
+TODO: What does this package do
+
+## Environment Setup
+
+TODO: What environment variables need to be set (if any)
+
+## Usage
+
+To use this package, you should first have the LangChain CLI installed:
+
+```shell
+pip install -U "langchain-cli[serve]"
+```
+
+To create a new LangChain project and install this as the only package, you can do:
+
+```shell
+langchain app new my-app --package oai-assistant
+```
+
+If you want to add this to an existing project, you can just run:
+
+```shell
+langchain app add oai-assistant
+```
+
+And add the following code to your `server.py` file:
+```python
+from oai_assistant import chain as oai_assistant_chain
+
+add_routes(app, oai_assistant_chain, path="/oai-assistant")
+```
+
+(Optional) Let's now configure LangSmith. 
+LangSmith will help us trace, monitor and debug LangChain applications. 
+LangSmith is currently in private beta, you can sign up [here](https://smith.langchain.com/). 
+If you don't have access, you can skip this section
+
+
+```shell
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_API_KEY=<your-api-key>
+export LANGCHAIN_PROJECT=<your-project>  # if not specified, defaults to "default"
+```
+
+If you are inside this directory, then you can spin up a LangServe instance directly by:
+
+```shell
+langchain serve
+```
+
+This will start the FastAPI app with a server is running locally at 
+[http://localhost:8000](http://localhost:8000)
+
+We can see all templates at [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs)
+We can access the playground at [http://127.0.0.1:8000/oai-assistant/playground](http://127.0.0.1:8000/oai-assistant/playground)  
+
+We can access the template from code with:
+
+```python
+from langserve.client import RemoteRunnable
+
+runnable = RemoteRunnable("http://localhost:8000/oai-assistant")
+```

archived/langchain-docs-benchmarking/packages/oai-assistant/oai_assistant/__init__.py

@@ -0,0 +1,3 @@
+from oai_assistant.chain import agent_executor
+
+__all__ = ["agent_executor"]

archived/langchain-docs-benchmarking/packages/oai-assistant/oai_assistant/chain.py

@@ -0,0 +1,36 @@
+import json
+
+from langchain.agents import AgentExecutor
+from langchain.tools import tool
+from langchain_docs_retriever.retriever import get_retriever
+from langchain_experimental.openai_assistant import OpenAIAssistantRunnable
+
+# This is used to tell the model how to best use the retriever.
+
+
+_RETRIEVER = get_retriever()
+
+
+@tool
+def search(query, callbacks=None) -> str:
+    """Search the LangChain docs with the retriever."""
+    docs = _RETRIEVER.get_relevant_documents(query, callbacks=callbacks)
+    return json.dumps([doc.dict() for doc in docs])
+
+
+tools = [search]
+
+agent = OpenAIAssistantRunnable.create_assistant(
+    name="langchain docs assistant",
+    instructions="You are a helpful assistant tasked with answering technical questions about LangChain.",
+    tools=tools,
+    model="gpt-4-1106-preview",
+    as_agent=True,
+)
+
+
+agent_executor = (
+    (lambda x: {"content": x["question"]})
+    | AgentExecutor(agent=agent, tools=tools)
+    | (lambda x: x["output"])
+)

archived/langchain-docs-benchmarking/packages/oai-assistant/pyproject.toml

@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "oai-assistant"
+version = "0.0.1"
+description = ""
+authors = []
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.8.1,<4.0"
+langchain = ">=0.0.332,<0.1.0"
+openai = ">1,<2"
+langchain-experimental = "^0.0.38"
+
+[tool.poetry.group.dev.dependencies]
+langchain-cli = ">=0.0.4"
+fastapi = "^0.104.0"
+sse-starlette = "^1.6.5"
+
+[tool.langserve]
+export_module = "oai_assistant"
+export_attr = "agent_executor"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

archived/langchain-docs-benchmarking/packages/openai-functions-agent/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 LangChain, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

archived/langchain-docs-benchmarking/packages/openai-functions-agent/README.md

@@ -0,0 +1,72 @@
+
+# openai-functions-agent
+
+This template creates an agent that uses OpenAI function calling to communicate its decisions on what actions to take. 
+
+This example creates an agent that can optionally look up information on the internet using Tavily's search engine.
+
+## Environment Setup
+
+The following environment variables need to be set:
+
+Set the `OPENAI_API_KEY` environment variable to access the OpenAI models.
+
+Set the `TAVILY_API_KEY` environment variable to access Tavily.
+
+## Usage
+
+To use this package, you should first have the LangChain CLI installed:
+
+```shell
+pip install -U "langchain-cli[serve]"
+```
+
+To create a new LangChain project and install this as the only package, you can do:
+
+```shell
+langchain app new my-app --package openai-functions-agent
+```
+
+If you want to add this to an existing project, you can just run:
+
+```shell
+langchain app add openai-functions-agent
+```
+
+And add the following code to your `server.py` file:
+```python
+from openai_functions_agent import chain as openai_functions_agent_chain
+
+add_routes(app, openai_functions_agent_chain, path="/openai-functions-agent")
+```
+
+(Optional) Let's now configure LangSmith. 
+LangSmith will help us trace, monitor and debug LangChain applications. 
+LangSmith is currently in private beta, you can sign up [here](https://smith.langchain.com/). 
+If you don't have access, you can skip this section
+
+```shell
+export LANGCHAIN_TRACING_V2=true
+export LANGCHAIN_API_KEY=<your-api-key>
+export LANGCHAIN_PROJECT=<your-project>  # if not specified, defaults to "default"
+```
+
+If you are inside this directory, then you can spin up a LangServe instance directly by:
+
+```shell
+langchain serve
+```
+
+This will start the FastAPI app with a server is running locally at 
+[http://localhost:8000](http://localhost:8000)
+
+We can see all templates at [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs)
+We can access the playground at [http://127.0.0.1:8000/openai-functions-agent/playground](http://127.0.0.1:8000/openai-functions-agent/playground)  
+
+We can access the template from code with:
+
+```python
+from langserve.client import RemoteRunnable
+
+runnable = RemoteRunnable("http://localhost:8000/openai-functions-agent")
+```

archived/langchain-docs-benchmarking/packages/openai-functions-agent/main.py

@@ -0,0 +1,5 @@
+from openai_functions_agent.agent import agent_executor
+
+if __name__ == "__main__":
+    question = "who won the womens world cup in 2023?"
+    print(agent_executor.invoke({"input": question, "chat_history": []}))

archived/langchain-docs-benchmarking/packages/openai-functions-agent/openai_functions_agent/__init__.py

@@ -0,0 +1,3 @@
+from openai_functions_agent.agent import agent_executor
+
+__all__ = ["agent_executor"]

archived/langchain-docs-benchmarking/packages/openai-functions-agent/openai_functions_agent/agent.py

@@ -0,0 +1,85 @@
+from typing import List, Tuple
+
+from langchain.agents import AgentExecutor
+from langchain.agents.format_scratchpad import format_to_openai_functions
+from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain.pydantic_v1 import BaseModel, Field
+from langchain.schema.messages import AIMessage, HumanMessage
+from langchain.tools import tool
+from langchain.tools.render import format_tool_to_openai_function
+from langchain_docs_retriever.retriever import get_retriever
+
+# This is used to tell the model how to best use the retriever.
+
+
+_RETRIEVER = get_retriever()
+
+
+@tool
+def search(query, callbacks=None):
+    """Search the LangChain docs with the retriever."""
+    return _RETRIEVER.get_relevant_documents(query, callbacks=callbacks)
+
+
+tools = [search]
+
+llm = ChatOpenAI(model="gpt-3.5-turbo-16k", temperature=0)
+assistant_system_message = """You are a helpful assistant tasked with answering technical questions about LangChain. \
+Use tools (only if necessary) to best answer the users questions. Do not make up information if you cannot find the answer using your tools."""
+prompt = ChatPromptTemplate.from_messages(
+    [
+        ("system", assistant_system_message),
+        MessagesPlaceholder(variable_name="chat_history"),
+        ("user", "{input}"),
+        MessagesPlaceholder(variable_name="agent_scratchpad"),
+    ]
+)
+
+llm_with_tools = llm.bind(functions=[format_tool_to_openai_function(t) for t in tools])
+
+
+def _format_chat_history(chat_history: List[Tuple[str, str]]):
+    buffer = []
+    for human, ai in chat_history:
+        buffer.append(HumanMessage(content=human))
+        buffer.append(AIMessage(content=ai))
+    return buffer
+
+
+agent = (
+    {
+        "input": lambda x: x["input"],
+        "chat_history": lambda x: _format_chat_history(x["chat_history"]),
+        "agent_scratchpad": lambda x: format_to_openai_functions(
+            x["intermediate_steps"]
+        ),
+    }
+    | prompt
+    | llm_with_tools
+    | OpenAIFunctionsAgentOutputParser()
+)
+
+
+class AgentInput(BaseModel):
+    input: str
+    chat_history: List[Tuple[str, str]] = Field(..., extra={"widget": {"type": "chat"}})
+
+
+agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=False).with_types(
+    input_type=AgentInput
+)
+
+
+class ChainInput(BaseModel):
+    question: str
+
+
+def mapper(input: dict):
+    return {"input": input["question"], "chat_history": []}
+
+
+agent_executor = (mapper | agent_executor | (lambda x: x["output"])).with_types(
+    input_type=ChainInput
+)

archived/langchain-docs-benchmarking/packages/openai-functions-agent/pyproject.toml

@@ -0,0 +1,24 @@
+[tool.poetry]
+name = "openai-functions-agent"
+version = "0.1.0"
+description = ""
+authors = [
+    "Lance Martin <lance@langchain.dev>",
+]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = ">=3.8.1,<4.0"
+langchain = ">=0.0.327,<0.1.0"
+openai = ">=0.5.0"
+tavily-python = "^0.1.9"
+
+[tool.langserve]
+export_module = "openai_functions_agent"
+export_attr = "agent_executor"
+
+[build-system]
+requires = [
+    "poetry-core",
+]
+build-backend = "poetry.core.masonry.api"

archived/langchain-docs-benchmarking/prepare_dataset.py

@@ -0,0 +1,57 @@
+"""Copy the public dataset to your own langsmith tenant."""
+from typing import Optional
+
+from langsmith import Client
+
+DATASET_NAME = "LangChain Docs Q&A"
+PUBLIC_DATASET_TOKEN = "452ccafc-18e1-4314-885b-edd735f17b9d"
+
+
+def create_langchain_docs_dataset(
+    dataset_name: str = DATASET_NAME,
+    public_dataset_token: str = PUBLIC_DATASET_TOKEN,
+    client: Optional[Client] = None,
+):
+    shared_client = Client(
+        api_url="https://api.smith.langchain.com", api_key="placeholder"
+    )
+    examples = list(shared_client.list_shared_examples(public_dataset_token))
+    client = client or Client()
+    if client.has_dataset(dataset_name=dataset_name):
+        loaded_examples = list(client.list_examples(dataset_name=dataset_name))
+        if len(loaded_examples) == len(examples):
+            return
+        else:
+            ds = client.read_dataset(dataset_name=dataset_name)
+    else:
+        ds = client.create_dataset(dataset_name=dataset_name)
+    client.create_examples(
+        inputs=[e.inputs for e in examples],
+        outputs=[e.outputs for e in examples],
+        dataset_id=ds.id,
+    )
+    print("Done creating dataset.")
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--target-api-key", type=str, required=False)
+    parser.add_argument("--target-endpoint", type=str, required=False)
+    parser.add_argument("--dataset-name", type=str, default=DATASET_NAME)
+    parser.add_argument(
+        "--public-dataset-token", type=str, default=PUBLIC_DATASET_TOKEN
+    )
+    args = parser.parse_args()
+    client = None
+    if args.target_api_key or args.target_endpoint:
+        client = Client(
+            api_key=args.target_api_key,
+            api_url=args.target_endpoint,
+        )
+    create_langchain_docs_dataset(
+        dataset_name=args.dataset_name,
+        public_dataset_token=args.public_dataset_token,
+        client=client,
+    )

archived/langchain-docs-benchmarking/pyproject.toml

@@ -0,0 +1,29 @@
+[tool.poetry]
+name = "langservehub-template"
+version = "0.1.0"
+description = ""
+authors = ["Your Name <you@example.com>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+langsmith = ">=0.0.64,<0.1.0"
+sse-starlette = "^1.6.5"
+tomli-w = "^1.0.0"
+uvicorn = "^0.23.2"
+fastapi = "^0.104"
+langserve = ">=0.0.16"
+chat-langchain = {path = "packages/chat-langchain", develop = true}
+langchain-docs-retriever = {path = "packages/langchain-docs-retriever", develop = true}
+anthropic-iterative-search = {path = "packages/anthropic-iterative-search", develop = true}
+oai-assistant = {path = "packages/oai-assistant", develop = true}
+openai-functions-agent = {path = "packages/openai-functions-agent", develop = true}
+
+[tool.poetry.group.dev.dependencies]
+uvicorn = "^0.23.2"
+pygithub = "^2.1.1"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

archived/langchain-docs-benchmarking/run_evals.py

@@ -0,0 +1,120 @@
+import argparse
+import importlib.util
+import sys
+import uuid
+from functools import partial
+from typing import Callable, Optional
+
+from anthropic_iterative_search.chain import chain as anthropic_agent_chain
+from chat_langchain.chain import create_chain
+from langchain.chat_models import ChatOpenAI
+from langchain.schema.runnable import Runnable
+from langchain.smith import RunEvalConfig, run_on_dataset
+from langsmith import Client
+from oai_assistant.chain import agent_executor as openai_assistant_chain
+from openai_functions_agent import agent_executor as openai_functions_agent_chain
+
+ls_client = Client()
+
+
+def import_from_path(path_name: str):
+    func_name = "create_chain"
+    if "::" in path_name:
+        path_name, func_name = path_name.split("::")
+    spec = importlib.util.spec_from_file_location("module_name", path_name)
+    module = importlib.util.module_from_spec(spec)
+    sys.modules["module_name"] = module
+    spec.loader.exec_module(module)
+    return getattr(module, func_name)
+
+
+def _get_chain_factory(arch: str) -> Callable:
+    _map = {
+        "chat": create_chain,
+        "anthropic-iterative-search": lambda _: anthropic_agent_chain,
+        "openai-functions-agent": lambda _: openai_functions_agent_chain,
+        "openai-assistant": lambda _: openai_assistant_chain,
+    }
+    if arch in _map:
+        return _map[arch]
+    else:
+        return import_from_path(arch)
+
+
+def create_runnable(
+    arch: str, model_config: Optional[dict], retry_config: Optional[dict] = None
+):
+    factory = _get_chain_factory(arch)
+    chain: Runnable = factory(model_config)
+    if retry_config:
+        return chain.with_retry(**retry_config)
+    return chain
+
+
+def get_eval_config():
+    accuracy_criteria = {
+        "accuracy": """
+Score 1: The answer is incorrect and unrelated to the question or reference document.
+Score 3: The answer shows slight relevance to the question or reference document but is largely incorrect.
+Score 5: The answer is partially correct but has significant errors or omissions.
+Score 7: The answer is mostly correct with minor errors or omissions, and aligns with the reference document.
+Score 10: The answer is correct, complete, and perfectly aligns with the reference document.
+
+If the reference answer contains multiple alternatives, the predicted answer must only match one of the alternatives to be considered correct.
+If the predicted answer contains additional helpful and accurate information that is not present in the reference answer, it should still be considered correct.
+"""  # noqa
+    }
+
+    eval_llm = ChatOpenAI(model="gpt-4", temperature=0.0)
+    return RunEvalConfig(
+        evaluators=[
+            RunEvalConfig.LabeledScoreString(
+                criteria=accuracy_criteria, llm=eval_llm, normalize_by=10.0
+            ),
+            # Mainly to compare with the above
+            # Suspected to be less reliable.
+            RunEvalConfig.EmbeddingDistance(),
+        ]
+    )
+
+
+def main(
+    arch: str,
+    dataset_name: str,
+    model_config: Optional[dict] = None,
+    max_concurrency: int = 5,
+    project_name: Optional[str] = None,
+    retry_config: Optional[dict] = None,
+):
+    eval_config = get_eval_config()
+    project_name = project_name or arch
+    project_name += f" {uuid.uuid4().hex[:4]}"
+    run_on_dataset(
+        client=ls_client,
+        dataset_name=dataset_name,
+        llm_or_chain_factory=partial(
+            create_runnable,
+            arch=arch,
+            model_config=model_config,
+            retry_config=retry_config,
+        ),
+        evaluation=eval_config,
+        concurrency_level=max_concurrency,
+        project_name=project_name,
+        project_metadata={"arch": arch, "model_config": model_config},
+    )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--url", type=str)
+    parser.add_argument("--dataset-name", type=str, default="Chat Langchain Pub")
+    parser.add_argument("--project-name", type=Optional[str], default=None)
+    parser.add_argument("--max-concurrency", type=int, default=5)
+    args = parser.parse_args()
+    main(
+        args.url,
+        args.dataset_name,
+        max_concurrency=args.max_concurrency,
+        project_name=args.project_name,
+    )

archived/langchain-docs-benchmarking/run_experiments.py

@@ -0,0 +1,127 @@
+import argparse
+import json
+
+from prepare_dataset import create_langchain_docs_dataset
+from run_evals import main
+
+experiments = [
+    {
+        # "server_url": "http://localhost:1983/openai-functions-agent",
+        "arch": "openai-functions-agent",
+        "project_name": "openai-functions-agent",
+    },
+    {
+        # "server_url": "http://localhost:1983/anthropic_chat",
+        "arch": "chat",
+        "model_config": {
+            "chat_cls": "ChatAnthropic",
+            "model": "claude-2",
+            "temperature": 1.0,
+        },
+        "project_name": "anthropic-chat",
+    },
+    {
+        "arch": "chat",
+        "model_config": {
+            "chat_cls": "ChatOpenAI",
+            "model": "gpt-3.5-turbo-16k",
+        },
+        # "server_url": "http://localhost:1983/chat",
+        "project_name": "chat-gpt-3.5",
+    },
+    {
+        "arch": "chat",
+        "model_config": {
+            "chat_cls": "ChatFireworks",
+            "model": "accounts/fireworks/models/mistral-7b-instruct-4k",
+        },
+        "project_name": "mistral-7b-instruct-4k",
+    },
+    {
+        "arch": "chat",
+        "model_config": {
+            "chat_cls": "ChatFireworks",
+            "model": "accounts/fireworks/models/llama-v2-34b-code-instruct-w8a16",
+        },
+        "project_name": "llama-v2-34b-code-instruct-w8a16",
+    },
+    {
+        "arch": "chat",
+        "model_config": {
+            "chat_cls": "ChatFireworks",
+            "model": "accounts/fireworks/models/zephyr-7b-beta",
+        },
+        "project_name": "zephyr-7b-beta",
+    },
+    {
+        "arch": "chat",
+        "model_config": {
+            "chat_cls": "ChatOpenAI",
+            "model": "gpt-4",
+        },
+        "project_name": "gpt-4-chat",
+    },
+    {
+        "arch": "openai-assistant",
+        "model_config": {},
+        "project_name": "openai-assistant",
+        "max_concurrency": 2,  # Rate limit is VERY low right now.
+        "retry_config": {
+            "stop_after_attempt": 10,
+        },
+    },
+    # Not worth our time it's so bad and slow
+    {
+        # "server_url": "http://localhost:1983/anthropic_iterative_search",
+        "arch": "anthropic-iterative-search",
+        "max_concurrency": 2,
+        "project_name": "anthropic-iterative-search",
+    },
+]
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--dataset-name", type=str, default="LangChain Docs Q&A")
+    parser.add_argument(
+        "--config",
+        type=str,
+        default=None,
+        nargs="*",
+        help="Path to a JSON file with experiment config."
+        " If specified, the include and exclude args are ignored",
+    )
+    parser.add_argument("--include", type=str, nargs="+", default=None)
+    parser.add_argument(
+        "--exclude",
+        type=str,
+        nargs="+",
+    )
+    args = parser.parse_args()
+    create_langchain_docs_dataset(dataset_name=args.dataset_name)
+    selected_experiments = experiments
+    if args.config:
+        selected_experiments = []
+        for config_path in args.config:
+            with open(config_path) as f:
+                selected_experiments.append(json.load(f))
+    elif args.include:
+        selected_experiments = [
+            e for e in selected_experiments if e["project_name"] in args.include
+        ]
+    to_exclude = args.exclude or []
+    if args.include and not to_exclude:
+        to_exclude = [
+            "anthropic-iterative-search",
+            "openai-assistant",
+        ]
+    if args.exclude:
+        selected_experiments = [
+            e for e in selected_experiments if e["project_name"] not in args.exclude
+        ]
+
+    for experiment in selected_experiments:
+        print("Running experiment:", experiment)
+        main(
+            **experiment,
+            dataset_name=args.dataset_name,
+        )

archived/meta-evals/README.md

@@ -0,0 +1,4 @@
+# Meta-Evaluations
+
+
+This folder holds some scripts/tests for evaluating some of LangChain's default evaluators.

archived/meta-evals/correctness/README.md

@@ -0,0 +1,17 @@
+# Correctness Meta-Evals
+
+This folder contains a test script to check the aggregate performance of the "correctness"-related evaluators.
+
+To upload the dataset to LangSmith, run:
+
+```bash
+python meta-evals/correctness/_upload_dataset.py
+```
+
+To test, run:
+
+```bash
+pytest --capture=no meta-evals/correctness/test_correctness_evaluator.py
+```
+
+Then navigate to the Web Q&A dataset to review the results.

archived/meta-evals/correctness/_upload_dataset.py

@@ -0,0 +1,36 @@
+import json
+import logging
+from pathlib import Path
+
+from langsmith import Client
+
+logging.basicConfig(level=logging.INFO)
+
+# Synthetic dataset adapted from https://aclanthology.org/D13-1160/
+
+_DATA_REPO = Path(__file__).parent / "data"
+_CLIENT = Client()
+
+
+def _upload_dataset(path: str):
+    with open(path, "r") as f:
+        data = json.load(f)
+        dataset_name = data["name"]
+        examples = data["examples"]
+        try:
+            dataset = _CLIENT.create_dataset(dataset_name)
+        except Exception:
+            logging.warning(f"Skipping {dataset_name}")
+            return
+        logging.info(f"Uploading dataset: {dataset_name}")
+        _CLIENT.create_examples(
+            inputs=[example["inputs"] for example in examples],
+            outputs=[example["outputs"] for example in examples],
+            dataset_id=dataset.id,
+        )
+
+
+if __name__ == "__main__":
+    for dataset in _DATA_REPO.glob("*.json"):
+        print("Uploading dataset:", dataset)
+        _upload_dataset(dataset)

archived/meta-evals/correctness/data/Web-Q&A-Dataset-Correct.json

@@ -0,0 +1,805 @@
+{
+  "name": "Web Q&A Dataset - Correct",
+  "examples": [
+    {
+      "inputs": {
+        "input": "who does the author stephenie meyer play in the movie twilight?",
+        "input_answer": "The answer to the question 'who does the author stephenie meyer play in the movie twilight?' is Woman at Wedding",
+        "input_prediction": "Woman at Wedding"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where is jefferson davis buried?",
+        "input_answer": "The answer to the question 'where is jefferson davis buried?' is Hollywood Cemetery",
+        "input_prediction": "Hollywood Cemetery"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where do the ny rangers play?",
+        "input_answer": "The answer to the question 'where do the ny rangers play?' is Madison Square Garden",
+        "input_prediction": "Madison Square Garden"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who is robert downey jr wife?",
+        "input_answer": "The answer to the question 'who is robert downey jr wife?' is Susan Downey",
+        "input_prediction": "Susan Downey"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what kind of money should i take to jamaica?",
+        "input_answer": "The answer to the question 'what kind of money should i take to jamaica?' is Jamaican dollar",
+        "input_prediction": "Jamaican dollar"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who founded the roanoke settlement?",
+        "input_answer": "The answer to the question 'who founded the roanoke settlement?' is walter raleigh",
+        "input_prediction": "Sir Walter Raleigh"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is charles darwin famous for?",
+        "input_answer": "The answer to the question 'what is charles darwin famous for?' is Evolution",
+        "input_prediction": "Evolution"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who miley cyrus engaged to?",
+        "input_answer": "The answer to the question 'who miley cyrus engaged to?' is Liam Hemsworth",
+        "input_prediction": "Liam Hemsworth"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what sea does the yangtze river flow into?",
+        "input_answer": "The answer to the question 'what sea does the yangtze river flow into?' is East China Sea",
+        "input_prediction": "East China Sea"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where did timothy mcveigh go to prison?",
+        "input_answer": "The answer to the question 'where did timothy mcveigh go to prison?' is Terre Haute",
+        "input_prediction": "Terre Haute"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "which part of korea is communist?",
+        "input_answer": "The answer to the question 'which part of korea is communist?' is North Korea",
+        "input_prediction": "North Korea"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where is santa clara university?",
+        "input_answer": "The answer to the question 'where is santa clara university?' is Santa Clara, California",
+        "input_prediction": "Santa Clara, California, USA"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what language do the speak in pakistan?",
+        "input_answer": "The answer to the question 'what language do they speak in pakistan?' is English and Urdu",
+        "input_prediction": "English and Urdu are the official languages. Other languages include punjabi and pashto"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where is jack daniels factory?",
+        "input_answer": "The answer to the question 'where is jack daniels factory?' is Tennessee",
+        "input_prediction": "Tennessee"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what was woodrow wilson's major accomplishments?",
+        "input_answer": "The answer to the question 'what was woodrow wilson's major accomplishments?' is being the United States President and starting the league of nations",
+        "input_prediction": "Starting the league of nations as President of the USA"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is the official language of china 2010?",
+        "input_answer": "The answer to the question 'what is the official language of china 2010?' is Chinese language",
+        "input_prediction": "Chinese language"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "when did the new york knicks win a championship?",
+        "input_answer": "The answer to the question 'when did the new york knicks win a championship?' is 1970 NBA Finals",
+        "input_prediction": "1970 NBA Finals"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what time zone am i in michigan?",
+        "input_answer": "The answer to the question 'what time zone am i in michigan?' is North American Eastern Time Zone",
+        "input_prediction": "North American Eastern Time Zone"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where to get married in jacksonville nc?",
+        "input_answer": "One nearby place to try is Atlantic Beach",
+        "input_prediction": "Atlantic Beach is a great weddling location you can check out"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who was vincent van gogh inspired by?",
+        "input_answer": "The answer to the question 'who was vincent van gogh inspired by?' is Jozef Isra\u00ebls",
+        "input_prediction": "Jozef Isra\u00ebls"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where did france colonise?",
+        "input_answer": "The answer to the question 'where did france colonise?' is R\u00e9union",
+        "input_prediction": "R\u00e9union"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what school did joan miro attend?",
+        "input_answer": "The answer to the question 'what school did joan miro attend?' is Cercle Art\u00edstic de Sant Lluc",
+        "input_prediction": "Cercle Art\u00edstic de Sant Lluc"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where to go in london england?",
+        "input_answer": "The answer to the question 'where to go in london england?' is Regent's Park",
+        "input_prediction": "Regent's Park"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what college did matt lauer attend?",
+        "input_answer": "The answer to the question 'what college did matt lauer attend?' is Ohio University",
+        "input_prediction": "Ohio University"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who was peter parker's first girlfriend?",
+        "input_answer": "The answer to the question 'who was peter parker's first girlfriend?' is Gwen",
+        "input_prediction": "gwen"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "when did the colorado rockies go to the world series?",
+        "input_answer": "The answer to the question 'when did the colorado rockies go to the world series?' is 2007 ",
+        "input_prediction": "The rockies went to the world series in 2007, where they also won the National League Championship Series"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what form of government was practiced in sparta?",
+        "input_answer": "The answer to the question 'what form of government was practiced in sparta?' is Monarchy",
+        "input_prediction": "Monarchy"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where does jackie french live?",
+        "input_answer": "The answer to the question 'where does jackie french live?' is Australia",
+        "input_prediction": "Australia"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what character did brian austin green play on 90210?",
+        "input_answer": "The answer to the question 'what character did brian austin green play on 90210?' is David Silver",
+        "input_prediction": "David Silver"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what timezone is denver co?",
+        "input_answer": "The answer to the question 'what timezone is denver co?' is Mountain Time Zone",
+        "input_prediction": "Mountain Time Zone"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what tv show did joey lawrence play on?",
+        "input_answer": "The answer to the question 'what tv show did joey lawrence play on?' is Blossom",
+        "input_prediction": "Blossom"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "when is summer in australia sydney?",
+        "input_answer": "The answer to the question 'when is summer in australia sydney?' is from November to March",
+        "input_prediction": "Nov-March"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what style of music did louis armstrong play?",
+        "input_answer": "The answer to the question 'what style of music did louis armstrong play?' is Jazz",
+        "input_prediction": "Jazz"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who does maggie grace play in taken?",
+        "input_answer": "The answer to the question 'who does maggie grace play in taken?' is Kim",
+        "input_prediction": "Kim"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what timezone is australia gmt?",
+        "input_answer": "The answer to the question 'what timezone is australia gmt?' is UTC+8",
+        "input_prediction": "UTC+8"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what was tupac name in juice?",
+        "input_answer": "The answer to the question 'what was tupac name in juice?' is Bishop",
+        "input_prediction": "Bishop"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who is in the band bush?",
+        "input_answer": "The answer to the question 'who is in the band bush?' is Dave Parsons",
+        "input_prediction": "Dave Parsons"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "name one of the names of walt disney kids?",
+        "input_answer": "The answer to the question 'name of of the names of walt disney kids?' is Diane Disney Miller",
+        "input_prediction": "Diane Disney Miller is one of Walt Disney's children"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "Which county is Flemburg, PA in?",
+        "input_answer": "The answer to the question 'Which county is Flemburg, PA in?' is Cumberland County",
+        "input_prediction": "Cumberland"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "Name a country where people speak portuguese?",
+        "input_answer": "The answer to the question is Brazil",
+        "input_prediction": "Brazil"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where does lena river begin?",
+        "input_answer": "The answer to the question 'where does lena river begin?' is Baikal Mountains",
+        "input_prediction": "Baikal Mountains"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what type of car does Charmandr Beaton drive?",
+        "input_answer": "The answer to the question 'what type of car does Charmandr Beaton drive?' is Wishcraft",
+        "input_prediction": "Wishcraft is the type of car"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is the first book sherlock holmes appeared in?",
+        "input_answer": "The answer to the question 'what is the first book sherlock holmes appeared in?' is A Study in Scarlet",
+        "input_prediction": "A Study in Scarlet"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what did sally field get in math class in 4th grade?",
+        "input_answer": "The answer to the question 'what did sally field get in math class in 4th grade?' is a B+",
+        "input_prediction": "Sally got a B+"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who was queen victoria's father called?",
+        "input_answer": "The answer to the question 'who was queen victoria's father called?' is Prince Edward, Duke of Kent and Strathearn",
+        "input_prediction": "Prince Edward, Duke of Kent and Strathearn"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what was the name of frederick douglass book?",
+        "input_answer": "The answer to the question 'what was the name of frederick douglass book?' is Narrative of the Life of Frederick Douglass, an American Slave",
+        "input_prediction": "Narrative of the Life of Frederick Douglass, an American Slave"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what year did Maureen Jenson run for office?",
+        "input_answer": "The answer to the question 'what year did Maureen Jenson run for office?' is 2018 in Flibbings, NJ",
+        "input_prediction": "2018"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where do the san francisco giants play home games?",
+        "input_answer": "The answer to the question 'where do the san francisco giants play home games?' is AT&T Park",
+        "input_prediction": "AT&T Park"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what season did tony soprano get shot?",
+        "input_answer": "The answer to the question 'what season did tony soprano get shot?' is the seventh season",
+        "input_prediction": "season 7"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "What were the action items for this meeting?",
+        "input_answer": "James was to send out the minutes, Anna was to share the reports with Avi, and Jane agreed to share the api keys for the deployment with Nora",
+        "input_prediction": "- Anna will send the reports to Avi\nJane will give Nora the information about the deployment's API keys\nJames will send the minutes "
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what movie is josh hutcherson in?",
+        "input_answer": "The answer to the question 'what movie is josh hutcherson in?' is Cirque du Freak: The Vampire's Assistant",
+        "input_prediction": "Cirque du Freak: The Vampire's Assistant"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where did dmitri mendeleev study science?",
+        "input_answer": "The answer to the question 'where did dmitri mendeleev study science?' is Saint Petersburg State University",
+        "input_prediction": "Saint Petersburg State University"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what county is heathrow airport in?",
+        "input_answer": "The answer to the question 'what county is heathrow airport in?' is London",
+        "input_prediction": "London"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what did william shakespeare do for a living?",
+        "input_answer": "The answer to the question 'what did william shakespeare do for a living?' is Poet",
+        "input_prediction": "Poet"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is the name of the first harry potter novel?",
+        "input_answer": "The answer to the question 'what is the name of the first harry potter novel?' is Harry Potter and the Philosopher's Stone",
+        "input_prediction": "Harry Potter and the Philosopher's Stone"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what county is the city of hampton va in?",
+        "input_answer": "The answer to the question 'what county is the city of hampton va in?' is Hampton",
+        "input_prediction": "Hampton"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is Project Ambiance??",
+        "input_answer": "The answer to the question 'waht is Project Ambiance?' is: the project is a secret engineering undertaking by HP focused on modernizing their fiber investments",
+        "input_prediction": "Project Ambiance is the code-name for the secret project to improve HP's fiber infrastructure investments"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what party was Sean?",
+        "input_answer": "The answer to the question 'what party was Sean?' is the purble eggs party",
+        "input_prediction": "He was part of PEP, the purple eggs party"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is shana cooper doing now?",
+        "input_answer": "The answer to the question 'what is shana cooper doing now?' is that she is an author",
+        "input_prediction": "an author"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is cher's son's name?",
+        "input_answer": "The answer to the question 'what is cher's son's name?' is Elijah Blue Allman",
+        "input_prediction": "Elijah Blue Allman"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "when is the new series of the only way is essex starting?",
+        "input_answer": "The answer to the question 'when is the new series of the only way is essex starting?' is next month",
+        "input_prediction": "next month"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is Egroeg known for?",
+        "input_answer": "The answer to the question 'what is Egroeg known for?' is that he is a rapper",
+        "input_prediction": "Egroeg is a rapper and musician"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what sport do the toronto maple leafs play?",
+        "input_answer": "The answer to the question 'what sport do the toronto maple leafs play?' is Ice Hockey",
+        "input_prediction": "Ice Hockey"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what city did esther live in?",
+        "input_answer": "The answer to the question 'what city did esther live in?' is Susa",
+        "input_prediction": "Susa"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "which department is Cemalitryx known for?",
+        "input_answer": "The answer to the question 'which department is Cemalitryx known for?' is their legal team.",
+        "input_prediction": "The attorneys at Cemalitryx are renown."
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is the president of brazil?",
+        "input_answer": "The answer to the question 'what is the president of brazil?' is Dilma Rousseff",
+        "input_prediction": "Dilma Rousseff"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where are the netherlands on a world map? What is its name?",
+        "input_answer": "The answer to the question 'where are the netherlands on a world map?' is west of germany. The full name is the Kingdom of the Netherlands",
+        "input_prediction": "Northern europe, west of geremany. The full name is Kingdom of the Netherlands"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what was malcolm x trying to accomplish?",
+        "input_answer": "The answer to the question 'what was malcolm x trying to accomplish?' is African Americans' rights",
+        "input_prediction": "African Americans' rights"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who was ishmael's mom?",
+        "input_answer": "The answer to the question 'who was ishmael's mom?' is Hagar",
+        "input_prediction": "Hagar"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "which wife did king henry behead?",
+        "input_answer": "The answer to the question 'which wife did king henry behead?' is Anne",
+        "input_prediction": "Anne"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what does thai mean?",
+        "input_answer": "The answer to the question 'what does thai mean?' is Language",
+        "input_prediction": "Language"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "how much did adriana lima earn in the month after the Spike award show ?",
+        "input_answer": "The answer to the question 'how much did adriana lima earn in the month after the Spike award show ?' is 32 grand ",
+        "input_prediction": "$32,000"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what did the german revolution lead to?",
+        "input_answer": "The answer to the question 'what did the german revolution lead to?' is German Democratic Republic",
+        "input_prediction": "German Democratic Republic"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what year lebron james came to the nba?",
+        "input_answer": "The answer to the question 'what year lebron james came to the nba?' is 2003\u201304 NBA season",
+        "input_prediction": "2003\u201304 NBA season"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what movies does taylor lautner play in?",
+        "input_answer": "The answer to the question 'what movies does taylor lautner play in?' is Abduction",
+        "input_prediction": "Abduction"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what form of government did Jen propose yesterday?",
+        "input_answer": "The answer to the question 'what form of government did Jen propose yesterday?' is Constitutional republic",
+        "input_prediction": "Constitutional republic"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what high school did president bill clinton attend?",
+        "input_answer": "The answer to the question 'what high school did president bill clinton attend?' is Hot Springs High School",
+        "input_prediction": "Hot Springs High School"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what to see near sedona arizona?",
+        "input_answer": "The answer to the question 'what to see near sedona arizona?' is Sedona Airport",
+        "input_prediction": "Sedona Airport"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who is the prime minister of ethiopia?",
+        "input_answer": "The answer to the question 'who is the prime minister of ethiopia?' is Hailemariam Desalegn",
+        "input_prediction": "Hailemariam Desalegn"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where to fly into bali?",
+        "input_answer": "The answer to the question 'where to fly into bali?' is Ngurah Rai Airport",
+        "input_prediction": "Ngurah Rai Airport"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who did the philippines gain independence from?",
+        "input_answer": "The answer to the question 'who did the philippines gain independence from?' is United States of America",
+        "input_prediction": "United States of America"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what books did beverly cleary right?",
+        "input_answer": "The answer to the question 'what books did beverly cleary right?' is Ramona's World",
+        "input_prediction": "Ramona's World"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "which kennedy died first?",
+        "input_answer": "The answer to the question 'which kennedy died first?' is John F. Kennedy",
+        "input_prediction": "John F. Kennedy"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what country does iceland belong to?",
+        "input_answer": "The answer to the question 'what country does iceland belong to?' is Iceland",
+        "input_prediction": "Iceland is its own country and doesn't belong to any other country"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is nina dobrev nationality?",
+        "input_answer": "The answer to the question 'what is nina dobrev nationality?' is Bulgaria",
+        "input_prediction": "Bulgaria"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where is rome italy located on a map?",
+        "input_answer": "The answer to the question 'where is rome italy located on a map?' is central italy",
+        "input_prediction": "in the center of italy"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "which countries border the us?",
+        "input_answer": "The answer to the question 'which countries border the us?' is mexico and Canada",
+        "input_prediction": "Canada and mexico"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who did draco malloy end up marrying?",
+        "input_answer": "The answer to the question 'who did draco malloy end up marrying?' is Astoria Greengrass",
+        "input_prediction": "Astoria Greengrass"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what time zone am i in cleveland ohio?",
+        "input_answer": "The answer to the question 'what time zone am i in cleveland ohio?' is North American Eastern Time Zone",
+        "input_prediction": "North American Eastern Time Zone"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what two countries invaded poland in the beginning of ww2?",
+        "input_answer": "The answer to the question 'what two countries invaded poland in the beginning of ww2?' is Germany and russia",
+        "input_prediction": "Germany + russia"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "how old is sacha andra smith?",
+        "input_answer": "The answer to the question 'how old is sacha andra smith?' is 32, found on http://www.buzznet.com/sacha-andra-age",
+        "input_prediction": "32"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where did saki live?",
+        "input_answer": "The answer to the question 'where did saki live?' is United Kingdom",
+        "input_prediction": "United Kingdom"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "where are the nfl redskins from?",
+        "input_answer": "The answer to the question 'where are the nfl redskins from?' is Washington Redskins",
+        "input_prediction": "Washington Redskins"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "who does joakim noah play for?",
+        "input_answer": "The answer to the question 'who does joakim noah play for?' is Chicago Bulls",
+        "input_prediction": "Chicago Bulls"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what character did john noble play in lord of the rings?",
+        "input_answer": "The answer to the question 'what character did john noble play in lord of the rings?' is Denethor II",
+        "input_prediction": "Denethor II"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what kind of money to take to bahamas?",
+        "input_answer": "The answer to the question 'what kind of money to take to bahamas?' is Bahamian dollar",
+        "input_prediction": "Bahamian dollar"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what country is the grand bahama island in?",
+        "input_answer": "The answer to the question 'what country is the grand bahama island in?' is Bahamas",
+        "input_prediction": "Bahamas"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what state does selena gomez?",
+        "input_answer": "The answer to the question 'what state does selena gomez?' is New York",
+        "input_prediction": "New York"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what character did natalie portman play in star wars?",
+        "input_answer": "The answer to the question 'what character did natalie portman play in star wars?' is Padm\u00e9 Amidala",
+        "input_prediction": "Padm\u00e9 Amidala"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    },
+    {
+      "inputs": {
+        "input": "what is the name of justin bieber sister?",
+        "input_answer": "The answer to the question 'what is the name of justin bieber sister?' is Jazmyn Bieber",
+        "input_prediction": "Jazmyn Bieber"
+      },
+      "outputs": { "output_correctness_score": 1 }
+    }
+  ]
+}

archived/meta-evals/correctness/test_correctness_evaluator.py

@@ -0,0 +1,206 @@
+# Tests for the criteria evaluator
+from typing import Tuple
+from uuid import uuid4
+
+import pytest
+from langchain import chat_models, hub, llms
+from langchain.evaluation import load_evaluator
+from langchain.schema import runnable
+from langchain.smith import RunEvalConfig, run_on_dataset
+from langsmith import Client, EvaluationResult
+from langsmith.evaluation import RunEvaluator
+from langsmith.schemas import Example, Run
+
+
+class ExactScoreMatch(RunEvaluator):
+    def evaluate_run(self, run: Run, example: Example) -> EvaluationResult:
+        predicted_score = run.outputs["score"]
+        return EvaluationResult(
+            key="exact_score_match",
+            score=predicted_score == example.outputs["output_correctness_score"],
+        )
+
+
+class AbsDistanceEvaluator(RunEvaluator):
+    def __init__(self) -> None:
+        super().__init__()
+
+    def evaluate_run(self, run: Run, example: Example) -> EvaluationResult:
+        predicted_score = run.outputs["score"]
+        if predicted_score is None:
+            return EvaluationResult(key="absolute_distance", score=None)
+        return EvaluationResult(
+            key="absolute_distance",
+            score=abs(predicted_score - example.outputs["output_correctness_score"]),
+        )
+
+
+class NullScore(RunEvaluator):
+    def __init__(self) -> None:
+        super().__init__()
+
+    def evaluate_run(self, run: Run, example: Example) -> EvaluationResult:
+        predicted_score = run.outputs["score"]
+        null_score = 1 if predicted_score is None else 0
+        return EvaluationResult(key="null_score", score=null_score)
+
+
+@pytest.fixture(scope="session")
+def uid() -> str:
+    return uuid4().hex[:8]
+
+
+def _check_dataset(
+    loader_kwargs: dict,
+    dataset_name: str,
+    project_name: str,
+    model_provider: str,
+    model_name: str,
+    tags: list,
+    metadata: dict,
+) -> Tuple[float, float]:
+    client = Client()
+    match model_provider:
+        case "openai":
+            llm = chat_models.ChatOpenAI(model_name=model_name, temperature=0)
+        case "openai-completion":
+            llm = llms.OpenAI(model_name=model_name, temperature=0)
+        case "anthropic":
+            llm = chat_models.ChatAnthropic(
+                model_name=model_name, max_tokens=1000, temperature=0
+            )
+    eval_chain = load_evaluator(**loader_kwargs, llm=llm)
+
+    def to_evaluate(input: dict, config: runnable.RunnableConfig) -> dict:
+        return eval_chain.evaluate_strings(
+            input=input["input"],
+            prediction=input["input_prediction"],
+            reference=input["input_answer"],
+            **config,
+        )
+
+    res = run_on_dataset(
+        dataset_name=dataset_name,
+        llm_or_chain_factory=runnable.RunnableLambda(to_evaluate).with_config(
+            {"metadata": metadata}
+        ),
+        evaluation=RunEvalConfig(
+            custom_evaluators=[
+                ExactScoreMatch(),
+                #   AbsDistanceEvaluator(),
+                NullScore(),
+            ],
+        ),
+        concurrency_level=8,
+        client=client,
+        verbose=True,
+        project_name=project_name,
+        tags=["int-test"] + tags,
+    )
+    df = res.to_dataframe()
+    feedback_cols = [
+        col for col in df.columns if col not in ["input", "output", "reference"]
+    ]
+    # Return averages
+    return df[feedback_cols].mean()
+
+
+def _get_project_name(
+    loader_kwargs: dict, uid: str, dataset_name: str, model: Tuple[str]
+) -> str:
+    other_args = "-".join(f"[{k}={v}]" for k, v in loader_kwargs.items())
+    return f"{model[0]}.{model[1]}-{loader_kwargs['evaluator']}{other_args} - {dataset_name} - {uid}"
+
+
+# prompt = "wfh/criteria_candidates"
+# commits = [
+#     "f470538b",
+#     "c92fcf90",
+# ]
+
+# anthropic_prompt = "wfh/criteria_candidates_anthropic"
+
+prompt_list = [
+    {
+        "openai": "wfh/criteria_candidates:f470538b",
+        "anthropic": "wfh/criteria_candidates_anthropic:fb037730",
+    },  # Here we'll try the anthropic one that inserts the "reasoning" step in the AI's mouth
+    {
+        "openai": "wfh/criteria_candidates:c92fcf90",
+    },  # It's the same as the openai one
+]
+
+
+@pytest.mark.parametrize(
+    "dataset_name",
+    [
+        "Web Q&A Dataset - Incorrect",
+        "Carb-IE-Test INCORRECT",
+        "Opus100 - Incorrect",
+    ]
+    + [
+        "Web Q&A Dataset - Correct",
+        "Opus100 - Correct",
+        "Carb-IE-Test CORRECT",
+    ],
+)
+@pytest.mark.parametrize(
+    "model",
+    [
+        ("openai", "gpt-4"),
+        ("openai", "gpt-3.5-turbo"),
+        ("openai-completion", "gpt-3.5-turbo-instruct"),
+        ("anthropic", "claude-2"),
+    ],
+)
+@pytest.mark.parametrize(
+    "loader_kwargs",
+    [
+        {"evaluator": "cot_qa"},
+        {"evaluator": "qa"},
+    ]
+    + [
+        {
+            "evaluator": "labeled_criteria",
+            "criteria": "correctness",
+            "prompt_lookup": pl,
+        }
+        for pl in prompt_list
+    ],
+)
+@pytest.mark.asyncio
+async def test_metaeval_correctness(
+    loader_kwargs: dict, uid: str, dataset_name: str, model: Tuple[str, str]
+):
+    project_name = _get_project_name(loader_kwargs, uid, dataset_name, model)
+    tags = ["test_metaeval_correctness", loader_kwargs["evaluator"]]
+    metadata = {
+        "model_provider": model[0],
+        "model_name": model[1],
+        "dataset_name": dataset_name,
+        "evaluator": loader_kwargs["evaluator"],
+        "uid": str(uid),
+    }
+    if "prompt_lookup" in loader_kwargs:
+        prompt_lookup = loader_kwargs.get("prompt_lookup")
+        prompt_repo = prompt_lookup.get(
+            model[0], prompt_lookup.get("openai")
+        )  # Fall back on openai prompt
+        commit = prompt_repo.split(":")[-1]
+        prompt = hub.pull(prompt_repo)
+        loader_kwargs["prompt"] = prompt
+        tags += [commit]
+        print("Using prompt:", prompt_repo, commit)
+        metadata["prompt"] = prompt_repo
+        metadata["commit"] = commit
+    scores = _check_dataset(
+        loader_kwargs,
+        dataset_name,
+        project_name,
+        model_provider=model[0],
+        model_name=model[1],
+        tags=tags,
+        metadata=metadata,
+    )
+    score = scores["exact_score_match"]
+    assert score >= 0.95

csv-qa/pandas_ai.py

@@ -1,44 +0,0 @@
-import pandas as pd
-from langchain.chat_models import ChatOpenAI
-from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
-from langchain.agents.agent_types import AgentType
-from langsmith import Client
-from langchain.smith import RunEvalConfig, run_on_dataset
-import pandas as pd
-from pandasai import PandasAI
-
-from langchain.chat_models import ChatOpenAI
-from langchain.prompts import ChatPromptTemplate
-from langchain.schema.output_parser import StrOutputParser
-
-if __name__ == "__main__":
-    df = pd.read_csv("titanic.csv")
-
-    pandas_ai = PandasAI(ChatOpenAI(temperature=0, model="gpt-4"), enable_cache=False)
-    prompt = ChatPromptTemplate.from_messages([
-        ("system",
-         "Answer the users question about some data. A data scientist will run some code and the results will be returned to you to use in your answer"),
-        ("human", "Question: {input}"),
-        ("human", "Data Scientist Result: {result}"),
-    ])
-
-    def get_chain():
-        chain = {
-                    "input": lambda x: x["input_question"],
-                    "result": lambda x: pandas_ai(df, prompt=x['input_question'])
-                } | prompt | ChatOpenAI(temperature=0, model="gpt-4") | StrOutputParser()
-        return chain
-
-
-    client = Client()
-    eval_config = RunEvalConfig(
-        evaluators=[
-            "qa"
-        ],
-    )
-    chain_results = run_on_dataset(
-        client,
-        dataset_name="Titanic CSV Data",
-        llm_or_chain_factory=get_chain,
-        evaluation=eval_config,
-    )

csv-qa/streamlit_app.py

@@ -1,42 +0,0 @@
-import pandas as pd
-import streamlit as st
-from langchain.chat_models import ChatOpenAI
-from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
-from langchain.agents.agent_types import AgentType
-
-df = pd.read_csv('titanic.csv')
-
-
-llm = ChatOpenAI(temperature=0)
-agent = create_pandas_dataframe_agent(llm, df, agent_type=AgentType.OPENAI_FUNCTIONS)
-
-
-from langsmith import Client
-client = Client()
-def send_feedback(run_id, score):
-    client.create_feedback(run_id, "user_score", score=score)
-
-st.set_page_config(page_title='🦜🔗 Ask the CSV App')
-st.title('🦜🔗 Ask the CSV App')
-st.info("Most 'question answering' applications run over unstructured text data. But a lot of the data in the world is tabular data! This is an attempt to create an application using [LangChain](https://github.com/langchain-ai/langchain) to let you ask questions of data in tabular format. For this demo application, we will use the Titanic Dataset. Please explore it [here](https://github.com/datasciencedojo/datasets/blob/master/titanic.csv) to get a sense for what questions you can ask. Please leave feedback on well the question is answered, and we will use that improve the application!")
-
-query_text = st.text_input('Enter your question:', placeholder = 'Who was in cabin C128?')
-# Form input and query
-result = None
-with st.form('myform', clear_on_submit=True):
-	submitted = st.form_submit_button('Submit')
-	if submitted:
-		with st.spinner('Calculating...'):
-			response = agent({"input": query_text}, include_run_info=True)
-			result = response["output"]
-			run_id = response["__run"].run_id
-if result is not None:
-	st.info(result)
-	col_blank, col_text, col1, col2 = st.columns([10, 2,1,1])
-	with col_text:
-		st.text("Feedback:")
-	with col1:
-		st.button("👍", on_click=send_feedback, args=(run_id, 1))
-	with col2:
-		st.button("👎", on_click=send_feedback, args=(run_id, 0))
-