infra: release permissions (#193)

Co-authored-by: Bagatur <baskaryan@gmail.com>

.github/actions/poetry_setup/action.yml

@@ -0,0 +1,91 @@
+# An action for setting up poetry install with caching.
+# Using a custom action since the default action does not
+# take poetry install groups into account.
+# Action code from:
+# https://github.com/actions/setup-python/issues/505#issuecomment-1273013236
+name: poetry-install-with-caching
+description: Poetry install with support for caching of dependency groups.
+
+inputs:
+  python-version:
+    description: Python version, supporting MAJOR.MINOR only
+    required: true
+
+  poetry-version:
+    description: Poetry version
+    required: true
+
+  cache-key:
+    description: Cache key to use for manual handling of caching
+    required: true
+
+  working-directory:
+    description: Directory whose poetry.lock file should be cached
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - uses: actions/setup-python@v4
+      name: Setup python ${{ inputs.python-version }}
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - uses: actions/cache@v3
+      id: cache-bin-poetry
+      name: Cache Poetry binary - Python ${{ inputs.python-version }}
+      env:
+        SEGMENT_DOWNLOAD_TIMEOUT_MIN: "1"
+      with:
+        path: |
+          /opt/pipx/venvs/poetry
+        # This step caches the poetry installation, so make sure it's keyed on the poetry version as well.
+        key: bin-poetry-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-${{ inputs.poetry-version }}
+
+    - name: Refresh shell hashtable and fixup softlinks
+      if: steps.cache-bin-poetry.outputs.cache-hit == 'true'
+      shell: bash
+      env:
+        POETRY_VERSION: ${{ inputs.poetry-version }}
+        PYTHON_VERSION: ${{ inputs.python-version }}
+      run: |
+        set -eux
+
+        # Refresh the shell hashtable, to ensure correct `which` output.
+        hash -r
+
+        # `actions/cache@v3` doesn't always seem able to correctly unpack softlinks.
+        # Delete and recreate the softlinks pipx expects to have.
+        rm /opt/pipx/venvs/poetry/bin/python
+        cd /opt/pipx/venvs/poetry/bin
+        ln -s "$(which "python$PYTHON_VERSION")" python
+        chmod +x python
+        cd /opt/pipx_bin/
+        ln -s /opt/pipx/venvs/poetry/bin/poetry poetry
+        chmod +x poetry
+
+        # Ensure everything got set up correctly.
+        /opt/pipx/venvs/poetry/bin/python --version
+        /opt/pipx_bin/poetry --version
+
+    - name: Install poetry
+      if: steps.cache-bin-poetry.outputs.cache-hit != 'true'
+      shell: bash
+      env:
+        POETRY_VERSION: ${{ inputs.poetry-version }}
+        PYTHON_VERSION: ${{ inputs.python-version }}
+      run: pipx install "poetry==$POETRY_VERSION" --python "python$PYTHON_VERSION" --verbose
+
+    - name: Restore pip and poetry cached dependencies
+      uses: actions/cache@v3
+      env:
+        SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4"
+        WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }}
+      with:
+        path: |
+          ~/.cache/pip
+          ~/.cache/pypoetry/virtualenvs
+          ~/.cache/pypoetry/cache
+          ~/.cache/pypoetry/artifacts
+          ${{ env.WORKDIR }}/.venv
+        key: py-deps-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-poetry-${{ inputs.poetry-version }}-${{ inputs.cache-key }}-${{ hashFiles(format('{0}/**/poetry.lock', env.WORKDIR)) }}

.github/workflows/_lint.yml

@@ -0,0 +1,83 @@
+name: lint
+
+on:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+
+env:
+  POETRY_VERSION: "1.6.1"
+  WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }}
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      # This number is set "by eye": we want it to be big enough
+      # so that it's bigger than the number of commits in any reasonable PR,
+      # and also as small as possible since increasing the number makes
+      # the initial `git fetch` slower.
+      FETCH_DEPTH: 50
+    strategy:
+      matrix:
+        # Only lint on the min and max supported Python versions.
+        # It's extremely unlikely that there's a lint issue on any version in between
+        # that doesn't show up on the min or max versions.
+        #
+        # GitHub rate-limits how many jobs can be running at any one time.
+        # Starting new jobs is also relatively slow,
+        # so linting on fewer versions makes CI faster.
+        python-version:
+          - "3.8"
+          - "3.11"
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
+          cache-key: lint-with-extras
+
+      - name: Check Poetry File
+        shell: bash
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          poetry check
+
+      - name: Check lock file
+        shell: bash
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          poetry lock --check
+
+      - name: Install dependencies
+        # Also installs dev/lint/test/typing dependencies, to ensure we have
+        # type hints for as many of our libraries as possible.
+        # This helps catch errors that require dependencies to be spotted, for example:
+        # https://github.com/langchain-ai/langchain/pull/10249/files#diff-935185cd488d015f026dcd9e19616ff62863e8cde8c0bee70318d3ccbca98341
+        #
+        # If you change this configuration, make sure to change the `cache-key`
+        # in the `poetry_setup` action above to stop using the old cache.
+        # It doesn't matter how you change it, any change will cause a cache-bust.
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          poetry install --with dev,lint,test,typing
+
+      - name: Get .mypy_cache to speed up mypy
+        uses: actions/cache@v3
+        env:
+          SEGMENT_DOWNLOAD_TIMEOUT_MIN: "2"
+        with:
+          path: |
+            ${{ env.WORKDIR }}/.mypy_cache
+          key: mypy-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', env.WORKDIR)) }}
+
+      - name: Analysing the code with our lint
+        working-directory: ${{ inputs.working-directory }}
+        run: |
+          make lint

.github/workflows/_pydantic_compatibility.yml

@@ -0,0 +1,94 @@
+name: pydantic v1/v2 compatibility
+
+on:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  build:
+    timeout-minutes: 5
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+    name: Pydantic v1/v2 compatibility - Python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
+          cache-key: pydantic-cross-compat
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install --with test
+
+      - name: Install the opposite major version of pydantic
+        # If normal tests use pydantic v1, here we'll use v2, and vice versa.
+        shell: bash
+        run: |
+          # Determine the major part of pydantic version
+          REGULAR_VERSION=$(poetry run python -c "import pydantic; print(pydantic.__version__)" | cut -d. -f1)
+
+          if [[ "$REGULAR_VERSION" == "1" ]]; then
+            PYDANTIC_DEP=">=2.1,<3"
+            TEST_WITH_VERSION="2"
+          elif [[ "$REGULAR_VERSION" == "2" ]]; then
+            PYDANTIC_DEP="<2"
+            TEST_WITH_VERSION="1"
+          else
+            echo "Unexpected pydantic major version '$REGULAR_VERSION', cannot determine which version to use for cross-compatibility test."
+            exit 1
+          fi
+
+          # Install via `pip` instead of `poetry add` to avoid changing lockfile,
+          # which would prevent caching from working: the cache would get saved
+          # to a different key than where it gets loaded from.
+          poetry run pip install "pydantic${PYDANTIC_DEP}"
+
+          # Ensure that the correct pydantic is installed now.
+          echo "Checking pydantic version... Expecting ${TEST_WITH_VERSION}"
+
+          # Determine the major part of pydantic version
+          CURRENT_VERSION=$(poetry run python -c "import pydantic; print(pydantic.__version__)" | cut -d. -f1)
+
+          # Check that the major part of pydantic version is as expected, if not
+          # raise an error
+          if [[ "$CURRENT_VERSION" != "$TEST_WITH_VERSION" ]]; then
+            echo "Error: expected pydantic version ${CURRENT_VERSION} to have been installed, but found: ${TEST_WITH_VERSION}"
+            exit 1
+          fi
+          echo "Found pydantic version ${CURRENT_VERSION}, as expected"
+      - name: Run pydantic compatibility tests
+        shell: bash
+        run: make test
+
+      - name: Ensure the tests did not create any additional files
+        shell: bash
+        run: |
+          set -eu
+
+          STATUS="$(git status)"
+          echo "$STATUS"
+
+          # grep will exit non-zero if the target message isn't found,
+          # and `set -e` above will cause the step to fail.
+          echo "$STATUS" | grep 'nothing to commit, working tree clean'

.github/workflows/_release.yml

@@ -0,0 +1,63 @@
+name: release
+
+on:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  if_release:
+    # Disallow publishing from branches that aren't `main`.
+    if: github.ref == 'refs/heads/main'
+    runs-on: ubuntu-latest
+    permissions:
+      # This permission is used for trusted publishing:
+      # https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/
+      #
+      # Trusted publishing has to also be configured on PyPI for each package:
+      # https://docs.pypi.org/trusted-publishers/adding-a-publisher/
+      id-token: write
+
+      # This permission is needed by `ncipollo/release-action` to create the GitHub release.
+      contents: write
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: "3.10"
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
+          cache-key: release
+
+      - name: Build project for distribution
+        run: poetry build
+      - name: Check Version
+        id: check-version
+        run: |
+          echo version=$(poetry version --short) >> $GITHUB_OUTPUT
+      - name: Create Release
+        uses: ncipollo/release-action@v1
+        with:
+          artifacts: "dist/*"
+          token: ${{ secrets.GITHUB_TOKEN }}
+          draft: false
+          generateReleaseNotes: true
+          tag: v${{ steps.check-version.outputs.version }}
+          commit: main
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: ${{ inputs.working-directory }}/dist/
+          verbose: true
+          print-hash: true

.github/workflows/_test.yml

@@ -0,0 +1,57 @@
+name: test
+
+on:
+  workflow_call:
+    inputs:
+      working-directory:
+        required: true
+        type: string
+        description: "From which folder this pipeline executes"
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  build:
+    defaults:
+      run:
+        working-directory: ${{ inputs.working-directory }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+    name: Python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: ${{ inputs.working-directory }}
+          cache-key: core
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install
+
+      - name: Run core tests
+        shell: bash
+        run: make test
+
+      - name: Ensure the tests did not create any additional files
+        shell: bash
+        run: |
+          set -eu
+
+          STATUS="$(git status)"
+          echo "$STATUS"
+
+          # grep will exit non-zero if the target message isn't found,
+          # and `set -e` above will cause the step to fail.
+          echo "$STATUS" | grep 'nothing to commit, working tree clean'

.github/workflows/ci.yml

@@ -0,0 +1,150 @@
+---
+name: Run CI Tests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    paths-ignore:
+      - 'README.md'
+  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
+
+# If another push to the same PR or branch happens while this workflow is still running,
+# cancel the earlier run in favor of the next run.
+#
+# There's no point in testing an outdated version of the code. GitHub only allows
+# a limited number of job runners to be active at the same time, so it's better to cancel
+# pointless jobs early so that more useful jobs can run sooner.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  POETRY_VERSION: "1.5.1"
+  WORKDIR: "."
+
+jobs:
+  lint:
+    uses:
+      ./.github/workflows/_lint.yml
+    with:
+      working-directory: .
+    secrets: inherit
+
+  pydantic-compatibility:
+    uses:
+      ./.github/workflows/_pydantic_compatibility.yml
+    with:
+      working-directory: .
+    secrets: inherit
+  test:
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ env.WORKDIR }}
+    strategy:
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+    name: Python ${{ matrix.python-version }} tests
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: .
+          cache-key: benchmarks-all
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running tests, installing dependencies with poetry..."
+          poetry install --with test,lint,typing,docs
+
+      - name: Run tests
+        run: make test
+
+      - name: Ensure the tests did not create any additional files
+        shell: bash
+        run: |
+          set -eu
+
+          STATUS="$(git status)"
+          echo "$STATUS"
+
+          # grep will exit non-zero if the target message isn't found,
+          # and `set -e` above will cause the step to fail.
+          echo "$STATUS" | grep 'nothing to commit, working tree clean'
+  test_docs:
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ env.WORKDIR }}
+    strategy:
+      matrix:
+        python-version:
+          - "3.11"
+    name: Documentation Build for Python ${{ matrix.python-version }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: .
+          cache-key: benchmarks-all
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running tests, installing dependencies with poetry..."
+          poetry install --with test,lint,typing,docs
+
+      - name: Test Sphinx Docs
+        shell: bash
+        run: |
+          echo "Attempting to build docs..."
+          make docs_build
+  test_datasets:
+    timeout-minutes: 5
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ${{ env.WORKDIR }}
+    strategy:
+      matrix:
+        python-version:
+          - "3.11"
+    name: Validate Public Datasets
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: .
+          cache-key: benchmarks-all
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running tests, installing dependencies with poetry..."
+          poetry install --with test,lint,typing,docs
+
+      - name: Request datasets
+        shell: bash
+        run: |
+          echo "Attempting to build docs..."
+          poetry run python -m scripts.check_datasets

.github/workflows/doc_publish.yaml

@@ -0,0 +1,44 @@
+name: Publish Docs
+on: [workflow_dispatch]
+permissions:
+    contents: write
+
+env:
+  POETRY_VERSION: "1.6.1"
+
+jobs:
+  docs:
+    strategy:
+      matrix:
+        python-version:
+          - "3.11"
+    runs-on: ubuntu-latest
+    name: Documentation Publish
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: ${{ matrix.python-version }}
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: .
+          cache-key: benchmarks-all
+
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running tests, installing dependencies with poetry..."
+          poetry install --with test,lint,typing,docs
+
+      - name: Sphinx build
+        shell: bash
+        run: |
+          make docs_build
+      - name: Publish Docs
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          publish_branch: gh-pages
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs/build
+          force_orphan: true

.github/workflows/release.yml

@@ -0,0 +1,14 @@
+---
+name: Publish Package to PyPi
+
+on:
+  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
+
+jobs:
+  release:
+    uses:
+      ./.github/workflows/_release.yml
+    permissions: write-all
+    with:
+      working-directory: .
+    secrets: inherit

.github/workflows/tool_benchmarks.yml

@@ -0,0 +1,33 @@
+name: Weekly Tool Benchmarks
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * 0'  # Runs at midnight (00:00) every Sunday (UTC time)
+
+jobs:
+  run_tool_benchmarks:
+    runs-on: ubuntu-latest
+    
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python 3.12 + Poetry ${{ env.POETRY_VERSION }}
+        uses: "./.github/actions/poetry_setup"
+        with:
+          python-version: '3.12'
+          poetry-version: ${{ env.POETRY_VERSION }}
+          working-directory: .
+          cache-key: benchmarks-all
+      
+      - name: Install dependencies
+        shell: bash
+        run: |
+          echo "Running tests, installing dependencies with poetry..."
+          poetry install --with test,lint,typing,docs
+
+      - name: Multiverse math benchmark
+        run: python scripts/multiverse_math_benchmark.py
+      
+      - name: Query analysis benchmark
+        run: python scripts/query_analysis_benchmark.py

.gitignore

@@ -158,5 +158,5 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
-
+.idea/
+.DS_Store

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Langchain AI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,67 @@
+.PHONY: all lint format test help
+
+# Default target executed when no arguments are given to make.
+all: help
+
+# LINTING AND FORMATTING:
+
+# Define a variable for Python and notebook files.
+lint format: PYTHON_FILES=.
+lint_diff format_diff: PYTHON_FILES=$(shell git diff --relative=. --name-only --diff-filter=d master | grep -E '\.py$$|\.ipynb$$')
+
+lint lint_diff:
+	[ "$(PYTHON_FILES)" = "" ] ||	poetry run ruff format $(PYTHON_FILES) --diff
+	# [ "$(PYTHON_FILES)" = "" ] || poetry run mypy $(PYTHON_FILES)
+
+format format_diff:
+	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff format $(PYTHON_FILES)
+	[ "$(PYTHON_FILES)" = "" ] || poetry run ruff --select I --fix $(PYTHON_FILES)
+
+spell_check:
+	poetry run codespell --toml pyproject.toml
+
+spell_fix:
+	poetry run codespell --toml pyproject.toml -w
+
+
+# TESTING AND COVERAGE:
+
+# Define a variable for the test file path.
+TEST_FILE ?= tests/unit_tests/
+
+test:
+	poetry run pytest --disable-socket --allow-unix-socket $(TEST_FILE)
+
+test_watch:
+	poetry run ptw . -- $(TEST_FILE)
+
+
+# DOCUMENTATION:
+
+docs_clean:
+	rm -rf ./docs/build
+
+docs_build:
+	# Copy README.md to docs/index.md
+	cp README.md ./docs/source/index.md
+	# Append to the table of contents the contents of the file
+	cat ./docs/source/toc.segment >> ./docs/source/index.md
+	poetry run sphinx-build "./docs/source" "./docs/build"
+
+
+# HELP:
+help:
+	@echo ''
+	@echo 'LINTING:'
+	@echo '  format             - run code formatters'
+	@echo '  lint               - run linters'
+	@echo '  spell_check        - run codespell'
+	@echo '  spell_fix          - run codespell and fix the errors'
+	@echo 'TESTS:'
+	@echo '  test               - run unit tests'
+	@echo '  test TEST_FILE=<test_file>   - run tests in <test_file>'
+	@echo '  coverage           - run unit tests and generate coverage report'
+	@echo 'DOCUMENTATION:'
+	@echo '  docs_clean         - delete the docs/build directory'
+	@echo '  docs_build         - build the documentation'
+	@echo ''

README.md

@@ -1,8 +1,19 @@
-# LangChain Benchmarks
+# 🦜💯 LangChain Benchmarks
 
-This repository shows how we benchmark some of our more popular chains and agents.
-The benchmarks are organized by end-to-end use cases.
-They utilize [LangSmith](https://smith.langchain.com/) heavily.
+[![Release Notes](https://img.shields.io/github/release/langchain-ai/langchain-benchmarks)](https://github.com/langchain-ai/langchain-benchmarks/releases)
+[![CI](https://github.com/langchain-ai/langchain-benchmarks/actions/workflows/ci.yml/badge.svg)](https://github.com/langchain-ai/langchain-benchmarks/actions/workflows/ci.yml)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Twitter](https://img.shields.io/twitter/url/https/twitter.com/langchainai.svg?style=social&label=Follow%20%40LangChainAI)](https://twitter.com/langchainai)
+[![](https://dcbadge.vercel.app/api/server/6adMQxSpJS?compact=true&style=flat)](https://discord.gg/6adMQxSpJS)
+[![Open Issues](https://img.shields.io/github/issues-raw/langchain-ai/langchain-benchmarks)](https://github.com/langchain-ai/langchain-benchmarks/issues)
+
+
+[📖 Documentation](https://langchain-ai.github.io/langchain-benchmarks/index.html)
+
+A package to help benchmark various LLM related tasks.
+
+The benchmarks are organized by end-to-end use cases, and
+utilize [LangSmith](https://smith.langchain.com/) heavily.
 
 We have several goals in open sourcing this:
 
@@ -11,8 +22,62 @@ We have several goals in open sourcing this:
 - Showing how we evaluate each task
 - Encouraging others to benchmark their solutions on these tasks (we are always looking for better ways of doing things!)
 
-We currently include the following tasks:
-- [CSV Question Answering](csv-qa)
-- [Extraction](extraction)
-- [Q&A over the LangChain docs](langchain-docs-benchmarking)
-- [Meta-evaluation of 'correctness' evaluators](meta-evals)
+## Benchmarking Results
+
+Read some of the articles about benchmarking results on our blog.
+
+* [Agent Tool Use](https://blog.langchain.dev/benchmarking-agent-tool-use/)
+* [Query Analysis in High Cardinality Situations](https://blog.langchain.dev/high-cardinality/)
+* [RAG on Tables](https://blog.langchain.dev/benchmarking-rag-on-tables/)
+* [Q&A over CSV data](https://blog.langchain.dev/benchmarking-question-answering-over-csv-data/)
+
+
+### Tool Usage (2024-04-18)
+
+See [tool usage docs](https://langchain-ai.github.io/langchain-benchmarks/notebooks/tool_usage/benchmark_all_tasks.html) to recreate!
+
+![download](https://github.com/langchain-ai/langchain-benchmarks/assets/3205522/0da33de8-e03f-49cf-bd48-e9ff945828a9)
+
+Explore Agent Traces on LangSmith:
+
+* [Relational Data](https://smith.langchain.com/public/22721064-dcf6-4e42-be65-e7c46e6835e7/d)
+* [Tool Usage (1-tool)](https://smith.langchain.com/public/ac23cb40-e392-471f-b129-a893a77b6f62/d)
+* [Tool Usage (26-tools)](https://smith.langchain.com/public/366bddca-62b3-4b6e-849b-a478abab73db/d)
+* [Mutiverse Math](https://smith.langchain.com/public/983faff2-54b9-4875-9bf2-c16913e7d489/d)
+
+## Installation
+
+To install the packages, run the following command:
+
+```bash
+pip install -U langchain-benchmarks
+```
+
+All the benchmarks come with an associated benchmark dataset stored in [LangSmith](https://smith.langchain.com). To take advantage of the eval and debugging experience, [sign up](https://smith.langchain.com), and set your API key in your environment:
+
+```bash
+export LANGCHAIN_API_KEY=ls-...
+```
+
+## Repo Structure
+
+The package is located within [langchain_benchmarks](./langchain_benchmarks/). Check out the [docs](https://langchain-ai.github.io/langchain-benchmarks/index.html) for information on how to get starte.
+
+The other directories are legacy and may be moved in the future.
+
+
+## Archived
+
+Below are archived benchmarks that require cloning this repo to run.
+
+- [CSV Question Answering](https://github.com/langchain-ai/langchain-benchmarks/tree/main/archived/csv-qa)
+- [Extraction](https://github.com/langchain-ai/langchain-benchmarks/tree/main/archived/extraction)
+- [Q&A over the LangChain docs](https://github.com/langchain-ai/langchain-benchmarks/tree/main/archived/langchain-docs-benchmarking)
+- [Meta-evaluation of 'correctness' evaluators](https://github.com/langchain-ai/langchain-benchmarks/tree/main/archived/meta-evals)
+
+
+## Related
+
+- For cookbooks on other ways to test, debug, monitor, and improve your LLM applications, check out the [LangSmith docs](https://docs.smith.langchain.com/)
+- For information on building with LangChain, check out the [python documentation](https://python.langchain.com/docs/get_started/introduction) or [JS documentation](https://js.langchain.com/docs/get_started/introduction)
+

agents/README.md

@@ -1,107 +0,0 @@
-# Testing Agents
-
-This directory contains environments that can be used to test agent's ability
-to use tools and make decisions.
-
-## Environments
-
-Environments are named in the style of e[env_number]_[name].py.
-
-### e01_alpha
-
-* Consists of 3 relational tables of users, locations and foods.
-* Defines a set of tools that can be used these tables.
-* Agent should use the given tools to answer questions.
-
-## Running Evaluation
-
-### Install dependencies
-
-```bash
-poetry install
-```
-
-### Run evaluation
-
-We'll make this more convenient in the future, but for now, you can run
-
-
-```python
-from langchain.agents.format_scratchpad import format_to_openai_functions
-from langchain.agents import AgentExecutor
-from langchain.agents.output_parsers import OpenAIFunctionsAgentOutputParser
-from langchain.chat_models import ChatOpenAI
-from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
-from langchain.tools.render import format_tool_to_openai_function
-
-from environments.e01_alpha import get_tools
-
-
-def agent_factory() -> AgentExecutor:
-    """This creates an OpenAI agent."""
-    llm = ChatOpenAI(
-        model="gpt-3.5-turbo-16k",
-        temperature=0,
-    )
-    tools = get_tools()
-    llm_with_tools = llm.bind(
-        functions=[format_tool_to_openai_function(t) for t in tools]
-    )
-    prompt = ChatPromptTemplate.from_messages(
-        [
-            ("system", "You are a helpful assistant. Use the given tools to answer the question. Keep in mind that an ID is distinct from a name for every entity."),
-            MessagesPlaceholder(variable_name="agent_scratchpad"),
-            ("user", "{input}"),
-        ]
-    )
-
-    runnable_agent = (
-        {
-            "input": lambda x: x["question"],
-            "agent_scratchpad": lambda x: format_to_openai_functions(
-                x["intermediate_steps"]
-            ),
-        }
-        | prompt
-        | llm_with_tools
-        | OpenAIFunctionsAgentOutputParser()
-    )
-
-    return AgentExecutor(
-        agent=runnable_agent,
-        tools=tools,
-        handle_parsing_errors=True,
-        return_intermediate_steps=True,
-    )
-```
-
-Confirm that the agent works and can use the environment tools:
-
-```python
-agent_factory().invoke({'question': "who is bob?"})
-```
-
-```python
-from langsmith import Client
-
-from environments.e01_alpha import DATASET_ID
-from evaluators import STANDARD_AGENT_EVALUATOR
-
-client = Client()
-
-dataset_name = client.read_dataset(dataset_id=DATASET_ID).name
-
-results = client.run_on_dataset(
-    dataset_name=dataset_name,
-    llm_or_chain_factory=agent_factory,
-    evaluation=STANDARD_AGENT_EVALUATOR,
-    verbose=True,
-    tags=["openai-agent", "gpt-3.5-turbo-16k"],
-)
- ```
-
-
-### Customize evaluation
-
-Please refer to the following example to see how to set up and run evaluation
-for agents using [LangSmith](https://github.com/langchain-ai/langsmith-cookbook/blob/main/testing-examples/agent_steps/evaluating_agents.ipynb).

agents/__init__.py

@@ -1 +0,0 @@
-"""Package for helping to evaluate agent runs."""

agents/evaluators.py

@@ -1,71 +0,0 @@
-"""Module contains standard evaluators for agents.
-
-Requirements:
-
-* Agents must output "intermediate_steps" in their run outputs.
-* The dataset must have "expected_steps" in its outputs.
-"""
-from typing import Optional
-
-from langchain.evaluation import EvaluatorType
-from langchain.smith import RunEvalConfig
-from langsmith.evaluation.evaluator import (
-    EvaluationResults,
-    RunEvaluator,
-    EvaluationResult,
-)
-from langsmith.schemas import Example, Run
-
-
-class AgentTrajectoryEvaluator(RunEvaluator):
-    """An evaluator that can be used in conjunction with a standard agent interface."""
-
-    def evaluate_run(
-        self, run: Run, example: Optional[Example] = None
-    ) -> EvaluationResults:
-        if run.outputs is None:
-            raise ValueError("Run outputs cannot be None")
-        # This is the output of each run
-        intermediate_steps = run.outputs["intermediate_steps"]
-        # Since we are comparing to the tool names, we now need to get that
-        # Intermediate steps is a Tuple[AgentAction, Any]
-        # The first element is the action taken
-        # The second element is the observation from taking that action
-        trajectory = [action.tool for action, _ in intermediate_steps]
-        # This is what we uploaded to the dataset
-        expected_trajectory = example.outputs["expected_steps"]
-
-        # Just score it based on whether it is correct or not
-        score = int(trajectory == expected_trajectory)
-        step_fraction = len(trajectory) / len(expected_trajectory)
-
-        return {
-            "results": [
-                EvaluationResult(
-                    key="Intermediate steps correctness",
-                    score=score,
-                ),
-                EvaluationResult(
-                    key="# steps / # expected steps",
-                    score=step_fraction,
-                ),
-            ]
-        }
-
-
-STANDARD_AGENT_EVALUATOR = RunEvalConfig(
-    # Evaluators can either be an evaluator type
-    # (e.g., "qa", "criteria", "embedding_distance", etc.) or a
-    # configuration for that evaluator
-    evaluators=[
-        # Measures whether a QA response is "Correct", based on a reference answer
-        # You can also select via the raw string "qa"
-        EvaluatorType.QA
-    ],
-    # You can add custom StringEvaluator or RunEvaluator objects
-    # here as well, which will automatically be
-    # applied to each prediction. Check out the docs for examples.
-    custom_evaluators=[AgentTrajectoryEvaluator()],
-    # We now need to specify this because we have multiple outputs in our dataset
-    reference_key="reference",
-)

agents/pyproject.toml

@@ -1,16 +0,0 @@
-[tool.poetry]
-name = "agents"
-version = "0.1.0"
-description = "Agent Benchmarking Environments"
-authors = ["Eugene Yurtsev <eyurtsev@gmail.com>"]
-license = "MIT"
-readme = "README.md"
-
-[tool.poetry.dependencies]
-python = "^3.8.1"
-langchain = "^0.0.335"
-
-
-[build-system]
-requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"

archived/csv-qa/custom_agent.py

@@ -1,22 +1,25 @@
-from langchain.agents import OpenAIFunctionsAgent, AgentExecutor
-from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
-from langchain.tools import PythonAstREPLTool
 import pandas as pd
+from langchain.agents import AgentExecutor, OpenAIFunctionsAgent
+from langchain.agents.agent_toolkits.conversational_retrieval.tool import (
+    create_retriever_tool,
+)
 from langchain.chat_models import ChatOpenAI
-from langsmith import Client
-from langchain.smith import RunEvalConfig, run_on_dataset
-from pydantic import BaseModel, Field
 from langchain.embeddings import OpenAIEmbeddings
+from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder
+from langchain.smith import RunEvalConfig, run_on_dataset
+from langchain.tools import PythonAstREPLTool
 from langchain.vectorstores import FAISS
-from langchain.agents.agent_toolkits.conversational_retrieval.tool import create_retriever_tool
+from langsmith import Client
+from pydantic import BaseModel, Field
 
-
-pd.set_option('display.max_rows', 20)
-pd.set_option('display.max_columns', 20)
+pd.set_option("display.max_rows", 20)
+pd.set_option("display.max_columns", 20)
 
 embedding_model = OpenAIEmbeddings()
 vectorstore = FAISS.load_local("titanic_data", embedding_model)
-retriever_tool = create_retriever_tool(vectorstore.as_retriever(), "person_name_search", "Search for a person by name")
+retriever_tool = create_retriever_tool(
+    vectorstore.as_retriever(), "person_name_search", "Search for a person by name"
+)
 
 
 TEMPLATE = """You are working with a pandas dataframe in Python. The name of the dataframe is `df`.
@@ -42,7 +45,6 @@ For example:
 """
 
 
-
 class PythonInputs(BaseModel):
     query: str = Field(description="code snippet to run")
 
@@ -51,27 +53,33 @@ if __name__ == "__main__":
     df = pd.read_csv("titanic.csv")
     template = TEMPLATE.format(dhead=df.head().to_markdown())
 
-    prompt = ChatPromptTemplate.from_messages([
-        ("system", template),
-        MessagesPlaceholder(variable_name="agent_scratchpad"),
-        ("human", "{input}")
-    ])
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            ("system", template),
+            MessagesPlaceholder(variable_name="agent_scratchpad"),
+            ("human", "{input}"),
+        ]
+    )
 
     def get_chain():
-        repl = PythonAstREPLTool(locals={"df": df}, name="python_repl",
-                                 description="Runs code and returns the output of the final line",
-                                 args_schema=PythonInputs)
+        repl = PythonAstREPLTool(
+            locals={"df": df},
+            name="python_repl",
+            description="Runs code and returns the output of the final line",
+            args_schema=PythonInputs,
+        )
         tools = [repl, retriever_tool]
-        agent = OpenAIFunctionsAgent(llm=ChatOpenAI(temperature=0, model="gpt-4"), prompt=prompt, tools=tools)
-        agent_executor = AgentExecutor(agent=agent, tools=tools, max_iterations=5, early_stopping_method="generate")
+        agent = OpenAIFunctionsAgent(
+            llm=ChatOpenAI(temperature=0, model="gpt-4"), prompt=prompt, tools=tools
+        )
+        agent_executor = AgentExecutor(
+            agent=agent, tools=tools, max_iterations=5, early_stopping_method="generate"
+        )
         return agent_executor
 
-
     client = Client()
     eval_config = RunEvalConfig(
-        evaluators=[
-            "qa"
-        ],
+        evaluators=["qa"],
     )
     chain_results = run_on_dataset(
         client,

archived/csv-qa/pandas_agent_gpt_35.py

@@ -1,9 +1,9 @@
 import pandas as pd
-from langchain.chat_models import ChatOpenAI
 from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
 from langchain.agents.agent_types import AgentType
-from langsmith import Client
+from langchain.chat_models import ChatOpenAI
 from langchain.smith import RunEvalConfig, run_on_dataset
+from langsmith import Client
 
 if __name__ == "__main__":
     df = pd.read_csv("titanic.csv")
@@ -18,20 +18,17 @@ if __name__ == "__main__":
             df,
             agent_type=AgentType.OPENAI_FUNCTIONS,
             agent_executor_kwargs=agent_executor_kwargs,
-            max_iterations=5
+            max_iterations=5,
         )
         return agent
 
-
     client = Client()
     eval_config = RunEvalConfig(
-        evaluators=[
-            "qa"
-        ],
+        evaluators=["qa"],
     )
     chain_results = run_on_dataset(
         client,
         dataset_name="Titanic CSV Data",
         llm_or_chain_factory=get_chain,
         evaluation=eval_config,
-    )
+    )

archived/csv-qa/pandas_agent_gpt_4.py

@@ -1,14 +1,13 @@
 import pandas as pd
-from langchain.chat_models import ChatOpenAI
 from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
 from langchain.agents.agent_types import AgentType
-from langsmith import Client
+from langchain.chat_models import ChatOpenAI
 from langchain.smith import RunEvalConfig, run_on_dataset
+from langsmith import Client
 
 if __name__ == "__main__":
     df = pd.read_csv("titanic.csv")
 
-
     def get_chain():
         llm = ChatOpenAI(temperature=0, model="gpt-4")
         agent_executor_kwargs = {
@@ -19,20 +18,17 @@ if __name__ == "__main__":
             df,
             agent_type=AgentType.OPENAI_FUNCTIONS,
             agent_executor_kwargs=agent_executor_kwargs,
-            max_iterations=5
+            max_iterations=5,
         )
         return agent
 
-
     client = Client()
     eval_config = RunEvalConfig(
-        evaluators=[
-            "qa"
-        ],
+        evaluators=["qa"],
     )
     chain_results = run_on_dataset(
         client,
         dataset_name="Titanic CSV Data",
         llm_or_chain_factory=get_chain,
         evaluation=eval_config,
-    )
+    )

archived/csv-qa/pandas_agent_instruct.py

@@ -1,22 +1,24 @@
-from langchain.agents import ZeroShotAgent, AgentExecutor
-from langchain.prompts import PromptTemplate
-from langchain.tools import PythonAstREPLTool
 import pandas as pd
-from langchain.llms import OpenAI
-from langsmith import Client
-from langchain.smith import RunEvalConfig, run_on_dataset
-from pydantic import BaseModel, Field
+from langchain.agents import AgentExecutor, ZeroShotAgent
+from langchain.agents.agent_toolkits.conversational_retrieval.tool import (
+    create_retriever_tool,
+)
 from langchain.embeddings import OpenAIEmbeddings
+from langchain.llms import OpenAI
+from langchain.smith import RunEvalConfig, run_on_dataset
+from langchain.tools import PythonAstREPLTool
 from langchain.vectorstores import FAISS
-from langchain.agents.agent_toolkits.conversational_retrieval.tool import create_retriever_tool
+from langsmith import Client
+from pydantic import BaseModel, Field
 
-
-pd.set_option('display.max_rows', 20)
-pd.set_option('display.max_columns', 20)
+pd.set_option("display.max_rows", 20)
+pd.set_option("display.max_columns", 20)
 
 embedding_model = OpenAIEmbeddings()
 vectorstore = FAISS.load_local("titanic_data", embedding_model)
-retriever_tool = create_retriever_tool(vectorstore.as_retriever(), "person_name_search", "Search for a person by name")
+retriever_tool = create_retriever_tool(
+    vectorstore.as_retriever(), "person_name_search", "Search for a person by name"
+)
 
 
 TEMPLATE = """You are working with a pandas dataframe in Python. The name of the dataframe is `df`.
@@ -41,7 +43,6 @@ For example:
 <logic>Use `python_repl` since even though the question is about a person, you don't know their name so you can't include it.</logic>"""
 
 
-
 class PythonInputs(BaseModel):
     query: str = Field(description="code snippet to run")
 
@@ -50,22 +51,27 @@ if __name__ == "__main__":
     df = pd.read_csv("titanic.csv")
     template = TEMPLATE.format(dhead=df.head().to_markdown())
 
-
     def get_chain():
-        repl = PythonAstREPLTool(locals={"df": df}, name="python_repl",
-                                 description="Runs code and returns the output of the final line",
-                                 args_schema=PythonInputs)
+        repl = PythonAstREPLTool(
+            locals={"df": df},
+            name="python_repl",
+            description="Runs code and returns the output of the final line",
+            args_schema=PythonInputs,
+        )
         tools = [repl, retriever_tool]
-        agent = ZeroShotAgent.from_llm_and_tools(llm=OpenAI(temperature=0, model="gpt-3.5-turbo-instruct"), tools=tools, prefix=template)
-        agent_executor = AgentExecutor(agent=agent, tools=tools, max_iterations=5, early_stopping_method="generate")
+        agent = ZeroShotAgent.from_llm_and_tools(
+            llm=OpenAI(temperature=0, model="gpt-3.5-turbo-instruct"),
+            tools=tools,
+            prefix=template,
+        )
+        agent_executor = AgentExecutor(
+            agent=agent, tools=tools, max_iterations=5, early_stopping_method="generate"
+        )
         return agent_executor
 
-
     client = Client()
     eval_config = RunEvalConfig(
-        evaluators=[
-            "qa"
-        ],
+        evaluators=["qa"],
     )
     chain_results = run_on_dataset(
         client,

archived/csv-qa/pandas_ai.py

@@ -0,0 +1,45 @@
+import pandas as pd
+from langchain.chat_models import ChatOpenAI
+from langchain.prompts import ChatPromptTemplate
+from langchain.schema.output_parser import StrOutputParser
+from langchain.smith import RunEvalConfig, run_on_dataset
+from langsmith import Client
+from pandasai import PandasAI
+
+if __name__ == "__main__":
+    df = pd.read_csv("titanic.csv")
+
+    pandas_ai = PandasAI(ChatOpenAI(temperature=0, model="gpt-4"), enable_cache=False)
+    prompt = ChatPromptTemplate.from_messages(
+        [
+            (
+                "system",
+                "Answer the users question about some data. A data scientist will run some code and the results will be returned to you to use in your answer",
+            ),
+            ("human", "Question: {input}"),
+            ("human", "Data Scientist Result: {result}"),
+        ]
+    )
+
+    def get_chain():
+        chain = (
+            {
+                "input": lambda x: x["input_question"],
+                "result": lambda x: pandas_ai(df, prompt=x["input_question"]),
+            }
+            | prompt
+            | ChatOpenAI(temperature=0, model="gpt-4")
+            | StrOutputParser()
+        )
+        return chain
+
+    client = Client()
+    eval_config = RunEvalConfig(
+        evaluators=["qa"],
+    )
+    chain_results = run_on_dataset(
+        client,
+        dataset_name="Titanic CSV Data",
+        llm_or_chain_factory=get_chain,
+        evaluation=eval_config,
+    )

archived/csv-qa/streamlit_app.py

@@ -0,0 +1,47 @@
+import pandas as pd
+import streamlit as st
+from langchain.agents.agent_toolkits import create_pandas_dataframe_agent
+from langchain.agents.agent_types import AgentType
+from langchain.chat_models import ChatOpenAI
+
+df = pd.read_csv("titanic.csv")
+
+
+llm = ChatOpenAI(temperature=0)
+agent = create_pandas_dataframe_agent(llm, df, agent_type=AgentType.OPENAI_FUNCTIONS)
+
+
+from langsmith import Client
+
+client = Client()
+
+
+def send_feedback(run_id, score):
+    client.create_feedback(run_id, "user_score", score=score)
+
+
+st.set_page_config(page_title="🦜🔗 Ask the CSV App")
+st.title("🦜🔗 Ask the CSV App")
+st.info(
+    "Most 'question answering' applications run over unstructured text data. But a lot of the data in the world is tabular data! This is an attempt to create an application using [LangChain](https://github.com/langchain-ai/langchain) to let you ask questions of data in tabular format. For this demo application, we will use the Titanic Dataset. Please explore it [here](https://github.com/datasciencedojo/datasets/blob/master/titanic.csv) to get a sense for what questions you can ask. Please leave feedback on well the question is answered, and we will use that improve the application!"
+)
+
+query_text = st.text_input("Enter your question:", placeholder="Who was in cabin C128?")
+# Form input and query
+result = None
+with st.form("myform", clear_on_submit=True):
+    submitted = st.form_submit_button("Submit")
+    if submitted:
+        with st.spinner("Calculating..."):
+            response = agent({"input": query_text}, include_run_info=True)
+            result = response["output"]
+            run_id = response["__run"].run_id
+if result is not None:
+    st.info(result)
+    col_blank, col_text, col1, col2 = st.columns([10, 2, 1, 1])
+    with col_text:
+        st.text("Feedback:")
+    with col1:
+        st.button("👍", on_click=send_feedback, args=(run_id, 1))
+    with col2:
+        st.button("👎", on_click=send_feedback, args=(run_id, 0))

archived/csv-qa/upload_data.py

@@ -8,5 +8,5 @@ if __name__ == "__main__":
         output_keys=["output_text"],
         name="Titanic CSV Data",
         description="QA over titanic data",
-        data_type = "kv"
+        data_type="kv",
     )

archived/extraction/streamlit_app.py

@@ -0,0 +1,79 @@
+import streamlit as st
+from langchain.chains import create_extraction_chain
+from langchain.chat_models import ChatOpenAI
+from langsmith import Client
+
+st.set_page_config(page_title="🦜🔗 Text-to-graph extraction")
+client = Client()
+
+
+def send_feedback(run_id, score):
+    client.create_feedback(run_id, "user_score", score=score)
+
+
+st.title("🦜🔗 Text-to-graph playground")
+st.info(
+    "This playground explores the use of [OpenAI functions](https://openai.com/blog/function-calling-and-other-api-updates) and [LangChain](https://github.com/langchain-ai/langchain) to build knowledge graphs from user-input text. It breaks down the user input text into knowledge graph triples of subject (primary entities or concepts in a sentence), predicate (actions or relationships that connect subjects to objects), and object (entities or concepts that interact with or are acted upon by the subjects)."
+)
+
+# Input text (optional default)
+oppenheimer_text = """'Julius Robert Oppenheimer, often known as Robert or "Oppie", is heralded as the father of the atomic bomb. Emerging from a non-practicing Jewish family in New York, he made several breakthroughs, such as the early black hole theory, before the monumental Manhattan Project. His wife, Katherine “Kitty” Oppenheimer, was a German-born woman with a complex past, including connections to the Communist Party. Oppenheimer\'s journey was beset by political adversaries, notably Lewis Strauss, chairman of the U.S. Atomic Energy Commission, and William Borden, an executive director with hawkish nuclear ambitions. These tensions culminated in the famous 1954 security hearing. Influential figures like lieutenant general Leslie Groves, who had also overseen the Pentagon\'s creation, stood by Oppenheimer\'s side, having earlier chosen him for the Manhattan Project and the Los Alamos location. Intimate relationships, like that with Jean Tatlock, a Communist and the possible muse behind the Trinity test\'s name, and colleagues like Frank, Oppenheimer\'s physicist brother, intertwined with his professional life. Scientists such as Ernest Lawrence, Edward Teller, David Hill, Richard Feynman, and Hans Bethe were some of Oppenheimer\'s contemporaries, each contributing to and contesting the atomic age\'s directions. Boris Pash\'s investigations, and the perspectives of figures like Leo Szilard, Niels Bohr, Harry Truman, and others, framed the broader sociopolitical context. Meanwhile, individuals like Robert Serber, Enrico Fermi, Albert Einstein, and Isidor Isaac Rabi, among many others, each played their parts in this narrative, from naming the atomic bombs to pivotal scientific contributions and advisory roles. All these figures, together with the backdrop of World War II, McCarthyism, and the dawn of the nuclear age, presented a complex mosaic of ambitions, loyalties, betrayals, and ideologies.oppenheimer_short.txt"""
+
+# Knowledge triplet schema
+default_schema = {
+    "properties": {
+        "subject": {"type": "string"},
+        "predicate": {"type": "string"},
+        "object": {"type": "string"},
+    },
+    "required": ["subject", "predicate", "object"],
+}
+
+# Create a text_area, set the default value to oppenheimer_text
+MAX_CHARS = 2000  # Maximum number of characters
+user_input_text = st.text_area("Enter your text (<2000 characters):", height=200)
+if len(user_input_text) > MAX_CHARS:
+    st.warning(f"Text is too long. Processing only the first {MAX_CHARS} characters")
+    user_input_text = user_input_text[:MAX_CHARS]
+
+
+# Output formatting of triples
+def json_to_markdown_table(json_list):
+    if not json_list:
+        return "No data available."
+
+    # Extract headers
+    headers = json_list[0].keys()
+    markdown_table = " | ".join(headers) + "\n"
+    markdown_table += " | ".join(["---"] * len(headers)) + "\n"
+
+    # Extract rows
+    for item in json_list:
+        row = " | ".join([str(item[header]) for header in headers])
+        markdown_table += row + "\n"
+
+    return markdown_table
+
+
+# Form input and query
+markdown_output = None
+with st.form("myform", clear_on_submit=True):
+    submitted = st.form_submit_button("Submit")
+    if submitted:
+        with st.spinner("Calculating..."):
+            llm = ChatOpenAI(temperature=0, model="gpt-3.5-turbo")
+            chain = create_extraction_chain(default_schema, llm)
+            extraction_output = chain(user_input_text, include_run_info=True)
+            markdown_output = json_to_markdown_table(extraction_output["text"])
+            run_id = extraction_output["__run"].run_id
+
+# Feeback
+if markdown_output is not None:
+    st.markdown(markdown_output)
+    col_blank, col_text, col1, col2 = st.columns([10, 2, 1, 1])
+    with col_text:
+        st.text("Feedback:")
+    with col1:
+        st.button("👍", on_click=send_feedback, args=(run_id, 1))
+    with col2:
+        st.button("👎", on_click=send_feedback, args=(run_id, 0))

archived/langchain-docs-benchmarking/app/server.py

@@ -1,9 +1,8 @@
+from chat_langchain.chain import chain
 from fastapi import FastAPI
 from langserve import add_routes
-from chat_langchain.chain import chain
 from openai_functions_agent import agent_executor as openai_functions_agent_chain
 
-
 app = FastAPI()
 
 # Edit this to add the chain you want to add
@@ -16,6 +15,7 @@ add_routes(
 
 add_routes(app, openai_functions_agent_chain, path="/openai-functions-agent")
 
+
 def run_server(port: int = 1983):
     import uvicorn
 

archived/langchain-docs-benchmarking/packages/anthropic-iterative-search/anthropic_iterative_search/retriever.py

@@ -12,6 +12,6 @@ RETRIEVER_TOOL_NAME = "search"
 
 
 @tool
-def search(query, callbacks = None):
+def search(query, callbacks=None):
     """Search the LangChain docs with the retriever."""
     return retriever.get_relevant_documents(query, callbacks=callbacks)

archived/langchain-docs-benchmarking/packages/chat-langchain/chat_langchain/chain.py

@@ -2,7 +2,7 @@
 from operator import itemgetter
 from typing import Dict, List, Optional, Sequence
 
-from langchain.chat_models import ChatAnthropic, ChatOpenAI, ChatFireworks
+from langchain.chat_models import ChatAnthropic, ChatFireworks, ChatOpenAI
 from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder, PromptTemplate
 from langchain.schema import Document
 from langchain.schema.language_model import BaseLanguageModel
@@ -18,7 +18,6 @@ from langchain.schema.runnable import (
 from langchain_docs_retriever.retriever import get_retriever
 from pydantic import BaseModel
 
-
 RESPONSE_TEMPLATE = """\
 You are an expert programmer and problem-solver, tasked with answering any question \
 about Langchain.
@@ -135,7 +134,7 @@ def create_response_chain(
         ]
     )
 
-    response_synthesizer = (prompt | llm | StrOutputParser()).with_config(
+    response_generator = (prompt | llm | StrOutputParser()).with_config(
         run_name="GenerateResponse",
     )
     return (
@@ -148,7 +147,7 @@ def create_response_chain(
             ),
         }
         | _context
-        | response_synthesizer
+        | response_generator
     )
 
 

archived/langchain-docs-benchmarking/packages/langchain-docs-retriever/langchain_docs_retriever/download_db.py

@@ -1,7 +1,8 @@
 import os
-import requests
 import zipfile
 
+import requests
+
 remote_url = "https://storage.googleapis.com/benchmarks-artifacts/langchain-docs-benchmarking/chroma_db.zip"
 directory = os.path.dirname(os.path.realpath(__file__))
 db_directory = os.path.join(directory, "db")

archived/langchain-docs-benchmarking/packages/langchain-docs-retriever/langchain_docs_retriever/retriever.py

@@ -2,12 +2,13 @@ import os
 from typing import Optional
 
 from langchain.embeddings import OpenAIEmbeddings
+
+# from langchain_docs_retriever.voyage import VoyageEmbeddings
+from langchain.embeddings.voyageai import VoyageEmbeddings
 from langchain.schema.embeddings import Embeddings
 from langchain.schema.retriever import BaseRetriever
 from langchain.vectorstores.chroma import Chroma
 
-# from langchain_docs_retriever.voyage import VoyageEmbeddings
-from langchain.embeddings.voyageai import VoyageEmbeddings
 from .download_db import fetch_langchain_docs_db
 
 WEAVIATE_DOCS_INDEX_NAME = "LangChain_agent_docs"

archived/langchain-docs-benchmarking/packages/oai-assistant/oai_assistant/chain.py

@@ -1,9 +1,10 @@
-from langchain_experimental.openai_assistant import OpenAIAssistantRunnable
-from langchain.agents import AgentExecutor
-from langchain_docs_retriever.retriever import get_retriever
-from langchain.tools import tool
 import json
 
+from langchain.agents import AgentExecutor
+from langchain.tools import tool
+from langchain_docs_retriever.retriever import get_retriever
+from langchain_experimental.openai_assistant import OpenAIAssistantRunnable
+
 # This is used to tell the model how to best use the retriever.
 
 
@@ -28,6 +29,8 @@ agent = OpenAIAssistantRunnable.create_assistant(
 )
 
 
-agent_executor = (lambda x: {"content": x["question"]}) | AgentExecutor(
-    agent=agent, tools=tools
-) | (lambda x: x["output"])
+agent_executor = (
+    (lambda x: {"content": x["question"]})
+    | AgentExecutor(agent=agent, tools=tools)
+    | (lambda x: x["output"])
+)

archived/langchain-docs-benchmarking/prepare_dataset.py

@@ -0,0 +1,57 @@
+"""Copy the public dataset to your own langsmith tenant."""
+from typing import Optional
+
+from langsmith import Client
+
+DATASET_NAME = "LangChain Docs Q&A"
+PUBLIC_DATASET_TOKEN = "452ccafc-18e1-4314-885b-edd735f17b9d"
+
+
+def create_langchain_docs_dataset(
+    dataset_name: str = DATASET_NAME,
+    public_dataset_token: str = PUBLIC_DATASET_TOKEN,
+    client: Optional[Client] = None,
+):
+    shared_client = Client(
+        api_url="https://api.smith.langchain.com", api_key="placeholder"
+    )
+    examples = list(shared_client.list_shared_examples(public_dataset_token))
+    client = client or Client()
+    if client.has_dataset(dataset_name=dataset_name):
+        loaded_examples = list(client.list_examples(dataset_name=dataset_name))
+        if len(loaded_examples) == len(examples):
+            return
+        else:
+            ds = client.read_dataset(dataset_name=dataset_name)
+    else:
+        ds = client.create_dataset(dataset_name=dataset_name)
+    client.create_examples(
+        inputs=[e.inputs for e in examples],
+        outputs=[e.outputs for e in examples],
+        dataset_id=ds.id,
+    )
+    print("Done creating dataset.")
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--target-api-key", type=str, required=False)
+    parser.add_argument("--target-endpoint", type=str, required=False)
+    parser.add_argument("--dataset-name", type=str, default=DATASET_NAME)
+    parser.add_argument(
+        "--public-dataset-token", type=str, default=PUBLIC_DATASET_TOKEN
+    )
+    args = parser.parse_args()
+    client = None
+    if args.target_api_key or args.target_endpoint:
+        client = Client(
+            api_key=args.target_api_key,
+            api_url=args.target_endpoint,
+        )
+    create_langchain_docs_dataset(
+        dataset_name=args.dataset_name,
+        public_dataset_token=args.public_dataset_token,
+        client=client,
+    )

archived/langchain-docs-benchmarking/run_evals.py

@@ -1,21 +1,18 @@
 import argparse
-from functools import partial
-from typing import Optional, Callable
-
-from langchain.chat_models import ChatOpenAI
-from langchain.smith import RunEvalConfig, run_on_dataset
-from langchain.schema.runnable import Runnable
-from langsmith import Client
-from chat_langchain.chain import create_chain
-from anthropic_iterative_search.chain import chain as anthropic_agent_chain
-from openai_functions_agent import agent_executor as openai_functions_agent_chain
-from oai_assistant.chain import agent_executor as openai_assistant_chain
-import os
 import importlib.util
 import sys
-
-
 import uuid
+from functools import partial
+from typing import Callable, Optional
+
+from anthropic_iterative_search.chain import chain as anthropic_agent_chain
+from chat_langchain.chain import create_chain
+from langchain.chat_models import ChatOpenAI
+from langchain.schema.runnable import Runnable
+from langchain.smith import RunEvalConfig, run_on_dataset
+from langsmith import Client
+from oai_assistant.chain import agent_executor as openai_assistant_chain
+from openai_functions_agent import agent_executor as openai_functions_agent_chain
 
 ls_client = Client()
 

archived/langchain-docs-benchmarking/run_experiments.py

@@ -1,8 +1,9 @@
 import argparse
-from run_evals import main
-from prepare_dataset import create_langchain_docs_dataset
 import json
 
+from prepare_dataset import create_langchain_docs_dataset
+from run_evals import main
+
 experiments = [
     {
         # "server_url": "http://localhost:1983/openai-functions-agent",
@@ -119,6 +120,7 @@ if __name__ == "__main__":
         ]
 
     for experiment in selected_experiments:
+        print("Running experiment:", experiment)
         main(
             **experiment,
             dataset_name=args.dataset_name,